diff options
Diffstat (limited to 'LoongArch-Add-ifunc-support-for-strcmp-aligned-lsx.patch')
| -rw-r--r-- | LoongArch-Add-ifunc-support-for-strcmp-aligned-lsx.patch | 499 |
1 files changed, 499 insertions, 0 deletions
diff --git a/LoongArch-Add-ifunc-support-for-strcmp-aligned-lsx.patch b/LoongArch-Add-ifunc-support-for-strcmp-aligned-lsx.patch new file mode 100644 index 0000000..d960bc6 --- /dev/null +++ b/LoongArch-Add-ifunc-support-for-strcmp-aligned-lsx.patch @@ -0,0 +1,499 @@ +From e258cfcf92f5e31e902fa045b41652f00fcf2521 Mon Sep 17 00:00:00 2001 +From: dengjianbo <dengjianbo@loongson.cn> +Date: Thu, 24 Aug 2023 16:50:18 +0800 +Subject: [PATCH 09/29] LoongArch: Add ifunc support for strcmp{aligned, lsx} + +Based on the glibc microbenchmark, strcmp-aligned implementation could +reduce the runtime 0%-10% for aligned comparison, 10%-20% for unaligned +comparison, strcmp-lsx implemenation could reduce the runtime 0%-50%. + +Signed-off-by: Peng Fan <fanpeng@loongson.cn> +Signed-off-by: ticat_fp <fanpeng@loongson.cn> +--- + sysdeps/loongarch/lp64/multiarch/Makefile | 2 + + .../lp64/multiarch/ifunc-impl-list.c | 7 + + .../loongarch/lp64/multiarch/ifunc-strcmp.h | 38 ++++ + .../loongarch/lp64/multiarch/strcmp-aligned.S | 179 ++++++++++++++++++ + sysdeps/loongarch/lp64/multiarch/strcmp-lsx.S | 165 ++++++++++++++++ + sysdeps/loongarch/lp64/multiarch/strcmp.c | 35 ++++ + 6 files changed, 426 insertions(+) + create mode 100644 sysdeps/loongarch/lp64/multiarch/ifunc-strcmp.h + create mode 100644 sysdeps/loongarch/lp64/multiarch/strcmp-aligned.S + create mode 100644 sysdeps/loongarch/lp64/multiarch/strcmp-lsx.S + create mode 100644 sysdeps/loongarch/lp64/multiarch/strcmp.c + +diff --git a/sysdeps/loongarch/lp64/multiarch/Makefile b/sysdeps/loongarch/lp64/multiarch/Makefile +index c4dd3143..d5a500de 100644 +--- a/sysdeps/loongarch/lp64/multiarch/Makefile ++++ b/sysdeps/loongarch/lp64/multiarch/Makefile +@@ -12,6 +12,8 @@ sysdep_routines += \ + strchrnul-aligned \ + strchrnul-lsx \ + strchrnul-lasx \ ++ strcmp-aligned \ ++ strcmp-lsx \ + memcpy-aligned \ + memcpy-unaligned \ + memmove-unaligned \ +diff --git a/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c b/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c +index 7cec0b77..9183b7da 100644 +--- a/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c ++++ b/sysdeps/loongarch/lp64/multiarch/ifunc-impl-list.c +@@ -62,6 +62,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, + IFUNC_IMPL_ADD (array, i, strchrnul, 1, __strchrnul_aligned) + ) + ++ IFUNC_IMPL (i, name, strcmp, ++#if !defined __loongarch_soft_float ++ IFUNC_IMPL_ADD (array, i, strcmp, SUPPORT_LSX, __strcmp_lsx) ++#endif ++ IFUNC_IMPL_ADD (array, i, strcmp, 1, __strcmp_aligned) ++ ) ++ + IFUNC_IMPL (i, name, memcpy, + #if !defined __loongarch_soft_float + IFUNC_IMPL_ADD (array, i, memcpy, SUPPORT_LASX, __memcpy_lasx) +diff --git a/sysdeps/loongarch/lp64/multiarch/ifunc-strcmp.h b/sysdeps/loongarch/lp64/multiarch/ifunc-strcmp.h +new file mode 100644 +index 00000000..ca26352b +--- /dev/null ++++ b/sysdeps/loongarch/lp64/multiarch/ifunc-strcmp.h +@@ -0,0 +1,38 @@ ++/* Common definition for strcmp ifunc selection. ++ All versions must be listed in ifunc-impl-list.c. ++ Copyright (C) 2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <ldsodefs.h> ++#include <ifunc-init.h> ++ ++#if !defined __loongarch_soft_float ++extern __typeof (REDIRECT_NAME) OPTIMIZE (lsx) attribute_hidden; ++#endif ++ ++extern __typeof (REDIRECT_NAME) OPTIMIZE (aligned) attribute_hidden; ++ ++static inline void * ++IFUNC_SELECTOR (void) ++{ ++#if !defined __loongarch_soft_float ++ if (SUPPORT_LSX) ++ return OPTIMIZE (lsx); ++ else ++#endif ++ return OPTIMIZE (aligned); ++} +diff --git a/sysdeps/loongarch/lp64/multiarch/strcmp-aligned.S b/sysdeps/loongarch/lp64/multiarch/strcmp-aligned.S +new file mode 100644 +index 00000000..f5f4f336 +--- /dev/null ++++ b/sysdeps/loongarch/lp64/multiarch/strcmp-aligned.S +@@ -0,0 +1,179 @@ ++/* Optimized strcmp implementation using basic Loongarch instructions. ++ Copyright (C) 2023 Free Software Foundation, Inc. ++ ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library. If not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <sysdep.h> ++#include <sys/regdef.h> ++#include <sys/asm.h> ++ ++#if IS_IN (libc) ++# define STRCMP_NAME __strcmp_aligned ++#else ++# define STRCMP_NAME strcmp ++#endif ++ ++LEAF(STRCMP_NAME, 6) ++ lu12i.w a4, 0x01010 ++ andi a2, a0, 0x7 ++ ori a4, a4, 0x101 ++ andi a3, a1, 0x7 ++ ++ bstrins.d a4, a4, 63, 32 ++ li.d t7, -1 ++ li.d t8, 8 ++ slli.d a5, a4, 7 ++ ++ bne a2, a3, L(unaligned) ++ bstrins.d a0, zero, 2, 0 ++ bstrins.d a1, zero, 2, 0 ++ ld.d t0, a0, 0 ++ ++ ld.d t1, a1, 0 ++ slli.d t3, a2, 3 ++ sll.d t2, t7, t3 ++ orn t0, t0, t2 ++ ++ ++ orn t1, t1, t2 ++ sub.d t2, t0, a4 ++ andn t3, a5, t0 ++ and t2, t2, t3 ++ ++ bne t0, t1, L(al_end) ++L(al_loop): ++ bnez t2, L(ret0) ++ ldx.d t0, a0, t8 ++ ldx.d t1, a1, t8 ++ ++ addi.d t8, t8, 8 ++ sub.d t2, t0, a4 ++ andn t3, a5, t0 ++ and t2, t2, t3 ++ ++ beq t0, t1, L(al_loop) ++L(al_end): ++ xor t3, t0, t1 ++ or t2, t2, t3 ++ ctz.d t3, t2 ++ ++ ++ bstrins.d t3, zero, 2, 0 ++ srl.d t0, t0, t3 ++ srl.d t1, t1, t3 ++ andi t0, t0, 0xff ++ ++ andi t1, t1, 0xff ++ sub.d a0, t0, t1 ++ jr ra ++ nop ++ ++L(ret0): ++ move a0, zero ++ jr ra ++ nop ++ nop ++ ++L(unaligned): ++ slt a6, a3, a2 ++ xor t0, a0, a1 ++ maskeqz t0, t0, a6 ++ xor a0, a0, t0 ++ ++ ++ xor a1, a1, t0 ++ andi a2, a0, 0x7 ++ andi a3, a1, 0x7 ++ bstrins.d a0, zero, 2, 0 ++ ++ bstrins.d a1, zero, 2, 0 ++ ld.d t4, a0, 0 ++ ld.d t1, a1, 0 ++ slli.d a2, a2, 3 ++ ++ slli.d a3, a3, 3 ++ srl.d t0, t4, a2 ++ srl.d t1, t1, a3 ++ srl.d t5, t7, a3 ++ ++ orn t0, t0, t5 ++ orn t1, t1, t5 ++ bne t0, t1, L(not_equal) ++ sll.d t5, t7, a2 ++ ++ ++ sub.d a3, a2, a3 ++ orn t4, t4, t5 ++ sub.d a2, zero, a3 ++ sub.d t2, t4, a4 ++ ++ andn t3, a5, t4 ++ and t2, t2, t3 ++ bnez t2, L(find_zero) ++L(un_loop): ++ srl.d t5, t4, a3 ++ ++ ldx.d t4, a0, t8 ++ ldx.d t1, a1, t8 ++ addi.d t8, t8, 8 ++ sll.d t0, t4, a2 ++ ++ or t0, t0, t5 ++ bne t0, t1, L(not_equal) ++ sub.d t2, t4, a4 ++ andn t3, a5, t4 ++ ++ ++ and t2, t2, t3 ++ beqz t2, L(un_loop) ++L(find_zero): ++ sub.d t2, t0, a4 ++ andn t3, a5, t0 ++ ++ and t2, t2, t3 ++ bnez t2, L(ret0) ++ ldx.d t1, a1, t8 ++ srl.d t0, t4, a3 ++ ++L(not_equal): ++ sub.d t2, t0, a4 ++ andn t3, a5, t0 ++ and t2, t2, t3 ++ xor t3, t0, t1 ++ ++ or t2, t2, t3 ++L(un_end): ++ ctz.d t3, t2 ++ bstrins.d t3, zero, 2, 0 ++ srl.d t0, t0, t3 ++ ++ ++ srl.d t1, t1, t3 ++ andi t0, t0, 0xff ++ andi t1, t1, 0xff ++ sub.d t2, t0, t1 ++ ++ ++ sub.d t3, t1, t0 ++ masknez t0, t2, a6 ++ maskeqz t1, t3, a6 ++ or a0, t0, t1 ++ ++ jr ra ++END(STRCMP_NAME) ++ ++libc_hidden_builtin_def (STRCMP_NAME) +diff --git a/sysdeps/loongarch/lp64/multiarch/strcmp-lsx.S b/sysdeps/loongarch/lp64/multiarch/strcmp-lsx.S +new file mode 100644 +index 00000000..2e177a38 +--- /dev/null ++++ b/sysdeps/loongarch/lp64/multiarch/strcmp-lsx.S +@@ -0,0 +1,165 @@ ++/* Optimized strcmp implementation using Loongarch LSX instructions. ++ Copyright (C) 2023 Free Software Foundation, Inc. ++ ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library. If not, see ++ <https://www.gnu.org/licenses/>. */ ++ ++#include <sysdep.h> ++#include <sys/regdef.h> ++#include <sys/asm.h> ++ ++#if IS_IN (libc) && !defined __loongarch_soft_float ++ ++# define STRCMP __strcmp_lsx ++ ++LEAF(STRCMP, 6) ++ pcalau12i t0, %pc_hi20(L(INDEX)) ++ andi a2, a0, 0xf ++ vld vr2, t0, %pc_lo12(L(INDEX)) ++ andi a3, a1, 0xf ++ ++ bne a2, a3, L(unaligned) ++ bstrins.d a0, zero, 3, 0 ++ bstrins.d a1, zero, 3, 0 ++ vld vr0, a0, 0 ++ ++ vld vr1, a1, 0 ++ vreplgr2vr.b vr3, a2 ++ vslt.b vr2, vr2, vr3 ++ vseq.b vr3, vr0, vr1 ++ ++ vmin.bu vr3, vr0, vr3 ++ vor.v vr3, vr3, vr2 ++ vsetanyeqz.b fcc0, vr3 ++ bcnez fcc0, L(al_out) ++ ++ ++L(al_loop): ++ vld vr0, a0, 16 ++ vld vr1, a1, 16 ++ addi.d a0, a0, 16 ++ addi.d a1, a1, 16 ++ ++ vseq.b vr3, vr0, vr1 ++ vmin.bu vr3, vr0, vr3 ++ vsetanyeqz.b fcc0, vr3 ++ bceqz fcc0, L(al_loop) ++ ++L(al_out): ++ vseqi.b vr3, vr3, 0 ++ vfrstpi.b vr3, vr3, 0 ++ vshuf.b vr0, vr0, vr0, vr3 ++ vshuf.b vr1, vr1, vr1, vr3 ++ ++ vpickve2gr.bu t0, vr0, 0 ++ vpickve2gr.bu t1, vr1, 0 ++ sub.d a0, t0, t1 ++ jr ra ++ ++ ++L(unaligned): ++ slt a4, a3, a2 ++ xor t0, a0, a1 ++ maskeqz t0, t0, a4 ++ xor a0, a0, t0 ++ ++ xor a1, a1, t0 ++ andi a2, a0, 0xf ++ andi a3, a1, 0xf ++ bstrins.d a0, zero, 3, 0 ++ ++ bstrins.d a1, zero, 3, 0 ++ vld vr3, a0, 0 ++ vld vr1, a1, 0 ++ vreplgr2vr.b vr4, a2 ++ ++ vreplgr2vr.b vr5, a3 ++ vslt.b vr7, vr2, vr5 ++ vsub.b vr5, vr5, vr4 ++ vaddi.bu vr6, vr2, 16 ++ ++ ++ vsub.b vr6, vr6, vr5 ++ vshuf.b vr0, vr3, vr3, vr6 ++ vor.v vr0, vr0, vr7 ++ vor.v vr1, vr1, vr7 ++ ++ vseq.b vr5, vr0, vr1 ++ vsetanyeqz.b fcc0, vr5 ++ bcnez fcc0, L(not_equal) ++ vslt.b vr4, vr2, vr4 ++ ++ vor.v vr0, vr3, vr4 ++ vsetanyeqz.b fcc0, vr0 ++ bcnez fcc0, L(find_zero) ++ nop ++ ++L(un_loop): ++ vld vr3, a0, 16 ++ vld vr1, a1, 16 ++ addi.d a0, a0, 16 ++ addi.d a1, a1, 16 ++ ++ ++ vshuf.b vr0, vr3, vr0, vr6 ++ vseq.b vr5, vr0, vr1 ++ vsetanyeqz.b fcc0, vr5 ++ bcnez fcc0, L(not_equal) ++ ++ vsetanyeqz.b fcc0, vr3 ++ vor.v vr0, vr3, vr3 ++ bceqz fcc0, L(un_loop) ++L(find_zero): ++ vmin.bu vr5, vr1, vr5 ++ ++ vsetanyeqz.b fcc0, vr5 ++ bcnez fcc0, L(ret0) ++ vld vr1, a1, 16 ++ vshuf.b vr0, vr3, vr3, vr6 ++ ++ vseq.b vr5, vr0, vr1 ++L(not_equal): ++ vmin.bu vr5, vr0, vr5 ++L(un_end): ++ vseqi.b vr5, vr5, 0 ++ vfrstpi.b vr5, vr5, 0 ++ ++ ++ vshuf.b vr0, vr0, vr0, vr5 ++ vshuf.b vr1, vr1, vr1, vr5 ++ vpickve2gr.bu t0, vr0, 0 ++ vpickve2gr.bu t1, vr1, 0 ++ ++ sub.d t3, t0, t1 ++ sub.d t4, t1, t0 ++ masknez t0, t3, a4 ++ maskeqz t1, t4, a4 ++ ++ or a0, t0, t1 ++ jr ra ++L(ret0): ++ move a0, zero ++ jr ra ++END(STRCMP) ++ ++ .section .rodata.cst16,"M",@progbits,16 ++ .align 4 ++L(INDEX): ++ .dword 0x0706050403020100 ++ .dword 0x0f0e0d0c0b0a0908 ++ ++libc_hidden_builtin_def (STRCMP) ++#endif +diff --git a/sysdeps/loongarch/lp64/multiarch/strcmp.c b/sysdeps/loongarch/lp64/multiarch/strcmp.c +new file mode 100644 +index 00000000..6f249c0b +--- /dev/null ++++ b/sysdeps/loongarch/lp64/multiarch/strcmp.c +@@ -0,0 +1,35 @@ ++/* Multiple versions of strcmp. ++ All versions must be listed in ifunc-impl-list.c. ++ Copyright (C) 2023 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <http://www.gnu.org/licenses/>. */ ++ ++/* Define multiple versions only for the definition in libc. */ ++#if IS_IN (libc) ++# define strcmp __redirect_strcmp ++# include <string.h> ++# undef strcmp ++ ++# define SYMBOL_NAME strcmp ++# include "ifunc-strcmp.h" ++ ++libc_ifunc_redirected (__redirect_strcmp, strcmp, IFUNC_SELECTOR ()); ++ ++# ifdef SHARED ++__hidden_ver1 (strcmp, __GI_strcmp, __redirect_strcmp) ++ __attribute__ ((visibility ("hidden"))) __attribute_copy__ (strcmp); ++# endif ++#endif +-- +2.33.0 + |