mirror of
https://sourceware.org/git/glibc.git
synced 2024-11-28 04:03:35 +08:00
LoongArch: Add ifunc support for strnlen{aligned, lsx, lasx}
Based on the glibc microbenchmark, strnlen-aligned implementation could reduce the runtime more than 10%, strnlen-lsx implementation could reduce the runtime about 50%-78%, strnlen-lasx implementation could reduce the runtime about 50%-88%.
This commit is contained in:
dengjianbo
caiyinyu
parent
1dc0bc8f07
commit
e74d959862
@ -3,6 +3,9 @@ sysdep_routines += \
|
||||
strlen-aligned \
|
||||
strlen-lsx \
|
||||
strlen-lasx \
|
||||
strnlen-aligned \
|
||||
strnlen-lsx \
|
||||
strnlen-lasx \
|
||||
strchr-aligned \
|
||||
strchr-lsx \
|
||||
strchr-lasx \
|
||||
|
||||
@ -38,6 +38,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_aligned)
|
||||
)
|
||||
|
||||
IFUNC_IMPL (i, name, strnlen,
|
||||
#if !defined __loongarch_soft_float
|
||||
IFUNC_IMPL_ADD (array, i, strnlen, SUPPORT_LASX, __strnlen_lasx)
|
||||
IFUNC_IMPL_ADD (array, i, strnlen, SUPPORT_LSX, __strnlen_lsx)
|
||||
#endif
|
||||
IFUNC_IMPL_ADD (array, i, strnlen, 1, __strnlen_aligned)
|
||||
)
|
||||
|
||||
IFUNC_IMPL (i, name, strchr,
|
||||
#if !defined __loongarch_soft_float
|
||||
IFUNC_IMPL_ADD (array, i, strchr, SUPPORT_LASX, __strchr_lasx)
|
||||
|
||||
41
sysdeps/loongarch/lp64/multiarch/ifunc-strnlen.h
Normal file
41
sysdeps/loongarch/lp64/multiarch/ifunc-strnlen.h
Normal file
@ -0,0 +1,41 @@
|
||||
/* Common definition for strnlen ifunc selections.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <ldsodefs.h>
|
||||
#include <ifunc-init.h>
|
||||
|
||||
#if !defined __loongarch_soft_float
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (lasx) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (lsx) attribute_hidden;
|
||||
#endif
|
||||
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (aligned) attribute_hidden;
|
||||
|
||||
static inline void *
|
||||
IFUNC_SELECTOR (void)
|
||||
{
|
||||
#if !defined __loongarch_soft_float
|
||||
if (SUPPORT_LASX)
|
||||
return OPTIMIZE (lasx);
|
||||
else if (SUPPORT_LSX)
|
||||
return OPTIMIZE (lsx);
|
||||
else
|
||||
#endif
|
||||
return OPTIMIZE (aligned);
|
||||
}
|
||||
102
sysdeps/loongarch/lp64/multiarch/strnlen-aligned.S
Normal file
102
sysdeps/loongarch/lp64/multiarch/strnlen-aligned.S
Normal file
@ -0,0 +1,102 @@
|
||||
/* Optimized strnlen implementation using basic Loongarch instructions.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library. If not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <sys/regdef.h>
|
||||
#include <sys/asm.h>
|
||||
|
||||
#if IS_IN (libc)
|
||||
# define STRNLEN __strnlen_aligned
|
||||
#else
|
||||
# define STRNLEN __strnlen
|
||||
#endif
|
||||
|
||||
LEAF(STRNLEN, 6)
|
||||
beqz a1, L(out)
|
||||
lu12i.w a2, 0x01010
|
||||
andi t1, a0, 0x7
|
||||
move t4, a0
|
||||
|
||||
bstrins.d a0, zero, 2, 0
|
||||
ori a2, a2, 0x101
|
||||
li.w t0, -1
|
||||
ld.d t2, a0, 0
|
||||
|
||||
slli.d t3, t1, 3
|
||||
bstrins.d a2, a2, 63, 32
|
||||
li.w t5, 8
|
||||
slli.d a3, a2, 7
|
||||
|
||||
sub.w t1, t5, t1
|
||||
sll.d t0, t0, t3
|
||||
orn t2, t2, t0
|
||||
sub.d t0, t2, a2
|
||||
|
||||
|
||||
andn t3, a3, t2
|
||||
and t0, t0, t3
|
||||
bnez t0, L(count_pos)
|
||||
sub.d t5, a1, t1
|
||||
|
||||
bgeu t1, a1, L(out)
|
||||
addi.d a0, a0, 8
|
||||
L(loop):
|
||||
ld.d t2, a0, 0
|
||||
sub.d t0, t2, a2
|
||||
|
||||
andn t1, a3, t2
|
||||
sltui t6, t5, 9
|
||||
and t0, t0, t1
|
||||
or t7, t0, t6
|
||||
|
||||
bnez t7, L(count_pos)
|
||||
ld.d t2, a0, 8
|
||||
addi.d a0, a0, 16
|
||||
sub.d t0, t2, a2
|
||||
|
||||
|
||||
andn t1, a3, t2
|
||||
sltui t6, t5, 17
|
||||
and t0, t0, t1
|
||||
addi.d t5, t5, -16
|
||||
|
||||
or t7, t0, t6
|
||||
beqz t7, L(loop)
|
||||
addi.d a0, a0, -8
|
||||
L(count_pos):
|
||||
ctz.d t1, t0
|
||||
|
||||
sub.d a0, a0, t4
|
||||
srli.d t1, t1, 3
|
||||
add.d a0, t1, a0
|
||||
sltu t0, a0, a1
|
||||
|
||||
masknez t1, a1, t0
|
||||
maskeqz a0, a0, t0
|
||||
or a0, a0, t1
|
||||
jr ra
|
||||
|
||||
|
||||
L(out):
|
||||
move a0, a1
|
||||
jr ra
|
||||
END(STRNLEN)
|
||||
|
||||
weak_alias (STRNLEN, strnlen)
|
||||
libc_hidden_builtin_def (STRNLEN)
|
||||
100
sysdeps/loongarch/lp64/multiarch/strnlen-lasx.S
Normal file
100
sysdeps/loongarch/lp64/multiarch/strnlen-lasx.S
Normal file
@ -0,0 +1,100 @@
|
||||
/* Optimized strnlen implementation using loongarch LASX instructions
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library. If not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <sys/regdef.h>
|
||||
#include <sys/asm.h>
|
||||
|
||||
#if IS_IN (libc) && !defined __loongarch_soft_float
|
||||
|
||||
# define STRNLEN __strnlen_lasx
|
||||
|
||||
LEAF(STRNLEN, 6)
|
||||
beqz a1, L(ret0)
|
||||
andi t1, a0, 0x3f
|
||||
li.d t3, 65
|
||||
sub.d a2, a0, t1
|
||||
|
||||
xvld xr0, a2, 0
|
||||
xvld xr1, a2, 32
|
||||
sub.d t1, t3, t1
|
||||
move a3, a0
|
||||
|
||||
sltu t1, a1, t1
|
||||
xvmsknz.b xr0, xr0
|
||||
xvmsknz.b xr1, xr1
|
||||
xvpickve.w xr2, xr0, 4
|
||||
|
||||
xvpickve.w xr3, xr1, 4
|
||||
vilvl.h vr0, vr2, vr0
|
||||
vilvl.h vr1, vr3, vr1
|
||||
vilvl.w vr0, vr1, vr0
|
||||
|
||||
|
||||
movfr2gr.d t0, fa0
|
||||
sra.d t0, t0, a0
|
||||
orn t1, t1, t0
|
||||
bnez t1, L(end)
|
||||
|
||||
add.d a4, a0, a1
|
||||
move a0, a2
|
||||
addi.d a4, a4, -1
|
||||
bstrins.d a4, zero, 5, 0
|
||||
|
||||
L(loop):
|
||||
xvld xr0, a0, 64
|
||||
xvld xr1, a0, 96
|
||||
addi.d a0, a0, 64
|
||||
beq a0, a4, L(out)
|
||||
|
||||
xvmin.bu xr2, xr0, xr1
|
||||
xvsetanyeqz.b fcc0, xr2
|
||||
bceqz fcc0, L(loop)
|
||||
L(out):
|
||||
xvmsknz.b xr0, xr0
|
||||
|
||||
|
||||
xvmsknz.b xr1, xr1
|
||||
xvpickve.w xr2, xr0, 4
|
||||
xvpickve.w xr3, xr1, 4
|
||||
vilvl.h vr0, vr2, vr0
|
||||
|
||||
vilvl.h vr1, vr3, vr1
|
||||
vilvl.w vr0, vr1, vr0
|
||||
movfr2gr.d t0, fa0
|
||||
L(end):
|
||||
sub.d a0, a0, a3
|
||||
|
||||
cto.d t0, t0
|
||||
add.d a0, a0, t0
|
||||
sltu t1, a0, a1
|
||||
masknez t0, a1, t1
|
||||
|
||||
maskeqz t1, a0, t1
|
||||
or a0, t0, t1
|
||||
jr ra
|
||||
L(ret0):
|
||||
move a0, zero
|
||||
|
||||
|
||||
jr ra
|
||||
END(STRNLEN)
|
||||
|
||||
libc_hidden_def (STRNLEN)
|
||||
#endif
|
||||
89
sysdeps/loongarch/lp64/multiarch/strnlen-lsx.S
Normal file
89
sysdeps/loongarch/lp64/multiarch/strnlen-lsx.S
Normal file
@ -0,0 +1,89 @@
|
||||
/* Optimized strnlen implementation using loongarch LSX instructions
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library. If not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <sys/regdef.h>
|
||||
#include <sys/asm.h>
|
||||
|
||||
#if IS_IN (libc) && !defined __loongarch_soft_float
|
||||
|
||||
# define STRNLEN __strnlen_lsx
|
||||
|
||||
LEAF(STRNLEN, 6)
|
||||
beqz a1, L(ret0)
|
||||
andi t1, a0, 0x1f
|
||||
li.d t3, 33
|
||||
sub.d a2, a0, t1
|
||||
|
||||
vld vr0, a2, 0
|
||||
vld vr1, a2, 16
|
||||
sub.d t1, t3, t1
|
||||
move a3, a0
|
||||
|
||||
sltu t1, a1, t1
|
||||
vmsknz.b vr0, vr0
|
||||
vmsknz.b vr1, vr1
|
||||
vilvl.h vr0, vr1, vr0
|
||||
|
||||
movfr2gr.s t0, fa0
|
||||
sra.w t0, t0, a0
|
||||
orn t1, t1, t0
|
||||
bnez t1, L(end)
|
||||
|
||||
|
||||
add.d a4, a0, a1
|
||||
move a0, a2
|
||||
addi.d a4, a4, -1
|
||||
bstrins.d a4, zero, 4, 0
|
||||
|
||||
L(loop):
|
||||
vld vr0, a0, 32
|
||||
vld vr1, a0, 48
|
||||
addi.d a0, a0, 32
|
||||
beq a0, a4, L(out)
|
||||
|
||||
vmin.bu vr2, vr0, vr1
|
||||
vsetanyeqz.b fcc0, vr2
|
||||
bceqz fcc0, L(loop)
|
||||
L(out):
|
||||
vmsknz.b vr0, vr0
|
||||
|
||||
vmsknz.b vr1, vr1
|
||||
vilvl.h vr0, vr1, vr0
|
||||
movfr2gr.s t0, fa0
|
||||
L(end):
|
||||
sub.d a0, a0, a3
|
||||
|
||||
|
||||
cto.w t0, t0
|
||||
add.d a0, a0, t0
|
||||
sltu t1, a0, a1
|
||||
masknez t0, a1, t1
|
||||
|
||||
maskeqz t1, a0, t1
|
||||
or a0, t0, t1
|
||||
jr ra
|
||||
L(ret0):
|
||||
move a0, zero
|
||||
|
||||
jr ra
|
||||
END(STRNLEN)
|
||||
|
||||
libc_hidden_builtin_def (STRNLEN)
|
||||
#endif
|
||||
39
sysdeps/loongarch/lp64/multiarch/strnlen.c
Normal file
39
sysdeps/loongarch/lp64/multiarch/strnlen.c
Normal file
@ -0,0 +1,39 @@
|
||||
/* Multiple versions of strnlen.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Define multiple versions only for the definition in libc. */
|
||||
#if IS_IN (libc)
|
||||
# define strnlen __redirect_strnlen
|
||||
# define __strnlen __redirect___strnlen
|
||||
# include <string.h>
|
||||
# undef __strnlen
|
||||
# undef strnlen
|
||||
|
||||
# define SYMBOL_NAME strnlen
|
||||
# include "ifunc-strnlen.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_strnlen, __strnlen, IFUNC_SELECTOR ());
|
||||
weak_alias (__strnlen, strnlen);
|
||||
# ifdef SHARED
|
||||
__hidden_ver1 (__strnlen, __GI___strnlen, __redirect___strnlen)
|
||||
__attribute__((visibility ("hidden"))) __attribute_copy__ (strnlen);
|
||||
__hidden_ver1 (strnlen, __GI_strnlen, __redirect_strnlen)
|
||||
__attribute__((weak, visibility ("hidden"))) __attribute_copy__ (strnlen);
|
||||
# endif
|
||||
#endif
|
||||
Loading…
Reference in New Issue
Block a user