mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/kdave/btrfs-progs.git
synced 2024-11-15 00:04:23 +08:00
0da635fd1b
On recent x86-64 system with march=native|<cpu>|<microarchitecture
level> gcc/clang will automatically define all the available vector
extensions macros. crypto/blake2-config.h then correctly set all the
HAVE_<EXTENSION> macros.
crypto/blake2-round.h then checks the HAVE_<EXTENSION> macros for
including further headers:
#if defined(HAVE_SSE41)
#include "blake2b-load-sse41.h"
#else
#include "blake2b-load-sse2.h"
#endif
which is wrong. On recent systems it always results in including
blake2b-load-sse41.h. crypto/blake2-round.h itself is included by
crypto/blake2b-sse2.c and now we have a SSE2/SSE4.1 code mixing
resulting in the incompatible type for argument build errors described
in #589.
The idea is to remove the lines above from crypto/blake2-round.h and put
the includes directly into crypto/blake2b-sse2.c and
crypto/blake2b-sse41.c respectively.
Note this slightly diverges from the upstream BLAKE2 sources.
Pull-request: #591
Author: Tino Mai <mai.tino@gmail.com>
Signed-off-by: David Sterba <dsterba@suse.com>
152 lines
4.8 KiB
C
152 lines
4.8 KiB
C
/*
|
|
BLAKE2 reference source code package - optimized C implementations
|
|
|
|
Copyright 2012, Samuel Neves <sneves@dei.uc.pt>. You may use this under the
|
|
terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at
|
|
your option. The terms of these licenses can be found at:
|
|
|
|
- CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0
|
|
- OpenSSL license : https://www.openssl.org/source/license.html
|
|
- Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
More information about the BLAKE2 hash function can be found at
|
|
https://blake2.net.
|
|
*/
|
|
#ifndef BLAKE2B_ROUND_H
|
|
#define BLAKE2B_ROUND_H
|
|
|
|
#define LOADU(p) _mm_loadu_si128( (const __m128i *)(p) )
|
|
#define STOREU(p,r) _mm_storeu_si128((__m128i *)(p), r)
|
|
|
|
#define TOF(reg) _mm_castsi128_ps((reg))
|
|
#define TOI(reg) _mm_castps_si128((reg))
|
|
|
|
#define LIKELY(x) __builtin_expect((x),1)
|
|
|
|
|
|
/* Microarchitecture-specific macros */
|
|
#ifndef HAVE_XOP
|
|
#ifdef HAVE_SSSE3
|
|
#define _mm_roti_epi64(x, c) \
|
|
(-(c) == 32) ? _mm_shuffle_epi32((x), _MM_SHUFFLE(2,3,0,1)) \
|
|
: (-(c) == 24) ? _mm_shuffle_epi8((x), r24) \
|
|
: (-(c) == 16) ? _mm_shuffle_epi8((x), r16) \
|
|
: (-(c) == 63) ? _mm_xor_si128(_mm_srli_epi64((x), -(c)), _mm_add_epi64((x), (x))) \
|
|
: _mm_xor_si128(_mm_srli_epi64((x), -(c)), _mm_slli_epi64((x), 64-(-(c))))
|
|
#else
|
|
#define _mm_roti_epi64(r, c) _mm_xor_si128(_mm_srli_epi64( (r), -(c) ),_mm_slli_epi64( (r), 64-(-(c)) ))
|
|
#endif
|
|
#else
|
|
/* ... */
|
|
#endif
|
|
|
|
|
|
|
|
#define G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \
|
|
row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \
|
|
row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \
|
|
\
|
|
row4l = _mm_xor_si128(row4l, row1l); \
|
|
row4h = _mm_xor_si128(row4h, row1h); \
|
|
\
|
|
row4l = _mm_roti_epi64(row4l, -32); \
|
|
row4h = _mm_roti_epi64(row4h, -32); \
|
|
\
|
|
row3l = _mm_add_epi64(row3l, row4l); \
|
|
row3h = _mm_add_epi64(row3h, row4h); \
|
|
\
|
|
row2l = _mm_xor_si128(row2l, row3l); \
|
|
row2h = _mm_xor_si128(row2h, row3h); \
|
|
\
|
|
row2l = _mm_roti_epi64(row2l, -24); \
|
|
row2h = _mm_roti_epi64(row2h, -24); \
|
|
|
|
#define G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \
|
|
row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \
|
|
row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \
|
|
\
|
|
row4l = _mm_xor_si128(row4l, row1l); \
|
|
row4h = _mm_xor_si128(row4h, row1h); \
|
|
\
|
|
row4l = _mm_roti_epi64(row4l, -16); \
|
|
row4h = _mm_roti_epi64(row4h, -16); \
|
|
\
|
|
row3l = _mm_add_epi64(row3l, row4l); \
|
|
row3h = _mm_add_epi64(row3h, row4h); \
|
|
\
|
|
row2l = _mm_xor_si128(row2l, row3l); \
|
|
row2h = _mm_xor_si128(row2h, row3h); \
|
|
\
|
|
row2l = _mm_roti_epi64(row2l, -63); \
|
|
row2h = _mm_roti_epi64(row2h, -63); \
|
|
|
|
#if defined(HAVE_SSSE3)
|
|
#define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \
|
|
t0 = _mm_alignr_epi8(row2h, row2l, 8); \
|
|
t1 = _mm_alignr_epi8(row2l, row2h, 8); \
|
|
row2l = t0; \
|
|
row2h = t1; \
|
|
\
|
|
t0 = row3l; \
|
|
row3l = row3h; \
|
|
row3h = t0; \
|
|
\
|
|
t0 = _mm_alignr_epi8(row4h, row4l, 8); \
|
|
t1 = _mm_alignr_epi8(row4l, row4h, 8); \
|
|
row4l = t1; \
|
|
row4h = t0;
|
|
|
|
#define UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \
|
|
t0 = _mm_alignr_epi8(row2l, row2h, 8); \
|
|
t1 = _mm_alignr_epi8(row2h, row2l, 8); \
|
|
row2l = t0; \
|
|
row2h = t1; \
|
|
\
|
|
t0 = row3l; \
|
|
row3l = row3h; \
|
|
row3h = t0; \
|
|
\
|
|
t0 = _mm_alignr_epi8(row4l, row4h, 8); \
|
|
t1 = _mm_alignr_epi8(row4h, row4l, 8); \
|
|
row4l = t1; \
|
|
row4h = t0;
|
|
#else
|
|
|
|
#define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \
|
|
t0 = row4l;\
|
|
t1 = row2l;\
|
|
row4l = row3l;\
|
|
row3l = row3h;\
|
|
row3h = row4l;\
|
|
row4l = _mm_unpackhi_epi64(row4h, _mm_unpacklo_epi64(t0, t0)); \
|
|
row4h = _mm_unpackhi_epi64(t0, _mm_unpacklo_epi64(row4h, row4h)); \
|
|
row2l = _mm_unpackhi_epi64(row2l, _mm_unpacklo_epi64(row2h, row2h)); \
|
|
row2h = _mm_unpackhi_epi64(row2h, _mm_unpacklo_epi64(t1, t1))
|
|
|
|
#define UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \
|
|
t0 = row3l;\
|
|
row3l = row3h;\
|
|
row3h = t0;\
|
|
t0 = row2l;\
|
|
t1 = row4l;\
|
|
row2l = _mm_unpackhi_epi64(row2h, _mm_unpacklo_epi64(row2l, row2l)); \
|
|
row2h = _mm_unpackhi_epi64(t0, _mm_unpacklo_epi64(row2h, row2h)); \
|
|
row4l = _mm_unpackhi_epi64(row4l, _mm_unpacklo_epi64(row4h, row4h)); \
|
|
row4h = _mm_unpackhi_epi64(row4h, _mm_unpacklo_epi64(t1, t1))
|
|
|
|
#endif
|
|
|
|
#define ROUND(r) \
|
|
LOAD_MSG_ ##r ##_1(b0, b1); \
|
|
G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \
|
|
LOAD_MSG_ ##r ##_2(b0, b1); \
|
|
G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \
|
|
DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h); \
|
|
LOAD_MSG_ ##r ##_3(b0, b1); \
|
|
G1(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \
|
|
LOAD_MSG_ ##r ##_4(b0, b1); \
|
|
G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1); \
|
|
UNDIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h);
|
|
|
|
#endif
|