mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-11-11 12:28:41 +08:00
6ab39f9992
The GFNI routines in the AVX version of the ARIA implementation now use
explicit VMOVDQA instructions to load the constant input vectors, which
means they must be 16 byte aligned. So ensure that this is the case, by
dropping the section split and the incorrect .align 8 directive, and
emitting the constants into the 16-byte aligned section instead.
Note that the AVX2 version of this code deviates from this pattern, and
does not require a similar fix, given that it loads these contants as
8-byte memory operands, for which AVX2 permits any alignment.
Cc: Taehee Yoo <ap420073@gmail.com>
Fixes: 8b84475318 ("crypto: x86/aria-avx - Do not use avx2 instructions")
Reported-by: syzbot+a6abcf08bad8b18fd198@syzkaller.appspotmail.com
Tested-by: syzbot+a6abcf08bad8b18fd198@syzkaller.appspotmail.com
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
1363 lines
40 KiB
ArmAsm
1363 lines
40 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
|
/*
|
|
* ARIA Cipher 16-way parallel algorithm (AVX)
|
|
*
|
|
* Copyright (c) 2022 Taehee Yoo <ap420073@gmail.com>
|
|
*
|
|
*/
|
|
|
|
#include <linux/linkage.h>
|
|
#include <linux/cfi_types.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/frame.h>
|
|
|
|
/* register macros */
|
|
#define CTX %rdi
|
|
|
|
|
|
#define BV8(a0, a1, a2, a3, a4, a5, a6, a7) \
|
|
( (((a0) & 1) << 0) | \
|
|
(((a1) & 1) << 1) | \
|
|
(((a2) & 1) << 2) | \
|
|
(((a3) & 1) << 3) | \
|
|
(((a4) & 1) << 4) | \
|
|
(((a5) & 1) << 5) | \
|
|
(((a6) & 1) << 6) | \
|
|
(((a7) & 1) << 7) )
|
|
|
|
#define BM8X8(l0, l1, l2, l3, l4, l5, l6, l7) \
|
|
( ((l7) << (0 * 8)) | \
|
|
((l6) << (1 * 8)) | \
|
|
((l5) << (2 * 8)) | \
|
|
((l4) << (3 * 8)) | \
|
|
((l3) << (4 * 8)) | \
|
|
((l2) << (5 * 8)) | \
|
|
((l1) << (6 * 8)) | \
|
|
((l0) << (7 * 8)) )
|
|
|
|
#define inc_le128(x, minus_one, tmp) \
|
|
vpcmpeqq minus_one, x, tmp; \
|
|
vpsubq minus_one, x, x; \
|
|
vpslldq $8, tmp, tmp; \
|
|
vpsubq tmp, x, x;
|
|
|
|
#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \
|
|
vpand x, mask4bit, tmp0; \
|
|
vpandn x, mask4bit, x; \
|
|
vpsrld $4, x, x; \
|
|
\
|
|
vpshufb tmp0, lo_t, tmp0; \
|
|
vpshufb x, hi_t, x; \
|
|
vpxor tmp0, x, x;
|
|
|
|
#define transpose_4x4(x0, x1, x2, x3, t1, t2) \
|
|
vpunpckhdq x1, x0, t2; \
|
|
vpunpckldq x1, x0, x0; \
|
|
\
|
|
vpunpckldq x3, x2, t1; \
|
|
vpunpckhdq x3, x2, x2; \
|
|
\
|
|
vpunpckhqdq t1, x0, x1; \
|
|
vpunpcklqdq t1, x0, x0; \
|
|
\
|
|
vpunpckhqdq x2, t2, x3; \
|
|
vpunpcklqdq x2, t2, x2;
|
|
|
|
#define byteslice_16x16b(a0, b0, c0, d0, \
|
|
a1, b1, c1, d1, \
|
|
a2, b2, c2, d2, \
|
|
a3, b3, c3, d3, \
|
|
st0, st1) \
|
|
vmovdqu d2, st0; \
|
|
vmovdqu d3, st1; \
|
|
transpose_4x4(a0, a1, a2, a3, d2, d3); \
|
|
transpose_4x4(b0, b1, b2, b3, d2, d3); \
|
|
vmovdqu st0, d2; \
|
|
vmovdqu st1, d3; \
|
|
\
|
|
vmovdqu a0, st0; \
|
|
vmovdqu a1, st1; \
|
|
transpose_4x4(c0, c1, c2, c3, a0, a1); \
|
|
transpose_4x4(d0, d1, d2, d3, a0, a1); \
|
|
\
|
|
vmovdqu .Lshufb_16x16b(%rip), a0; \
|
|
vmovdqu st1, a1; \
|
|
vpshufb a0, a2, a2; \
|
|
vpshufb a0, a3, a3; \
|
|
vpshufb a0, b0, b0; \
|
|
vpshufb a0, b1, b1; \
|
|
vpshufb a0, b2, b2; \
|
|
vpshufb a0, b3, b3; \
|
|
vpshufb a0, a1, a1; \
|
|
vpshufb a0, c0, c0; \
|
|
vpshufb a0, c1, c1; \
|
|
vpshufb a0, c2, c2; \
|
|
vpshufb a0, c3, c3; \
|
|
vpshufb a0, d0, d0; \
|
|
vpshufb a0, d1, d1; \
|
|
vpshufb a0, d2, d2; \
|
|
vpshufb a0, d3, d3; \
|
|
vmovdqu d3, st1; \
|
|
vmovdqu st0, d3; \
|
|
vpshufb a0, d3, a0; \
|
|
vmovdqu d2, st0; \
|
|
\
|
|
transpose_4x4(a0, b0, c0, d0, d2, d3); \
|
|
transpose_4x4(a1, b1, c1, d1, d2, d3); \
|
|
vmovdqu st0, d2; \
|
|
vmovdqu st1, d3; \
|
|
\
|
|
vmovdqu b0, st0; \
|
|
vmovdqu b1, st1; \
|
|
transpose_4x4(a2, b2, c2, d2, b0, b1); \
|
|
transpose_4x4(a3, b3, c3, d3, b0, b1); \
|
|
vmovdqu st0, b0; \
|
|
vmovdqu st1, b1; \
|
|
/* does not adjust output bytes inside vectors */
|
|
|
|
#define debyteslice_16x16b(a0, b0, c0, d0, \
|
|
a1, b1, c1, d1, \
|
|
a2, b2, c2, d2, \
|
|
a3, b3, c3, d3, \
|
|
st0, st1) \
|
|
vmovdqu d2, st0; \
|
|
vmovdqu d3, st1; \
|
|
transpose_4x4(a0, a1, a2, a3, d2, d3); \
|
|
transpose_4x4(b0, b1, b2, b3, d2, d3); \
|
|
vmovdqu st0, d2; \
|
|
vmovdqu st1, d3; \
|
|
\
|
|
vmovdqu a0, st0; \
|
|
vmovdqu a1, st1; \
|
|
transpose_4x4(c0, c1, c2, c3, a0, a1); \
|
|
transpose_4x4(d0, d1, d2, d3, a0, a1); \
|
|
\
|
|
vmovdqu .Lshufb_16x16b(%rip), a0; \
|
|
vmovdqu st1, a1; \
|
|
vpshufb a0, a2, a2; \
|
|
vpshufb a0, a3, a3; \
|
|
vpshufb a0, b0, b0; \
|
|
vpshufb a0, b1, b1; \
|
|
vpshufb a0, b2, b2; \
|
|
vpshufb a0, b3, b3; \
|
|
vpshufb a0, a1, a1; \
|
|
vpshufb a0, c0, c0; \
|
|
vpshufb a0, c1, c1; \
|
|
vpshufb a0, c2, c2; \
|
|
vpshufb a0, c3, c3; \
|
|
vpshufb a0, d0, d0; \
|
|
vpshufb a0, d1, d1; \
|
|
vpshufb a0, d2, d2; \
|
|
vpshufb a0, d3, d3; \
|
|
vmovdqu d3, st1; \
|
|
vmovdqu st0, d3; \
|
|
vpshufb a0, d3, a0; \
|
|
vmovdqu d2, st0; \
|
|
\
|
|
transpose_4x4(c0, d0, a0, b0, d2, d3); \
|
|
transpose_4x4(c1, d1, a1, b1, d2, d3); \
|
|
vmovdqu st0, d2; \
|
|
vmovdqu st1, d3; \
|
|
\
|
|
vmovdqu b0, st0; \
|
|
vmovdqu b1, st1; \
|
|
transpose_4x4(c2, d2, a2, b2, b0, b1); \
|
|
transpose_4x4(c3, d3, a3, b3, b0, b1); \
|
|
vmovdqu st0, b0; \
|
|
vmovdqu st1, b1; \
|
|
/* does not adjust output bytes inside vectors */
|
|
|
|
/* load blocks to registers and apply pre-whitening */
|
|
#define inpack16_pre(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
rio) \
|
|
vmovdqu (0 * 16)(rio), x0; \
|
|
vmovdqu (1 * 16)(rio), x1; \
|
|
vmovdqu (2 * 16)(rio), x2; \
|
|
vmovdqu (3 * 16)(rio), x3; \
|
|
vmovdqu (4 * 16)(rio), x4; \
|
|
vmovdqu (5 * 16)(rio), x5; \
|
|
vmovdqu (6 * 16)(rio), x6; \
|
|
vmovdqu (7 * 16)(rio), x7; \
|
|
vmovdqu (8 * 16)(rio), y0; \
|
|
vmovdqu (9 * 16)(rio), y1; \
|
|
vmovdqu (10 * 16)(rio), y2; \
|
|
vmovdqu (11 * 16)(rio), y3; \
|
|
vmovdqu (12 * 16)(rio), y4; \
|
|
vmovdqu (13 * 16)(rio), y5; \
|
|
vmovdqu (14 * 16)(rio), y6; \
|
|
vmovdqu (15 * 16)(rio), y7;
|
|
|
|
/* byteslice pre-whitened blocks and store to temporary memory */
|
|
#define inpack16_post(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_ab, mem_cd) \
|
|
byteslice_16x16b(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
(mem_ab), (mem_cd)); \
|
|
\
|
|
vmovdqu x0, 0 * 16(mem_ab); \
|
|
vmovdqu x1, 1 * 16(mem_ab); \
|
|
vmovdqu x2, 2 * 16(mem_ab); \
|
|
vmovdqu x3, 3 * 16(mem_ab); \
|
|
vmovdqu x4, 4 * 16(mem_ab); \
|
|
vmovdqu x5, 5 * 16(mem_ab); \
|
|
vmovdqu x6, 6 * 16(mem_ab); \
|
|
vmovdqu x7, 7 * 16(mem_ab); \
|
|
vmovdqu y0, 0 * 16(mem_cd); \
|
|
vmovdqu y1, 1 * 16(mem_cd); \
|
|
vmovdqu y2, 2 * 16(mem_cd); \
|
|
vmovdqu y3, 3 * 16(mem_cd); \
|
|
vmovdqu y4, 4 * 16(mem_cd); \
|
|
vmovdqu y5, 5 * 16(mem_cd); \
|
|
vmovdqu y6, 6 * 16(mem_cd); \
|
|
vmovdqu y7, 7 * 16(mem_cd);
|
|
|
|
#define write_output(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem) \
|
|
vmovdqu x0, 0 * 16(mem); \
|
|
vmovdqu x1, 1 * 16(mem); \
|
|
vmovdqu x2, 2 * 16(mem); \
|
|
vmovdqu x3, 3 * 16(mem); \
|
|
vmovdqu x4, 4 * 16(mem); \
|
|
vmovdqu x5, 5 * 16(mem); \
|
|
vmovdqu x6, 6 * 16(mem); \
|
|
vmovdqu x7, 7 * 16(mem); \
|
|
vmovdqu y0, 8 * 16(mem); \
|
|
vmovdqu y1, 9 * 16(mem); \
|
|
vmovdqu y2, 10 * 16(mem); \
|
|
vmovdqu y3, 11 * 16(mem); \
|
|
vmovdqu y4, 12 * 16(mem); \
|
|
vmovdqu y5, 13 * 16(mem); \
|
|
vmovdqu y6, 14 * 16(mem); \
|
|
vmovdqu y7, 15 * 16(mem); \
|
|
|
|
#define aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, idx) \
|
|
vmovdqu x0, ((idx + 0) * 16)(mem_tmp); \
|
|
vmovdqu x1, ((idx + 1) * 16)(mem_tmp); \
|
|
vmovdqu x2, ((idx + 2) * 16)(mem_tmp); \
|
|
vmovdqu x3, ((idx + 3) * 16)(mem_tmp); \
|
|
vmovdqu x4, ((idx + 4) * 16)(mem_tmp); \
|
|
vmovdqu x5, ((idx + 5) * 16)(mem_tmp); \
|
|
vmovdqu x6, ((idx + 6) * 16)(mem_tmp); \
|
|
vmovdqu x7, ((idx + 7) * 16)(mem_tmp);
|
|
|
|
#define aria_load_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, idx) \
|
|
vmovdqu ((idx + 0) * 16)(mem_tmp), x0; \
|
|
vmovdqu ((idx + 1) * 16)(mem_tmp), x1; \
|
|
vmovdqu ((idx + 2) * 16)(mem_tmp), x2; \
|
|
vmovdqu ((idx + 3) * 16)(mem_tmp), x3; \
|
|
vmovdqu ((idx + 4) * 16)(mem_tmp), x4; \
|
|
vmovdqu ((idx + 5) * 16)(mem_tmp), x5; \
|
|
vmovdqu ((idx + 6) * 16)(mem_tmp), x6; \
|
|
vmovdqu ((idx + 7) * 16)(mem_tmp), x7;
|
|
|
|
#define aria_ark_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
t0, t1, t2, rk, \
|
|
idx, round) \
|
|
/* AddRoundKey */ \
|
|
vbroadcastss ((round * 16) + idx + 0)(rk), t0; \
|
|
vpsrld $24, t0, t2; \
|
|
vpshufb t1, t2, t2; \
|
|
vpxor t2, x0, x0; \
|
|
vpsrld $16, t0, t2; \
|
|
vpshufb t1, t2, t2; \
|
|
vpxor t2, x1, x1; \
|
|
vpsrld $8, t0, t2; \
|
|
vpshufb t1, t2, t2; \
|
|
vpxor t2, x2, x2; \
|
|
vpshufb t1, t0, t2; \
|
|
vpxor t2, x3, x3; \
|
|
vbroadcastss ((round * 16) + idx + 4)(rk), t0; \
|
|
vpsrld $24, t0, t2; \
|
|
vpshufb t1, t2, t2; \
|
|
vpxor t2, x4, x4; \
|
|
vpsrld $16, t0, t2; \
|
|
vpshufb t1, t2, t2; \
|
|
vpxor t2, x5, x5; \
|
|
vpsrld $8, t0, t2; \
|
|
vpshufb t1, t2, t2; \
|
|
vpxor t2, x6, x6; \
|
|
vpshufb t1, t0, t2; \
|
|
vpxor t2, x7, x7;
|
|
|
|
#ifdef CONFIG_AS_GFNI
|
|
#define aria_sbox_8way_gfni(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
t0, t1, t2, t3, \
|
|
t4, t5, t6, t7) \
|
|
vmovdqa .Ltf_s2_bitmatrix(%rip), t0; \
|
|
vmovdqa .Ltf_inv_bitmatrix(%rip), t1; \
|
|
vmovdqa .Ltf_id_bitmatrix(%rip), t2; \
|
|
vmovdqa .Ltf_aff_bitmatrix(%rip), t3; \
|
|
vmovdqa .Ltf_x2_bitmatrix(%rip), t4; \
|
|
vgf2p8affineinvqb $(tf_s2_const), t0, x1, x1; \
|
|
vgf2p8affineinvqb $(tf_s2_const), t0, x5, x5; \
|
|
vgf2p8affineqb $(tf_inv_const), t1, x2, x2; \
|
|
vgf2p8affineqb $(tf_inv_const), t1, x6, x6; \
|
|
vgf2p8affineinvqb $0, t2, x2, x2; \
|
|
vgf2p8affineinvqb $0, t2, x6, x6; \
|
|
vgf2p8affineinvqb $(tf_aff_const), t3, x0, x0; \
|
|
vgf2p8affineinvqb $(tf_aff_const), t3, x4, x4; \
|
|
vgf2p8affineqb $(tf_x2_const), t4, x3, x3; \
|
|
vgf2p8affineqb $(tf_x2_const), t4, x7, x7; \
|
|
vgf2p8affineinvqb $0, t2, x3, x3; \
|
|
vgf2p8affineinvqb $0, t2, x7, x7
|
|
|
|
#endif /* CONFIG_AS_GFNI */
|
|
|
|
#define aria_sbox_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
t0, t1, t2, t3, \
|
|
t4, t5, t6, t7) \
|
|
vmovdqa .Linv_shift_row(%rip), t0; \
|
|
vmovdqa .Lshift_row(%rip), t1; \
|
|
vbroadcastss .L0f0f0f0f(%rip), t6; \
|
|
vmovdqa .Ltf_lo__inv_aff__and__s2(%rip), t2; \
|
|
vmovdqa .Ltf_hi__inv_aff__and__s2(%rip), t3; \
|
|
vmovdqa .Ltf_lo__x2__and__fwd_aff(%rip), t4; \
|
|
vmovdqa .Ltf_hi__x2__and__fwd_aff(%rip), t5; \
|
|
\
|
|
vaesenclast t7, x0, x0; \
|
|
vaesenclast t7, x4, x4; \
|
|
vaesenclast t7, x1, x1; \
|
|
vaesenclast t7, x5, x5; \
|
|
vaesdeclast t7, x2, x2; \
|
|
vaesdeclast t7, x6, x6; \
|
|
\
|
|
/* AES inverse shift rows */ \
|
|
vpshufb t0, x0, x0; \
|
|
vpshufb t0, x4, x4; \
|
|
vpshufb t0, x1, x1; \
|
|
vpshufb t0, x5, x5; \
|
|
vpshufb t1, x3, x3; \
|
|
vpshufb t1, x7, x7; \
|
|
vpshufb t1, x2, x2; \
|
|
vpshufb t1, x6, x6; \
|
|
\
|
|
/* affine transformation for S2 */ \
|
|
filter_8bit(x1, t2, t3, t6, t0); \
|
|
/* affine transformation for S2 */ \
|
|
filter_8bit(x5, t2, t3, t6, t0); \
|
|
\
|
|
/* affine transformation for X2 */ \
|
|
filter_8bit(x3, t4, t5, t6, t0); \
|
|
/* affine transformation for X2 */ \
|
|
filter_8bit(x7, t4, t5, t6, t0); \
|
|
vaesdeclast t7, x3, x3; \
|
|
vaesdeclast t7, x7, x7;
|
|
|
|
#define aria_diff_m(x0, x1, x2, x3, \
|
|
t0, t1, t2, t3) \
|
|
/* T = rotr32(X, 8); */ \
|
|
/* X ^= T */ \
|
|
vpxor x0, x3, t0; \
|
|
vpxor x1, x0, t1; \
|
|
vpxor x2, x1, t2; \
|
|
vpxor x3, x2, t3; \
|
|
/* X = T ^ rotr(X, 16); */ \
|
|
vpxor t2, x0, x0; \
|
|
vpxor x1, t3, t3; \
|
|
vpxor t0, x2, x2; \
|
|
vpxor t1, x3, x1; \
|
|
vmovdqu t3, x3;
|
|
|
|
#define aria_diff_word(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7) \
|
|
/* t1 ^= t2; */ \
|
|
vpxor y0, x4, x4; \
|
|
vpxor y1, x5, x5; \
|
|
vpxor y2, x6, x6; \
|
|
vpxor y3, x7, x7; \
|
|
\
|
|
/* t2 ^= t3; */ \
|
|
vpxor y4, y0, y0; \
|
|
vpxor y5, y1, y1; \
|
|
vpxor y6, y2, y2; \
|
|
vpxor y7, y3, y3; \
|
|
\
|
|
/* t0 ^= t1; */ \
|
|
vpxor x4, x0, x0; \
|
|
vpxor x5, x1, x1; \
|
|
vpxor x6, x2, x2; \
|
|
vpxor x7, x3, x3; \
|
|
\
|
|
/* t3 ^= t1; */ \
|
|
vpxor x4, y4, y4; \
|
|
vpxor x5, y5, y5; \
|
|
vpxor x6, y6, y6; \
|
|
vpxor x7, y7, y7; \
|
|
\
|
|
/* t2 ^= t0; */ \
|
|
vpxor x0, y0, y0; \
|
|
vpxor x1, y1, y1; \
|
|
vpxor x2, y2, y2; \
|
|
vpxor x3, y3, y3; \
|
|
\
|
|
/* t1 ^= t2; */ \
|
|
vpxor y0, x4, x4; \
|
|
vpxor y1, x5, x5; \
|
|
vpxor y2, x6, x6; \
|
|
vpxor y3, x7, x7;
|
|
|
|
#define aria_fe(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, rk, round) \
|
|
vpxor y7, y7, y7; \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 8, round); \
|
|
\
|
|
aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
|
|
y0, y1, y2, y3, y4, y5, y6, y7); \
|
|
\
|
|
aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
|
|
aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
|
|
aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 8); \
|
|
\
|
|
aria_load_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 0); \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 0, round); \
|
|
\
|
|
aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
|
|
y0, y1, y2, y3, y4, y5, y6, y7); \
|
|
\
|
|
aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
|
|
aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
|
|
aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 0); \
|
|
aria_load_state_8way(y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, 8); \
|
|
aria_diff_word(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7); \
|
|
/* aria_diff_byte() \
|
|
* T3 = ABCD -> BADC \
|
|
* T3 = y4, y5, y6, y7 -> y5, y4, y7, y6 \
|
|
* T0 = ABCD -> CDAB \
|
|
* T0 = x0, x1, x2, x3 -> x2, x3, x0, x1 \
|
|
* T1 = ABCD -> DCBA \
|
|
* T1 = x4, x5, x6, x7 -> x7, x6, x5, x4 \
|
|
*/ \
|
|
aria_diff_word(x2, x3, x0, x1, \
|
|
x7, x6, x5, x4, \
|
|
y0, y1, y2, y3, \
|
|
y5, y4, y7, y6); \
|
|
aria_store_state_8way(x3, x2, x1, x0, \
|
|
x6, x7, x4, x5, \
|
|
mem_tmp, 0);
|
|
|
|
#define aria_fo(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, rk, round) \
|
|
vpxor y7, y7, y7; \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 8, round); \
|
|
\
|
|
aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, y4, y5, y6, y7); \
|
|
\
|
|
aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
|
|
aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
|
|
aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 8); \
|
|
\
|
|
aria_load_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 0); \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 0, round); \
|
|
\
|
|
aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, y4, y5, y6, y7); \
|
|
\
|
|
aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
|
|
aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
|
|
aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 0); \
|
|
aria_load_state_8way(y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, 8); \
|
|
aria_diff_word(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7); \
|
|
/* aria_diff_byte() \
|
|
* T1 = ABCD -> BADC \
|
|
* T1 = x4, x5, x6, x7 -> x5, x4, x7, x6 \
|
|
* T2 = ABCD -> CDAB \
|
|
* T2 = y0, y1, y2, y3, -> y2, y3, y0, y1 \
|
|
* T3 = ABCD -> DCBA \
|
|
* T3 = y4, y5, y6, y7 -> y7, y6, y5, y4 \
|
|
*/ \
|
|
aria_diff_word(x0, x1, x2, x3, \
|
|
x5, x4, x7, x6, \
|
|
y2, y3, y0, y1, \
|
|
y7, y6, y5, y4); \
|
|
aria_store_state_8way(x3, x2, x1, x0, \
|
|
x6, x7, x4, x5, \
|
|
mem_tmp, 0);
|
|
|
|
#define aria_ff(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, rk, round, last_round) \
|
|
vpxor y7, y7, y7; \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 8, round); \
|
|
\
|
|
aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
|
|
y0, y1, y2, y3, y4, y5, y6, y7); \
|
|
\
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 8, last_round); \
|
|
\
|
|
aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 8); \
|
|
\
|
|
aria_load_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 0); \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 0, round); \
|
|
\
|
|
aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
|
|
y0, y1, y2, y3, y4, y5, y6, y7); \
|
|
\
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 0, last_round); \
|
|
\
|
|
aria_load_state_8way(y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, 8);
|
|
|
|
#ifdef CONFIG_AS_GFNI
|
|
#define aria_fe_gfni(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, rk, round) \
|
|
vpxor y7, y7, y7; \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 8, round); \
|
|
\
|
|
aria_sbox_8way_gfni(x2, x3, x0, x1, \
|
|
x6, x7, x4, x5, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7); \
|
|
\
|
|
aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
|
|
aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
|
|
aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 8); \
|
|
\
|
|
aria_load_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 0); \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 0, round); \
|
|
\
|
|
aria_sbox_8way_gfni(x2, x3, x0, x1, \
|
|
x6, x7, x4, x5, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7); \
|
|
\
|
|
aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
|
|
aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
|
|
aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 0); \
|
|
aria_load_state_8way(y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, 8); \
|
|
aria_diff_word(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7); \
|
|
/* aria_diff_byte() \
|
|
* T3 = ABCD -> BADC \
|
|
* T3 = y4, y5, y6, y7 -> y5, y4, y7, y6 \
|
|
* T0 = ABCD -> CDAB \
|
|
* T0 = x0, x1, x2, x3 -> x2, x3, x0, x1 \
|
|
* T1 = ABCD -> DCBA \
|
|
* T1 = x4, x5, x6, x7 -> x7, x6, x5, x4 \
|
|
*/ \
|
|
aria_diff_word(x2, x3, x0, x1, \
|
|
x7, x6, x5, x4, \
|
|
y0, y1, y2, y3, \
|
|
y5, y4, y7, y6); \
|
|
aria_store_state_8way(x3, x2, x1, x0, \
|
|
x6, x7, x4, x5, \
|
|
mem_tmp, 0);
|
|
|
|
#define aria_fo_gfni(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, rk, round) \
|
|
vpxor y7, y7, y7; \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 8, round); \
|
|
\
|
|
aria_sbox_8way_gfni(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7); \
|
|
\
|
|
aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
|
|
aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
|
|
aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 8); \
|
|
\
|
|
aria_load_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 0); \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 0, round); \
|
|
\
|
|
aria_sbox_8way_gfni(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7); \
|
|
\
|
|
aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
|
|
aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
|
|
aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 0); \
|
|
aria_load_state_8way(y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, 8); \
|
|
aria_diff_word(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7); \
|
|
/* aria_diff_byte() \
|
|
* T1 = ABCD -> BADC \
|
|
* T1 = x4, x5, x6, x7 -> x5, x4, x7, x6 \
|
|
* T2 = ABCD -> CDAB \
|
|
* T2 = y0, y1, y2, y3, -> y2, y3, y0, y1 \
|
|
* T3 = ABCD -> DCBA \
|
|
* T3 = y4, y5, y6, y7 -> y7, y6, y5, y4 \
|
|
*/ \
|
|
aria_diff_word(x0, x1, x2, x3, \
|
|
x5, x4, x7, x6, \
|
|
y2, y3, y0, y1, \
|
|
y7, y6, y5, y4); \
|
|
aria_store_state_8way(x3, x2, x1, x0, \
|
|
x6, x7, x4, x5, \
|
|
mem_tmp, 0);
|
|
|
|
#define aria_ff_gfni(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, rk, round, last_round) \
|
|
vpxor y7, y7, y7; \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 8, round); \
|
|
\
|
|
aria_sbox_8way_gfni(x2, x3, x0, x1, \
|
|
x6, x7, x4, x5, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7); \
|
|
\
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 8, last_round); \
|
|
\
|
|
aria_store_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 8); \
|
|
\
|
|
aria_load_state_8way(x0, x1, x2, x3, \
|
|
x4, x5, x6, x7, \
|
|
mem_tmp, 0); \
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 0, round); \
|
|
\
|
|
aria_sbox_8way_gfni(x2, x3, x0, x1, \
|
|
x6, x7, x4, x5, \
|
|
y0, y1, y2, y3, \
|
|
y4, y5, y6, y7); \
|
|
\
|
|
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
|
y0, y7, y2, rk, 0, last_round); \
|
|
\
|
|
aria_load_state_8way(y0, y1, y2, y3, \
|
|
y4, y5, y6, y7, \
|
|
mem_tmp, 8);
|
|
|
|
#endif /* CONFIG_AS_GFNI */
|
|
|
|
/* NB: section is mergeable, all elements must be aligned 16-byte blocks */
|
|
.section .rodata.cst16, "aM", @progbits, 16
|
|
.align 16
|
|
|
|
#define SHUFB_BYTES(idx) \
|
|
0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx)
|
|
|
|
.Lshufb_16x16b:
|
|
.byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3);
|
|
/* For isolating SubBytes from AESENCLAST, inverse shift row */
|
|
.Linv_shift_row:
|
|
.byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b
|
|
.byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03
|
|
.Lshift_row:
|
|
.byte 0x00, 0x05, 0x0a, 0x0f, 0x04, 0x09, 0x0e, 0x03
|
|
.byte 0x08, 0x0d, 0x02, 0x07, 0x0c, 0x01, 0x06, 0x0b
|
|
/* For CTR-mode IV byteswap */
|
|
.Lbswap128_mask:
|
|
.byte 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08
|
|
.byte 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
|
|
|
|
/* AES inverse affine and S2 combined:
|
|
* 1 1 0 0 0 0 0 1 x0 0
|
|
* 0 1 0 0 1 0 0 0 x1 0
|
|
* 1 1 0 0 1 1 1 1 x2 0
|
|
* 0 1 1 0 1 0 0 1 x3 1
|
|
* 0 1 0 0 1 1 0 0 * x4 + 0
|
|
* 0 1 0 1 1 0 0 0 x5 0
|
|
* 0 0 0 0 0 1 0 1 x6 0
|
|
* 1 1 1 0 0 1 1 1 x7 1
|
|
*/
|
|
.Ltf_lo__inv_aff__and__s2:
|
|
.octa 0x92172DA81A9FA520B2370D883ABF8500
|
|
.Ltf_hi__inv_aff__and__s2:
|
|
.octa 0x2B15FFC1AF917B45E6D8320C625CB688
|
|
|
|
/* X2 and AES forward affine combined:
|
|
* 1 0 1 1 0 0 0 1 x0 0
|
|
* 0 1 1 1 1 0 1 1 x1 0
|
|
* 0 0 0 1 1 0 1 0 x2 1
|
|
* 0 1 0 0 0 1 0 0 x3 0
|
|
* 0 0 1 1 1 0 1 1 * x4 + 0
|
|
* 0 1 0 0 1 0 0 0 x5 0
|
|
* 1 1 0 1 0 0 1 1 x6 0
|
|
* 0 1 0 0 1 0 1 0 x7 0
|
|
*/
|
|
.Ltf_lo__x2__and__fwd_aff:
|
|
.octa 0xEFAE0544FCBD1657B8F95213ABEA4100
|
|
.Ltf_hi__x2__and__fwd_aff:
|
|
.octa 0x3F893781E95FE1576CDA64D2BA0CB204
|
|
|
|
#ifdef CONFIG_AS_GFNI
|
|
/* AES affine: */
|
|
#define tf_aff_const BV8(1, 1, 0, 0, 0, 1, 1, 0)
|
|
.Ltf_aff_bitmatrix:
|
|
.quad BM8X8(BV8(1, 0, 0, 0, 1, 1, 1, 1),
|
|
BV8(1, 1, 0, 0, 0, 1, 1, 1),
|
|
BV8(1, 1, 1, 0, 0, 0, 1, 1),
|
|
BV8(1, 1, 1, 1, 0, 0, 0, 1),
|
|
BV8(1, 1, 1, 1, 1, 0, 0, 0),
|
|
BV8(0, 1, 1, 1, 1, 1, 0, 0),
|
|
BV8(0, 0, 1, 1, 1, 1, 1, 0),
|
|
BV8(0, 0, 0, 1, 1, 1, 1, 1))
|
|
.quad BM8X8(BV8(1, 0, 0, 0, 1, 1, 1, 1),
|
|
BV8(1, 1, 0, 0, 0, 1, 1, 1),
|
|
BV8(1, 1, 1, 0, 0, 0, 1, 1),
|
|
BV8(1, 1, 1, 1, 0, 0, 0, 1),
|
|
BV8(1, 1, 1, 1, 1, 0, 0, 0),
|
|
BV8(0, 1, 1, 1, 1, 1, 0, 0),
|
|
BV8(0, 0, 1, 1, 1, 1, 1, 0),
|
|
BV8(0, 0, 0, 1, 1, 1, 1, 1))
|
|
|
|
/* AES inverse affine: */
|
|
#define tf_inv_const BV8(1, 0, 1, 0, 0, 0, 0, 0)
|
|
.Ltf_inv_bitmatrix:
|
|
.quad BM8X8(BV8(0, 0, 1, 0, 0, 1, 0, 1),
|
|
BV8(1, 0, 0, 1, 0, 0, 1, 0),
|
|
BV8(0, 1, 0, 0, 1, 0, 0, 1),
|
|
BV8(1, 0, 1, 0, 0, 1, 0, 0),
|
|
BV8(0, 1, 0, 1, 0, 0, 1, 0),
|
|
BV8(0, 0, 1, 0, 1, 0, 0, 1),
|
|
BV8(1, 0, 0, 1, 0, 1, 0, 0),
|
|
BV8(0, 1, 0, 0, 1, 0, 1, 0))
|
|
.quad BM8X8(BV8(0, 0, 1, 0, 0, 1, 0, 1),
|
|
BV8(1, 0, 0, 1, 0, 0, 1, 0),
|
|
BV8(0, 1, 0, 0, 1, 0, 0, 1),
|
|
BV8(1, 0, 1, 0, 0, 1, 0, 0),
|
|
BV8(0, 1, 0, 1, 0, 0, 1, 0),
|
|
BV8(0, 0, 1, 0, 1, 0, 0, 1),
|
|
BV8(1, 0, 0, 1, 0, 1, 0, 0),
|
|
BV8(0, 1, 0, 0, 1, 0, 1, 0))
|
|
|
|
/* S2: */
|
|
#define tf_s2_const BV8(0, 1, 0, 0, 0, 1, 1, 1)
|
|
.Ltf_s2_bitmatrix:
|
|
.quad BM8X8(BV8(0, 1, 0, 1, 0, 1, 1, 1),
|
|
BV8(0, 0, 1, 1, 1, 1, 1, 1),
|
|
BV8(1, 1, 1, 0, 1, 1, 0, 1),
|
|
BV8(1, 1, 0, 0, 0, 0, 1, 1),
|
|
BV8(0, 1, 0, 0, 0, 0, 1, 1),
|
|
BV8(1, 1, 0, 0, 1, 1, 1, 0),
|
|
BV8(0, 1, 1, 0, 0, 0, 1, 1),
|
|
BV8(1, 1, 1, 1, 0, 1, 1, 0))
|
|
.quad BM8X8(BV8(0, 1, 0, 1, 0, 1, 1, 1),
|
|
BV8(0, 0, 1, 1, 1, 1, 1, 1),
|
|
BV8(1, 1, 1, 0, 1, 1, 0, 1),
|
|
BV8(1, 1, 0, 0, 0, 0, 1, 1),
|
|
BV8(0, 1, 0, 0, 0, 0, 1, 1),
|
|
BV8(1, 1, 0, 0, 1, 1, 1, 0),
|
|
BV8(0, 1, 1, 0, 0, 0, 1, 1),
|
|
BV8(1, 1, 1, 1, 0, 1, 1, 0))
|
|
|
|
/* X2: */
|
|
#define tf_x2_const BV8(0, 0, 1, 1, 0, 1, 0, 0)
|
|
.Ltf_x2_bitmatrix:
|
|
.quad BM8X8(BV8(0, 0, 0, 1, 1, 0, 0, 0),
|
|
BV8(0, 0, 1, 0, 0, 1, 1, 0),
|
|
BV8(0, 0, 0, 0, 1, 0, 1, 0),
|
|
BV8(1, 1, 1, 0, 0, 0, 1, 1),
|
|
BV8(1, 1, 1, 0, 1, 1, 0, 0),
|
|
BV8(0, 1, 1, 0, 1, 0, 1, 1),
|
|
BV8(1, 0, 1, 1, 1, 1, 0, 1),
|
|
BV8(1, 0, 0, 1, 0, 0, 1, 1))
|
|
.quad BM8X8(BV8(0, 0, 0, 1, 1, 0, 0, 0),
|
|
BV8(0, 0, 1, 0, 0, 1, 1, 0),
|
|
BV8(0, 0, 0, 0, 1, 0, 1, 0),
|
|
BV8(1, 1, 1, 0, 0, 0, 1, 1),
|
|
BV8(1, 1, 1, 0, 1, 1, 0, 0),
|
|
BV8(0, 1, 1, 0, 1, 0, 1, 1),
|
|
BV8(1, 0, 1, 1, 1, 1, 0, 1),
|
|
BV8(1, 0, 0, 1, 0, 0, 1, 1))
|
|
|
|
/* Identity matrix: */
|
|
.Ltf_id_bitmatrix:
|
|
.quad BM8X8(BV8(1, 0, 0, 0, 0, 0, 0, 0),
|
|
BV8(0, 1, 0, 0, 0, 0, 0, 0),
|
|
BV8(0, 0, 1, 0, 0, 0, 0, 0),
|
|
BV8(0, 0, 0, 1, 0, 0, 0, 0),
|
|
BV8(0, 0, 0, 0, 1, 0, 0, 0),
|
|
BV8(0, 0, 0, 0, 0, 1, 0, 0),
|
|
BV8(0, 0, 0, 0, 0, 0, 1, 0),
|
|
BV8(0, 0, 0, 0, 0, 0, 0, 1))
|
|
.quad BM8X8(BV8(1, 0, 0, 0, 0, 0, 0, 0),
|
|
BV8(0, 1, 0, 0, 0, 0, 0, 0),
|
|
BV8(0, 0, 1, 0, 0, 0, 0, 0),
|
|
BV8(0, 0, 0, 1, 0, 0, 0, 0),
|
|
BV8(0, 0, 0, 0, 1, 0, 0, 0),
|
|
BV8(0, 0, 0, 0, 0, 1, 0, 0),
|
|
BV8(0, 0, 0, 0, 0, 0, 1, 0),
|
|
BV8(0, 0, 0, 0, 0, 0, 0, 1))
|
|
#endif /* CONFIG_AS_GFNI */
|
|
|
|
/* 4-bit mask */
|
|
.section .rodata.cst4.L0f0f0f0f, "aM", @progbits, 4
|
|
.align 4
|
|
.L0f0f0f0f:
|
|
.long 0x0f0f0f0f
|
|
|
|
.text
|
|
|
|
SYM_FUNC_START_LOCAL(__aria_aesni_avx_crypt_16way)
|
|
/* input:
|
|
* %r9: rk
|
|
* %rsi: dst
|
|
* %rdx: src
|
|
* %xmm0..%xmm15: 16 byte-sliced blocks
|
|
*/
|
|
|
|
FRAME_BEGIN
|
|
|
|
movq %rsi, %rax;
|
|
leaq 8 * 16(%rax), %r8;
|
|
|
|
inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r8);
|
|
aria_fo(%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 0);
|
|
aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 1);
|
|
aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 2);
|
|
aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 3);
|
|
aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 4);
|
|
aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 5);
|
|
aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 6);
|
|
aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 7);
|
|
aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 8);
|
|
aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 9);
|
|
aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 10);
|
|
cmpl $12, ARIA_CTX_rounds(CTX);
|
|
jne .Laria_192;
|
|
aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 11, 12);
|
|
jmp .Laria_end;
|
|
.Laria_192:
|
|
aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 11);
|
|
aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 12);
|
|
cmpl $14, ARIA_CTX_rounds(CTX);
|
|
jne .Laria_256;
|
|
aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 13, 14);
|
|
jmp .Laria_end;
|
|
.Laria_256:
|
|
aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 13);
|
|
aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 14);
|
|
aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 15, 16);
|
|
.Laria_end:
|
|
debyteslice_16x16b(%xmm8, %xmm12, %xmm1, %xmm4,
|
|
%xmm9, %xmm13, %xmm0, %xmm5,
|
|
%xmm10, %xmm14, %xmm3, %xmm6,
|
|
%xmm11, %xmm15, %xmm2, %xmm7,
|
|
(%rax), (%r8));
|
|
|
|
FRAME_END
|
|
RET;
|
|
SYM_FUNC_END(__aria_aesni_avx_crypt_16way)
|
|
|
|
SYM_TYPED_FUNC_START(aria_aesni_avx_encrypt_16way)
|
|
/* input:
|
|
* %rdi: ctx, CTX
|
|
* %rsi: dst
|
|
* %rdx: src
|
|
*/
|
|
|
|
FRAME_BEGIN
|
|
|
|
leaq ARIA_CTX_enc_key(CTX), %r9;
|
|
|
|
inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rdx);
|
|
|
|
call __aria_aesni_avx_crypt_16way;
|
|
|
|
write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax);
|
|
|
|
FRAME_END
|
|
RET;
|
|
SYM_FUNC_END(aria_aesni_avx_encrypt_16way)
|
|
|
|
SYM_TYPED_FUNC_START(aria_aesni_avx_decrypt_16way)
|
|
/* input:
|
|
* %rdi: ctx, CTX
|
|
* %rsi: dst
|
|
* %rdx: src
|
|
*/
|
|
|
|
FRAME_BEGIN
|
|
|
|
leaq ARIA_CTX_dec_key(CTX), %r9;
|
|
|
|
inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rdx);
|
|
|
|
call __aria_aesni_avx_crypt_16way;
|
|
|
|
write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax);
|
|
|
|
FRAME_END
|
|
RET;
|
|
SYM_FUNC_END(aria_aesni_avx_decrypt_16way)
|
|
|
|
SYM_FUNC_START_LOCAL(__aria_aesni_avx_ctr_gen_keystream_16way)
|
|
/* input:
|
|
* %rdi: ctx
|
|
* %rsi: dst
|
|
* %rdx: src
|
|
* %rcx: keystream
|
|
* %r8: iv (big endian, 128bit)
|
|
*/
|
|
|
|
FRAME_BEGIN
|
|
/* load IV and byteswap */
|
|
vmovdqu (%r8), %xmm8;
|
|
|
|
vmovdqa .Lbswap128_mask (%rip), %xmm1;
|
|
vpshufb %xmm1, %xmm8, %xmm3; /* be => le */
|
|
|
|
vpcmpeqd %xmm0, %xmm0, %xmm0;
|
|
vpsrldq $8, %xmm0, %xmm0; /* low: -1, high: 0 */
|
|
|
|
/* construct IVs */
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm9;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm10;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm11;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm12;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm13;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm14;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm15;
|
|
vmovdqu %xmm8, (0 * 16)(%rcx);
|
|
vmovdqu %xmm9, (1 * 16)(%rcx);
|
|
vmovdqu %xmm10, (2 * 16)(%rcx);
|
|
vmovdqu %xmm11, (3 * 16)(%rcx);
|
|
vmovdqu %xmm12, (4 * 16)(%rcx);
|
|
vmovdqu %xmm13, (5 * 16)(%rcx);
|
|
vmovdqu %xmm14, (6 * 16)(%rcx);
|
|
vmovdqu %xmm15, (7 * 16)(%rcx);
|
|
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm8;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm9;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm10;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm11;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm12;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm13;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm14;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm15;
|
|
inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
|
|
vpshufb %xmm1, %xmm3, %xmm4;
|
|
vmovdqu %xmm4, (%r8);
|
|
|
|
vmovdqu (0 * 16)(%rcx), %xmm0;
|
|
vmovdqu (1 * 16)(%rcx), %xmm1;
|
|
vmovdqu (2 * 16)(%rcx), %xmm2;
|
|
vmovdqu (3 * 16)(%rcx), %xmm3;
|
|
vmovdqu (4 * 16)(%rcx), %xmm4;
|
|
vmovdqu (5 * 16)(%rcx), %xmm5;
|
|
vmovdqu (6 * 16)(%rcx), %xmm6;
|
|
vmovdqu (7 * 16)(%rcx), %xmm7;
|
|
|
|
FRAME_END
|
|
RET;
|
|
SYM_FUNC_END(__aria_aesni_avx_ctr_gen_keystream_16way)
|
|
|
|
SYM_TYPED_FUNC_START(aria_aesni_avx_ctr_crypt_16way)
|
|
/* input:
|
|
* %rdi: ctx
|
|
* %rsi: dst
|
|
* %rdx: src
|
|
* %rcx: keystream
|
|
* %r8: iv (big endian, 128bit)
|
|
*/
|
|
FRAME_BEGIN
|
|
|
|
call __aria_aesni_avx_ctr_gen_keystream_16way;
|
|
|
|
leaq (%rsi), %r10;
|
|
leaq (%rdx), %r11;
|
|
leaq (%rcx), %rsi;
|
|
leaq (%rcx), %rdx;
|
|
leaq ARIA_CTX_enc_key(CTX), %r9;
|
|
|
|
call __aria_aesni_avx_crypt_16way;
|
|
|
|
vpxor (0 * 16)(%r11), %xmm1, %xmm1;
|
|
vpxor (1 * 16)(%r11), %xmm0, %xmm0;
|
|
vpxor (2 * 16)(%r11), %xmm3, %xmm3;
|
|
vpxor (3 * 16)(%r11), %xmm2, %xmm2;
|
|
vpxor (4 * 16)(%r11), %xmm4, %xmm4;
|
|
vpxor (5 * 16)(%r11), %xmm5, %xmm5;
|
|
vpxor (6 * 16)(%r11), %xmm6, %xmm6;
|
|
vpxor (7 * 16)(%r11), %xmm7, %xmm7;
|
|
vpxor (8 * 16)(%r11), %xmm8, %xmm8;
|
|
vpxor (9 * 16)(%r11), %xmm9, %xmm9;
|
|
vpxor (10 * 16)(%r11), %xmm10, %xmm10;
|
|
vpxor (11 * 16)(%r11), %xmm11, %xmm11;
|
|
vpxor (12 * 16)(%r11), %xmm12, %xmm12;
|
|
vpxor (13 * 16)(%r11), %xmm13, %xmm13;
|
|
vpxor (14 * 16)(%r11), %xmm14, %xmm14;
|
|
vpxor (15 * 16)(%r11), %xmm15, %xmm15;
|
|
write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %r10);
|
|
|
|
FRAME_END
|
|
RET;
|
|
SYM_FUNC_END(aria_aesni_avx_ctr_crypt_16way)
|
|
|
|
#ifdef CONFIG_AS_GFNI
|
|
SYM_FUNC_START_LOCAL(__aria_aesni_avx_gfni_crypt_16way)
|
|
/* input:
|
|
* %r9: rk
|
|
* %rsi: dst
|
|
* %rdx: src
|
|
* %xmm0..%xmm15: 16 byte-sliced blocks
|
|
*/
|
|
|
|
FRAME_BEGIN
|
|
|
|
movq %rsi, %rax;
|
|
leaq 8 * 16(%rax), %r8;
|
|
|
|
inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r8);
|
|
aria_fo_gfni(%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 0);
|
|
aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 1);
|
|
aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
|
|
%xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 2);
|
|
aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 3);
|
|
aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
|
|
%xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 4);
|
|
aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 5);
|
|
aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
|
|
%xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 6);
|
|
aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 7);
|
|
aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
|
|
%xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 8);
|
|
aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 9);
|
|
aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
|
|
%xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 10);
|
|
cmpl $12, ARIA_CTX_rounds(CTX);
|
|
jne .Laria_gfni_192;
|
|
aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 11, 12);
|
|
jmp .Laria_gfni_end;
|
|
.Laria_gfni_192:
|
|
aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 11);
|
|
aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
|
|
%xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 12);
|
|
cmpl $14, ARIA_CTX_rounds(CTX);
|
|
jne .Laria_gfni_256;
|
|
aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 13, 14);
|
|
jmp .Laria_gfni_end;
|
|
.Laria_gfni_256:
|
|
aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 13);
|
|
aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
|
|
%xmm12, %xmm13, %xmm14, %xmm15,
|
|
%xmm0, %xmm1, %xmm2, %xmm3,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%rax, %r9, 14);
|
|
aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
|
|
%xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11,
|
|
%xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax, %r9, 15, 16);
|
|
.Laria_gfni_end:
|
|
debyteslice_16x16b(%xmm8, %xmm12, %xmm1, %xmm4,
|
|
%xmm9, %xmm13, %xmm0, %xmm5,
|
|
%xmm10, %xmm14, %xmm3, %xmm6,
|
|
%xmm11, %xmm15, %xmm2, %xmm7,
|
|
(%rax), (%r8));
|
|
|
|
FRAME_END
|
|
RET;
|
|
SYM_FUNC_END(__aria_aesni_avx_gfni_crypt_16way)
|
|
|
|
SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_encrypt_16way)
|
|
/* input:
|
|
* %rdi: ctx, CTX
|
|
* %rsi: dst
|
|
* %rdx: src
|
|
*/
|
|
|
|
FRAME_BEGIN
|
|
|
|
leaq ARIA_CTX_enc_key(CTX), %r9;
|
|
|
|
inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rdx);
|
|
|
|
call __aria_aesni_avx_gfni_crypt_16way;
|
|
|
|
write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax);
|
|
|
|
FRAME_END
|
|
RET;
|
|
SYM_FUNC_END(aria_aesni_avx_gfni_encrypt_16way)
|
|
|
|
SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_decrypt_16way)
|
|
/* input:
|
|
* %rdi: ctx, CTX
|
|
* %rsi: dst
|
|
* %rdx: src
|
|
*/
|
|
|
|
FRAME_BEGIN
|
|
|
|
leaq ARIA_CTX_dec_key(CTX), %r9;
|
|
|
|
inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rdx);
|
|
|
|
call __aria_aesni_avx_gfni_crypt_16way;
|
|
|
|
write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %rax);
|
|
|
|
FRAME_END
|
|
RET;
|
|
SYM_FUNC_END(aria_aesni_avx_gfni_decrypt_16way)
|
|
|
|
SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_ctr_crypt_16way)
|
|
/* input:
|
|
* %rdi: ctx
|
|
* %rsi: dst
|
|
* %rdx: src
|
|
* %rcx: keystream
|
|
* %r8: iv (big endian, 128bit)
|
|
*/
|
|
FRAME_BEGIN
|
|
|
|
call __aria_aesni_avx_ctr_gen_keystream_16way
|
|
|
|
leaq (%rsi), %r10;
|
|
leaq (%rdx), %r11;
|
|
leaq (%rcx), %rsi;
|
|
leaq (%rcx), %rdx;
|
|
leaq ARIA_CTX_enc_key(CTX), %r9;
|
|
|
|
call __aria_aesni_avx_gfni_crypt_16way;
|
|
|
|
vpxor (0 * 16)(%r11), %xmm1, %xmm1;
|
|
vpxor (1 * 16)(%r11), %xmm0, %xmm0;
|
|
vpxor (2 * 16)(%r11), %xmm3, %xmm3;
|
|
vpxor (3 * 16)(%r11), %xmm2, %xmm2;
|
|
vpxor (4 * 16)(%r11), %xmm4, %xmm4;
|
|
vpxor (5 * 16)(%r11), %xmm5, %xmm5;
|
|
vpxor (6 * 16)(%r11), %xmm6, %xmm6;
|
|
vpxor (7 * 16)(%r11), %xmm7, %xmm7;
|
|
vpxor (8 * 16)(%r11), %xmm8, %xmm8;
|
|
vpxor (9 * 16)(%r11), %xmm9, %xmm9;
|
|
vpxor (10 * 16)(%r11), %xmm10, %xmm10;
|
|
vpxor (11 * 16)(%r11), %xmm11, %xmm11;
|
|
vpxor (12 * 16)(%r11), %xmm12, %xmm12;
|
|
vpxor (13 * 16)(%r11), %xmm13, %xmm13;
|
|
vpxor (14 * 16)(%r11), %xmm14, %xmm14;
|
|
vpxor (15 * 16)(%r11), %xmm15, %xmm15;
|
|
write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
|
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
|
%xmm15, %r10);
|
|
|
|
FRAME_END
|
|
RET;
|
|
SYM_FUNC_END(aria_aesni_avx_gfni_ctr_crypt_16way)
|
|
#endif /* CONFIG_AS_GFNI */
|