korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-11-24 04:34:08 +08:00
Code Issues Packages Projects Releases Wiki Activity
c0f08d6b5a
linux/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
Nathan Huckleberry fd94fcf099 crypto: x86/aesni-xctr - Add accelerated implementation of XCTR 
Add hardware accelerated version of XCTR for x86-64 CPUs with AESNI
support.

More information on XCTR can be found in the HCTR2 paper:
"Length-preserving encryption with HCTR2":
https://eprint.iacr.org/2021/1441.pdf

Signed-off-by: Nathan Huckleberry <nhuck@google.com>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
2022-06-10 16:40:17 +08:00

598 lines
12 KiB
ArmAsm
Raw Blame History

/* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */
/*
* AES CTR mode by8 optimization with AVX instructions. (x86_64)
*
* Copyright(c) 2014 Intel Corporation.
*
* Contact Information:
* James Guilford <james.guilford@intel.com>
* Sean Gulley <sean.m.gulley@intel.com>
* Chandramouli Narayanan <mouli@linux.intel.com>
*/
/*
* This is AES128/192/256 CTR mode optimization implementation. It requires
* the support of Intel(R) AESNI and AVX instructions.
*
* This work was inspired by the AES CTR mode optimization published
* in Intel Optimized IPSEC Cryptographic library.
* Additional information on it can be found at:
* https://github.com/intel/intel-ipsec-mb
*/
#include <linux/linkage.h>
#define VMOVDQ vmovdqu
/*
* Note: the "x" prefix in these aliases means "this is an xmm register". The
* alias prefixes have no relation to XCTR where the "X" prefix means "XOR
* counter".
*/
#define xdata0 %xmm0
#define xdata1 %xmm1
#define xdata2 %xmm2
#define xdata3 %xmm3
#define xdata4 %xmm4
#define xdata5 %xmm5
#define xdata6 %xmm6
#define xdata7 %xmm7
#define xcounter %xmm8 // CTR mode only
#define xiv %xmm8 // XCTR mode only
#define xbyteswap %xmm9 // CTR mode only
#define xtmp %xmm9 // XCTR mode only
#define xkey0 %xmm10
#define xkey4 %xmm11
#define xkey8 %xmm12
#define xkey12 %xmm13
#define xkeyA %xmm14
#define xkeyB %xmm15
#define p_in %rdi
#define p_iv %rsi
#define p_keys %rdx
#define p_out %rcx
#define num_bytes %r8
#define counter %r9 // XCTR mode only
#define tmp %r10
#define DDQ_DATA 0
#define XDATA 1
#define KEY_128 1
#define KEY_192 2
#define KEY_256 3
.section .rodata
.align 16
byteswap_const:
.octa 0x000102030405060708090A0B0C0D0E0F
ddq_low_msk:
.octa 0x0000000000000000FFFFFFFFFFFFFFFF
ddq_high_add_1:
.octa 0x00000000000000010000000000000000
ddq_add_1:
.octa 0x00000000000000000000000000000001
ddq_add_2:
.octa 0x00000000000000000000000000000002
ddq_add_3:
.octa 0x00000000000000000000000000000003
ddq_add_4:
.octa 0x00000000000000000000000000000004
ddq_add_5:
.octa 0x00000000000000000000000000000005
ddq_add_6:
.octa 0x00000000000000000000000000000006
ddq_add_7:
.octa 0x00000000000000000000000000000007
ddq_add_8:
.octa 0x00000000000000000000000000000008
.text
/* generate a unique variable for ddq_add_x */
/* generate a unique variable for xmm register */
.macro setxdata n
var_xdata = %xmm\n
.endm
/* club the numeric 'id' to the symbol 'name' */
.macro club name, id
.altmacro
.if \name == XDATA
setxdata %\id
.endif
.noaltmacro
.endm
/*
* do_aes num_in_par load_keys key_len
* This increments p_in, but not p_out
*/
.macro do_aes b, k, key_len, xctr
.set by, \b
.set load_keys, \k
.set klen, \key_len
.if (load_keys)
vmovdqa 0*16(p_keys), xkey0
.endif
.if \xctr
movq counter, xtmp
.set i, 0
.rept (by)
club XDATA, i
vpaddq (ddq_add_1 + 16 * i)(%rip), xtmp, var_xdata
.set i, (i +1)
.endr
.set i, 0
.rept (by)
club XDATA, i
vpxor xiv, var_xdata, var_xdata
.set i, (i +1)
.endr
.else
vpshufb xbyteswap, xcounter, xdata0
.set i, 1
.rept (by - 1)
club XDATA, i
vpaddq (ddq_add_1 + 16 * (i - 1))(%rip), xcounter, var_xdata
vptest ddq_low_msk(%rip), var_xdata
jnz 1f
vpaddq ddq_high_add_1(%rip), var_xdata, var_xdata
vpaddq ddq_high_add_1(%rip), xcounter, xcounter
1:
vpshufb xbyteswap, var_xdata, var_xdata
.set i, (i +1)
.endr
.endif
vmovdqa 1*16(p_keys), xkeyA
vpxor xkey0, xdata0, xdata0
.if \xctr
add $by, counter
.else
vpaddq (ddq_add_1 + 16 * (by - 1))(%rip), xcounter, xcounter
vptest ddq_low_msk(%rip), xcounter
jnz 1f
vpaddq ddq_high_add_1(%rip), xcounter, xcounter
1:
.endif
.set i, 1
.rept (by - 1)
club XDATA, i
vpxor xkey0, var_xdata, var_xdata
.set i, (i +1)
.endr
vmovdqa 2*16(p_keys), xkeyB
.set i, 0
.rept by
club XDATA, i
vaesenc xkeyA, var_xdata, var_xdata /* key 1 */
.set i, (i +1)
.endr
.if (klen == KEY_128)
.if (load_keys)
vmovdqa 3*16(p_keys), xkey4
.endif
.else
vmovdqa 3*16(p_keys), xkeyA
.endif
.set i, 0
.rept by
club XDATA, i
vaesenc xkeyB, var_xdata, var_xdata /* key 2 */
.set i, (i +1)
.endr
add $(16*by), p_in
.if (klen == KEY_128)
vmovdqa 4*16(p_keys), xkeyB
.else
.if (load_keys)
vmovdqa 4*16(p_keys), xkey4
.endif
.endif
.set i, 0
.rept by
club XDATA, i
/* key 3 */
.if (klen == KEY_128)
vaesenc xkey4, var_xdata, var_xdata
.else
vaesenc xkeyA, var_xdata, var_xdata
.endif
.set i, (i +1)
.endr
vmovdqa 5*16(p_keys), xkeyA
.set i, 0
.rept by
club XDATA, i
/* key 4 */
.if (klen == KEY_128)
vaesenc xkeyB, var_xdata, var_xdata
.else
vaesenc xkey4, var_xdata, var_xdata
.endif
.set i, (i +1)
.endr
.if (klen == KEY_128)
.if (load_keys)
vmovdqa 6*16(p_keys), xkey8
.endif
.else
vmovdqa 6*16(p_keys), xkeyB
.endif
.set i, 0
.rept by
club XDATA, i
vaesenc xkeyA, var_xdata, var_xdata /* key 5 */
.set i, (i +1)
.endr
vmovdqa 7*16(p_keys), xkeyA
.set i, 0
.rept by
club XDATA, i
/* key 6 */
.if (klen == KEY_128)
vaesenc xkey8, var_xdata, var_xdata
.else
vaesenc xkeyB, var_xdata, var_xdata
.endif
.set i, (i +1)
.endr
.if (klen == KEY_128)
vmovdqa 8*16(p_keys), xkeyB
.else
.if (load_keys)
vmovdqa 8*16(p_keys), xkey8
.endif
.endif
.set i, 0
.rept by
club XDATA, i
vaesenc xkeyA, var_xdata, var_xdata /* key 7 */
.set i, (i +1)
.endr
.if (klen == KEY_128)
.if (load_keys)
vmovdqa 9*16(p_keys), xkey12
.endif
.else
vmovdqa 9*16(p_keys), xkeyA
.endif
.set i, 0
.rept by
club XDATA, i
/* key 8 */
.if (klen == KEY_128)
vaesenc xkeyB, var_xdata, var_xdata
.else
vaesenc xkey8, var_xdata, var_xdata
.endif
.set i, (i +1)
.endr
vmovdqa 10*16(p_keys), xkeyB
.set i, 0
.rept by
club XDATA, i
/* key 9 */
.if (klen == KEY_128)
vaesenc xkey12, var_xdata, var_xdata
.else
vaesenc xkeyA, var_xdata, var_xdata
.endif
.set i, (i +1)
.endr
.if (klen != KEY_128)
vmovdqa 11*16(p_keys), xkeyA
.endif
.set i, 0
.rept by
club XDATA, i
/* key 10 */
.if (klen == KEY_128)
vaesenclast xkeyB, var_xdata, var_xdata
.else
vaesenc xkeyB, var_xdata, var_xdata
.endif
.set i, (i +1)
.endr
.if (klen != KEY_128)
.if (load_keys)
vmovdqa 12*16(p_keys), xkey12
.endif
.set i, 0
.rept by
club XDATA, i
vaesenc xkeyA, var_xdata, var_xdata /* key 11 */
.set i, (i +1)
.endr
.if (klen == KEY_256)
vmovdqa 13*16(p_keys), xkeyA
.endif
.set i, 0
.rept by
club XDATA, i
.if (klen == KEY_256)
/* key 12 */
vaesenc xkey12, var_xdata, var_xdata
.else
vaesenclast xkey12, var_xdata, var_xdata
.endif
.set i, (i +1)
.endr
.if (klen == KEY_256)
vmovdqa 14*16(p_keys), xkeyB
.set i, 0
.rept by
club XDATA, i
/* key 13 */
vaesenc xkeyA, var_xdata, var_xdata
.set i, (i +1)
.endr
.set i, 0
.rept by
club XDATA, i
/* key 14 */
vaesenclast xkeyB, var_xdata, var_xdata
.set i, (i +1)
.endr
.endif
.endif
.set i, 0
.rept (by / 2)
.set j, (i+1)
VMOVDQ (i*16 - 16*by)(p_in), xkeyA
VMOVDQ (j*16 - 16*by)(p_in), xkeyB
club XDATA, i
vpxor xkeyA, var_xdata, var_xdata
club XDATA, j
vpxor xkeyB, var_xdata, var_xdata
.set i, (i+2)
.endr
.if (i < by)
VMOVDQ (i*16 - 16*by)(p_in), xkeyA
club XDATA, i
vpxor xkeyA, var_xdata, var_xdata
.endif
.set i, 0
.rept by
club XDATA, i
VMOVDQ var_xdata, i*16(p_out)
.set i, (i+1)
.endr
.endm
.macro do_aes_load val, key_len, xctr
do_aes \val, 1, \key_len, \xctr
.endm
.macro do_aes_noload val, key_len, xctr
do_aes \val, 0, \key_len, \xctr
.endm
/* main body of aes ctr load */
.macro do_aes_ctrmain key_len, xctr
cmp $16, num_bytes
jb .Ldo_return2\xctr\key_len
.if \xctr
shr $4, counter
vmovdqu (p_iv), xiv
.else
vmovdqa byteswap_const(%rip), xbyteswap
vmovdqu (p_iv), xcounter
vpshufb xbyteswap, xcounter, xcounter
.endif
mov num_bytes, tmp
and $(7*16), tmp
jz .Lmult_of_8_blks\xctr\key_len
/* 1 <= tmp <= 7 */
cmp $(4*16), tmp
jg .Lgt4\xctr\key_len
je .Leq4\xctr\key_len
.Llt4\xctr\key_len:
cmp $(2*16), tmp
jg .Leq3\xctr\key_len
je .Leq2\xctr\key_len
.Leq1\xctr\key_len:
do_aes_load 1, \key_len, \xctr
add $(1*16), p_out
and $(~7*16), num_bytes
jz .Ldo_return2\xctr\key_len
jmp .Lmain_loop2\xctr\key_len
.Leq2\xctr\key_len:
do_aes_load 2, \key_len, \xctr
add $(2*16), p_out
and $(~7*16), num_bytes
jz .Ldo_return2\xctr\key_len
jmp .Lmain_loop2\xctr\key_len
.Leq3\xctr\key_len:
do_aes_load 3, \key_len, \xctr
add $(3*16), p_out
and $(~7*16), num_bytes
jz .Ldo_return2\xctr\key_len
jmp .Lmain_loop2\xctr\key_len
.Leq4\xctr\key_len:
do_aes_load 4, \key_len, \xctr
add $(4*16), p_out
and $(~7*16), num_bytes
jz .Ldo_return2\xctr\key_len
jmp .Lmain_loop2\xctr\key_len
.Lgt4\xctr\key_len:
cmp $(6*16), tmp
jg .Leq7\xctr\key_len
je .Leq6\xctr\key_len
.Leq5\xctr\key_len:
do_aes_load 5, \key_len, \xctr
add $(5*16), p_out
and $(~7*16), num_bytes
jz .Ldo_return2\xctr\key_len
jmp .Lmain_loop2\xctr\key_len
.Leq6\xctr\key_len:
do_aes_load 6, \key_len, \xctr
add $(6*16), p_out
and $(~7*16), num_bytes
jz .Ldo_return2\xctr\key_len
jmp .Lmain_loop2\xctr\key_len
.Leq7\xctr\key_len:
do_aes_load 7, \key_len, \xctr
add $(7*16), p_out
and $(~7*16), num_bytes
jz .Ldo_return2\xctr\key_len
jmp .Lmain_loop2\xctr\key_len
.Lmult_of_8_blks\xctr\key_len:
.if (\key_len != KEY_128)
vmovdqa 0*16(p_keys), xkey0
vmovdqa 4*16(p_keys), xkey4
vmovdqa 8*16(p_keys), xkey8
vmovdqa 12*16(p_keys), xkey12
.else
vmovdqa 0*16(p_keys), xkey0
vmovdqa 3*16(p_keys), xkey4
vmovdqa 6*16(p_keys), xkey8
vmovdqa 9*16(p_keys), xkey12
.endif
.align 16
.Lmain_loop2\xctr\key_len:
/* num_bytes is a multiple of 8 and >0 */
do_aes_noload 8, \key_len, \xctr
add $(8*16), p_out
sub $(8*16), num_bytes
jne .Lmain_loop2\xctr\key_len
.Ldo_return2\xctr\key_len:
.if !\xctr
/* return updated IV */
vpshufb xbyteswap, xcounter, xcounter
vmovdqu xcounter, (p_iv)
.endif
RET
.endm
/*
* routine to do AES128 CTR enc/decrypt "by8"
* XMM registers are clobbered.
* Saving/restoring must be done at a higher level
* aes_ctr_enc_128_avx_by8(void *in, void *iv, void *keys, void *out,
* unsigned int num_bytes)
*/
SYM_FUNC_START(aes_ctr_enc_128_avx_by8)
/* call the aes main loop */
do_aes_ctrmain KEY_128 0
SYM_FUNC_END(aes_ctr_enc_128_avx_by8)
/*
* routine to do AES192 CTR enc/decrypt "by8"
* XMM registers are clobbered.
* Saving/restoring must be done at a higher level
* aes_ctr_enc_192_avx_by8(void *in, void *iv, void *keys, void *out,
* unsigned int num_bytes)
*/
SYM_FUNC_START(aes_ctr_enc_192_avx_by8)
/* call the aes main loop */
do_aes_ctrmain KEY_192 0
SYM_FUNC_END(aes_ctr_enc_192_avx_by8)
/*
* routine to do AES256 CTR enc/decrypt "by8"
* XMM registers are clobbered.
* Saving/restoring must be done at a higher level
* aes_ctr_enc_256_avx_by8(void *in, void *iv, void *keys, void *out,
* unsigned int num_bytes)
*/
SYM_FUNC_START(aes_ctr_enc_256_avx_by8)
/* call the aes main loop */
do_aes_ctrmain KEY_256 0
SYM_FUNC_END(aes_ctr_enc_256_avx_by8)
/*
* routine to do AES128 XCTR enc/decrypt "by8"
* XMM registers are clobbered.
* Saving/restoring must be done at a higher level
* aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv, const void *keys,
* u8* out, unsigned int num_bytes, unsigned int byte_ctr)
*/
SYM_FUNC_START(aes_xctr_enc_128_avx_by8)
/* call the aes main loop */
do_aes_ctrmain KEY_128 1
SYM_FUNC_END(aes_xctr_enc_128_avx_by8)
/*
* routine to do AES192 XCTR enc/decrypt "by8"
* XMM registers are clobbered.
* Saving/restoring must be done at a higher level
* aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv, const void *keys,
* u8* out, unsigned int num_bytes, unsigned int byte_ctr)
*/
SYM_FUNC_START(aes_xctr_enc_192_avx_by8)
/* call the aes main loop */
do_aes_ctrmain KEY_192 1
SYM_FUNC_END(aes_xctr_enc_192_avx_by8)
/*
* routine to do AES256 XCTR enc/decrypt "by8"
* XMM registers are clobbered.
* Saving/restoring must be done at a higher level
* aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv, const void *keys,
* u8* out, unsigned int num_bytes, unsigned int byte_ctr)
*/
SYM_FUNC_START(aes_xctr_enc_256_avx_by8)
/* call the aes main loop */
do_aes_ctrmain KEY_256 1
SYM_FUNC_END(aes_xctr_enc_256_avx_by8)
Reference in New Issue View Git Blame Copy Permalink