mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-11-17 09:14:19 +08:00
34f7f6c301
Add hardware accelerated version of POLYVAL for x86-64 CPUs with PCLMULQDQ support. This implementation is accelerated using PCLMULQDQ instructions to perform the finite field computations. For added efficiency, 8 blocks of the message are processed simultaneously by precomputing the first 8 powers of the key. Schoolbook multiplication is used instead of Karatsuba multiplication because it was found to be slightly faster on x86-64 machines. Montgomery reduction must be used instead of Barrett reduction due to the difference in modulus between POLYVAL's field and other finite fields. More information on POLYVAL can be found in the HCTR2 paper: "Length-preserving encryption with HCTR2": https://eprint.iacr.org/2021/1441.pdf Signed-off-by: Nathan Huckleberry <nhuck@google.com> Reviewed-by: Ard Biesheuvel <ardb@kernel.org> Reviewed-by: Eric Biggers <ebiggers@google.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
204 lines
4.8 KiB
C
204 lines
4.8 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* Glue code for POLYVAL using PCMULQDQ-NI
|
|
*
|
|
* Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
|
|
* Copyright (c) 2009 Intel Corp.
|
|
* Author: Huang Ying <ying.huang@intel.com>
|
|
* Copyright 2021 Google LLC
|
|
*/
|
|
|
|
/*
|
|
* Glue code based on ghash-clmulni-intel_glue.c.
|
|
*
|
|
* This implementation of POLYVAL uses montgomery multiplication
|
|
* accelerated by PCLMULQDQ-NI to implement the finite field
|
|
* operations.
|
|
*/
|
|
|
|
#include <crypto/algapi.h>
|
|
#include <crypto/internal/hash.h>
|
|
#include <crypto/internal/simd.h>
|
|
#include <crypto/polyval.h>
|
|
#include <linux/crypto.h>
|
|
#include <linux/init.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/module.h>
|
|
#include <asm/cpu_device_id.h>
|
|
#include <asm/simd.h>
|
|
|
|
#define NUM_KEY_POWERS 8
|
|
|
|
struct polyval_tfm_ctx {
|
|
/*
|
|
* These powers must be in the order h^8, ..., h^1.
|
|
*/
|
|
u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE];
|
|
};
|
|
|
|
struct polyval_desc_ctx {
|
|
u8 buffer[POLYVAL_BLOCK_SIZE];
|
|
u32 bytes;
|
|
};
|
|
|
|
asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys,
|
|
const u8 *in, size_t nblocks, u8 *accumulator);
|
|
asmlinkage void clmul_polyval_mul(u8 *op1, const u8 *op2);
|
|
|
|
static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
|
|
const u8 *in, size_t nblocks, u8 *accumulator)
|
|
{
|
|
if (likely(crypto_simd_usable())) {
|
|
kernel_fpu_begin();
|
|
clmul_polyval_update(keys, in, nblocks, accumulator);
|
|
kernel_fpu_end();
|
|
} else {
|
|
polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in,
|
|
nblocks, accumulator);
|
|
}
|
|
}
|
|
|
|
static void internal_polyval_mul(u8 *op1, const u8 *op2)
|
|
{
|
|
if (likely(crypto_simd_usable())) {
|
|
kernel_fpu_begin();
|
|
clmul_polyval_mul(op1, op2);
|
|
kernel_fpu_end();
|
|
} else {
|
|
polyval_mul_non4k(op1, op2);
|
|
}
|
|
}
|
|
|
|
static int polyval_x86_setkey(struct crypto_shash *tfm,
|
|
const u8 *key, unsigned int keylen)
|
|
{
|
|
struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm);
|
|
int i;
|
|
|
|
if (keylen != POLYVAL_BLOCK_SIZE)
|
|
return -EINVAL;
|
|
|
|
memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE);
|
|
|
|
for (i = NUM_KEY_POWERS-2; i >= 0; i--) {
|
|
memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE);
|
|
internal_polyval_mul(tctx->key_powers[i],
|
|
tctx->key_powers[i+1]);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int polyval_x86_init(struct shash_desc *desc)
|
|
{
|
|
struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
|
|
|
|
memset(dctx, 0, sizeof(*dctx));
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int polyval_x86_update(struct shash_desc *desc,
|
|
const u8 *src, unsigned int srclen)
|
|
{
|
|
struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
|
|
const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
|
|
u8 *pos;
|
|
unsigned int nblocks;
|
|
unsigned int n;
|
|
|
|
if (dctx->bytes) {
|
|
n = min(srclen, dctx->bytes);
|
|
pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes;
|
|
|
|
dctx->bytes -= n;
|
|
srclen -= n;
|
|
|
|
while (n--)
|
|
*pos++ ^= *src++;
|
|
|
|
if (!dctx->bytes)
|
|
internal_polyval_mul(dctx->buffer,
|
|
tctx->key_powers[NUM_KEY_POWERS-1]);
|
|
}
|
|
|
|
while (srclen >= POLYVAL_BLOCK_SIZE) {
|
|
/* Allow rescheduling every 4K bytes. */
|
|
nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
|
|
internal_polyval_update(tctx, src, nblocks, dctx->buffer);
|
|
srclen -= nblocks * POLYVAL_BLOCK_SIZE;
|
|
src += nblocks * POLYVAL_BLOCK_SIZE;
|
|
}
|
|
|
|
if (srclen) {
|
|
dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
|
|
pos = dctx->buffer;
|
|
while (srclen--)
|
|
*pos++ ^= *src++;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int polyval_x86_final(struct shash_desc *desc, u8 *dst)
|
|
{
|
|
struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
|
|
const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
|
|
|
|
if (dctx->bytes) {
|
|
internal_polyval_mul(dctx->buffer,
|
|
tctx->key_powers[NUM_KEY_POWERS-1]);
|
|
}
|
|
|
|
memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct shash_alg polyval_alg = {
|
|
.digestsize = POLYVAL_DIGEST_SIZE,
|
|
.init = polyval_x86_init,
|
|
.update = polyval_x86_update,
|
|
.final = polyval_x86_final,
|
|
.setkey = polyval_x86_setkey,
|
|
.descsize = sizeof(struct polyval_desc_ctx),
|
|
.base = {
|
|
.cra_name = "polyval",
|
|
.cra_driver_name = "polyval-clmulni",
|
|
.cra_priority = 200,
|
|
.cra_blocksize = POLYVAL_BLOCK_SIZE,
|
|
.cra_ctxsize = sizeof(struct polyval_tfm_ctx),
|
|
.cra_module = THIS_MODULE,
|
|
},
|
|
};
|
|
|
|
__maybe_unused static const struct x86_cpu_id pcmul_cpu_id[] = {
|
|
X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
|
|
{}
|
|
};
|
|
MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);
|
|
|
|
static int __init polyval_clmulni_mod_init(void)
|
|
{
|
|
if (!x86_match_cpu(pcmul_cpu_id))
|
|
return -ENODEV;
|
|
|
|
if (!boot_cpu_has(X86_FEATURE_AVX))
|
|
return -ENODEV;
|
|
|
|
return crypto_register_shash(&polyval_alg);
|
|
}
|
|
|
|
static void __exit polyval_clmulni_mod_exit(void)
|
|
{
|
|
crypto_unregister_shash(&polyval_alg);
|
|
}
|
|
|
|
module_init(polyval_clmulni_mod_init);
|
|
module_exit(polyval_clmulni_mod_exit);
|
|
|
|
MODULE_LICENSE("GPL");
|
|
MODULE_DESCRIPTION("POLYVAL hash function accelerated by PCLMULQDQ-NI");
|
|
MODULE_ALIAS_CRYPTO("polyval");
|
|
MODULE_ALIAS_CRYPTO("polyval-clmulni");
|