archives/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-03-08 03:04:51 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

lib/crypto: arm64/nh: Migrate optimized code into library

Browse source 
Migrate the arm64 NEON implementation of NH into lib/crypto/.  This
makes the nh() function be optimized on arm64 kernels.

Note: this temporarily makes the adiantum template not utilize the arm64
optimized NH code.  This is resolved in a later commit that converts the
adiantum template to use nh() instead of "nhpoly1305".

Link: https://lore.kernel.org/r/20251211011846.8179-5-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers 2025-12-10 17:18:36 -08:00
parent 29e39a11f5
commit b4a8528d17
7 changed files with 37 additions and 94 deletions
Download patch file Download diff file Expand all files Collapse all files

10
arch/arm64/crypto/Kconfig
View file

@ -15,16 +15,6 @@ config CRYPTO_GHASH_ARM64_CE
Architecture: arm64 using:
- ARMv8 Crypto Extensions
config CRYPTO_NHPOLY1305_NEON
tristate "Hash functions: NHPoly1305 (NEON)"
depends on KERNEL_MODE_NEON
select CRYPTO_NHPOLY1305
help
NHPoly1305 hash function (Adiantum)
Architecture: arm64 using:
- NEON (Advanced SIMD) extensions
config CRYPTO_SM3_NEON
tristate "Hash functions: SM3 (NEON)"
depends on KERNEL_MODE_NEON

3
arch/arm64/crypto/Makefile
View file

@ -41,9 +41,6 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o
obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
aes-neon-blk-y := aes-glue-neon.o aes-neon.o
obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o
nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o
obj-$(CONFIG_CRYPTO_AES_ARM64) += aes-arm64.o
aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o

79
arch/arm64/crypto/nhpoly1305-neon-glue.c
View file

@ -1,79 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/*
* NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
* (ARM64 NEON accelerated version)
*
* Copyright 2018 Google LLC
*/
#include <asm/neon.h>
#include <asm/simd.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/simd.h>
#include <crypto/nhpoly1305.h>
#include <linux/module.h>
asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
__le64 hash[NH_NUM_PASSES]);
static int nhpoly1305_neon_update(struct shash_desc *desc,
const u8 *src, unsigned int srclen)
{
if (srclen < 64 || !crypto_simd_usable())
return crypto_nhpoly1305_update(desc, src, srclen);
do {
unsigned int n = min_t(unsigned int, srclen, SZ_4K);
scoped_ksimd()
crypto_nhpoly1305_update_helper(desc, src, n, nh_neon);
src += n;
srclen -= n;
} while (srclen);
return 0;
}
static int nhpoly1305_neon_digest(struct shash_desc *desc,
const u8 *src, unsigned int srclen, u8 *out)
{
return crypto_nhpoly1305_init(desc) ?:
nhpoly1305_neon_update(desc, src, srclen) ?:
crypto_nhpoly1305_final(desc, out);
}
static struct shash_alg nhpoly1305_alg = {
.base.cra_name = "nhpoly1305",
.base.cra_driver_name = "nhpoly1305-neon",
.base.cra_priority = 200,
.base.cra_ctxsize = sizeof(struct nhpoly1305_key),
.base.cra_module = THIS_MODULE,
.digestsize = POLY1305_DIGEST_SIZE,
.init = crypto_nhpoly1305_init,
.update = nhpoly1305_neon_update,
.final = crypto_nhpoly1305_final,
.digest = nhpoly1305_neon_digest,
.setkey = crypto_nhpoly1305_setkey,
.descsize = sizeof(struct nhpoly1305_state),
};
static int __init nhpoly1305_mod_init(void)
{
if (!cpu_have_named_feature(ASIMD))
return -ENODEV;
return crypto_register_shash(&nhpoly1305_alg);
}
static void __exit nhpoly1305_mod_exit(void)
{
crypto_unregister_shash(&nhpoly1305_alg);
}
module_init(nhpoly1305_mod_init);
module_exit(nhpoly1305_mod_exit);
MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (NEON-accelerated)");
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
MODULE_ALIAS_CRYPTO("nhpoly1305");
MODULE_ALIAS_CRYPTO("nhpoly1305-neon");

1
lib/crypto/Kconfig
View file

@ -118,6 +118,7 @@ config CRYPTO_LIB_NH_ARCH
bool
depends on CRYPTO_LIB_NH && !UML
default y if ARM && KERNEL_MODE_NEON
default y if ARM64 && KERNEL_MODE_NEON
config CRYPTO_LIB_POLY1305
tristate

1
lib/crypto/Makefile
View file

@ -136,6 +136,7 @@ libnh-y := nh.o
ifeq ($(CONFIG_CRYPTO_LIB_NH_ARCH),y)
CFLAGS_nh.o += -I$(src)/$(SRCARCH)
libnh-$(CONFIG_ARM) += arm/nh-neon-core.o
libnh-$(CONFIG_ARM64) += arm64/nh-neon-core.o
endif
################################################################################

3
arch/arm64/crypto/nh-neon-core.S → lib/crypto/arm64/nh-neon-core.S
View file

@ -8,7 +8,6 @@
*/
#include <linux/linkage.h>
#include <linux/cfi_types.h>
KEY .req x0
MESSAGE .req x1
@ -63,7 +62,7 @@
*
* It's guaranteed that message_len % 16 == 0.
*/
SYM_TYPED_FUNC_START(nh_neon)
SYM_FUNC_START(nh_neon)
ld1 {K0.4s,K1.4s}, [KEY], #32
movi PASS0_SUMS.2d, #0

34
lib/crypto/arm64/nh.h Normal file
View file

@ -0,0 +1,34 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* ARM64 accelerated implementation of NH
*
* Copyright 2018 Google LLC
*/
#include <asm/hwcap.h>
#include <asm/simd.h>
#include <linux/cpufeature.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
__le64 hash[NH_NUM_PASSES]);
static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
__le64 hash[NH_NUM_PASSES])
{
if (static_branch_likely(&have_neon) && message_len >= 64 &&
may_use_simd()) {
scoped_ksimd()
nh_neon(key, message, message_len, hash);
return true;
}
return false;
}
#define nh_mod_init_arch nh_mod_init_arch
static void nh_mod_init_arch(void)
{
if (cpu_have_named_feature(ASIMD))
static_branch_enable(&have_neon);
}