archives/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-03-08 01:04:41 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

lib/crypto: sparc/aes: Migrate optimized code into library

Browse source 
Move the SPARC64 AES assembly code into lib/crypto/, wire the key
expansion and single-block en/decryption functions up to the AES library
API, and remove the "aes-sparc64" crypto_cipher algorithm.

The result is that both the AES library and crypto_cipher APIs use the
SPARC64 AES opcodes, whereas previously only crypto_cipher did (and it
wasn't enabled by default, which this commit fixes as well).

Note that some of the functions in the SPARC64 AES assembly code are
still used by the AES mode implementations in
arch/sparc/crypto/aes_glue.c.  For now, just export these functions.
These exports will go away once the AES mode implementations are
migrated to the library as well.  (Trying to split up the assembly file
seemed like much more trouble than it would be worth.)

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260112192035.10427-17-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers 2026-01-12 11:20:14 -08:00
parent 0cab15611e
commit 293c7cd5c6
8 changed files with 200 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

2
arch/sparc/crypto/Kconfig
View file

@ -19,9 +19,9 @@ config CRYPTO_DES_SPARC64
config CRYPTO_AES_SPARC64
tristate "Ciphers: AES, modes: ECB, CBC, CTR"
depends on SPARC64
select CRYPTO_LIB_AES
select CRYPTO_SKCIPHER
help
Block ciphers: AES cipher algorithms (FIPS-197)
Length-preseving ciphers: AES with ECB, CBC, and CTR modes
Architecture: sparc64 using crypto instructions

2
arch/sparc/crypto/Makefile
View file

@ -7,6 +7,6 @@ obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o
obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o
obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o
aes-sparc64-y := aes_asm.o aes_glue.o
aes-sparc64-y := aes_glue.o
des-sparc64-y := des_asm.o des_glue.o
camellia-sparc64-y := camellia_asm.o camellia_glue.o

140
arch/sparc/crypto/aes_glue.c
View file

@ -32,8 +32,6 @@
#include <asm/elf.h>
struct aes_ops {
void (*encrypt)(const u64 *key, const u32 *input, u32 *output);
void (*decrypt)(const u64 *key, const u32 *input, u32 *output);
void (*load_encrypt_keys)(const u64 *key);
void (*load_decrypt_keys)(const u64 *key);
void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
@ -55,79 +53,7 @@ struct crypto_sparc64_aes_ctx {
u32 expanded_key_length;
};
extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,
u32 *output);
extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,
u32 *output);
extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,
u32 *output);
extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,
u32 *output);
extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,
u32 *output);
extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,
u32 *output);
extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);
extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);
extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);
extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);
extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);
extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);
extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,
u64 *output, unsigned int len);
extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,
u64 *output, unsigned int len);
extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,
u64 *output, unsigned int len);
extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,
u64 *output, unsigned int len);
extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,
u64 *output, unsigned int len);
extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,
u64 *output, unsigned int len);
extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,
u64 *output, unsigned int len,
u64 *iv);
extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,
u64 *output, unsigned int len,
u64 *iv);
extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,
u64 *output, unsigned int len,
u64 *iv);
extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,
u64 *output, unsigned int len,
u64 *iv);
extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,
u64 *output, unsigned int len,
u64 *iv);
extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,
u64 *output, unsigned int len,
u64 *iv);
extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input,
u64 *output, unsigned int len,
u64 *iv);
extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input,
u64 *output, unsigned int len,
u64 *iv);
extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input,
u64 *output, unsigned int len,
u64 *iv);
static struct aes_ops aes128_ops = {
.encrypt = aes_sparc64_encrypt_128,
.decrypt = aes_sparc64_decrypt_128,
.load_encrypt_keys = aes_sparc64_load_encrypt_keys_128,
.load_decrypt_keys = aes_sparc64_load_decrypt_keys_128,
.ecb_encrypt = aes_sparc64_ecb_encrypt_128,
@ -138,8 +64,6 @@ static struct aes_ops aes128_ops = {
};
static struct aes_ops aes192_ops = {
.encrypt = aes_sparc64_encrypt_192,
.decrypt = aes_sparc64_decrypt_192,
.load_encrypt_keys = aes_sparc64_load_encrypt_keys_192,
.load_decrypt_keys = aes_sparc64_load_decrypt_keys_192,
.ecb_encrypt = aes_sparc64_ecb_encrypt_192,
@ -150,8 +74,6 @@ static struct aes_ops aes192_ops = {
};
static struct aes_ops aes256_ops = {
.encrypt = aes_sparc64_encrypt_256,
.decrypt = aes_sparc64_decrypt_256,
.load_encrypt_keys = aes_sparc64_load_encrypt_keys_256,
.load_decrypt_keys = aes_sparc64_load_decrypt_keys_256,
.ecb_encrypt = aes_sparc64_ecb_encrypt_256,
@ -161,13 +83,10 @@ static struct aes_ops aes256_ops = {
.ctr_crypt = aes_sparc64_ctr_crypt_256,
};
extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
unsigned int key_len);
static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
unsigned int key_len)
static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
unsigned int key_len)
{
struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
switch (key_len) {
case AES_KEYSIZE_128:
@ -195,26 +114,6 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
return 0;
}
static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
unsigned int key_len)
{
return aes_set_key(crypto_skcipher_tfm(tfm), in_key, key_len);
}
static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
}
static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
}
static int ecb_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@ -358,26 +257,6 @@ static int ctr_crypt(struct skcipher_request *req)
return err;
}
static struct crypto_alg cipher_alg = {
.cra_name = "aes",
.cra_driver_name = "aes-sparc64",
.cra_priority = SPARC_CR_OPCODE_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
.cra_alignmask = 3,
.cra_module = THIS_MODULE,
.cra_u = {
.cipher = {
.cia_min_keysize = AES_MIN_KEY_SIZE,
.cia_max_keysize = AES_MAX_KEY_SIZE,
.cia_setkey = aes_set_key,
.cia_encrypt = crypto_aes_encrypt,
.cia_decrypt = crypto_aes_decrypt
}
}
};
static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ecb(aes)",
@ -440,26 +319,17 @@ static bool __init sparc64_has_aes_opcode(void)
static int __init aes_sparc64_mod_init(void)
{
int err;
if (!sparc64_has_aes_opcode()) {
pr_info("sparc64 aes opcodes not available.\n");
return -ENODEV;
}
pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
err = crypto_register_alg(&cipher_alg);
if (err)
return err;
err = crypto_register_skciphers(skcipher_algs,
ARRAY_SIZE(skcipher_algs));
if (err)
crypto_unregister_alg(&cipher_alg);
return err;
return crypto_register_skciphers(skcipher_algs,
ARRAY_SIZE(skcipher_algs));
}
static void __exit aes_sparc64_mod_fini(void)
{
crypto_unregister_alg(&cipher_alg);
crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
}

42
include/crypto/aes.h
View file

@ -49,6 +49,9 @@ union aes_enckey_arch {
#elif defined(CONFIG_S390)
/* Used when the CPU supports CPACF AES for this key's length */
u8 raw_key[AES_MAX_KEY_SIZE];
#elif defined(CONFIG_SPARC64)
/* Used when the CPU supports the SPARC64 AES opcodes */
u64 sparc_rndkeys[AES_MAX_KEYLENGTH / sizeof(u64)];
#endif
#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */
};
@ -199,6 +202,45 @@ void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len,
void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len,
const struct p8_aes_key *key1,
const struct p8_aes_key *key2, u8 *iv);
#elif defined(CONFIG_SPARC64)
void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
unsigned int key_len);
void aes_sparc64_load_encrypt_keys_128(const u64 *key);
void aes_sparc64_load_encrypt_keys_192(const u64 *key);
void aes_sparc64_load_encrypt_keys_256(const u64 *key);
void aes_sparc64_load_decrypt_keys_128(const u64 *key);
void aes_sparc64_load_decrypt_keys_192(const u64 *key);
void aes_sparc64_load_decrypt_keys_256(const u64 *key);
void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, u64 *output,
unsigned int len);
void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, u64 *output,
unsigned int len);
void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, u64 *output,
unsigned int len);
void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input, u64 *output,
unsigned int len);
void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input, u64 *output,
unsigned int len);
void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input, u64 *output,
unsigned int len);
void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
#endif
/**

1
lib/crypto/Kconfig
View file

@ -20,6 +20,7 @@ config CRYPTO_LIB_AES_ARCH
default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
default y if S390
default y if SPARC64
config CRYPTO_LIB_AESCFB
tristate

1
lib/crypto/Makefile
View file

@ -51,6 +51,7 @@ endif # !CONFIG_SPE
endif # CONFIG_PPC
libaes-$(CONFIG_RISCV) += riscv/aes-riscv64-zvkned.o
libaes-$(CONFIG_SPARC) += sparc/aes_asm.o
endif # CONFIG_CRYPTO_LIB_AES_ARCH
################################################################################

149
lib/crypto/sparc/aes.h Normal file
View file

@ -0,0 +1,149 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* AES accelerated using the sparc64 aes opcodes
*
* Copyright (C) 2008, Intel Corp.
* Copyright (c) 2010, Intel Corporation.
* Copyright 2026 Google LLC
*/
#include <asm/fpumacro.h>
#include <asm/opcodes.h>
#include <asm/pstate.h>
#include <asm/elf.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes_opcodes);
EXPORT_SYMBOL_GPL(aes_sparc64_key_expand);
EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_128);
EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_192);
EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_256);
EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_128);
EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_192);
EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_256);
EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_128);
EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_192);
EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_256);
EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_128);
EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_192);
EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_256);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_128);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_192);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_256);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_128);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_192);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_256);
EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_128);
EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_192);
EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_256);
void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_encrypt_256(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_decrypt_128(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_decrypt_192(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_decrypt_256(const u64 *key, const u32 *input, u32 *output);
static void aes_preparekey_arch(union aes_enckey_arch *k,
union aes_invkey_arch *inv_k,
const u8 *in_key, int key_len, int nrounds)
{
if (static_branch_likely(&have_aes_opcodes)) {
u32 aligned_key[AES_MAX_KEY_SIZE / 4];
if (IS_ALIGNED((uintptr_t)in_key, 4)) {
aes_sparc64_key_expand((const u32 *)in_key,
k->sparc_rndkeys, key_len);
} else {
memcpy(aligned_key, in_key, key_len);
aes_sparc64_key_expand(aligned_key,
k->sparc_rndkeys, key_len);
memzero_explicit(aligned_key, key_len);
}
/*
* Note that nothing needs to be written to inv_k (if it's
* non-NULL) here, since the SPARC64 assembly code uses
* k->sparc_rndkeys for both encryption and decryption.
*/
} else {
aes_expandkey_generic(k->rndkeys,
inv_k ? inv_k->inv_rndkeys : NULL,
in_key, key_len);
}
}
static void aes_sparc64_encrypt(const struct aes_enckey *key,
const u32 *input, u32 *output)
{
if (key->len == AES_KEYSIZE_128)
aes_sparc64_encrypt_128(key->k.sparc_rndkeys, input, output);
else if (key->len == AES_KEYSIZE_192)
aes_sparc64_encrypt_192(key->k.sparc_rndkeys, input, output);
else
aes_sparc64_encrypt_256(key->k.sparc_rndkeys, input, output);
}
static void aes_encrypt_arch(const struct aes_enckey *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
u32 bounce_buf[AES_BLOCK_SIZE / 4];
if (static_branch_likely(&have_aes_opcodes)) {
if (IS_ALIGNED((uintptr_t)in | (uintptr_t)out, 4)) {
aes_sparc64_encrypt(key, (const u32 *)in, (u32 *)out);
} else {
memcpy(bounce_buf, in, AES_BLOCK_SIZE);
aes_sparc64_encrypt(key, bounce_buf, bounce_buf);
memcpy(out, bounce_buf, AES_BLOCK_SIZE);
}
} else {
aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
}
}
static void aes_sparc64_decrypt(const struct aes_key *key,
const u32 *input, u32 *output)
{
if (key->len == AES_KEYSIZE_128)
aes_sparc64_decrypt_128(key->k.sparc_rndkeys, input, output);
else if (key->len == AES_KEYSIZE_192)
aes_sparc64_decrypt_192(key->k.sparc_rndkeys, input, output);
else
aes_sparc64_decrypt_256(key->k.sparc_rndkeys, input, output);
}
static void aes_decrypt_arch(const struct aes_key *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
u32 bounce_buf[AES_BLOCK_SIZE / 4];
if (static_branch_likely(&have_aes_opcodes)) {
if (IS_ALIGNED((uintptr_t)in | (uintptr_t)out, 4)) {
aes_sparc64_decrypt(key, (const u32 *)in, (u32 *)out);
} else {
memcpy(bounce_buf, in, AES_BLOCK_SIZE);
aes_sparc64_decrypt(key, bounce_buf, bounce_buf);
memcpy(out, bounce_buf, AES_BLOCK_SIZE);
}
} else {
aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
out, in);
}
}
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
unsigned long cfr;
if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
return;
__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
if (!(cfr & CFR_AES))
return;
static_branch_enable(&have_aes_opcodes);
}

0
arch/sparc/crypto/aes_asm.S → lib/crypto/sparc/aes_asm.S
View file