mirror of
https://github.com/torvalds/linux.git
synced 2026-03-08 07:44:49 +01:00
nvme-tcp: use crc32c() and skb_copy_and_crc32c_datagram_iter()
Now that the crc32c() library function directly takes advantage of architecture-specific optimizations and there also now exists a function skb_copy_and_crc32c_datagram_iter(), it is unnecessary to go through the crypto_ahash API. Just use those functions. This is much simpler, and it also improves performance due to eliminating the crypto API overhead. Reviewed-by: Christoph Hellwig <hch@lst.de> Reviewed-by: Sagi Grimberg <sagi@grimberg.me> Signed-off-by: Eric Biggers <ebiggers@google.com> Reviewed-by: Hannes Reinecke <hare@suse.de> Link: https://patch.msgid.link/20250519175012.36581-10-ebiggers@kernel.org Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Eric Biggers
Jakub Kicinski
parent
ea6342d989
commit
427fff9aff
2 changed files with 42 additions and 86 deletions
|
|
@ -84,9 +84,9 @@ config NVME_TCP
|
|||
tristate "NVM Express over Fabrics TCP host driver"
|
||||
depends on INET
|
||||
depends on BLOCK
|
||||
select CRC32
|
||||
select NET_CRC32C
|
||||
select NVME_FABRICS
|
||||
select CRYPTO
|
||||
select CRYPTO_CRC32C
|
||||
help
|
||||
This provides support for the NVMe over Fabrics protocol using
|
||||
the TCP transport. This allows you to use remote block devices
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
#include <linux/init.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/crc32.h>
|
||||
#include <linux/nvme-tcp.h>
|
||||
#include <linux/nvme-keyring.h>
|
||||
#include <net/sock.h>
|
||||
|
|
@ -16,7 +17,6 @@
|
|||
#include <net/tls_prot.h>
|
||||
#include <net/handshake.h>
|
||||
#include <linux/blk-mq.h>
|
||||
#include <crypto/hash.h>
|
||||
#include <net/busy_poll.h>
|
||||
#include <trace/events/sock.h>
|
||||
|
||||
|
|
@ -168,8 +168,8 @@ struct nvme_tcp_queue {
|
|||
bool hdr_digest;
|
||||
bool data_digest;
|
||||
bool tls_enabled;
|
||||
struct ahash_request *rcv_hash;
|
||||
struct ahash_request *snd_hash;
|
||||
u32 rcv_crc;
|
||||
u32 snd_crc;
|
||||
__le32 exp_ddgst;
|
||||
__le32 recv_ddgst;
|
||||
struct completion tls_complete;
|
||||
|
|
@ -456,32 +456,38 @@ nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
|
|||
return req;
|
||||
}
|
||||
|
||||
static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
|
||||
__le32 *dgst)
|
||||
#define NVME_TCP_CRC_SEED (~0)
|
||||
|
||||
static inline void nvme_tcp_ddgst_update(u32 *crcp,
|
||||
struct page *page, size_t off, size_t len)
|
||||
{
|
||||
ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
|
||||
crypto_ahash_final(hash);
|
||||
page += off / PAGE_SIZE;
|
||||
off %= PAGE_SIZE;
|
||||
while (len) {
|
||||
const void *vaddr = kmap_local_page(page);
|
||||
size_t n = min(len, (size_t)PAGE_SIZE - off);
|
||||
|
||||
*crcp = crc32c(*crcp, vaddr + off, n);
|
||||
kunmap_local(vaddr);
|
||||
page++;
|
||||
off = 0;
|
||||
len -= n;
|
||||
}
|
||||
}
|
||||
|
||||
static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
|
||||
struct page *page, off_t off, size_t len)
|
||||
static inline __le32 nvme_tcp_ddgst_final(u32 crc)
|
||||
{
|
||||
struct scatterlist sg;
|
||||
|
||||
sg_init_table(&sg, 1);
|
||||
sg_set_page(&sg, page, len, off);
|
||||
ahash_request_set_crypt(hash, &sg, NULL, len);
|
||||
crypto_ahash_update(hash);
|
||||
return cpu_to_le32(~crc);
|
||||
}
|
||||
|
||||
static inline void nvme_tcp_hdgst(struct ahash_request *hash,
|
||||
void *pdu, size_t len)
|
||||
static inline __le32 nvme_tcp_hdgst(const void *pdu, size_t len)
|
||||
{
|
||||
struct scatterlist sg;
|
||||
return cpu_to_le32(~crc32c(NVME_TCP_CRC_SEED, pdu, len));
|
||||
}
|
||||
|
||||
sg_init_one(&sg, pdu, len);
|
||||
ahash_request_set_crypt(hash, &sg, pdu + len, len);
|
||||
crypto_ahash_digest(hash);
|
||||
static inline void nvme_tcp_set_hdgst(void *pdu, size_t len)
|
||||
{
|
||||
*(__le32 *)(pdu + len) = nvme_tcp_hdgst(pdu, len);
|
||||
}
|
||||
|
||||
static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
|
||||
|
|
@ -499,8 +505,7 @@ static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
|
|||
}
|
||||
|
||||
recv_digest = *(__le32 *)(pdu + hdr->hlen);
|
||||
nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
|
||||
exp_digest = *(__le32 *)(pdu + hdr->hlen);
|
||||
exp_digest = nvme_tcp_hdgst(pdu, pdu_len);
|
||||
if (recv_digest != exp_digest) {
|
||||
dev_err(queue->ctrl->ctrl.device,
|
||||
"header digest error: recv %#x expected %#x\n",
|
||||
|
|
@ -526,7 +531,7 @@ static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
|
|||
nvme_tcp_queue_id(queue));
|
||||
return -EPROTO;
|
||||
}
|
||||
crypto_ahash_init(queue->rcv_hash);
|
||||
queue->rcv_crc = NVME_TCP_CRC_SEED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -926,8 +931,8 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
|
|||
iov_iter_count(&req->iter));
|
||||
|
||||
if (queue->data_digest)
|
||||
ret = skb_copy_and_hash_datagram_iter(skb, *offset,
|
||||
&req->iter, recv_len, queue->rcv_hash);
|
||||
ret = skb_copy_and_crc32c_datagram_iter(skb, *offset,
|
||||
&req->iter, recv_len, &queue->rcv_crc);
|
||||
else
|
||||
ret = skb_copy_datagram_iter(skb, *offset,
|
||||
&req->iter, recv_len);
|
||||
|
|
@ -945,7 +950,7 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
|
|||
|
||||
if (!queue->data_remaining) {
|
||||
if (queue->data_digest) {
|
||||
nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
|
||||
queue->exp_ddgst = nvme_tcp_ddgst_final(queue->rcv_crc);
|
||||
queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
|
||||
} else {
|
||||
if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
|
||||
|
|
@ -1147,7 +1152,7 @@ static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
|
|||
return ret;
|
||||
|
||||
if (queue->data_digest)
|
||||
nvme_tcp_ddgst_update(queue->snd_hash, page,
|
||||
nvme_tcp_ddgst_update(&queue->snd_crc, page,
|
||||
offset, ret);
|
||||
|
||||
/*
|
||||
|
|
@ -1161,8 +1166,8 @@ static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
|
|||
/* fully successful last send in current PDU */
|
||||
if (last && ret == len) {
|
||||
if (queue->data_digest) {
|
||||
nvme_tcp_ddgst_final(queue->snd_hash,
|
||||
&req->ddgst);
|
||||
req->ddgst =
|
||||
nvme_tcp_ddgst_final(queue->snd_crc);
|
||||
req->state = NVME_TCP_SEND_DDGST;
|
||||
req->offset = 0;
|
||||
} else {
|
||||
|
|
@ -1194,7 +1199,7 @@ static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
|
|||
msg.msg_flags |= MSG_EOR;
|
||||
|
||||
if (queue->hdr_digest && !req->offset)
|
||||
nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
|
||||
nvme_tcp_set_hdgst(pdu, sizeof(*pdu));
|
||||
|
||||
bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
|
||||
iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
|
||||
|
|
@ -1207,7 +1212,7 @@ static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
|
|||
if (inline_data) {
|
||||
req->state = NVME_TCP_SEND_DATA;
|
||||
if (queue->data_digest)
|
||||
crypto_ahash_init(queue->snd_hash);
|
||||
queue->snd_crc = NVME_TCP_CRC_SEED;
|
||||
} else {
|
||||
nvme_tcp_done_send_req(queue);
|
||||
}
|
||||
|
|
@ -1229,7 +1234,7 @@ static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
|
|||
int ret;
|
||||
|
||||
if (queue->hdr_digest && !req->offset)
|
||||
nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
|
||||
nvme_tcp_set_hdgst(pdu, sizeof(*pdu));
|
||||
|
||||
if (!req->h2cdata_left)
|
||||
msg.msg_flags |= MSG_SPLICE_PAGES;
|
||||
|
|
@ -1244,7 +1249,7 @@ static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
|
|||
if (!len) {
|
||||
req->state = NVME_TCP_SEND_DATA;
|
||||
if (queue->data_digest)
|
||||
crypto_ahash_init(queue->snd_hash);
|
||||
queue->snd_crc = NVME_TCP_CRC_SEED;
|
||||
return 1;
|
||||
}
|
||||
req->offset += ret;
|
||||
|
|
@ -1384,41 +1389,6 @@ static void nvme_tcp_io_work(struct work_struct *w)
|
|||
queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
|
||||
}
|
||||
|
||||
static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
|
||||
{
|
||||
struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
|
||||
|
||||
ahash_request_free(queue->rcv_hash);
|
||||
ahash_request_free(queue->snd_hash);
|
||||
crypto_free_ahash(tfm);
|
||||
}
|
||||
|
||||
static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
|
||||
{
|
||||
struct crypto_ahash *tfm;
|
||||
|
||||
tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
|
||||
if (IS_ERR(tfm))
|
||||
return PTR_ERR(tfm);
|
||||
|
||||
queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
|
||||
if (!queue->snd_hash)
|
||||
goto free_tfm;
|
||||
ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
|
||||
|
||||
queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
|
||||
if (!queue->rcv_hash)
|
||||
goto free_snd_hash;
|
||||
ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
|
||||
|
||||
return 0;
|
||||
free_snd_hash:
|
||||
ahash_request_free(queue->snd_hash);
|
||||
free_tfm:
|
||||
crypto_free_ahash(tfm);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
|
||||
{
|
||||
struct nvme_tcp_request *async = &ctrl->async_req;
|
||||
|
|
@ -1451,9 +1421,6 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
|
|||
if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
|
||||
return;
|
||||
|
||||
if (queue->hdr_digest || queue->data_digest)
|
||||
nvme_tcp_free_crypto(queue);
|
||||
|
||||
page_frag_cache_drain(&queue->pf_cache);
|
||||
|
||||
noreclaim_flag = memalloc_noreclaim_save();
|
||||
|
|
@ -1867,21 +1834,13 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
|
|||
|
||||
queue->hdr_digest = nctrl->opts->hdr_digest;
|
||||
queue->data_digest = nctrl->opts->data_digest;
|
||||
if (queue->hdr_digest || queue->data_digest) {
|
||||
ret = nvme_tcp_alloc_crypto(queue);
|
||||
if (ret) {
|
||||
dev_err(nctrl->device,
|
||||
"failed to allocate queue %d crypto\n", qid);
|
||||
goto err_sock;
|
||||
}
|
||||
}
|
||||
|
||||
rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
|
||||
nvme_tcp_hdgst_len(queue);
|
||||
queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
|
||||
if (!queue->pdu) {
|
||||
ret = -ENOMEM;
|
||||
goto err_crypto;
|
||||
goto err_sock;
|
||||
}
|
||||
|
||||
dev_dbg(nctrl->device, "connecting queue %d\n",
|
||||
|
|
@ -1914,9 +1873,6 @@ err_init_connect:
|
|||
kernel_sock_shutdown(queue->sock, SHUT_RDWR);
|
||||
err_rcv_pdu:
|
||||
kfree(queue->pdu);
|
||||
err_crypto:
|
||||
if (queue->hdr_digest || queue->data_digest)
|
||||
nvme_tcp_free_crypto(queue);
|
||||
err_sock:
|
||||
/* ->sock will be released by fput() */
|
||||
fput(queue->sock->file);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue