summaryrefslogtreecommitdiffstats
path: root/drivers/crypto/chelsio/chtls
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/crypto/chelsio/chtls')
-rw-r--r--drivers/crypto/chelsio/chtls/chtls.h11
-rw-r--r--drivers/crypto/chelsio/chtls/chtls_cm.c28
-rw-r--r--drivers/crypto/chelsio/chtls/chtls_hw.c6
-rw-r--r--drivers/crypto/chelsio/chtls/chtls_io.c158
-rw-r--r--drivers/crypto/chelsio/chtls/chtls_main.c16
5 files changed, 145 insertions, 74 deletions
diff --git a/drivers/crypto/chelsio/chtls/chtls.h b/drivers/crypto/chelsio/chtls/chtls.h
index f4b8f1ec00610..a53a0e6ba024e 100644
--- a/drivers/crypto/chelsio/chtls/chtls.h
+++ b/drivers/crypto/chelsio/chtls/chtls.h
@@ -67,11 +67,6 @@ enum {
CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */
};
-#define TLS_RCV_ST_READ_HEADER 0xF0
-#define TLS_RCV_ST_READ_BODY 0xF1
-#define TLS_RCV_ST_READ_DONE 0xF2
-#define TLS_RCV_ST_READ_NB 0xF3
-
#define LISTEN_INFO_HASH_SIZE 32
#define RSPQ_HASH_BITS 5
struct listen_info {
@@ -149,6 +144,7 @@ struct chtls_dev {
struct list_head rcu_node;
struct list_head na_node;
unsigned int send_page_order;
+ int max_host_sndbuf;
struct key_map kmap;
};
@@ -278,6 +274,7 @@ struct tlsrx_cmp_hdr {
#define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U)
#define TLSRX_HDR_PKT_ERROR_M 0x1F
+#define CONTENT_TYPE_ERROR 0x7F
struct ulp_mem_rw {
__be32 cmd;
@@ -347,8 +344,8 @@ enum {
ULPCB_FLAG_HOLD = 1 << 3, /* skb not ready for Tx yet */
ULPCB_FLAG_COMPL = 1 << 4, /* request WR completion */
ULPCB_FLAG_URG = 1 << 5, /* urgent data */
- ULPCB_FLAG_TLS_ND = 1 << 6, /* payload of zero length */
- ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */
+ ULPCB_FLAG_TLS_HDR = 1 << 6, /* payload with tls hdr */
+ ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */
};
/* The ULP mode/submode of an skbuff */
diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c b/drivers/crypto/chelsio/chtls/chtls_cm.c
index 82a473a0cefa9..2bb6f03807588 100644
--- a/drivers/crypto/chelsio/chtls/chtls_cm.c
+++ b/drivers/crypto/chelsio/chtls/chtls_cm.c
@@ -1537,6 +1537,10 @@ static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
struct sock *sk;
sk = lookup_tid(cdev->tids, hwtid);
+ if (unlikely(!sk)) {
+ pr_err("can't find conn. for hwtid %u.\n", hwtid);
+ return -EINVAL;
+ }
skb_dst_set(skb, NULL);
process_cpl_msg(chtls_recv_data, sk, skb);
return 0;
@@ -1585,6 +1589,10 @@ static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
struct sock *sk;
sk = lookup_tid(cdev->tids, hwtid);
+ if (unlikely(!sk)) {
+ pr_err("can't find conn. for hwtid %u.\n", hwtid);
+ return -EINVAL;
+ }
skb_dst_set(skb, NULL);
process_cpl_msg(chtls_recv_pdu, sk, skb);
return 0;
@@ -1600,12 +1608,14 @@ static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
{
- struct cpl_rx_tls_cmp *cmp_cpl = cplhdr(skb);
+ struct tlsrx_cmp_hdr *tls_hdr_pkt;
+ struct cpl_rx_tls_cmp *cmp_cpl;
struct sk_buff *skb_rec;
struct chtls_sock *csk;
struct chtls_hws *tlsk;
struct tcp_sock *tp;
+ cmp_cpl = cplhdr(skb);
csk = rcu_dereference_sk_user_data(sk);
tlsk = &csk->tlshws;
tp = tcp_sk(sk);
@@ -1615,16 +1625,18 @@ static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
skb_reset_transport_header(skb);
__skb_pull(skb, sizeof(*cmp_cpl));
+ tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
+ if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
+ tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
if (!skb->data_len)
- __skb_trim(skb, CPL_RX_TLS_CMP_LENGTH_G
- (ntohl(cmp_cpl->pdulength_length)));
+ __skb_trim(skb, TLS_HEADER_LENGTH);
tp->rcv_nxt +=
CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
+ ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
if (!skb_rec) {
- ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_ND;
__skb_queue_tail(&sk->sk_receive_queue, skb);
} else {
chtls_set_hdrlen(skb, tlsk->pldlen);
@@ -1646,6 +1658,10 @@ static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
struct sock *sk;
sk = lookup_tid(cdev->tids, hwtid);
+ if (unlikely(!sk)) {
+ pr_err("can't find conn. for hwtid %u.\n", hwtid);
+ return -EINVAL;
+ }
skb_dst_set(skb, NULL);
process_cpl_msg(chtls_rx_hdr, sk, skb);
@@ -2105,6 +2121,10 @@ static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
struct sock *sk;
sk = lookup_tid(cdev->tids, hwtid);
+ if (unlikely(!sk)) {
+ pr_err("can't find conn. for hwtid %u.\n", hwtid);
+ return -EINVAL;
+ }
process_cpl_msg(chtls_rx_ack, sk, skb);
return 0;
diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c b/drivers/crypto/chelsio/chtls/chtls_hw.c
index 54a13aa99121a..55d50140f9e5a 100644
--- a/drivers/crypto/chelsio/chtls/chtls_hw.c
+++ b/drivers/crypto/chelsio/chtls/chtls_hw.c
@@ -213,7 +213,7 @@ static int chtls_key_info(struct chtls_sock *csk,
struct _key_ctx *kctx,
u32 keylen, u32 optname)
{
- unsigned char key[CHCR_KEYCTX_CIPHER_KEY_SIZE_256];
+ unsigned char key[AES_KEYSIZE_128];
struct tls12_crypto_info_aes_gcm_128 *gcm_ctx;
unsigned char ghash_h[AEAD_H_SIZE];
struct crypto_cipher *cipher;
@@ -228,10 +228,6 @@ static int chtls_key_info(struct chtls_sock *csk,
if (keylen == AES_KEYSIZE_128) {
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
- } else if (keylen == AES_KEYSIZE_192) {
- ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
- } else if (keylen == AES_KEYSIZE_256) {
- ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
} else {
pr_err("GCM: Invalid key length %d\n", keylen);
return -EINVAL;
diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c b/drivers/crypto/chelsio/chtls/chtls_io.c
index 5a75be43950f6..51fc6821cbbf4 100644
--- a/drivers/crypto/chelsio/chtls/chtls_io.c
+++ b/drivers/crypto/chelsio/chtls/chtls_io.c
@@ -907,11 +907,83 @@ static int chtls_skb_copy_to_page_nocache(struct sock *sk,
}
/* Read TLS header to find content type and data length */
-static u16 tls_header_read(struct tls_hdr *thdr, struct iov_iter *from)
+static int tls_header_read(struct tls_hdr *thdr, struct iov_iter *from)
{
if (copy_from_iter(thdr, sizeof(*thdr), from) != sizeof(*thdr))
return -EFAULT;
- return (__force u16)cpu_to_be16(thdr->length);
+ return (__force int)cpu_to_be16(thdr->length);
+}
+
+static int csk_mem_free(struct chtls_dev *cdev, struct sock *sk)
+{
+ return (cdev->max_host_sndbuf - sk->sk_wmem_queued);
+}
+
+static int csk_wait_memory(struct chtls_dev *cdev,
+ struct sock *sk, long *timeo_p)
+{
+ DEFINE_WAIT_FUNC(wait, woken_wake_function);
+ int sndbuf, err = 0;
+ long current_timeo;
+ long vm_wait = 0;
+ bool noblock;
+
+ current_timeo = *timeo_p;
+ noblock = (*timeo_p ? false : true);
+ sndbuf = cdev->max_host_sndbuf;
+ if (csk_mem_free(cdev, sk)) {
+ current_timeo = (prandom_u32() % (HZ / 5)) + 2;
+ vm_wait = (prandom_u32() % (HZ / 5)) + 2;
+ }
+
+ add_wait_queue(sk_sleep(sk), &wait);
+ while (1) {
+ sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
+
+ if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
+ goto do_error;
+ if (!*timeo_p) {
+ if (noblock)
+ set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
+ goto do_nonblock;
+ }
+ if (signal_pending(current))
+ goto do_interrupted;
+ sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
+ if (csk_mem_free(cdev, sk) && !vm_wait)
+ break;
+
+ set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
+ sk->sk_write_pending++;
+ sk_wait_event(sk, &current_timeo, sk->sk_err ||
+ (sk->sk_shutdown & SEND_SHUTDOWN) ||
+ (csk_mem_free(cdev, sk) && !vm_wait), &wait);
+ sk->sk_write_pending--;
+
+ if (vm_wait) {
+ vm_wait -= current_timeo;
+ current_timeo = *timeo_p;
+ if (current_timeo != MAX_SCHEDULE_TIMEOUT) {
+ current_timeo -= vm_wait;
+ if (current_timeo < 0)
+ current_timeo = 0;
+ }
+ vm_wait = 0;
+ }
+ *timeo_p = current_timeo;
+ }
+do_rm_wq:
+ remove_wait_queue(sk_sleep(sk), &wait);
+ return err;
+do_error:
+ err = -EPIPE;
+ goto do_rm_wq;
+do_nonblock:
+ err = -EAGAIN;
+ goto do_rm_wq;
+do_interrupted:
+ err = sock_intr_errno(*timeo_p);
+ goto do_rm_wq;
}
int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
@@ -952,6 +1024,8 @@ int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
copy = mss - skb->len;
skb->ip_summed = CHECKSUM_UNNECESSARY;
}
+ if (!csk_mem_free(cdev, sk))
+ goto wait_for_sndbuf;
if (is_tls_tx(csk) && !csk->tlshws.txleft) {
struct tls_hdr hdr;
@@ -1009,9 +1083,10 @@ new_buf:
int off = TCP_OFF(sk);
bool merge;
- if (page)
- pg_size <<= compound_order(page);
+ if (!page)
+ goto wait_for_memory;
+ pg_size <<= compound_order(page);
if (off < pg_size &&
skb_can_coalesce(skb, i, page, off)) {
merge = 1;
@@ -1099,8 +1174,10 @@ copy:
if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND)
push_frames_if_head(sk);
continue;
+wait_for_sndbuf:
+ set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
wait_for_memory:
- err = sk_stream_wait_memory(sk, &timeo);
+ err = csk_wait_memory(cdev, sk, &timeo);
if (err)
goto do_error;
}
@@ -1131,6 +1208,7 @@ int chtls_sendpage(struct sock *sk, struct page *page,
int offset, size_t size, int flags)
{
struct chtls_sock *csk;
+ struct chtls_dev *cdev;
int mss, err, copied;
struct tcp_sock *tp;
long timeo;
@@ -1138,6 +1216,7 @@ int chtls_sendpage(struct sock *sk, struct page *page,
tp = tcp_sk(sk);
copied = 0;
csk = rcu_dereference_sk_user_data(sk);
+ cdev = csk->cdev;
timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
err = sk_stream_wait_connect(sk, &timeo);
@@ -1152,10 +1231,11 @@ int chtls_sendpage(struct sock *sk, struct page *page,
struct sk_buff *skb = skb_peek_tail(&csk->txq);
int copy, i;
- copy = mss - skb->len;
if (!skb || (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND) ||
- copy <= 0) {
+ (copy = mss - skb->len) <= 0) {
new_buf:
+ if (!csk_mem_free(cdev, sk))
+ goto wait_for_sndbuf;
if (is_tls_tx(csk)) {
skb = get_record_skb(sk,
@@ -1167,7 +1247,7 @@ new_buf:
skb = get_tx_skb(sk, 0);
}
if (!skb)
- goto do_error;
+ goto wait_for_memory;
copy = mss;
}
if (copy > size)
@@ -1206,8 +1286,12 @@ new_buf:
if (unlikely(ULP_SKB_CB(skb)->flags & ULPCB_FLAG_NO_APPEND))
push_frames_if_head(sk);
continue;
-
+wait_for_sndbuf:
set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
+wait_for_memory:
+ err = csk_wait_memory(cdev, sk, &timeo);
+ if (err)
+ goto do_error;
}
out:
csk_reset_flag(csk, CSK_TX_MORE_DATA);
@@ -1409,7 +1493,7 @@ static int chtls_pt_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
break;
chtls_cleanup_rbuf(sk, copied);
sk_wait_data(sk, &timeo, NULL);
- continue;
+ continue;
found_ok_skb:
if (!skb->len) {
skb_dst_set(skb, NULL);
@@ -1449,31 +1533,13 @@ found_ok_skb:
}
}
}
- if (hws->rstate == TLS_RCV_ST_READ_BODY) {
- if (skb_copy_datagram_msg(skb, offset,
- msg, avail)) {
- if (!copied) {
- copied = -EFAULT;
- break;
- }
- }
- } else {
- struct tlsrx_cmp_hdr *tls_hdr_pkt =
- (struct tlsrx_cmp_hdr *)skb->data;
-
- if ((tls_hdr_pkt->res_to_mac_error &
- TLSRX_HDR_PKT_ERROR_M))
- tls_hdr_pkt->type = 0x7F;
-
- /* CMP pld len is for recv seq */
- hws->rcvpld = skb->hdr_len;
- if (skb_copy_datagram_msg(skb, offset, msg, avail)) {
- if (!copied) {
- copied = -EFAULT;
- break;
- }
+ if (skb_copy_datagram_msg(skb, offset, msg, avail)) {
+ if (!copied) {
+ copied = -EFAULT;
+ break;
}
}
+
copied += avail;
len -= avail;
hws->copied_seq += avail;
@@ -1481,32 +1547,20 @@ skip_copy:
if (tp->urg_data && after(tp->copied_seq, tp->urg_seq))
tp->urg_data = 0;
- if (hws->rstate == TLS_RCV_ST_READ_BODY &&
- (avail + offset) >= skb->len) {
+ if ((avail + offset) >= skb->len) {
if (likely(skb))
chtls_free_skb(sk, skb);
buffers_freed++;
- hws->rstate = TLS_RCV_ST_READ_HEADER;
- atomic_inc(&adap->chcr_stats.tls_pdu_rx);
- tp->copied_seq += hws->rcvpld;
+ if (ULP_SKB_CB(skb)->flags & ULPCB_FLAG_TLS_HDR) {
+ tp->copied_seq += skb->len;
+ hws->rcvpld = skb->hdr_len;
+ } else {
+ tp->copied_seq += hws->rcvpld;
+ }
hws->copied_seq = 0;
if (copied >= target &&
!skb_peek(&sk->sk_receive_queue))
break;
- } else {
- if (likely(skb)) {
- if (ULP_SKB_CB(skb)->flags &
- ULPCB_FLAG_TLS_ND)
- hws->rstate =
- TLS_RCV_ST_READ_HEADER;
- else
- hws->rstate =
- TLS_RCV_ST_READ_BODY;
- chtls_free_skb(sk, skb);
- }
- buffers_freed++;
- tp->copied_seq += avail;
- hws->copied_seq = 0;
}
} while (len > 0);
diff --git a/drivers/crypto/chelsio/chtls/chtls_main.c b/drivers/crypto/chelsio/chtls/chtls_main.c
index 007c45c38fc71..9b07f9165658b 100644
--- a/drivers/crypto/chelsio/chtls/chtls_main.c
+++ b/drivers/crypto/chelsio/chtls/chtls_main.c
@@ -216,7 +216,6 @@ static void *chtls_uld_add(const struct cxgb4_lld_info *info)
cdev->lldi = lldi;
cdev->pdev = lldi->pdev;
cdev->tids = lldi->tids;
- cdev->ports = (struct net_device **)(cdev + 1);
cdev->ports = lldi->ports;
cdev->mtus = lldi->mtus;
cdev->tids = lldi->tids;
@@ -239,6 +238,7 @@ static void *chtls_uld_add(const struct cxgb4_lld_info *info)
spin_lock_init(&cdev->idr_lock);
cdev->send_page_order = min_t(uint, get_order(32768),
send_page_order);
+ cdev->max_host_sndbuf = 48 * 1024;
if (lldi->vr->key.size)
if (chtls_init_kmap(cdev, lldi))
@@ -250,7 +250,7 @@ static void *chtls_uld_add(const struct cxgb4_lld_info *info)
return cdev;
out_rspq_skb:
- for (j = 0; j <= i; j++)
+ for (j = 0; j < i; j++)
kfree_skb(cdev->rspq_skb_cache[j]);
kfree_skb(cdev->askb);
out_skb:
@@ -441,7 +441,7 @@ nomem:
static int do_chtls_getsockopt(struct sock *sk, char __user *optval,
int __user *optlen)
{
- struct tls_crypto_info crypto_info;
+ struct tls_crypto_info crypto_info = { 0 };
crypto_info.version = TLS_1_2_VERSION;
if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info)))
@@ -491,9 +491,13 @@ static int do_chtls_setsockopt(struct sock *sk, int optname,
switch (tmp_crypto_info.cipher_type) {
case TLS_CIPHER_AES_GCM_128: {
- rc = copy_from_user(crypto_info, optval,
- sizeof(struct
- tls12_crypto_info_aes_gcm_128));
+ /* Obtain version and type from previous copy */
+ crypto_info[0] = tmp_crypto_info;
+ /* Now copy the following data */
+ rc = copy_from_user((char *)crypto_info + sizeof(*crypto_info),
+ optval + sizeof(*crypto_info),
+ sizeof(struct tls12_crypto_info_aes_gcm_128)
+ - sizeof(*crypto_info));
if (rc) {
rc = -EFAULT;
generated by cgit v1.2.3 (git 2.39.1) at 2024-04-30 12:40:53 +0000