[PATCH v9 16/20] tcp: Adjust iov_len before filling headers

[David Gibson <david@gibson.dropbear.id.au>]

Currently tcp_fill_headers[46] return the size of the IP payload,
which we use to adjust the size of the last IOV entry for the frame,
so that it only includes the expected data.  This was originally done
to isolate knowledge of the header layout to the header building
functions.  However, we've since reorganised things so that the
callers have knowledge of the layout anyway.

Use that knowledge to adjust iov_len *before* we call tcp_fill_headers*().
This means that the header building functions are called with the IOV
containing the frame and only the frame, which will be useful later on.

For the vhost-user flags, this also simplifies things a bit, since
we'd actually already separately calculated the header length.  For
the vhost-user data case, we'd already shortened the final IOV from
tcp_vu_data_from_sock().

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 tcp.c          | 24 ++++++++----------------
 tcp_buf.c      | 36 +++++++++++++++++-------------------
 tcp_internal.h | 17 +++++++++--------
 tcp_vu.c       | 24 +++++++++++-------------
 4 files changed, 45 insertions(+), 56 deletions(-)

diff --git a/tcp.c b/tcp.c
index 345f65c..8f09a3c 100644
--- a/tcp.c
+++ b/tcp.c
@@ -977,13 +977,11 @@ static void tcp_fill_header(struct tcphdr *th,
  * @check:		Checksum, if already known
  * @seq:		Sequence number for this segment
  * @no_tcp_csum:	Do not set TCP checksum
- *
- * Return: The IPv4 payload length, host order
  */
-size_t tcp_fill_headers4(const struct tcp_tap_conn *conn,
-			 struct tap_hdr *taph, struct iphdr *iph,
-			 struct tcp_payload_t *bp, size_t dlen,
-			 const uint16_t *check, uint32_t seq, bool no_tcp_csum)
+void tcp_fill_headers4(const struct tcp_tap_conn *conn,
+		       struct tap_hdr *taph, struct iphdr *iph,
+		       struct tcp_payload_t *bp, size_t dlen,
+		       const uint16_t *check, uint32_t seq, bool no_tcp_csum)
 {
 	const struct flowside *tapside = TAPFLOW(conn);
 	const struct in_addr *src4 = inany_v4(&tapside->oaddr);
@@ -1014,8 +1012,6 @@ size_t tcp_fill_headers4(const struct tcp_tap_conn *conn,
 	}
 
 	tap_hdr_update(taph, l3len + sizeof(struct ethhdr));
-
-	return l4len;
 }
 
 /**
@@ -1028,13 +1024,11 @@ size_t tcp_fill_headers4(const struct tcp_tap_conn *conn,
  * @check:		Checksum, if already known
  * @seq:		Sequence number for this segment
  * @no_tcp_csum:	Do not set TCP checksum
- *
- * Return: The IPv6 payload length, host order
  */
-size_t tcp_fill_headers6(const struct tcp_tap_conn *conn,
-			 struct tap_hdr *taph, struct ipv6hdr *ip6h,
-			 struct tcp_payload_t *bp, size_t dlen,
-			 uint32_t seq, bool no_tcp_csum)
+void tcp_fill_headers6(const struct tcp_tap_conn *conn,
+		       struct tap_hdr *taph, struct ipv6hdr *ip6h,
+		       struct tcp_payload_t *bp, size_t dlen,
+		       uint32_t seq, bool no_tcp_csum)
 {
 	const struct flowside *tapside = TAPFLOW(conn);
 	size_t l4len = dlen + sizeof(bp->th);
@@ -1065,8 +1059,6 @@ size_t tcp_fill_headers6(const struct tcp_tap_conn *conn,
 	}
 
 	tap_hdr_update(taph, l4len + sizeof(*ip6h) + sizeof(struct ethhdr));
-
-	return l4len;
 }
 
 /**
diff --git a/tcp_buf.c b/tcp_buf.c
index 6f8f5bb..52a19ed 100644
--- a/tcp_buf.c
+++ b/tcp_buf.c
@@ -194,28 +194,26 @@ void tcp_payload_flush(const struct ctx *c)
  * @check:	Checksum, if already known
  * @seq:	Sequence number for this segment
  * @no_tcp_csum: Do not set TCP checksum
- *
- * Return: IP payload length, host order
  */
-static size_t tcp_l2_buf_fill_headers(const struct tcp_tap_conn *conn,
-				      struct iovec *iov, size_t dlen,
-				      const uint16_t *check, uint32_t seq,
-				      bool no_tcp_csum)
+static void tcp_l2_buf_fill_headers(const struct tcp_tap_conn *conn,
+				    struct iovec *iov, size_t dlen,
+				    const uint16_t *check, uint32_t seq,
+				    bool no_tcp_csum)
 {
 	const struct flowside *tapside = TAPFLOW(conn);
 	const struct in_addr *a4 = inany_v4(&tapside->oaddr);
 
 	if (a4) {
-		return tcp_fill_headers4(conn, iov[TCP_IOV_TAP].iov_base,
-					 iov[TCP_IOV_IP].iov_base,
-					 iov[TCP_IOV_PAYLOAD].iov_base, dlen,
-					 check, seq, no_tcp_csum);
+		tcp_fill_headers4(conn, iov[TCP_IOV_TAP].iov_base,
+				  iov[TCP_IOV_IP].iov_base,
+				  iov[TCP_IOV_PAYLOAD].iov_base, dlen,
+				  check, seq, no_tcp_csum);
+	} else {
+		tcp_fill_headers6(conn, iov[TCP_IOV_TAP].iov_base,
+				  iov[TCP_IOV_IP].iov_base,
+				  iov[TCP_IOV_PAYLOAD].iov_base, dlen,
+				  seq, no_tcp_csum);
 	}
-
-	return tcp_fill_headers6(conn, iov[TCP_IOV_TAP].iov_base,
-				 iov[TCP_IOV_IP].iov_base,
-				 iov[TCP_IOV_PAYLOAD].iov_base, dlen,
-				 seq, no_tcp_csum);
 }
 
 /**
@@ -252,8 +250,9 @@ int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
 		return ret;
 
 	tcp_flags_used++;
-	l4len = tcp_l2_buf_fill_headers(conn, iov, optlen, NULL, seq, false);
+	l4len = optlen + sizeof(struct tcphdr);
 	iov[TCP_IOV_PAYLOAD].iov_len = l4len;
+	tcp_l2_buf_fill_headers(conn, iov, optlen, NULL, seq, false);
 
 	if (flags & DUP_ACK) {
 		struct iovec *dup_iov;
@@ -287,7 +286,6 @@ static void tcp_data_to_tap(const struct ctx *c, struct tcp_tap_conn *conn,
 {
 	const uint16_t *check = NULL;
 	struct iovec *iov;
-	size_t l4len;
 
 	conn->seq_to_tap = seq + dlen;
 	tcp_frame_conns[tcp_payload_used] = conn;
@@ -305,8 +303,8 @@ static void tcp_data_to_tap(const struct ctx *c, struct tcp_tap_conn *conn,
 		iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[tcp_payload_used]);
 		iov[TCP_IOV_ETH].iov_base = &tcp6_eth_src;
 	}
-	l4len = tcp_l2_buf_fill_headers(conn, iov, dlen, check, seq, false);
-	iov[TCP_IOV_PAYLOAD].iov_len = l4len;
+	iov[TCP_IOV_PAYLOAD].iov_len = dlen + sizeof(struct tcphdr);
+	tcp_l2_buf_fill_headers(conn, iov, dlen, check, seq, false);
 	if (++tcp_payload_used > TCP_FRAMES_MEM - 1)
 		tcp_payload_flush(c);
 }
diff --git a/tcp_internal.h b/tcp_internal.h
index 21e31a0..2989db7 100644
--- a/tcp_internal.h
+++ b/tcp_internal.h
@@ -183,14 +183,15 @@ void tcp_update_check_tcp4(const struct iphdr *iph,
 void tcp_update_check_tcp6(const struct ipv6hdr *ip6h,
 			   const struct iovec *iov, int iov_cnt,
 			   size_t l4offset);
-size_t tcp_fill_headers4(const struct tcp_tap_conn *conn,
-			 struct tap_hdr *taph, struct iphdr *iph,
-			 struct tcp_payload_t *bp, size_t dlen,
-			 const uint16_t *check, uint32_t seq, bool no_tcp_csum);
-size_t tcp_fill_headers6(const struct tcp_tap_conn *conn,
-			 struct tap_hdr *taph, struct ipv6hdr *ip6h,
-			 struct tcp_payload_t *bp, size_t dlen,
-			 uint32_t seq, bool no_tcp_csum);
+void tcp_fill_headers4(const struct tcp_tap_conn *conn,
+		       struct tap_hdr *taph, struct iphdr *iph,
+		       struct tcp_payload_t *bp, size_t dlen,
+		       const uint16_t *check, uint32_t seq, bool no_tcp_csum);
+void tcp_fill_headers6(const struct tcp_tap_conn *conn,
+		       struct tap_hdr *taph, struct ipv6hdr *ip6h,
+		       struct tcp_payload_t *bp, size_t dlen,
+		       uint32_t seq, bool no_tcp_csum);
+
 int tcp_update_seqack_wnd(const struct ctx *c, struct tcp_tap_conn *conn,
 			  bool force_seq, struct tcp_info_linux *tinfo);
 int tcp_prepare_flags(const struct ctx *c, struct tcp_tap_conn *conn,
diff --git a/tcp_vu.c b/tcp_vu.c
index f779279..7a446b6 100644
--- a/tcp_vu.c
+++ b/tcp_vu.c
@@ -94,7 +94,7 @@ int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
 	struct vu_dev *vdev = c->vdev;
 	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
 	const struct flowside *tapside = TAPFLOW(conn);
-	size_t l2len, l4len, optlen, hdrlen;
+	size_t l2len, optlen, hdrlen;
 	struct tcp_flags_t *payload;
 	struct ipv6hdr *ip6h = NULL;
 	struct iphdr *iph = NULL;
@@ -149,22 +149,20 @@ int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
 		return ret;
 	}
 
+	l2len = hdrlen + optlen;
+	elem[0].in_sg[0].iov_len = l2len +
+				   sizeof(struct virtio_net_hdr_mrg_rxbuf);
+
 	if (CONN_V4(conn)) {
-		l4len = tcp_fill_headers4(conn, NULL, iph,
-					  (struct tcp_payload_t *)payload, optlen,
-					  NULL, seq, true);
-		l2len = sizeof(*iph);
+		tcp_fill_headers4(conn, NULL, iph,
+				  (struct tcp_payload_t *)payload, optlen,
+				  NULL, seq, true);
 	} else {
-		l4len = tcp_fill_headers6(conn, NULL, ip6h,
-					  (struct tcp_payload_t *)payload, optlen,
-					  seq, true);
-		l2len = sizeof(*ip6h);
+		tcp_fill_headers6(conn, NULL, ip6h,
+				  (struct tcp_payload_t *)payload, optlen,
+				  seq, true);
 	}
 
-	l2len += l4len + sizeof(struct ethhdr);
-
-	elem[0].in_sg[0].iov_len = l2len +
-				   sizeof(struct virtio_net_hdr_mrg_rxbuf);
 	if (*c->pcap) {
 		tcp_vu_update_check(tapside, &elem[0].in_sg[0], 1);
 		pcap_iov(&elem[0].in_sg[0], 1,
-- 
@@ -94,7 +94,7 @@ int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
 	struct vu_dev *vdev = c->vdev;
 	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
 	const struct flowside *tapside = TAPFLOW(conn);
-	size_t l2len, l4len, optlen, hdrlen;
+	size_t l2len, optlen, hdrlen;
 	struct tcp_flags_t *payload;
 	struct ipv6hdr *ip6h = NULL;
 	struct iphdr *iph = NULL;
@@ -149,22 +149,20 @@ int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
 		return ret;
 	}
 
+	l2len = hdrlen + optlen;
+	elem[0].in_sg[0].iov_len = l2len +
+				   sizeof(struct virtio_net_hdr_mrg_rxbuf);
+
 	if (CONN_V4(conn)) {
-		l4len = tcp_fill_headers4(conn, NULL, iph,
-					  (struct tcp_payload_t *)payload, optlen,
-					  NULL, seq, true);
-		l2len = sizeof(*iph);
+		tcp_fill_headers4(conn, NULL, iph,
+				  (struct tcp_payload_t *)payload, optlen,
+				  NULL, seq, true);
 	} else {
-		l4len = tcp_fill_headers6(conn, NULL, ip6h,
-					  (struct tcp_payload_t *)payload, optlen,
-					  seq, true);
-		l2len = sizeof(*ip6h);
+		tcp_fill_headers6(conn, NULL, ip6h,
+				  (struct tcp_payload_t *)payload, optlen,
+				  seq, true);
 	}
 
-	l2len += l4len + sizeof(struct ethhdr);
-
-	elem[0].in_sg[0].iov_len = l2len +
-				   sizeof(struct virtio_net_hdr_mrg_rxbuf);
 	if (*c->pcap) {
 		tcp_vu_update_check(tapside, &elem[0].in_sg[0], 1);
 		pcap_iov(&elem[0].in_sg[0], 1,
-- 
2.47.0