[PATCH v2 0/7] Rework UDP buffers

[PATCH v2 0/7] Rework UDP buffers

[David Gibson]

Laurent recently reworked the TCP buffer handling to be split into
various pieces tracked by iovecs.  We'll want that for various future
changes.  This series makes a similar split for UDP buffers, which
we'll want in order to allow dual-stack UDP sockets, amongst other
things.

This is based on my earlier series of cleanups for the TCP buffer
handling.

Changes since v1:
 * Rebase on v2 of the TCP iov and buffer handling cleanups
 * Rename udp_payload and udp_meta structures to _t for more
   consistency with tcp.c
 * Other minor changes based on Stefano's review

David Gibson (7):
  test: Allow sftp via vsock-ssh in tests
  udp: Split tap-bound UDP packets into multiple buffers using io vector
  udp: Combine initialisation of IPv4 and IPv6 iovs
  udp: Explicitly set checksum in guest-bound UDP headers
  udp: Share payload buffers between IPv4 and IPv6
  udp: Use the same buffer for the L2 header for all frames
  udp: Single buffer for IPv4, IPv6 headers and metadata

 tap.h            |  38 ------
 test/passt.mbuto |   6 +-
 udp.c            | 299 +++++++++++++++++++++++------------------------
 3 files changed, 152 insertions(+), 191 deletions(-)

-- 
2.44.0

[PATCH v2 1/7] test: Allow sftp via vsock-ssh in tests

[David Gibson]

During some debugging recently, I wanted to extact a file from a test
guest and found it was tricky, since the ssh-over-vsock setup we had didn't
allow sftp/scp.  We can fix this by adding a line to the guest side sshd
config from mbuto.  While we're there correct an inaccurate comment.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 test/passt.mbuto | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/passt.mbuto b/test/passt.mbuto
index 6240d5c1..436eecc5 100755
--- a/test/passt.mbuto
+++ b/test/passt.mbuto
@@ -54,7 +54,7 @@ EOF
 	ln -s /run /var/run
 	:> /etc/fstab
 
-	# sshd(dropbear) via vsock
+	# sshd via vsock
 	cat > /etc/passwd << EOF
 root:x:0:0:root:/root:/bin/sh
 sshd:x:100:100:Privilege-separated SSH:/var/empty/sshd:/sbin/nologin
@@ -64,7 +64,9 @@ root:::0:99999:7:::
 EOF
 	chmod 000 /etc/shadow
 
-	:> /etc/ssh/sshd_config
+	cat > /etc/ssh/sshd_config << EOF
+Subsystem sftp internal-sftp
+EOF
 	ssh-keygen -A
 	chmod 700 /root/.ssh
 	chmod 700 /run/sshd
-- 
@@ -54,7 +54,7 @@ EOF
 	ln -s /run /var/run
 	:> /etc/fstab
 
-	# sshd(dropbear) via vsock
+	# sshd via vsock
 	cat > /etc/passwd << EOF
 root:x:0:0:root:/root:/bin/sh
 sshd:x:100:100:Privilege-separated SSH:/var/empty/sshd:/sbin/nologin
@@ -64,7 +64,9 @@ root:::0:99999:7:::
 EOF
 	chmod 000 /etc/shadow
 
-	:> /etc/ssh/sshd_config
+	cat > /etc/ssh/sshd_config << EOF
+Subsystem sftp internal-sftp
+EOF
 	ssh-keygen -A
 	chmod 700 /root/.ssh
 	chmod 700 /run/sshd
-- 
2.44.0

[PATCH v2 2/7] udp: Split tap-bound UDP packets into multiple buffers using io vector

[David Gibson]

When sending to the tap device, currently we assemble the headers and
payload into a single contiguous buffer.  Those are described by a single
struct iovec, then a batch of frames is sent to the device with
tap_send_frames().

In order to better integrate the IPv4 and IPv6 paths, we want the IP
header in a different buffer that might not be contiguous with the
payload.  To prepare for that, split the UDP packet into an iovec of
buffers.  We use the same split that Laurent recently introduced for
TCP for convenience.

This removes the last use of tap_hdr_len_(), tap_frame_base() and
tap_frame_len(), so remove those too.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 tap.h | 38 ------------------------------
 udp.c | 74 +++++++++++++++++++++++++++++++++++++++--------------------
 2 files changed, 49 insertions(+), 63 deletions(-)

diff --git a/tap.h b/tap.h
index 9216d5af..d146d2f1 100644
--- a/tap.h
+++ b/tap.h
@@ -43,44 +43,6 @@ static inline void tap_hdr_update(struct tap_hdr *thdr, size_t l2len)
 	thdr->vnet_len = htonl(l2len);
 }
 
-static inline size_t tap_hdr_len_(const struct ctx *c)
-{
-	if (c->mode == MODE_PASST)
-		return sizeof(struct tap_hdr);
-	else
-		return 0;
-}
-
-/**
- * tap_frame_base() - Find start of tap frame
- * @c:		Execution context
- * @taph:	Pointer to tap specific header buffer
- *
- * Returns: pointer to the start of tap frame - suitable for an
- *          iov_base to be passed to tap_send_frames())
- */
-static inline void *tap_frame_base(const struct ctx *c, struct tap_hdr *taph)
-{
-	return (char *)(taph + 1) - tap_hdr_len_(c);
-}
-
-/**
- * tap_frame_len() - Finalize tap frame and return total length
- * @c:		Execution context
- * @taph:	Tap header to finalize
- * @l2len:	L2 packet length (includes L2, excludes tap specific headers)
- *
- * Returns: length of the tap frame including tap specific headers - suitable
- *          for an iov_len to be passed to tap_send_frames()
- */
-static inline size_t tap_frame_len(const struct ctx *c, struct tap_hdr *taph,
-				   size_t l2len)
-{
-	if (c->mode == MODE_PASST)
-		taph->vnet_len = htonl(l2len);
-	return l2len + tap_hdr_len_(c);
-}
-
 struct in_addr tap_ip4_daddr(const struct ctx *c);
 void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport,
 		   struct in_addr dst, in_port_t dport,
diff --git a/udp.c b/udp.c
index 7186fae9..d293bc5d 100644
--- a/udp.c
+++ b/udp.c
@@ -222,12 +222,28 @@ struct udp6_l2_buf_t {
 #endif
 udp6_l2_buf[UDP_MAX_FRAMES];
 
+/**
+ * enum udp_iov_idx - Indices for the buffers making up a single UDP frame
+ * @UDP_IOV_TAP         tap specific header
+ * @UDP_IOV_ETH         Ethernet header
+ * @UDP_IOV_IP          IP (v4/v6) header
+ * @UDP_IOV_PAYLOAD     IP payload (UDP header + data)
+ * @UDP_NUM_IOVS        the number of entries in the iovec array
+ */
+enum udp_iov_idx {
+	UDP_IOV_TAP	= 0,
+	UDP_IOV_ETH	= 1,
+	UDP_IOV_IP	= 2,
+	UDP_IOV_PAYLOAD	= 3,
+	UDP_NUM_IOVS
+};
+
 /* recvmmsg()/sendmmsg() data for tap */
 static struct iovec	udp4_l2_iov_sock	[UDP_MAX_FRAMES];
 static struct iovec	udp6_l2_iov_sock	[UDP_MAX_FRAMES];
 
-static struct iovec	udp4_l2_iov_tap		[UDP_MAX_FRAMES];
-static struct iovec	udp6_l2_iov_tap		[UDP_MAX_FRAMES];
+static struct iovec	udp4_l2_iov_tap		[UDP_MAX_FRAMES][UDP_NUM_IOVS];
+static struct iovec	udp6_l2_iov_tap		[UDP_MAX_FRAMES][UDP_NUM_IOVS];
 
 static struct mmsghdr	udp4_l2_mh_sock		[UDP_MAX_FRAMES];
 static struct mmsghdr	udp6_l2_mh_sock		[UDP_MAX_FRAMES];
@@ -309,7 +325,7 @@ static void udp_sock4_iov_init_one(const struct ctx *c, size_t i)
 	struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
 	struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
 	struct iovec *siov = &udp4_l2_iov_sock[i];
-	struct iovec *tiov = &udp4_l2_iov_tap[i];
+	struct iovec *tiov = udp4_l2_iov_tap[i];
 
 	*buf = (struct udp4_l2_buf_t) {
 		.eh  = ETH_HDR_INIT(ETH_P_IP),
@@ -323,7 +339,10 @@ static void udp_sock4_iov_init_one(const struct ctx *c, size_t i)
 	mh->msg_iov	= siov;
 	mh->msg_iovlen	= 1;
 
-	tiov->iov_base	= tap_frame_base(c, &buf->taph);
+	tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+	tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+	tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
+	tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
 }
 
 /**
@@ -336,7 +355,7 @@ static void udp_sock6_iov_init_one(const struct ctx *c, size_t i)
 	struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
 	struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
 	struct iovec *siov = &udp6_l2_iov_sock[i];
-	struct iovec *tiov = &udp6_l2_iov_tap[i];
+	struct iovec *tiov = udp6_l2_iov_tap[i];
 
 	*buf = (struct udp6_l2_buf_t) {
 		.eh   = ETH_HDR_INIT(ETH_P_IPV6),
@@ -350,7 +369,10 @@ static void udp_sock6_iov_init_one(const struct ctx *c, size_t i)
 	mh->msg_iov	= siov;
 	mh->msg_iovlen	= 1;
 
-	tiov->iov_base	= tap_frame_base(c, &buf->taph);
+	tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+	tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+	tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
+	tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
 }
 
 /**
@@ -572,13 +594,14 @@ static void udp_splice_sendfrom(const struct ctx *c, unsigned start, unsigned n,
  * @dlen:	Length of UDP payload
  * @now:	Current timestamp
  *
- * Return: size of tap frame with headers
+ * Return: size of IPv4 payload (UDP header + data)
  */
 static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
-	size_t l3len = dlen + sizeof(b->iph) + sizeof(b->uh);
+	size_t l4len = dlen + sizeof(b->uh);
+	size_t l3len = l4len + sizeof(b->iph);
 	in_port_t srcport = ntohs(b->s_in.sin_port);
 	struct in_addr src = b->s_in.sin_addr;
 
@@ -609,9 +632,10 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
 
 	b->uh.source = b->s_in.sin_port;
 	b->uh.dest = htons(dstport);
-	b->uh.len = htons(dlen + sizeof(b->uh));
+	b->uh.len = htons(l4len);
 
-	return tap_frame_len(c, &b->taph, l3len + sizeof(b->eh));
+	tap_hdr_update(&b->taph, l3len + sizeof(b->eh));
+	return l4len;
 }
 
 /**
@@ -622,7 +646,7 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
  * @dlen:	Length of UDP payload
  * @now:	Current timestamp
  *
- * Return: size of tap frame with headers
+ * Return: size of IPv6 payload (UDP header + data)
  */
 static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *b,
 			      in_port_t dstport, size_t dlen,
@@ -679,8 +703,8 @@ static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *b,
 	b->uh.len = b->ip6h.payload_len;
 	csum_udp6(&b->uh, src, dst, b->data, dlen);
 
-	return tap_frame_len(c, &b->taph, l4len +
-			                  sizeof(b->ip6h) + sizeof(b->eh));
+	tap_hdr_update(&b->taph, l4len + sizeof(b->ip6h) + sizeof(b->eh));
+	return l4len;
 }
 
 /**
@@ -698,8 +722,8 @@ static void udp_tap_send(const struct ctx *c,
 			 unsigned int start, unsigned int n,
 			 in_port_t dstport, bool v6, const struct timespec *now)
 {
-	struct iovec *tap_iov;
-	unsigned int i;
+	struct iovec (*tap_iov)[UDP_NUM_IOVS];
+	size_t i;
 
 	if (v6)
 		tap_iov = udp6_l2_iov_tap;
@@ -707,19 +731,19 @@ static void udp_tap_send(const struct ctx *c,
 		tap_iov = udp4_l2_iov_tap;
 
 	for (i = start; i < start + n; i++) {
-		size_t buf_len;
-
-		if (v6)
-			buf_len = udp_update_hdr6(c, &udp6_l2_buf[i], dstport,
-						  udp6_l2_mh_sock[i].msg_len, now);
-		else
-			buf_len = udp_update_hdr4(c, &udp4_l2_buf[i], dstport,
-						  udp4_l2_mh_sock[i].msg_len, now);
+		size_t l4len;
 
-		tap_iov[i].iov_len = buf_len;
+		if (v6) {
+			l4len = udp_update_hdr6(c, &udp6_l2_buf[i], dstport,
+						udp6_l2_mh_sock[i].msg_len, now);
+		} else {
+			l4len = udp_update_hdr4(c, &udp4_l2_buf[i], dstport,
+						udp4_l2_mh_sock[i].msg_len, now);
+		}
+		tap_iov[i][UDP_IOV_PAYLOAD].iov_len = l4len;
 	}
 
-	tap_send_frames(c, tap_iov + start, 1, n);
+	tap_send_frames(c, &tap_iov[start][0], UDP_NUM_IOVS, n);
 }
 
 /**
-- 
@@ -222,12 +222,28 @@ struct udp6_l2_buf_t {
 #endif
 udp6_l2_buf[UDP_MAX_FRAMES];
 
+/**
+ * enum udp_iov_idx - Indices for the buffers making up a single UDP frame
+ * @UDP_IOV_TAP         tap specific header
+ * @UDP_IOV_ETH         Ethernet header
+ * @UDP_IOV_IP          IP (v4/v6) header
+ * @UDP_IOV_PAYLOAD     IP payload (UDP header + data)
+ * @UDP_NUM_IOVS        the number of entries in the iovec array
+ */
+enum udp_iov_idx {
+	UDP_IOV_TAP	= 0,
+	UDP_IOV_ETH	= 1,
+	UDP_IOV_IP	= 2,
+	UDP_IOV_PAYLOAD	= 3,
+	UDP_NUM_IOVS
+};
+
 /* recvmmsg()/sendmmsg() data for tap */
 static struct iovec	udp4_l2_iov_sock	[UDP_MAX_FRAMES];
 static struct iovec	udp6_l2_iov_sock	[UDP_MAX_FRAMES];
 
-static struct iovec	udp4_l2_iov_tap		[UDP_MAX_FRAMES];
-static struct iovec	udp6_l2_iov_tap		[UDP_MAX_FRAMES];
+static struct iovec	udp4_l2_iov_tap		[UDP_MAX_FRAMES][UDP_NUM_IOVS];
+static struct iovec	udp6_l2_iov_tap		[UDP_MAX_FRAMES][UDP_NUM_IOVS];
 
 static struct mmsghdr	udp4_l2_mh_sock		[UDP_MAX_FRAMES];
 static struct mmsghdr	udp6_l2_mh_sock		[UDP_MAX_FRAMES];
@@ -309,7 +325,7 @@ static void udp_sock4_iov_init_one(const struct ctx *c, size_t i)
 	struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
 	struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
 	struct iovec *siov = &udp4_l2_iov_sock[i];
-	struct iovec *tiov = &udp4_l2_iov_tap[i];
+	struct iovec *tiov = udp4_l2_iov_tap[i];
 
 	*buf = (struct udp4_l2_buf_t) {
 		.eh  = ETH_HDR_INIT(ETH_P_IP),
@@ -323,7 +339,10 @@ static void udp_sock4_iov_init_one(const struct ctx *c, size_t i)
 	mh->msg_iov	= siov;
 	mh->msg_iovlen	= 1;
 
-	tiov->iov_base	= tap_frame_base(c, &buf->taph);
+	tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+	tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+	tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
+	tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
 }
 
 /**
@@ -336,7 +355,7 @@ static void udp_sock6_iov_init_one(const struct ctx *c, size_t i)
 	struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
 	struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
 	struct iovec *siov = &udp6_l2_iov_sock[i];
-	struct iovec *tiov = &udp6_l2_iov_tap[i];
+	struct iovec *tiov = udp6_l2_iov_tap[i];
 
 	*buf = (struct udp6_l2_buf_t) {
 		.eh   = ETH_HDR_INIT(ETH_P_IPV6),
@@ -350,7 +369,10 @@ static void udp_sock6_iov_init_one(const struct ctx *c, size_t i)
 	mh->msg_iov	= siov;
 	mh->msg_iovlen	= 1;
 
-	tiov->iov_base	= tap_frame_base(c, &buf->taph);
+	tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+	tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+	tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
+	tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
 }
 
 /**
@@ -572,13 +594,14 @@ static void udp_splice_sendfrom(const struct ctx *c, unsigned start, unsigned n,
  * @dlen:	Length of UDP payload
  * @now:	Current timestamp
  *
- * Return: size of tap frame with headers
+ * Return: size of IPv4 payload (UDP header + data)
  */
 static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
-	size_t l3len = dlen + sizeof(b->iph) + sizeof(b->uh);
+	size_t l4len = dlen + sizeof(b->uh);
+	size_t l3len = l4len + sizeof(b->iph);
 	in_port_t srcport = ntohs(b->s_in.sin_port);
 	struct in_addr src = b->s_in.sin_addr;
 
@@ -609,9 +632,10 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
 
 	b->uh.source = b->s_in.sin_port;
 	b->uh.dest = htons(dstport);
-	b->uh.len = htons(dlen + sizeof(b->uh));
+	b->uh.len = htons(l4len);
 
-	return tap_frame_len(c, &b->taph, l3len + sizeof(b->eh));
+	tap_hdr_update(&b->taph, l3len + sizeof(b->eh));
+	return l4len;
 }
 
 /**
@@ -622,7 +646,7 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
  * @dlen:	Length of UDP payload
  * @now:	Current timestamp
  *
- * Return: size of tap frame with headers
+ * Return: size of IPv6 payload (UDP header + data)
  */
 static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *b,
 			      in_port_t dstport, size_t dlen,
@@ -679,8 +703,8 @@ static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *b,
 	b->uh.len = b->ip6h.payload_len;
 	csum_udp6(&b->uh, src, dst, b->data, dlen);
 
-	return tap_frame_len(c, &b->taph, l4len +
-			                  sizeof(b->ip6h) + sizeof(b->eh));
+	tap_hdr_update(&b->taph, l4len + sizeof(b->ip6h) + sizeof(b->eh));
+	return l4len;
 }
 
 /**
@@ -698,8 +722,8 @@ static void udp_tap_send(const struct ctx *c,
 			 unsigned int start, unsigned int n,
 			 in_port_t dstport, bool v6, const struct timespec *now)
 {
-	struct iovec *tap_iov;
-	unsigned int i;
+	struct iovec (*tap_iov)[UDP_NUM_IOVS];
+	size_t i;
 
 	if (v6)
 		tap_iov = udp6_l2_iov_tap;
@@ -707,19 +731,19 @@ static void udp_tap_send(const struct ctx *c,
 		tap_iov = udp4_l2_iov_tap;
 
 	for (i = start; i < start + n; i++) {
-		size_t buf_len;
-
-		if (v6)
-			buf_len = udp_update_hdr6(c, &udp6_l2_buf[i], dstport,
-						  udp6_l2_mh_sock[i].msg_len, now);
-		else
-			buf_len = udp_update_hdr4(c, &udp4_l2_buf[i], dstport,
-						  udp4_l2_mh_sock[i].msg_len, now);
+		size_t l4len;
 
-		tap_iov[i].iov_len = buf_len;
+		if (v6) {
+			l4len = udp_update_hdr6(c, &udp6_l2_buf[i], dstport,
+						udp6_l2_mh_sock[i].msg_len, now);
+		} else {
+			l4len = udp_update_hdr4(c, &udp4_l2_buf[i], dstport,
+						udp4_l2_mh_sock[i].msg_len, now);
+		}
+		tap_iov[i][UDP_IOV_PAYLOAD].iov_len = l4len;
 	}
 
-	tap_send_frames(c, tap_iov + start, 1, n);
+	tap_send_frames(c, &tap_iov[start][0], UDP_NUM_IOVS, n);
 }
 
 /**
-- 
2.44.0

[PATCH v2 3/7] udp: Combine initialisation of IPv4 and IPv6 iovs

[David Gibson]

We're going to introduce more sharing between the IPv4 and IPv6 buffer
structures.  Prepare for this by combinng the initialisation functions.
While we're at it remove the misleading "sock" from the name since these
initialise both tap side and sock side structures.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 udp.c | 114 +++++++++++++++++++++++++++-------------------------------
 1 file changed, 53 insertions(+), 61 deletions(-)

diff --git a/udp.c b/udp.c
index d293bc5d..bb7d161d 100644
--- a/udp.c
+++ b/udp.c
@@ -316,79 +316,71 @@ void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s)
 }
 
 /**
- * udp_sock4_iov_init_one() - Initialise a scatter-gather L2 buffer for IPv4
+ * udp_iov_init_one() - Initialise scatter-gather lists for one buffer
  * @c:		Execution context
  * @i:		Index of buffer to initialize
  */
-static void udp_sock4_iov_init_one(const struct ctx *c, size_t i)
+static void udp_iov_init_one(const struct ctx *c, size_t i)
 {
-	struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
-	struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
-	struct iovec *siov = &udp4_l2_iov_sock[i];
-	struct iovec *tiov = udp4_l2_iov_tap[i];
-
-	*buf = (struct udp4_l2_buf_t) {
-		.eh  = ETH_HDR_INIT(ETH_P_IP),
-		.iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
-	};
-
-	*siov		= IOV_OF_LVALUE(buf->data);
-
-	mh->msg_name	= &buf->s_in;
-	mh->msg_namelen	= sizeof(buf->s_in);
-	mh->msg_iov	= siov;
-	mh->msg_iovlen	= 1;
-
-	tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
-	tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
-	tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
-	tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
-}
+	if (c->ifi4) {
+		struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
+		struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
+		struct iovec *siov = &udp4_l2_iov_sock[i];
+		struct iovec *tiov = udp4_l2_iov_tap[i];
+
+		*buf = (struct udp4_l2_buf_t) {
+			.eh  = ETH_HDR_INIT(ETH_P_IP),
+			.iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
+		};
 
-/**
- * udp_sock6_iov_init_one() - Initialise a scatter-gather L2 buffer for IPv6
- * @c:		Execution context
- * @i:		Index of buffer to initialize
- */
-static void udp_sock6_iov_init_one(const struct ctx *c, size_t i)
-{
-	struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
-	struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
-	struct iovec *siov = &udp6_l2_iov_sock[i];
-	struct iovec *tiov = udp6_l2_iov_tap[i];
-
-	*buf = (struct udp6_l2_buf_t) {
-		.eh   = ETH_HDR_INIT(ETH_P_IPV6),
-		.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
-	};
-
-	*siov		= IOV_OF_LVALUE(buf->data);
-
-	mh->msg_name	= &buf->s_in6;
-	mh->msg_namelen	= sizeof(buf->s_in6);
-	mh->msg_iov	= siov;
-	mh->msg_iovlen	= 1;
-
-	tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
-	tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
-	tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
-	tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
+		*siov		= IOV_OF_LVALUE(buf->data);
+
+		mh->msg_name	= &buf->s_in;
+		mh->msg_namelen	= sizeof(buf->s_in);
+		mh->msg_iov	= siov;
+		mh->msg_iovlen	= 1;
+
+		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
+		tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
+	}
+
+	if (c->ifi6) {
+		struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
+		struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
+		struct iovec *siov = &udp6_l2_iov_sock[i];
+		struct iovec *tiov = udp6_l2_iov_tap[i];
+
+		*buf = (struct udp6_l2_buf_t) {
+			.eh   = ETH_HDR_INIT(ETH_P_IPV6),
+			.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
+		};
+
+		*siov		 = IOV_OF_LVALUE(buf->data);
+
+		mh->msg_name	= &buf->s_in6;
+		mh->msg_namelen	= sizeof(buf->s_in6);
+		mh->msg_iov	= siov;
+		mh->msg_iovlen	= 1;
+
+		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
+		tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
+	}
 }
 
 /**
- * udp_sock_iov_init() - Initialise scatter-gather L2 buffers
+ * udp_iov_init() - Initialise scatter-gather L2 buffers
  * @c:		Execution context
  */
-static void udp_sock_iov_init(const struct ctx *c)
+static void udp_iov_init(const struct ctx *c)
 {
 	size_t i;
 
-	for (i = 0; i < UDP_MAX_FRAMES; i++) {
-		if (c->ifi4)
-			udp_sock4_iov_init_one(c, i);
-		if (c->ifi6)
-			udp_sock6_iov_init_one(c, i);
-	}
+	for (i = 0; i < UDP_MAX_FRAMES; i++)
+		udp_iov_init_one(c, i);
 }
 
 /**
@@ -1259,7 +1251,7 @@ v6:
  */
 int udp_init(struct ctx *c)
 {
-	udp_sock_iov_init(c);
+	udp_iov_init(c);
 
 	udp_invert_portmap(&c->udp.fwd_in);
 	udp_invert_portmap(&c->udp.fwd_out);
-- 
@@ -316,79 +316,71 @@ void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s)
 }
 
 /**
- * udp_sock4_iov_init_one() - Initialise a scatter-gather L2 buffer for IPv4
+ * udp_iov_init_one() - Initialise scatter-gather lists for one buffer
  * @c:		Execution context
  * @i:		Index of buffer to initialize
  */
-static void udp_sock4_iov_init_one(const struct ctx *c, size_t i)
+static void udp_iov_init_one(const struct ctx *c, size_t i)
 {
-	struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
-	struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
-	struct iovec *siov = &udp4_l2_iov_sock[i];
-	struct iovec *tiov = udp4_l2_iov_tap[i];
-
-	*buf = (struct udp4_l2_buf_t) {
-		.eh  = ETH_HDR_INIT(ETH_P_IP),
-		.iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
-	};
-
-	*siov		= IOV_OF_LVALUE(buf->data);
-
-	mh->msg_name	= &buf->s_in;
-	mh->msg_namelen	= sizeof(buf->s_in);
-	mh->msg_iov	= siov;
-	mh->msg_iovlen	= 1;
-
-	tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
-	tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
-	tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
-	tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
-}
+	if (c->ifi4) {
+		struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
+		struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
+		struct iovec *siov = &udp4_l2_iov_sock[i];
+		struct iovec *tiov = udp4_l2_iov_tap[i];
+
+		*buf = (struct udp4_l2_buf_t) {
+			.eh  = ETH_HDR_INIT(ETH_P_IP),
+			.iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
+		};
 
-/**
- * udp_sock6_iov_init_one() - Initialise a scatter-gather L2 buffer for IPv6
- * @c:		Execution context
- * @i:		Index of buffer to initialize
- */
-static void udp_sock6_iov_init_one(const struct ctx *c, size_t i)
-{
-	struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
-	struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
-	struct iovec *siov = &udp6_l2_iov_sock[i];
-	struct iovec *tiov = udp6_l2_iov_tap[i];
-
-	*buf = (struct udp6_l2_buf_t) {
-		.eh   = ETH_HDR_INIT(ETH_P_IPV6),
-		.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
-	};
-
-	*siov		= IOV_OF_LVALUE(buf->data);
-
-	mh->msg_name	= &buf->s_in6;
-	mh->msg_namelen	= sizeof(buf->s_in6);
-	mh->msg_iov	= siov;
-	mh->msg_iovlen	= 1;
-
-	tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
-	tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
-	tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
-	tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
+		*siov		= IOV_OF_LVALUE(buf->data);
+
+		mh->msg_name	= &buf->s_in;
+		mh->msg_namelen	= sizeof(buf->s_in);
+		mh->msg_iov	= siov;
+		mh->msg_iovlen	= 1;
+
+		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
+		tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
+	}
+
+	if (c->ifi6) {
+		struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
+		struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
+		struct iovec *siov = &udp6_l2_iov_sock[i];
+		struct iovec *tiov = udp6_l2_iov_tap[i];
+
+		*buf = (struct udp6_l2_buf_t) {
+			.eh   = ETH_HDR_INIT(ETH_P_IPV6),
+			.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
+		};
+
+		*siov		 = IOV_OF_LVALUE(buf->data);
+
+		mh->msg_name	= &buf->s_in6;
+		mh->msg_namelen	= sizeof(buf->s_in6);
+		mh->msg_iov	= siov;
+		mh->msg_iovlen	= 1;
+
+		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
+		tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
+	}
 }
 
 /**
- * udp_sock_iov_init() - Initialise scatter-gather L2 buffers
+ * udp_iov_init() - Initialise scatter-gather L2 buffers
  * @c:		Execution context
  */
-static void udp_sock_iov_init(const struct ctx *c)
+static void udp_iov_init(const struct ctx *c)
 {
 	size_t i;
 
-	for (i = 0; i < UDP_MAX_FRAMES; i++) {
-		if (c->ifi4)
-			udp_sock4_iov_init_one(c, i);
-		if (c->ifi6)
-			udp_sock6_iov_init_one(c, i);
-	}
+	for (i = 0; i < UDP_MAX_FRAMES; i++)
+		udp_iov_init_one(c, i);
 }
 
 /**
@@ -1259,7 +1251,7 @@ v6:
  */
 int udp_init(struct ctx *c)
 {
-	udp_sock_iov_init(c);
+	udp_iov_init(c);
 
 	udp_invert_portmap(&c->udp.fwd_in);
 	udp_invert_portmap(&c->udp.fwd_out);
-- 
2.44.0

[PATCH v2 4/7] udp: Explicitly set checksum in guest-bound UDP headers

[David Gibson]

For IPv4, UDP checksums are optional and can just be set to 0.
udp_update_hdr4() ignores the checksum field entirely.  Since these are set
to 0 during startup, this works as intended for now.

However, we'd like to share payload and UDP header buffers betweem IPv4 and
IPv6, which does calculate UDP checksums.  Therefore, for robustness, we
should explicitly set the checksum field to 0 for guest-bound UDP packets.

In the tap_udp4_send() slow path, however, we do allow IPv4 UDP checksums
to be calculated as a compile time option.  For consistency, use the same
thing in the udp_update_hdr4() path, which will typically initialize to 0,
but calculate a real checksum if configured to do so.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 udp.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/udp.c b/udp.c
index bb7d161d..cc938bbe 100644
--- a/udp.c
+++ b/udp.c
@@ -592,6 +592,7 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
+	const struct in_addr dst = c->ip4.addr_seen;
 	size_t l4len = dlen + sizeof(b->uh);
 	size_t l3len = l4len + sizeof(b->iph);
 	in_port_t srcport = ntohs(b->s_in.sin_port);
@@ -617,14 +618,14 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
 	}
 
 	b->iph.tot_len = htons(l3len);
-	b->iph.daddr = c->ip4.addr_seen.s_addr;
+	b->iph.daddr = dst.s_addr;
 	b->iph.saddr = src.s_addr;
-	b->iph.check = csum_ip4_header(l3len, IPPROTO_UDP,
-				       src, c->ip4.addr_seen);
+	b->iph.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
 
 	b->uh.source = b->s_in.sin_port;
 	b->uh.dest = htons(dstport);
 	b->uh.len = htons(l4len);
+	csum_udp4(&b->uh, src, dst, b->data, dlen);
 
 	tap_hdr_update(&b->taph, l3len + sizeof(b->eh));
 	return l4len;
-- 
@@ -592,6 +592,7 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
+	const struct in_addr dst = c->ip4.addr_seen;
 	size_t l4len = dlen + sizeof(b->uh);
 	size_t l3len = l4len + sizeof(b->iph);
 	in_port_t srcport = ntohs(b->s_in.sin_port);
@@ -617,14 +618,14 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
 	}
 
 	b->iph.tot_len = htons(l3len);
-	b->iph.daddr = c->ip4.addr_seen.s_addr;
+	b->iph.daddr = dst.s_addr;
 	b->iph.saddr = src.s_addr;
-	b->iph.check = csum_ip4_header(l3len, IPPROTO_UDP,
-				       src, c->ip4.addr_seen);
+	b->iph.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
 
 	b->uh.source = b->s_in.sin_port;
 	b->uh.dest = htons(dstport);
 	b->uh.len = htons(l4len);
+	csum_udp4(&b->uh, src, dst, b->data, dlen);
 
 	tap_hdr_update(&b->taph, l3len + sizeof(b->eh));
 	return l4len;
-- 
2.44.0

[PATCH v2 5/7] udp: Share payload buffers between IPv4 and IPv6

[David Gibson]

Currently the IPv4 and IPv6 paths unnecessarily use different buffers for
the UDP payload.  Now that we're handling the various pieces of the UDP
packets with an iov, we can split the payload part of the buffers off into
its own array shared between IPv4 and IPv6.  As well as saving a little
memory, this allows the payload buffers to be neatly page aligned.

With the buffers merged, udp[46]_l2_iov_sock contain exactly the same thing
as each other and can also be merged.  Likewise udp[46]_iov_splice can be
merged together.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 udp.c | 126 +++++++++++++++++++++++++++++++---------------------------
 1 file changed, 67 insertions(+), 59 deletions(-)

diff --git a/udp.c b/udp.c
index cc938bbe..adca8196 100644
--- a/udp.c
+++ b/udp.c
@@ -171,14 +171,27 @@ static uint8_t udp_act[IP_VERSIONS][UDP_ACT_TYPE_MAX][DIV_ROUND_UP(NUM_PORTS, 8)
 
 /* Static buffers */
 
+/**
+ * struct udp_payload_t - UDP header and data for inbound messages
+ * @uh:		UDP header
+ * @data:	UDP data
+ */
+static struct udp_payload_t {
+	struct udphdr uh;
+	char data[USHRT_MAX - sizeof(struct udphdr)];
+#ifdef __AVX2__
+} __attribute__ ((packed, aligned(32)))
+#else
+} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
+#endif
+udp_payload[UDP_MAX_FRAMES];
+
 /**
  * udp4_l2_buf_t - Pre-cooked IPv4 packet buffers for tap connections
  * @s_in:	Source socket address, filled in by recvmmsg()
  * @taph:	Tap backend specific header
  * @eh:		Prefilled ethernet header
  * @iph:	Pre-filled IP header (except for tot_len and saddr)
- * @uh:		Headroom for UDP header
- * @data:	Storage for UDP payload
  */
 static struct udp4_l2_buf_t {
 	struct sockaddr_in s_in;
@@ -186,9 +199,6 @@ static struct udp4_l2_buf_t {
 	struct tap_hdr taph;
 	struct ethhdr eh;
 	struct iphdr iph;
-	struct udphdr uh;
-	uint8_t data[USHRT_MAX -
-		     (sizeof(struct iphdr) + sizeof(struct udphdr))];
 } __attribute__ ((packed, aligned(__alignof__(unsigned int))))
 udp4_l2_buf[UDP_MAX_FRAMES];
 
@@ -198,8 +208,6 @@ udp4_l2_buf[UDP_MAX_FRAMES];
  * @taph:	Tap backend specific header
  * @eh:		Pre-filled ethernet header
  * @ip6h:	Pre-filled IP header (except for payload_len and addresses)
- * @uh:		Headroom for UDP header
- * @data:	Storage for UDP payload
  */
 struct udp6_l2_buf_t {
 	struct sockaddr_in6 s_in6;
@@ -212,9 +220,6 @@ struct udp6_l2_buf_t {
 	struct tap_hdr taph;
 	struct ethhdr eh;
 	struct ipv6hdr ip6h;
-	struct udphdr uh;
-	uint8_t data[USHRT_MAX -
-		     (sizeof(struct ipv6hdr) + sizeof(struct udphdr))];
 #ifdef __AVX2__
 } __attribute__ ((packed, aligned(32)))
 #else
@@ -239,8 +244,7 @@ enum udp_iov_idx {
 };
 
 /* recvmmsg()/sendmmsg() data for tap */
-static struct iovec	udp4_l2_iov_sock	[UDP_MAX_FRAMES];
-static struct iovec	udp6_l2_iov_sock	[UDP_MAX_FRAMES];
+static struct iovec	udp_l2_iov_sock		[UDP_MAX_FRAMES];
 
 static struct iovec	udp4_l2_iov_tap		[UDP_MAX_FRAMES][UDP_NUM_IOVS];
 static struct iovec	udp6_l2_iov_tap		[UDP_MAX_FRAMES][UDP_NUM_IOVS];
@@ -249,8 +253,7 @@ static struct mmsghdr	udp4_l2_mh_sock		[UDP_MAX_FRAMES];
 static struct mmsghdr	udp6_l2_mh_sock		[UDP_MAX_FRAMES];
 
 /* recvmmsg()/sendmmsg() data for "spliced" connections */
-static struct iovec	udp4_iov_splice		[UDP_MAX_FRAMES];
-static struct iovec	udp6_iov_splice		[UDP_MAX_FRAMES];
+static struct iovec	udp_iov_splice		[UDP_MAX_FRAMES];
 
 static struct sockaddr_in udp4_localname = {
 	.sin_family = AF_INET,
@@ -322,10 +325,14 @@ void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s)
  */
 static void udp_iov_init_one(const struct ctx *c, size_t i)
 {
+	struct udp_payload_t *payload = &udp_payload[i];
+	struct iovec *siov = &udp_l2_iov_sock[i];
+
+	*siov = IOV_OF_LVALUE(payload->data);
+
 	if (c->ifi4) {
 		struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
 		struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
-		struct iovec *siov = &udp4_l2_iov_sock[i];
 		struct iovec *tiov = udp4_l2_iov_tap[i];
 
 		*buf = (struct udp4_l2_buf_t) {
@@ -333,8 +340,6 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 			.iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
 		};
 
-		*siov		= IOV_OF_LVALUE(buf->data);
-
 		mh->msg_name	= &buf->s_in;
 		mh->msg_namelen	= sizeof(buf->s_in);
 		mh->msg_iov	= siov;
@@ -343,13 +348,12 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
 		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
 		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
-		tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
+		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
 
 	if (c->ifi6) {
 		struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
 		struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
-		struct iovec *siov = &udp6_l2_iov_sock[i];
 		struct iovec *tiov = udp6_l2_iov_tap[i];
 
 		*buf = (struct udp6_l2_buf_t) {
@@ -357,8 +361,6 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 			.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
 		};
 
-		*siov		 = IOV_OF_LVALUE(buf->data);
-
 		mh->msg_name	= &buf->s_in6;
 		mh->msg_namelen	= sizeof(buf->s_in6);
 		mh->msg_iov	= siov;
@@ -367,7 +369,7 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
 		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
 		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
-		tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
+		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
 }
 
@@ -581,22 +583,24 @@ static void udp_splice_sendfrom(const struct ctx *c, unsigned start, unsigned n,
 /**
  * udp_update_hdr4() - Update headers for one IPv4 datagram
  * @c:		Execution context
- * @b:		Pointer to udp4_l2_buf to update
+ * @bh:		Pointer to udp4_l2_buf to update
+ * @bp:		Pointer to udp_payload_t to update
  * @dstport:	Destination port number
  * @dlen:	Length of UDP payload
  * @now:	Current timestamp
  *
  * Return: size of IPv4 payload (UDP header + data)
  */
-static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
+static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *bh,
+			      struct udp_payload_t *bp,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
+	in_port_t srcport = ntohs(bh->s_in.sin_port);
 	const struct in_addr dst = c->ip4.addr_seen;
-	size_t l4len = dlen + sizeof(b->uh);
-	size_t l3len = l4len + sizeof(b->iph);
-	in_port_t srcport = ntohs(b->s_in.sin_port);
-	struct in_addr src = b->s_in.sin_addr;
+	struct in_addr src = bh->s_in.sin_addr;
+	size_t l4len = dlen + sizeof(bp->uh);
+	size_t l3len = l4len + sizeof(bh->iph);
 
 	if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_match) &&
 	    IN4_ARE_ADDR_EQUAL(&src, &c->ip4.dns_host) && srcport == 53 &&
@@ -617,38 +621,40 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
 		src = c->ip4.gw;
 	}
 
-	b->iph.tot_len = htons(l3len);
-	b->iph.daddr = dst.s_addr;
-	b->iph.saddr = src.s_addr;
-	b->iph.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
+	bh->iph.tot_len = htons(l3len);
+	bh->iph.daddr = dst.s_addr;
+	bh->iph.saddr = src.s_addr;
+	bh->iph.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
 
-	b->uh.source = b->s_in.sin_port;
-	b->uh.dest = htons(dstport);
-	b->uh.len = htons(l4len);
-	csum_udp4(&b->uh, src, dst, b->data, dlen);
+	bp->uh.source = bh->s_in.sin_port;
+	bp->uh.dest = htons(dstport);
+	bp->uh.len = htons(l4len);
+	csum_udp4(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&b->taph, l3len + sizeof(b->eh));
+	tap_hdr_update(&bh->taph, l3len + sizeof(bh->eh));
 	return l4len;
 }
 
 /**
  * udp_update_hdr6() - Update headers for one IPv6 datagram
  * @c:		Execution context
- * @b:		Pointer to udp6_l2_buf to update
+ * @bh:		Pointer to udp6_l2_buf to update
+ * @bp:		Pointer to udp_payload_t to update
  * @dstport:	Destination port number
  * @dlen:	Length of UDP payload
  * @now:	Current timestamp
  *
  * Return: size of IPv6 payload (UDP header + data)
  */
-static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *b,
+static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *bh,
+			      struct udp_payload_t *bp,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
-	const struct in6_addr *src = &b->s_in6.sin6_addr;
+	const struct in6_addr *src = &bh->s_in6.sin6_addr;
 	const struct in6_addr *dst = &c->ip6.addr_seen;
-	in_port_t srcport = ntohs(b->s_in6.sin6_port);
-	uint16_t l4len = dlen + sizeof(b->uh);
+	in_port_t srcport = ntohs(bh->s_in6.sin6_port);
+	uint16_t l4len = dlen + sizeof(bp->uh);
 
 	if (IN6_IS_ADDR_LINKLOCAL(src)) {
 		dst = &c->ip6.addr_ll_seen;
@@ -684,19 +690,19 @@ static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *b,
 
 	}
 
-	b->ip6h.payload_len = htons(l4len);
-	b->ip6h.daddr = *dst;
-	b->ip6h.saddr = *src;
-	b->ip6h.version = 6;
-	b->ip6h.nexthdr = IPPROTO_UDP;
-	b->ip6h.hop_limit = 255;
+	bh->ip6h.payload_len = htons(l4len);
+	bh->ip6h.daddr = *dst;
+	bh->ip6h.saddr = *src;
+	bh->ip6h.version = 6;
+	bh->ip6h.nexthdr = IPPROTO_UDP;
+	bh->ip6h.hop_limit = 255;
 
-	b->uh.source = b->s_in6.sin6_port;
-	b->uh.dest = htons(dstport);
-	b->uh.len = b->ip6h.payload_len;
-	csum_udp6(&b->uh, src, dst, b->data, dlen);
+	bp->uh.source = bh->s_in6.sin6_port;
+	bp->uh.dest = htons(dstport);
+	bp->uh.len = bh->ip6h.payload_len;
+	csum_udp6(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&b->taph, l4len + sizeof(b->ip6h) + sizeof(b->eh));
+	tap_hdr_update(&bh->taph, l4len + sizeof(bh->ip6h) + sizeof(bh->eh));
 	return l4len;
 }
 
@@ -724,13 +730,16 @@ static void udp_tap_send(const struct ctx *c,
 		tap_iov = udp4_l2_iov_tap;
 
 	for (i = start; i < start + n; i++) {
+		struct udp_payload_t *bp = &udp_payload[i];
 		size_t l4len;
 
 		if (v6) {
-			l4len = udp_update_hdr6(c, &udp6_l2_buf[i], dstport,
+			l4len = udp_update_hdr6(c, &udp6_l2_buf[i], bp,
+						dstport,
 						udp6_l2_mh_sock[i].msg_len, now);
 		} else {
-			l4len = udp_update_hdr4(c, &udp4_l2_buf[i], dstport,
+			l4len = udp_update_hdr4(c, &udp4_l2_buf[i], bp,
+						dstport,
 						udp4_l2_mh_sock[i].msg_len, now);
 		}
 		tap_iov[i][UDP_IOV_PAYLOAD].iov_len = l4len;
@@ -1077,11 +1086,10 @@ static void udp_splice_iov_init(void)
 		mh6->msg_name = &udp6_localname;
 		mh6->msg_namelen = sizeof(udp6_localname);
 
-		udp4_iov_splice[i].iov_base = udp4_l2_buf[i].data;
-		udp6_iov_splice[i].iov_base = udp6_l2_buf[i].data;
+		udp_iov_splice[i].iov_base = udp_payload[i].data;
 
-		mh4->msg_iov = &udp4_iov_splice[i];
-		mh6->msg_iov = &udp6_iov_splice[i];
+		mh4->msg_iov = &udp_iov_splice[i];
+		mh6->msg_iov = &udp_iov_splice[i];
 		mh4->msg_iovlen = mh6->msg_iovlen = 1;
 	}
 }
-- 
@@ -171,14 +171,27 @@ static uint8_t udp_act[IP_VERSIONS][UDP_ACT_TYPE_MAX][DIV_ROUND_UP(NUM_PORTS, 8)
 
 /* Static buffers */
 
+/**
+ * struct udp_payload_t - UDP header and data for inbound messages
+ * @uh:		UDP header
+ * @data:	UDP data
+ */
+static struct udp_payload_t {
+	struct udphdr uh;
+	char data[USHRT_MAX - sizeof(struct udphdr)];
+#ifdef __AVX2__
+} __attribute__ ((packed, aligned(32)))
+#else
+} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
+#endif
+udp_payload[UDP_MAX_FRAMES];
+
 /**
  * udp4_l2_buf_t - Pre-cooked IPv4 packet buffers for tap connections
  * @s_in:	Source socket address, filled in by recvmmsg()
  * @taph:	Tap backend specific header
  * @eh:		Prefilled ethernet header
  * @iph:	Pre-filled IP header (except for tot_len and saddr)
- * @uh:		Headroom for UDP header
- * @data:	Storage for UDP payload
  */
 static struct udp4_l2_buf_t {
 	struct sockaddr_in s_in;
@@ -186,9 +199,6 @@ static struct udp4_l2_buf_t {
 	struct tap_hdr taph;
 	struct ethhdr eh;
 	struct iphdr iph;
-	struct udphdr uh;
-	uint8_t data[USHRT_MAX -
-		     (sizeof(struct iphdr) + sizeof(struct udphdr))];
 } __attribute__ ((packed, aligned(__alignof__(unsigned int))))
 udp4_l2_buf[UDP_MAX_FRAMES];
 
@@ -198,8 +208,6 @@ udp4_l2_buf[UDP_MAX_FRAMES];
  * @taph:	Tap backend specific header
  * @eh:		Pre-filled ethernet header
  * @ip6h:	Pre-filled IP header (except for payload_len and addresses)
- * @uh:		Headroom for UDP header
- * @data:	Storage for UDP payload
  */
 struct udp6_l2_buf_t {
 	struct sockaddr_in6 s_in6;
@@ -212,9 +220,6 @@ struct udp6_l2_buf_t {
 	struct tap_hdr taph;
 	struct ethhdr eh;
 	struct ipv6hdr ip6h;
-	struct udphdr uh;
-	uint8_t data[USHRT_MAX -
-		     (sizeof(struct ipv6hdr) + sizeof(struct udphdr))];
 #ifdef __AVX2__
 } __attribute__ ((packed, aligned(32)))
 #else
@@ -239,8 +244,7 @@ enum udp_iov_idx {
 };
 
 /* recvmmsg()/sendmmsg() data for tap */
-static struct iovec	udp4_l2_iov_sock	[UDP_MAX_FRAMES];
-static struct iovec	udp6_l2_iov_sock	[UDP_MAX_FRAMES];
+static struct iovec	udp_l2_iov_sock		[UDP_MAX_FRAMES];
 
 static struct iovec	udp4_l2_iov_tap		[UDP_MAX_FRAMES][UDP_NUM_IOVS];
 static struct iovec	udp6_l2_iov_tap		[UDP_MAX_FRAMES][UDP_NUM_IOVS];
@@ -249,8 +253,7 @@ static struct mmsghdr	udp4_l2_mh_sock		[UDP_MAX_FRAMES];
 static struct mmsghdr	udp6_l2_mh_sock		[UDP_MAX_FRAMES];
 
 /* recvmmsg()/sendmmsg() data for "spliced" connections */
-static struct iovec	udp4_iov_splice		[UDP_MAX_FRAMES];
-static struct iovec	udp6_iov_splice		[UDP_MAX_FRAMES];
+static struct iovec	udp_iov_splice		[UDP_MAX_FRAMES];
 
 static struct sockaddr_in udp4_localname = {
 	.sin_family = AF_INET,
@@ -322,10 +325,14 @@ void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s)
  */
 static void udp_iov_init_one(const struct ctx *c, size_t i)
 {
+	struct udp_payload_t *payload = &udp_payload[i];
+	struct iovec *siov = &udp_l2_iov_sock[i];
+
+	*siov = IOV_OF_LVALUE(payload->data);
+
 	if (c->ifi4) {
 		struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
 		struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
-		struct iovec *siov = &udp4_l2_iov_sock[i];
 		struct iovec *tiov = udp4_l2_iov_tap[i];
 
 		*buf = (struct udp4_l2_buf_t) {
@@ -333,8 +340,6 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 			.iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
 		};
 
-		*siov		= IOV_OF_LVALUE(buf->data);
-
 		mh->msg_name	= &buf->s_in;
 		mh->msg_namelen	= sizeof(buf->s_in);
 		mh->msg_iov	= siov;
@@ -343,13 +348,12 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
 		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
 		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
-		tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
+		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
 
 	if (c->ifi6) {
 		struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
 		struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
-		struct iovec *siov = &udp6_l2_iov_sock[i];
 		struct iovec *tiov = udp6_l2_iov_tap[i];
 
 		*buf = (struct udp6_l2_buf_t) {
@@ -357,8 +361,6 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 			.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
 		};
 
-		*siov		 = IOV_OF_LVALUE(buf->data);
-
 		mh->msg_name	= &buf->s_in6;
 		mh->msg_namelen	= sizeof(buf->s_in6);
 		mh->msg_iov	= siov;
@@ -367,7 +369,7 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
 		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
 		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
-		tiov[UDP_IOV_PAYLOAD].iov_base = &buf->uh;
+		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
 }
 
@@ -581,22 +583,24 @@ static void udp_splice_sendfrom(const struct ctx *c, unsigned start, unsigned n,
 /**
  * udp_update_hdr4() - Update headers for one IPv4 datagram
  * @c:		Execution context
- * @b:		Pointer to udp4_l2_buf to update
+ * @bh:		Pointer to udp4_l2_buf to update
+ * @bp:		Pointer to udp_payload_t to update
  * @dstport:	Destination port number
  * @dlen:	Length of UDP payload
  * @now:	Current timestamp
  *
  * Return: size of IPv4 payload (UDP header + data)
  */
-static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
+static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *bh,
+			      struct udp_payload_t *bp,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
+	in_port_t srcport = ntohs(bh->s_in.sin_port);
 	const struct in_addr dst = c->ip4.addr_seen;
-	size_t l4len = dlen + sizeof(b->uh);
-	size_t l3len = l4len + sizeof(b->iph);
-	in_port_t srcport = ntohs(b->s_in.sin_port);
-	struct in_addr src = b->s_in.sin_addr;
+	struct in_addr src = bh->s_in.sin_addr;
+	size_t l4len = dlen + sizeof(bp->uh);
+	size_t l3len = l4len + sizeof(bh->iph);
 
 	if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_match) &&
 	    IN4_ARE_ADDR_EQUAL(&src, &c->ip4.dns_host) && srcport == 53 &&
@@ -617,38 +621,40 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *b,
 		src = c->ip4.gw;
 	}
 
-	b->iph.tot_len = htons(l3len);
-	b->iph.daddr = dst.s_addr;
-	b->iph.saddr = src.s_addr;
-	b->iph.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
+	bh->iph.tot_len = htons(l3len);
+	bh->iph.daddr = dst.s_addr;
+	bh->iph.saddr = src.s_addr;
+	bh->iph.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
 
-	b->uh.source = b->s_in.sin_port;
-	b->uh.dest = htons(dstport);
-	b->uh.len = htons(l4len);
-	csum_udp4(&b->uh, src, dst, b->data, dlen);
+	bp->uh.source = bh->s_in.sin_port;
+	bp->uh.dest = htons(dstport);
+	bp->uh.len = htons(l4len);
+	csum_udp4(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&b->taph, l3len + sizeof(b->eh));
+	tap_hdr_update(&bh->taph, l3len + sizeof(bh->eh));
 	return l4len;
 }
 
 /**
  * udp_update_hdr6() - Update headers for one IPv6 datagram
  * @c:		Execution context
- * @b:		Pointer to udp6_l2_buf to update
+ * @bh:		Pointer to udp6_l2_buf to update
+ * @bp:		Pointer to udp_payload_t to update
  * @dstport:	Destination port number
  * @dlen:	Length of UDP payload
  * @now:	Current timestamp
  *
  * Return: size of IPv6 payload (UDP header + data)
  */
-static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *b,
+static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *bh,
+			      struct udp_payload_t *bp,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
-	const struct in6_addr *src = &b->s_in6.sin6_addr;
+	const struct in6_addr *src = &bh->s_in6.sin6_addr;
 	const struct in6_addr *dst = &c->ip6.addr_seen;
-	in_port_t srcport = ntohs(b->s_in6.sin6_port);
-	uint16_t l4len = dlen + sizeof(b->uh);
+	in_port_t srcport = ntohs(bh->s_in6.sin6_port);
+	uint16_t l4len = dlen + sizeof(bp->uh);
 
 	if (IN6_IS_ADDR_LINKLOCAL(src)) {
 		dst = &c->ip6.addr_ll_seen;
@@ -684,19 +690,19 @@ static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *b,
 
 	}
 
-	b->ip6h.payload_len = htons(l4len);
-	b->ip6h.daddr = *dst;
-	b->ip6h.saddr = *src;
-	b->ip6h.version = 6;
-	b->ip6h.nexthdr = IPPROTO_UDP;
-	b->ip6h.hop_limit = 255;
+	bh->ip6h.payload_len = htons(l4len);
+	bh->ip6h.daddr = *dst;
+	bh->ip6h.saddr = *src;
+	bh->ip6h.version = 6;
+	bh->ip6h.nexthdr = IPPROTO_UDP;
+	bh->ip6h.hop_limit = 255;
 
-	b->uh.source = b->s_in6.sin6_port;
-	b->uh.dest = htons(dstport);
-	b->uh.len = b->ip6h.payload_len;
-	csum_udp6(&b->uh, src, dst, b->data, dlen);
+	bp->uh.source = bh->s_in6.sin6_port;
+	bp->uh.dest = htons(dstport);
+	bp->uh.len = bh->ip6h.payload_len;
+	csum_udp6(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&b->taph, l4len + sizeof(b->ip6h) + sizeof(b->eh));
+	tap_hdr_update(&bh->taph, l4len + sizeof(bh->ip6h) + sizeof(bh->eh));
 	return l4len;
 }
 
@@ -724,13 +730,16 @@ static void udp_tap_send(const struct ctx *c,
 		tap_iov = udp4_l2_iov_tap;
 
 	for (i = start; i < start + n; i++) {
+		struct udp_payload_t *bp = &udp_payload[i];
 		size_t l4len;
 
 		if (v6) {
-			l4len = udp_update_hdr6(c, &udp6_l2_buf[i], dstport,
+			l4len = udp_update_hdr6(c, &udp6_l2_buf[i], bp,
+						dstport,
 						udp6_l2_mh_sock[i].msg_len, now);
 		} else {
-			l4len = udp_update_hdr4(c, &udp4_l2_buf[i], dstport,
+			l4len = udp_update_hdr4(c, &udp4_l2_buf[i], bp,
+						dstport,
 						udp4_l2_mh_sock[i].msg_len, now);
 		}
 		tap_iov[i][UDP_IOV_PAYLOAD].iov_len = l4len;
@@ -1077,11 +1086,10 @@ static void udp_splice_iov_init(void)
 		mh6->msg_name = &udp6_localname;
 		mh6->msg_namelen = sizeof(udp6_localname);
 
-		udp4_iov_splice[i].iov_base = udp4_l2_buf[i].data;
-		udp6_iov_splice[i].iov_base = udp6_l2_buf[i].data;
+		udp_iov_splice[i].iov_base = udp_payload[i].data;
 
-		mh4->msg_iov = &udp4_iov_splice[i];
-		mh6->msg_iov = &udp6_iov_splice[i];
+		mh4->msg_iov = &udp_iov_splice[i];
+		mh6->msg_iov = &udp_iov_splice[i];
 		mh4->msg_iovlen = mh6->msg_iovlen = 1;
 	}
 }
-- 
2.44.0

[PATCH v2 6/7] udp: Use the same buffer for the L2 header for all frames

[David Gibson]

Currently each tap-bound frame buffer has room for its own ethernet header.
However the ethernet header is always the same for such frames, so now
that we're indirectly referencing the ethernet header via iov, we can use
a single buffer for all of them.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 udp.c | 36 +++++++++++++++---------------------
 1 file changed, 15 insertions(+), 21 deletions(-)

diff --git a/udp.c b/udp.c
index adca8196..c940749a 100644
--- a/udp.c
+++ b/udp.c
@@ -186,39 +186,40 @@ static struct udp_payload_t {
 #endif
 udp_payload[UDP_MAX_FRAMES];
 
+/* Ethernet header for IPv4 frames */
+static struct ethhdr udp4_eth_hdr;
+
 /**
  * udp4_l2_buf_t - Pre-cooked IPv4 packet buffers for tap connections
  * @s_in:	Source socket address, filled in by recvmmsg()
  * @taph:	Tap backend specific header
- * @eh:		Prefilled ethernet header
  * @iph:	Pre-filled IP header (except for tot_len and saddr)
  */
 static struct udp4_l2_buf_t {
 	struct sockaddr_in s_in;
 
 	struct tap_hdr taph;
-	struct ethhdr eh;
 	struct iphdr iph;
 } __attribute__ ((packed, aligned(__alignof__(unsigned int))))
 udp4_l2_buf[UDP_MAX_FRAMES];
 
+/* Ethernet header for IPv6 frames */
+static struct ethhdr udp6_eth_hdr;
+
 /**
  * udp6_l2_buf_t - Pre-cooked IPv6 packet buffers for tap connections
  * @s_in6:	Source socket address, filled in by recvmmsg()
  * @taph:	Tap backend specific header
- * @eh:		Pre-filled ethernet header
  * @ip6h:	Pre-filled IP header (except for payload_len and addresses)
  */
 struct udp6_l2_buf_t {
 	struct sockaddr_in6 s_in6;
 #ifdef __AVX2__
 	/* Align ip6h to 32-byte boundary. */
-	uint8_t pad[64 - (sizeof(struct sockaddr_in6) + sizeof(struct ethhdr) +
-			  sizeof(struct tap_hdr))];
+	uint8_t pad[64 - (sizeof(struct sockaddr_in6) + sizeof(struct tap_hdr))];
 #endif
 
 	struct tap_hdr taph;
-	struct ethhdr eh;
 	struct ipv6hdr ip6h;
 #ifdef __AVX2__
 } __attribute__ ((packed, aligned(32)))
@@ -307,15 +308,8 @@ static void udp_invert_portmap(struct udp_fwd_ports *fwd)
  */
 void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s)
 {
-	int i;
-
-	for (i = 0; i < UDP_MAX_FRAMES; i++) {
-		struct udp4_l2_buf_t *b4 = &udp4_l2_buf[i];
-		struct udp6_l2_buf_t *b6 = &udp6_l2_buf[i];
-
-		eth_update_mac(&b4->eh, eth_d, eth_s);
-		eth_update_mac(&b6->eh, eth_d, eth_s);
-	}
+	eth_update_mac(&udp4_eth_hdr, eth_d, eth_s);
+	eth_update_mac(&udp6_eth_hdr, eth_d, eth_s);
 }
 
 /**
@@ -329,6 +323,8 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 	struct iovec *siov = &udp_l2_iov_sock[i];
 
 	*siov = IOV_OF_LVALUE(payload->data);
+	udp4_eth_hdr.h_proto = htons_constant(ETH_P_IP);
+	udp6_eth_hdr.h_proto = htons_constant(ETH_P_IPV6);
 
 	if (c->ifi4) {
 		struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
@@ -336,7 +332,6 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		struct iovec *tiov = udp4_l2_iov_tap[i];
 
 		*buf = (struct udp4_l2_buf_t) {
-			.eh  = ETH_HDR_INIT(ETH_P_IP),
 			.iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
 		};
 
@@ -346,7 +341,7 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		mh->msg_iovlen	= 1;
 
 		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
-		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp4_eth_hdr);
 		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
 		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
@@ -357,7 +352,6 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		struct iovec *tiov = udp6_l2_iov_tap[i];
 
 		*buf = (struct udp6_l2_buf_t) {
-			.eh   = ETH_HDR_INIT(ETH_P_IPV6),
 			.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
 		};
 
@@ -367,7 +361,7 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		mh->msg_iovlen	= 1;
 
 		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
-		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp6_eth_hdr);
 		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
 		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
@@ -631,7 +625,7 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *bh,
 	bp->uh.len = htons(l4len);
 	csum_udp4(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&bh->taph, l3len + sizeof(bh->eh));
+	tap_hdr_update(&bh->taph, l3len + sizeof(udp4_eth_hdr));
 	return l4len;
 }
 
@@ -702,7 +696,7 @@ static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *bh,
 	bp->uh.len = bh->ip6h.payload_len;
 	csum_udp6(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&bh->taph, l4len + sizeof(bh->ip6h) + sizeof(bh->eh));
+	tap_hdr_update(&bh->taph, l4len + sizeof(bh->ip6h) + sizeof(udp6_eth_hdr));
 	return l4len;
 }
 
-- 
@@ -186,39 +186,40 @@ static struct udp_payload_t {
 #endif
 udp_payload[UDP_MAX_FRAMES];
 
+/* Ethernet header for IPv4 frames */
+static struct ethhdr udp4_eth_hdr;
+
 /**
  * udp4_l2_buf_t - Pre-cooked IPv4 packet buffers for tap connections
  * @s_in:	Source socket address, filled in by recvmmsg()
  * @taph:	Tap backend specific header
- * @eh:		Prefilled ethernet header
  * @iph:	Pre-filled IP header (except for tot_len and saddr)
  */
 static struct udp4_l2_buf_t {
 	struct sockaddr_in s_in;
 
 	struct tap_hdr taph;
-	struct ethhdr eh;
 	struct iphdr iph;
 } __attribute__ ((packed, aligned(__alignof__(unsigned int))))
 udp4_l2_buf[UDP_MAX_FRAMES];
 
+/* Ethernet header for IPv6 frames */
+static struct ethhdr udp6_eth_hdr;
+
 /**
  * udp6_l2_buf_t - Pre-cooked IPv6 packet buffers for tap connections
  * @s_in6:	Source socket address, filled in by recvmmsg()
  * @taph:	Tap backend specific header
- * @eh:		Pre-filled ethernet header
  * @ip6h:	Pre-filled IP header (except for payload_len and addresses)
  */
 struct udp6_l2_buf_t {
 	struct sockaddr_in6 s_in6;
 #ifdef __AVX2__
 	/* Align ip6h to 32-byte boundary. */
-	uint8_t pad[64 - (sizeof(struct sockaddr_in6) + sizeof(struct ethhdr) +
-			  sizeof(struct tap_hdr))];
+	uint8_t pad[64 - (sizeof(struct sockaddr_in6) + sizeof(struct tap_hdr))];
 #endif
 
 	struct tap_hdr taph;
-	struct ethhdr eh;
 	struct ipv6hdr ip6h;
 #ifdef __AVX2__
 } __attribute__ ((packed, aligned(32)))
@@ -307,15 +308,8 @@ static void udp_invert_portmap(struct udp_fwd_ports *fwd)
  */
 void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s)
 {
-	int i;
-
-	for (i = 0; i < UDP_MAX_FRAMES; i++) {
-		struct udp4_l2_buf_t *b4 = &udp4_l2_buf[i];
-		struct udp6_l2_buf_t *b6 = &udp6_l2_buf[i];
-
-		eth_update_mac(&b4->eh, eth_d, eth_s);
-		eth_update_mac(&b6->eh, eth_d, eth_s);
-	}
+	eth_update_mac(&udp4_eth_hdr, eth_d, eth_s);
+	eth_update_mac(&udp6_eth_hdr, eth_d, eth_s);
 }
 
 /**
@@ -329,6 +323,8 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 	struct iovec *siov = &udp_l2_iov_sock[i];
 
 	*siov = IOV_OF_LVALUE(payload->data);
+	udp4_eth_hdr.h_proto = htons_constant(ETH_P_IP);
+	udp6_eth_hdr.h_proto = htons_constant(ETH_P_IPV6);
 
 	if (c->ifi4) {
 		struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
@@ -336,7 +332,6 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		struct iovec *tiov = udp4_l2_iov_tap[i];
 
 		*buf = (struct udp4_l2_buf_t) {
-			.eh  = ETH_HDR_INIT(ETH_P_IP),
 			.iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
 		};
 
@@ -346,7 +341,7 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		mh->msg_iovlen	= 1;
 
 		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
-		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp4_eth_hdr);
 		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
 		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
@@ -357,7 +352,6 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		struct iovec *tiov = udp6_l2_iov_tap[i];
 
 		*buf = (struct udp6_l2_buf_t) {
-			.eh   = ETH_HDR_INIT(ETH_P_IPV6),
 			.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
 		};
 
@@ -367,7 +361,7 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 		mh->msg_iovlen	= 1;
 
 		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
-		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(buf->eh);
+		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp6_eth_hdr);
 		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
 		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
@@ -631,7 +625,7 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *bh,
 	bp->uh.len = htons(l4len);
 	csum_udp4(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&bh->taph, l3len + sizeof(bh->eh));
+	tap_hdr_update(&bh->taph, l3len + sizeof(udp4_eth_hdr));
 	return l4len;
 }
 
@@ -702,7 +696,7 @@ static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *bh,
 	bp->uh.len = bh->ip6h.payload_len;
 	csum_udp6(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&bh->taph, l4len + sizeof(bh->ip6h) + sizeof(bh->eh));
+	tap_hdr_update(&bh->taph, l4len + sizeof(bh->ip6h) + sizeof(udp6_eth_hdr));
 	return l4len;
 }
 
-- 
2.44.0

[PATCH v2 7/7] udp: Single buffer for IPv4, IPv6 headers and metadata

[David Gibson]

Currently we have separate arrays for IPv4 and IPv6 which contain the
headers for guest-bound packets, and also the originating socket address.
We can combine these into a single array of "metadata" structures with
space for both pre-cooked IPv4 and IPv6 headers, as well as shared space
for the tap specific header and socket address (using sockaddr_inany).

Because we're using IOVs to separately address the pieces of each frame,
these structures don't need to be packed to keep the headers contiguous
so we can more naturally arrange for the alignment we want.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 udp.c | 132 ++++++++++++++++++++++++----------------------------------
 1 file changed, 55 insertions(+), 77 deletions(-)

diff --git a/udp.c b/udp.c
index c940749a..3abafc99 100644
--- a/udp.c
+++ b/udp.c
@@ -189,44 +189,27 @@ udp_payload[UDP_MAX_FRAMES];
 /* Ethernet header for IPv4 frames */
 static struct ethhdr udp4_eth_hdr;
 
-/**
- * udp4_l2_buf_t - Pre-cooked IPv4 packet buffers for tap connections
- * @s_in:	Source socket address, filled in by recvmmsg()
- * @taph:	Tap backend specific header
- * @iph:	Pre-filled IP header (except for tot_len and saddr)
- */
-static struct udp4_l2_buf_t {
-	struct sockaddr_in s_in;
-
-	struct tap_hdr taph;
-	struct iphdr iph;
-} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
-udp4_l2_buf[UDP_MAX_FRAMES];
-
 /* Ethernet header for IPv6 frames */
 static struct ethhdr udp6_eth_hdr;
 
 /**
- * udp6_l2_buf_t - Pre-cooked IPv6 packet buffers for tap connections
- * @s_in6:	Source socket address, filled in by recvmmsg()
+ * struct udp_meta_t - Pre-cooked headers and metadata for UDP packets
+ * @ip6h:	Pre-filled IPv6 header (except for payload_len and addresses)
+ * @ip4h:	Pre-filled IPv4 header (except for tot_len and saddr)
  * @taph:	Tap backend specific header
- * @ip6h:	Pre-filled IP header (except for payload_len and addresses)
+ * @s_in:	Source socket address, filled in by recvmmsg()
  */
-struct udp6_l2_buf_t {
-	struct sockaddr_in6 s_in6;
-#ifdef __AVX2__
-	/* Align ip6h to 32-byte boundary. */
-	uint8_t pad[64 - (sizeof(struct sockaddr_in6) + sizeof(struct tap_hdr))];
-#endif
-
-	struct tap_hdr taph;
+static struct udp_meta_t {
 	struct ipv6hdr ip6h;
+	struct iphdr ip4h;
+	struct tap_hdr taph;
+
+	union sockaddr_inany s_in;
+}
 #ifdef __AVX2__
-} __attribute__ ((packed, aligned(32)))
-#else
-} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
+__attribute__ ((aligned(32)))
 #endif
-udp6_l2_buf[UDP_MAX_FRAMES];
+udp_meta[UDP_MAX_FRAMES];
 
 /**
  * enum udp_iov_idx - Indices for the buffers making up a single UDP frame
@@ -321,6 +304,12 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 {
 	struct udp_payload_t *payload = &udp_payload[i];
 	struct iovec *siov = &udp_l2_iov_sock[i];
+	struct udp_meta_t *meta = &udp_meta[i];
+
+	*meta = (struct udp_meta_t) {
+		.ip4h = L2_BUF_IP4_INIT(IPPROTO_UDP),
+		.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP),
+	};
 
 	*siov = IOV_OF_LVALUE(payload->data);
 	udp4_eth_hdr.h_proto = htons_constant(ETH_P_IP);
@@ -328,41 +317,31 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 
 	if (c->ifi4) {
 		struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
-		struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
 		struct iovec *tiov = udp4_l2_iov_tap[i];
 
-		*buf = (struct udp4_l2_buf_t) {
-			.iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
-		};
-
-		mh->msg_name	= &buf->s_in;
-		mh->msg_namelen	= sizeof(buf->s_in);
+		mh->msg_name	= &meta->s_in;
+		mh->msg_namelen	= sizeof(struct sockaddr_in);
 		mh->msg_iov	= siov;
 		mh->msg_iovlen	= 1;
 
-		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &meta->taph);
 		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp4_eth_hdr);
-		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
+		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(meta->ip4h);
 		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
 
 	if (c->ifi6) {
 		struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
-		struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
 		struct iovec *tiov = udp6_l2_iov_tap[i];
 
-		*buf = (struct udp6_l2_buf_t) {
-			.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
-		};
-
-		mh->msg_name	= &buf->s_in6;
-		mh->msg_namelen	= sizeof(buf->s_in6);
+		mh->msg_name	= &meta->s_in;
+		mh->msg_namelen	= sizeof(struct sockaddr_in6);
 		mh->msg_iov	= siov;
 		mh->msg_iovlen	= 1;
 
-		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &meta->taph);
 		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp6_eth_hdr);
-		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
+		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(meta->ip6h);
 		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
 }
@@ -577,7 +556,7 @@ static void udp_splice_sendfrom(const struct ctx *c, unsigned start, unsigned n,
 /**
  * udp_update_hdr4() - Update headers for one IPv4 datagram
  * @c:		Execution context
- * @bh:		Pointer to udp4_l2_buf to update
+ * @bm:		Pointer to udp_meta_t to update
  * @bp:		Pointer to udp_payload_t to update
  * @dstport:	Destination port number
  * @dlen:	Length of UDP payload
@@ -585,16 +564,16 @@ static void udp_splice_sendfrom(const struct ctx *c, unsigned start, unsigned n,
  *
  * Return: size of IPv4 payload (UDP header + data)
  */
-static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *bh,
-			      struct udp_payload_t *bp,
+static size_t udp_update_hdr4(const struct ctx *c,
+			      struct udp_meta_t *bm, struct udp_payload_t *bp,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
-	in_port_t srcport = ntohs(bh->s_in.sin_port);
+	in_port_t srcport = ntohs(bm->s_in.sa4.sin_port);
 	const struct in_addr dst = c->ip4.addr_seen;
-	struct in_addr src = bh->s_in.sin_addr;
+	struct in_addr src = bm->s_in.sa4.sin_addr;
 	size_t l4len = dlen + sizeof(bp->uh);
-	size_t l3len = l4len + sizeof(bh->iph);
+	size_t l3len = l4len + sizeof(bm->ip4h);
 
 	if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_match) &&
 	    IN4_ARE_ADDR_EQUAL(&src, &c->ip4.dns_host) && srcport == 53 &&
@@ -615,24 +594,24 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *bh,
 		src = c->ip4.gw;
 	}
 
-	bh->iph.tot_len = htons(l3len);
-	bh->iph.daddr = dst.s_addr;
-	bh->iph.saddr = src.s_addr;
-	bh->iph.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
+	bm->ip4h.tot_len = htons(l3len);
+	bm->ip4h.daddr = dst.s_addr;
+	bm->ip4h.saddr = src.s_addr;
+	bm->ip4h.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
 
-	bp->uh.source = bh->s_in.sin_port;
+	bp->uh.source = bm->s_in.sa4.sin_port;
 	bp->uh.dest = htons(dstport);
 	bp->uh.len = htons(l4len);
 	csum_udp4(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&bh->taph, l3len + sizeof(udp4_eth_hdr));
+	tap_hdr_update(&bm->taph, l3len + sizeof(udp4_eth_hdr));
 	return l4len;
 }
 
 /**
  * udp_update_hdr6() - Update headers for one IPv6 datagram
  * @c:		Execution context
- * @bh:		Pointer to udp6_l2_buf to update
+ * @bm:		Pointer to udp_meta_t to update
  * @bp:		Pointer to udp_payload_t to update
  * @dstport:	Destination port number
  * @dlen:	Length of UDP payload
@@ -640,14 +619,14 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *bh,
  *
  * Return: size of IPv6 payload (UDP header + data)
  */
-static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *bh,
-			      struct udp_payload_t *bp,
+static size_t udp_update_hdr6(const struct ctx *c,
+			      struct udp_meta_t *bm, struct udp_payload_t *bp,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
-	const struct in6_addr *src = &bh->s_in6.sin6_addr;
+	const struct in6_addr *src = &bm->s_in.sa6.sin6_addr;
 	const struct in6_addr *dst = &c->ip6.addr_seen;
-	in_port_t srcport = ntohs(bh->s_in6.sin6_port);
+	in_port_t srcport = ntohs(bm->s_in.sa6.sin6_port);
 	uint16_t l4len = dlen + sizeof(bp->uh);
 
 	if (IN6_IS_ADDR_LINKLOCAL(src)) {
@@ -684,19 +663,19 @@ static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *bh,
 
 	}
 
-	bh->ip6h.payload_len = htons(l4len);
-	bh->ip6h.daddr = *dst;
-	bh->ip6h.saddr = *src;
-	bh->ip6h.version = 6;
-	bh->ip6h.nexthdr = IPPROTO_UDP;
-	bh->ip6h.hop_limit = 255;
+	bm->ip6h.payload_len = htons(l4len);
+	bm->ip6h.daddr = *dst;
+	bm->ip6h.saddr = *src;
+	bm->ip6h.version = 6;
+	bm->ip6h.nexthdr = IPPROTO_UDP;
+	bm->ip6h.hop_limit = 255;
 
-	bp->uh.source = bh->s_in6.sin6_port;
+	bp->uh.source = bm->s_in.sa6.sin6_port;
 	bp->uh.dest = htons(dstport);
-	bp->uh.len = bh->ip6h.payload_len;
+	bp->uh.len = bm->ip6h.payload_len;
 	csum_udp6(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&bh->taph, l4len + sizeof(bh->ip6h) + sizeof(udp6_eth_hdr));
+	tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip6h) + sizeof(udp6_eth_hdr));
 	return l4len;
 }
 
@@ -725,15 +704,14 @@ static void udp_tap_send(const struct ctx *c,
 
 	for (i = start; i < start + n; i++) {
 		struct udp_payload_t *bp = &udp_payload[i];
+		struct udp_meta_t *bm = &udp_meta[i];
 		size_t l4len;
 
 		if (v6) {
-			l4len = udp_update_hdr6(c, &udp6_l2_buf[i], bp,
-						dstport,
+			l4len = udp_update_hdr6(c, bm, bp, dstport,
 						udp6_l2_mh_sock[i].msg_len, now);
 		} else {
-			l4len = udp_update_hdr4(c, &udp4_l2_buf[i], bp,
-						dstport,
+			l4len = udp_update_hdr4(c, bm, bp, dstport,
 						udp4_l2_mh_sock[i].msg_len, now);
 		}
 		tap_iov[i][UDP_IOV_PAYLOAD].iov_len = l4len;
-- 
@@ -189,44 +189,27 @@ udp_payload[UDP_MAX_FRAMES];
 /* Ethernet header for IPv4 frames */
 static struct ethhdr udp4_eth_hdr;
 
-/**
- * udp4_l2_buf_t - Pre-cooked IPv4 packet buffers for tap connections
- * @s_in:	Source socket address, filled in by recvmmsg()
- * @taph:	Tap backend specific header
- * @iph:	Pre-filled IP header (except for tot_len and saddr)
- */
-static struct udp4_l2_buf_t {
-	struct sockaddr_in s_in;
-
-	struct tap_hdr taph;
-	struct iphdr iph;
-} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
-udp4_l2_buf[UDP_MAX_FRAMES];
-
 /* Ethernet header for IPv6 frames */
 static struct ethhdr udp6_eth_hdr;
 
 /**
- * udp6_l2_buf_t - Pre-cooked IPv6 packet buffers for tap connections
- * @s_in6:	Source socket address, filled in by recvmmsg()
+ * struct udp_meta_t - Pre-cooked headers and metadata for UDP packets
+ * @ip6h:	Pre-filled IPv6 header (except for payload_len and addresses)
+ * @ip4h:	Pre-filled IPv4 header (except for tot_len and saddr)
  * @taph:	Tap backend specific header
- * @ip6h:	Pre-filled IP header (except for payload_len and addresses)
+ * @s_in:	Source socket address, filled in by recvmmsg()
  */
-struct udp6_l2_buf_t {
-	struct sockaddr_in6 s_in6;
-#ifdef __AVX2__
-	/* Align ip6h to 32-byte boundary. */
-	uint8_t pad[64 - (sizeof(struct sockaddr_in6) + sizeof(struct tap_hdr))];
-#endif
-
-	struct tap_hdr taph;
+static struct udp_meta_t {
 	struct ipv6hdr ip6h;
+	struct iphdr ip4h;
+	struct tap_hdr taph;
+
+	union sockaddr_inany s_in;
+}
 #ifdef __AVX2__
-} __attribute__ ((packed, aligned(32)))
-#else
-} __attribute__ ((packed, aligned(__alignof__(unsigned int))))
+__attribute__ ((aligned(32)))
 #endif
-udp6_l2_buf[UDP_MAX_FRAMES];
+udp_meta[UDP_MAX_FRAMES];
 
 /**
  * enum udp_iov_idx - Indices for the buffers making up a single UDP frame
@@ -321,6 +304,12 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 {
 	struct udp_payload_t *payload = &udp_payload[i];
 	struct iovec *siov = &udp_l2_iov_sock[i];
+	struct udp_meta_t *meta = &udp_meta[i];
+
+	*meta = (struct udp_meta_t) {
+		.ip4h = L2_BUF_IP4_INIT(IPPROTO_UDP),
+		.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP),
+	};
 
 	*siov = IOV_OF_LVALUE(payload->data);
 	udp4_eth_hdr.h_proto = htons_constant(ETH_P_IP);
@@ -328,41 +317,31 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 
 	if (c->ifi4) {
 		struct msghdr *mh = &udp4_l2_mh_sock[i].msg_hdr;
-		struct udp4_l2_buf_t *buf = &udp4_l2_buf[i];
 		struct iovec *tiov = udp4_l2_iov_tap[i];
 
-		*buf = (struct udp4_l2_buf_t) {
-			.iph = L2_BUF_IP4_INIT(IPPROTO_UDP)
-		};
-
-		mh->msg_name	= &buf->s_in;
-		mh->msg_namelen	= sizeof(buf->s_in);
+		mh->msg_name	= &meta->s_in;
+		mh->msg_namelen	= sizeof(struct sockaddr_in);
 		mh->msg_iov	= siov;
 		mh->msg_iovlen	= 1;
 
-		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &meta->taph);
 		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp4_eth_hdr);
-		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->iph);
+		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(meta->ip4h);
 		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
 
 	if (c->ifi6) {
 		struct msghdr *mh = &udp6_l2_mh_sock[i].msg_hdr;
-		struct udp6_l2_buf_t *buf = &udp6_l2_buf[i];
 		struct iovec *tiov = udp6_l2_iov_tap[i];
 
-		*buf = (struct udp6_l2_buf_t) {
-			.ip6h = L2_BUF_IP6_INIT(IPPROTO_UDP)
-		};
-
-		mh->msg_name	= &buf->s_in6;
-		mh->msg_namelen	= sizeof(buf->s_in6);
+		mh->msg_name	= &meta->s_in;
+		mh->msg_namelen	= sizeof(struct sockaddr_in6);
 		mh->msg_iov	= siov;
 		mh->msg_iovlen	= 1;
 
-		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &buf->taph);
+		tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &meta->taph);
 		tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp6_eth_hdr);
-		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(buf->ip6h);
+		tiov[UDP_IOV_IP] = IOV_OF_LVALUE(meta->ip6h);
 		tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 	}
 }
@@ -577,7 +556,7 @@ static void udp_splice_sendfrom(const struct ctx *c, unsigned start, unsigned n,
 /**
  * udp_update_hdr4() - Update headers for one IPv4 datagram
  * @c:		Execution context
- * @bh:		Pointer to udp4_l2_buf to update
+ * @bm:		Pointer to udp_meta_t to update
  * @bp:		Pointer to udp_payload_t to update
  * @dstport:	Destination port number
  * @dlen:	Length of UDP payload
@@ -585,16 +564,16 @@ static void udp_splice_sendfrom(const struct ctx *c, unsigned start, unsigned n,
  *
  * Return: size of IPv4 payload (UDP header + data)
  */
-static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *bh,
-			      struct udp_payload_t *bp,
+static size_t udp_update_hdr4(const struct ctx *c,
+			      struct udp_meta_t *bm, struct udp_payload_t *bp,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
-	in_port_t srcport = ntohs(bh->s_in.sin_port);
+	in_port_t srcport = ntohs(bm->s_in.sa4.sin_port);
 	const struct in_addr dst = c->ip4.addr_seen;
-	struct in_addr src = bh->s_in.sin_addr;
+	struct in_addr src = bm->s_in.sa4.sin_addr;
 	size_t l4len = dlen + sizeof(bp->uh);
-	size_t l3len = l4len + sizeof(bh->iph);
+	size_t l3len = l4len + sizeof(bm->ip4h);
 
 	if (!IN4_IS_ADDR_UNSPECIFIED(&c->ip4.dns_match) &&
 	    IN4_ARE_ADDR_EQUAL(&src, &c->ip4.dns_host) && srcport == 53 &&
@@ -615,24 +594,24 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *bh,
 		src = c->ip4.gw;
 	}
 
-	bh->iph.tot_len = htons(l3len);
-	bh->iph.daddr = dst.s_addr;
-	bh->iph.saddr = src.s_addr;
-	bh->iph.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
+	bm->ip4h.tot_len = htons(l3len);
+	bm->ip4h.daddr = dst.s_addr;
+	bm->ip4h.saddr = src.s_addr;
+	bm->ip4h.check = csum_ip4_header(l3len, IPPROTO_UDP, src, dst);
 
-	bp->uh.source = bh->s_in.sin_port;
+	bp->uh.source = bm->s_in.sa4.sin_port;
 	bp->uh.dest = htons(dstport);
 	bp->uh.len = htons(l4len);
 	csum_udp4(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&bh->taph, l3len + sizeof(udp4_eth_hdr));
+	tap_hdr_update(&bm->taph, l3len + sizeof(udp4_eth_hdr));
 	return l4len;
 }
 
 /**
  * udp_update_hdr6() - Update headers for one IPv6 datagram
  * @c:		Execution context
- * @bh:		Pointer to udp6_l2_buf to update
+ * @bm:		Pointer to udp_meta_t to update
  * @bp:		Pointer to udp_payload_t to update
  * @dstport:	Destination port number
  * @dlen:	Length of UDP payload
@@ -640,14 +619,14 @@ static size_t udp_update_hdr4(const struct ctx *c, struct udp4_l2_buf_t *bh,
  *
  * Return: size of IPv6 payload (UDP header + data)
  */
-static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *bh,
-			      struct udp_payload_t *bp,
+static size_t udp_update_hdr6(const struct ctx *c,
+			      struct udp_meta_t *bm, struct udp_payload_t *bp,
 			      in_port_t dstport, size_t dlen,
 			      const struct timespec *now)
 {
-	const struct in6_addr *src = &bh->s_in6.sin6_addr;
+	const struct in6_addr *src = &bm->s_in.sa6.sin6_addr;
 	const struct in6_addr *dst = &c->ip6.addr_seen;
-	in_port_t srcport = ntohs(bh->s_in6.sin6_port);
+	in_port_t srcport = ntohs(bm->s_in.sa6.sin6_port);
 	uint16_t l4len = dlen + sizeof(bp->uh);
 
 	if (IN6_IS_ADDR_LINKLOCAL(src)) {
@@ -684,19 +663,19 @@ static size_t udp_update_hdr6(const struct ctx *c, struct udp6_l2_buf_t *bh,
 
 	}
 
-	bh->ip6h.payload_len = htons(l4len);
-	bh->ip6h.daddr = *dst;
-	bh->ip6h.saddr = *src;
-	bh->ip6h.version = 6;
-	bh->ip6h.nexthdr = IPPROTO_UDP;
-	bh->ip6h.hop_limit = 255;
+	bm->ip6h.payload_len = htons(l4len);
+	bm->ip6h.daddr = *dst;
+	bm->ip6h.saddr = *src;
+	bm->ip6h.version = 6;
+	bm->ip6h.nexthdr = IPPROTO_UDP;
+	bm->ip6h.hop_limit = 255;
 
-	bp->uh.source = bh->s_in6.sin6_port;
+	bp->uh.source = bm->s_in.sa6.sin6_port;
 	bp->uh.dest = htons(dstport);
-	bp->uh.len = bh->ip6h.payload_len;
+	bp->uh.len = bm->ip6h.payload_len;
 	csum_udp6(&bp->uh, src, dst, bp->data, dlen);
 
-	tap_hdr_update(&bh->taph, l4len + sizeof(bh->ip6h) + sizeof(udp6_eth_hdr));
+	tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip6h) + sizeof(udp6_eth_hdr));
 	return l4len;
 }
 
@@ -725,15 +704,14 @@ static void udp_tap_send(const struct ctx *c,
 
 	for (i = start; i < start + n; i++) {
 		struct udp_payload_t *bp = &udp_payload[i];
+		struct udp_meta_t *bm = &udp_meta[i];
 		size_t l4len;
 
 		if (v6) {
-			l4len = udp_update_hdr6(c, &udp6_l2_buf[i], bp,
-						dstport,
+			l4len = udp_update_hdr6(c, bm, bp, dstport,
 						udp6_l2_mh_sock[i].msg_len, now);
 		} else {
-			l4len = udp_update_hdr4(c, &udp4_l2_buf[i], bp,
-						dstport,
+			l4len = udp_update_hdr4(c, bm, bp, dstport,
 						udp4_l2_mh_sock[i].msg_len, now);
 		}
 		tap_iov[i][UDP_IOV_PAYLOAD].iov_len = l4len;
-- 
2.44.0

Re: [PATCH v2 0/7] Rework UDP buffers

[Stefano Brivio]

On Wed,  1 May 2024 18:31:03 +1000
David Gibson <david@gibson.dropbear.id.au> wrote:

> Laurent recently reworked the TCP buffer handling to be split into
> various pieces tracked by iovecs.  We'll want that for various future
> changes.  This series makes a similar split for UDP buffers, which
> we'll want in order to allow dual-stack UDP sockets, amongst other
> things.
> 
> This is based on my earlier series of cleanups for the TCP buffer
> handling.
> 
> Changes since v1:
>  * Rebase on v2 of the TCP iov and buffer handling cleanups
>  * Rename udp_payload and udp_meta structures to _t for more
>    consistency with tcp.c
>  * Other minor changes based on Stefano's review
> 
> David Gibson (7):
>   test: Allow sftp via vsock-ssh in tests
>   udp: Split tap-bound UDP packets into multiple buffers using io vector
>   udp: Combine initialisation of IPv4 and IPv6 iovs
>   udp: Explicitly set checksum in guest-bound UDP headers
>   udp: Share payload buffers between IPv4 and IPv6
>   udp: Use the same buffer for the L2 header for all frames
>   udp: Single buffer for IPv4, IPv6 headers and metadata

Applied.

-- 
Stefano