[PATCH v3 17/18] tap: Improve handling of partial frame sends

[David Gibson <david@gibson.dropbear.id.au>]

In passt mode, when writing frames to the qemu socket, we might get a short
send.  If we ignored this and carried on, the qemu socket would get out of
sync, because the bytes we actually sent wouldn't correspond  to the length
header we already sent.  tap_send_frames_passt() handles that by doing a
a blocking send to complete the message, but it has a few flaws:
 * We only attempt to resend once: although it's unlikely in practice,
   nothing prevents the blocking send() from also being short
 * We print a debug error if send() returns non-zero.. but send() returns
   the number of bytes sent, so we actually want it to return the length
   of the remaining data.

Correct those flaws and also be a bit more thorough about reporting
problems here.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 tap.c | 49 ++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/tap.c b/tap.c
index 5ec6b70..af9bc15 100644
--- a/tap.c
+++ b/tap.c
@@ -326,13 +326,37 @@ static void tap_send_frames_pasta(struct ctx *c,
 	}
 }
 
+/**
+ * tap_send_remainder() - Send remainder of a partially sent frame
+ * @c:		Execution context
+ * @iov:	Partially sent buffer
+ * @offset:	Number of bytes already sent from @iov
+ */
+static void tap_send_remainder(const struct ctx *c, const struct iovec *iov,
+			       size_t offset)
+{
+	const char *base = (char *)iov->iov_base;
+	size_t len = iov->iov_len;
+
+	while (offset < len) {
+		ssize_t sent = send(c->fd_tap, base + offset, len - offset,
+				    MSG_NOSIGNAL);
+		if (sent < 0) {
+			err("tap: partial frame send (missing %lu bytes): %s",
+			    len - offset, strerror(errno));
+			return;
+		}
+		offset += sent;
+	}
+}
+
 /**
  * tap_send_frames_passt() - Send multiple frames to the passt tap
  * @c:		Execution context
  * @iov:	Array of buffers, each containing one frame
  * @n:		Number of buffers/frames in @iov
  *
- * #syscalls:passt sendmsg send
+ * #syscalls:passt sendmsg
  */
 static void tap_send_frames_passt(const struct ctx *c,
 				  const struct iovec *iov, size_t n)
@@ -341,29 +365,28 @@ static void tap_send_frames_passt(const struct ctx *c,
 		.msg_iov = (void *)iov,
 		.msg_iovlen = n,
 	};
-	size_t end = 0, missing;
 	unsigned int i;
 	ssize_t sent;
-	char *p;
 
 	sent = sendmsg(c->fd_tap, &mh, MSG_NOSIGNAL | MSG_DONTWAIT);
 	if (sent < 0)
 		return;
 
-	/* Ensure a complete last message on partial sendmsg() */
-	for (i = 0; i < n; i++, iov++) {
-		end += iov->iov_len;
-		if (end >= (size_t)sent)
+	/* Check for any partial frames due to short send */
+	for (i = 0; i < n; i++) {
+		if ((size_t)sent < iov[i].iov_len)
 			break;
+		sent -= iov[i].iov_len;
 	}
 
-	missing = end - sent;
-	if (!missing)
-		return;
+	if (i < n && sent) {
+		/* A partial frame was sent */
+		tap_send_remainder(c, &iov[i], sent);
+		i++;
+	}
 
-	p = (char *)iov->iov_base + iov->iov_len - missing;
-	if (send(c->fd_tap, p, missing, MSG_NOSIGNAL))
-		debug("tap: failed to flush %lu missing bytes to tap", missing);
+	if (i < n)
+		debug("tap: dropped %lu frames due to short send", n - i);
 }
 
 /**
-- 
@@ -326,13 +326,37 @@ static void tap_send_frames_pasta(struct ctx *c,
 	}
 }
 
+/**
+ * tap_send_remainder() - Send remainder of a partially sent frame
+ * @c:		Execution context
+ * @iov:	Partially sent buffer
+ * @offset:	Number of bytes already sent from @iov
+ */
+static void tap_send_remainder(const struct ctx *c, const struct iovec *iov,
+			       size_t offset)
+{
+	const char *base = (char *)iov->iov_base;
+	size_t len = iov->iov_len;
+
+	while (offset < len) {
+		ssize_t sent = send(c->fd_tap, base + offset, len - offset,
+				    MSG_NOSIGNAL);
+		if (sent < 0) {
+			err("tap: partial frame send (missing %lu bytes): %s",
+			    len - offset, strerror(errno));
+			return;
+		}
+		offset += sent;
+	}
+}
+
 /**
  * tap_send_frames_passt() - Send multiple frames to the passt tap
  * @c:		Execution context
  * @iov:	Array of buffers, each containing one frame
  * @n:		Number of buffers/frames in @iov
  *
- * #syscalls:passt sendmsg send
+ * #syscalls:passt sendmsg
  */
 static void tap_send_frames_passt(const struct ctx *c,
 				  const struct iovec *iov, size_t n)
@@ -341,29 +365,28 @@ static void tap_send_frames_passt(const struct ctx *c,
 		.msg_iov = (void *)iov,
 		.msg_iovlen = n,
 	};
-	size_t end = 0, missing;
 	unsigned int i;
 	ssize_t sent;
-	char *p;
 
 	sent = sendmsg(c->fd_tap, &mh, MSG_NOSIGNAL | MSG_DONTWAIT);
 	if (sent < 0)
 		return;
 
-	/* Ensure a complete last message on partial sendmsg() */
-	for (i = 0; i < n; i++, iov++) {
-		end += iov->iov_len;
-		if (end >= (size_t)sent)
+	/* Check for any partial frames due to short send */
+	for (i = 0; i < n; i++) {
+		if ((size_t)sent < iov[i].iov_len)
 			break;
+		sent -= iov[i].iov_len;
 	}
 
-	missing = end - sent;
-	if (!missing)
-		return;
+	if (i < n && sent) {
+		/* A partial frame was sent */
+		tap_send_remainder(c, &iov[i], sent);
+		i++;
+	}
 
-	p = (char *)iov->iov_base + iov->iov_len - missing;
-	if (send(c->fd_tap, p, missing, MSG_NOSIGNAL))
-		debug("tap: failed to flush %lu missing bytes to tap", missing);
+	if (i < n)
+		debug("tap: dropped %lu frames due to short send", n - i);
 }
 
 /**
-- 
2.39.0