[PATCH] tcp_vu: head_cnt need not be global

[PATCH] tcp_vu: head_cnt need not be global

[David Gibson]

head_cnt is a global variable which tracks how many entries in head[] are
currently used.  The fact that it's global obscures the fact that the
lifetime over which it has a meaningful value is quite short: a single
call to of tcp_vu_data_from_sock().

Make it a local to tcp_vu_data_from_sock() to make that lifetime clearer.
We keep the head[] array global for now - although technically it has the
same valid lifetime - because it's large enough we might not want to put
it on the stack.

Cc: Laurent Vivier <lvivier@redhat.com>

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 tcp_vu.c | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/tcp_vu.c b/tcp_vu.c
index 0622f173..6891ed13 100644
--- a/tcp_vu.c
+++ b/tcp_vu.c
@@ -38,7 +38,6 @@
 static struct iovec iov_vu[VIRTQUEUE_MAX_SIZE + 1];
 static struct vu_virtq_element elem[VIRTQUEUE_MAX_SIZE];
 static int head[VIRTQUEUE_MAX_SIZE + 1];
-static int head_cnt;
 
 /**
  * tcp_vu_hdrlen() - return the size of the header in level 2 frame (TCP)
@@ -183,7 +182,7 @@ int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
 static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 				const struct tcp_tap_conn *conn, bool v6,
 				uint32_t already_sent, size_t fillsize,
-				int *iov_cnt)
+				int *iov_cnt, int *head_cnt)
 {
 	struct vu_dev *vdev = c->vdev;
 	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
@@ -202,7 +201,7 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 	vu_init_elem(elem, &iov_vu[1], VIRTQUEUE_MAX_SIZE);
 
 	elem_cnt = 0;
-	head_cnt = 0;
+	*head_cnt = 0;
 	while (fillsize > 0 && elem_cnt < VIRTQUEUE_MAX_SIZE) {
 		struct iovec *iov;
 		size_t frame_size, dlen;
@@ -221,7 +220,7 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 		ASSERT(iov->iov_len >= hdrlen);
 		iov->iov_base = (char *)iov->iov_base + hdrlen;
 		iov->iov_len -= hdrlen;
-		head[head_cnt++] = elem_cnt;
+		head[(*head_cnt)++] = elem_cnt;
 
 		fillsize -= dlen;
 		elem_cnt += cnt;
@@ -261,17 +260,18 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 		len -= iov->iov_len;
 	}
 	/* adjust head count */
-	while (head_cnt > 0 && head[head_cnt - 1] >= i)
-		head_cnt--;
+	while (*head_cnt > 0 && head[*head_cnt - 1] >= i)
+		(*head_cnt)--;
+
 	/* mark end of array */
-	head[head_cnt] = i;
+	head[*head_cnt] = i;
 	*iov_cnt = i;
 
 	/* release unused buffers */
 	vu_queue_rewind(vq, elem_cnt - i);
 
 	/* restore space for headers in iov */
-	for (i = 0; i < head_cnt; i++) {
+	for (i = 0; i < *head_cnt; i++) {
 		struct iovec *iov = &elem[head[i]].in_sg[0];
 
 		iov->iov_base = (char *)iov->iov_base - hdrlen;
@@ -357,11 +357,11 @@ int tcp_vu_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn)
 	struct vu_dev *vdev = c->vdev;
 	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
 	ssize_t len, previous_dlen;
+	int i, iov_cnt, head_cnt;
 	size_t hdrlen, fillsize;
 	int v6 = CONN_V6(conn);
 	uint32_t already_sent;
 	const uint16_t *check;
-	int i, iov_cnt;
 
 	if (!vu_queue_enabled(vq) || !vu_queue_started(vq)) {
 		debug("Got packet, but RX virtqueue not usable yet");
@@ -396,7 +396,8 @@ int tcp_vu_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn)
 	/* collect the buffers from vhost-user and fill them with the
 	 * data from the socket
 	 */
-	len = tcp_vu_sock_recv(c, conn, v6, already_sent, fillsize, &iov_cnt);
+	len = tcp_vu_sock_recv(c, conn, v6, already_sent, fillsize,
+			       &iov_cnt, &head_cnt);
 	if (len < 0) {
 		if (len != -EAGAIN && len != -EWOULDBLOCK) {
 			tcp_rst(c, conn);
-- 
@@ -38,7 +38,6 @@
 static struct iovec iov_vu[VIRTQUEUE_MAX_SIZE + 1];
 static struct vu_virtq_element elem[VIRTQUEUE_MAX_SIZE];
 static int head[VIRTQUEUE_MAX_SIZE + 1];
-static int head_cnt;
 
 /**
  * tcp_vu_hdrlen() - return the size of the header in level 2 frame (TCP)
@@ -183,7 +182,7 @@ int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
 static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 				const struct tcp_tap_conn *conn, bool v6,
 				uint32_t already_sent, size_t fillsize,
-				int *iov_cnt)
+				int *iov_cnt, int *head_cnt)
 {
 	struct vu_dev *vdev = c->vdev;
 	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
@@ -202,7 +201,7 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 	vu_init_elem(elem, &iov_vu[1], VIRTQUEUE_MAX_SIZE);
 
 	elem_cnt = 0;
-	head_cnt = 0;
+	*head_cnt = 0;
 	while (fillsize > 0 && elem_cnt < VIRTQUEUE_MAX_SIZE) {
 		struct iovec *iov;
 		size_t frame_size, dlen;
@@ -221,7 +220,7 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 		ASSERT(iov->iov_len >= hdrlen);
 		iov->iov_base = (char *)iov->iov_base + hdrlen;
 		iov->iov_len -= hdrlen;
-		head[head_cnt++] = elem_cnt;
+		head[(*head_cnt)++] = elem_cnt;
 
 		fillsize -= dlen;
 		elem_cnt += cnt;
@@ -261,17 +260,18 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 		len -= iov->iov_len;
 	}
 	/* adjust head count */
-	while (head_cnt > 0 && head[head_cnt - 1] >= i)
-		head_cnt--;
+	while (*head_cnt > 0 && head[*head_cnt - 1] >= i)
+		(*head_cnt)--;
+
 	/* mark end of array */
-	head[head_cnt] = i;
+	head[*head_cnt] = i;
 	*iov_cnt = i;
 
 	/* release unused buffers */
 	vu_queue_rewind(vq, elem_cnt - i);
 
 	/* restore space for headers in iov */
-	for (i = 0; i < head_cnt; i++) {
+	for (i = 0; i < *head_cnt; i++) {
 		struct iovec *iov = &elem[head[i]].in_sg[0];
 
 		iov->iov_base = (char *)iov->iov_base - hdrlen;
@@ -357,11 +357,11 @@ int tcp_vu_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn)
 	struct vu_dev *vdev = c->vdev;
 	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
 	ssize_t len, previous_dlen;
+	int i, iov_cnt, head_cnt;
 	size_t hdrlen, fillsize;
 	int v6 = CONN_V6(conn);
 	uint32_t already_sent;
 	const uint16_t *check;
-	int i, iov_cnt;
 
 	if (!vu_queue_enabled(vq) || !vu_queue_started(vq)) {
 		debug("Got packet, but RX virtqueue not usable yet");
@@ -396,7 +396,8 @@ int tcp_vu_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn)
 	/* collect the buffers from vhost-user and fill them with the
 	 * data from the socket
 	 */
-	len = tcp_vu_sock_recv(c, conn, v6, already_sent, fillsize, &iov_cnt);
+	len = tcp_vu_sock_recv(c, conn, v6, already_sent, fillsize,
+			       &iov_cnt, &head_cnt);
 	if (len < 0) {
 		if (len != -EAGAIN && len != -EWOULDBLOCK) {
 			tcp_rst(c, conn);
-- 
2.48.1

Re: [PATCH] tcp_vu: head_cnt need not be global

[Laurent Vivier]

On 18/02/2025 03:50, David Gibson wrote:
> head_cnt is a global variable which tracks how many entries in head[] are
> currently used.  The fact that it's global obscures the fact that the
> lifetime over which it has a meaningful value is quite short: a single
> call to of tcp_vu_data_from_sock().
> 
> Make it a local to tcp_vu_data_from_sock() to make that lifetime clearer.
> We keep the head[] array global for now - although technically it has the
> same valid lifetime - because it's large enough we might not want to put
> it on the stack.

We can make the array local but on the heap using a static declaration in the function.

Reviewed-by: Laurent Vivier <lvivier@redhat.com>

> 
> Cc: Laurent Vivier <lvivier@redhat.com>
> 
> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
> ---
>   tcp_vu.c | 21 +++++++++++----------
>   1 file changed, 11 insertions(+), 10 deletions(-)
> 
> diff --git a/tcp_vu.c b/tcp_vu.c
> index 0622f173..6891ed13 100644
> --- a/tcp_vu.c
> +++ b/tcp_vu.c
> @@ -38,7 +38,6 @@
>   static struct iovec iov_vu[VIRTQUEUE_MAX_SIZE + 1];
>   static struct vu_virtq_element elem[VIRTQUEUE_MAX_SIZE];
>   static int head[VIRTQUEUE_MAX_SIZE + 1];
> -static int head_cnt;
>   
>   /**
>    * tcp_vu_hdrlen() - return the size of the header in level 2 frame (TCP)
> @@ -183,7 +182,7 @@ int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
>   static ssize_t tcp_vu_sock_recv(const struct ctx *c,
>   				const struct tcp_tap_conn *conn, bool v6,
>   				uint32_t already_sent, size_t fillsize,
> -				int *iov_cnt)
> +				int *iov_cnt, int *head_cnt)
>   {
>   	struct vu_dev *vdev = c->vdev;
>   	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
> @@ -202,7 +201,7 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
>   	vu_init_elem(elem, &iov_vu[1], VIRTQUEUE_MAX_SIZE);
>   
>   	elem_cnt = 0;
> -	head_cnt = 0;
> +	*head_cnt = 0;
>   	while (fillsize > 0 && elem_cnt < VIRTQUEUE_MAX_SIZE) {
>   		struct iovec *iov;
>   		size_t frame_size, dlen;
> @@ -221,7 +220,7 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
>   		ASSERT(iov->iov_len >= hdrlen);
>   		iov->iov_base = (char *)iov->iov_base + hdrlen;
>   		iov->iov_len -= hdrlen;
> -		head[head_cnt++] = elem_cnt;
> +		head[(*head_cnt)++] = elem_cnt;
>   
>   		fillsize -= dlen;
>   		elem_cnt += cnt;
> @@ -261,17 +260,18 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
>   		len -= iov->iov_len;
>   	}
>   	/* adjust head count */
> -	while (head_cnt > 0 && head[head_cnt - 1] >= i)
> -		head_cnt--;
> +	while (*head_cnt > 0 && head[*head_cnt - 1] >= i)
> +		(*head_cnt)--;
> +
>   	/* mark end of array */
> -	head[head_cnt] = i;
> +	head[*head_cnt] = i;
>   	*iov_cnt = i;
>   
>   	/* release unused buffers */
>   	vu_queue_rewind(vq, elem_cnt - i);
>   
>   	/* restore space for headers in iov */
> -	for (i = 0; i < head_cnt; i++) {
> +	for (i = 0; i < *head_cnt; i++) {
>   		struct iovec *iov = &elem[head[i]].in_sg[0];
>   
>   		iov->iov_base = (char *)iov->iov_base - hdrlen;
> @@ -357,11 +357,11 @@ int tcp_vu_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn)
>   	struct vu_dev *vdev = c->vdev;
>   	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
>   	ssize_t len, previous_dlen;
> +	int i, iov_cnt, head_cnt;
>   	size_t hdrlen, fillsize;
>   	int v6 = CONN_V6(conn);
>   	uint32_t already_sent;
>   	const uint16_t *check;
> -	int i, iov_cnt;
>   
>   	if (!vu_queue_enabled(vq) || !vu_queue_started(vq)) {
>   		debug("Got packet, but RX virtqueue not usable yet");
> @@ -396,7 +396,8 @@ int tcp_vu_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn)
>   	/* collect the buffers from vhost-user and fill them with the
>   	 * data from the socket
>   	 */
> -	len = tcp_vu_sock_recv(c, conn, v6, already_sent, fillsize, &iov_cnt);
> +	len = tcp_vu_sock_recv(c, conn, v6, already_sent, fillsize,
> +			       &iov_cnt, &head_cnt);
>   	if (len < 0) {
>   		if (len != -EAGAIN && len != -EWOULDBLOCK) {
>   			tcp_rst(c, conn);

Re: [PATCH] tcp_vu: head_cnt need not be global

[Stefano Brivio]

On Tue, 18 Feb 2025 09:01:30 +0100
Laurent Vivier <lvivier@redhat.com> wrote:

> On 18/02/2025 03:50, David Gibson wrote:
> > head_cnt is a global variable which tracks how many entries in head[] are
> > currently used.  The fact that it's global obscures the fact that the
> > lifetime over which it has a meaningful value is quite short: a single
> > call to of tcp_vu_data_from_sock().
> > 
> > Make it a local to tcp_vu_data_from_sock() to make that lifetime clearer.
> > We keep the head[] array global for now - although technically it has the
> > same valid lifetime - because it's large enough we might not want to put
> > it on the stack.  
> 
> We can make the array local but on the heap using a static declaration in the function.

Well, it would still be in the data segment, not actually on the heap
[1], but it would be somewhat hidden in the function.

I'd rather suggest that we keep this stuff visible at the "top level".


[1] I had a doubt for a moment so I tried it:

$ git diff
diff --git a/tcp_vu.c b/tcp_vu.c
index 0622f17..c40b102 100644
--- a/tcp_vu.c
+++ b/tcp_vu.c
@@ -187,6 +187,7 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 {
 	struct vu_dev *vdev = c->vdev;
 	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
+	static char am_i_heap_or_am_i_data[64 << 20];
 	struct msghdr mh_sock = { 0 };
 	uint16_t mss = MSS_GET(conn);
 	int s = conn->sock;
@@ -195,6 +196,8 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 	int elem_cnt;
 	int i;
 
+	memset(am_i_heap_or_am_i_data, 0, sizeof(am_i_heap_or_am_i_data));
+
 	*iov_cnt = 0;
 
 	hdrlen = tcp_vu_hdrlen(v6);

$ nm -td -S passt | grep am_i_heap_or_am_i_data
0000000202163552 0000000067108864 b am_i_heap_or_am_i_data.2

-- 
@@ -187,6 +187,7 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 {
 	struct vu_dev *vdev = c->vdev;
 	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
+	static char am_i_heap_or_am_i_data[64 << 20];
 	struct msghdr mh_sock = { 0 };
 	uint16_t mss = MSS_GET(conn);
 	int s = conn->sock;
@@ -195,6 +196,8 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
 	int elem_cnt;
 	int i;
 
+	memset(am_i_heap_or_am_i_data, 0, sizeof(am_i_heap_or_am_i_data));
+
 	*iov_cnt = 0;
 
 	hdrlen = tcp_vu_hdrlen(v6);

$ nm -td -S passt | grep am_i_heap_or_am_i_data
0000000202163552 0000000067108864 b am_i_heap_or_am_i_data.2

-- 
Stefano

Re: [PATCH] tcp_vu: head_cnt need not be global

[David Gibson]

[-- Attachment #1: Type: text/plain, Size: 4704 bytes --]

On Tue, Feb 18, 2025 at 09:01:30AM +0100, Laurent Vivier wrote:
> On 18/02/2025 03:50, David Gibson wrote:
> > head_cnt is a global variable which tracks how many entries in head[] are
> > currently used.  The fact that it's global obscures the fact that the
> > lifetime over which it has a meaningful value is quite short: a single
> > call to of tcp_vu_data_from_sock().
> > 
> > Make it a local to tcp_vu_data_from_sock() to make that lifetime clearer.
> > We keep the head[] array global for now - although technically it has the
> > same valid lifetime - because it's large enough we might not want to put
> > it on the stack.
> 
> We can make the array local but on the heap using a static
> declaration in the function.

We could.  I kind of dislike static locals, as a rule, because it's
pretty easy to miss them and make things non-reentrant by accident.

> Reviewed-by: Laurent Vivier <lvivier@redhat.com>
> 
> > 
> > Cc: Laurent Vivier <lvivier@redhat.com>
> > 
> > Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
> > ---
> >   tcp_vu.c | 21 +++++++++++----------
> >   1 file changed, 11 insertions(+), 10 deletions(-)
> > 
> > diff --git a/tcp_vu.c b/tcp_vu.c
> > index 0622f173..6891ed13 100644
> > --- a/tcp_vu.c
> > +++ b/tcp_vu.c
> > @@ -38,7 +38,6 @@
> >   static struct iovec iov_vu[VIRTQUEUE_MAX_SIZE + 1];
> >   static struct vu_virtq_element elem[VIRTQUEUE_MAX_SIZE];
> >   static int head[VIRTQUEUE_MAX_SIZE + 1];
> > -static int head_cnt;
> >   /**
> >    * tcp_vu_hdrlen() - return the size of the header in level 2 frame (TCP)
> > @@ -183,7 +182,7 @@ int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
> >   static ssize_t tcp_vu_sock_recv(const struct ctx *c,
> >   				const struct tcp_tap_conn *conn, bool v6,
> >   				uint32_t already_sent, size_t fillsize,
> > -				int *iov_cnt)
> > +				int *iov_cnt, int *head_cnt)
> >   {
> >   	struct vu_dev *vdev = c->vdev;
> >   	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
> > @@ -202,7 +201,7 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
> >   	vu_init_elem(elem, &iov_vu[1], VIRTQUEUE_MAX_SIZE);
> >   	elem_cnt = 0;
> > -	head_cnt = 0;
> > +	*head_cnt = 0;
> >   	while (fillsize > 0 && elem_cnt < VIRTQUEUE_MAX_SIZE) {
> >   		struct iovec *iov;
> >   		size_t frame_size, dlen;
> > @@ -221,7 +220,7 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
> >   		ASSERT(iov->iov_len >= hdrlen);
> >   		iov->iov_base = (char *)iov->iov_base + hdrlen;
> >   		iov->iov_len -= hdrlen;
> > -		head[head_cnt++] = elem_cnt;
> > +		head[(*head_cnt)++] = elem_cnt;
> >   		fillsize -= dlen;
> >   		elem_cnt += cnt;
> > @@ -261,17 +260,18 @@ static ssize_t tcp_vu_sock_recv(const struct ctx *c,
> >   		len -= iov->iov_len;
> >   	}
> >   	/* adjust head count */
> > -	while (head_cnt > 0 && head[head_cnt - 1] >= i)
> > -		head_cnt--;
> > +	while (*head_cnt > 0 && head[*head_cnt - 1] >= i)
> > +		(*head_cnt)--;
> > +
> >   	/* mark end of array */
> > -	head[head_cnt] = i;
> > +	head[*head_cnt] = i;
> >   	*iov_cnt = i;
> >   	/* release unused buffers */
> >   	vu_queue_rewind(vq, elem_cnt - i);
> >   	/* restore space for headers in iov */
> > -	for (i = 0; i < head_cnt; i++) {
> > +	for (i = 0; i < *head_cnt; i++) {
> >   		struct iovec *iov = &elem[head[i]].in_sg[0];
> >   		iov->iov_base = (char *)iov->iov_base - hdrlen;
> > @@ -357,11 +357,11 @@ int tcp_vu_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn)
> >   	struct vu_dev *vdev = c->vdev;
> >   	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
> >   	ssize_t len, previous_dlen;
> > +	int i, iov_cnt, head_cnt;
> >   	size_t hdrlen, fillsize;
> >   	int v6 = CONN_V6(conn);
> >   	uint32_t already_sent;
> >   	const uint16_t *check;
> > -	int i, iov_cnt;
> >   	if (!vu_queue_enabled(vq) || !vu_queue_started(vq)) {
> >   		debug("Got packet, but RX virtqueue not usable yet");
> > @@ -396,7 +396,8 @@ int tcp_vu_data_from_sock(const struct ctx *c, struct tcp_tap_conn *conn)
> >   	/* collect the buffers from vhost-user and fill them with the
> >   	 * data from the socket
> >   	 */
> > -	len = tcp_vu_sock_recv(c, conn, v6, already_sent, fillsize, &iov_cnt);
> > +	len = tcp_vu_sock_recv(c, conn, v6, already_sent, fillsize,
> > +			       &iov_cnt, &head_cnt);
> >   	if (len < 0) {
> >   		if (len != -EAGAIN && len != -EWOULDBLOCK) {
> >   			tcp_rst(c, conn);
> 

-- 
David Gibson (he or they)	| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you, not the other way
				| around.
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH] tcp_vu: head_cnt need not be global

[Stefano Brivio]

On Tue, 18 Feb 2025 13:50:13 +1100
David Gibson <david@gibson.dropbear.id.au> wrote:

> head_cnt is a global variable which tracks how many entries in head[] are
> currently used.  The fact that it's global obscures the fact that the
> lifetime over which it has a meaningful value is quite short: a single
> call to of tcp_vu_data_from_sock().
> 
> Make it a local to tcp_vu_data_from_sock() to make that lifetime clearer.
> We keep the head[] array global for now - although technically it has the
> same valid lifetime - because it's large enough we might not want to put
> it on the stack.
> 
> Cc: Laurent Vivier <lvivier@redhat.com>
> 
> Signed-off-by: David Gibson <david@gibson.dropbear.id.au>

Applied.

-- 
Stefano