Re: [PATCH v2 2/6] vhost-user: Enable multiqueue

[David Gibson <david@gibson.dropbear.id.au>]

[-- Attachment #1: Type: text/plain, Size: 10717 bytes --]

On Fri, Nov 21, 2025 at 05:58:58PM +0100, Laurent Vivier wrote:
> Add the --max-queues parameter to specify the maximum number of
> queue

Nit: you updated the option to be --max-qpairs, which I think is good,
but now the commit message is out of date.


One other query: what makes it "max qpairs" rather than just "qpairs"
- are there (now or in planned work) circumstances where you'd end up
with less qpairs than specified here?

> pairs supported in vhost-user mode. This enables multi-queue support
> by allowing configuration of up to 16 queue pairs (32 virtqueues).
> 
> For the moment, only the first RX queue is used, the TX queue is
> selected by the guest kernel.

IIUC, with this patch (but not the ones after) things will break if
the guest uses a qpair other than 0, right?  AFAICT vu_kick_cb() isn't
updated so will ignore anything on the other qpairs.

> 
> Signed-off-by: Laurent Vivier <lvivier@redhat.com>
> ---
>  conf.c       | 31 ++++++++++++++++++++++++++++++-
>  passt.h      |  2 ++
>  tap.c        | 15 +++++++++++++--
>  vhost_user.c | 38 +++++++++++++++++++++-----------------
>  virtio.h     |  2 +-
>  5 files changed, 67 insertions(+), 21 deletions(-)
> 
> diff --git a/conf.c b/conf.c
> index 66b9e63400ec..99a7995f8038 100644
> --- a/conf.c
> +++ b/conf.c
> @@ -862,7 +862,9 @@ static void usage(const char *name, FILE *f, int status)
>  			"  --vhost-user		Enable vhost-user mode\n"
>  			"    UNIX domain socket is provided by -s option\n"
>  			"  --print-capabilities	print back-end capabilities in JSON format,\n"
> -			"    only meaningful for vhost-user mode\n");
> +			"    only meaningful for vhost-user mode\n"
> +			"  --max-qpairs		Specify the maximum number of queue pairs\n"
> +			);
>  		FPRINTF(f,
>  			"  --repair-path PATH	path for passt-repair(1)\n"
>  			"    default: append '.repair' to UNIX domain path\n");
> @@ -1483,6 +1485,7 @@ void conf(struct ctx *c, int argc, char **argv)
>  		{"migrate-exit", no_argument,		NULL,		29 },
>  		{"migrate-no-linger", no_argument,	NULL,		30 },
>  		{"stats", required_argument,		NULL,		31 },
> +		{"max-qpairs", required_argument,	NULL,		32 },
>  		{ 0 },
>  	};
>  	const char *optstring = "+dqfel:hs:F:I:p:P:m:a:n:M:g:i:o:D:S:H:461t:u:T:U:";
> @@ -1514,6 +1517,7 @@ void conf(struct ctx *c, int argc, char **argv)
>  	c->tcp.fwd_in.mode = c->tcp.fwd_out.mode = FWD_UNSET;
>  	c->udp.fwd_in.mode = c->udp.fwd_out.mode = FWD_UNSET;
>  	memcpy(c->our_tap_mac, MAC_OUR_LAA, ETH_ALEN);
> +	c->max_qpairs = 1;
>  
>  	optind = 0;
>  	do {
> @@ -1717,6 +1721,31 @@ void conf(struct ctx *c, int argc, char **argv)
>  				die("Can't display statistics if not running in foreground");
>  			c->stats = strtol(optarg, NULL, 0);
>  			break;
> +		case 32: {
> +			unsigned long max_qpairs;
> +			char *e;
> +
> +			if (c->mode != MODE_VU)
> +				die("--max-qpairs is for vhost-user mode only");
> +
> +			errno = 0;
> +			max_qpairs = strtoul(optarg, &e, 0);
> +
> +			if (errno || *e)
> +				die("Invalid max-qpairs: %s", optarg);
> +
> +			if (max_qpairs < 1) {
> +				die("max-qpairs %lu too small (min 1)",
> +				    max_qpairs);
> +			}
> +
> +			if (max_qpairs * 2 > VHOST_USER_MAX_VQS) {
> +				die("max-qpairs %lu too big (maximum %u)",
> +				    max_qpairs, VHOST_USER_MAX_VQS / 2);
> +			}
> +			c->max_qpairs = max_qpairs;
> +			break;
> +		}
>  		case 'd':
>  			c->debug = 1;
>  			c->quiet = 0;
> diff --git a/passt.h b/passt.h
> index 15801b44bfa8..c7b6dad69190 100644
> --- a/passt.h
> +++ b/passt.h
> @@ -205,6 +205,7 @@ struct ip6_ctx {
>   * @low_wmem:		Low probed net.core.wmem_max
>   * @low_rmem:		Low probed net.core.rmem_max
>   * @vdev:		vhost-user device
> + * @max_qpairs:		Maximum number of queue pairs
>   * @device_state_fd:	Device state migration channel
>   * @device_state_result: Device state migration result
>   * @migrate_target:	Are we the target, on the next migration request?
> @@ -283,6 +284,7 @@ struct ctx {
>  	int low_rmem;
>  
>  	struct vu_dev *vdev;
> +	unsigned int max_qpairs;
>  
>  	/* Migration */
>  	int device_state_fd;
> diff --git a/tap.c b/tap.c
> index d7f777fd0b19..d098061ed559 100644
> --- a/tap.c
> +++ b/tap.c
> @@ -1314,8 +1314,19 @@ static void tap_backend_show_hints(struct ctx *c)
>  		break;
>  	case MODE_VU:
>  		info("You can start qemu with:");
> -		info("    kvm ... -chardev socket,id=chr0,path=%s -netdev vhost-user,id=netdev0,chardev=chr0 -device virtio-net,netdev=netdev0 -object memory-backend-memfd,id=memfd0,share=on,size=$RAMSIZE -numa node,memdev=memfd0\n",
> -		     c->sock_path);
> +		if (c->max_qpairs > 1) {
> +			info("    kvm ... -chardev socket,id=chr0,path=%s "
> +			     "-netdev vhost-user,id=netdev0,chardev=chr0,queues=%d "
> +			     "-device virtio-net,netdev=netdev0,mq=true "
> +			     "-object memory-backend-memfd,id=memfd0,share=on,size=$RAMSIZE "
> +			     "-numa node,memdev=memfd0\n", c->sock_path, c->max_qpairs);
> +		} else {
> +			info("    kvm ... -chardev socket,id=chr0,path=%s "
> +			     "-netdev vhost-user,id=netdev0,chardev=chr0 "
> +			     "-device virtio-net,netdev=netdev0 "
> +			     "-object memory-backend-memfd,id=memfd0,share=on,size=$RAMSIZE "
> +			     "-numa node,memdev=memfd0\n", c->sock_path);
> +		}
>  		break;
>  	}
>  }
> diff --git a/vhost_user.c b/vhost_user.c
> index aa7c869d9e56..6d3fa04d2119 100644
> --- a/vhost_user.c
> +++ b/vhost_user.c
> @@ -323,6 +323,7 @@ static bool vu_get_features_exec(struct vu_dev *vdev,
>  	uint64_t features =
>  		1ULL << VIRTIO_F_VERSION_1 |
>  		1ULL << VIRTIO_NET_F_MRG_RXBUF |
> +		1ULL << VIRTIO_NET_F_MQ |
>  		1ULL << VHOST_F_LOG_ALL |
>  		1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
>  
> @@ -342,9 +343,9 @@ static bool vu_get_features_exec(struct vu_dev *vdev,
>   */
>  static void vu_set_enable_all_rings(struct vu_dev *vdev, bool enable)
>  {
> -	uint16_t i;
> +	unsigned int i;
>  
> -	for (i = 0; i < VHOST_USER_MAX_VQS; i++)
> +	for (i = 0; i < vdev->context->max_qpairs * 2; i++)
>  		vdev->vq[i].enable = enable;
>  }
>  
> @@ -476,7 +477,7 @@ static bool vu_set_mem_table_exec(struct vu_dev *vdev,
>  		close(vmsg->fds[i]);
>  	}
>  
> -	for (i = 0; i < VHOST_USER_MAX_VQS; i++) {
> +	for (i = 0; i < vdev->context->max_qpairs * 2; i++) {
>  		if (vdev->vq[i].vring.desc) {
>  			if (map_ring(vdev, &vdev->vq[i]))
>  				die("remapping queue %d during setmemtable", i);
> @@ -759,15 +760,18 @@ static void vu_set_watch(const struct vu_dev *vdev, int idx)
>  /**
>   * vu_check_queue_msg_file() - Check if a message is valid,
>   * 			       close fds if NOFD bit is set
> + * @vdev:	vhost-user device
>   * @vmsg:	vhost-user message
>   */
> -static void vu_check_queue_msg_file(struct vhost_user_msg *vmsg)
> +static void vu_check_queue_msg_file(const struct vu_dev *vdev,
> +				    struct vhost_user_msg *vmsg)
>  {
>  	bool nofd = vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK;
> -	int idx = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
> +	unsigned int idx = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
>  
> -	if (idx >= VHOST_USER_MAX_VQS)
> -		die("Invalid vhost-user queue index: %u", idx);
> +	if (idx >= vdev->context->max_qpairs * 2)
> +		die("Invalid vhost-user queue index: %u (maximum %u)", idx,
> +		    vdev->context->max_qpairs * 2);
>  
>  	if (nofd) {
>  		vmsg_close_fds(vmsg);
> @@ -794,7 +798,7 @@ static bool vu_set_vring_kick_exec(struct vu_dev *vdev,
>  
>  	debug("u64: 0x%016"PRIx64, vmsg->payload.u64);
>  
> -	vu_check_queue_msg_file(vmsg);
> +	vu_check_queue_msg_file(vdev, vmsg);
>  
>  	if (vdev->vq[idx].kick_fd != -1) {
>  		epoll_del(vdev->context->epollfd, vdev->vq[idx].kick_fd);
> @@ -834,7 +838,7 @@ static bool vu_set_vring_call_exec(struct vu_dev *vdev,
>  
>  	debug("u64: 0x%016"PRIx64, vmsg->payload.u64);
>  
> -	vu_check_queue_msg_file(vmsg);
> +	vu_check_queue_msg_file(vdev, vmsg);
>  
>  	if (vdev->vq[idx].call_fd != -1) {
>  		close(vdev->vq[idx].call_fd);
> @@ -869,7 +873,7 @@ static bool vu_set_vring_err_exec(struct vu_dev *vdev,
>  
>  	debug("u64: 0x%016"PRIx64, vmsg->payload.u64);
>  
> -	vu_check_queue_msg_file(vmsg);
> +	vu_check_queue_msg_file(vdev, vmsg);
>  
>  	if (vdev->vq[idx].err_fd != -1) {
>  		close(vdev->vq[idx].err_fd);
> @@ -896,7 +900,8 @@ static bool vu_get_protocol_features_exec(struct vu_dev *vdev,
>  	uint64_t features = 1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK |
>  			    1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD |
>  			    1ULL << VHOST_USER_PROTOCOL_F_DEVICE_STATE |
> -			    1ULL << VHOST_USER_PROTOCOL_F_RARP;
> +			    1ULL << VHOST_USER_PROTOCOL_F_RARP |
> +			    1ULL << VHOST_USER_PROTOCOL_F_MQ;
>  
>  	(void)vdev;
>  	vmsg_set_reply_u64(vmsg, features);
> @@ -935,10 +940,9 @@ static bool vu_get_queue_num_exec(struct vu_dev *vdev,
>  {
>  	(void)vdev;
>  
> -	/* NOLINTNEXTLINE(misc-redundant-expression) */
> -	vmsg_set_reply_u64(vmsg, VHOST_USER_MAX_VQS / 2);
> +	vmsg_set_reply_u64(vmsg, vdev->context->max_qpairs);
>  
> -	debug("VHOST_USER_MAX_VQS  %u", VHOST_USER_MAX_VQS / 2);
> +	debug("max_qpairs  %u", vdev->context->max_qpairs);
>  
>  	return true;
>  }
> @@ -959,7 +963,7 @@ static bool vu_set_vring_enable_exec(struct vu_dev *vdev,
>  	debug("State.index:  %u", idx);
>  	debug("State.enable: %u", enable);
>  
> -	if (idx >= VHOST_USER_MAX_VQS)
> +	if (idx >= vdev->context->max_qpairs * 2)
>  		die("Invalid vring_enable index: %u", idx);
>  
>  	vdev->vq[idx].enable = enable;
> @@ -1047,7 +1051,7 @@ static bool vu_check_device_state_exec(struct vu_dev *vdev,
>   */
>  void vu_init(struct ctx *c)
>  {
> -	int i;
> +	unsigned int i;
>  
>  	c->vdev = &vdev_storage;
>  	c->vdev->context = c;
> @@ -1074,7 +1078,7 @@ void vu_cleanup(struct vu_dev *vdev)
>  {
>  	unsigned int i;
>  
> -	for (i = 0; i < VHOST_USER_MAX_VQS; i++) {
> +	for (i = 0; i < vdev->context->max_qpairs * 2; i++) {
>  		struct vu_virtq *vq = &vdev->vq[i];
>  
>  		vq->started = false;
> diff --git a/virtio.h b/virtio.h
> index 12caaa0b6def..176c935cecc7 100644
> --- a/virtio.h
> +++ b/virtio.h
> @@ -88,7 +88,7 @@ struct vu_dev_region {
>  	uint64_t mmap_addr;
>  };
>  
> -#define VHOST_USER_MAX_VQS 2
> +#define VHOST_USER_MAX_VQS 32
>  
>  /*
>   * Set a reasonable maximum number of ram slots, which will be supported by
> -- 
> 2.51.0
> 

-- 
David Gibson (he or they)	| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you, not the other way
				| around.
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]