[PATCH 2/4] vhost-user: Add queue parameter throughout the network stack

[Laurent Vivier <lvivier@redhat.com>]

Add a queue parameter to vu_send_single() and propagate this parameter
through the entire network stack call chain. The queue parameter specifies
which virtqueue to use for sending packets in vhost-user mode.

Functions modified to accept and propagate the queue parameter:
- Core sending: tap_send_single(), vu_send_single()
- UDP/ICMP helpers: tap_udp4_send(), tap_udp6_send(), tap_icmp4_send(),
  tap_icmp6_send()
- Protocol handlers: arp(), dhcp(), dhcpv6(), ndp(), tcp_rst_no_conn()
- NDP helpers: ndp_send(), ndp_na(), ndp_ra()
- UDP error handling: udp_send_tap_icmp4(), udp_send_tap_icmp6()

All callers currently pass VHOST_USER_RX_QUEUE to preserve existing
behavior. This is a preparatory step for enabling multi-queue and
per-queue worker threads in vhost-user mode.

No functional change.

Signed-off-by: Laurent Vivier <lvivier@redhat.com>
---
 arp.c       | 12 +++++++-----
 arp.h       |  4 ++--
 dhcp.c      |  5 +++--
 dhcp.h      |  2 +-
 dhcpv6.c    | 12 +++++++-----
 dhcpv6.h    |  2 +-
 fwd.c       |  4 ++--
 icmp.c      |  6 ++++--
 ndp.c       | 32 +++++++++++++++++++-------------
 ndp.h       |  5 +++--
 tap.c       | 38 +++++++++++++++++++++++---------------
 tap.h       | 13 +++++++------
 tcp.c       | 15 +++++++++------
 tcp.h       | 11 ++++++-----
 udp.c       | 17 ++++++++++-------
 vu_common.c |  9 ++++++---
 vu_common.h |  3 ++-
 17 files changed, 112 insertions(+), 78 deletions(-)

diff --git a/arp.c b/arp.c
index 33b03cf6c316..c57e0471aee8 100644
--- a/arp.c
+++ b/arp.c
@@ -63,11 +63,12 @@ static bool ignore_arp(const struct ctx *c,
 /**
  * arp() - Check if this is a supported ARP message, reply as needed
  * @c:		Execution context
+ * @queue:	Queue to use to send the reply
  * @data:	Single packet with Ethernet buffer
  *
  * Return: 1 if handled, -1 on failure
  */
-int arp(const struct ctx *c, struct iov_tail *data)
+int arp(const struct ctx *c, int queue, struct iov_tail *data)
 {
 	union inany_addr tgt;
 	struct {
@@ -112,7 +113,7 @@ int arp(const struct ctx *c, struct iov_tail *data)
 	memcpy(resp.am.tha,		am->sha,	sizeof(resp.am.tha));
 	memcpy(resp.am.tip,		am->sip,	sizeof(resp.am.tip));
 
-	tap_send_single(c, &resp, sizeof(resp));
+	tap_send_single(c, queue, &resp, sizeof(resp));
 
 	return 1;
 }
@@ -148,16 +149,17 @@ void arp_send_init_req(const struct ctx *c)
 	memcpy(req.am.tip,	&c->ip4.addr,		sizeof(req.am.tip));
 
 	debug("Sending initial ARP request for guest MAC address");
-	tap_send_single(c, &req, sizeof(req));
+	tap_send_single(c, VHOST_USER_RX_QUEUE, &req, sizeof(req));
 }
 
 /**
  * arp_announce() - Send an ARP announcement for an IPv4 host
  * @c:		Execution context
+ * @queue:	Queue to use to send the announcement
  * @ip:	IPv4 address we announce as owned by @mac
  * @mac:	MAC address to advertise for @ip
  */
-void arp_announce(const struct ctx *c, struct in_addr *ip,
+void arp_announce(const struct ctx *c, int queue, struct in_addr *ip,
 		  const unsigned char *mac)
 {
 	char ip_str[INET_ADDRSTRLEN];
@@ -199,5 +201,5 @@ void arp_announce(const struct ctx *c, struct in_addr *ip,
 	eth_ntop(mac, mac_str, sizeof(mac_str));
 	debug("ARP announcement for %s / %s", ip_str, mac_str);
 
-	tap_send_single(c, &msg, sizeof(msg));
+	tap_send_single(c, queue, &msg, sizeof(msg));
 }
diff --git a/arp.h b/arp.h
index 4862e90a14ee..ba2b1253a924 100644
--- a/arp.h
+++ b/arp.h
@@ -20,9 +20,9 @@ struct arpmsg {
 	unsigned char tip[4];
 } __attribute__((__packed__));
 
-int arp(const struct ctx *c, struct iov_tail *data);
+int arp(const struct ctx *c, int queue, struct iov_tail *data);
 void arp_send_init_req(const struct ctx *c);
-void arp_announce(const struct ctx *c, struct in_addr *ip,
+void arp_announce(const struct ctx *c, int queue, struct in_addr *ip,
 		  const unsigned char *mac);
 
 #endif /* ARP_H */
diff --git a/dhcp.c b/dhcp.c
index 6b9c2e3b9e5a..3d84065db9cb 100644
--- a/dhcp.c
+++ b/dhcp.c
@@ -296,11 +296,12 @@ static void opt_set_dns_search(const struct ctx *c, size_t max_len)
 /**
  * dhcp() - Check if this is a DHCP message, reply as needed
  * @c:		Execution context
+ * @queue:	Queue to use to send the reply
  * @data:	Single packet with Ethernet buffer
  *
  * Return: 0 if it's not a DHCP message, 1 if handled, -1 on failure
  */
-int dhcp(const struct ctx *c, struct iov_tail *data)
+int dhcp(const struct ctx *c, int queue, struct iov_tail *data)
 {
 	char macstr[ETH_ADDRSTRLEN];
 	size_t mlen, dlen, opt_len;
@@ -471,7 +472,7 @@ int dhcp(const struct ctx *c, struct iov_tail *data)
 	else
 		dst = c->ip4.addr;
 
-	tap_udp4_send(c, c->ip4.our_tap_addr, 67, dst, 68, &reply, dlen);
+	tap_udp4_send(c, queue, c->ip4.our_tap_addr, 67, dst, 68, &reply, dlen);
 
 	return 1;
 }
diff --git a/dhcp.h b/dhcp.h
index cd50c99b8856..847d957ff7d1 100644
--- a/dhcp.h
+++ b/dhcp.h
@@ -6,7 +6,7 @@
 #ifndef DHCP_H
 #define DHCP_H
 
-int dhcp(const struct ctx *c, struct iov_tail *data);
+int dhcp(const struct ctx *c, int queue, struct iov_tail *data);
 void dhcp_init(void);
 
 #endif /* DHCP_H */
diff --git a/dhcpv6.c b/dhcpv6.c
index e4df0db562e6..f1fa1d047c86 100644
--- a/dhcpv6.c
+++ b/dhcpv6.c
@@ -369,12 +369,13 @@ notonlink:
 /**
  * dhcpv6_send_ia_notonlink() - Send NotOnLink status
  * @c:			Execution context
+ * @queue:		Queue to use to send the reply
  * @ia_base:		Non-appropriate IA_NA or IA_TA base
  * @client_id_base:	Client ID message option base
  * @len:		Client ID length
  * @xid:		Transaction ID for message exchange
  */
-static void dhcpv6_send_ia_notonlink(struct ctx *c,
+static void dhcpv6_send_ia_notonlink(struct ctx *c, int queue,
 				     const struct iov_tail *ia_base,
 				     const struct iov_tail *client_id_base,
 				     int len, uint32_t xid)
@@ -404,7 +405,7 @@ static void dhcpv6_send_ia_notonlink(struct ctx *c,
 
 	resp_not_on_link.hdr.xid = xid;
 
-	tap_udp6_send(c, src, 547, tap_ip6_daddr(c, src), 546,
+	tap_udp6_send(c, queue, src, 547, tap_ip6_daddr(c, src), 546,
 		      xid, &resp_not_on_link, n);
 }
 
@@ -539,13 +540,14 @@ static size_t dhcpv6_client_fqdn_fill(const struct iov_tail *data,
 /**
  * dhcpv6() - Check if this is a DHCPv6 message, reply as needed
  * @c:		Execution context
+ * @queue:	Queue to use to send the reply
  * @data:	Single packet starting from UDP header
  * @saddr:	Source IPv6 address of original message
  * @daddr:	Destination IPv6 address of original message
  *
  * Return: 0 if it's not a DHCPv6 message, 1 if handled, -1 on failure
  */
-int dhcpv6(struct ctx *c, struct iov_tail *data,
+int dhcpv6(struct ctx *c, int queue, struct iov_tail *data,
 	   const struct in6_addr *saddr, const struct in6_addr *daddr)
 {
 	const struct opt_server_id *server_id = NULL;
@@ -627,7 +629,7 @@ int dhcpv6(struct ctx *c, struct iov_tail *data,
 
 		if (dhcpv6_ia_notonlink(data, &c->ip6.addr)) {
 
-			dhcpv6_send_ia_notonlink(c, data, &client_id_base,
+			dhcpv6_send_ia_notonlink(c, queue, data, &client_id_base,
 						 ntohs(client_id->l), mh->xid);
 
 			return 1;
@@ -677,7 +679,7 @@ int dhcpv6(struct ctx *c, struct iov_tail *data,
 
 	resp.hdr.xid = mh->xid;
 
-	tap_udp6_send(c, src, 547, tap_ip6_daddr(c, src), 546,
+	tap_udp6_send(c, queue, src, 547, tap_ip6_daddr(c, src), 546,
 		      mh->xid, &resp, n);
 	c->ip6.addr_seen = c->ip6.addr;
 
diff --git a/dhcpv6.h b/dhcpv6.h
index c706dfdbb2ac..049f338fa133 100644
--- a/dhcpv6.h
+++ b/dhcpv6.h
@@ -6,7 +6,7 @@
 #ifndef DHCPV6_H
 #define DHCPV6_H
 
-int dhcpv6(struct ctx *c, struct iov_tail *data,
+int dhcpv6(struct ctx *c, int queue, struct iov_tail *data,
 	   struct in6_addr *saddr, struct in6_addr *daddr);
 void dhcpv6_init(const struct ctx *c);
 
diff --git a/fwd.c b/fwd.c
index 68bb11663c46..30667ab10204 100644
--- a/fwd.c
+++ b/fwd.c
@@ -147,9 +147,9 @@ void fwd_neigh_table_update(const struct ctx *c, const union inany_addr *addr,
 		return;
 
 	if (inany_v4(addr))
-		arp_announce(c, inany_v4(addr), e->mac);
+		arp_announce(c, VHOST_USER_RX_QUEUE, inany_v4(addr), e->mac);
 	else
-		ndp_unsolicited_na(c, &addr->a6);
+		ndp_unsolicited_na(c, VHOST_USER_RX_QUEUE, &addr->a6);
 }
 
 /**
diff --git a/icmp.c b/icmp.c
index 35faefb91870..d58499c3bf5c 100644
--- a/icmp.c
+++ b/icmp.c
@@ -132,12 +132,14 @@ void icmp_sock_handler(const struct ctx *c, union epoll_ref ref)
 		const struct in_addr *daddr = inany_v4(&ini->eaddr);
 
 		ASSERT(saddr && daddr); /* Must have IPv4 addresses */
-		tap_icmp4_send(c, *saddr, *daddr, buf, pingf->f.tap_omac, n);
+		tap_icmp4_send(c, VHOST_USER_RX_QUEUE, *saddr, *daddr, buf,
+			       pingf->f.tap_omac, n);
 	} else if (pingf->f.type == FLOW_PING6) {
 		const struct in6_addr *saddr = &ini->oaddr.a6;
 		const struct in6_addr *daddr = &ini->eaddr.a6;
 
-		tap_icmp6_send(c, saddr, daddr, buf, pingf->f.tap_omac, n);
+		tap_icmp6_send(c, VHOST_USER_RX_QUEUE, saddr, daddr, buf,
+			       pingf->f.tap_omac, n);
 	}
 	return;
 
diff --git a/ndp.c b/ndp.c
index a33239d4aa81..a5f1b946ae0b 100644
--- a/ndp.c
+++ b/ndp.c
@@ -175,25 +175,27 @@ struct ndp_ns {
 /**
  * ndp_send() - Send an NDP message
  * @c:		Execution context
+ * @queue:	Queue to use to send the message
  * @dst:	IPv6 address to send the message to
  * @buf:	ICMPv6 header + message payload
  * @l4len:	Length of message, including ICMPv6 header
  */
-static void ndp_send(const struct ctx *c, const struct in6_addr *dst,
+static void ndp_send(const struct ctx *c, int queue, const struct in6_addr *dst,
 		     const void *buf, size_t l4len)
 {
 	const struct in6_addr *src = &c->ip6.our_tap_ll;
 
-	tap_icmp6_send(c, src, dst, buf, c->our_tap_mac, l4len);
+	tap_icmp6_send(c, queue, src, dst, buf, c->our_tap_mac, l4len);
 }
 
 /**
  * ndp_na() - Send an NDP Neighbour Advertisement (NA) message
  * @c:		Execution context
+ * @queue:	Queue to use to send the NA
  * @dst:	IPv6 address to send the NA to
  * @addr:	IPv6 address to advertise
  */
-static void ndp_na(const struct ctx *c, const struct in6_addr *dst,
+static void ndp_na(const struct ctx *c, int queue, const struct in6_addr *dst,
 	    const struct in6_addr *addr)
 {
 	union inany_addr tgt;
@@ -217,25 +219,28 @@ static void ndp_na(const struct ctx *c, const struct in6_addr *dst,
 	inany_from_af(&tgt, AF_INET6, addr);
 	fwd_neigh_mac_get(c, &tgt, na.target_l2_addr.mac);
 
-	ndp_send(c, dst, &na, sizeof(na));
+	ndp_send(c, queue, dst, &na, sizeof(na));
 }
 
 /**
  * ndp_unsolicited_na() - Send unsolicited NA
  * @c:         Execution context
+ * @queue:	Queue to use to send the RA
  * @addr:      IPv6 address to advertise
  */
-void ndp_unsolicited_na(const struct ctx *c, const struct in6_addr *addr)
+void ndp_unsolicited_na(const struct ctx *c, int queue,
+			const struct in6_addr *addr)
 {
-	ndp_na(c, &in6addr_ll_all_nodes, addr);
+	ndp_na(c, queue, &in6addr_ll_all_nodes, addr);
 }
 
 /**
  * ndp_ra() - Send an NDP Router Advertisement (RA) message
  * @c:		Execution context
+ * @queue:	Queue to use to send the RA
  * @dst:	IPv6 address to send the RA to
  */
-static void ndp_ra(const struct ctx *c, const struct in6_addr *dst)
+static void ndp_ra(const struct ctx *c, int queue, const struct in6_addr *dst)
 {
 	struct ndp_ra ra = {
 		.ih = {
@@ -341,18 +346,19 @@ static void ndp_ra(const struct ctx *c, const struct in6_addr *dst)
 	memcpy(&ra.source_ll.mac, c->our_tap_mac, ETH_ALEN);
 
 	/* NOLINTNEXTLINE(clang-analyzer-security.PointerSub) */
-	ndp_send(c, dst, &ra, ptr - (unsigned char *)&ra);
+	ndp_send(c, queue, dst, &ra, ptr - (unsigned char *)&ra);
 }
 
 /**
  * ndp() - Check for NDP solicitations, reply as needed
  * @c:		Execution context
+ * @queue:	Queue to use to send replies
  * @saddr:	Source IPv6 address
  * @data:	Single packet with ICMPv6 header
  *
  * Return: 0 if not handled here, 1 if handled, -1 on failure
  */
-int ndp(const struct ctx *c, const struct in6_addr *saddr,
+int ndp(const struct ctx *c, int queue, const struct in6_addr *saddr,
 	struct iov_tail *data)
 {
 	struct icmp6hdr ih_storage;
@@ -381,13 +387,13 @@ int ndp(const struct ctx *c, const struct in6_addr *saddr,
 
 		info("NDP: received NS, sending NA");
 
-		ndp_na(c, saddr, &ns->target_addr);
+		ndp_na(c, queue, saddr, &ns->target_addr);
 	} else if (ih->icmp6_type == RS) {
 		if (c->no_ra)
 			return 1;
 
 		info("NDP: received RS, sending RA");
-		ndp_ra(c, saddr);
+		ndp_ra(c, queue, saddr);
 	}
 
 	return 1;
@@ -445,7 +451,7 @@ void ndp_timer(const struct ctx *c, const struct timespec *now)
 
 	info("NDP: sending unsolicited RA, next in %llds", (long long)interval);
 
-	ndp_ra(c, &in6addr_ll_all_nodes);
+	ndp_ra(c, VHOST_USER_RX_QUEUE, &in6addr_ll_all_nodes);
 
 first:
 	next_ra = now->tv_sec + interval;
@@ -468,5 +474,5 @@ void ndp_send_init_req(const struct ctx *c)
 		.target_addr = c->ip6.addr
 	};
 	debug("Sending initial NDP NS request for guest MAC address");
-	ndp_send(c, &c->ip6.addr, &ns, sizeof(ns));
+	ndp_send(c, VHOST_USER_RX_QUEUE, &c->ip6.addr, &ns, sizeof(ns));
 }
diff --git a/ndp.h b/ndp.h
index 56b756d8400b..1a9622ee17e7 100644
--- a/ndp.h
+++ b/ndp.h
@@ -8,10 +8,11 @@
 
 struct icmp6hdr;
 
-int ndp(const struct ctx *c, const struct in6_addr *saddr,
+int ndp(const struct ctx *c, int queue, const struct in6_addr *saddr,
 	struct iov_tail *data);
 void ndp_timer(const struct ctx *c, const struct timespec *now);
 void ndp_send_init_req(const struct ctx *c);
-void ndp_unsolicited_na(const struct ctx *c, const struct in6_addr *addr);
+void ndp_unsolicited_na(const struct ctx *c, int queue,
+			const struct in6_addr *addr);
 
 #endif /* NDP_H */
diff --git a/tap.c b/tap.c
index e18d693a665a..1308d49242e8 100644
--- a/tap.c
+++ b/tap.c
@@ -125,10 +125,12 @@ unsigned long tap_l2_max_len(const struct ctx *c)
 /**
  * tap_send_single() - Send a single frame
  * @c:		Execution context
+ * @queue:	Queue to use to send the frame
  * @data:	Packet buffer
  * @l2len:	Total L2 packet length
  */
-void tap_send_single(const struct ctx *c, const void *data, size_t l2len)
+void tap_send_single(const struct ctx *c, int queue, const void *data,
+		     size_t l2len)
 {
 	uint32_t vnet_len = htonl(l2len);
 	struct iovec iov[2];
@@ -147,7 +149,7 @@ void tap_send_single(const struct ctx *c, const void *data, size_t l2len)
 		tap_send_frames(c, iov, iovcnt, 1);
 		break;
 	case MODE_VU:
-		vu_send_single(c, data, l2len);
+		vu_send_single(c, queue, data, l2len);
 		break;
 	}
 }
@@ -250,6 +252,7 @@ void *tap_push_uh4(struct udphdr *uh, struct in_addr src, in_port_t sport,
 /**
  * tap_udp4_send() - Send UDP over IPv4 packet
  * @c:		Execution context
+ * @queue:	Queue to use to send packet
  * @src:	IPv4 source address
  * @sport:	UDP source port
  * @dst:	IPv4 destination address
@@ -257,7 +260,7 @@ void *tap_push_uh4(struct udphdr *uh, struct in_addr src, in_port_t sport,
  * @in:	UDP payload contents (not including UDP header)
  * @dlen:	UDP payload length (not including UDP header)
  */
-void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport,
+void tap_udp4_send(const struct ctx *c, int queue, struct in_addr src, in_port_t sport,
 		   struct in_addr dst, in_port_t dport,
 		   const void *in, size_t dlen)
 {
@@ -268,20 +271,22 @@ void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport,
 	char *data = tap_push_uh4(uh, src, sport, dst, dport, in, dlen);
 
 	memcpy(data, in, dlen);
-	tap_send_single(c, buf, dlen + (data - buf));
+	tap_send_single(c, queue, buf, dlen + (data - buf));
 }
 
 /**
  * tap_icmp4_send() - Send ICMPv4 packet
  * @c:		Execution context
+ * @queue:	Queue to use to send packet
  * @src:	IPv4 source address
  * @dst:	IPv4 destination address
  * @in:		ICMP packet, including ICMP header
  * @src_mac:	MAC address to be used as source for message
  * @l4len:	ICMP packet length, including ICMP header
  */
-void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst,
-		    const void *in, const void *src_mac, size_t l4len)
+void tap_icmp4_send(const struct ctx *c, int queue, struct in_addr src,
+		    struct in_addr dst, const void *in, const void *src_mac,
+		    size_t l4len)
 {
 	char buf[USHRT_MAX];
 	struct iphdr *ip4h = tap_push_l2h(c, buf, src_mac, ETH_P_IP);
@@ -291,7 +296,7 @@ void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst,
 	memcpy(icmp4h, in, l4len);
 	csum_icmp4(icmp4h, icmp4h + 1, l4len - sizeof(*icmp4h));
 
-	tap_send_single(c, buf, l4len + ((char *)icmp4h - buf));
+	tap_send_single(c, queue, buf, l4len + ((char *)icmp4h - buf));
 }
 
 /**
@@ -355,6 +360,7 @@ void *tap_push_uh6(struct udphdr *uh,
 /**
  * tap_udp6_send() - Send UDP over IPv6 packet
  * @c:		Execution context
+ * @queue:	Queue to use to send packet
  * @src:	IPv6 source address
  * @sport:	UDP source port
  * @dst:	IPv6 destination address
@@ -363,7 +369,7 @@ void *tap_push_uh6(struct udphdr *uh,
  * @in:	UDP payload contents (not including UDP header)
  * @dlen:	UDP payload length (not including UDP header)
  */
-void tap_udp6_send(const struct ctx *c,
+void tap_udp6_send(const struct ctx *c, int queue,
 		   const struct in6_addr *src, in_port_t sport,
 		   const struct in6_addr *dst, in_port_t dport,
 		   uint32_t flow, void *in, size_t dlen)
@@ -376,19 +382,20 @@ void tap_udp6_send(const struct ctx *c,
 	char *data = tap_push_uh6(uh, src, sport, dst, dport, in, dlen);
 
 	memcpy(data, in, dlen);
-	tap_send_single(c, buf, dlen + (data - buf));
+	tap_send_single(c, queue, buf, dlen + (data - buf));
 }
 
 /**
  * tap_icmp6_send() - Send ICMPv6 packet
  * @c:		Execution context
+ * @queue:	Queue to use to send packet
  * @src:	IPv6 source address
  * @dst:	IPv6 destination address
  * @in:		ICMP packet, including ICMP header
  * @src_mac:	MAC address to be used as source for message
  * @l4len:	ICMP packet length, including ICMP header
  */
-void tap_icmp6_send(const struct ctx *c,
+void tap_icmp6_send(const struct ctx *c, int queue,
 		    const struct in6_addr *src, const struct in6_addr *dst,
 		    const void *in, const void *src_mac, size_t l4len)
 {
@@ -400,7 +407,7 @@ void tap_icmp6_send(const struct ctx *c,
 	memcpy(icmp6h, in, l4len);
 	csum_icmp6(icmp6h, src, dst, icmp6h + 1, l4len - sizeof(*icmp6h));
 
-	tap_send_single(c, buf, l4len + ((char *)icmp6h - buf));
+	tap_send_single(c, queue, buf, l4len + ((char *)icmp6h - buf));
 }
 
 /**
@@ -729,7 +736,7 @@ resume:
 		if (!eh)
 			continue;
 		if (ntohs(eh->h_proto) == ETH_P_ARP) {
-			arp(c, &data);
+			arp(c, VHOST_USER_RX_QUEUE, &data);
 			continue;
 		}
 
@@ -790,7 +797,7 @@ resume:
 			struct iov_tail eh_data;
 
 			packet_get(in, i, &eh_data);
-			if (dhcp(c, &eh_data))
+			if (dhcp(c, VHOST_USER_RX_QUEUE, &eh_data))
 				continue;
 		}
 
@@ -958,7 +965,7 @@ resume:
 				continue;
 
 			ndp_data = data;
-			if (ndp(c, saddr, &ndp_data))
+			if (ndp(c, VHOST_USER_RX_QUEUE, saddr, &ndp_data))
 				continue;
 
 			tap_packet_debug(NULL, ip6h, NULL, proto, NULL, 1);
@@ -977,7 +984,8 @@ resume:
 		if (proto == IPPROTO_UDP) {
 			struct iov_tail uh_data = data;
 
-			if (dhcpv6(c, &uh_data, saddr, daddr))
+			if (dhcpv6(c, VHOST_USER_RX_QUEUE, &uh_data, saddr,
+				   daddr))
 				continue;
 		}
 
diff --git a/tap.h b/tap.h
index 1864173cc9b0..76403a43edbc 100644
--- a/tap.h
+++ b/tap.h
@@ -87,24 +87,25 @@ void *tap_push_ip6h(struct ipv6hdr *ip6h,
 		    const struct in6_addr *src,
 		    const struct in6_addr *dst,
 		    size_t l4len, uint8_t proto, uint32_t flow);
-void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport,
+void tap_udp4_send(const struct ctx *c, int queue, struct in_addr src, in_port_t sport,
 		   struct in_addr dst, in_port_t dport,
 		   const void *in, size_t dlen);
-void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst,
-		    const void *in, const void *src_mac, size_t l4len);
+void tap_icmp4_send(const struct ctx *c, int queue, struct in_addr src,
+		    struct in_addr dst, const void *in, const void *src_mac,
+		    size_t l4len);
 const struct in6_addr *tap_ip6_daddr(const struct ctx *c,
 				     const struct in6_addr *src);
 void *tap_push_ip6h(struct ipv6hdr *ip6h,
 		    const struct in6_addr *src, const struct in6_addr *dst,
 		    size_t l4len, uint8_t proto, uint32_t flow);
-void tap_udp6_send(const struct ctx *c,
+void tap_udp6_send(const struct ctx *c, int queue,
 		   const struct in6_addr *src, in_port_t sport,
 		   const struct in6_addr *dst, in_port_t dport,
 		   uint32_t flow, void *in, size_t dlen);
-void tap_icmp6_send(const struct ctx *c,
+void tap_icmp6_send(const struct ctx *c, int queue,
 		    const struct in6_addr *src, const struct in6_addr *dst,
 		    const void *in, const void *src_mac, size_t l4len);
-void tap_send_single(const struct ctx *c, const void *data, size_t l2len);
+void tap_send_single(const struct ctx *c, int queue, const void *data, size_t l2len);
 size_t tap_send_frames(const struct ctx *c, const struct iovec *iov,
 		       size_t bufs_per_frame, size_t nframes);
 void eth_update_mac(struct ethhdr *eh,
diff --git a/tcp.c b/tcp.c
index e91c0cf5a441..5ce34baa8a5a 100644
--- a/tcp.c
+++ b/tcp.c
@@ -1984,6 +1984,7 @@ static void tcp_conn_from_sock_finish(const struct ctx *c,
 /**
  * tcp_rst_no_conn() - Send RST in response to a packet with no connection
  * @c:		Execution context
+ * @queue:	Queue to use to send the reply
  * @af:		Address family, AF_INET or AF_INET6
  * @saddr:	Source address of the packet we're responding to
  * @daddr:	Destination address of the packet we're responding to
@@ -1991,7 +1992,7 @@ static void tcp_conn_from_sock_finish(const struct ctx *c,
  * @th:		TCP header of the packet we're responding to
  * @l4len:	Packet length, including TCP header
  */
-static void tcp_rst_no_conn(const struct ctx *c, int af,
+static void tcp_rst_no_conn(const struct ctx *c, int queue, int af,
 			    const void *saddr, const void *daddr,
 			    uint32_t flow_lbl,
 			    const struct tcphdr *th, size_t l4len)
@@ -2049,7 +2050,7 @@ static void tcp_rst_no_conn(const struct ctx *c, int af,
 
 	tcp_update_csum(psum, rsth, &payload);
 	rst_l2len = ((char *)rsth - buf) + sizeof(*rsth);
-	tap_send_single(c, buf, rst_l2len);
+	tap_send_single(c, queue, buf, rst_l2len);
 }
 
 /**
@@ -2066,9 +2067,10 @@ static void tcp_rst_no_conn(const struct ctx *c, int af,
  *
  * Return: count of consumed packets
  */
-int tcp_tap_handler(const struct ctx *c, uint8_t pif, sa_family_t af,
-		    const void *saddr, const void *daddr, uint32_t flow_lbl,
-		    const struct pool *p, int idx, const struct timespec *now)
+int tcp_tap_handler(const struct ctx *c, uint8_t pif,
+		    sa_family_t af, const void *saddr, const void *daddr,
+		    uint32_t flow_lbl, const struct pool *p, int idx,
+		    const struct timespec *now)
 {
 	struct tcp_tap_conn *conn;
 	struct tcphdr th_storage;
@@ -2108,7 +2110,8 @@ int tcp_tap_handler(const struct ctx *c, uint8_t pif, sa_family_t af,
 			tcp_conn_from_tap(c, af, saddr, daddr, th,
 					  opts, optlen, now);
 		else
-			tcp_rst_no_conn(c, af, saddr, daddr, flow_lbl, th, l4len);
+			tcp_rst_no_conn(c, VHOST_USER_RX_QUEUE, af, saddr,
+					daddr, flow_lbl, th, l4len);
 		return 1;
 	}
 
diff --git a/tcp.h b/tcp.h
index 0082386725c2..320683ce5679 100644
--- a/tcp.h
+++ b/tcp.h
@@ -13,11 +13,12 @@ struct ctx;
 void tcp_timer_handler(const struct ctx *c, union epoll_ref ref);
 void tcp_listen_handler(const struct ctx *c, union epoll_ref ref,
 			const struct timespec *now);
-void tcp_sock_handler(const struct ctx *c, union epoll_ref ref,
-		      uint32_t events);
-int tcp_tap_handler(const struct ctx *c, uint8_t pif, sa_family_t af,
-		    const void *saddr, const void *daddr, uint32_t flow_lbl,
-		    const struct pool *p, int idx, const struct timespec *now);
+void tcp_sock_handler(const struct ctx *c,
+		      union epoll_ref ref, uint32_t events);
+int tcp_tap_handler(const struct ctx *c, uint8_t pif,
+		    sa_family_t af, const void *saddr, const void *daddr,
+		    uint32_t flow_lbl, const struct pool *p, int idx,
+		    const struct timespec *now);
 int tcp_sock_init(const struct ctx *c, const union inany_addr *addr,
 		  const char *ifname, in_port_t port);
 int tcp_init(struct ctx *c);
diff --git a/udp.c b/udp.c
index 9c00950250a0..868ffebb5802 100644
--- a/udp.c
+++ b/udp.c
@@ -384,13 +384,14 @@ static void udp_tap_prepare(const struct mmsghdr *mmh,
 /**
  * udp_send_tap_icmp4() - Construct and send ICMPv4 to local peer
  * @c:		Execution context
+ * @queue:	Queue to send the ICMPv4 packet to
  * @ee:	Extended error descriptor
  * @toside:	Destination side of flow
  * @saddr:	Address of ICMP generating node
  * @in:	First bytes (max 8) of original UDP message body
  * @dlen:	Length of the read part of original UDP message body
  */
-static void udp_send_tap_icmp4(const struct ctx *c,
+static void udp_send_tap_icmp4(const struct ctx *c, int queue,
 			       const struct sock_extended_err *ee,
 			       const struct flowside *toside,
 			       struct in_addr saddr,
@@ -426,13 +427,14 @@ static void udp_send_tap_icmp4(const struct ctx *c,
 	/* Try to obtain the MAC address of the generating node */
 	saddr_any = inany_from_v4(saddr);
 	fwd_neigh_mac_get(c, &saddr_any, tap_omac);
-	tap_icmp4_send(c, saddr, eaddr, &msg, tap_omac, msglen);
+	tap_icmp4_send(c, queue, saddr, eaddr, &msg, tap_omac, msglen);
 }
 
 
 /**
  * udp_send_tap_icmp6() - Construct and send ICMPv6 to local peer
  * @c:		Execution context
+ * @queue:	Queue to send the ICMPv6 packet to
  * @ee:	Extended error descriptor
  * @toside:	Destination side of flow
  * @saddr:	Address of ICMP generating node
@@ -440,7 +442,7 @@ static void udp_send_tap_icmp4(const struct ctx *c,
  * @dlen:	Length of the read part of original UDP message body
  * @flow:	IPv6 flow identifier
  */
-static void udp_send_tap_icmp6(const struct ctx *c,
+static void udp_send_tap_icmp6(const struct ctx *c, int queue,
 			       const struct sock_extended_err *ee,
 			       const struct flowside *toside,
 			       const struct in6_addr *saddr,
@@ -474,7 +476,7 @@ static void udp_send_tap_icmp6(const struct ctx *c,
 
 	/* Try to obtain the MAC address of the generating node */
 	fwd_neigh_mac_get(c, (union inany_addr *) saddr, tap_omac);
-	tap_icmp6_send(c, saddr, eaddr, &msg, tap_omac, msglen);
+	tap_icmp6_send(c, queue, saddr, eaddr, &msg, tap_omac, msglen);
 }
 
 /**
@@ -634,13 +636,14 @@ static int udp_sock_recverr(const struct ctx *c, int s, flow_sidx_t sidx,
 	if (hdr->cmsg_level == IPPROTO_IP &&
 	    (o4 = inany_v4(&otap)) && inany_v4(&toside->eaddr)) {
 		dlen = MIN(dlen, ICMP4_MAX_DLEN);
-		udp_send_tap_icmp4(c, ee, toside, *o4, data, dlen);
+		udp_send_tap_icmp4(c, VHOST_USER_RX_QUEUE, ee, toside, *o4,
+				   data, dlen);
 		return 1;
 	}
 
 	if (hdr->cmsg_level == IPPROTO_IPV6 && !inany_v4(&toside->eaddr)) {
-		udp_send_tap_icmp6(c, ee, toside, &otap.a6, data, dlen,
-				   FLOW_IDX(uflow));
+		udp_send_tap_icmp6(c, VHOST_USER_RX_QUEUE, ee, toside,
+				   &otap.a6, data, dlen, FLOW_IDX(uflow));
 		return 1;
 	}
 
diff --git a/vu_common.c b/vu_common.c
index b13b7c308fd8..8904403e66af 100644
--- a/vu_common.c
+++ b/vu_common.c
@@ -235,23 +235,26 @@ void vu_kick_cb(struct vu_dev *vdev, union epoll_ref ref,
 }
 
 /**
- * vu_send_single() - Send a buffer to the front-end using the RX virtqueue
+ * vu_send_single() - Send a buffer to the front-end using a specified virtqueue
  * @c:		execution context
+ * @queue:	queue to use to send the buffer
  * @buf:	address of the buffer
  * @size:	size of the buffer
  *
  * Return: number of bytes sent, -1 if there is an error
  */
-int vu_send_single(const struct ctx *c, const void *buf, size_t size)
+int vu_send_single(const struct ctx *c, int queue, const void *buf, size_t size)
 {
 	struct vu_dev *vdev = c->vdev;
-	struct vu_virtq *vq = &vdev->vq[VHOST_USER_RX_QUEUE];
 	struct vu_virtq_element elem[VIRTQUEUE_MAX_SIZE];
 	struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
+	struct vu_virtq *vq;
 	size_t total;
 	int elem_cnt;
 	int i;
 
+	vq = &vdev->vq[queue];
+
 	trace("vu_send_single size %zu", size);
 
 	if (!vu_queue_enabled(vq) || !vu_queue_started(vq)) {
diff --git a/vu_common.h b/vu_common.h
index f538f237790b..2637c67f913c 100644
--- a/vu_common.h
+++ b/vu_common.h
@@ -56,6 +56,7 @@ void vu_flush(const struct vu_dev *vdev, struct vu_virtq *vq,
 	      struct vu_virtq_element *elem, int elem_cnt);
 void vu_kick_cb(struct vu_dev *vdev, union epoll_ref ref,
 		const struct timespec *now);
-int vu_send_single(const struct ctx *c, const void *buf, size_t size);
+int vu_send_single(const struct ctx *c, int queue, const void *buf,
+		   size_t size);
 
 #endif /* VU_COMMON_H */
-- 
2.51.0