[PATCH v3 0/8] use true MAC address of LAN local remote hosts

[PATCH v3 0/8] use true MAC address of LAN local remote hosts

[Jon Maloy]

Bug #120 asks us to use the true MAC addresses of LAN local
remote hosts, since some programs need this information.
These commits introduces this for ARP, NDP, UDP, TCP and
ICMP.

---
v3: Updated according to feedback from Stefano and David:
    - Made the ARP/NDP lookup call filter out the requested address
      by itself, qualified by the index if the template interface
    - Moved the flow specific MAC address from struct flowside to
      struct flow_common.

Jon Maloy (8):
  netlink: Add function to extract MAC addresses from NDP/ARP table
  arp/ndp: respond with true MAC address of LAN local remote hosts
  flow: add MAC address of LAN local remote hosts to flow
  udp: forward external source MAC address through tap interface
  tcp: forward external source MAC address through tap interface
  tap: change signature of function tap_push_l2h()
  tcp: make tcp_rst_no_conn() respond with correct MAC address
  icmp: let icmp use mac address from flowside structure

 arp.c          |  9 ++++++++
 flow.c         | 21 ++++++++++++++++-
 flow.h         |  2 ++
 fwd.c          |  2 +-
 fwd.h          |  3 ++-
 icmp.c         |  4 ++--
 inany.c        | 15 ++++++++++++
 inany.h        |  1 +
 ndp.c          | 11 ++++++++-
 netlink.c      | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++
 netlink.h      |  2 ++
 tap.c          | 24 +++++++++++--------
 tap.h          |  7 +++---
 tcp.c          | 22 +++++++++++++++---
 tcp_buf.c      | 27 +++++++++++-----------
 tcp_internal.h |  2 +-
 tcp_vu.c       |  5 ++--
 udp.c          | 49 +++++++++++++++++++++------------------
 18 files changed, 207 insertions(+), 61 deletions(-)

-- 
2.48.1

[PATCH v3 1/8] netlink: add function to extract MAC addresses from NDP/ARP table

[Jon Maloy]

The solution to bug https://bugs.passt.top/show_bug.cgi?id=120
requires the ability to translate from an IP address to its
corresponding MAC address in cases where those are present in
the ARP/NDP table.

We add this feature here.

Signed-off-by: Jon Maloy <jmaloy@redhat.com>

---
v3: - Added an attribute contianing NDA_DST to sent message, so
      that we let the kernel do the filtering of the IP address
      and return only one entry.
    - Added interface index to the call signature. Since the only
      interface we know is the template interface, this limits
      the number of hosts that will be seen as 'network segment
      local' from a PASST viewpoint.
---
 netlink.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 netlink.h |  2 ++
 2 files changed, 64 insertions(+)

diff --git a/netlink.c b/netlink.c
index ee9325a..16bb995 100644
--- a/netlink.c
+++ b/netlink.c
@@ -800,6 +800,68 @@ int nl_addr_get(int s, unsigned int ifi, sa_family_t af,
 	return status;
 }
 
+/**
+ * nl_mac_get() - Get MAC address corresponding to given IP address
+ * @s:		Netlink socket
+ * @addr:	IPv4 or IPv6 address
+ * @ifi:	Interface index
+ * @mac:	Array to place the returned MAC address
+ *
+ * Return: 0 if found or not in table, negative error code on failure.
+ *         Leaves MAC array unchanged if no match found
+ */
+int nl_mac_get(int s, const union inany_addr *addr, int ifi, unsigned char *mac)
+{
+	struct req_t {
+		struct nlmsghdr nlh;
+		struct ndmsg ndm;
+		struct rtattr rta;
+		char ip[RTA_ALIGN(sizeof(struct in6_addr))];
+	} req = {
+		.ndm.ndm_ifindex = ifi,
+		.rta.rta_type = NDA_DST
+	};
+	struct nlmsghdr *nh;
+	char buf[NLBUFSIZ];
+	const void *ip;
+	ssize_t status;
+	uint32_t seq;
+	int msglen;
+	int iplen;
+
+	if (inany_v4(addr)) {
+		ip = &addr->v4mapped.a4;
+		iplen = sizeof(struct in_addr);
+		req.ndm.ndm_family = AF_INET;
+	} else {
+		ip = &addr->a6;
+		iplen = sizeof(struct in6_addr);
+		req.ndm.ndm_family = AF_INET6;
+	}
+	req.rta.rta_len = RTA_LENGTH(iplen);
+	memcpy(RTA_DATA(&req.rta), ip, iplen);
+	msglen = NLMSG_ALIGN(sizeof(req.nlh) + sizeof(req.ndm) + RTA_LENGTH(iplen));
+
+	seq = nl_send(s, &req, RTM_GETNEIGH, 0, msglen);
+	nl_foreach_oftype(nh, status, s, buf, seq, RTM_NEWNEIGH) {
+		struct ndmsg *ndm = NLMSG_DATA(nh);
+		struct rtattr *rta = (struct rtattr *)(ndm + 1);
+		size_t na = RTM_PAYLOAD(nh);
+		bool found = false;
+
+		for (; RTA_OK(rta, na); rta = RTA_NEXT(rta, na)) {
+			if (rta->rta_type == NDA_DST) {
+				if (memcmp(RTA_DATA(rta), ip, iplen) == 0)
+					found = true;
+			} else if (rta->rta_type == NDA_LLADDR && found) {
+				memcpy(mac, RTA_DATA(rta), ETH_ALEN);
+			}
+		}
+	}
+
+	return status;
+}
+
 /**
  * nl_addr_get_ll() - Get first IPv6 link-local address for a given interface
  * @s:		Netlink socket
diff --git a/netlink.h b/netlink.h
index b51e99c..51ba49f 100644
--- a/netlink.h
+++ b/netlink.h
@@ -17,6 +17,8 @@ int nl_route_dup(int s_src, unsigned int ifi_src,
 		 int s_dst, unsigned int ifi_dst, sa_family_t af);
 int nl_addr_get(int s, unsigned int ifi, sa_family_t af,
 		void *addr, int *prefix_len, void *addr_l);
+int nl_mac_get(int s, const union inany_addr *addr, int ifi,
+	       unsigned char *mac);
 int nl_addr_set(int s, unsigned int ifi, sa_family_t af,
 		const void *addr, int prefix_len);
 int nl_addr_get_ll(int s, unsigned int ifi, struct in6_addr *addr);
-- 
@@ -17,6 +17,8 @@ int nl_route_dup(int s_src, unsigned int ifi_src,
 		 int s_dst, unsigned int ifi_dst, sa_family_t af);
 int nl_addr_get(int s, unsigned int ifi, sa_family_t af,
 		void *addr, int *prefix_len, void *addr_l);
+int nl_mac_get(int s, const union inany_addr *addr, int ifi,
+	       unsigned char *mac);
 int nl_addr_set(int s, unsigned int ifi, sa_family_t af,
 		const void *addr, int prefix_len);
 int nl_addr_get_ll(int s, unsigned int ifi, struct in6_addr *addr);
-- 
2.48.1

[PATCH v3 2/8] arp/ndp: respond with true MAC address of LAN local remote hosts

[Jon Maloy]

When we receive an ARP request or NDP neigbor solicitation over
the tap interface for a host on the local network segment attached
to the template interface, we respond with that host's real MAC
address.

The local host, which is acting as a proxy for the default gateway,
is still exempted from this rule.

Signed-off-by: Jon Maloy <jmaloy@redhat.com>

---
v3: - Added helper function to find out if a remote ip address is subject
      to NAT. This filters out local host addresses which should be
      presented with the passt/pasta local MAC address 9a:55:9a:55:9a:55 even
      though it is on the local segment.
    - Adapted to the change in nl_mac_get() function, so that we now consider
      only the template interface when checking the ARP/NDP table.
---
 arp.c   |  9 +++++++++
 fwd.c   |  2 +-
 fwd.h   |  3 ++-
 inany.c | 15 +++++++++++++++
 inany.h |  1 +
 ndp.c   |  9 +++++++++
 6 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/arp.c b/arp.c
index fc482bb..1952a63 100644
--- a/arp.c
+++ b/arp.c
@@ -29,6 +29,7 @@
 #include "dhcp.h"
 #include "passt.h"
 #include "tap.h"
+#include "netlink.h"
 
 /**
  * arp() - Check if this is a supported ARP message, reply as needed
@@ -40,6 +41,7 @@
 int arp(const struct ctx *c, const struct pool *p)
 {
 	unsigned char swap[4];
+	union inany_addr tgt;
 	struct ethhdr *eh;
 	struct arphdr *ah;
 	struct arpmsg *am;
@@ -72,6 +74,13 @@ int arp(const struct ctx *c, const struct pool *p)
 	memcpy(am->tha,		am->sha,	sizeof(am->tha));
 	memcpy(am->sha,		c->our_tap_mac,	sizeof(am->sha));
 
+	/* Respond with true MAC address if remote host is on
+	 * the template interface's network segment
+	 */
+	inany_from_af(&tgt, AF_INET, am->tip);
+	if (!inany_nat(c, &tgt))
+		nl_mac_get(nl_sock, &tgt, c->ifi4, am->sha);
+
 	memcpy(swap,		am->tip,	sizeof(am->tip));
 	memcpy(am->tip,		am->sip,	sizeof(am->tip));
 	memcpy(am->sip,		swap,		sizeof(am->sip));
diff --git a/fwd.c b/fwd.c
index 250cf56..02ebc9d 100644
--- a/fwd.c
+++ b/fwd.c
@@ -332,7 +332,7 @@ static bool fwd_guest_accessible(const struct ctx *c,
  * Only handles translations that depend *only* on the address.  Anything
  * related to specific ports or flows is handled elsewhere.
  */
-static void nat_outbound(const struct ctx *c, const union inany_addr *addr,
+void nat_outbound(const struct ctx *c, const union inany_addr *addr,
 			 union inany_addr *translated)
 {
 	if (inany_equals4(addr, &c->ip4.map_host_loopback))
diff --git a/fwd.h b/fwd.h
index 0458a3c..61632f2 100644
--- a/fwd.h
+++ b/fwd.h
@@ -56,5 +56,6 @@ uint8_t fwd_nat_from_splice(const struct ctx *c, uint8_t proto,
 			    const struct flowside *ini, struct flowside *tgt);
 uint8_t fwd_nat_from_host(const struct ctx *c, uint8_t proto,
 			  const struct flowside *ini, struct flowside *tgt);
-
+void nat_outbound(const struct ctx *c, const union inany_addr *addr,
+		  union inany_addr *translated);
 #endif /* FWD_H */
diff --git a/inany.c b/inany.c
index f5483bf..0ecf10a 100644
--- a/inany.c
+++ b/inany.c
@@ -16,6 +16,7 @@
 #include "ip.h"
 #include "siphash.h"
 #include "inany.h"
+#include "fwd.h"
 
 const union inany_addr inany_loopback4 = INANY_INIT4(IN4ADDR_LOOPBACK_INIT);
 const union inany_addr inany_any4 = INANY_INIT4(IN4ADDR_ANY_INIT);
@@ -56,3 +57,17 @@ int inany_pton(const char *src, union inany_addr *dst)
 
 	return 0;
 }
+
+/** inany_nat - Find if a remote IPv[46] address is subject to NAT
+ * @c:		Execution context
+ * @addr:	IPv[46] address
+ *
+ * Return: true if translated, false otherwise
+ */
+bool inany_nat(const struct ctx *c, const union inany_addr *addr)
+{
+	union inany_addr addr_nat;
+
+	nat_outbound(c, addr, &addr_nat);
+	return !inany_equals(addr, &addr_nat);
+}
diff --git a/inany.h b/inany.h
index 7ca5cbd..b2a66b0 100644
--- a/inany.h
+++ b/inany.h
@@ -278,5 +278,6 @@ static inline void inany_siphash_feed(struct siphash_state *state,
 
 const char *inany_ntop(const union inany_addr *src, char *dst, socklen_t size);
 int inany_pton(const char *src, union inany_addr *dst);
+bool inany_nat(const struct ctx *c, const union inany_addr *addr);
 
 #endif /* INANY_H */
diff --git a/ndp.c b/ndp.c
index 3e15494..2e7f0bc 100644
--- a/ndp.c
+++ b/ndp.c
@@ -32,6 +32,7 @@
 #include "passt.h"
 #include "tap.h"
 #include "log.h"
+#include "netlink.h"
 
 #define	RT_LIFETIME	65535
 
@@ -196,6 +197,7 @@ static void ndp_send(const struct ctx *c, const struct in6_addr *dst,
 static void ndp_na(const struct ctx *c, const struct in6_addr *dst,
 	    const struct in6_addr *addr)
 {
+	union inany_addr tgt;
 	struct ndp_na na = {
 		.ih = {
 			.icmp6_type		= NA,
@@ -215,6 +217,13 @@ static void ndp_na(const struct ctx *c, const struct in6_addr *dst,
 
 	memcpy(na.target_l2_addr.mac, c->our_tap_mac, ETH_ALEN);
 
+	/* Respond with true link-layer address if remote host is on
+	 * the template interface's network segment
+	 */
+	inany_from_af(&tgt, AF_INET6, addr);
+	if (!inany_nat(c, &tgt))
+		nl_mac_get(nl_sock, &tgt, c->ifi6, na.target_l2_addr.mac);
+
 	ndp_send(c, dst, &na, sizeof(na));
 }
 
-- 
@@ -32,6 +32,7 @@
 #include "passt.h"
 #include "tap.h"
 #include "log.h"
+#include "netlink.h"
 
 #define	RT_LIFETIME	65535
 
@@ -196,6 +197,7 @@ static void ndp_send(const struct ctx *c, const struct in6_addr *dst,
 static void ndp_na(const struct ctx *c, const struct in6_addr *dst,
 	    const struct in6_addr *addr)
 {
+	union inany_addr tgt;
 	struct ndp_na na = {
 		.ih = {
 			.icmp6_type		= NA,
@@ -215,6 +217,13 @@ static void ndp_na(const struct ctx *c, const struct in6_addr *dst,
 
 	memcpy(na.target_l2_addr.mac, c->our_tap_mac, ETH_ALEN);
 
+	/* Respond with true link-layer address if remote host is on
+	 * the template interface's network segment
+	 */
+	inany_from_af(&tgt, AF_INET6, addr);
+	if (!inany_nat(c, &tgt))
+		nl_mac_get(nl_sock, &tgt, c->ifi6, na.target_l2_addr.mac);
+
 	ndp_send(c, dst, &na, sizeof(na));
 }
 
-- 
2.48.1

[PATCH v3 3/8] flow: add MAC address of LAN local remote hosts to flow

[Jon Maloy]

When communicating with remote hosts on the local network, some guest
applications want to see the real MAC address of that host instead
of PASST/PASTA's own tap address. The flow_common structure is a
convenient location for storing that address, so we do that in this
commit.

Note that we don´t add actual usage of this address here, that will
be done in later commits.

Signed-off-by: Jon Maloy <jmaloy@redhat.com>

---
v3: - Moved the remote host macaddress from struct flowside to
      struct flow_common. I chose to call it 'omac' as suggested
      by David, although in my understanding the correct name would be
      'emac'. (In general I find the address naming scheme confusing.)
    - Adapted to new signature of function nl_mac_get(), now passing
      it the index of the template interface.
---
 flow.c | 21 ++++++++++++++++++++-
 flow.h |  2 ++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/flow.c b/flow.c
index da5c813..dcda1a7 100644
--- a/flow.c
+++ b/flow.c
@@ -20,6 +20,7 @@
 #include "flow.h"
 #include "flow_table.h"
 #include "repair.h"
+#include "netlink.h"
 
 const char *flow_state_str[] = {
 	[FLOW_STATE_FREE]	= "FREE",
@@ -438,18 +439,28 @@ struct flowside *flow_target(const struct ctx *c, union flow *flow,
 {
 	char estr[INANY_ADDRSTRLEN], fstr[INANY_ADDRSTRLEN];
 	struct flow_common *f = &flow->f;
-	const struct flowside *ini = &f->side[INISIDE];
+	struct flowside *ini = &f->side[INISIDE];
 	struct flowside *tgt = &f->side[TGTSIDE];
 	uint8_t tgtpif = PIF_NONE;
+	int ifi;
 
 	ASSERT(flow_new_entry == flow && f->state == FLOW_STATE_INI);
 	ASSERT(f->type == FLOW_TYPE_NONE);
 	ASSERT(f->pif[INISIDE] != PIF_NONE && f->pif[TGTSIDE] == PIF_NONE);
 	ASSERT(flow->f.state == FLOW_STATE_INI);
+	memcpy(f->omac, c->our_tap_mac, ETH_ALEN);
 
 	switch (f->pif[INISIDE]) {
 	case PIF_TAP:
 		tgtpif = fwd_nat_from_tap(c, proto, ini, tgt);
+
+		/* If there was no NAT, chances are this is a remote host
+		 * on the template interface's local network segment.
+		 * If so, insert its MAC address
+		 */
+		ifi = inany_v4(&ini->oaddr) ? c->ifi4 : c->ifi6;
+		if (inany_equals(&ini->oaddr, &tgt->eaddr))
+			nl_mac_get(nl_sock, &ini->oaddr, ifi, f->omac);
 		break;
 
 	case PIF_SPLICE:
@@ -458,6 +469,14 @@ struct flowside *flow_target(const struct ctx *c, union flow *flow,
 
 	case PIF_HOST:
 		tgtpif = fwd_nat_from_host(c, proto, ini, tgt);
+
+		/* If there was no NAT, chances are this is a remote host
+		 * on the template interface's local network segment.
+		 * If so, insert its MAC address
+		 */
+		ifi = inany_v4(&ini->eaddr) ? c->ifi4 : c->ifi6;
+		if (inany_equals(&ini->eaddr, &tgt->oaddr))
+			nl_mac_get(nl_sock, &ini->eaddr, ifi, f->omac);
 		break;
 
 	default:
diff --git a/flow.h b/flow.h
index cac618a..3240fb7 100644
--- a/flow.h
+++ b/flow.h
@@ -177,6 +177,7 @@ int flowside_connect(const struct ctx *c, int s,
  * @type:	Type of packet flow
  * @pif[]:	Interface for each side of the flow
  * @side[]:	Information for each side of the flow
+ * @omac:       MAC address of remote endpoint as seen from the guest
  */
 struct flow_common {
 #ifdef __GNUC__
@@ -192,6 +193,7 @@ struct flow_common {
 #endif
 	uint8_t		pif[SIDES];
 	struct flowside	side[SIDES];
+	unsigned char		omac[6];
 };
 
 #define FLOW_INDEX_BITS		17	/* 128k - 1 */
-- 
@@ -177,6 +177,7 @@ int flowside_connect(const struct ctx *c, int s,
  * @type:	Type of packet flow
  * @pif[]:	Interface for each side of the flow
  * @side[]:	Information for each side of the flow
+ * @omac:       MAC address of remote endpoint as seen from the guest
  */
 struct flow_common {
 #ifdef __GNUC__
@@ -192,6 +193,7 @@ struct flow_common {
 #endif
 	uint8_t		pif[SIDES];
 	struct flowside	side[SIDES];
+	unsigned char		omac[6];
 };
 
 #define FLOW_INDEX_BITS		17	/* 128k - 1 */
-- 
2.48.1

[PATCH v3 4/8] udp: forward external source MAC address through tap interface

[Jon Maloy]

We forward the incoming MAC address through the tap interface when
receiving incoming packets from network local hosts. Packets from
the own host are excepted from this rule, and are still forwarded
with the default PASST/PASTA MAC address as source.

This is a part of the solution to bug
https://bugs.passt.top/show_bug.cgi?id=120

Signed-off-by: Jon Maloy <jmaloy@redhat.com>

---
v3: - Adapted to the move of external MAC address from struct flowside
      to struct flow_common
---
 udp.c | 37 +++++++++++++++++++------------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/udp.c b/udp.c
index 65a52e0..5942088 100644
--- a/udp.c
+++ b/udp.c
@@ -133,11 +133,8 @@ static int udp_splice_init[IP_VERSIONS][NUM_PORTS];
 /* UDP header and data for inbound messages */
 static struct udp_payload_t udp_payload[UDP_MAX_FRAMES];
 
-/* Ethernet header for IPv4 frames */
-static struct ethhdr udp4_eth_hdr;
-
-/* Ethernet header for IPv6 frames */
-static struct ethhdr udp6_eth_hdr;
+/* Ethernet headers for IPv4 and IPv6 frames */
+static struct ethhdr udp_eth_hdr[UDP_MAX_FRAMES];
 
 /**
  * struct udp_meta_t - Pre-cooked headers for UDP packets
@@ -214,8 +211,10 @@ void udp_portmap_clear(void)
  */
 void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s)
 {
-	eth_update_mac(&udp4_eth_hdr, eth_d, eth_s);
-	eth_update_mac(&udp6_eth_hdr, eth_d, eth_s);
+	int i;
+
+	for (i = 0; i < UDP_MAX_FRAMES; i++)
+		eth_update_mac(&udp_eth_hdr[i], eth_d, eth_s);
 }
 
 /**
@@ -238,6 +237,7 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 
 	*siov = IOV_OF_LVALUE(payload->data);
 
+	tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp_eth_hdr[i]);
 	tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &meta->taph);
 	tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 
@@ -253,9 +253,6 @@ static void udp_iov_init(const struct ctx *c)
 {
 	size_t i;
 
-	udp4_eth_hdr.h_proto = htons_constant(ETH_P_IP);
-	udp6_eth_hdr.h_proto = htons_constant(ETH_P_IPV6);
-
 	for (i = 0; i < UDP_MAX_FRAMES; i++)
 		udp_iov_init_one(c, i);
 }
@@ -352,31 +349,34 @@ size_t udp_update_hdr6(struct ipv6hdr *ip6h, struct udp_payload_t *bp,
  * udp_tap_prepare() - Convert one datagram into a tap frame
  * @mmh:	Receiving mmsghdr array
  * @idx:	Index of the datagram to prepare
+ * @uflow:	UDP flow
  * @toside:	Flowside for destination side
  * @no_udp_csum: Do not set UDP checksum
  */
 static void udp_tap_prepare(const struct mmsghdr *mmh,
-			    unsigned idx, const struct flowside *toside,
+			    unsigned int idx,
+			    const struct udp_flow *uflow,
+			    const struct flowside *toside,
 			    bool no_udp_csum)
 {
 	struct iovec (*tap_iov)[UDP_NUM_IOVS] = &udp_l2_iov[idx];
 	struct udp_payload_t *bp = &udp_payload[idx];
 	struct udp_meta_t *bm = &udp_meta[idx];
+	struct ethhdr *eh = (*tap_iov)[UDP_IOV_ETH].iov_base;
 	size_t l4len;
 
+	eth_update_mac(eh, 0, uflow->f.omac);
 	if (!inany_v4(&toside->eaddr) || !inany_v4(&toside->oaddr)) {
 		l4len = udp_update_hdr6(&bm->ip6h, bp, toside,
 					mmh[idx].msg_len, no_udp_csum);
-		tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip6h) +
-			       sizeof(udp6_eth_hdr));
-		(*tap_iov)[UDP_IOV_ETH] = IOV_OF_LVALUE(udp6_eth_hdr);
+		tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip6h) + ETH_HLEN);
+		eh->h_proto = htons_constant(ETH_P_IPV6);
 		(*tap_iov)[UDP_IOV_IP] = IOV_OF_LVALUE(bm->ip6h);
 	} else {
 		l4len = udp_update_hdr4(&bm->ip4h, bp, toside,
 					mmh[idx].msg_len, no_udp_csum);
-		tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip4h) +
-			       sizeof(udp4_eth_hdr));
-		(*tap_iov)[UDP_IOV_ETH] = IOV_OF_LVALUE(udp4_eth_hdr);
+		tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip4h) + ETH_HLEN);
+		eh->h_proto = htons_constant(ETH_P_IP);
 		(*tap_iov)[UDP_IOV_IP] = IOV_OF_LVALUE(bm->ip4h);
 	}
 	(*tap_iov)[UDP_IOV_PAYLOAD].iov_len = l4len;
@@ -801,13 +801,14 @@ static void udp_buf_sock_to_tap(const struct ctx *c, int s, int n,
 				flow_sidx_t tosidx)
 {
 	const struct flowside *toside = flowside_at_sidx(tosidx);
+	const struct udp_flow *uflow = udp_at_sidx(tosidx);
 	int i;
 
 	if ((n = udp_sock_recv(c, s, udp_mh_recv, n)) <= 0)
 		return;
 
 	for (i = 0; i < n; i++)
-		udp_tap_prepare(udp_mh_recv, i, toside, false);
+		udp_tap_prepare(udp_mh_recv, i, uflow, toside, false);
 
 	tap_send_frames(c, &udp_l2_iov[0][0], UDP_NUM_IOVS, n);
 }
-- 
@@ -133,11 +133,8 @@ static int udp_splice_init[IP_VERSIONS][NUM_PORTS];
 /* UDP header and data for inbound messages */
 static struct udp_payload_t udp_payload[UDP_MAX_FRAMES];
 
-/* Ethernet header for IPv4 frames */
-static struct ethhdr udp4_eth_hdr;
-
-/* Ethernet header for IPv6 frames */
-static struct ethhdr udp6_eth_hdr;
+/* Ethernet headers for IPv4 and IPv6 frames */
+static struct ethhdr udp_eth_hdr[UDP_MAX_FRAMES];
 
 /**
  * struct udp_meta_t - Pre-cooked headers for UDP packets
@@ -214,8 +211,10 @@ void udp_portmap_clear(void)
  */
 void udp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s)
 {
-	eth_update_mac(&udp4_eth_hdr, eth_d, eth_s);
-	eth_update_mac(&udp6_eth_hdr, eth_d, eth_s);
+	int i;
+
+	for (i = 0; i < UDP_MAX_FRAMES; i++)
+		eth_update_mac(&udp_eth_hdr[i], eth_d, eth_s);
 }
 
 /**
@@ -238,6 +237,7 @@ static void udp_iov_init_one(const struct ctx *c, size_t i)
 
 	*siov = IOV_OF_LVALUE(payload->data);
 
+	tiov[UDP_IOV_ETH] = IOV_OF_LVALUE(udp_eth_hdr[i]);
 	tiov[UDP_IOV_TAP] = tap_hdr_iov(c, &meta->taph);
 	tiov[UDP_IOV_PAYLOAD].iov_base = payload;
 
@@ -253,9 +253,6 @@ static void udp_iov_init(const struct ctx *c)
 {
 	size_t i;
 
-	udp4_eth_hdr.h_proto = htons_constant(ETH_P_IP);
-	udp6_eth_hdr.h_proto = htons_constant(ETH_P_IPV6);
-
 	for (i = 0; i < UDP_MAX_FRAMES; i++)
 		udp_iov_init_one(c, i);
 }
@@ -352,31 +349,34 @@ size_t udp_update_hdr6(struct ipv6hdr *ip6h, struct udp_payload_t *bp,
  * udp_tap_prepare() - Convert one datagram into a tap frame
  * @mmh:	Receiving mmsghdr array
  * @idx:	Index of the datagram to prepare
+ * @uflow:	UDP flow
  * @toside:	Flowside for destination side
  * @no_udp_csum: Do not set UDP checksum
  */
 static void udp_tap_prepare(const struct mmsghdr *mmh,
-			    unsigned idx, const struct flowside *toside,
+			    unsigned int idx,
+			    const struct udp_flow *uflow,
+			    const struct flowside *toside,
 			    bool no_udp_csum)
 {
 	struct iovec (*tap_iov)[UDP_NUM_IOVS] = &udp_l2_iov[idx];
 	struct udp_payload_t *bp = &udp_payload[idx];
 	struct udp_meta_t *bm = &udp_meta[idx];
+	struct ethhdr *eh = (*tap_iov)[UDP_IOV_ETH].iov_base;
 	size_t l4len;
 
+	eth_update_mac(eh, 0, uflow->f.omac);
 	if (!inany_v4(&toside->eaddr) || !inany_v4(&toside->oaddr)) {
 		l4len = udp_update_hdr6(&bm->ip6h, bp, toside,
 					mmh[idx].msg_len, no_udp_csum);
-		tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip6h) +
-			       sizeof(udp6_eth_hdr));
-		(*tap_iov)[UDP_IOV_ETH] = IOV_OF_LVALUE(udp6_eth_hdr);
+		tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip6h) + ETH_HLEN);
+		eh->h_proto = htons_constant(ETH_P_IPV6);
 		(*tap_iov)[UDP_IOV_IP] = IOV_OF_LVALUE(bm->ip6h);
 	} else {
 		l4len = udp_update_hdr4(&bm->ip4h, bp, toside,
 					mmh[idx].msg_len, no_udp_csum);
-		tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip4h) +
-			       sizeof(udp4_eth_hdr));
-		(*tap_iov)[UDP_IOV_ETH] = IOV_OF_LVALUE(udp4_eth_hdr);
+		tap_hdr_update(&bm->taph, l4len + sizeof(bm->ip4h) + ETH_HLEN);
+		eh->h_proto = htons_constant(ETH_P_IP);
 		(*tap_iov)[UDP_IOV_IP] = IOV_OF_LVALUE(bm->ip4h);
 	}
 	(*tap_iov)[UDP_IOV_PAYLOAD].iov_len = l4len;
@@ -801,13 +801,14 @@ static void udp_buf_sock_to_tap(const struct ctx *c, int s, int n,
 				flow_sidx_t tosidx)
 {
 	const struct flowside *toside = flowside_at_sidx(tosidx);
+	const struct udp_flow *uflow = udp_at_sidx(tosidx);
 	int i;
 
 	if ((n = udp_sock_recv(c, s, udp_mh_recv, n)) <= 0)
 		return;
 
 	for (i = 0; i < n; i++)
-		udp_tap_prepare(udp_mh_recv, i, toside, false);
+		udp_tap_prepare(udp_mh_recv, i, uflow, toside, false);
 
 	tap_send_frames(c, &udp_l2_iov[0][0], UDP_NUM_IOVS, n);
 }
-- 
2.48.1

[PATCH v3 5/8] tcp: forward external source MAC address through tap interface

[Jon Maloy]

We forward the incoming mac address through the tap interface when
receiving incoming packets from network local hosts. Packets from
the own host are excepted from this rule, and are still forwarded
with the default PASST/PASTA MAC address as source.

This is a part of the solution to bug
https://bugs.passt.top/show_bug.cgi?id=120

Signed-off-by: Jon Maloy <jmaloy@redhat.com>

---
v3: - Adapted to the move of external MAC address from struct flowside
      to struct flow_common
---
 tcp.c          |  5 ++++-
 tcp_buf.c      | 27 +++++++++++++--------------
 tcp_internal.h |  2 +-
 tcp_vu.c       |  5 ++---
 4 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/tcp.c b/tcp.c
index f43c1e2..057ee93 100644
--- a/tcp.c
+++ b/tcp.c
@@ -920,7 +920,7 @@ static void tcp_fill_header(struct tcphdr *th,
  * @no_tcp_csum:	Do not set TCP checksum
  */
 void tcp_fill_headers(const struct tcp_tap_conn *conn,
-		      struct tap_hdr *taph,
+		      struct tap_hdr *taph, struct ethhdr *eh,
 		      struct iphdr *ip4h, struct ipv6hdr *ip6h,
 		      struct tcphdr *th, struct iov_tail *payload,
 		      const uint16_t *ip4_check, uint32_t seq, bool no_tcp_csum)
@@ -952,6 +952,7 @@ void tcp_fill_headers(const struct tcp_tap_conn *conn,
 			psum = proto_ipv4_header_psum(l4len, IPPROTO_TCP,
 						      *src4, *dst4);
 		}
+		eh->h_proto = htons_constant(ETH_P_IP);
 	}
 
 	if (ip6h) {
@@ -972,7 +973,9 @@ void tcp_fill_headers(const struct tcp_tap_conn *conn,
 						      &ip6h->saddr,
 						      &ip6h->daddr);
 		}
+		eh->h_proto = htons_constant(ETH_P_IPV6);
 	}
+	eth_update_mac(eh, 0, conn->f.omac);
 
 	tcp_fill_header(th, conn, seq);
 
diff --git a/tcp_buf.c b/tcp_buf.c
index d1fca67..85df5c7 100644
--- a/tcp_buf.c
+++ b/tcp_buf.c
@@ -40,8 +40,7 @@
 /* Static buffers */
 
 /* Ethernet header for IPv4 and IPv6 frames */
-static struct ethhdr		tcp4_eth_src;
-static struct ethhdr		tcp6_eth_src;
+static struct ethhdr		tcp_eth_hdr[TCP_FRAMES_MEM];
 
 static struct tap_hdr		tcp_payload_tap_hdr[TCP_FRAMES_MEM];
 
@@ -71,8 +70,10 @@ static struct iovec	tcp_l2_iov[TCP_FRAMES_MEM][TCP_NUM_IOVS];
  */
 void tcp_update_l2_buf(const unsigned char *eth_d, const unsigned char *eth_s)
 {
-	eth_update_mac(&tcp4_eth_src, eth_d, eth_s);
-	eth_update_mac(&tcp6_eth_src, eth_d, eth_s);
+	int i;
+
+	for (i = 0; i < TCP_FRAMES_MEM; i++)
+		eth_update_mac(&tcp_eth_hdr[i], eth_d, eth_s);
 }
 
 /**
@@ -85,8 +86,8 @@ void tcp_sock_iov_init(const struct ctx *c)
 	struct iphdr iph = L2_BUF_IP4_INIT(IPPROTO_TCP);
 	int i;
 
-	tcp6_eth_src.h_proto = htons_constant(ETH_P_IPV6);
-	tcp4_eth_src.h_proto = htons_constant(ETH_P_IP);
+	for (i = 0; i < TCP_FRAMES_MEM; i++)
+		eth_update_mac(&tcp_eth_hdr[i], NULL, c->our_tap_mac);
 
 	for (i = 0; i < ARRAY_SIZE(tcp_payload); i++) {
 		tcp6_payload_ip[i] = ip6;
@@ -164,6 +165,7 @@ static void tcp_l2_buf_fill_headers(const struct tcp_tap_conn *conn,
 	struct tap_hdr *taph = iov[TCP_IOV_TAP].iov_base;
 	const struct flowside *tapside = TAPFLOW(conn);
 	const struct in_addr *a4 = inany_v4(&tapside->oaddr);
+	struct ethhdr *eh = iov[TCP_IOV_ETH].iov_base;
 	struct ipv6hdr *ip6h = NULL;
 	struct iphdr *ip4h = NULL;
 
@@ -172,7 +174,7 @@ static void tcp_l2_buf_fill_headers(const struct tcp_tap_conn *conn,
 	else
 		ip6h = iov[TCP_IOV_IP].iov_base;
 
-	tcp_fill_headers(conn, taph, ip4h, ip6h, th, &tail,
+	tcp_fill_headers(conn, taph, eh, ip4h, ip6h, th, &tail,
 			 check, seq, no_tcp_csum);
 }
 
@@ -194,14 +196,12 @@ int tcp_buf_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
 	int ret;
 
 	iov = tcp_l2_iov[tcp_payload_used];
-	if (CONN_V4(conn)) {
+	if (CONN_V4(conn))
 		iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[tcp_payload_used]);
-		iov[TCP_IOV_ETH].iov_base = &tcp4_eth_src;
-	} else {
+	else
 		iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[tcp_payload_used]);
-		iov[TCP_IOV_ETH].iov_base = &tcp6_eth_src;
-	}
 
+	iov[TCP_IOV_ETH] = IOV_OF_LVALUE(tcp_eth_hdr[tcp_payload_used]);
 	payload = iov[TCP_IOV_PAYLOAD].iov_base;
 	seq = conn->seq_to_tap;
 	ret = tcp_prepare_flags(c, conn, flags, &payload->th,
@@ -259,11 +259,10 @@ static void tcp_data_to_tap(const struct ctx *c, struct tcp_tap_conn *conn,
 			check = &iph->check;
 		}
 		iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp4_payload_ip[tcp_payload_used]);
-		iov[TCP_IOV_ETH].iov_base = &tcp4_eth_src;
 	} else if (CONN_V6(conn)) {
 		iov[TCP_IOV_IP] = IOV_OF_LVALUE(tcp6_payload_ip[tcp_payload_used]);
-		iov[TCP_IOV_ETH].iov_base = &tcp6_eth_src;
 	}
+	iov[TCP_IOV_ETH].iov_base = &tcp_eth_hdr[tcp_payload_used];
 	payload = iov[TCP_IOV_PAYLOAD].iov_base;
 	payload->th.th_off = sizeof(struct tcphdr) / 4;
 	payload->th.th_x2 = 0;
diff --git a/tcp_internal.h b/tcp_internal.h
index 36c6533..6c2d1ef 100644
--- a/tcp_internal.h
+++ b/tcp_internal.h
@@ -167,7 +167,7 @@ void tcp_rst_do(const struct ctx *c, struct tcp_tap_conn *conn);
 struct tcp_info_linux;
 
 void tcp_fill_headers(const struct tcp_tap_conn *conn,
-		      struct tap_hdr *taph,
+		      struct tap_hdr *taph, struct ethhdr *eh,
 		      struct iphdr *ip4h, struct ipv6hdr *ip6h,
 		      struct tcphdr *th, struct iov_tail *payload,
 		      const uint16_t *ip4_check, uint32_t seq, bool no_tcp_csum);
diff --git a/tcp_vu.c b/tcp_vu.c
index f3914c7..da1fb37 100644
--- a/tcp_vu.c
+++ b/tcp_vu.c
@@ -135,7 +135,7 @@ int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
 	flags_elem[0].in_sg[0].iov_len = hdrlen + optlen;
 	payload = IOV_TAIL(flags_elem[0].in_sg, 1, hdrlen);
 
-	tcp_fill_headers(conn, NULL, ip4h, ip6h, th, &payload,
+	tcp_fill_headers(conn, NULL, eh, ip4h, ip6h, th, &payload,
 			 NULL, seq, !*c->pcap);
 
 	if (*c->pcap) {
@@ -315,7 +315,6 @@ static void tcp_vu_prepare(const struct ctx *c, struct tcp_tap_conn *conn,
 	eh = vu_eth(base);
 
 	memcpy(eh->h_dest, c->guest_mac, sizeof(eh->h_dest));
-	memcpy(eh->h_source, c->our_tap_mac, sizeof(eh->h_source));
 
 	/* initialize header */
 
@@ -339,7 +338,7 @@ static void tcp_vu_prepare(const struct ctx *c, struct tcp_tap_conn *conn,
 	th->ack = 1;
 	th->psh = push;
 
-	tcp_fill_headers(conn, NULL, ip4h, ip6h, th, &payload,
+	tcp_fill_headers(conn, NULL, eh, ip4h, ip6h, th, &payload,
 			 *check, conn->seq_to_tap, no_tcp_csum);
 	if (ip4h)
 		*check = &ip4h->check;
-- 
@@ -135,7 +135,7 @@ int tcp_vu_send_flag(const struct ctx *c, struct tcp_tap_conn *conn, int flags)
 	flags_elem[0].in_sg[0].iov_len = hdrlen + optlen;
 	payload = IOV_TAIL(flags_elem[0].in_sg, 1, hdrlen);
 
-	tcp_fill_headers(conn, NULL, ip4h, ip6h, th, &payload,
+	tcp_fill_headers(conn, NULL, eh, ip4h, ip6h, th, &payload,
 			 NULL, seq, !*c->pcap);
 
 	if (*c->pcap) {
@@ -315,7 +315,6 @@ static void tcp_vu_prepare(const struct ctx *c, struct tcp_tap_conn *conn,
 	eh = vu_eth(base);
 
 	memcpy(eh->h_dest, c->guest_mac, sizeof(eh->h_dest));
-	memcpy(eh->h_source, c->our_tap_mac, sizeof(eh->h_source));
 
 	/* initialize header */
 
@@ -339,7 +338,7 @@ static void tcp_vu_prepare(const struct ctx *c, struct tcp_tap_conn *conn,
 	th->ack = 1;
 	th->psh = push;
 
-	tcp_fill_headers(conn, NULL, ip4h, ip6h, th, &payload,
+	tcp_fill_headers(conn, NULL, eh, ip4h, ip6h, th, &payload,
 			 *check, conn->seq_to_tap, no_tcp_csum);
 	if (ip4h)
 		*check = &ip4h->check;
-- 
2.48.1

[PATCH v3 6/8] tap: change signature of function tap_push_l2h()

[Jon Maloy]

In the following commits it must be possible for the callers of
function tap_push_l2h() to specify which source MAC address should
be added to the ethernet header sent over the tap interface. As
a preparation, we now add a new argument to that function, still
without actually using it.

Signed-off-by: Jon Maloy <jmaloy@redhat.com>

---
v3: Improved comment for src_mac argument, as suggested by Stefano.
---
 tap.c | 18 +++++++++++-------
 tap.h |  3 ++-
 tcp.c |  4 ++--
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/tap.c b/tap.c
index 6db5d88..ceb9d9f 100644
--- a/tap.c
+++ b/tap.c
@@ -171,17 +171,21 @@ const struct in6_addr *tap_ip6_daddr(const struct ctx *c,
  * tap_push_l2h() - Build an L2 header for an inbound packet
  * @c:		Execution context
  * @buf:	Buffer address at which to generate header
+ * @src_mac:	MAC address to be used as source for message. NULL means default
  * @proto:	Ethernet protocol number for L3
  *
  * Return: pointer at which to write the packet's payload
  */
-void *tap_push_l2h(const struct ctx *c, void *buf, uint16_t proto)
+void *tap_push_l2h(const struct ctx *c, void *buf,
+		   const void *src_mac, uint16_t proto)
 {
 	struct ethhdr *eh = (struct ethhdr *)buf;
 
-	/* TODO: ARP table lookup */
 	memcpy(eh->h_dest, c->guest_mac, ETH_ALEN);
-	memcpy(eh->h_source, c->our_tap_mac, ETH_ALEN);
+	if (src_mac)
+		memcpy(eh->h_source, src_mac, ETH_ALEN);
+	else
+		memcpy(eh->h_source, c->our_tap_mac, ETH_ALEN);
 	eh->h_proto = ntohs(proto);
 	return eh + 1;
 }
@@ -261,7 +265,7 @@ void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport,
 {
 	size_t l4len = dlen + sizeof(struct udphdr);
 	char buf[USHRT_MAX];
-	struct iphdr *ip4h = tap_push_l2h(c, buf, ETH_P_IP);
+	struct iphdr *ip4h = tap_push_l2h(c, buf, NULL, ETH_P_IP);
 	struct udphdr *uh = tap_push_ip4h(ip4h, src, dst, l4len, IPPROTO_UDP);
 	char *data = tap_push_uh4(uh, src, sport, dst, dport, in, dlen);
 
@@ -281,7 +285,7 @@ void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst,
 		    const void *in, size_t l4len)
 {
 	char buf[USHRT_MAX];
-	struct iphdr *ip4h = tap_push_l2h(c, buf, ETH_P_IP);
+	struct iphdr *ip4h = tap_push_l2h(c, buf, NULL, ETH_P_IP);
 	struct icmphdr *icmp4h = tap_push_ip4h(ip4h, src, dst,
 					       l4len, IPPROTO_ICMP);
 
@@ -367,7 +371,7 @@ void tap_udp6_send(const struct ctx *c,
 {
 	size_t l4len = dlen + sizeof(struct udphdr);
 	char buf[USHRT_MAX];
-	struct ipv6hdr *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6);
+	struct ipv6hdr *ip6h = tap_push_l2h(c, buf, NULL, ETH_P_IPV6);
 	struct udphdr *uh = tap_push_ip6h(ip6h, src, dst,
 					  l4len, IPPROTO_UDP, flow);
 	char *data = tap_push_uh6(uh, src, sport, dst, dport, in, dlen);
@@ -389,7 +393,7 @@ void tap_icmp6_send(const struct ctx *c,
 		    const void *in, size_t l4len)
 {
 	char buf[USHRT_MAX];
-	struct ipv6hdr *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6);
+	struct ipv6hdr *ip6h = tap_push_l2h(c, buf, NULL, ETH_P_IPV6);
 	struct icmp6hdr *icmp6h = tap_push_ip6h(ip6h, src, dst, l4len,
 						IPPROTO_ICMPV6, 0);
 
diff --git a/tap.h b/tap.h
index 6fe3d15..e640d95 100644
--- a/tap.h
+++ b/tap.h
@@ -70,7 +70,8 @@ static inline void tap_hdr_update(struct tap_hdr *thdr, size_t l2len)
 }
 
 unsigned long tap_l2_max_len(const struct ctx *c);
-void *tap_push_l2h(const struct ctx *c, void *buf, uint16_t proto);
+void *tap_push_l2h(const struct ctx *c, void *buf,
+		   const void *src_mac, uint16_t proto);
 void *tap_push_ip4h(struct iphdr *ip4h, struct in_addr src,
 		     struct in_addr dst, size_t l4len, uint8_t proto);
 void *tap_push_uh4(struct udphdr *uh, struct in_addr src, in_port_t sport,
diff --git a/tcp.c b/tcp.c
index 057ee93..3ecf9e8 100644
--- a/tcp.c
+++ b/tcp.c
@@ -1898,7 +1898,7 @@ static void tcp_rst_no_conn(const struct ctx *c, int af,
 		return;
 
 	if (af == AF_INET) {
-		struct iphdr *ip4h = tap_push_l2h(c, buf, ETH_P_IP);
+		struct iphdr *ip4h = tap_push_l2h(c, buf, NULL, ETH_P_IP);
 		const struct in_addr *rst_src = daddr;
 		const struct in_addr *rst_dst = saddr;
 
@@ -1908,7 +1908,7 @@ static void tcp_rst_no_conn(const struct ctx *c, int af,
 					      *rst_src, *rst_dst);
 
 	} else {
-		struct ipv6hdr *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6);
+		struct ipv6hdr *ip6h = tap_push_l2h(c, buf, NULL, ETH_P_IPV6);
 		const struct in6_addr *rst_src = daddr;
 		const struct in6_addr *rst_dst = saddr;
 
-- 
@@ -1898,7 +1898,7 @@ static void tcp_rst_no_conn(const struct ctx *c, int af,
 		return;
 
 	if (af == AF_INET) {
-		struct iphdr *ip4h = tap_push_l2h(c, buf, ETH_P_IP);
+		struct iphdr *ip4h = tap_push_l2h(c, buf, NULL, ETH_P_IP);
 		const struct in_addr *rst_src = daddr;
 		const struct in_addr *rst_dst = saddr;
 
@@ -1908,7 +1908,7 @@ static void tcp_rst_no_conn(const struct ctx *c, int af,
 					      *rst_src, *rst_dst);
 
 	} else {
-		struct ipv6hdr *ip6h = tap_push_l2h(c, buf, ETH_P_IPV6);
+		struct ipv6hdr *ip6h = tap_push_l2h(c, buf, NULL, ETH_P_IPV6);
 		const struct in6_addr *rst_src = daddr;
 		const struct in6_addr *rst_dst = saddr;
 
-- 
2.48.1

[PATCH v3 7/8] tcp: make tcp_rst_no_conn() respond with correct MAC address

[Jon Maloy]

tcp_rst_no_conn() needs to identify and specify which source MAC
address to use when sending an RST to the guest. This is because
it doesn't have access to any flow structure where this address
could be fetched.

Signed-off-by: Jon Maloy <jmaloy@redhat.com>

---
v3: - Adapted to the signature change in nl_mac_get() function, so that
      we now consider only the template interface when checking the
      ARP/NDP table.
---
 tcp.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/tcp.c b/tcp.c
index 3ecf9e8..8c502ea 100644
--- a/tcp.c
+++ b/tcp.c
@@ -309,6 +309,7 @@
 #include "tcp_internal.h"
 #include "tcp_buf.h"
 #include "tcp_vu.h"
+#include "netlink.h"
 
 #ifndef __USE_MISC
 /* From Linux UAPI, missing in netinet/tcp.h provided by musl */
@@ -1888,17 +1889,29 @@ static void tcp_rst_no_conn(const struct ctx *c, int af,
 			    const struct tcphdr *th, size_t l4len)
 {
 	struct iov_tail payload = IOV_TAIL(NULL, 0, 0);
+	unsigned char src_mac[ETH_ALEN];
+	union inany_addr tgt;
 	struct tcphdr *rsth;
 	char buf[USHRT_MAX];
 	uint32_t psum = 0;
 	size_t rst_l2len;
+	int ifi;
 
 	/* Don't respond to RSTs without a connection */
 	if (th->rst)
 		return;
 
+	/* Respond with true MAC address if remote host is on
+	 * the template interface's network segment
+	 */
+	ifi = af == AF_INET ? c->ifi4 : c->ifi6;
+	memcpy(src_mac, c->our_tap_mac, ETH_ALEN);
+	inany_from_af(&tgt, af, daddr);
+	if (!inany_nat(c, &tgt))
+		nl_mac_get(nl_sock, &tgt, ifi, src_mac);
+
 	if (af == AF_INET) {
-		struct iphdr *ip4h = tap_push_l2h(c, buf, NULL, ETH_P_IP);
+		struct iphdr *ip4h = tap_push_l2h(c, buf, src_mac, ETH_P_IP);
 		const struct in_addr *rst_src = daddr;
 		const struct in_addr *rst_dst = saddr;
 
@@ -1908,7 +1921,7 @@ static void tcp_rst_no_conn(const struct ctx *c, int af,
 					      *rst_src, *rst_dst);
 
 	} else {
-		struct ipv6hdr *ip6h = tap_push_l2h(c, buf, NULL, ETH_P_IPV6);
+		struct ipv6hdr *ip6h = tap_push_l2h(c, buf, src_mac, ETH_P_IPV6);
 		const struct in6_addr *rst_src = daddr;
 		const struct in6_addr *rst_dst = saddr;
 
-- 
@@ -309,6 +309,7 @@
 #include "tcp_internal.h"
 #include "tcp_buf.h"
 #include "tcp_vu.h"
+#include "netlink.h"
 
 #ifndef __USE_MISC
 /* From Linux UAPI, missing in netinet/tcp.h provided by musl */
@@ -1888,17 +1889,29 @@ static void tcp_rst_no_conn(const struct ctx *c, int af,
 			    const struct tcphdr *th, size_t l4len)
 {
 	struct iov_tail payload = IOV_TAIL(NULL, 0, 0);
+	unsigned char src_mac[ETH_ALEN];
+	union inany_addr tgt;
 	struct tcphdr *rsth;
 	char buf[USHRT_MAX];
 	uint32_t psum = 0;
 	size_t rst_l2len;
+	int ifi;
 
 	/* Don't respond to RSTs without a connection */
 	if (th->rst)
 		return;
 
+	/* Respond with true MAC address if remote host is on
+	 * the template interface's network segment
+	 */
+	ifi = af == AF_INET ? c->ifi4 : c->ifi6;
+	memcpy(src_mac, c->our_tap_mac, ETH_ALEN);
+	inany_from_af(&tgt, af, daddr);
+	if (!inany_nat(c, &tgt))
+		nl_mac_get(nl_sock, &tgt, ifi, src_mac);
+
 	if (af == AF_INET) {
-		struct iphdr *ip4h = tap_push_l2h(c, buf, NULL, ETH_P_IP);
+		struct iphdr *ip4h = tap_push_l2h(c, buf, src_mac, ETH_P_IP);
 		const struct in_addr *rst_src = daddr;
 		const struct in_addr *rst_dst = saddr;
 
@@ -1908,7 +1921,7 @@ static void tcp_rst_no_conn(const struct ctx *c, int af,
 					      *rst_src, *rst_dst);
 
 	} else {
-		struct ipv6hdr *ip6h = tap_push_l2h(c, buf, NULL, ETH_P_IPV6);
+		struct ipv6hdr *ip6h = tap_push_l2h(c, buf, src_mac, ETH_P_IPV6);
 		const struct in6_addr *rst_src = daddr;
 		const struct in6_addr *rst_dst = saddr;
 
-- 
2.48.1

[PATCH v3 8/8] icmp: let icmp use mac address from flowside structure

[Jon Maloy]

Even ICMP needs to be updated to use the external MAC address instead
of just the own tap address when applicable. We do that here.

Signed-off-by: Jon Maloy <jmaloy@redhat.com>

---
v3: - Adapted to the move of external MAC address from struct flowside
      to struct flow_common
---
 icmp.c |  4 ++--
 ndp.c  |  2 +-
 tap.c  | 10 ++++++----
 tap.h  |  4 ++--
 udp.c  | 12 ++++++++----
 5 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/icmp.c b/icmp.c
index 7e2b342..6a3b636 100644
--- a/icmp.c
+++ b/icmp.c
@@ -129,12 +129,12 @@ void icmp_sock_handler(const struct ctx *c, union epoll_ref ref)
 		const struct in_addr *daddr = inany_v4(&ini->eaddr);
 
 		ASSERT(saddr && daddr); /* Must have IPv4 addresses */
-		tap_icmp4_send(c, *saddr, *daddr, buf, n);
+		tap_icmp4_send(c, *saddr, *daddr, buf, pingf->f.omac, n);
 	} else if (pingf->f.type == FLOW_PING6) {
 		const struct in6_addr *saddr = &ini->oaddr.a6;
 		const struct in6_addr *daddr = &ini->eaddr.a6;
 
-		tap_icmp6_send(c, saddr, daddr, buf, n);
+		tap_icmp6_send(c, saddr, daddr, buf, pingf->f.omac, n);
 	}
 	return;
 
diff --git a/ndp.c b/ndp.c
index 2e7f0bc..09a9e0a 100644
--- a/ndp.c
+++ b/ndp.c
@@ -185,7 +185,7 @@ static void ndp_send(const struct ctx *c, const struct in6_addr *dst,
 {
 	const struct in6_addr *src = &c->ip6.our_tap_ll;
 
-	tap_icmp6_send(c, src, dst, buf, l4len);
+	tap_icmp6_send(c, src, dst, buf, c->our_tap_mac, l4len);
 }
 
 /**
diff --git a/tap.c b/tap.c
index ceb9d9f..e62c9a6 100644
--- a/tap.c
+++ b/tap.c
@@ -279,13 +279,14 @@ void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport,
  * @src:	IPv4 source address
  * @dst:	IPv4 destination address
  * @in:		ICMP packet, including ICMP header
+ * @src_mac:	MAC address to be used as source for message
  * @l4len:	ICMP packet length, including ICMP header
  */
 void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst,
-		    const void *in, size_t l4len)
+		    const void *in, const void *src_mac, size_t l4len)
 {
 	char buf[USHRT_MAX];
-	struct iphdr *ip4h = tap_push_l2h(c, buf, NULL, ETH_P_IP);
+	struct iphdr *ip4h = tap_push_l2h(c, buf, src_mac, ETH_P_IP);
 	struct icmphdr *icmp4h = tap_push_ip4h(ip4h, src, dst,
 					       l4len, IPPROTO_ICMP);
 
@@ -386,14 +387,15 @@ void tap_udp6_send(const struct ctx *c,
  * @src:	IPv6 source address
  * @dst:	IPv6 destination address
  * @in:		ICMP packet, including ICMP header
+ * @src_mac:	MAC address to be used as source for message
  * @l4len:	ICMP packet length, including ICMP header
  */
 void tap_icmp6_send(const struct ctx *c,
 		    const struct in6_addr *src, const struct in6_addr *dst,
-		    const void *in, size_t l4len)
+		    const void *in, const void *src_mac, size_t l4len)
 {
 	char buf[USHRT_MAX];
-	struct ipv6hdr *ip6h = tap_push_l2h(c, buf, NULL, ETH_P_IPV6);
+	struct ipv6hdr *ip6h = tap_push_l2h(c, buf, src_mac, ETH_P_IPV6);
 	struct icmp6hdr *icmp6h = tap_push_ip6h(ip6h, src, dst, l4len,
 						IPPROTO_ICMPV6, 0);
 
diff --git a/tap.h b/tap.h
index e640d95..7025eaa 100644
--- a/tap.h
+++ b/tap.h
@@ -91,7 +91,7 @@ void tap_udp4_send(const struct ctx *c, struct in_addr src, in_port_t sport,
 		   struct in_addr dst, in_port_t dport,
 		   const void *in, size_t dlen);
 void tap_icmp4_send(const struct ctx *c, struct in_addr src, struct in_addr dst,
-		    const void *in, size_t l4len);
+		    const void *in, const void *src_mac, size_t l4len);
 const struct in6_addr *tap_ip6_daddr(const struct ctx *c,
 				     const struct in6_addr *src);
 void *tap_push_ip6h(struct ipv6hdr *ip6h,
@@ -103,7 +103,7 @@ void tap_udp6_send(const struct ctx *c,
 		   uint32_t flow, void *in, size_t dlen);
 void tap_icmp6_send(const struct ctx *c,
 		    const struct in6_addr *src, const struct in6_addr *dst,
-		    const void *in, size_t l4len);
+		    const void *in, const void *src_mac, size_t l4len);
 void tap_send_single(const struct ctx *c, const void *data, size_t l2len);
 size_t tap_send_frames(const struct ctx *c, const struct iovec *iov,
 		       size_t bufs_per_frame, size_t nframes);
diff --git a/udp.c b/udp.c
index 5942088..453bb51 100644
--- a/udp.c
+++ b/udp.c
@@ -386,6 +386,7 @@ static void udp_tap_prepare(const struct mmsghdr *mmh,
  * udp_send_tap_icmp4() - Construct and send ICMPv4 to local peer
  * @c:		Execution context
  * @ee:	Extended error descriptor
+ * @uflow:	UDP flow
  * @toside:	Destination side of flow
  * @saddr:	Address of ICMP generating node
  * @in:	First bytes (max 8) of original UDP message body
@@ -393,6 +394,7 @@ static void udp_tap_prepare(const struct mmsghdr *mmh,
  */
 static void udp_send_tap_icmp4(const struct ctx *c,
 			       const struct sock_extended_err *ee,
+			       const struct udp_flow *uflow,
 			       const struct flowside *toside,
 			       struct in_addr saddr,
 			       const void *in, size_t dlen)
@@ -422,7 +424,7 @@ static void udp_send_tap_icmp4(const struct ctx *c,
 	tap_push_uh4(&msg.uh, eaddr, eport, oaddr, oport, in, dlen);
 	memcpy(&msg.data, in, dlen);
 
-	tap_icmp4_send(c, saddr, eaddr, &msg, msglen);
+	tap_icmp4_send(c, saddr, eaddr, &msg, uflow->f.omac, msglen);
 }
 
 
@@ -430,6 +432,7 @@ static void udp_send_tap_icmp4(const struct ctx *c,
  * udp_send_tap_icmp6() - Construct and send ICMPv6 to local peer
  * @c:		Execution context
  * @ee:	Extended error descriptor
+ * @uflow:	UDP flow
  * @toside:	Destination side of flow
  * @saddr:	Address of ICMP generating node
  * @in:	First bytes (max 1232) of original UDP message body
@@ -438,6 +441,7 @@ static void udp_send_tap_icmp4(const struct ctx *c,
  */
 static void udp_send_tap_icmp6(const struct ctx *c,
 			       const struct sock_extended_err *ee,
+			       const struct udp_flow *uflow,
 			       const struct flowside *toside,
 			       const struct in6_addr *saddr,
 			       void *in, size_t dlen, uint32_t flow)
@@ -467,7 +471,7 @@ static void udp_send_tap_icmp6(const struct ctx *c,
 	tap_push_uh6(&msg.uh, eaddr, eport, oaddr, oport, in, dlen);
 	memcpy(&msg.data, in, dlen);
 
-	tap_icmp6_send(c, saddr, eaddr, &msg, msglen);
+	tap_icmp6_send(c, saddr, eaddr, &msg, uflow->f.omac, msglen);
 }
 
 /**
@@ -627,12 +631,12 @@ static int udp_sock_recverr(const struct ctx *c, int s, flow_sidx_t sidx,
 	if (hdr->cmsg_level == IPPROTO_IP &&
 	    (o4 = inany_v4(&otap)) && inany_v4(&toside->eaddr)) {
 		dlen = MIN(dlen, ICMP4_MAX_DLEN);
-		udp_send_tap_icmp4(c, ee, toside, *o4, data, dlen);
+		udp_send_tap_icmp4(c, ee, uflow, toside, *o4, data, dlen);
 		return 1;
 	}
 
 	if (hdr->cmsg_level == IPPROTO_IPV6 && !inany_v4(&toside->eaddr)) {
-		udp_send_tap_icmp6(c, ee, toside, &otap.a6, data, dlen,
+		udp_send_tap_icmp6(c, ee, uflow, toside, &otap.a6, data, dlen,
 				   FLOW_IDX(uflow));
 		return 1;
 	}
-- 
@@ -386,6 +386,7 @@ static void udp_tap_prepare(const struct mmsghdr *mmh,
  * udp_send_tap_icmp4() - Construct and send ICMPv4 to local peer
  * @c:		Execution context
  * @ee:	Extended error descriptor
+ * @uflow:	UDP flow
  * @toside:	Destination side of flow
  * @saddr:	Address of ICMP generating node
  * @in:	First bytes (max 8) of original UDP message body
@@ -393,6 +394,7 @@ static void udp_tap_prepare(const struct mmsghdr *mmh,
  */
 static void udp_send_tap_icmp4(const struct ctx *c,
 			       const struct sock_extended_err *ee,
+			       const struct udp_flow *uflow,
 			       const struct flowside *toside,
 			       struct in_addr saddr,
 			       const void *in, size_t dlen)
@@ -422,7 +424,7 @@ static void udp_send_tap_icmp4(const struct ctx *c,
 	tap_push_uh4(&msg.uh, eaddr, eport, oaddr, oport, in, dlen);
 	memcpy(&msg.data, in, dlen);
 
-	tap_icmp4_send(c, saddr, eaddr, &msg, msglen);
+	tap_icmp4_send(c, saddr, eaddr, &msg, uflow->f.omac, msglen);
 }
 
 
@@ -430,6 +432,7 @@ static void udp_send_tap_icmp4(const struct ctx *c,
  * udp_send_tap_icmp6() - Construct and send ICMPv6 to local peer
  * @c:		Execution context
  * @ee:	Extended error descriptor
+ * @uflow:	UDP flow
  * @toside:	Destination side of flow
  * @saddr:	Address of ICMP generating node
  * @in:	First bytes (max 1232) of original UDP message body
@@ -438,6 +441,7 @@ static void udp_send_tap_icmp4(const struct ctx *c,
  */
 static void udp_send_tap_icmp6(const struct ctx *c,
 			       const struct sock_extended_err *ee,
+			       const struct udp_flow *uflow,
 			       const struct flowside *toside,
 			       const struct in6_addr *saddr,
 			       void *in, size_t dlen, uint32_t flow)
@@ -467,7 +471,7 @@ static void udp_send_tap_icmp6(const struct ctx *c,
 	tap_push_uh6(&msg.uh, eaddr, eport, oaddr, oport, in, dlen);
 	memcpy(&msg.data, in, dlen);
 
-	tap_icmp6_send(c, saddr, eaddr, &msg, msglen);
+	tap_icmp6_send(c, saddr, eaddr, &msg, uflow->f.omac, msglen);
 }
 
 /**
@@ -627,12 +631,12 @@ static int udp_sock_recverr(const struct ctx *c, int s, flow_sidx_t sidx,
 	if (hdr->cmsg_level == IPPROTO_IP &&
 	    (o4 = inany_v4(&otap)) && inany_v4(&toside->eaddr)) {
 		dlen = MIN(dlen, ICMP4_MAX_DLEN);
-		udp_send_tap_icmp4(c, ee, toside, *o4, data, dlen);
+		udp_send_tap_icmp4(c, ee, uflow, toside, *o4, data, dlen);
 		return 1;
 	}
 
 	if (hdr->cmsg_level == IPPROTO_IPV6 && !inany_v4(&toside->eaddr)) {
-		udp_send_tap_icmp6(c, ee, toside, &otap.a6, data, dlen,
+		udp_send_tap_icmp6(c, ee, uflow, toside, &otap.a6, data, dlen,
 				   FLOW_IDX(uflow));
 		return 1;
 	}
-- 
2.48.1