[PATCH v3 09/11] ip: Track observed guest IPv4 addresses in unified address array

[Jon Maloy <jmaloy@redhat.com>]

We remove the addr_seen field in struct ip4_ctx and replace it by
setting a new INANY_ADDR_OBSERVED flag in the corresponding entry in
the unified address array. If the seen address is not present in the
array we add it first.

We also update the migration protocol to v3, so that it can distribute
multiple addresses from the unified array.

Signed-off-by: Jon Maloy <jmaloy@redhat.com>

---
v3: - Updated migration protocol
---
 conf.c    |   2 -
 fwd.c     |  50 ++++++++++++----
 migrate.c | 172 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 passt.h   |   3 +-
 tap.c     |  40 +++++++++++--
 5 files changed, 241 insertions(+), 26 deletions(-)

diff --git a/conf.c b/conf.c
index d73a3dd..c5818a8 100644
--- a/conf.c
+++ b/conf.c
@@ -734,7 +734,6 @@ static unsigned int conf_ip4(unsigned int ifi, struct ctx *c)
 		c->addrs[c->addr_count].prefix_len = prefix_len + 96;
 		c->addrs[c->addr_count].flags = INANY_ADDR_HOST;
 		c->addr_count++;
-		c->ip4.addr_seen = addr;
 	}
 
 	c->ip4.our_tap_addr = c->ip4.guest_gw;
@@ -749,7 +748,6 @@ static unsigned int conf_ip4(unsigned int ifi, struct ctx *c)
 static void conf_ip4_local(struct ctx *c)
 {
 	c->addrs[c->addr_count].addr = inany_from_v4(IP4_LL_GUEST_ADDR);
-	c->ip4.addr_seen = *inany_v4(&c->addrs[c->addr_count].addr);
 	c->ip4.our_tap_addr = c->ip4.guest_gw = IP4_LL_GUEST_GW;
 	c->addrs[c->addr_count].prefix_len = IP4_LL_PREFIX_LEN;
 	c->addr_count++;
diff --git a/fwd.c b/fwd.c
index 20d581d..5af482d 100644
--- a/fwd.c
+++ b/fwd.c
@@ -491,6 +491,30 @@ static bool is_dns_flow(uint8_t proto, const struct flowside *ini)
 		((ini->oport == 53) || (ini->oport == 853));
 }
 
+/**
+ * fwd_guest_addr4() - Get first observed IPv4 guest address
+ * @c:		Execution context
+ *
+ * Return: first observed IPv4 address, if any, otherwise
+ *         first configured, if any, otherwise NULL
+ */
+static const struct in_addr *fwd_guest_addr4(const struct ctx *c)
+{
+	const struct inany_addr_entry *e;
+
+	/* Find first observed address */
+	for_each_addr(c, e, AF_INET)
+		if (e->flags & INANY_ADDR_OBSERVED)
+			return inany_v4(&e->addr);
+
+	/* Fallback to first address */
+	e = first_v4(c);
+	if (e)
+		return inany_v4(&e->addr);
+
+	return NULL;
+}
+
 /**
  * fwd_guest_accessible4() - Is IPv4 address guest-accessible
  * @c:		Execution context
@@ -515,17 +539,11 @@ static bool fwd_guest_accessible4(const struct ctx *c,
 	if (IN4_IS_ADDR_UNSPECIFIED(addr))
 		return false;
 
-	/* Check against all configured guest addresses */
+	/* Check against all guest addresses (configured and observed) */
 	for_each_addr(c, e, AF_INET)
 		if (IN4_ARE_ADDR_EQUAL(addr, inany_v4(&e->addr)))
 			return false;
 
-	/* Also check addr_seen: it tracks the address the guest is actually
-	 * using, which may differ from configured addresses.
-	 */
-	if (IN4_ARE_ADDR_EQUAL(addr, &c->ip4.addr_seen))
-		return false;
-
 	return true;
 }
 
@@ -768,10 +786,16 @@ uint8_t fwd_nat_from_host(const struct ctx *c, uint8_t proto,
 		 * match.
 		 */
 		if (inany_v4(&ini->eaddr)) {
-			if (c->host_lo_to_ns_lo)
+			if (c->host_lo_to_ns_lo) {
 				tgt->eaddr = inany_loopback4;
-			else
-				tgt->eaddr = inany_from_v4(c->ip4.addr_seen);
+			} else {
+				const struct in_addr *guest_addr;
+
+				guest_addr = fwd_guest_addr4(c);
+				if (!guest_addr)
+					return PIF_NONE;
+				tgt->eaddr = inany_from_v4(*guest_addr);
+			}
 			tgt->oaddr = inany_any4;
 		} else {
 			if (c->host_lo_to_ns_lo)
@@ -803,7 +827,11 @@ uint8_t fwd_nat_from_host(const struct ctx *c, uint8_t proto,
 	tgt->oport = ini->eport;
 
 	if (inany_v4(&tgt->oaddr)) {
-		tgt->eaddr = inany_from_v4(c->ip4.addr_seen);
+		const struct in_addr *guest_addr = fwd_guest_addr4(c);
+
+		if (!guest_addr)
+			return PIF_NONE;
+		tgt->eaddr = inany_from_v4(*guest_addr);
 	} else {
 		if (inany_is_linklocal6(&tgt->oaddr))
 			tgt->eaddr.a6 = c->ip6.addr_ll_seen;
diff --git a/migrate.c b/migrate.c
index 7398d26..a84af1c 100644
--- a/migrate.c
+++ b/migrate.c
@@ -29,7 +29,7 @@
 #define MIGRATE_MAGIC		0xB1BB1D1B0BB1D1B0
 
 /**
- * struct migrate_seen_addrs_v2 - Migratable guest addresses for v2 protocol
+ * struct migrate_seen_addrs_v2 - Migratable observed guest addresses
  * @addr6:	Observed guest IPv6 address
  * @addr6_ll:	Observed guest IPv6 link-local address
  * @addr4:	Observed guest IPv4 address
@@ -43,7 +43,19 @@ struct migrate_seen_addrs_v2 {
 } __attribute__((packed));
 
 /**
- * seen_addrs_source_v2() - Copy and send guest observed addresses from source
+ * struct migrate_seen_addrs_v3 - Migratable observed guest addresses
+ * @addr_count:	Number of address entries
+ * @addrs:	Array of observed address entries
+ * @mac:	Observed guest MAC address
+ */
+struct migrate_seen_addrs_v3 {
+	uint8_t addr_count;
+	struct inany_addr_entry addrs[INANY_MAX_ADDRS];
+	unsigned char mac[ETH_ALEN];
+} __packed;
+
+/**
+ * seen_addrs_source_v2() - Send observed addresses from source
  * @c:		Execution context
  * @stage:	Migration stage, unused
  * @fd:		File descriptor for state transfer
@@ -57,11 +69,19 @@ static int seen_addrs_source_v2(struct ctx *c,
 	struct migrate_seen_addrs_v2 addrs = {
 		.addr6 = c->ip6.addr_seen,
 		.addr6_ll = c->ip6.addr_ll_seen,
-		.addr4 = c->ip4.addr_seen,
 	};
+	const struct inany_addr_entry *e;
 
 	(void)stage;
 
+	/* Find first observed IPv4 address */
+	for_each_addr(c, e, AF_INET) {
+		if (e->flags & INANY_ADDR_OBSERVED) {
+			addrs.addr4 = *inany_v4(&e->addr);
+			break;
+		}
+	}
+
 	memcpy(addrs.mac, c->guest_mac, sizeof(addrs.mac));
 
 	if (write_all_buf(fd, &addrs, sizeof(addrs)))
@@ -71,7 +91,34 @@ static int seen_addrs_source_v2(struct ctx *c,
 }
 
 /**
- * seen_addrs_target_v2() - Receive and use guest observed addresses on target
+ * migrate_observed_addr4() - Add observed IPv4 address to address array
+ * @c:		Execution context
+ * @addr:	IPv4 address to add
+ */
+static void migrate_observed_addr4(struct ctx *c, const struct in_addr *addr)
+{
+	struct inany_addr_entry *e;
+
+	if (IN4_IS_ADDR_UNSPECIFIED(addr))
+		return;
+
+	for_each_addr(c, e, AF_INET) {
+		if (IN4_ARE_ADDR_EQUAL(addr, inany_v4(&e->addr))) {
+			e->flags |= INANY_ADDR_OBSERVED;
+			return;
+		}
+	}
+
+	if (c->addr_count < INANY_MAX_ADDRS) {
+		c->addrs[c->addr_count].addr = inany_from_v4(*addr);
+		c->addrs[c->addr_count].prefix_len = 0;
+		c->addrs[c->addr_count].flags = INANY_ADDR_OBSERVED;
+		c->addr_count++;
+	}
+}
+
+/**
+ * seen_addrs_target_v2() - Receive observed addresses on target
  * @c:		Execution context
  * @stage:	Migration stage, unused
  * @fd:		File descriptor for state transfer
@@ -82,6 +129,7 @@ static int seen_addrs_target_v2(struct ctx *c,
 				const struct migrate_stage *stage, int fd)
 {
 	struct migrate_seen_addrs_v2 addrs;
+	struct in_addr addr4;
 
 	(void)stage;
 
@@ -90,7 +138,100 @@ static int seen_addrs_target_v2(struct ctx *c,
 
 	c->ip6.addr_seen = addrs.addr6;
 	c->ip6.addr_ll_seen = addrs.addr6_ll;
-	c->ip4.addr_seen = addrs.addr4;
+
+	/* Avoid alignment warning */
+	addr4 = addrs.addr4;
+	migrate_observed_addr4(c, &addr4);
+
+	memcpy(c->guest_mac, addrs.mac, sizeof(c->guest_mac));
+
+	return 0;
+}
+
+/**
+ * seen_addrs_source_v3() - Send observed addresses from source
+ * @c:		Execution context
+ * @stage:	Migration stage, unused
+ * @fd:		File descriptor for state transfer
+ *
+ * Return: 0 on success, positive error code on failure
+ */
+/* cppcheck-suppress [constParameterCallback, unmatchedSuppression] */
+static int seen_addrs_source_v3(struct ctx *c,
+				const struct migrate_stage *stage, int fd)
+{
+	struct migrate_seen_addrs_v3 addrs = { .addr_count = 0 };
+	const struct inany_addr_entry *e;
+
+	(void)stage;
+
+	/* Copy only observed address entries */
+	for_each_addr(c, e, 0) {
+		if ((e->flags & INANY_ADDR_OBSERVED) &&
+		    addrs.addr_count < INANY_MAX_ADDRS)
+			addrs.addrs[addrs.addr_count++] = *e;
+	}
+
+	memcpy(addrs.mac, c->guest_mac, sizeof(addrs.mac));
+
+	if (write_all_buf(fd, &addrs, sizeof(addrs)))
+		return errno;
+
+	return 0;
+}
+
+/**
+ * migrate_observed_addr() - Add observed address to address array
+ * @c:		Execution context
+ * @addr:	Address entry to add
+ */
+static void migrate_observed_addr(struct ctx *c,
+				  const struct inany_addr_entry *addr)
+{
+	struct inany_addr_entry *e;
+
+	if (inany_is_unspecified(&addr->addr))
+		return;
+
+	/* Check if address already exists, just mark it observed */
+	for_each_addr(c, e, 0) {
+		if (inany_equals(&e->addr, &addr->addr)) {
+			e->flags |= INANY_ADDR_OBSERVED;
+			return;
+		}
+	}
+
+	/* Add new entry if there's room */
+	if (c->addr_count < INANY_MAX_ADDRS)
+		c->addrs[c->addr_count++] = *addr;
+}
+
+/**
+ * seen_addrs_target_v3() - Receive observed addresses on target
+ * @c:		Execution context
+ * @stage:	Migration stage, unused
+ * @fd:		File descriptor for state transfer
+ *
+ * Return: 0 on success, positive error code on failure
+ */
+static int seen_addrs_target_v3(struct ctx *c,
+				const struct migrate_stage *stage, int fd)
+{
+	struct migrate_seen_addrs_v3 addrs;
+	uint8_t i;
+
+	(void)stage;
+
+	if (read_all_buf(fd, &addrs, sizeof(addrs)))
+		return errno;
+
+	/* Merge observed addresses into existing array */
+	for (i = 0; i < addrs.addr_count && i < INANY_MAX_ADDRS; i++) {
+		struct inany_addr_entry e = addrs.addrs[i];
+
+		migrate_observed_addr(c, &e);
+	}
+
 	memcpy(c->guest_mac, addrs.mac, sizeof(c->guest_mac));
 
 	return 0;
@@ -116,8 +257,29 @@ static const struct migrate_stage stages_v2[] = {
 	{ 0 },
 };
 
+/* Stages for version 3 (multiple observed IPv4 addresses) */
+static const struct migrate_stage stages_v3[] = {
+	{
+		.name = "observed addresses",
+		.source = seen_addrs_source_v3,
+		.target = seen_addrs_target_v3,
+	},
+	{
+		.name = "prepare flows",
+		.source = flow_migrate_source_pre,
+		.target = NULL,
+	},
+	{
+		.name = "transfer flows",
+		.source = flow_migrate_source,
+		.target = flow_migrate_target,
+	},
+	{ 0 },
+};
+
 /* Supported encoding versions, from latest (most preferred) to oldest */
 static const struct migrate_version versions[] = {
+	{ 3,	stages_v3, },
 	{ 2,	stages_v2, },
 	/* v1 was released, but not widely used.  It had bad endianness for the
 	 * MSS and omitted timestamps, which meant it usually wouldn't work.
diff --git a/passt.h b/passt.h
index ec5517d..73084ae 100644
--- a/passt.h
+++ b/passt.h
@@ -69,6 +69,7 @@ enum passt_modes {
  */
 #define INANY_ADDR_CONFIGURED	BIT(0)		/* User set via -a */
 #define INANY_ADDR_HOST		BIT(1)		/* From host interface */
+#define INANY_ADDR_OBSERVED	BIT(2)		/* Seen in guest traffic */
 
 /**
  * for_each_addr() - Iterate over addresses in unified array
@@ -99,7 +100,6 @@ struct inany_addr_entry {
 
 /**
  * struct ip4_ctx - IPv4 execution context
- * @addr_seen:		Latest IPv4 address seen as source from tap
  * @guest_gw:		IPv4 gateway as seen by the guest
  * @map_host_loopback:	Outbound connections to this address are NATted to the
  *                      host's 127.0.0.1
@@ -115,7 +115,6 @@ struct inany_addr_entry {
  * @no_copy_addrs:	Don't copy all addresses when configuring namespace
  */
 struct ip4_ctx {
-	struct in_addr addr_seen;
 	struct in_addr guest_gw;
 	struct in_addr map_host_loopback;
 	struct in_addr map_guest_addr;
diff --git a/tap.c b/tap.c
index f3073c6..f2512f2 100644
--- a/tap.c
+++ b/tap.c
@@ -161,6 +161,38 @@ void tap_send_single(const struct ctx *c, const void *data, size_t l2len)
 	}
 }
 
+/**
+ * tap_check_src_addr4() - Note an IPv4 address seen in guest traffic
+ * @c:		Execution context
+ * @addr:	IPv4 address seen as source from guest
+ *
+ * Add the address to addrs[] with OBSERVED flag if not already present
+ */
+static void tap_check_src_addr4(struct ctx *c, const struct in_addr *addr)
+{
+	struct inany_addr_entry *e;
+
+	/* Check if already in array */
+	for_each_addr(c, e, AF_INET) {
+		if (IN4_ARE_ADDR_EQUAL(addr, inany_v4(&e->addr))) {
+			e->flags |= INANY_ADDR_OBSERVED;
+			return;
+		}
+	}
+
+	/* Add new entry if space available */
+	if (c->addr_count >= INANY_MAX_ADDRS) {
+		debug("Address table full, can't add new IPv4 address");
+		return;
+	}
+
+	c->addrs[c->addr_count].addr = inany_from_v4(*addr);
+	c->addrs[c->addr_count].prefix_len = 0;
+	c->addrs[c->addr_count].flags = INANY_ADDR_OBSERVED;
+	c->addr_count++;
+	debug("Added new observed IPv4 address");
+}
+
 /**
  * tap_ip6_daddr() - Normal IPv6 destination address for inbound packets
  * @c:		Execution context
@@ -771,8 +803,8 @@ resume:
 			continue;
 		}
 
-		if (iph->saddr && c->ip4.addr_seen.s_addr != iph->saddr)
-			c->ip4.addr_seen.s_addr = iph->saddr;
+		if (iph->saddr)
+			tap_check_src_addr4(c, (const struct in_addr *)&iph->saddr);
 
 		if (!iov_drop_header(&data, hlen))
 			continue;
@@ -950,10 +982,6 @@ resume:
 			if (IN6_IS_ADDR_UNSPECIFIED(&c->ip6.addr_seen)) {
 				c->ip6.addr_seen = *saddr;
 			}
-
-			if (first_v6(c) &&
-			    IN6_IS_ADDR_UNSPECIFIED(&first_v6(c)->addr.a6))
-				first_v6(c)->addr.a6 = *saddr;
 		} else if (!IN6_IS_ADDR_UNSPECIFIED(saddr)){
 			c->ip6.addr_seen = *saddr;
 		}
-- 
2.52.0