[PATCH v4 4/8] migrate: Don't handle the migration channel through epoll

[Stefano Brivio <sbrivio@redhat.com>]

From: David Gibson <david@gibson.dropbear.id.au>

Currently, once a migration device state fd is assigned, we wait for
EPOLLIN or EPOLLOUT events on it to actually perform the migration.  Change
it so that once a migration is requested it we complete it synchronously
at the end of the current epoll cycle.  This has several advantages:

  1. It makes it clear that everything about the migration must be dealt
     with at once, not split between multiple epoll events on the channel
  2. It ensures the migration always takes place between epoll cycles,
     rather than, for example, between handling TCP events and their
     deferred handling in post_handler().
  3. It reduces code setting up the epoll watch on the fd.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 epoll_type.h |  2 --
 migrate.c    | 43 +++++++++++--------------------------------
 migrate.h    |  2 +-
 passt.c      |  8 ++++----
 passt.h      |  2 ++
 5 files changed, 18 insertions(+), 39 deletions(-)

diff --git a/epoll_type.h b/epoll_type.h
index b1d3a1d..f3ef415 100644
--- a/epoll_type.h
+++ b/epoll_type.h
@@ -40,8 +40,6 @@ enum epoll_type {
 	EPOLL_TYPE_VHOST_CMD,
 	/* vhost-user kick event socket */
 	EPOLL_TYPE_VHOST_KICK,
-	/* migration device state channel */
-	EPOLL_TYPE_DEVICE_STATE,
 
 	EPOLL_NUM_TYPES,
 };
diff --git a/migrate.c b/migrate.c
index b7e5431..f64ffc8 100644
--- a/migrate.c
+++ b/migrate.c
@@ -250,26 +250,6 @@ static int migrate_target(struct ctx *c, int fd)
 	return 0;
 }
 
-/**
- * set_migration_watch() - Add the migration file descriptor to epoll
- * @c:		Execution context
- * @fd:		File descriptor to add
- * @target:	Are we the target of the migration?
- */
-static void set_migration_watch(const struct ctx *c, int fd, bool target)
-{
-	union epoll_ref ref = {
-		.type = EPOLL_TYPE_DEVICE_STATE,
-		.fd = fd,
-	 };
-	struct epoll_event ev = { 0 };
-
-	ev.data.u64 = ref.u64;
-	ev.events = target ? EPOLLIN : EPOLLOUT;
-
-	epoll_ctl(c->epollfd, EPOLL_CTL_ADD, ref.fd, &ev);
-}
-
 /**
  * migrate_init() - Set up things necessary for migration
  * @c:		Execution context
@@ -288,7 +268,6 @@ void migrate_close(struct ctx *c)
 {
 	if (c->device_state_fd != -1) {
 		debug("Closing migration channel, fd: %d", c->device_state_fd);
-		epoll_del(c, c->device_state_fd);
 		close(c->device_state_fd);
 		c->device_state_fd = -1;
 		c->device_state_result = -1;
@@ -310,27 +289,27 @@ void migrate_request(struct ctx *c, int fd, bool target)
 		migrate_close(c);
 
 	c->device_state_fd = fd;
-	set_migration_watch(c, c->device_state_fd, target);
-
+	c->migrate_target = target;
 }
 
 /**
  * migrate_handler() - Send/receive passt internal state to/from QEMU
  * @c:		Execution context
- * @events:	epoll events
  */
-void migrate_handler(struct ctx *c, uint32_t events)
+void migrate_handler(struct ctx *c)
 {
-	int rc = EIO;
+	int rc;
 
-	debug("migrate_handler fd %d events %x", c->device_state_fd, events);
+	if (c->device_state_fd < 0)
+		return;
 
-	if (events & EPOLLOUT)
-		rc = migrate_source(c, c->device_state_fd);
-	else if (events & EPOLLIN)
-		rc = migrate_target(c, c->device_state_fd);
+	debug("migrate_handler fd %d target %d",
+	      c->device_state_fd, c->migrate_target);
 
-	/* EPOLLHUP without EPOLLIN/EPOLLOUT, or EPOLLERR? Migration failed */
+	if (c->migrate_target)
+		rc = migrate_target(c, c->device_state_fd);
+	else
+		rc = migrate_source(c, c->device_state_fd);
 
 	migrate_close(c);
 
diff --git a/migrate.h b/migrate.h
index ac15d0f..9c49e78 100644
--- a/migrate.h
+++ b/migrate.h
@@ -91,6 +91,6 @@ struct migrate_version {
 void migrate_init(struct ctx *c);
 void migrate_close(struct ctx *c);
 void migrate_request(struct ctx *c, int fd, bool target);
-void migrate_handler(struct ctx *c, uint32_t events);
+void migrate_handler(struct ctx *c);
 
 #endif /* MIGRATE_H */
diff --git a/passt.c b/passt.c
index afd63c2..8682022 100644
--- a/passt.c
+++ b/passt.c
@@ -75,7 +75,8 @@ char *epoll_type_str[] = {
 	[EPOLL_TYPE_TAP_LISTEN]		= "listening qemu socket",
 	[EPOLL_TYPE_VHOST_CMD]		= "vhost-user command socket",
 	[EPOLL_TYPE_VHOST_KICK]		= "vhost-user kick socket",
-	[EPOLL_TYPE_DEVICE_STATE]	= "migration device state channel",
+	[EPOLL_TYPE_REPAIR_LISTEN]	= "TCP_REPAIR helper listening socket",
+	[EPOLL_TYPE_REPAIR]		= "TCP_REPAIR helper socket",
 };
 static_assert(ARRAY_SIZE(epoll_type_str) == EPOLL_NUM_TYPES,
 	      "epoll_type_str[] doesn't match enum epoll_type");
@@ -357,9 +358,6 @@ loop:
 		case EPOLL_TYPE_VHOST_KICK:
 			vu_kick_cb(c.vdev, ref, &now);
 			break;
-		case EPOLL_TYPE_DEVICE_STATE:
-			migrate_handler(&c, eventmask);
-			break;
 		default:
 			/* Can't happen */
 			ASSERT(0);
@@ -368,5 +366,7 @@ loop:
 
 	post_handler(&c, &now);
 
+	migrate_handler(&c);
+
 	goto loop;
 }
diff --git a/passt.h b/passt.h
index 0b8a26c..2255f18 100644
--- a/passt.h
+++ b/passt.h
@@ -237,6 +237,7 @@ struct ip6_ctx {
  * @vdev:		vhost-user device
  * @device_state_fd:	Device state migration channel
  * @device_state_result: Device state migration result
+ * @migrate_target:	Is this the target for next migration?
  */
 struct ctx {
 	enum passt_modes mode;
@@ -306,6 +307,7 @@ struct ctx {
 	/* Migration */
 	int device_state_fd;
 	int device_state_result;
+	bool migrate_target;
 };
 
 void proto_update_l2_buf(const unsigned char *eth_d,
-- 
@@ -237,6 +237,7 @@ struct ip6_ctx {
  * @vdev:		vhost-user device
  * @device_state_fd:	Device state migration channel
  * @device_state_result: Device state migration result
+ * @migrate_target:	Is this the target for next migration?
  */
 struct ctx {
 	enum passt_modes mode;
@@ -306,6 +307,7 @@ struct ctx {
 	/* Migration */
 	int device_state_fd;
 	int device_state_result;
+	bool migrate_target;
 };
 
 void proto_update_l2_buf(const unsigned char *eth_d,
-- 
2.43.0