[PATCH 5/5] migrate: Don't handle the migration channel through epoll

[David Gibson <david@gibson.dropbear.id.au>]

Currently, once a migration device state fd is assigned, we wait for
EPOLLIN or EPOLLOUT events on it to actually perform the migration.  Change
it so that once a migration is requested it we complete it synchronously
at the end of the current epoll cycle.  This has several advantages:

  1. It makes it clear that everything about the migration must be dealt
     with at once, not split between multiple epoll events on the channel
  2. It ensures the migration always takes place between epoll cycles,
     rather than, for example, between handling TCP events and their
     deferred handling in post_handler().
  3. It reduces code setting up the epoll watch on the fd.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 epoll_type.h |  2 --
 migrate.c    | 42 +++++++++++-------------------------------
 migrate.h    |  2 +-
 passt.c      |  6 ++----
 passt.h      |  2 ++
 5 files changed, 16 insertions(+), 38 deletions(-)

diff --git a/epoll_type.h b/epoll_type.h
index b981d30e..7f2a1217 100644
--- a/epoll_type.h
+++ b/epoll_type.h
@@ -40,8 +40,6 @@ enum epoll_type {
 	EPOLL_TYPE_VHOST_CMD,
 	/* vhost-user kick event socket */
 	EPOLL_TYPE_VHOST_KICK,
-	/* migration device state channel */
-	EPOLL_TYPE_DEVICE_STATE,
 	/* TCP_REPAIR helper listening socket */
 	EPOLL_TYPE_REPAIR_LISTEN,
 	/* TCP_REPAIR helper socket */
diff --git a/migrate.c b/migrate.c
index 04df6822..4279d1f8 100644
--- a/migrate.c
+++ b/migrate.c
@@ -327,26 +327,6 @@ static int migrate_target(struct ctx *c, int fd)
 	return rc;
 }
 
-/**
- * set_migration_watch() - Add the migration file descriptor to epoll
- * @c:		Execution context
- * @fd:		File descriptor to add
- * @target:	Are we the target of the migration?
- */
-static void set_migration_watch(const struct ctx *c, int fd, bool target)
-{
-	union epoll_ref ref = {
-		.type = EPOLL_TYPE_DEVICE_STATE,
-		.fd = fd,
-	 };
-	struct epoll_event ev = { 0 };
-
-	ev.data.u64 = ref.u64;
-	ev.events = target ? EPOLLIN : EPOLLOUT;
-
-	epoll_ctl(c->epollfd, EPOLL_CTL_ADD, ref.fd, &ev);
-}
-
 /**
  * migrate_init() - Set up things necessary for migration
  * @c:		Execution context
@@ -387,27 +367,27 @@ void migrate_request(struct ctx *c, int fd, bool target)
 		migrate_close(c);
 
 	c->device_state_fd = fd;
-	set_migration_watch(c, c->device_state_fd, target);
-
+	c->migrate_target = target;
 }
 
 /**
  * migrate_handler() - Send/receive passt internal state to/from QEMU
  * @c:		Execution context
- * @events:	epoll events
  */
-void migrate_handler(struct ctx *c, uint32_t events)
+void migrate_handler(struct ctx *c)
 {
-	int rc = EIO;
+	int rc;
 
-	debug("migrate_handler fd %d events %x", c->device_state_fd, events);
+	if (c->device_state_fd < 0)
+		return;
 
-	if (events & EPOLLOUT)
-		rc = migrate_source(c, c->device_state_fd);
-	else if (events & EPOLLIN)
-		rc = migrate_target(c, c->device_state_fd);
+	debug("migrate_handler fd %d target %d",
+	      c->device_state_fd, c->migrate_target);
 
-	/* EPOLLHUP without EPOLLIN/EPOLLOUT, or EPOLLERR? Migration failed */
+	if (c->migrate_target)
+		rc = migrate_target(c, c->device_state_fd);
+	else
+		rc = migrate_source(c, c->device_state_fd);
 
 	migrate_close(c);
 
diff --git a/migrate.h b/migrate.h
index b97e55e7..3432940d 100644
--- a/migrate.h
+++ b/migrate.h
@@ -79,6 +79,6 @@ struct migrate_target_handlers {
 void migrate_init(struct ctx *c);
 void migrate_close(struct ctx *c);
 void migrate_request(struct ctx *c, int fd, bool target);
-void migrate_handler(struct ctx *c, uint32_t events);
+void migrate_handler(struct ctx *c);
 
 #endif /* MIGRATE_H */
diff --git a/passt.c b/passt.c
index 3c3a3313..1938290f 100644
--- a/passt.c
+++ b/passt.c
@@ -76,7 +76,6 @@ char *epoll_type_str[] = {
 	[EPOLL_TYPE_TAP_LISTEN]		= "listening qemu socket",
 	[EPOLL_TYPE_VHOST_CMD]		= "vhost-user command socket",
 	[EPOLL_TYPE_VHOST_KICK]		= "vhost-user kick socket",
-	[EPOLL_TYPE_DEVICE_STATE]	= "migration device state channel",
 	[EPOLL_TYPE_REPAIR_LISTEN]	= "TCP_REPAIR helper listening socket",
 	[EPOLL_TYPE_REPAIR]		= "TCP_REPAIR helper socket",
 };
@@ -360,9 +359,6 @@ loop:
 		case EPOLL_TYPE_VHOST_KICK:
 			vu_kick_cb(c.vdev, ref, &now);
 			break;
-		case EPOLL_TYPE_DEVICE_STATE:
-			migrate_handler(&c, eventmask);
-			break;
 		case EPOLL_TYPE_REPAIR_LISTEN:
 			repair_listen_handler(&c, eventmask);
 			break;
@@ -377,5 +373,7 @@ loop:
 
 	post_handler(&c, &now);
 
+	migrate_handler(&c);
+
 	goto loop;
 }
diff --git a/passt.h b/passt.h
index 5992cbeb..4189a4a5 100644
--- a/passt.h
+++ b/passt.h
@@ -241,6 +241,7 @@ struct ip6_ctx {
  * @vdev:		vhost-user device
  * @device_state_fd:	Device state migration channel
  * @device_state_result: Device state migration result
+ * @migrate_target:	Is this the target for next migration?
  */
 struct ctx {
 	enum passt_modes mode;
@@ -313,6 +314,7 @@ struct ctx {
 	/* Migration */
 	int device_state_fd;
 	int device_state_result;
+	bool migrate_target;
 };
 
 void proto_update_l2_buf(const unsigned char *eth_d,
-- 
@@ -241,6 +241,7 @@ struct ip6_ctx {
  * @vdev:		vhost-user device
  * @device_state_fd:	Device state migration channel
  * @device_state_result: Device state migration result
+ * @migrate_target:	Is this the target for next migration?
  */
 struct ctx {
 	enum passt_modes mode;
@@ -313,6 +314,7 @@ struct ctx {
 	/* Migration */
 	int device_state_fd;
 	int device_state_result;
+	bool migrate_target;
 };
 
 void proto_update_l2_buf(const unsigned char *eth_d,
-- 
2.48.1