libxl: Receive migration data in a thread

The libxl driver receives migration data within an IO callback invoked
by the event loop, effectively disabling the event loop while migration
occurs.

This patch moves receving of the migration data to a thread.  The
incoming connection is still accepted in the IO callback, but control
is immediately returned to the event loop after spawning the thread.

Signed-off-by: Jim Fehlig <jfehlig@suse.com>

src/libxl/libxl_migration.c

@@ -35,6 +35,7 @@
 #include "vircommand.h"
 #include "virstring.h"
 #include "virobject.h"
+#include "virthread.h"
 #include "rpc/virnetsocket.h"
 #include "libxl_domain.h"
 #include "libxl_driver.h"
@@ -48,6 +49,7 @@ VIR_LOG_INIT("libxl.libxl_migration");
 typedef struct _libxlMigrationDstArgs {
     virObject parent;
 
+    int recvfd;
     virConnectPtr conn;
     virDomainObjPtr vm;
     unsigned int flags;
@@ -82,31 +84,18 @@ libxlMigrationDstArgsOnceInit(void)
 VIR_ONCE_GLOBAL_INIT(libxlMigrationDstArgs)
 
 static void
-libxlDoMigrateReceive(virNetSocketPtr sock,
+libxlDoMigrateReceive(void *opaque)
-                      int events ATTRIBUTE_UNUSED,
-                      void *opaque)
 {
     libxlMigrationDstArgs *args = opaque;
-    virConnectPtr conn = args->conn;
     virDomainObjPtr vm = args->vm;
     virNetSocketPtr *socks = args->socks;
     size_t nsocks = args->nsocks;
     bool paused = args->flags & VIR_MIGRATE_PAUSED;
-    libxlDriverPrivatePtr driver = conn->privateData;
+    libxlDriverPrivatePtr driver = args->conn->privateData;
-    virNetSocketPtr client_sock;
+    int recvfd = args->recvfd;
-    int recvfd = -1;
     size_t i;
     int ret;
 
-    if (virNetSocketAccept(sock, &client_sock) < 0) {
-        virReportError(VIR_ERR_OPERATION_INVALID, "%s",
-                       _("Fail to accept migration connection"));
-        goto cleanup;
-    }
-    VIR_DEBUG("Accepted migration connection\n");
-    recvfd = virNetSocketDupFD(client_sock, true);
-    virObjectUnref(client_sock);
-
     virObjectLock(vm);
     ret = libxlDomainStart(driver, vm, paused, recvfd);
     virObjectUnlock(vm);
@@ -114,7 +103,59 @@ libxlDoMigrateReceive(virNetSocketPtr sock,
     if (ret < 0 && !vm->persistent)
         virDomainObjListRemove(driver->domains, vm);
 
- cleanup:
+    /* Remove all listen socks from event handler, and close them. */
+    for (i = 0; i < nsocks; i++) {
+        virNetSocketUpdateIOCallback(socks[i], 0);
+        virNetSocketRemoveIOCallback(socks[i]);
+        virNetSocketClose(socks[i]);
+        virObjectUnref(socks[i]);
+        socks[i] = NULL;
+    }
+    args->nsocks = 0;
+    VIR_FORCE_CLOSE(recvfd);
+}
+
+
+static void
+libxlMigrateReceive(virNetSocketPtr sock,
+                    int events ATTRIBUTE_UNUSED,
+                    void *opaque)
+{
+    libxlMigrationDstArgs *args = opaque;
+    virNetSocketPtr *socks = args->socks;
+    size_t nsocks = args->nsocks;
+    virNetSocketPtr client_sock;
+    int recvfd = -1;
+    virThread thread;
+    size_t i;
+
+    /* Accept migration connection */
+    virNetSocketAccept(sock, &client_sock);
+    if (client_sock == NULL) {
+        virReportError(VIR_ERR_OPERATION_FAILED, "%s",
+                       _("Failed to accept migration connection"));
+        goto fail;
+    }
+    VIR_DEBUG("Accepted migration connection."
+              "  Spawing thread to process migration data");
+    recvfd = virNetSocketDupFD(client_sock, true);
+    virObjectUnref(client_sock);
+
+    /*
+     * Avoid blocking the event loop.  Start a thread to receive
+     * the migration data
+     */
+    args->recvfd = recvfd;
+    if (virThreadCreate(&thread, false,
+                        libxlDoMigrateReceive, args) < 0) {
+        virReportError(VIR_ERR_OPERATION_FAILED, "%s",
+                       _("Failed to create thread for receiving migration data"));
+        goto fail;
+    }
+
+    return;
+
+ fail:
     /* Remove all listen socks from event handler, and close them. */
     for (i = 0; i < nsocks; i++) {
         virNetSocketUpdateIOCallback(socks[i], 0);
@@ -376,7 +417,7 @@ libxlDomainMigrationPrepare(virConnectPtr dconn,
 
         if (virNetSocketAddIOCallback(socks[i],
                                       VIR_EVENT_HANDLE_READABLE,
-                                      libxlDoMigrateReceive,
+                                      libxlMigrateReceive,
                                       args,
                                       virObjectFreeCallback) < 0)
             continue;