qemu: live migration with non-shared storage for kvm

Support for live migration between hosts that do not share storage was
added to qemu-kvm release 0.12.1.
It supports two flags:
-b migration without shared storage with full disk copy
-i migration without shared storage with incremental copy (same base image
shared between source and destination).

I tested the live migration without shared storage (both flags) for native
and p2p with and without tunnelling.  I also verified that the fix doesn't
affect normal migration with shared storage.

include/libvirt/libvirt.h.in

@@ -411,6 +411,10 @@ typedef enum {
     VIR_MIGRATE_PERSIST_DEST      = (1 << 3), /* persist the VM on the destination */
     VIR_MIGRATE_UNDEFINE_SOURCE   = (1 << 4), /* undefine the VM on the source */
     VIR_MIGRATE_PAUSED            = (1 << 5), /* pause on remote side */
+    VIR_MIGRATE_NON_SHARED_DISK   = (1 << 6), /* migration with non-shared storage with full disk copy */
+    VIR_MIGRATE_NON_SHARED_INC    = (1 << 7), /* migration with non-shared storage with incremental copy */
+                                              /* (same base image shared between source and destination) */
+
 } virDomainMigrateFlags;
 
 /* Domain migration. */

src/qemu/qemu_driver.c

@@ -4958,7 +4958,9 @@ static int qemudDomainSaveFlag(virDomainPtr dom, const char *path,
     if (header.compressed == QEMUD_SAVE_FORMAT_RAW) {
         const char *args[] = { "cat", NULL };
         qemuDomainObjEnterMonitorWithDriver(driver, vm);
-        rc = qemuMonitorMigrateToFile(priv->mon, 1, args, path, offset);
+        rc = qemuMonitorMigrateToFile(priv->mon,
+                                      QEMU_MONITOR_MIGRATE_BACKGROUND,
+                                      args, path, offset);
         qemuDomainObjExitMonitorWithDriver(driver, vm);
     } else {
         const char *prog = qemudSaveCompressionTypeToString(header.compressed);
@@ -4968,7 +4970,9 @@ static int qemudDomainSaveFlag(virDomainPtr dom, const char *path,
             NULL
         };
         qemuDomainObjEnterMonitorWithDriver(driver, vm);
-        rc = qemuMonitorMigrateToFile(priv->mon, 1, args, path, offset);
+        rc = qemuMonitorMigrateToFile(priv->mon,
+                                      QEMU_MONITOR_MIGRATE_BACKGROUND,
+                                      args, path, offset);
         qemuDomainObjExitMonitorWithDriver(driver, vm);
     }
 
@@ -5286,9 +5290,10 @@ static int qemudDomainCoreDump(virDomainPtr dom,
     }
 
     qemuDomainObjEnterMonitorWithDriver(driver, vm);
-    ret = qemuMonitorMigrateToFile(priv->mon, 1, args, path, 0);
+    ret = qemuMonitorMigrateToFile(priv->mon,
+                                   QEMU_MONITOR_MIGRATE_BACKGROUND,
+                                   args, path, 0);
     qemuDomainObjExitMonitorWithDriver(driver, vm);
-
     if (ret < 0)
         goto endjob;
 
@@ -9885,13 +9890,17 @@ cleanup:
 static int doNativeMigrate(struct qemud_driver *driver,
                            virDomainObjPtr vm,
                            const char *uri,
-                           unsigned long flags ATTRIBUTE_UNUSED,
+                           unsigned int flags,
                            const char *dname ATTRIBUTE_UNUSED,
                            unsigned long resource)
 {
     int ret = -1;
     xmlURIPtr uribits = NULL;
     qemuDomainObjPrivatePtr priv = vm->privateData;
+    unsigned int background_flags = 0;
+
+    virCheckFlags(VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC,
+                  -1);
 
     /* Issue the migrate command. */
     if (STRPREFIX(uri, "tcp:") && !STRPREFIX(uri, "tcp://")) {
@@ -9919,7 +9928,14 @@ static int doNativeMigrate(struct qemud_driver *driver,
         goto cleanup;
     }
 
-    if (qemuMonitorMigrateToHost(priv->mon, 1, uribits->server, uribits->port) < 0) {
+    if (flags & VIR_MIGRATE_NON_SHARED_DISK)
+        background_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_DISK;
+
+    if (flags & VIR_MIGRATE_NON_SHARED_INC)
+        background_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_INC;
+
+    if (qemuMonitorMigrateToHost(priv->mon, background_flags, uribits->server,
+                                 uribits->port) < 0) {
         qemuDomainObjExitMonitorWithDriver(driver, vm);
         goto cleanup;
     }
@@ -10004,6 +10020,7 @@ static int doTunnelMigrate(virDomainPtr dom,
     unsigned long long qemuCmdFlags;
     int status;
     unsigned long long transferred, remaining, total;
+    unsigned int background_flags = QEMU_MONITOR_MIGRATE_BACKGROUND;
 
     /*
      * The order of operations is important here to avoid touching
@@ -10089,11 +10106,17 @@ static int doTunnelMigrate(virDomainPtr dom,
 
     /*   3. start migration on source */
     qemuDomainObjEnterMonitorWithDriver(driver, vm);
-    if (qemuCmdFlags & QEMUD_CMD_FLAG_MIGRATE_QEMU_UNIX)
+    if (flags & VIR_MIGRATE_NON_SHARED_DISK)
-        internalret = qemuMonitorMigrateToUnix(priv->mon, 1, unixfile);
+        background_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_DISK;
+    if (flags & VIR_MIGRATE_NON_SHARED_INC)
+        background_flags |= QEMU_MONITOR_MIGRATE_NON_SHARED_INC;
+    if (qemuCmdFlags & QEMUD_CMD_FLAG_MIGRATE_QEMU_UNIX){
+        internalret = qemuMonitorMigrateToUnix(priv->mon, background_flags,
+                                               unixfile);
+    }
     else if (qemuCmdFlags & QEMUD_CMD_FLAG_MIGRATE_QEMU_EXEC) {
         const char *args[] = { "nc", "-U", unixfile, NULL };
-        internalret = qemuMonitorMigrateToCommand(priv->mon, 1, args);
+        internalret = qemuMonitorMigrateToCommand(priv->mon, QEMU_MONITOR_MIGRATE_BACKGROUND, args);
     } else {
         internalret = -1;
     }

src/qemu/qemu_monitor.c

@@ -1161,7 +1161,7 @@ int qemuMonitorGetMigrationStatus(qemuMonitorPtr mon,
 
 
 int qemuMonitorMigrateToHost(qemuMonitorPtr mon,
-                             int background,
+                             unsigned int background,
                              const char *hostname,
                              int port)
 {
@@ -1178,7 +1178,7 @@ int qemuMonitorMigrateToHost(qemuMonitorPtr mon,
 
 
 int qemuMonitorMigrateToCommand(qemuMonitorPtr mon,
-                                int background,
+                                unsigned int background,
                                 const char * const *argv)
 {
     int ret;
@@ -1193,7 +1193,7 @@ int qemuMonitorMigrateToCommand(qemuMonitorPtr mon,
 }
 
 int qemuMonitorMigrateToFile(qemuMonitorPtr mon,
-                             int background,
+                             unsigned int background,
                              const char * const *argv,
                              const char *target,
                              unsigned long long offset)
@@ -1217,7 +1217,7 @@ int qemuMonitorMigrateToFile(qemuMonitorPtr mon,
 }
 
 int qemuMonitorMigrateToUnix(qemuMonitorPtr mon,
-                             int background,
+                             unsigned int background,
                              const char *unixfile)
 {
     int ret;

src/qemu/qemu_monitor.h

@@ -241,25 +241,32 @@ int qemuMonitorGetMigrationStatus(qemuMonitorPtr mon,
                                   unsigned long long *remaining,
                                   unsigned long long *total);
 
+typedef enum {
+  QEMU_MONITOR_MIGRATE_BACKGROUND 		= 1 << 0,
+  QEMU_MONITOR_MIGRATE_NON_SHARED_DISK  = 1 << 1, /* migration with non-shared storage with full disk copy */
+  QEMU_MONITOR_MIGRATE_NON_SHARED_INC   = 1 << 2, /* migration with non-shared storage with incremental copy */
+  QEMU_MONITOR_MIGRATION_FLAGS_LAST
+} QEMU_MONITOR_MIGRATE;
+
 int qemuMonitorMigrateToHost(qemuMonitorPtr mon,
-                             int background,
+                             unsigned int background,
                              const char *hostname,
                              int port);
 
 int qemuMonitorMigrateToCommand(qemuMonitorPtr mon,
-                                int background,
+                                unsigned int background,
                                 const char * const *argv);
 
 # define QEMU_MONITOR_MIGRATE_TO_FILE_BS 512llu
 
 int qemuMonitorMigrateToFile(qemuMonitorPtr mon,
-                             int background,
+                             unsigned int background,
                              const char * const *argv,
                              const char *target,
                              unsigned long long offset);
 
 int qemuMonitorMigrateToUnix(qemuMonitorPtr mon,
-                             int background,
+                             unsigned int background,
                              const char *unixfile);
 
 int qemuMonitorMigrateCancel(qemuMonitorPtr mon);

src/qemu/qemu_monitor_text.c

@@ -38,6 +38,7 @@
 #include "driver.h"
 #include "datatypes.h"
 #include "virterror_internal.h"
+#include "buf.h"
 
 #define VIR_FROM_THIS VIR_FROM_QEMU
 
@@ -1125,26 +1126,32 @@ cleanup:
 
 
 static int qemuMonitorTextMigrate(qemuMonitorPtr mon,
-                                  int background,
+                                  unsigned int background,
                                   const char *dest)
 {
     char *cmd = NULL;
     char *info = NULL;
     int ret = -1;
     char *safedest = qemuMonitorEscapeArg(dest);
-    const char *extra;
+    virBuffer extra = VIR_BUFFER_INITIALIZER;
 
     if (!safedest) {
         virReportOOMError();
         return -1;
     }
 
-    if (background)
+    if (background & QEMU_MONITOR_MIGRATE_BACKGROUND)
-        extra = "-d ";
+        virBufferAddLit(&extra, " -d");
-    else
+    if (background & QEMU_MONITOR_MIGRATE_NON_SHARED_DISK)
-        extra = " ";
+        virBufferAddLit(&extra, " -b");
-
+    if (background & QEMU_MONITOR_MIGRATE_NON_SHARED_INC)
-    if (virAsprintf(&cmd, "migrate %s\"%s\"", extra, safedest) < 0) {
+        virBufferAddLit(&extra, " -i");
+    if (virBufferError(&extra)) {
+        virBufferFreeAndReset(&extra);
+        virReportOOMError();
+        return -1;
+    }
+    if (virAsprintf(&cmd, "migrate %s\"%s\"", virBufferContentAndReset(&extra), safedest) < 0) {
         virReportOOMError();
         goto cleanup;
     }
@@ -1180,7 +1187,7 @@ cleanup:
 }
 
 int qemuMonitorTextMigrateToHost(qemuMonitorPtr mon,
-                                 int background,
+                                 unsigned int background,
                                  const char *hostname,
                                  int port)
 {
@@ -1201,7 +1208,7 @@ int qemuMonitorTextMigrateToHost(qemuMonitorPtr mon,
 
 
 int qemuMonitorTextMigrateToCommand(qemuMonitorPtr mon,
-                                    int background,
+                                    unsigned int background,
                                     const char * const *argv)
 {
     char *argstr;
@@ -1228,7 +1235,7 @@ cleanup:
 }
 
 int qemuMonitorTextMigrateToFile(qemuMonitorPtr mon,
-                                 int background,
+                                 unsigned int background,
                                  const char * const *argv,
                                  const char *target,
                                  unsigned long long offset)
@@ -1269,7 +1276,7 @@ cleanup:
 }
 
 int qemuMonitorTextMigrateToUnix(qemuMonitorPtr mon,
-                                 int background,
+                                 unsigned int background,
                                  const char *unixfile)
 {
     char *dest = NULL;

src/qemu/qemu_monitor_text.h

@@ -93,22 +93,22 @@ int qemuMonitorTextGetMigrationStatus(qemuMonitorPtr mon,
                                       unsigned long long *total);
 
 int qemuMonitorTextMigrateToHost(qemuMonitorPtr mon,
-                                 int background,
+                                 unsigned int background,
                                  const char *hostname,
                                  int port);
 
 int qemuMonitorTextMigrateToCommand(qemuMonitorPtr mon,
-                                    int background,
+                                    unsigned int background,
                                     const char * const *argv);
 
 int qemuMonitorTextMigrateToFile(qemuMonitorPtr mon,
-                                 int background,
+                                 unsigned int background,
                                  const char * const *argv,
                                  const char *target,
                                  unsigned long long offset);
 
 int qemuMonitorTextMigrateToUnix(qemuMonitorPtr mon,
-                                 int background,
+                                 unsigned int background,
                                  const char *unixfile);
 
 int qemuMonitorTextMigrateCancel(qemuMonitorPtr mon);

tools/virsh.c

@@ -2851,6 +2851,8 @@ static const vshCmdOptDef opts_migrate[] = {
     {"persistent", VSH_OT_BOOL, 0, N_("persist VM on destination")},
     {"undefinesource", VSH_OT_BOOL, 0, N_("undefine VM on source")},
     {"suspend", VSH_OT_BOOL, 0, N_("do not restart the domain on the destination host")},
+    {"copy-storage-all", VSH_OT_BOOL, 0, N_("migration with non-shared storage with full disk copy")},
+    {"copy-storage-inc", VSH_OT_BOOL, 0, N_("migration with non-shared storage with incremental copy (same base image shared between source and destination)")},
     {"domain", VSH_OT_DATA, VSH_OFLAG_REQ, N_("domain name, id or uuid")},
     {"desturi", VSH_OT_DATA, VSH_OFLAG_REQ, N_("connection URI of the destination host")},
     {"migrateuri", VSH_OT_DATA, 0, N_("migration URI, usually can be omitted")},
@@ -2898,6 +2900,12 @@ cmdMigrate (vshControl *ctl, const vshCmd *cmd)
     if (vshCommandOptBool (cmd, "suspend"))
         flags |= VIR_MIGRATE_PAUSED;
 
+    if (vshCommandOptBool (cmd, "copy-storage-all"))
+        flags |= VIR_MIGRATE_NON_SHARED_DISK;
+
+    if (vshCommandOptBool (cmd, "copy-storage-inc"))
+        flags |= VIR_MIGRATE_NON_SHARED_INC;
+
     if ((flags & VIR_MIGRATE_PEER2PEER) ||
         vshCommandOptBool (cmd, "direct")) {
         /* For peer2peer migration or direct migration we only expect one URI