qemu: RDMA migration support

This patch adds support for RDMA protocol in migration URIs.

USAGE: $ virsh migrate --live --migrateuri rdma://hostname domain qemu+ssh://hostname/system

Since libvirt runs QEMU in a pretty restricted environment, several
files needs to be added to cgroup_device_acl (in qemu.conf) for QEMU to
be able to access the host's infiniband hardware. Full documenation of
the feature can be found on QEMU wiki:
http://wiki.qemu.org/Features/RDMALiveMigration

Signed-off-by: Michael R. Hines <mrhines@us.ibm.com>
Signed-off-by: Jiri Denemark <jdenemar@redhat.com>

src/qemu/qemu.conf

@@ -274,6 +274,14 @@
 #    "/dev/ptmx", "/dev/kvm", "/dev/kqemu",
 #    "/dev/rtc","/dev/hpet", "/dev/vfio/vfio"
 #]
+#
+# RDMA migration requires the following extra files to be added to the list:
+#   "/dev/infiniband/rdma_cm",
+#   "/dev/infiniband/issm0",
+#   "/dev/infiniband/issm1",
+#   "/dev/infiniband/umad0",
+#   "/dev/infiniband/umad1",
+#   "/dev/infiniband/uverbs0"
 
 
 # The default format for Qemu/KVM guest save images is raw; that is, the

src/qemu/qemu_command.c

@@ -9476,6 +9476,14 @@ qemuBuildCommandLine(virConnectPtr conn,
                 goto error;
             }
             virCommandAddArg(cmd, migrateFrom);
+        } else if (STRPREFIX(migrateFrom, "rdma")) {
+            if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) {
+                virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
+                               _("incoming RDMA migration is not supported "
+                                 "with this QEMU binary"));
+                goto error;
+            }
+            virCommandAddArg(cmd, migrateFrom);
         } else if (STREQ(migrateFrom, "stdio")) {
             if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD)) {
                 virCommandAddArgFormat(cmd, "fd:%d", migrateFd);

src/qemu/qemu_migration.c

@@ -56,6 +56,7 @@
 #include "virhook.h"
 #include "virstring.h"
 #include "virtypedparam.h"
+#include "virprocess.h"
 
 #define VIR_FROM_THIS VIR_FROM_QEMU
 
@@ -2653,6 +2654,13 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
                                        QEMU_MIGRATION_COOKIE_NBD)))
         goto cleanup;
 
+    if (STREQ(protocol, "rdma") && !vm->def->mem.hard_limit) {
+        virReportError(VIR_ERR_OPERATION_INVALID, "%s",
+                       _("cannot start RDMA migration with no memory hard "
+                         "limit set"));
+        goto cleanup;
+    }
+
     if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
         goto cleanup;
     qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PREPARE);
@@ -2696,6 +2704,11 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
                                     QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
         goto stop;
 
+    if (STREQ(protocol, "rdma") &&
+        virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) {
+        goto stop;
+    }
+
     if (mig->lockState) {
         VIR_DEBUG("Received lockstate %s", mig->lockState);
         VIR_FREE(priv->lockState);
@@ -2926,7 +2939,8 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver,
         if (!(uri = qemuMigrationParseURI(uri_in, &well_formed_uri)))
             goto cleanup;
 
-        if (STRNEQ(uri->scheme, "tcp")) {
+        if (STRNEQ(uri->scheme, "tcp") &&
+            STRNEQ(uri->scheme, "rdma")) {
             virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED,
                            _("unsupported scheme %s in migration URI %s"),
                            uri->scheme, uri_in);
@@ -3547,6 +3561,11 @@ qemuMigrationRun(virQEMUDriverPtr driver,
 
     switch (spec->destType) {
     case MIGRATION_DEST_HOST:
+        if (STREQ(spec->dest.host.protocol, "rdma") &&
+            virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) {
+            qemuDomainObjExitMonitor(driver, vm);
+            goto cleanup;
+        }
         ret = qemuMonitorMigrateToHost(priv->mon, migrate_flags,
                                        spec->dest.host.protocol,
                                        spec->dest.host.name,
@@ -3719,7 +3738,23 @@ static int doNativeMigrate(virQEMUDriverPtr driver,
     if (!(uribits = qemuMigrationParseURI(uri, NULL)))
         return -1;
 
-    if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD))
+    if (STREQ(uribits->scheme, "rdma")) {
+        if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) {
+            virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
+                           _("outgoing RDMA migration is not supported "
+                             "with this QEMU binary"));
+            return -1;
+        }
+        if (!vm->def->mem.hard_limit) {
+            virReportError(VIR_ERR_OPERATION_INVALID, "%s",
+                           _("cannot start RDMA migration with no memory hard "
+                             "limit set"));
+            return -1;
+        }
+    }
+
+    if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD) &&
+        STRNEQ(uribits->scheme, "rdma"))
         spec.destType = MIGRATION_DEST_CONNECT_HOST;
     else
         spec.destType = MIGRATION_DEST_HOST;