qemu: Allow forcing VFIO when computing memlock limit

With NVMe disks, one can start a blockjob with a NVMe disk
that is not visible in domain XML (at least right away). Usually,
it's fairly easy to override this limitation of
qemuDomainGetMemLockLimitBytes() - for instance for hostdevs we
temporarily add the device to domain def, let the function
calculate the limit and then remove the device. But it's not so
easy with virStorageSourcePtr - in some cases they don't
necessarily are attached to a disk. And even if they are it's
done later in the process and frankly, I find it too complicated
to be able to use the simple trick we use with hostdevs.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
Reviewed-by: Cole Robinson <crobinso@redhat.com>

src/qemu/qemu_command.c

@@ -10009,7 +10009,7 @@ qemuBuildCommandLine(virQEMUDriverPtr driver,
 
     /* In some situations, eg. VFIO passthrough, QEMU might need to lock a
      * significant amount of memory, so we need to set the limit accordingly */
-    virCommandSetMaxMemLock(cmd, qemuDomainGetMemLockLimitBytes(def));
+    virCommandSetMaxMemLock(cmd, qemuDomainGetMemLockLimitBytes(def, false));
 
     if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MSG_TIMESTAMP) &&
         cfg->logTimestamp)

src/qemu/qemu_domain.c

@@ -12836,12 +12836,14 @@ ppc64VFIODeviceIsNV2Bridge(const char *device)
 /**
  * getPPC64MemLockLimitBytes:
  * @def: domain definition
+ * @forceVFIO: force VFIO usage
  *
  * A PPC64 helper that calculates the memory locking limit in order for
  * the guest to operate properly.
  */
 static unsigned long long
-getPPC64MemLockLimitBytes(virDomainDefPtr def)
+getPPC64MemLockLimitBytes(virDomainDefPtr def,
+                          bool forceVFIO)
 {
     unsigned long long memKB = 0;
     unsigned long long baseLimit = 0;
@@ -12932,7 +12934,7 @@ getPPC64MemLockLimitBytes(virDomainDefPtr def)
         passthroughLimit = maxMemory +
                            128 * (1ULL<<30) / 512 * nPCIHostBridges +
                            8192;
-    } else if (usesVFIO) {
+    } else if (usesVFIO || forceVFIO) {
         /* For regular (non-NVLink2 present) VFIO passthrough, the value
          * of passthroughLimit is:
          *
@@ -12970,16 +12972,20 @@ getPPC64MemLockLimitBytes(virDomainDefPtr def)
 /**
  * qemuDomainGetMemLockLimitBytes:
  * @def: domain definition
+ * @forceVFIO: force VFIO calculation
  *
  * Calculate the memory locking limit that needs to be set in order for
  * the guest to operate properly. The limit depends on a number of factors,
  * including certain configuration options and less immediately apparent ones
  * such as the guest architecture or the use of certain devices.
+ * The @forceVFIO argument can be used to tell this function will use VFIO even
+ * though @def doesn't indicates so right now.
  *
  * Returns: the memory locking limit, or 0 if setting the limit is not needed
  */
 unsigned long long
-qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
+qemuDomainGetMemLockLimitBytes(virDomainDefPtr def,
+                               bool forceVFIO)
 {
     unsigned long long memKB = 0;
     bool usesVFIO = false;
@@ -13000,7 +13006,7 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
         return VIR_DOMAIN_MEMORY_PARAM_UNLIMITED;
 
     if (ARCH_IS_PPC64(def->os.arch) && def->virtType == VIR_DOMAIN_VIRT_KVM)
-        return getPPC64MemLockLimitBytes(def);
+        return getPPC64MemLockLimitBytes(def, forceVFIO);
 
     /* For device passthrough using VFIO the guest memory and MMIO memory
      * regions need to be locked persistent in order to allow DMA.
@@ -13020,6 +13026,7 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
      *
      * Note that this may not be valid for all platforms.
      */
+    if (!forceVFIO) {
         for (i = 0; i < def->nhostdevs; i++) {
             if (virHostdevIsVFIODevice(def->hostdevs[i]) ||
                 virHostdevIsMdevDevice(def->hostdevs[i])) {
@@ -13030,8 +13037,9 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
 
         if (virDomainDefHasNVMeDisk(def))
             usesVFIO = true;
+    }
 
-    if (usesVFIO)
+    if (usesVFIO || forceVFIO)
         memKB = virDomainDefGetMemoryTotal(def) + 1024 * 1024;
 
  done:
@@ -13042,9 +13050,12 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
 /**
  * qemuDomainAdjustMaxMemLock:
  * @vm: domain
+ * @forceVFIO: apply VFIO requirements even if vm's def doesn't require it
  *
  * Adjust the memory locking limit for the QEMU process associated to @vm, in
- * order to comply with VFIO or architecture requirements.
+ * order to comply with VFIO or architecture requirements. If @forceVFIO is
+ * true then the limit is changed even if nothing in @vm's definition indicates
+ * so.
  *
  * The limit will not be changed unless doing so is needed; the first time
  * the limit is changed, the original (default) limit is stored in @vm and
@@ -13054,12 +13065,13 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
  * Returns: 0 on success, <0 on failure
  */
 int
-qemuDomainAdjustMaxMemLock(virDomainObjPtr vm)
+qemuDomainAdjustMaxMemLock(virDomainObjPtr vm,
+                           bool forceVFIO)
 {
     unsigned long long bytes = 0;
     int ret = -1;
 
-    bytes = qemuDomainGetMemLockLimitBytes(vm->def);
+    bytes = qemuDomainGetMemLockLimitBytes(vm->def, forceVFIO);
 
     if (bytes) {
         /* If this is the first time adjusting the limit, save the current
@@ -13108,7 +13120,7 @@ qemuDomainAdjustMaxMemLockHostdev(virDomainObjPtr vm,
     int ret = 0;
 
     vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
-    if (qemuDomainAdjustMaxMemLock(vm) < 0)
+    if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
         ret = -1;
 
     vm->def->hostdevs[--(vm->def->nhostdevs)] = NULL;

src/qemu/qemu_domain.h

@@ -975,8 +975,10 @@ bool qemuDomainSupportsPCI(virDomainDefPtr def,
 
 void qemuDomainUpdateCurrentMemorySize(virDomainObjPtr vm);
 
-unsigned long long qemuDomainGetMemLockLimitBytes(virDomainDefPtr def);
-int qemuDomainAdjustMaxMemLock(virDomainObjPtr vm);
+unsigned long long qemuDomainGetMemLockLimitBytes(virDomainDefPtr def,
+                                                  bool forceVFIO);
+int qemuDomainAdjustMaxMemLock(virDomainObjPtr vm,
+                               bool forceVFIO);
 int qemuDomainAdjustMaxMemLockHostdev(virDomainObjPtr vm,
                                       virDomainHostdevDefPtr hostdev);
 

src/qemu/qemu_hotplug.c

@@ -1646,7 +1646,7 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver,
     if (teardowndevice &&
         qemuDomainNamespaceTeardownHostdev(vm, hostdev) < 0)
         VIR_WARN("Unable to remove host device from /dev");
-    if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm) < 0)
+    if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm, false) < 0)
         VIR_WARN("Unable to reset maximum locked memory on hotplug fail");
 
     if (releaseaddr)
@@ -2383,7 +2383,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
     if (virDomainMemoryInsert(vm->def, mem) < 0)
         goto cleanup;
 
-    if (qemuDomainAdjustMaxMemLock(vm) < 0)
+    if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
         goto removedef;
 
     qemuDomainObjEnterMonitor(driver, vm);
@@ -2454,7 +2454,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
 
     /* reset the mlock limit */
     virErrorPreserveLast(&orig_err);
-    ignore_value(qemuDomainAdjustMaxMemLock(vm));
+    ignore_value(qemuDomainAdjustMaxMemLock(vm, false));
     virErrorRestore(&orig_err);
 
     goto audit;
@@ -2857,7 +2857,7 @@ qemuDomainAttachMediatedDevice(virQEMUDriverPtr driver,
     ret = 0;
  cleanup:
     if (ret < 0) {
-        if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm) < 0)
+        if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm, false) < 0)
             VIR_WARN("Unable to reset maximum locked memory on hotplug fail");
         if (teardowncgroup && qemuTeardownHostdevCgroup(vm, hostdev) < 0)
             VIR_WARN("Unable to remove host device cgroup ACL on hotplug fail");
@@ -4356,7 +4356,7 @@ qemuDomainRemoveMemoryDevice(virQEMUDriverPtr driver,
     ignore_value(qemuProcessRefreshBalloonState(driver, vm, QEMU_ASYNC_JOB_NONE));
 
     /* decrease the mlock limit after memory unplug if necessary */
-    ignore_value(qemuDomainAdjustMaxMemLock(vm));
+    ignore_value(qemuDomainAdjustMaxMemLock(vm, false));
 
     return 0;
 }
@@ -4483,7 +4483,7 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver,
         qemuDomainRemovePCIHostDevice(driver, vm, hostdev);
         /* QEMU might no longer need to lock as much memory, eg. we just
          * detached the last VFIO device, so adjust the limit here */
-        if (qemuDomainAdjustMaxMemLock(vm) < 0)
+        if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
             VIR_WARN("Failed to adjust locked memory limit");
         break;
     case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:

tests/qemumemlocktest.c

@@ -42,7 +42,7 @@ testCompareMemLock(const void *data)
         goto cleanup;
     }
 
-    ret = virTestCompareToULL(info->memlock, qemuDomainGetMemLockLimitBytes(def));
+    ret = virTestCompareToULL(info->memlock, qemuDomainGetMemLockLimitBytes(def, false));
 
  cleanup:
     virDomainDefFree(def);