mirror of
https://gitlab.com/libvirt/libvirt.git
synced 2024-10-05 05:45:46 +00:00
qemu: Allow forcing VFIO when computing memlock limit
With NVMe disks, one can start a blockjob with a NVMe disk that is not visible in domain XML (at least right away). Usually, it's fairly easy to override this limitation of qemuDomainGetMemLockLimitBytes() - for instance for hostdevs we temporarily add the device to domain def, let the function calculate the limit and then remove the device. But it's not so easy with virStorageSourcePtr - in some cases they don't necessarily are attached to a disk. And even if they are it's done later in the process and frankly, I find it too complicated to be able to use the simple trick we use with hostdevs. Signed-off-by: Michal Privoznik <mprivozn@redhat.com> Reviewed-by: Cole Robinson <crobinso@redhat.com>
This commit is contained in:
parent
da27be1b09
commit
6edb4321b2
@ -10009,7 +10009,7 @@ qemuBuildCommandLine(virQEMUDriverPtr driver,
|
|||||||
|
|
||||||
/* In some situations, eg. VFIO passthrough, QEMU might need to lock a
|
/* In some situations, eg. VFIO passthrough, QEMU might need to lock a
|
||||||
* significant amount of memory, so we need to set the limit accordingly */
|
* significant amount of memory, so we need to set the limit accordingly */
|
||||||
virCommandSetMaxMemLock(cmd, qemuDomainGetMemLockLimitBytes(def));
|
virCommandSetMaxMemLock(cmd, qemuDomainGetMemLockLimitBytes(def, false));
|
||||||
|
|
||||||
if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MSG_TIMESTAMP) &&
|
if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MSG_TIMESTAMP) &&
|
||||||
cfg->logTimestamp)
|
cfg->logTimestamp)
|
||||||
|
@ -12836,12 +12836,14 @@ ppc64VFIODeviceIsNV2Bridge(const char *device)
|
|||||||
/**
|
/**
|
||||||
* getPPC64MemLockLimitBytes:
|
* getPPC64MemLockLimitBytes:
|
||||||
* @def: domain definition
|
* @def: domain definition
|
||||||
|
* @forceVFIO: force VFIO usage
|
||||||
*
|
*
|
||||||
* A PPC64 helper that calculates the memory locking limit in order for
|
* A PPC64 helper that calculates the memory locking limit in order for
|
||||||
* the guest to operate properly.
|
* the guest to operate properly.
|
||||||
*/
|
*/
|
||||||
static unsigned long long
|
static unsigned long long
|
||||||
getPPC64MemLockLimitBytes(virDomainDefPtr def)
|
getPPC64MemLockLimitBytes(virDomainDefPtr def,
|
||||||
|
bool forceVFIO)
|
||||||
{
|
{
|
||||||
unsigned long long memKB = 0;
|
unsigned long long memKB = 0;
|
||||||
unsigned long long baseLimit = 0;
|
unsigned long long baseLimit = 0;
|
||||||
@ -12932,7 +12934,7 @@ getPPC64MemLockLimitBytes(virDomainDefPtr def)
|
|||||||
passthroughLimit = maxMemory +
|
passthroughLimit = maxMemory +
|
||||||
128 * (1ULL<<30) / 512 * nPCIHostBridges +
|
128 * (1ULL<<30) / 512 * nPCIHostBridges +
|
||||||
8192;
|
8192;
|
||||||
} else if (usesVFIO) {
|
} else if (usesVFIO || forceVFIO) {
|
||||||
/* For regular (non-NVLink2 present) VFIO passthrough, the value
|
/* For regular (non-NVLink2 present) VFIO passthrough, the value
|
||||||
* of passthroughLimit is:
|
* of passthroughLimit is:
|
||||||
*
|
*
|
||||||
@ -12970,16 +12972,20 @@ getPPC64MemLockLimitBytes(virDomainDefPtr def)
|
|||||||
/**
|
/**
|
||||||
* qemuDomainGetMemLockLimitBytes:
|
* qemuDomainGetMemLockLimitBytes:
|
||||||
* @def: domain definition
|
* @def: domain definition
|
||||||
|
* @forceVFIO: force VFIO calculation
|
||||||
*
|
*
|
||||||
* Calculate the memory locking limit that needs to be set in order for
|
* Calculate the memory locking limit that needs to be set in order for
|
||||||
* the guest to operate properly. The limit depends on a number of factors,
|
* the guest to operate properly. The limit depends on a number of factors,
|
||||||
* including certain configuration options and less immediately apparent ones
|
* including certain configuration options and less immediately apparent ones
|
||||||
* such as the guest architecture or the use of certain devices.
|
* such as the guest architecture or the use of certain devices.
|
||||||
|
* The @forceVFIO argument can be used to tell this function will use VFIO even
|
||||||
|
* though @def doesn't indicates so right now.
|
||||||
*
|
*
|
||||||
* Returns: the memory locking limit, or 0 if setting the limit is not needed
|
* Returns: the memory locking limit, or 0 if setting the limit is not needed
|
||||||
*/
|
*/
|
||||||
unsigned long long
|
unsigned long long
|
||||||
qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
|
qemuDomainGetMemLockLimitBytes(virDomainDefPtr def,
|
||||||
|
bool forceVFIO)
|
||||||
{
|
{
|
||||||
unsigned long long memKB = 0;
|
unsigned long long memKB = 0;
|
||||||
bool usesVFIO = false;
|
bool usesVFIO = false;
|
||||||
@ -13000,7 +13006,7 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
|
|||||||
return VIR_DOMAIN_MEMORY_PARAM_UNLIMITED;
|
return VIR_DOMAIN_MEMORY_PARAM_UNLIMITED;
|
||||||
|
|
||||||
if (ARCH_IS_PPC64(def->os.arch) && def->virtType == VIR_DOMAIN_VIRT_KVM)
|
if (ARCH_IS_PPC64(def->os.arch) && def->virtType == VIR_DOMAIN_VIRT_KVM)
|
||||||
return getPPC64MemLockLimitBytes(def);
|
return getPPC64MemLockLimitBytes(def, forceVFIO);
|
||||||
|
|
||||||
/* For device passthrough using VFIO the guest memory and MMIO memory
|
/* For device passthrough using VFIO the guest memory and MMIO memory
|
||||||
* regions need to be locked persistent in order to allow DMA.
|
* regions need to be locked persistent in order to allow DMA.
|
||||||
@ -13020,6 +13026,7 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
|
|||||||
*
|
*
|
||||||
* Note that this may not be valid for all platforms.
|
* Note that this may not be valid for all platforms.
|
||||||
*/
|
*/
|
||||||
|
if (!forceVFIO) {
|
||||||
for (i = 0; i < def->nhostdevs; i++) {
|
for (i = 0; i < def->nhostdevs; i++) {
|
||||||
if (virHostdevIsVFIODevice(def->hostdevs[i]) ||
|
if (virHostdevIsVFIODevice(def->hostdevs[i]) ||
|
||||||
virHostdevIsMdevDevice(def->hostdevs[i])) {
|
virHostdevIsMdevDevice(def->hostdevs[i])) {
|
||||||
@ -13030,8 +13037,9 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
|
|||||||
|
|
||||||
if (virDomainDefHasNVMeDisk(def))
|
if (virDomainDefHasNVMeDisk(def))
|
||||||
usesVFIO = true;
|
usesVFIO = true;
|
||||||
|
}
|
||||||
|
|
||||||
if (usesVFIO)
|
if (usesVFIO || forceVFIO)
|
||||||
memKB = virDomainDefGetMemoryTotal(def) + 1024 * 1024;
|
memKB = virDomainDefGetMemoryTotal(def) + 1024 * 1024;
|
||||||
|
|
||||||
done:
|
done:
|
||||||
@ -13042,9 +13050,12 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
|
|||||||
/**
|
/**
|
||||||
* qemuDomainAdjustMaxMemLock:
|
* qemuDomainAdjustMaxMemLock:
|
||||||
* @vm: domain
|
* @vm: domain
|
||||||
|
* @forceVFIO: apply VFIO requirements even if vm's def doesn't require it
|
||||||
*
|
*
|
||||||
* Adjust the memory locking limit for the QEMU process associated to @vm, in
|
* Adjust the memory locking limit for the QEMU process associated to @vm, in
|
||||||
* order to comply with VFIO or architecture requirements.
|
* order to comply with VFIO or architecture requirements. If @forceVFIO is
|
||||||
|
* true then the limit is changed even if nothing in @vm's definition indicates
|
||||||
|
* so.
|
||||||
*
|
*
|
||||||
* The limit will not be changed unless doing so is needed; the first time
|
* The limit will not be changed unless doing so is needed; the first time
|
||||||
* the limit is changed, the original (default) limit is stored in @vm and
|
* the limit is changed, the original (default) limit is stored in @vm and
|
||||||
@ -13054,12 +13065,13 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
|
|||||||
* Returns: 0 on success, <0 on failure
|
* Returns: 0 on success, <0 on failure
|
||||||
*/
|
*/
|
||||||
int
|
int
|
||||||
qemuDomainAdjustMaxMemLock(virDomainObjPtr vm)
|
qemuDomainAdjustMaxMemLock(virDomainObjPtr vm,
|
||||||
|
bool forceVFIO)
|
||||||
{
|
{
|
||||||
unsigned long long bytes = 0;
|
unsigned long long bytes = 0;
|
||||||
int ret = -1;
|
int ret = -1;
|
||||||
|
|
||||||
bytes = qemuDomainGetMemLockLimitBytes(vm->def);
|
bytes = qemuDomainGetMemLockLimitBytes(vm->def, forceVFIO);
|
||||||
|
|
||||||
if (bytes) {
|
if (bytes) {
|
||||||
/* If this is the first time adjusting the limit, save the current
|
/* If this is the first time adjusting the limit, save the current
|
||||||
@ -13108,7 +13120,7 @@ qemuDomainAdjustMaxMemLockHostdev(virDomainObjPtr vm,
|
|||||||
int ret = 0;
|
int ret = 0;
|
||||||
|
|
||||||
vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
|
vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
|
||||||
if (qemuDomainAdjustMaxMemLock(vm) < 0)
|
if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
|
||||||
ret = -1;
|
ret = -1;
|
||||||
|
|
||||||
vm->def->hostdevs[--(vm->def->nhostdevs)] = NULL;
|
vm->def->hostdevs[--(vm->def->nhostdevs)] = NULL;
|
||||||
|
@ -975,8 +975,10 @@ bool qemuDomainSupportsPCI(virDomainDefPtr def,
|
|||||||
|
|
||||||
void qemuDomainUpdateCurrentMemorySize(virDomainObjPtr vm);
|
void qemuDomainUpdateCurrentMemorySize(virDomainObjPtr vm);
|
||||||
|
|
||||||
unsigned long long qemuDomainGetMemLockLimitBytes(virDomainDefPtr def);
|
unsigned long long qemuDomainGetMemLockLimitBytes(virDomainDefPtr def,
|
||||||
int qemuDomainAdjustMaxMemLock(virDomainObjPtr vm);
|
bool forceVFIO);
|
||||||
|
int qemuDomainAdjustMaxMemLock(virDomainObjPtr vm,
|
||||||
|
bool forceVFIO);
|
||||||
int qemuDomainAdjustMaxMemLockHostdev(virDomainObjPtr vm,
|
int qemuDomainAdjustMaxMemLockHostdev(virDomainObjPtr vm,
|
||||||
virDomainHostdevDefPtr hostdev);
|
virDomainHostdevDefPtr hostdev);
|
||||||
|
|
||||||
|
@ -1646,7 +1646,7 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver,
|
|||||||
if (teardowndevice &&
|
if (teardowndevice &&
|
||||||
qemuDomainNamespaceTeardownHostdev(vm, hostdev) < 0)
|
qemuDomainNamespaceTeardownHostdev(vm, hostdev) < 0)
|
||||||
VIR_WARN("Unable to remove host device from /dev");
|
VIR_WARN("Unable to remove host device from /dev");
|
||||||
if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm) < 0)
|
if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm, false) < 0)
|
||||||
VIR_WARN("Unable to reset maximum locked memory on hotplug fail");
|
VIR_WARN("Unable to reset maximum locked memory on hotplug fail");
|
||||||
|
|
||||||
if (releaseaddr)
|
if (releaseaddr)
|
||||||
@ -2383,7 +2383,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
|
|||||||
if (virDomainMemoryInsert(vm->def, mem) < 0)
|
if (virDomainMemoryInsert(vm->def, mem) < 0)
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
|
|
||||||
if (qemuDomainAdjustMaxMemLock(vm) < 0)
|
if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
|
||||||
goto removedef;
|
goto removedef;
|
||||||
|
|
||||||
qemuDomainObjEnterMonitor(driver, vm);
|
qemuDomainObjEnterMonitor(driver, vm);
|
||||||
@ -2454,7 +2454,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
|
|||||||
|
|
||||||
/* reset the mlock limit */
|
/* reset the mlock limit */
|
||||||
virErrorPreserveLast(&orig_err);
|
virErrorPreserveLast(&orig_err);
|
||||||
ignore_value(qemuDomainAdjustMaxMemLock(vm));
|
ignore_value(qemuDomainAdjustMaxMemLock(vm, false));
|
||||||
virErrorRestore(&orig_err);
|
virErrorRestore(&orig_err);
|
||||||
|
|
||||||
goto audit;
|
goto audit;
|
||||||
@ -2857,7 +2857,7 @@ qemuDomainAttachMediatedDevice(virQEMUDriverPtr driver,
|
|||||||
ret = 0;
|
ret = 0;
|
||||||
cleanup:
|
cleanup:
|
||||||
if (ret < 0) {
|
if (ret < 0) {
|
||||||
if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm) < 0)
|
if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm, false) < 0)
|
||||||
VIR_WARN("Unable to reset maximum locked memory on hotplug fail");
|
VIR_WARN("Unable to reset maximum locked memory on hotplug fail");
|
||||||
if (teardowncgroup && qemuTeardownHostdevCgroup(vm, hostdev) < 0)
|
if (teardowncgroup && qemuTeardownHostdevCgroup(vm, hostdev) < 0)
|
||||||
VIR_WARN("Unable to remove host device cgroup ACL on hotplug fail");
|
VIR_WARN("Unable to remove host device cgroup ACL on hotplug fail");
|
||||||
@ -4356,7 +4356,7 @@ qemuDomainRemoveMemoryDevice(virQEMUDriverPtr driver,
|
|||||||
ignore_value(qemuProcessRefreshBalloonState(driver, vm, QEMU_ASYNC_JOB_NONE));
|
ignore_value(qemuProcessRefreshBalloonState(driver, vm, QEMU_ASYNC_JOB_NONE));
|
||||||
|
|
||||||
/* decrease the mlock limit after memory unplug if necessary */
|
/* decrease the mlock limit after memory unplug if necessary */
|
||||||
ignore_value(qemuDomainAdjustMaxMemLock(vm));
|
ignore_value(qemuDomainAdjustMaxMemLock(vm, false));
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@ -4483,7 +4483,7 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver,
|
|||||||
qemuDomainRemovePCIHostDevice(driver, vm, hostdev);
|
qemuDomainRemovePCIHostDevice(driver, vm, hostdev);
|
||||||
/* QEMU might no longer need to lock as much memory, eg. we just
|
/* QEMU might no longer need to lock as much memory, eg. we just
|
||||||
* detached the last VFIO device, so adjust the limit here */
|
* detached the last VFIO device, so adjust the limit here */
|
||||||
if (qemuDomainAdjustMaxMemLock(vm) < 0)
|
if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
|
||||||
VIR_WARN("Failed to adjust locked memory limit");
|
VIR_WARN("Failed to adjust locked memory limit");
|
||||||
break;
|
break;
|
||||||
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:
|
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:
|
||||||
|
@ -42,7 +42,7 @@ testCompareMemLock(const void *data)
|
|||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
ret = virTestCompareToULL(info->memlock, qemuDomainGetMemLockLimitBytes(def));
|
ret = virTestCompareToULL(info->memlock, qemuDomainGetMemLockLimitBytes(def, false));
|
||||||
|
|
||||||
cleanup:
|
cleanup:
|
||||||
virDomainDefFree(def);
|
virDomainDefFree(def);
|
||||||
|
Loading…
Reference in New Issue
Block a user