conf: Introduce virtio-pmem <memory/> model

The virtio-pmem is a virtio variant of NVDIMM and just like
NVDIMM virtio-pmem also allows accessing host pages bypassing
guest page cache. The difference is that if a regular file is
used to back guest's NVDIMM (model='nvdimm') the persistence of
guest writes might not be guaranteed while with virtio-pmem it
is.

To express this new model at domain XML level, I've chosen the
following:

  <memory model='virtio-pmem' access='shared'>
    <source>
      <path>/tmp/virtio_pmem</path>
    </source>
    <target>
      <size unit='KiB'>524288</size>
    </target>
    <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
  </memory>

Another difference between NVDIMM and virtio-pmem is that while
the former supports NUMA node locality the latter doesn't. And
also, the latter goes onto PCI bus and not into a DIMM module.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>
This commit is contained in:
Michal Privoznik 2020-12-07 15:24:17 +01:00
parent f06c1d908f
commit 173733b7a8
17 changed files with 274 additions and 27 deletions

View File

@ -7257,20 +7257,30 @@ Example: usage of the memory devices
</label>
</target>
</memory>
<memory model='virtio-pmem' access='shared'>
<source>
<path>/tmp/virtio_pmem</path>
</source>
<target>
<size unit='KiB'>524288</size>
</target>
</memory>
</devices>
...
``model``
Provide ``dimm`` to add a virtual DIMM module to the guest. :since:`Since
1.2.14` Provide ``nvdimm`` model adds a Non-Volatile DIMM module.
:since:`Since 3.2.0`
1.2.14` Provide ``nvdimm`` model that adds a Non-Volatile DIMM module.
:since:`Since 3.2.0` Provide ``virtio-pmem`` model to add a paravirtualized
persistent memory device. :since:`Since 7.1.0`
``access``
An optional attribute ``access`` ( :since:`since 3.2.0` ) that provides
capability to fine tune mapping of the memory on per module basis. Values are
the same as `Memory Backing <#elementsMemoryBacking>`__: ``shared`` and
``private``. For ``nvdimm`` model, if using real NVDIMM DAX device as
backend, ``shared`` is required.
backend, ``shared`` is required. For ``virtio-pmem`` model ``shared`` is
required.
``discard``
An optional attribute ``discard`` ( :since:`since 4.4.0` ) that provides
@ -7299,9 +7309,9 @@ Example: usage of the memory devices
This element can be used to override the default set of NUMA nodes where
the memory would be allocated.
For model ``nvdimm`` this element is mandatory. The mandatory child element
``path`` represents a path in the host that backs the nvdimm module in the
guest. The following optional elements may be used:
For model ``nvdimm`` the ``source`` element is mandatory. The mandatory
child element ``path`` represents a path in the host that backs the nvdimm
module in the guest. The following optional elements may be used:
``alignsize``
The ``alignsize`` element defines the page size alignment used to mmap the
@ -7315,6 +7325,13 @@ Example: usage of the memory devices
to guarantee the persistence of writes to the vNVDIMM backend, then use
the ``pmem`` element in order to utilize the feature. :since:`Since 5.0.0`
For model ``virtio-pmem`` the ``source`` element is mandatory. The following
optional elements may be used:
``path``
Represents a path in the host that backs the virtio memory module in the
guest. It is mandatory.
``target``
The mandatory ``target`` element configures the placement and sizing of the
added memory from the perspective of the guest.

View File

@ -6019,6 +6019,7 @@
<choice>
<value>dimm</value>
<value>nvdimm</value>
<value>virtio-pmem</value>
</choice>
</attribute>
<optional>

View File

@ -1309,6 +1309,7 @@ VIR_ENUM_IMPL(virDomainMemoryModel,
"",
"dimm",
"nvdimm",
"virtio-pmem",
);
VIR_ENUM_IMPL(virDomainShmemModel,
@ -5331,6 +5332,28 @@ virDomainVsockDefPostParse(virDomainVsockDefPtr vsock)
}
static int
virDomainMemoryDefPostParse(virDomainMemoryDefPtr mem)
{
switch (mem->model) {
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
/* Virtio-pmem mandates shared access so that guest writes get
* reflected in the underlying file. */
if (mem->access == VIR_DOMAIN_MEMORY_ACCESS_DEFAULT)
mem->access = VIR_DOMAIN_MEMORY_ACCESS_SHARED;
break;
case VIR_DOMAIN_MEMORY_MODEL_NVDIMM:
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
break;
}
return 0;
}
static int
virDomainDeviceDefPostParseCommon(virDomainDeviceDefPtr dev,
const virDomainDef *def,
@ -5376,6 +5399,10 @@ virDomainDeviceDefPostParseCommon(virDomainDeviceDefPtr dev,
ret = 0;
break;
case VIR_DOMAIN_DEVICE_MEMORY:
ret = virDomainMemoryDefPostParse(dev->data.memory);
break;
case VIR_DOMAIN_DEVICE_LEASE:
case VIR_DOMAIN_DEVICE_FS:
case VIR_DOMAIN_DEVICE_INPUT:
@ -5390,7 +5417,6 @@ virDomainDeviceDefPostParseCommon(virDomainDeviceDefPtr dev,
case VIR_DOMAIN_DEVICE_SHMEM:
case VIR_DOMAIN_DEVICE_TPM:
case VIR_DOMAIN_DEVICE_PANIC:
case VIR_DOMAIN_DEVICE_MEMORY:
case VIR_DOMAIN_DEVICE_IOMMU:
case VIR_DOMAIN_DEVICE_AUDIO:
ret = 0;
@ -15310,6 +15336,10 @@ virDomainMemorySourceDefParseXML(xmlNodePtr node,
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
def->nvdimmPath = virXPathString("string(./path)", ctxt);
break;
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
break;
@ -17194,6 +17224,11 @@ virDomainMemoryFindByDefInternal(virDomainDefPtr def,
continue;
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
if (STRNEQ(tmp->nvdimmPath, mem->nvdimmPath))
continue;
break;
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
break;
@ -26486,6 +26521,10 @@ virDomainMemorySourceDefFormat(virBufferPtr buf,
virBufferAddLit(&childBuf, "<pmem/>\n");
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
virBufferEscapeString(&childBuf, "<path>%s</path>\n", def->nvdimmPath);
break;
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
break;

View File

@ -2307,6 +2307,7 @@ typedef enum {
VIR_DOMAIN_MEMORY_MODEL_NONE,
VIR_DOMAIN_MEMORY_MODEL_DIMM, /* dimm hotpluggable memory device */
VIR_DOMAIN_MEMORY_MODEL_NVDIMM, /* nvdimm memory device */
VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM, /* virtio-pmem memory device */
VIR_DOMAIN_MEMORY_MODEL_LAST
} virDomainMemoryModel;
@ -2318,7 +2319,7 @@ struct _virDomainMemoryDef {
/* source */
virBitmapPtr sourceNodes;
unsigned long long pagesize; /* kibibytes */
char *nvdimmPath;
char *nvdimmPath; /* valid for NVDIMM an VIRTIO_PMEM */
unsigned long long alignsize; /* kibibytes; valid only for NVDIMM */
bool nvdimmPmem; /* valid only for NVDIMM */

View File

@ -1389,7 +1389,8 @@ static int
virDomainMemoryDefValidate(const virDomainMemoryDef *mem,
const virDomainDef *def)
{
if (mem->model == VIR_DOMAIN_MEMORY_MODEL_NVDIMM) {
switch (mem->model) {
case VIR_DOMAIN_MEMORY_MODEL_NVDIMM:
if (!mem->nvdimmPath) {
virReportError(VIR_ERR_XML_DETAIL, "%s",
_("path is required for model 'nvdimm'"));
@ -1407,6 +1408,43 @@ virDomainMemoryDefValidate(const virDomainMemoryDef *mem,
_("label size is required for NVDIMM device"));
return -1;
}
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
if (!mem->nvdimmPath) {
virReportError(VIR_ERR_XML_DETAIL,
_("path is required for model '%s'"),
virDomainMemoryModelTypeToString(mem->model));
return -1;
}
if (mem->discard == VIR_TRISTATE_BOOL_YES) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
_("discard is not supported for model '%s'"),
virDomainMemoryModelTypeToString(mem->model));
return -1;
}
if (mem->access != VIR_DOMAIN_MEMORY_ACCESS_SHARED) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("shared access mode required for virtio-pmem device"));
return -1;
}
if (mem->targetNode != -1) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("virtio-pmem does not support NUMA nodes"));
return -1;
}
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
break;
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
default:
virReportEnumRangeError(virDomainMemoryModel, mem->model);
return -1;
}
return 0;

View File

@ -3325,6 +3325,7 @@ qemuBuildMemoryDeviceStr(const virDomainDef *def,
return NULL;
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
break;

View File

@ -8605,6 +8605,8 @@ static int
qemuDomainDefValidateMemoryHotplugDevice(const virDomainMemoryDef *mem,
const virDomainDef *def)
{
bool needsNuma = true;
switch (mem->model) {
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
case VIR_DOMAIN_MEMORY_MODEL_NVDIMM:
@ -8640,11 +8642,34 @@ qemuDomainDefValidateMemoryHotplugDevice(const virDomainMemoryDef *mem,
}
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
if (mem->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI &&
mem->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("only 'pci' addresses are supported for the "
"virtio-pmem device"));
return -1;
}
/* virtio-pmem doesn't have .node attribute. */
needsNuma = false;
break;
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
return -1;
}
if (needsNuma &&
virDomainNumaGetNodeCount(def->numa) != 0) {
if (mem->targetNode == -1) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("target NUMA node needs to be specified for "
"memory device"));
return -1;
}
}
return 0;
}

View File

@ -313,11 +313,11 @@ qemuDomainPrimeVirtioDeviceAddresses(virDomainDefPtr def,
virDomainDeviceAddressType type)
{
/*
declare address-less virtio devices to be of address type 'type'
disks, networks, videos, consoles, controllers, memballoon and rng
in this order
if type is ccw filesystem and vsock devices are declared to be of
address type ccw
Declare address-less virtio devices to be of address type 'type'
disks, networks, videos, consoles, controllers, hostdevs, memballoon,
rngs and memories in this order.
If type is ccw filesystem and vsock devices are declared to be of
address type ccw.
*/
size_t i;
@ -379,6 +379,12 @@ qemuDomainPrimeVirtioDeviceAddresses(virDomainDefPtr def,
def->rngs[i]->info.type = type;
}
for (i = 0; i < def->nmems; i++) {
if (def->mems[i]->model == VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM &&
def->mems[i]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE)
def->mems[i]->info.type = type;
}
if (type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW) {
for (i = 0; i < def->nfss; i++) {
if (def->fss[i]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE)
@ -1005,11 +1011,23 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev,
}
break;
case VIR_DOMAIN_DEVICE_MEMORY:
switch (dev->data.memory->model) {
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
return virtioFlags;
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
case VIR_DOMAIN_MEMORY_MODEL_NVDIMM:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
return 0;
}
break;
/* These devices don't ever connect with PCI */
case VIR_DOMAIN_DEVICE_NVRAM:
case VIR_DOMAIN_DEVICE_TPM:
case VIR_DOMAIN_DEVICE_PANIC:
case VIR_DOMAIN_DEVICE_MEMORY:
case VIR_DOMAIN_DEVICE_HUB:
case VIR_DOMAIN_DEVICE_REDIRDEV:
case VIR_DOMAIN_DEVICE_SMARTCARD:
@ -2420,6 +2438,17 @@ qemuDomainAssignDevicePCISlots(virDomainDefPtr def,
return -1;
}
for (i = 0; i < def->nmems; i++) {
virDomainMemoryDefPtr mem = def->mems[i];
if (mem->model != VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM ||
!virDeviceInfoPCIAddressIsWanted(&mem->info))
continue;
if (qemuDomainPCIAddressReserveNextAddr(addrs, &mem->info) < 0)
return -1;
}
return 0;
}
@ -3067,19 +3096,32 @@ qemuAssignMemoryDeviceSlot(virDomainMemoryDefPtr mem,
int
qemuDomainAssignMemoryDeviceSlot(virDomainDefPtr def,
qemuDomainAssignMemoryDeviceSlot(virQEMUDriverPtr driver,
virDomainObjPtr vm,
virDomainMemoryDefPtr mem)
{
virBitmapPtr slotmap = NULL;
int ret;
g_autoptr(virBitmap) slotmap = NULL;
virDomainDeviceDef dev = {.type = VIR_DOMAIN_DEVICE_MEMORY, .data.memory = mem};
if (!(slotmap = qemuDomainGetMemorySlotMap(def)))
return -1;
switch (mem->model) {
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
case VIR_DOMAIN_MEMORY_MODEL_NVDIMM:
if (!(slotmap = qemuDomainGetMemorySlotMap(vm->def)))
return -1;
ret = qemuAssignMemoryDeviceSlot(mem, slotmap);
return qemuAssignMemoryDeviceSlot(mem, slotmap);
break;
virBitmapFree(slotmap);
return ret;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
return qemuDomainEnsurePCIAddress(vm, &dev, driver);
break;
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
break;
}
return 0;
}
@ -3097,8 +3139,22 @@ qemuDomainAssignMemorySlots(virDomainDefPtr def)
return -1;
for (i = 0; i < def->nmems; i++) {
if (qemuAssignMemoryDeviceSlot(def->mems[i], slotmap) < 0)
goto cleanup;
virDomainMemoryDefPtr mem = def->mems[i];
switch (mem->model) {
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
case VIR_DOMAIN_MEMORY_MODEL_NVDIMM:
if (qemuAssignMemoryDeviceSlot(def->mems[i], slotmap) < 0)
goto cleanup;
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
/* handled in qemuDomainAssignPCIAddresses() */
break;
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
break;
}
}
ret = 0;

View File

@ -56,7 +56,8 @@ void qemuDomainFillDeviceIsolationGroup(virDomainDefPtr def,
void qemuDomainReleaseDeviceAddress(virDomainObjPtr vm,
virDomainDeviceInfoPtr info);
int qemuDomainAssignMemoryDeviceSlot(virDomainDefPtr def,
int qemuDomainAssignMemoryDeviceSlot(virQEMUDriverPtr driver,
virDomainObjPtr vm,
virDomainMemoryDefPtr mem);
int qemuDomainEnsureVirtioAddress(bool *releaseAddr,

View File

@ -2412,7 +2412,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
if (qemuDomainDefValidateMemoryHotplug(vm->def, mem) < 0)
goto cleanup;
if (qemuDomainAssignMemoryDeviceSlot(vm->def, mem) < 0)
if (qemuDomainAssignMemoryDeviceSlot(driver, vm, mem) < 0)
goto cleanup;
/* in cases where we are using a VM with aliases generated according to the

View File

@ -4624,6 +4624,14 @@ qemuValidateDomainDeviceDefMemory(virDomainMemoryDefPtr mem,
}
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_VIRTIO_PMEM_PCI)) {
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
_("virtio-pmem isn't supported by this QEMU binary"));
return -1;
}
break;
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
break;

View File

@ -690,6 +690,7 @@ AppArmorSetMemoryLabel(virSecurityManagerPtr mgr,
return -1;
}
return reload_profile(mgr, def, mem->nvdimmPath, true);
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
case VIR_DOMAIN_MEMORY_MODEL_LAST:

View File

@ -1892,6 +1892,7 @@ virSecurityDACRestoreMemoryLabel(virSecurityManagerPtr mgr,
ret = virSecurityDACRestoreFileLabel(mgr, mem->nvdimmPath);
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
case VIR_DOMAIN_MEMORY_MODEL_NONE:
@ -2074,6 +2075,7 @@ virSecurityDACSetMemoryLabel(virSecurityManagerPtr mgr,
user, group, true);
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
case VIR_DOMAIN_MEMORY_MODEL_NONE:

View File

@ -1581,6 +1581,7 @@ virSecuritySELinuxSetMemoryLabel(virSecurityManagerPtr mgr,
return -1;
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
case VIR_DOMAIN_MEMORY_MODEL_LAST:
@ -1608,6 +1609,7 @@ virSecuritySELinuxRestoreMemoryLabel(virSecurityManagerPtr mgr,
ret = virSecuritySELinuxRestoreFileLabel(mgr, mem->nvdimmPath, true);
break;
case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
case VIR_DOMAIN_MEMORY_MODEL_DIMM:
case VIR_DOMAIN_MEMORY_MODEL_NONE:
case VIR_DOMAIN_MEMORY_MODEL_LAST:

View File

@ -0,0 +1,53 @@
<domain type='kvm'>
<name>QEMUGuest1</name>
<uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
<maxMemory slots='16' unit='KiB'>1099511627776</maxMemory>
<memory unit='KiB'>2095104</memory>
<currentMemory unit='KiB'>2095104</currentMemory>
<vcpu placement='static' cpuset='0-1'>2</vcpu>
<os>
<type arch='x86_64' machine='pc'>hvm</type>
<boot dev='hd'/>
</os>
<cpu mode='custom' match='exact' check='none'>
<model fallback='forbid'>qemu64</model>
<topology sockets='2' dies='1' cores='1' threads='1'/>
<numa>
<cell id='0' cpus='0-1' memory='2095104' unit='KiB'/>
</numa>
</cpu>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<devices>
<emulator>/usr/bin/qemu-system-i386</emulator>
<disk type='block' device='disk'>
<driver name='qemu' type='raw'/>
<source dev='/dev/HostVG/QEMUGuest1'/>
<target dev='hda' bus='ide'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<controller type='ide' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
</controller>
<controller type='usb' index='0' model='piix3-uhci'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pci-root'/>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<memballoon model='virtio'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</memballoon>
<memory model='virtio-pmem' access='shared'>
<source>
<path>/tmp/virtio_pmem</path>
</source>
<target>
<size unit='KiB'>524288</size>
</target>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
</memory>
</devices>
</domain>

View File

@ -0,0 +1 @@
../qemuxml2argvdata/memory-hotplug-virtio-pmem.xml

View File

@ -1261,6 +1261,7 @@ mymain(void)
QEMU_CAPS_OBJECT_MEMORY_FILE,
QEMU_CAPS_DEVICE_NVDIMM,
QEMU_CAPS_LAST);
DO_TEST_CAPS_LATEST("memory-hotplug-virtio-pmem");
DO_TEST("net-udp", NONE);