qemu: Implement ./hugepages/page/[@size, @unit, @nodeset]

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
2025-02-21 10:52:22 +00:00 · 2014-07-23 17:37:21 +02:00 · 2014-07-23 17:37:21 +02:00 · 3517e1b2f2
commit 3517e1b2f2
parent 136ad49740
11 changed files with 210 additions and 13 deletions
--- a/src/qemu/qemu_capabilities.c
+++ b/src/qemu/qemu_capabilities.c
@ -263,6 +263,7 @@ VIR_ENUM_IMPL(virQEMUCaps, QEMU_CAPS_LAST,

              "memory-backend-ram", /* 170 */
              "numa",
+              "memory-backend-file",
    );


@ -1481,6 +1482,7 @@ struct virQEMUCapsStringFlags virQEMUCapsObjectTypes[] = {
    { "pvpanic", QEMU_CAPS_DEVICE_PANIC },
    { "usb-kbd", QEMU_CAPS_DEVICE_USB_KBD },
    { "memory-backend-ram", QEMU_CAPS_OBJECT_MEMORY_RAM },
+    { "memory-backend-file", QEMU_CAPS_OBJECT_MEMORY_FILE },
 };

 static struct virQEMUCapsStringFlags virQEMUCapsObjectPropsVirtioBlk[] = {
--- a/src/qemu/qemu_capabilities.h
+++ b/src/qemu/qemu_capabilities.h
@ -211,6 +211,7 @@ typedef enum {
    QEMU_CAPS_CHANGE_BACKING_FILE = 169, /* change name of backing file in metadata */
    QEMU_CAPS_OBJECT_MEMORY_RAM  = 170, /* -object memory-backend-ram */
    QEMU_CAPS_NUMA               = 171, /* newer -numa handling with disjoint cpu ranges */
+    QEMU_CAPS_OBJECT_MEMORY_FILE = 172, /* -object memory-backend-file */

    QEMU_CAPS_LAST,                   /* this must always be the last item */
 } virQEMUCapsFlags;
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@ -6432,24 +6432,36 @@ qemuBuildSmpArgStr(const virDomainDef *def,
 }

 static int
-qemuBuildNumaArgStr(const virDomainDef *def,
+qemuBuildNumaArgStr(virQEMUDriverConfigPtr cfg,
+                    const virDomainDef *def,
                    virCommandPtr cmd,
                    virQEMUCapsPtr qemuCaps)
 {
-    size_t i;
+    size_t i, j;
    virBuffer buf = VIR_BUFFER_INITIALIZER;
+    virDomainHugePagePtr master_hugepage = NULL;
    char *cpumask = NULL, *tmpmask = NULL, *next = NULL;
    char *nodemask = NULL;
+    char *mem_path = NULL;
    int ret = -1;

    if (virDomainNumatuneHasPerNodeBinding(def->numatune) &&
-        !virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM)) {
+        !(virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM) ||
+          virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_FILE))) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Per-node memory binding is not supported "
                         "with this QEMU"));
        goto cleanup;
    }

+    if (def->mem.nhugepages && def->mem.hugepages[0].size &&
+        !virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_FILE)) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                       _("huge pages per NUMA node are not "
+                         "supported with this QEMU"));
+        goto cleanup;
+    }
+
    for (i = 0; i < def->cpu->ncells; i++) {
        int cellmem = VIR_DIV_UP(def->cpu->cells[i].mem, 1024);
        def->cpu->cells[i].mem = cellmem * 1024;
@ -6468,15 +6480,74 @@ qemuBuildNumaArgStr(const virDomainDef *def,
            goto cleanup;
        }

-        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM)) {
+        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM) ||
+            virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_FILE)) {
            virDomainNumatuneMemMode mode;
+            virDomainHugePagePtr hugepage = NULL;
            const char *policy = NULL;

            mode = virDomainNumatuneGetMode(def->numatune, i);
            policy = qemuNumaPolicyTypeToString(mode);

-            virBufferAsprintf(&buf, "memory-backend-ram,size=%dM,id=ram-node%zu",
-                              cellmem, i);
+            /* Find the huge page size we want to use */
+            for (j = 0; j < def->mem.nhugepages; j++) {
+                bool thisHugepage = false;
+
+                hugepage = &def->mem.hugepages[j];
+
+                if (!hugepage->nodemask) {
+                    master_hugepage = hugepage;
+                    continue;
+                }
+
+                if (virBitmapGetBit(hugepage->nodemask, i, &thisHugepage) < 0) {
+                    /* Ignore this error. It's not an error after all. Well,
+                     * the nodemask for this <page/> can contain lower NUMA
+                     * nodes than we are querying in here. */
+                    continue;
+                }
+
+                if (thisHugepage) {
+                    /* Hooray, we've found the page size */
+                    break;
+                }
+            }
+
+            if (j == def->mem.nhugepages) {
+                /* We have not found specific huge page to be used with this
+                 * NUMA node. Use the generic setting then (<page/> without any
+                 * @nodemask) if possible. */
+                hugepage = master_hugepage;
+            }
+
+            if (hugepage) {
+                /* Now lets see, if the huge page we want to use is even mounted
+                 * and ready to use */
+
+                for (j = 0; j < cfg->nhugetlbfs; j++) {
+                    if (cfg->hugetlbfs[j].size == hugepage->size)
+                        break;
+                }
+
+                if (j == cfg->nhugetlbfs) {
+                    virReportError(VIR_ERR_INTERNAL_ERROR,
+                                   _("Unable to find any usable hugetlbfs mount for %llu KiB"),
+                                   hugepage->size);
+                    goto cleanup;
+                }
+
+                VIR_FREE(mem_path);
+                if (!(mem_path = qemuGetHugepagePath(&cfg->hugetlbfs[j])))
+                    goto cleanup;
+
+                virBufferAsprintf(&buf,
+                                  "memory-backend-file,prealloc=yes,mem-path=%s",
+                                  mem_path);
+            } else {
+                virBufferAddLit(&buf, "memory-backend-ram");
+            }
+
+            virBufferAsprintf(&buf, ",size=%dM,id=ram-node%zu", cellmem, i);

            if (virDomainNumatuneMaybeFormatNodeset(def->numatune, NULL,
                                                    &nodemask, i) < 0)
@ -6515,7 +6586,8 @@ qemuBuildNumaArgStr(const virDomainDef *def,
            virBufferAdd(&buf, tmpmask, -1);
        }

-        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM)) {
+        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_RAM) ||
+            virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_FILE)) {
            virBufferAsprintf(&buf, ",memdev=ram-node%zu", i);
        } else {
            virBufferAsprintf(&buf, ",mem=%d", cellmem);
@ -6528,6 +6600,7 @@ qemuBuildNumaArgStr(const virDomainDef *def,
 cleanup:
    VIR_FREE(cpumask);
    VIR_FREE(nodemask);
+    VIR_FREE(mem_path);
    virBufferFreeAndReset(&buf);
    return ret;
 }
@ -7383,7 +7456,7 @@ qemuBuildCommandLine(virConnectPtr conn,
    virCommandAddArg(cmd, "-m");
    def->mem.max_balloon = VIR_DIV_UP(def->mem.max_balloon, 1024) * 1024;
    virCommandAddArgFormat(cmd, "%llu", def->mem.max_balloon / 1024);
-    if (def->mem.nhugepages) {
+    if (def->mem.nhugepages && !def->mem.hugepages[0].size) {
        char *mem_path;

        if (!cfg->nhugetlbfs) {
@ -7427,7 +7500,7 @@ qemuBuildCommandLine(virConnectPtr conn,
    VIR_FREE(smp);

    if (def->cpu && def->cpu->ncells)
-        if (qemuBuildNumaArgStr(def, cmd, qemuCaps) < 0)
+        if (qemuBuildNumaArgStr(cfg, def, cmd, qemuCaps) < 0)
            goto error;

    if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_UUID))
--- a/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages.args
+++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages.args
@ -0,0 +1,16 @@
+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
+/usr/bin/qemu -S -M pc -m 4096 -smp 4 \
+-object memory-backend-file,prealloc=yes,mem-path=/dev/hugepages1G/libvirt/qemu,\
+size=1024M,id=ram-node0,host-nodes=0-3,policy=bind \
+-numa node,nodeid=0,cpus=0,memdev=ram-node0 \
+-object memory-backend-file,prealloc=yes,mem-path=/dev/hugepages2M/libvirt/qemu,\
+size=1024M,id=ram-node1,host-nodes=0-3,policy=bind \
+-numa node,nodeid=1,cpus=1,memdev=ram-node1 \
+-object memory-backend-file,prealloc=yes,mem-path=/dev/hugepages1G/libvirt/qemu,\
+size=1024M,id=ram-node2,host-nodes=0-3,policy=bind \
+-numa node,nodeid=2,cpus=2,memdev=ram-node2 \
+-object memory-backend-file,prealloc=yes,mem-path=/dev/hugepages1G/libvirt/qemu,\
+size=1024M,id=ram-node3,host-nodes=3,policy=bind \
+-numa node,nodeid=3,cpus=3,memdev=ram-node3 \
+-nographic -monitor unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -usb \
+-hda /dev/HostVG/QEMUGuest1 -net none -serial none -parallel none
--- a/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages2.args
+++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages2.args
@ -0,0 +1,10 @@
+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
+/usr/bin/qemu -S -M pc -m 1024 -smp 2 \
+-object memory-backend-file,prealloc=yes,\
+mem-path=/dev/hugepages2M/libvirt/qemu,size=256M,id=ram-node0 \
+-numa node,nodeid=0,cpus=0,memdev=ram-node0 \
+-object memory-backend-file,prealloc=yes,\
+mem-path=/dev/hugepages2M/libvirt/qemu,size=768M,id=ram-node1 \
+-numa node,nodeid=1,cpus=1,memdev=ram-node1 \
+-nographic -monitor unix:/tmp/test-monitor,server,nowait -no-acpi -boot c \
+-usb -hda /dev/HostVG/QEMUGuest1 -net none -serial none -parallel none
--- a/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages2.xml
+++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages2.xml
@ -0,0 +1,38 @@
+<domain type='qemu'>
+  <name>SomeDummyHugepagesGuest</name>
+  <uuid>ef1bdff4-27f3-4e85-a807-5fb4d58463cc</uuid>
+  <memory unit='KiB'>1048576</memory>
+  <currentMemory unit='KiB'>1048576</currentMemory>
+  <memoryBacking>
+    <hugepages>
+      <page size='2048' unit='KiB'/>
+    </hugepages>
+  </memoryBacking>
+  <vcpu placement='static'>2</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <cpu>
+    <numa>
+      <cell id='0' cpus='0' memory='262144'/>
+      <cell id='1' cpus='1' memory='786432'/>
+    </numa>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='usb' index='0'/>
+    <controller type='ide' index='0'/>
+    <controller type='pci' index='0' model='pci-root'/>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>
--- a/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages3.args
+++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages3.args
@ -0,0 +1,9 @@
+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
+/usr/bin/qemu -S -M pc -m 1024 -smp 2 \
+-object memory-backend-ram,size=256M,id=ram-node0 \
+-numa node,nodeid=0,cpus=0,memdev=ram-node0 \
+-object memory-backend-file,prealloc=yes,\
+mem-path=/dev/hugepages1G/libvirt/qemu,size=768M,id=ram-node1 \
+-numa node,nodeid=1,cpus=1,memdev=ram-node1 \
+-nographic -monitor unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -usb \
+-hda /dev/HostVG/QEMUGuest1 -net none -serial none -parallel none
--- a/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages3.xml
+++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages3.xml
@ -0,0 +1,38 @@
+<domain type='qemu'>
+  <name>SomeDummyHugepagesGuest</name>
+  <uuid>ef1bdff4-27f3-4e85-a807-5fb4d58463cc</uuid>
+  <memory unit='KiB'>1048576</memory>
+  <currentMemory unit='KiB'>1048576</currentMemory>
+  <memoryBacking>
+    <hugepages>
+      <page size='1048576' unit='KiB' nodeset='1'/>
+    </hugepages>
+  </memoryBacking>
+  <vcpu placement='static'>2</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <cpu>
+    <numa>
+      <cell id='0' cpus='0' memory='262144'/>
+      <cell id='1' cpus='1' memory='786432'/>
+    </numa>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='usb' index='0'/>
+    <controller type='ide' index='0'/>
+    <controller type='pci' index='0' model='pci-root'/>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>
--- a/tests/qemuxml2argvdata/qemuxml2argv-hugepages.args
+++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages.args
@ -1,5 +1,5 @@
 LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
 /usr/bin/qemu -S -M \
-pc -m 214 -mem-prealloc -mem-path /dev/hugepages/libvirt/qemu -smp 1 \
+pc -m 214 -mem-prealloc -mem-path /dev/hugepages2M/libvirt/qemu -smp 1 \
 -nographic -monitor unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -usb -hda \
 /dev/HostVG/QEMUGuest1 -net none -serial none -parallel none
--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@ -525,13 +525,15 @@ mymain(void)
    if (VIR_STRDUP_QUIET(driver.config->stateDir, "/nowhere") < 0)
        return EXIT_FAILURE;
    VIR_FREE(driver.config->hugetlbfs);
-    if (VIR_ALLOC_N(driver.config->hugetlbfs, 1) < 0)
+    if (VIR_ALLOC_N(driver.config->hugetlbfs, 2) < 0)
        return EXIT_FAILURE;
-    driver.config->nhugetlbfs = 1;
-    if (VIR_STRDUP(driver.config->hugetlbfs[0].mnt_dir, "/dev/hugepages") < 0)
+    driver.config->nhugetlbfs = 2;
+    if (VIR_STRDUP(driver.config->hugetlbfs[0].mnt_dir, "/dev/hugepages2M") < 0 ||
+        VIR_STRDUP(driver.config->hugetlbfs[1].mnt_dir, "/dev/hugepages1G") < 0)
        return EXIT_FAILURE;
    driver.config->hugetlbfs[0].size = 2048;
    driver.config->hugetlbfs[0].deflt = true;
+    driver.config->hugetlbfs[1].size = 1048576;
    driver.config->spiceTLS = 1;
    if (VIR_STRDUP_QUIET(driver.config->spicePassword, "123456") < 0)
        return EXIT_FAILURE;
@ -665,6 +667,12 @@ mymain(void)
    DO_TEST("hyperv-off", NONE);

    DO_TEST("hugepages", QEMU_CAPS_MEM_PATH);
+    DO_TEST("hugepages-pages", QEMU_CAPS_MEM_PATH, QEMU_CAPS_OBJECT_MEMORY_RAM,
+            QEMU_CAPS_OBJECT_MEMORY_FILE);
+    DO_TEST("hugepages-pages2", QEMU_CAPS_MEM_PATH, QEMU_CAPS_OBJECT_MEMORY_RAM,
+            QEMU_CAPS_OBJECT_MEMORY_FILE);
+    DO_TEST("hugepages-pages3", QEMU_CAPS_MEM_PATH, QEMU_CAPS_OBJECT_MEMORY_RAM,
+            QEMU_CAPS_OBJECT_MEMORY_FILE);
    DO_TEST("nosharepages", QEMU_CAPS_MACHINE_OPT, QEMU_CAPS_MEM_MERGE);
    DO_TEST("disk-cdrom", NONE);
    DO_TEST("disk-cdrom-network-http", QEMU_CAPS_KVM, QEMU_CAPS_DEVICE,
--- a/tests/qemuxml2xmltest.c
+++ b/tests/qemuxml2xmltest.c
@ -198,6 +198,8 @@ mymain(void)

    DO_TEST("hugepages");
    DO_TEST("hugepages-pages");
+    DO_TEST("hugepages-pages2");
+    DO_TEST("hugepages-pages3");
    DO_TEST("nosharepages");
    DO_TEST("disk-aio");
    DO_TEST("disk-cdrom");