From d823a05aef2baa0927e792322f1af45e18c325bd Mon Sep 17 00:00:00 2001 From: "Daniel P. Berrange" Date: Tue, 25 Aug 2009 15:05:18 +0100 Subject: [PATCH] Support configuration of huge pages in guests Add option to domain XML for * configure.in: Add check for mntent.h * qemud/libvirtd_qemu.aug, qemud/test_libvirtd_qemu.aug, src/qemu.conf Add 'hugetlbfs_mount' config parameter * src/qemu_conf.c, src/qemu_conf.h: Check for -mem-path flag in QEMU, and pass it when hugepages are requested. Load hugetlbfs_mount config parameter, search for mount if not given. * src/qemu_driver.c: Free hugetlbfs_mount/path parameter in driver shutdown. Create directory for QEMU hugepage usage, chowning if required. * docs/formatdomain.html.in: Document memoryBacking/hugepages elements * docs/schemas/domain.rng: Add memoryBacking/hugepages elements to schema * src/util.c, src/util.h, src/libvirt_private.syms: Add virFileFindMountPoint helper API * tests/qemuhelptest.c: Add -mem-path constants * tests/qemuxml2argvtest.c, tests/qemuxml2xmltest.c: Add tests for hugepage handling * tests/qemuxml2argvdata/qemuxml2argv-hugepages.xml, tests/qemuxml2argvdata/qemuxml2argv-hugepages.args: Data files for hugepage tests --- configure.in | 2 +- docs/formatdomain.html | 8 ++- docs/formatdomain.html.in | 8 +++ docs/schemas/domain.rng | 9 ++++ qemud/libvirtd_qemu.aug | 1 + qemud/test_libvirtd_qemu.aug | 4 ++ src/domain_conf.c | 10 +++- src/domain_conf.h | 1 + src/libvirt_private.syms | 1 + src/qemu.conf | 12 +++++ src/qemu_conf.c | 50 +++++++++++++++++++ src/qemu_conf.h | 3 ++ src/qemu_driver.c | 33 ++++++++++++ src/util.c | 38 +++++++++++++- src/util.h | 4 ++ tests/qemuhelptest.c | 6 ++- .../qemuxml2argv-hugepages.args | 1 + .../qemuxml2argv-hugepages.xml | 25 ++++++++++ tests/qemuxml2argvtest.c | 7 ++- tests/qemuxml2xmltest.c | 1 + 20 files changed, 217 insertions(+), 7 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-hugepages.xml diff --git a/configure.in b/configure.in index d28c44a235..43b9b462e8 100644 --- a/configure.in +++ b/configure.in @@ -83,7 +83,7 @@ dnl Availability of various not common threadsafe functions AC_CHECK_FUNCS([strerror_r strtok_r getmntent_r getgrnam_r getpwuid_r]) dnl Availability of various common headers (non-fatal if missing). -AC_CHECK_HEADERS([pwd.h paths.h regex.h sys/syslimits.h sys/utsname.h sys/wait.h winsock2.h sched.h termios.h sys/poll.h syslog.h]) +AC_CHECK_HEADERS([pwd.h paths.h regex.h sys/syslimits.h sys/utsname.h sys/wait.h winsock2.h sched.h termios.h sys/poll.h syslog.h mntent.h]) dnl Where are the XDR functions? dnl If portablexdr is installed, prefer that. diff --git a/docs/formatdomain.html b/docs/formatdomain.html index 3368ad5004..a7154577c2 100644 --- a/docs/formatdomain.html +++ b/docs/formatdomain.html @@ -338,13 +338,19 @@ ... <memory>524288</memory> <currentMemory>524288</currentMemory> + <memoryBacking> + <hugepages/> + </memoryBacking> <vcpu>1</vcpu> ...
memory
The maximum allocation of memory for the guest at boot time. The units for this value are kilobytes (i.e. blocks of 1024 bytes)
currentMemory
The actual allocation of memory for the guest. This value be less than the maximum allocation, to allow for ballooning up the guests memory on the fly. If this is omitted, it defaults - to the same value as the memory element
vcpu
The content of this element defines the number of virtual + to the same value as the memory element
memoryBacking
The optional memoryBacking element, may have an + hugepages element set within it. This tells the + hypervisor that the guest should have its memory allocated using + hugepages instead of the normal native page size.
vcpu
The content of this element defines the number of virtual CPUs allocated for the guest OS.

Lifecycle control diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index 211f7edb5d..915814d792 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -176,6 +176,9 @@ ... <memory>524288</memory> <currentMemory>524288</currentMemory> + <memoryBacking> + <hugepages/> + </memoryBacking> <vcpu>1</vcpu> ... @@ -188,6 +191,11 @@ be less than the maximum allocation, to allow for ballooning up the guests memory on the fly. If this is omitted, it defaults to the same value as the memory element +
memoryBacking
+
The optional memoryBacking element, may have an + hugepages element set within it. This tells the + hypervisor that the guest should have its memory allocated using + hugepages instead of the normal native page size.
vcpu
The content of this element defines the number of virtual CPUs allocated for the guest OS.
diff --git a/docs/schemas/domain.rng b/docs/schemas/domain.rng index 4bd301a68d..e5780615a5 100644 --- a/docs/schemas/domain.rng +++ b/docs/schemas/domain.rng @@ -245,6 +245,15 @@ + + + + + + + + + diff --git a/qemud/libvirtd_qemu.aug b/qemud/libvirtd_qemu.aug index 2175e14128..f0b2a5e3c1 100644 --- a/qemud/libvirtd_qemu.aug +++ b/qemud/libvirtd_qemu.aug @@ -35,6 +35,7 @@ module Libvirtd_qemu = | str_array_entry "cgroup_controllers" | str_array_entry "cgroup_device_acl" | str_entry "save_image_format" + | str_entry "hugetlbfs_mount" (* Each enty in the config is one of the following three ... *) let entry = vnc_entry diff --git a/qemud/test_libvirtd_qemu.aug b/qemud/test_libvirtd_qemu.aug index 72f2227932..ac89438651 100644 --- a/qemud/test_libvirtd_qemu.aug +++ b/qemud/test_libvirtd_qemu.aug @@ -91,6 +91,8 @@ cgroup_controllers = [ \"cpu\", \"devices\" ] cgroup_device_acl = [ \"/dev/null\", \"/dev/full\", \"/dev/zero\" ] save_image_format = \"gzip\" + +hugetlbfs_mount = \"/dev/hugepages\" " test Libvirtd_qemu.lns get conf = @@ -192,3 +194,5 @@ save_image_format = \"gzip\" } { "#empty" } { "save_image_format" = "gzip" } +{ "#empty" } +{ "hugetlbfs_mount" = "/dev/hugepages" } \ No newline at end of file diff --git a/src/domain_conf.c b/src/domain_conf.c index 79225a89bb..e95fbe0364 100644 --- a/src/domain_conf.c +++ b/src/domain_conf.c @@ -2537,6 +2537,10 @@ static virDomainDefPtr virDomainDefParseXML(virConnectPtr conn, if (virXPathULong(conn, "string(./currentMemory[1])", ctxt, &def->memory) < 0) def->memory = def->maxmem; + node = virXPathNode(conn, "./memoryBacking/hugepages", ctxt); + if (node) + def->hugepage_backed = 1; + if (virXPathULong(conn, "string(./vcpu[1])", ctxt, &def->vcpus) < 0) def->vcpus = 1; @@ -4161,7 +4165,11 @@ char *virDomainDefFormat(virConnectPtr conn, virBufferVSprintf(&buf, " %lu\n", def->maxmem); virBufferVSprintf(&buf, " %lu\n", def->memory); - + if (def->hugepage_backed) { + virBufferAddLit(&buf, " \n"); + virBufferAddLit(&buf, " \n"); + virBufferAddLit(&buf, " \n"); + } for (n = 0 ; n < def->cpumasklen ; n++) if (def->cpumask[n] != 1) allones = 0; diff --git a/src/domain_conf.h b/src/domain_conf.h index 53d30d24d5..661773e427 100644 --- a/src/domain_conf.h +++ b/src/domain_conf.h @@ -535,6 +535,7 @@ struct _virDomainDef { unsigned long memory; unsigned long maxmem; + unsigned char hugepage_backed; unsigned long vcpus; int cpumasklen; char *cpumask; diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index ead3390e7b..eec8be64c7 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -408,6 +408,7 @@ virGetUserDirectory; virGetUserName; virGetUserID; virGetGroupID; +virFileFindMountPoint; # uuid.h diff --git a/src/qemu.conf b/src/qemu.conf index 9aecc2637e..06babc4a35 100644 --- a/src/qemu.conf +++ b/src/qemu.conf @@ -139,3 +139,15 @@ # the process of saving a domain in order to save disk space. # # save_image_format = "raw" + +# If provided by the host and a hugetlbfs mount point is configured, +# a guest may request huge page backing. When this mount point is +# unspecified here, determination of a host mount point in /proc/mounts +# will be attempted. Specifying an explicit mount overrides detection +# of the same in /proc/mounts. Setting the mount point to "" will +# disable guest hugepage backing. +# +# NB, within this mount point, guests will create memory backing files +# in a location of $MOUNTPOINT/libvirt/qemu + +# hugetlbfs_mount = "/dev/hugepages" diff --git a/src/qemu_conf.c b/src/qemu_conf.c index 22f5edd248..918ccf2143 100644 --- a/src/qemu_conf.c +++ b/src/qemu_conf.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "c-ctype.h" #include "virterror_internal.h" @@ -87,6 +88,7 @@ VIR_ENUM_IMPL(qemuVideo, VIR_DOMAIN_VIDEO_TYPE_LAST, NULL, /* no arg needed for xen */ NULL /* don't support vbox */); +#define PROC_MOUNT_BUF_LEN 255 int qemudLoadDriverConfig(struct qemud_driver *driver, const char *filename) { @@ -106,6 +108,21 @@ int qemudLoadDriverConfig(struct qemud_driver *driver, return -1; } +#ifdef HAVE_MNTENT_H + /* For privileged driver, try and find hugepage mount automatically. + * Non-privileged driver requires admin to create a dir for the + * user, chown it, and then let user configure it manually */ + if (driver->privileged && + !(driver->hugetlbfs_mount = virFileFindMountPoint("hugetlbfs"))) { + if (errno != ENOENT) { + virReportSystemError(NULL, errno, "%s", + _("unable to find hugetlbfs mountpoint")); + return -1; + } + } +#endif + + /* Just check the file is readable before opening it, otherwise * libvirt emits an error. */ @@ -290,6 +307,17 @@ int qemudLoadDriverConfig(struct qemud_driver *driver, } } + p = virConfGetValue (conf, "hugetlbfs_mount"); + CHECK_TYPE ("hugetlbfs_mount", VIR_CONF_STRING); + if (p && p->str) { + VIR_FREE(driver->hugetlbfs_mount); + if (!(driver->hugetlbfs_mount = strdup(p->str))) { + virReportOOMError(NULL); + virConfFree(conf); + return -1; + } + } + virConfFree (conf); return 0; } @@ -784,6 +812,8 @@ static unsigned int qemudComputeCmdFlags(const char *help, flags |= QEMUD_CMD_FLAG_DRIVE_BOOT; if (strstr(help, "-pcidevice")) flags |= QEMUD_CMD_FLAG_PCIDEVICE; + if (strstr(help, "-mem-path")) + flags |= QEMUD_CMD_FLAG_MEM_PATH; if (version >= 9000) flags |= QEMUD_CMD_FLAG_VNC_COLON; @@ -1583,6 +1613,26 @@ int qemudBuildCommandLine(virConnectPtr conn, ADD_ARG_LIT("-no-kvm"); ADD_ARG_LIT("-m"); ADD_ARG_LIT(memory); + if (def->hugepage_backed) { + if (!driver->hugetlbfs_mount) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + "%s", _("hugetlbfs filesystem is not mounted")); + goto error; + } + if (!driver->hugepage_path) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + "%s", _("hugepages are disabled by administrator config")); + goto error; + } + if (!(qemuCmdFlags & QEMUD_CMD_FLAG_MEM_PATH)) { + qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("hugepage backing not supported by '%s'"), + def->emulator); + goto error; + } + ADD_ARG_LIT("-mem-path"); + ADD_ARG_LIT(driver->hugepage_path); + } ADD_ARG_LIT("-smp"); ADD_ARG_LIT(vcpus); diff --git a/src/qemu_conf.h b/src/qemu_conf.h index a126dacfd7..f12694092c 100644 --- a/src/qemu_conf.h +++ b/src/qemu_conf.h @@ -67,6 +67,7 @@ enum qemud_cmd_flags { QEMUD_CMD_FLAG_HOST_NET_ADD = QEMUD_CMD_FLAG_0_10, /* host_net_add monitor command */ QEMUD_CMD_FLAG_PCIDEVICE = (1 << 17), /* PCI device assignment only supported by qemu-kvm */ + QEMUD_CMD_FLAG_MEM_PATH = (1 << 18), /* mmap'ped guest backing supported */ }; /* Main driver state */ @@ -99,6 +100,8 @@ struct qemud_driver { char *vncListen; char *vncPassword; char *vncSASLdir; + char *hugetlbfs_mount; + char *hugepage_path; virCapsPtr caps; diff --git a/src/qemu_driver.c b/src/qemu_driver.c index b921b71f26..099cbe9c28 100644 --- a/src/qemu_driver.c +++ b/src/qemu_driver.c @@ -555,6 +555,37 @@ qemudStartup(int privileged) { goto error; } + /* If hugetlbfs is present, then we need to create a sub-directory within + * it, since we can't assume the root mount point has permissions that + * will let our spawned QEMU instances use it. + * + * NB the check for '/', since user may config "" to disable hugepages + * even when mounted + */ + if (qemu_driver->hugetlbfs_mount && + qemu_driver->hugetlbfs_mount[0] == '/') { + char *mempath = NULL; + if (virAsprintf(&mempath, "%s/libvirt/qemu", qemu_driver->hugetlbfs_mount) < 0) + goto out_of_memory; + + if ((rc = virFileMakePath(mempath)) != 0) { + virReportSystemError(NULL, rc, + _("unable to create hugepage path %s"), mempath); + VIR_FREE(mempath); + goto error; + } + if (qemu_driver->privileged && + chown(mempath, qemu_driver->user, qemu_driver->group) < 0) { + virReportSystemError(NULL, errno, + _("unable to set ownership on %s to %d:%d"), + mempath, qemu_driver->user, qemu_driver->group); + VIR_FREE(mempath); + goto error; + } + + qemu_driver->hugepage_path = mempath; + } + /* Get all the running persistent or transient configs first */ if (virDomainLoadAllConfigs(NULL, qemu_driver->caps, @@ -686,6 +717,8 @@ qemudShutdown(void) { VIR_FREE(qemu_driver->vncPassword); VIR_FREE(qemu_driver->vncSASLdir); VIR_FREE(qemu_driver->saveImageFormat); + VIR_FREE(qemu_driver->hugetlbfs_mount); + VIR_FREE(qemu_driver->hugepage_path); /* Free domain callback list */ virDomainEventCallbackListFree(qemu_driver->domainEventCallbacks); diff --git a/src/util.c b/src/util.c index 0d4f3fa495..282f7d9188 100644 --- a/src/util.c +++ b/src/util.c @@ -60,7 +60,9 @@ #if HAVE_CAPNG #include #endif - +#ifdef HAVE_MNTENT_H +#include +#endif #include "virterror_internal.h" #include "logging.h" @@ -1983,3 +1985,37 @@ int virGetGroupID(virConnectPtr conn, return 0; } #endif + + +#ifdef HAVE_MNTENT_H +/* search /proc/mounts for mount point of *type; return pointer to + * malloc'ed string of the path if found, otherwise return NULL + * with errno set to an appropriate value. + */ +char *virFileFindMountPoint(const char *type) +{ + FILE *f; + struct mntent mb; + char mntbuf[1024]; + char *ret = NULL; + + f = setmntent("/proc/mounts", "r"); + if (!f) + return NULL; + + while (getmntent_r(f, &mb, mntbuf, sizeof(mntbuf))) { + if (STREQ(mb.mnt_type, type)) { + ret = strdup(mb.mnt_dir); + goto cleanup; + } + } + + if (!ret) + errno = ENOENT; + +cleanup: + endmntent(f); + + return ret; +} +#endif diff --git a/src/util.h b/src/util.h index b3e628ae77..896e1b4af8 100644 --- a/src/util.h +++ b/src/util.h @@ -233,4 +233,8 @@ int virGetGroupID(virConnectPtr conn, int virRandomInitialize(unsigned int seed); int virRandom(int max); +#ifdef HAVE_MNTENT_H +char *virFileFindMountPoint(const char *type); +#endif + #endif /* __VIR_UTIL_H__ */ diff --git a/tests/qemuhelptest.c b/tests/qemuhelptest.c index ad2045fa5e..a42a1ba09a 100644 --- a/tests/qemuhelptest.c +++ b/tests/qemuhelptest.c @@ -105,7 +105,8 @@ mymain(int argc, char **argv) QEMUD_CMD_FLAG_VNET_HDR | QEMUD_CMD_FLAG_MIGRATE_KVM_STDIO | QEMUD_CMD_FLAG_KVM | - QEMUD_CMD_FLAG_DRIVE_FORMAT, + QEMUD_CMD_FLAG_DRIVE_FORMAT | + QEMUD_CMD_FLAG_MEM_PATH, 9001, 1, 74); DO_TEST("qemu-0.10.5", QEMUD_CMD_FLAG_KQEMU | @@ -136,7 +137,8 @@ mymain(int argc, char **argv) QEMUD_CMD_FLAG_DRIVE_FORMAT | QEMUD_CMD_FLAG_VGA | QEMUD_CMD_FLAG_0_10 | - QEMUD_CMD_FLAG_PCIDEVICE, + QEMUD_CMD_FLAG_PCIDEVICE | + QEMUD_CMD_FLAG_MEM_PATH, 10005, 1, 0); DO_TEST("kvm-86", QEMUD_CMD_FLAG_VNC_COLON | diff --git a/tests/qemuxml2argvdata/qemuxml2argv-hugepages.args b/tests/qemuxml2argvdata/qemuxml2argv-hugepages.args new file mode 100644 index 0000000000..f10a40e573 --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages.args @@ -0,0 +1 @@ +LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test /usr/bin/qemu -S -M pc -m 214 -mem-path /dev/hugepages/libvirt/qemu -smp 1 -nographic -monitor unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -hda /dev/HostVG/QEMUGuest1 -net none -serial none -parallel none -usb diff --git a/tests/qemuxml2argvdata/qemuxml2argv-hugepages.xml b/tests/qemuxml2argvdata/qemuxml2argv-hugepages.xml new file mode 100644 index 0000000000..e25286feee --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-hugepages.xml @@ -0,0 +1,25 @@ + + QEMUGuest1 + c7a5fdbd-edaf-9455-926a-d65c16db1809 + 219200 + 219200 + + + + 1 + + hvm + + + + destroy + restart + destroy + + /usr/bin/qemu + + + + + + diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 6f25e7d77f..ade57b16ec 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -160,7 +160,11 @@ mymain(int argc, char **argv) if ((driver.caps = testQemuCapsInit()) == NULL) return EXIT_FAILURE; - if((driver.stateDir = strdup("/nowhere")) == NULL) + if ((driver.stateDir = strdup("/nowhere")) == NULL) + return EXIT_FAILURE; + if ((driver.hugetlbfs_mount = strdup("/dev/hugepages")) == NULL) + return EXIT_FAILURE; + if ((driver.hugepage_path = strdup("/dev/hugepages/libvirt/qemu")) == NULL) return EXIT_FAILURE; #define DO_TEST_FULL(name, extraFlags, migrateFrom) \ @@ -189,6 +193,7 @@ mymain(int argc, char **argv) DO_TEST("bootloader", 0); DO_TEST("clock-utc", 0); DO_TEST("clock-localtime", 0); + DO_TEST("hugepages", QEMUD_CMD_FLAG_MEM_PATH); DO_TEST("disk-cdrom", 0); DO_TEST("disk-cdrom-empty", QEMUD_CMD_FLAG_DRIVE); DO_TEST("disk-floppy", 0); diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c index 7db76118c6..7f19f78af9 100644 --- a/tests/qemuxml2xmltest.c +++ b/tests/qemuxml2xmltest.c @@ -92,6 +92,7 @@ mymain(int argc, char **argv) DO_TEST("bootloader"); DO_TEST("clock-utc"); DO_TEST("clock-localtime"); + DO_TEST("hugepages"); DO_TEST("disk-cdrom"); DO_TEST("disk-floppy"); DO_TEST("disk-many");