qemu: Utilize virFileFindHugeTLBFS

Use better detection of hugetlbfs mount points. Yes, there can be
multiple mount points each serving different huge page size.

Since we already have ability to override the mount point in the
qemu.conf file, this crazy backward compatibility code is brought in.
Now we allow multiple mount points, so the "hugetlbfs_mount" option
must take an list of strings (mount points). But previously, it was
just a string, so we must accept both types now.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

src/qemu/qemu.conf

@@ -330,7 +330,14 @@
 # unspecified here, determination of a host mount point in /proc/mounts
 # will be attempted.  Specifying an explicit mount overrides detection
 # of the same in /proc/mounts.  Setting the mount point to "" will
-# disable guest hugepage backing.
+# disable guest hugepage backing. If desired, multiple mount points can
+# be specified at once, separated by comma and enclosed in square
+# brackets, for example:
+#
+#     hugetlbfs_mount = ["/dev/hugepages2M", "/dev/hugepages1G"]
+#
+# The size of huge page served by specific mount point is determined by
+# libvirt at the daemon startup.
 #
 # NB, within this mount point, guests will create memory backing files
 # in a location of $MOUNTPOINT/libvirt/qemu

src/qemu/qemu_command.c

@@ -7384,14 +7384,12 @@ qemuBuildCommandLine(virConnectPtr conn,
     def->mem.max_balloon = VIR_DIV_UP(def->mem.max_balloon, 1024) * 1024;
     virCommandAddArgFormat(cmd, "%llu", def->mem.max_balloon / 1024);
     if (def->mem.hugepage_backed) {
-        if (!cfg->hugetlbfsMount) {
+        char *mem_path;
+
+        if (!cfg->nhugetlbfs) {
             virReportError(VIR_ERR_INTERNAL_ERROR,
-                           "%s", _("hugetlbfs filesystem is not mounted"));
+                           "%s", _("hugetlbfs filesystem is not mounted "
-            goto error;
+                                   "or disabled by administrator config"));
-        }
-        if (!cfg->hugepagePath) {
-            virReportError(VIR_ERR_INTERNAL_ERROR,
-                           "%s", _("hugepages are disabled by administrator config"));
             goto error;
         }
         if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MEM_PATH)) {
@@ -7400,8 +7398,14 @@ qemuBuildCommandLine(virConnectPtr conn,
                            def->emulator);
             goto error;
         }
+
+        if (!(mem_path = qemuGetDefaultHugepath(cfg->hugetlbfs,
+                                                cfg->nhugetlbfs)))
+            goto error;
+
         virCommandAddArgList(cmd, "-mem-prealloc", "-mem-path",
-                             cfg->hugepagePath, NULL);
+                             mem_path, NULL);
+        VIR_FREE(mem_path);
     }
 
     if (def->mem.locked && !virQEMUCapsGet(qemuCaps, QEMU_CAPS_MLOCK)) {

src/qemu/qemu_conf.c

@@ -230,19 +230,17 @@ virQEMUDriverConfigPtr virQEMUDriverConfigNew(bool privileged)
     cfg->migrationPortMin = QEMU_MIGRATION_PORT_MIN;
     cfg->migrationPortMax = QEMU_MIGRATION_PORT_MAX;
 
-#if defined HAVE_MNTENT_H && defined HAVE_GETMNTENT_R
+    /* For privileged driver, try and find hugetlbfs mounts automatically.
-    /* For privileged driver, try and find hugepage mount automatically.
      * Non-privileged driver requires admin to create a dir for the
-     * user, chown it, and then let user configure it manually */
+     * user, chown it, and then let user configure it manually. */
     if (privileged &&
-        !(cfg->hugetlbfsMount = virFileFindMountPoint("hugetlbfs"))) {
+        virFileFindHugeTLBFS(&cfg->hugetlbfs, &cfg->nhugetlbfs) < 0) {
-        if (errno != ENOENT) {
+        /* This however is not implemented on all platforms. */
-            virReportSystemError(errno, "%s",
+        virErrorPtr err = virGetLastError();
-                                 _("unable to find hugetlbfs mountpoint"));
+        if (err && err->code != VIR_ERR_NO_SUPPORT)
             goto error;
     }
-    }
+
-#endif
     if (VIR_STRDUP(cfg->bridgeHelperName, "/usr/libexec/qemu-bridge-helper") < 0)
         goto error;
 
@@ -293,8 +291,11 @@ static void virQEMUDriverConfigDispose(void *obj)
     VIR_FREE(cfg->spicePassword);
     VIR_FREE(cfg->spiceSASLdir);
 
-    VIR_FREE(cfg->hugetlbfsMount);
+    while (cfg->nhugetlbfs) {
-    VIR_FREE(cfg->hugepagePath);
+        cfg->nhugetlbfs--;
+        VIR_FREE(cfg->hugetlbfs[cfg->nhugetlbfs].mnt_dir);
+    }
+    VIR_FREE(cfg->hugetlbfs);
     VIR_FREE(cfg->bridgeHelperName);
 
     VIR_FREE(cfg->saveImageFormat);
@@ -307,6 +308,26 @@ static void virQEMUDriverConfigDispose(void *obj)
 }
 
 
+static int
+virQEMUDriverConfigHugeTLBFSInit(virHugeTLBFSPtr hugetlbfs,
+                                 const char *path,
+                                 bool deflt)
+{
+    int ret = -1;
+
+    if (VIR_STRDUP(hugetlbfs->mnt_dir, path) < 0)
+        goto cleanup;
+
+    if (virFileGetHugepageSize(path, &hugetlbfs->size) < 0)
+        goto cleanup;
+
+    hugetlbfs->deflt = deflt;
+    ret = 0;
+ cleanup:
+    return ret;
+}
+
+
 int virQEMUDriverConfigLoadFile(virQEMUDriverConfigPtr cfg,
                                 const char *filename)
 {
@@ -555,7 +576,59 @@ int virQEMUDriverConfigLoadFile(virQEMUDriverConfigPtr cfg,
     GET_VALUE_BOOL("auto_dump_bypass_cache", cfg->autoDumpBypassCache);
     GET_VALUE_BOOL("auto_start_bypass_cache", cfg->autoStartBypassCache);
 
-    GET_VALUE_STR("hugetlbfs_mount", cfg->hugetlbfsMount);
+    /* Some crazy backcompat. Back in the old days, this was just a pure
+     * string. We must continue supporting it. These days however, this may be
+     * an array of strings. */
+    p = virConfGetValue(conf, "hugetlbfs_mount");
+    if (p) {
+        /* There already might be something autodetected. Avoid leaking it. */
+        while (cfg->nhugetlbfs) {
+            cfg->nhugetlbfs--;
+            VIR_FREE(cfg->hugetlbfs[cfg->nhugetlbfs].mnt_dir);
+        }
+        VIR_FREE(cfg->hugetlbfs);
+
+        if (p->type == VIR_CONF_LIST) {
+            size_t len = 0;
+            virConfValuePtr pp = p->list;
+
+            /* Calc length and check items */
+            while (pp) {
+                if (pp->type != VIR_CONF_STRING) {
+                    virReportError(VIR_ERR_CONF_SYNTAX, "%s",
+                                   _("hugetlbfs_mount must be a list of strings"));
+                    goto cleanup;
+                }
+                len++;
+                pp = pp->next;
+            }
+
+            if (len && VIR_ALLOC_N(cfg->hugetlbfs, len) < 0)
+                goto cleanup;
+            cfg->nhugetlbfs = len;
+
+            pp = p->list;
+            len = 0;
+            while (pp) {
+                if (virQEMUDriverConfigHugeTLBFSInit(&cfg->hugetlbfs[len],
+                                                     pp->str, !len) < 0)
+                    goto cleanup;
+                len++;
+                pp = pp->next;
+            }
+        } else {
+            CHECK_TYPE("hugetlbfs_mount", VIR_CONF_STRING);
+            if (STRNEQ(p->str, "")) {
+                if (VIR_ALLOC_N(cfg->hugetlbfs, 1) < 0)
+                    goto cleanup;
+                cfg->nhugetlbfs = 1;
+                if (virQEMUDriverConfigHugeTLBFSInit(&cfg->hugetlbfs[0],
+                                                     p->str, true) < 0)
+                    goto cleanup;
+            }
+        }
+    }
+
     GET_VALUE_STR("bridge_helper", cfg->bridgeHelperName);
 
     GET_VALUE_BOOL("mac_filter", cfg->macFilter);
@@ -1406,3 +1479,30 @@ qemuTranslateSnapshotDiskSourcePool(virConnectPtr conn ATTRIBUTE_UNUSED,
                    _("Snapshots are not yet supported with 'pool' volumes"));
     return -1;
 }
+
+char *
+qemuGetHugepagePath(virHugeTLBFSPtr hugepage)
+{
+    char *ret;
+
+    if (virAsprintf(&ret, "%s/libvirt/qemu", hugepage->mnt_dir) < 0)
+        return NULL;
+
+    return ret;
+}
+
+char *
+qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs,
+                       size_t nhugetlbfs)
+{
+    size_t i;
+
+    for (i = 0; i < nhugetlbfs; i++)
+        if (hugetlbfs[i].deflt)
+            break;
+
+    if (i == nhugetlbfs)
+        i = 0;
+
+    return qemuGetHugepagePath(&hugetlbfs[i]);
+}

src/qemu/qemu_conf.h

@@ -45,6 +45,7 @@
 # include "qemu_capabilities.h"
 # include "virclosecallbacks.h"
 # include "virhostdev.h"
+# include "virfile.h"
 
 # ifdef CPU_SETSIZE /* Linux */
 #  define QEMUD_CPUMASK_LEN CPU_SETSIZE
@@ -126,8 +127,9 @@ struct _virQEMUDriverConfig {
     int webSocketPortMin;
     int webSocketPortMax;
 
-    char *hugetlbfsMount;
+    virHugeTLBFSPtr hugetlbfs;
-    char *hugepagePath;
+    size_t nhugetlbfs;
+
     char *bridgeHelperName;
 
     bool macFilter;
@@ -311,4 +313,7 @@ int qemuTranslateDiskSourcePool(virConnectPtr conn,
 int qemuTranslateSnapshotDiskSourcePool(virConnectPtr conn,
                                         virDomainSnapshotDiskDefPtr def);
 
+char * qemuGetHugepagePath(virHugeTLBFSPtr hugepage);
+char * qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs,
+                              size_t nhugetlbfs);
 #endif /* __QEMUD_CONF_H */

src/qemu/qemu_driver.c

@@ -627,11 +627,11 @@ qemuStateInitialize(bool privileged,
     char *driverConf = NULL;
     virConnectPtr conn = NULL;
     char ebuf[1024];
-    char *membase = NULL;
-    char *mempath = NULL;
     virQEMUDriverConfigPtr cfg;
     uid_t run_uid = -1;
     gid_t run_gid = -1;
+    char *hugepagePath = NULL;
+    size_t i;
 
     if (VIR_ALLOC(qemu_driver) < 0)
         return -1;
@@ -803,37 +803,33 @@ qemuStateInitialize(bool privileged,
 
     /* If hugetlbfs is present, then we need to create a sub-directory within
      * it, since we can't assume the root mount point has permissions that
-     * will let our spawned QEMU instances use it.
+     * will let our spawned QEMU instances use it. */
-     *
+    for (i = 0; i < cfg->nhugetlbfs; i++) {
-     * NB the check for '/', since user may config "" to disable hugepages
+        hugepagePath = qemuGetHugepagePath(&cfg->hugetlbfs[i]);
-     * even when mounted
+
-     */
+        if (!hugepagePath)
-    if (cfg->hugetlbfsMount &&
-        cfg->hugetlbfsMount[0] == '/') {
-        if (virAsprintf(&membase, "%s/libvirt",
-                        cfg->hugetlbfsMount) < 0 ||
-            virAsprintf(&mempath, "%s/qemu", membase) < 0)
             goto error;
 
-        if (virFileMakePath(mempath) < 0) {
+        if (virFileMakePath(hugepagePath) < 0) {
             virReportSystemError(errno,
-                                 _("unable to create hugepage path %s"), mempath);
+                                 _("unable to create hugepage path %s"),
+                                 hugepagePath);
             goto error;
         }
         if (cfg->privileged) {
-            if (virFileUpdatePerm(membase, 0, S_IXGRP | S_IXOTH) < 0)
+            if (virFileUpdatePerm(cfg->hugetlbfs[i].mnt_dir,
+                                  0, S_IXGRP | S_IXOTH) < 0)
                 goto error;
-            if (chown(mempath, cfg->user, cfg->group) < 0) {
+            if (chown(hugepagePath, cfg->user, cfg->group) < 0) {
                 virReportSystemError(errno,
                                      _("unable to set ownership on %s to %d:%d"),
-                                     mempath, (int) cfg->user,
+                                     hugepagePath,
+                                     (int) cfg->user,
                                      (int) cfg->group);
                 goto error;
             }
         }
-        VIR_FREE(membase);
+        VIR_FREE(hugepagePath);
-
-        cfg->hugepagePath = mempath;
     }
 
     if (!(qemu_driver->closeCallbacks = virCloseCallbacksNew()))
@@ -894,8 +890,7 @@ qemuStateInitialize(bool privileged,
  error:
     virObjectUnref(conn);
     VIR_FREE(driverConf);
-    VIR_FREE(membase);
+    VIR_FREE(hugepagePath);
-    VIR_FREE(mempath);
     qemuStateCleanup();
     return -1;
 }

src/qemu/qemu_process.c

@@ -3791,13 +3791,22 @@ int qemuProcessStart(virConnectPtr conn,
     }
     virDomainAuditSecurityLabel(vm, true);
 
-    if (cfg->hugepagePath && vm->def->mem.hugepage_backed) {
+    if (vm->def->mem.hugepage_backed) {
+        for (i = 0; i < cfg->nhugetlbfs; i++) {
+            char *hugepagePath = qemuGetHugepagePath(&cfg->hugetlbfs[i]);
+
+            if (!hugepagePath)
+                goto cleanup;
+
             if (virSecurityManagerSetHugepages(driver->securityManager,
-                                           vm->def, cfg->hugepagePath) < 0) {
+                                               vm->def, hugepagePath) < 0) {
                 virReportError(VIR_ERR_INTERNAL_ERROR,
                                "%s", _("Unable to set huge path in security driver"));
+                VIR_FREE(hugepagePath);
                 goto cleanup;
             }
+            VIR_FREE(hugepagePath);
+        }
     }
 
     /* Ensure no historical cgroup for this VM is lying around bogus

tests/qemuxml2argvtest.c

@@ -524,12 +524,14 @@ mymain(void)
     VIR_FREE(driver.config->stateDir);
     if (VIR_STRDUP_QUIET(driver.config->stateDir, "/nowhere") < 0)
         return EXIT_FAILURE;
-    VIR_FREE(driver.config->hugetlbfsMount);
+    VIR_FREE(driver.config->hugetlbfs);
-    if (VIR_STRDUP_QUIET(driver.config->hugetlbfsMount, "/dev/hugepages") < 0)
+    if (VIR_ALLOC_N(driver.config->hugetlbfs, 1) < 0)
         return EXIT_FAILURE;
-    VIR_FREE(driver.config->hugepagePath);
+    driver.config->nhugetlbfs = 1;
-    if (VIR_STRDUP_QUIET(driver.config->hugepagePath, "/dev/hugepages/libvirt/qemu") < 0)
+    if (VIR_STRDUP(driver.config->hugetlbfs[0].mnt_dir, "/dev/hugepages") < 0)
         return EXIT_FAILURE;
+    driver.config->hugetlbfs[0].size = 2048;
+    driver.config->hugetlbfs[0].deflt = true;
     driver.config->spiceTLS = 1;
     if (VIR_STRDUP_QUIET(driver.config->spicePassword, "123456") < 0)
         return EXIT_FAILURE;