qemu: Create hugepage path on per domain basis

If you've ever tried running a huge page backed guest under
different user than in qemu.conf, you probably failed. Problem is
even though we have corresponding APIs in the security drivers,
there's no implementation and thus we don't relabel the huge page
path. But even if we did, so far all of the domains share the
same path:

   /hugepageMount/libvirt/qemu

Our only option there would be to set 0777 mode on the qemu dir
which is totally unsafe. Therefore, we can create dir on
per-domain basis, i.e.:

   /hugepageMount/libvirt/qemu/domainName

and chown domainName dir to the user that domain is configured to
run under.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

src/qemu/qemu_command.c

@@ -3316,7 +3316,7 @@ qemuBuildMemoryBackendStr(unsigned long long size,
         return -1;
 
     if (pagesize) {
-        if (qemuGetHupageMemPath(cfg, pagesize, &mem_path) < 0)
+        if (qemuGetDomainHupageMemPath(def, cfg, pagesize, &mem_path) < 0)
             goto cleanup;
 
         *backendType = "memory-backend-file";
@@ -7270,7 +7270,7 @@ qemuBuildMemPathStr(virQEMUDriverConfigPtr cfg,
         return -1;
     }
 
-    if (qemuGetHupageMemPath(cfg, def->mem.hugepages[0].size, &mem_path) < 0)
+    if (qemuGetDomainHupageMemPath(def, cfg, def->mem.hugepages[0].size, &mem_path) < 0)
         return -1;
 
     virCommandAddArgList(cmd, "-mem-prealloc", "-mem-path", mem_path, NULL);

src/qemu/qemu_conf.c

@@ -1456,7 +1456,7 @@ qemuTranslateSnapshotDiskSourcePool(virConnectPtr conn ATTRIBUTE_UNUSED,
 }
 
 char *
-qemuGetHugepagePath(virHugeTLBFSPtr hugepage)
+qemuGetBaseHugepagePath(virHugeTLBFSPtr hugepage)
 {
     char *ret;
 
@@ -1467,8 +1467,25 @@ qemuGetHugepagePath(virHugeTLBFSPtr hugepage)
 }
 
 
+char *
+qemuGetDomainHugepagePath(const virDomainDef *def,
+                          virHugeTLBFSPtr hugepage)
+{
+    char *base = qemuGetBaseHugepagePath(hugepage);
+    char *domPath = virDomainObjGetShortName(def);
+    char *ret;
+
+    if (base && domPath)
+        ignore_value(virAsprintf(&ret, "%s/%s", base, domPath));
+    VIR_FREE(domPath);
+    VIR_FREE(base);
+    return ret;
+}
+
+
 /**
- * qemuGetDefaultHugepath:
+ * qemuGetDomainDefaultHugepath:
+ * @def: domain definition
  * @hugetlbfs: array of configured hugepages
  * @nhugetlbfs: number of item in the array
  *
@@ -1477,8 +1494,9 @@ qemuGetHugepagePath(virHugeTLBFSPtr hugepage)
  * Returns 0 on success, -1 otherwise.
  * */
 char *
-qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs,
-                       size_t nhugetlbfs)
+qemuGetDomainDefaultHugepath(const virDomainDef *def,
+                             virHugeTLBFSPtr hugetlbfs,
+                             size_t nhugetlbfs)
 {
     size_t i;
 
@@ -1489,12 +1507,12 @@ qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs,
     if (i == nhugetlbfs)
         i = 0;
 
-    return qemuGetHugepagePath(&hugetlbfs[i]);
+    return qemuGetDomainHugepagePath(def, &hugetlbfs[i]);
 }
 
 
 /**
- * qemuGetHupageMemPath: Construct HP enabled memory backend path
+ * qemuGetDomainHupageMemPath: Construct HP enabled memory backend path
  *
  * If no specific hugepage size is requested (@pagesize is zero)
  * the default hugepage size is used).
@@ -1504,9 +1522,10 @@ qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs,
  *        -1 otherwise.
  */
 int
-qemuGetHupageMemPath(virQEMUDriverConfigPtr cfg,
-                     unsigned long long pagesize,
-                     char **memPath)
+qemuGetDomainHupageMemPath(const virDomainDef *def,
+                           virQEMUDriverConfigPtr cfg,
+                           unsigned long long pagesize,
+                           char **memPath)
 {
     size_t i = 0;
 
@@ -1518,8 +1537,9 @@ qemuGetHupageMemPath(virQEMUDriverConfigPtr cfg,
     }
 
     if (!pagesize) {
-        if (!(*memPath = qemuGetDefaultHugepath(cfg->hugetlbfs,
-                                                cfg->nhugetlbfs)))
+        if (!(*memPath = qemuGetDomainDefaultHugepath(def,
+                                                      cfg->hugetlbfs,
+                                                      cfg->nhugetlbfs)))
             return -1;
     } else {
         for (i = 0; i < cfg->nhugetlbfs; i++) {
@@ -1535,7 +1555,7 @@ qemuGetHupageMemPath(virQEMUDriverConfigPtr cfg,
             return -1;
         }
 
-        if (!(*memPath = qemuGetHugepagePath(&cfg->hugetlbfs[i])))
+        if (!(*memPath = qemuGetDomainHugepagePath(def, &cfg->hugetlbfs[i])))
             return -1;
     }
 

src/qemu/qemu_conf.h

@@ -335,11 +335,15 @@ virDomainXMLOptionPtr virQEMUDriverCreateXMLConf(virQEMUDriverPtr driver);
 int qemuTranslateSnapshotDiskSourcePool(virConnectPtr conn,
                                         virDomainSnapshotDiskDefPtr def);
 
-char * qemuGetHugepagePath(virHugeTLBFSPtr hugepage);
-char * qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs,
-                              size_t nhugetlbfs);
+char * qemuGetBaseHugepagePath(virHugeTLBFSPtr hugepage);
+char * qemuGetDomainHugepagePath(const virDomainDef *def,
+                                 virHugeTLBFSPtr hugepage);
+char * qemuGetDomainDefaultHugepath(const virDomainDef *def,
+                                    virHugeTLBFSPtr hugetlbfs,
+                                    size_t nhugetlbfs);
 
-int qemuGetHupageMemPath(virQEMUDriverConfigPtr cfg,
-                         unsigned long long pagesize,
-                         char **memPath);
+int qemuGetDomainHupageMemPath(const virDomainDef *def,
+                               virQEMUDriverConfigPtr cfg,
+                               unsigned long long pagesize,
+                               char **memPath);
 #endif /* __QEMUD_CONF_H */

src/qemu/qemu_driver.c

@@ -857,7 +857,7 @@ qemuStateInitialize(bool privileged,
      * it, since we can't assume the root mount point has permissions that
      * will let our spawned QEMU instances use it. */
     for (i = 0; i < cfg->nhugetlbfs; i++) {
-        hugepagePath = qemuGetHugepagePath(&cfg->hugetlbfs[i]);
+        hugepagePath = qemuGetBaseHugepagePath(&cfg->hugetlbfs[i]);
 
         if (!hugepagePath)
             goto error;
@@ -868,19 +868,10 @@ qemuStateInitialize(bool privileged,
                                  hugepagePath);
             goto error;
         }
-        if (privileged) {
-            if (virFileUpdatePerm(cfg->hugetlbfs[i].mnt_dir,
-                                  0, S_IXGRP | S_IXOTH) < 0)
-                goto error;
-            if (chown(hugepagePath, cfg->user, cfg->group) < 0) {
-                virReportSystemError(errno,
-                                     _("unable to set ownership on %s to %d:%d"),
-                                     hugepagePath,
-                                     (int) cfg->user,
-                                     (int) cfg->group);
-                goto error;
-            }
-        }
+        if (privileged &&
+            virFileUpdatePerm(cfg->hugetlbfs[i].mnt_dir,
+                              0, S_IXGRP | S_IXOTH) < 0)
+            goto error;
         VIR_FREE(hugepagePath);
     }
 

src/qemu/qemu_process.c

@@ -3224,6 +3224,54 @@ qemuProcessReconnectCheckMemAliasOrderMismatch(virDomainObjPtr vm)
 }
 
 
+static int
+qemuProcessBuildDestroyHugepagesPath(virQEMUDriverPtr driver,
+                                     virDomainObjPtr vm,
+                                     bool build)
+{
+    virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
+    char *hugepagePath = NULL;
+    size_t i;
+    int ret = -1;
+
+    if (vm->def->mem.nhugepages) {
+        for (i = 0; i < cfg->nhugetlbfs; i++) {
+            VIR_FREE(hugepagePath);
+            hugepagePath = qemuGetDomainHugepagePath(vm->def, &cfg->hugetlbfs[i]);
+
+            if (!hugepagePath)
+                goto cleanup;
+
+            if (build) {
+                if (virFileMakePathWithMode(hugepagePath, 0700) < 0) {
+                    virReportSystemError(errno,
+                                         _("Unable to create %s"),
+                                         hugepagePath);
+                    goto cleanup;
+                }
+
+                if (virSecurityManagerSetHugepages(driver->securityManager,
+                                                   vm->def, hugepagePath) < 0) {
+                    virReportError(VIR_ERR_INTERNAL_ERROR,
+                                   "%s", _("Unable to set huge path in security driver"));
+                    goto cleanup;
+                }
+            } else {
+                if (rmdir(hugepagePath) < 0)
+                    VIR_WARN("Unable to remove hugepage path: %s (errno=%d)",
+                             hugepagePath, errno);
+            }
+        }
+    }
+
+    ret = 0;
+ cleanup:
+    VIR_FREE(hugepagePath);
+    virObjectUnref(cfg);
+    return ret;
+}
+
+
 struct qemuProcessReconnectData {
     virConnectPtr conn;
     virQEMUDriverPtr driver;
@@ -3368,6 +3416,9 @@ qemuProcessReconnect(void *opaque)
         goto cleanup;
     }
 
+    if (qemuProcessBuildDestroyHugepagesPath(driver, obj, true) < 0)
+        goto error;
+
     if ((qemuDomainAssignAddresses(obj->def, priv->qemuCaps,
                                    driver, obj, false)) < 0) {
         goto error;
@@ -5233,7 +5284,6 @@ qemuProcessPrepareHost(virQEMUDriverPtr driver,
 {
     int ret = -1;
     unsigned int hostdev_flags = 0;
-    size_t i;
     qemuDomainObjPrivatePtr priv = vm->privateData;
     virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
 
@@ -5265,23 +5315,8 @@ qemuProcessPrepareHost(virQEMUDriverPtr driver,
                                NULL) < 0)
         goto cleanup;
 
-    if (vm->def->mem.nhugepages) {
-        for (i = 0; i < cfg->nhugetlbfs; i++) {
-            char *hugepagePath = qemuGetHugepagePath(&cfg->hugetlbfs[i]);
-
-            if (!hugepagePath)
-                goto cleanup;
-
-            if (virSecurityManagerSetHugepages(driver->securityManager,
-                                               vm->def, hugepagePath) < 0) {
-                virReportError(VIR_ERR_INTERNAL_ERROR,
-                               "%s", _("Unable to set huge path in security driver"));
-                VIR_FREE(hugepagePath);
-                goto cleanup;
-            }
-            VIR_FREE(hugepagePath);
-        }
-    }
+    if (qemuProcessBuildDestroyHugepagesPath(driver, vm, true) < 0)
+        goto cleanup;
 
     /* Ensure no historical cgroup for this VM is lying around bogus
      * settings */
@@ -5959,6 +5994,8 @@ void qemuProcessStop(virQEMUDriverPtr driver,
         goto endjob;
     }
 
+    qemuProcessBuildDestroyHugepagesPath(driver, vm, false);
+
     vm->def->id = -1;
 
     if (virAtomicIntDecAndTest(&driver->nactive) && driver->inhibitCallback)

tests/qemuxml2argvdata/qemuxml2argv-hugepages-numa.args

@@ -11,11 +11,11 @@ QEMU_AUDIO_DRV=spice \
 -m size=1048576k,slots=16,maxmem=1099511627776k \
 -smp 2,sockets=2,cores=1,threads=1 \
 -mem-prealloc \
--mem-path /dev/hugepages2M/libvirt/qemu \
+-mem-path /dev/hugepages2M/libvirt/qemu/-1-fedora \
 -numa node,nodeid=0,cpus=0-1,mem=1024 \
 -object memory-backend-file,id=memdimm0,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=1-3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-fedora,size=1073741824,\
+host-nodes=1-3,policy=bind \
 -device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0 \
 -uuid 63840878-0deb-4095-97e6-fc444d9bc9fa \
 -nodefaults \

tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages.args

@@ -11,20 +11,20 @@ QEMU_AUDIO_DRV=none \
 -m 4096 \
 -smp 4,sockets=4,cores=1,threads=1 \
 -object memory-backend-file,id=ram-node0,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=0-3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=0-3,policy=bind \
 -numa node,nodeid=0,cpus=0,memdev=ram-node0 \
 -object memory-backend-file,id=ram-node1,prealloc=yes,\
-mem-path=/dev/hugepages2M/libvirt/qemu,size=1073741824,host-nodes=0-3,\
-policy=bind \
+mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=0-3,policy=bind \
 -numa node,nodeid=1,cpus=1,memdev=ram-node1 \
 -object memory-backend-file,id=ram-node2,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=0-3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=0-3,policy=bind \
 -numa node,nodeid=2,cpus=2,memdev=ram-node2 \
 -object memory-backend-file,id=ram-node3,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=3,policy=bind \
 -numa node,nodeid=3,cpus=3,memdev=ram-node3 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -nographic \

tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages2.args

@@ -11,7 +11,7 @@ QEMU_AUDIO_DRV=none \
 -m 1024 \
 -smp 2,sockets=2,cores=1,threads=1 \
 -mem-prealloc \
--mem-path /dev/hugepages2M/libvirt/qemu \
+-mem-path /dev/hugepages2M/libvirt/qemu/-1-SomeDummyHugepagesGu \
 -numa node,nodeid=0,cpus=0,mem=256 \
 -numa node,nodeid=1,cpus=1,mem=768 \
 -uuid ef1bdff4-27f3-4e85-a807-5fb4d58463cc \

tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages3.args

@@ -13,7 +13,7 @@ QEMU_AUDIO_DRV=none \
 -object memory-backend-ram,id=ram-node0,size=268435456 \
 -numa node,nodeid=0,cpus=0,memdev=ram-node0 \
 -object memory-backend-file,id=ram-node1,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=805306368 \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-SomeDummyHugepagesGu,size=805306368 \
 -numa node,nodeid=1,cpus=1,memdev=ram-node1 \
 -uuid ef1bdff4-27f3-4e85-a807-5fb4d58463cc \
 -nographic \

tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.args

@@ -10,7 +10,7 @@ QEMU_AUDIO_DRV=none \
 -M pc \
 -m 1024 \
 -mem-prealloc \
--mem-path /dev/hugepages2M/libvirt/qemu \
+-mem-path /dev/hugepages2M/libvirt/qemu/-1-SomeDummyHugepagesGu \
 -smp 2,sockets=2,cores=1,threads=1 \
 -uuid ef1bdff4-27f3-4e85-a807-5fb4d58463cc \
 -nographic \

tests/qemuxml2argvdata/qemuxml2argv-hugepages-shared.args

@@ -11,20 +11,20 @@ QEMU_AUDIO_DRV=none \
 -m 4096 \
 -smp 4,sockets=4,cores=1,threads=1 \
 -object memory-backend-file,id=ram-node0,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=0-3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=0-3,policy=bind \
 -numa node,nodeid=0,cpus=0,memdev=ram-node0 \
 -object memory-backend-file,id=ram-node1,prealloc=yes,\
-mem-path=/dev/hugepages2M/libvirt/qemu,share=yes,size=1073741824,\
+mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,share=yes,size=1073741824,\
 host-nodes=0-3,policy=bind \
 -numa node,nodeid=1,cpus=1,memdev=ram-node1 \
 -object memory-backend-file,id=ram-node2,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,share=no,size=1073741824,host-nodes=0-3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,share=no,size=1073741824,\
+host-nodes=0-3,policy=bind \
 -numa node,nodeid=2,cpus=2,memdev=ram-node2 \
 -object memory-backend-file,id=ram-node3,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=3,policy=bind \
 -numa node,nodeid=3,cpus=3,memdev=ram-node3 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -nographic \

tests/qemuxml2argvdata/qemuxml2argv-hugepages.args

@@ -10,7 +10,7 @@ QEMU_AUDIO_DRV=none \
 -M pc \
 -m 214 \
 -mem-prealloc \
--mem-path /dev/hugepages2M/libvirt/qemu \
+-mem-path /dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1 \
 -smp 1,sockets=1,cores=1,threads=1 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -nographic \

tests/qemuxml2argvdata/qemuxml2argv-memory-hotplug-dimm-addr.args

@@ -12,8 +12,8 @@ QEMU_AUDIO_DRV=none \
 -smp 2,sockets=2,cores=1,threads=1 \
 -numa node,nodeid=0,cpus=0-1,mem=214 \
 -object memory-backend-file,id=memdimm0,prealloc=yes,\
-mem-path=/dev/hugepages2M/libvirt/qemu,size=536870912,host-nodes=1-3,\
-policy=bind \
+mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,size=536870912,\
+host-nodes=1-3,policy=bind \
 -device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0,addr=4294967296 \
 -object memory-backend-ram,id=memdimm2,size=536870912 \
 -device pc-dimm,node=0,memdev=memdimm2,id=dimm2,slot=2 \

tests/qemuxml2argvdata/qemuxml2argv-memory-hotplug-dimm.args

@@ -14,8 +14,8 @@ QEMU_AUDIO_DRV=none \
 -object memory-backend-ram,id=memdimm0,size=536870912 \
 -device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0 \
 -object memory-backend-file,id=memdimm1,prealloc=yes,\
-mem-path=/dev/hugepages2M/libvirt/qemu,size=536870912,host-nodes=1-3,\
-policy=bind \
+mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,size=536870912,\
+host-nodes=1-3,policy=bind \
 -device pc-dimm,node=0,memdev=memdimm1,id=dimm1,slot=1 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -nographic \