qemu: Allow NVMe disk in CGroups

If a domain has an NVMe disk configured, then we need to allow it
on devices CGroup so that qemu can access it. There is one caveat
though - if an NVMe disk is read only we need CGroup to allow
write too. This is because when opening the device, qemu does
couple of ioctl()-s which are considered as write.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
Reviewed-by: Cole Robinson <crobinso@redhat.com>

src/qemu/qemu_cgroup.c

@@ -118,18 +118,37 @@ qemuSetupImageCgroupInternal(virDomainObjPtr vm,
                              virStorageSourcePtr src,
                              bool forceReadonly)
 {
+    g_autofree char *path = NULL;
+    bool readonly = src->readonly || forceReadonly;
+
+    if (src->type == VIR_STORAGE_TYPE_NVME) {
+        /* Even though disk is R/O we can't make it so in
+         * CGroups. QEMU will try to do some ioctl()-s over the
+         * device and such operations are considered R/W by the
+         * kernel */
+        readonly = false;
+
+        if (!(path = virPCIDeviceAddressGetIOMMUGroupDev(&src->nvme->pciAddr)))
+            return -1;
+
+        if (qemuSetupImagePathCgroup(vm, QEMU_DEV_VFIO, false) < 0)
+            return -1;
+    } else {
         if (!src->path || !virStorageSourceIsLocalStorage(src)) {
             VIR_DEBUG("Not updating cgroups for disk path '%s', type: %s",
                       NULLSTR(src->path), virStorageTypeToString(src->type));
             return 0;
         }
 
+        path = g_strdup(src->path);
+    }
+
     if (virStoragePRDefIsManaged(src->pr) &&
         virFileExists(QEMU_DEVICE_MAPPER_CONTROL_PATH) &&
         qemuSetupImagePathCgroup(vm, QEMU_DEVICE_MAPPER_CONTROL_PATH, false) < 0)
         return -1;
 
-    return qemuSetupImagePathCgroup(vm, src->path, src->readonly || forceReadonly);
+    return qemuSetupImagePathCgroup(vm, path, readonly);
 }
 
 
@@ -146,7 +165,10 @@ qemuTeardownImageCgroup(virDomainObjPtr vm,
                         virStorageSourcePtr src)
 {
     qemuDomainObjPrivatePtr priv = vm->privateData;
+    g_autofree char *path = NULL;
     int perms = VIR_CGROUP_DEVICE_RWM;
+    bool hasPR = false;
+    bool hasNVMe = false;
     size_t i;
     int ret;
 
@@ -154,13 +176,6 @@ qemuTeardownImageCgroup(virDomainObjPtr vm,
                                 VIR_CGROUP_CONTROLLER_DEVICES))
         return 0;
 
-    if (!src->path || !virStorageSourceIsLocalStorage(src)) {
-        VIR_DEBUG("Not updating cgroups for disk path '%s', type: %s",
-                  NULLSTR(src->path), virStorageTypeToString(src->type));
-        return 0;
-    }
-
-    if (virFileExists(QEMU_DEVICE_MAPPER_CONTROL_PATH)) {
     for (i = 0; i < vm->def->ndisks; i++) {
         virStorageSourcePtr diskSrc = vm->def->disks[i]->src;
 
@@ -168,10 +183,37 @@ qemuTeardownImageCgroup(virDomainObjPtr vm,
             continue;
 
         if (virStoragePRDefIsManaged(diskSrc->pr))
-                break;
+            hasPR = true;
+
+        if (virStorageSourceChainHasNVMe(diskSrc))
+            hasNVMe = true;
     }
 
-        if (i == vm->def->ndisks) {
+    if (src->type == VIR_STORAGE_TYPE_NVME) {
+        if (!(path = virPCIDeviceAddressGetIOMMUGroupDev(&src->nvme->pciAddr)))
+            return -1;
+
+        if (!hasNVMe &&
+            !qemuDomainNeedsVFIO(vm->def)) {
+            ret = virCgroupDenyDevicePath(priv->cgroup, QEMU_DEV_VFIO, perms, true);
+            virDomainAuditCgroupPath(vm, priv->cgroup, "deny",
+                                     QEMU_DEV_VFIO,
+                                     virCgroupGetDevicePermsString(perms), ret);
+            if (ret < 0)
+                return -1;
+        }
+    } else {
+        if (!src->path || !virStorageSourceIsLocalStorage(src)) {
+            VIR_DEBUG("Not updating cgroups for disk path '%s', type: %s",
+                      NULLSTR(src->path), virStorageTypeToString(src->type));
+            return 0;
+        }
+
+        path = g_strdup(src->path);
+    }
+
+    if (!hasPR &&
+        virFileExists(QEMU_DEVICE_MAPPER_CONTROL_PATH)) {
         VIR_DEBUG("Disabling device mapper control");
         ret = virCgroupDenyDevicePath(priv->cgroup,
                                       QEMU_DEVICE_MAPPER_CONTROL_PATH,
@@ -182,13 +224,12 @@ qemuTeardownImageCgroup(virDomainObjPtr vm,
         if (ret < 0)
             return ret;
     }
-    }
 
-    VIR_DEBUG("Deny path %s", src->path);
+    VIR_DEBUG("Deny path %s", path);
 
-    ret = virCgroupDenyDevicePath(priv->cgroup, src->path, perms, true);
+    ret = virCgroupDenyDevicePath(priv->cgroup, path, perms, true);
 
-    virDomainAuditCgroupPath(vm, priv->cgroup, "deny", src->path,
+    virDomainAuditCgroupPath(vm, priv->cgroup, "deny", path,
                              virCgroupGetDevicePermsString(perms), ret);
 
     /* If you're looking for a counter part to