qemu: Implement support for configuring iothread to virtqueue mapping for disks

Add validation and formatting of the commandline arguments for
'iothread-vq-mapping' parameter. The validation logic mirrors what qemu
allows.

Signed-off-by: Peter Krempa <pkrempa@redhat.com>
Reviewed-by: Ján Tomko <jtomko@redhat.com>

src/hypervisor/domain_driver.c

@@ -555,7 +555,20 @@ virDomainDriverDelIOThreadCheck(virDomainDef *def,
     }
 
     for (i = 0; i < def->ndisks; i++) {
-        if (def->disks[i]->iothread == iothread_id) {
+        GSList *n;
+        bool inuse = false;
+
+        for (n = def->disks[i]->iothreads; n; n = n->next) {
+            virDomainDiskIothreadDef *iothread = n->data;
+
+            if (iothread->id == iothread_id) {
+                inuse = true;
+                break;
+            }
+        }
+
+        if (inuse ||
+            def->disks[i]->iothread == iothread_id) {
             virReportError(VIR_ERR_INVALID_ARG,
                            _("cannot remove IOThread %1$u since it is being used by disk '%2$s'"),
                            iothread_id, def->disks[i]->dst);

src/qemu/qemu_command.c

@@ -1731,6 +1731,45 @@ qemuBuildDriveStr(virDomainDiskDef *disk)
 }
 
 
+static virJSONValue *
+qemuBuildDiskDeviceIothreadMappingProps(GSList *iothreads)
+{
+    g_autoptr(virJSONValue) ret = virJSONValueNewArray();
+    GSList *n;
+
+    for (n = iothreads; n; n = n->next) {
+        virDomainDiskIothreadDef *ioth = n->data;
+        g_autoptr(virJSONValue) props = NULL;
+        g_autoptr(virJSONValue) queues = NULL;
+        g_autofree char *alias = g_strdup_printf("iothread%u", ioth->id);
+        size_t i;
+
+        if (ioth->nqueues > 0) {
+            queues = virJSONValueNewArray();
+
+            for (i = 0; i < ioth->nqueues; i++) {
+                g_autoptr(virJSONValue) vq = virJSONValueNewNumberUint(ioth->queues[i]);
+
+                if (virJSONValueArrayAppend(queues, &vq))
+                    return NULL;
+            }
+        }
+
+        if (virJSONValueObjectAdd(&props,
+                                  "s:iothread", alias,
+                                  "A:vqs", &queues,
+                                  NULL) < 0)
+            return NULL;
+
+
+        if (virJSONValueArrayAppend(ret, &props))
+            return NULL;
+    }
+
+    return g_steal_pointer(&ret);
+}
+
+
 virJSONValue *
 qemuBuildDiskDeviceProps(const virDomainDef *def,
                          virDomainDiskDef *disk,
@@ -1792,11 +1831,16 @@ qemuBuildDiskDeviceProps(const virDomainDef *def,
 
     case VIR_DOMAIN_DISK_BUS_VIRTIO: {
         virTristateSwitch scsi = VIR_TRISTATE_SWITCH_ABSENT;
+        g_autoptr(virJSONValue) iothreadMapping = NULL;
         g_autofree char *iothread = NULL;
 
         if (disk->iothread > 0)
             iothread = g_strdup_printf("iothread%u", disk->iothread);
 
+        if (disk->iothreads &&
+            !(iothreadMapping = qemuBuildDiskDeviceIothreadMappingProps(disk->iothreads)))
+            return NULL;
+
         if (virStorageSourceGetActualType(disk->src) != VIR_STORAGE_TYPE_VHOST_USER &&
             virQEMUCapsGet(qemuCaps, QEMU_CAPS_VIRTIO_BLK_SCSI)) {
             /* if sg_io is true but the scsi option isn't supported,
@@ -1820,6 +1864,7 @@ qemuBuildDiskDeviceProps(const virDomainDef *def,
                                   "T:scsi", scsi,
                                   "p:num-queues", disk->queues,
                                   "p:queue-size", disk->queue_size,
+                                  "A:iothread-vq-mapping", &iothreadMapping,
                                   NULL) < 0)
             return NULL;
     }

src/qemu/qemu_validate.c

@@ -2669,12 +2669,24 @@ qemuValidateDomainDeviceDefDiskSerial(const char *value)
 }
 
 
-static bool
+static int
 qemuValidateDomainDeviceDefDiskIOThreads(const virDomainDef *def,
-                                         const virDomainDiskDef *disk)
+                                         const virDomainDiskDef *disk,
+                                         virQEMUCaps *qemuCaps)
 {
+    if (disk->iothread == 0 && !disk->iothreads)
+        return 0;
+
     switch (disk->bus) {
     case VIR_DOMAIN_DISK_BUS_VIRTIO:
+        if (disk->iothreads) {
+            if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_VIRTIO_BLK_IOTHREAD_MAPPING)) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                               _("IOThread mapping for disk '%1$s' is not available with this QEMU binary"),
+                               disk->dst);
+                return -1;
+            }
+        }
         break;
 
     case VIR_DOMAIN_DISK_BUS_IDE:
@@ -2690,18 +2702,101 @@ qemuValidateDomainDeviceDefDiskIOThreads(const virDomainDef *def,
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                        _("IOThreads not available for bus %1$s target %2$s"),
                        virDomainDiskBusTypeToString(disk->bus), disk->dst);
-        return false;
+        return -1;
     }
 
-    /* Can we find the disk iothread in the iothreadid list? */
+    if (disk->iothreads) {
+        virDomainDiskIothreadDef *first_ioth = disk->iothreads->data;
+        g_autoptr(virBitmap) queueMap = NULL;
+        g_autoptr(GHashTable) iothreads = virHashNew(NULL);
+        ssize_t unused;
+        GSList *n;
+
+        if (first_ioth->queues) {
+            if (disk->queues == 0) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                               _("disk 'queue' count must be configured for explicit iothread to queue mapping"));
+                return -1;
+            }
+
+            queueMap = virBitmapNew(disk->queues);
+        }
+
+        /* we are validating that:
+         * - there are no duplicate iothreads
+         * - there are only valid iothreads
+         * - if queue mapping is provided
+         *    - queue is in range
+         *    - it must be provided for all assigned iothreads
+         *    - it must be provided for all queues
+         *    - queue must be assigned only once
+         */
+        for (n = disk->iothreads; n; n = n->next) {
+            virDomainDiskIothreadDef *ioth = n->data;
+            g_autofree char *alias = g_strdup_printf("iothread%u", ioth->id);
+            size_t i;
+
+            if (g_hash_table_contains(iothreads, alias)) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                               _("Duplicate mapping for iothread '%1$u'"), ioth->id);
+                return -1;
+            }
+
+            g_hash_table_insert(iothreads, g_steal_pointer(&alias), NULL);
+
+            if (!virDomainIOThreadIDFind(def, ioth->id)) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                               _("Disk iothread '%1$u' not defined in iothreadid"),
+                               ioth->id);
+                return -1;
+            }
+
+            if (!!queueMap != !!ioth->queues) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                               _("iothread to queue mapping must be provided for all iothreads or for none"));
+                return -1;
+            }
+
+            for (i = 0; i < ioth->nqueues; i++) {
+                bool hasMapping;
+
+                if (virBitmapGetBit(queueMap, ioth->queues[i], &hasMapping) < 0) {
+                    virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                                   _("disk iothread queue '%1$u' mapping out of range"),
+                                   ioth->queues[i]);
+                    return -1;
+                }
+
+                if (hasMapping) {
+                    virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                                   _("disk iothread queue '%1$u' is already assigned"),
+                                   ioth->queues[i]);
+                    return -1;
+                }
+
+                ignore_value(virBitmapSetBit(queueMap, ioth->queues[i]));
+
+            }
+        }
+
+        if (queueMap) {
+            if ((unused = virBitmapNextClearBit(queueMap, -1)) >= 0) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                               _("missing iothread mapping for queue '%1$zd'"),
+                               unused);
+                return -1;
+            }
+        }
+    } else {
         if (!virDomainIOThreadIDFind(def, disk->iothread)) {
             virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                            _("Disk iothread '%1$u' not defined in iothreadid"),
                            disk->iothread);
-        return false;
+            return -1;
+        }
     }
 
-    return true;
+    return 0;
 }
 
 
@@ -2956,7 +3051,7 @@ qemuValidateDomainDeviceDefDiskFrontend(const virDomainDiskDef *disk,
         qemuValidateDomainDeviceDefDiskSerial(disk->serial) < 0)
         return -1;
 
-    if (disk->iothread && !qemuValidateDomainDeviceDefDiskIOThreads(def, disk))
+    if (qemuValidateDomainDeviceDefDiskIOThreads(def, disk, qemuCaps) < 0)
         return -1;
 
     return 0;