S390: QEMU driver support for CCW addresses

This commit adds the QEMU driver support for CCW addresses. The
current QEMU only allows virtio devices to be attached to the
CCW bus. We named the new capability indicating that support
QEMU_CAPS_VIRTIO_CCW accordingly.

The fact that CCW devices can only be assigned to domains with a
machine type of s390-ccw-virtio requires a few extra checks for
machine type in qemu_command.c on top of querying
QEMU_CAPS_VIRTIO_{CCW|S390}.

The majority of the new functions deals with CCW address generation
and management.

Signed-off-by: Viktor Mihajlovski <mihajlov@linux.vnet.ibm.com>

src/qemu/qemu_capabilities.c

@@ -210,6 +210,7 @@ VIR_ENUM_IMPL(virQEMUCaps, QEMU_CAPS_LAST,
 
               "rng-random", /* 130 */
               "rng-egd",
+              "virtio-ccw"
     );
 
 struct _virQEMUCaps {
@@ -1318,6 +1319,7 @@ struct virQEMUCapsStringFlags virQEMUCapsObjectTypes[] = {
     { "usb-hub", QEMU_CAPS_USB_HUB },
     { "ich9-ahci", QEMU_CAPS_ICH9_AHCI },
     { "virtio-blk-s390", QEMU_CAPS_VIRTIO_S390 },
+    { "virtio-blk-ccw", QEMU_CAPS_VIRTIO_CCW },
     { "sclpconsole", QEMU_CAPS_SCLP_S390 },
     { "lsi53c895a", QEMU_CAPS_SCSI_LSI },
     { "virtio-scsi-pci", QEMU_CAPS_VIRTIO_SCSI_PCI },
@@ -1338,7 +1340,6 @@ struct virQEMUCapsStringFlags virQEMUCapsObjectTypes[] = {
     { "rng-egd", QEMU_CAPS_OBJECT_RNG_EGD },
 };
 
-
 static struct virQEMUCapsStringFlags virQEMUCapsObjectPropsVirtioBlk[] = {
     { "multifunction", QEMU_CAPS_PCI_MULTIFUNCTION },
     { "bootindex", QEMU_CAPS_BOOTINDEX },
@@ -1393,6 +1394,10 @@ static struct virQEMUCapsObjectTypeProps virQEMUCapsObjectProps[] = {
       ARRAY_CARDINALITY(virQEMUCapsObjectPropsVirtioBlk) },
     { "virtio-net-pci", virQEMUCapsObjectPropsVirtioNet,
       ARRAY_CARDINALITY(virQEMUCapsObjectPropsVirtioNet) },
+    { "virtio-blk-ccw", virQEMUCapsObjectPropsVirtioBlk,
+      ARRAY_CARDINALITY(virQEMUCapsObjectPropsVirtioBlk) },
+    { "virtio-net-ccw", virQEMUCapsObjectPropsVirtioNet,
+      ARRAY_CARDINALITY(virQEMUCapsObjectPropsVirtioNet) },
     { "virtio-blk-s390", virQEMUCapsObjectPropsVirtioBlk,
       ARRAY_CARDINALITY(virQEMUCapsObjectPropsVirtioBlk) },
     { "virtio-net-s390", virQEMUCapsObjectPropsVirtioNet,

src/qemu/qemu_capabilities.h

@@ -171,6 +171,7 @@ enum virQEMUCapsFlags {
     QEMU_CAPS_OBJECT_RNG_RANDOM  = 130, /* the rng-random backend for
                                            virtio rng */
     QEMU_CAPS_OBJECT_RNG_EGD     = 131, /* EGD protocol daemon for rng */
+    QEMU_CAPS_VIRTIO_CCW         = 132, /* -device virtio-*-ccw */
 
     QEMU_CAPS_LAST,                   /* this must always be the last item */
 };

src/qemu/qemu_command.c

@@ -799,6 +799,100 @@ no_memory:
     return -1;
 }
 
+/* S390 ccw bus support */
+
+struct _qemuDomainCCWAddressSet {
+    virHashTablePtr defined;
+    virDomainDeviceCCWAddress next;
+};
+
+static char*
+qemuCCWAddressAsString(virDomainDeviceCCWAddressPtr addr)
+{
+    char *addrstr = NULL;
+
+    if (virAsprintf(&addrstr, "%x.%x.%04x",
+                    addr->cssid,
+                    addr->ssid,
+                    addr->devno) < 0) {
+        virReportOOMError();
+        return NULL;
+    }
+
+    return addrstr;
+}
+
+static int
+qemuCCWAdressIncrement(virDomainDeviceCCWAddressPtr addr)
+{
+    virDomainDeviceCCWAddress ccwaddr = *addr;
+
+    /* We are not touching subchannel sets and channel subsystems */
+    if (++ccwaddr.devno > VIR_DOMAIN_DEVICE_CCW_MAX_DEVNO)
+        return -1;
+
+    *addr = ccwaddr;
+    return 0;
+}
+
+static void
+qemuDomainCCWAddressSetFreeEntry(void *payload,
+                                 const void *name ATTRIBUTE_UNUSED)
+{
+    VIR_FREE(payload);
+}
+
+int qemuDomainCCWAddressAssign(virDomainDeviceInfoPtr dev,
+                               qemuDomainCCWAddressSetPtr addrs,
+                               bool autoassign)
+{
+    int ret = -1;
+    char *addr = NULL;
+
+    if (dev->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW)
+        return 0;
+
+    if (!autoassign && dev->addr.ccw.assigned) {
+        if (!(addr = qemuCCWAddressAsString(&dev->addr.ccw)))
+            goto cleanup;
+
+        if (virHashLookup(addrs->defined, addr)) {
+            virReportError(VIR_ERR_XML_ERROR,
+                           _("The CCW devno '%s' is in use already "),
+                           addr);
+            goto cleanup;
+        }
+    } else if (autoassign && !dev->addr.ccw.assigned) {
+        if (!(addr = qemuCCWAddressAsString(&addrs->next)) < 0)
+            goto cleanup;
+
+        while (virHashLookup(addrs->defined, addr)) {
+            if (qemuCCWAdressIncrement(&addrs->next) < 0) {
+                virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                               _("There are no more free CCW devnos."));
+                goto cleanup;
+            }
+            VIR_FREE(addr);
+            addr = qemuCCWAddressAsString(&addrs->next);
+        }
+        dev->addr.ccw = addrs->next;
+        dev->addr.ccw.assigned = true;
+    } else {
+        return 0;
+    }
+
+    if (virHashAddEntry(addrs->defined,addr,addr) < 0)
+        goto cleanup;
+    else
+        addr = NULL; /* memory will be freed by hash table */
+
+    ret = 0;
+
+cleanup:
+    VIR_FREE(addr);
+    return ret;
+}
+
 static void
 qemuDomainPrimeS390VirtioDevices(virDomainDefPtr def,
                                  enum virDomainDeviceAddressType type)
@@ -841,13 +935,135 @@ qemuDomainPrimeS390VirtioDevices(virDomainDefPtr def,
         def->memballoon->info.type = type;
 }
 
-static void
+static int
-qemuDomainAssignS390Addresses(virDomainDefPtr def, virQEMUCapsPtr qemuCaps)
+qemuDomainCCWAddressAllocate(virDomainDefPtr def ATTRIBUTE_UNUSED,
+                             virDomainDeviceDefPtr dev ATTRIBUTE_UNUSED,
+                             virDomainDeviceInfoPtr info,
+                             void *data)
 {
+    return qemuDomainCCWAddressAssign(info, data, true);
+}
+
+static int
+qemuDomainCCWAddressValidate(virDomainDefPtr def ATTRIBUTE_UNUSED,
+                             virDomainDeviceDefPtr dev ATTRIBUTE_UNUSED,
+                             virDomainDeviceInfoPtr info,
+                             void *data)
+{
+    return qemuDomainCCWAddressAssign(info, data, false);
+}
+
+int qemuDomainCCWAddressReleaseAddr(qemuDomainCCWAddressSetPtr addrs,
+                                    virDomainDeviceInfoPtr dev)
+{
+    char *addr;
+    int ret;
+
+    addr = qemuCCWAddressAsString(&(dev->addr.ccw));
+    if (!addr)
+        return -1;
+
+    if ((ret = virHashRemoveEntry(addrs->defined, addr)) == 0 &&
+        dev->addr.ccw.cssid == addrs->next.cssid &&
+        dev->addr.ccw.ssid == addrs->next.ssid &&
+        dev->addr.ccw.devno < addrs->next.devno) {
+        addrs->next.devno = dev->addr.ccw.devno;
+        addrs->next.assigned = false;
+    }
+
+    VIR_FREE(addr);
+
+    return ret;
+}
+
+void qemuDomainCCWAddressSetFree(qemuDomainCCWAddressSetPtr addrs)
+{
+    if (!addrs)
+        return;
+
+    virHashFree(addrs->defined);
+    VIR_FREE(addrs);
+}
+
+static qemuDomainCCWAddressSetPtr
+qemuDomainCCWAddressSetCreate(void)
+{
+     qemuDomainCCWAddressSetPtr addrs = NULL;
+
+    if (VIR_ALLOC(addrs) < 0)
+        goto no_memory;
+
+    if (!(addrs->defined = virHashCreate(10, qemuDomainCCWAddressSetFreeEntry)))
+        goto cleanup;
+
+    /* must use cssid = 0xfe (254) for virtio-ccw devices */
+    addrs->next.cssid = 254;
+    addrs->next.ssid = 0;
+    addrs->next.devno = 0;
+    addrs->next.assigned = 0;
+    return addrs;
+
+no_memory:
+    virReportOOMError();
+cleanup:
+    qemuDomainCCWAddressSetFree(addrs);
+    return addrs;
+}
+
+/*
+ * Three steps populating CCW devnos
+ * 1. Allocate empty address set
+ * 2. Gather addresses with explicit devno
+ * 3. Assign defaults to the rest
+ */
+static int
+qemuDomainAssignS390Addresses(virDomainDefPtr def,
+                              virQEMUCapsPtr qemuCaps,
+                              virDomainObjPtr obj)
+{
+    int ret = -1;
+    qemuDomainCCWAddressSetPtr addrs = NULL;
+    qemuDomainObjPrivatePtr priv = NULL;
+
+    if (STREQLEN(def->os.machine, "s390-ccw", 8) &&
+        virQEMUCapsGet(qemuCaps, QEMU_CAPS_VIRTIO_CCW)) {
+        qemuDomainPrimeS390VirtioDevices(
+            def, VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW);
+
+        if (!(addrs = qemuDomainCCWAddressSetCreate()))
+            goto cleanup;
+
+        if (virDomainDeviceInfoIterate(def, qemuDomainCCWAddressValidate,
+                                       addrs) < 0)
+            goto cleanup;
+
+        if (virDomainDeviceInfoIterate(def, qemuDomainCCWAddressAllocate,
+                                       addrs) < 0)
+            goto cleanup;
+    } else if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_VIRTIO_S390)) {
         /* deal with legacy virtio-s390 */
-    if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_VIRTIO_S390))
         qemuDomainPrimeS390VirtioDevices(
             def, VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390);
+    }
+
+    if (obj && obj->privateData) {
+        priv = obj->privateData;
+        if (addrs) {
+            /* if this is the live domain object, we persist the CCW addresses*/
+            qemuDomainCCWAddressSetFree(priv->ccwaddrs);
+            priv->persistentAddrs = 1;
+            priv->ccwaddrs = addrs;
+            addrs = NULL;
+        } else {
+            priv->persistentAddrs = 0;
+        }
+    }
+    ret = 0;
+
+cleanup:
+    qemuDomainCCWAddressSetFree(addrs);
+
+    return ret;
 }
 
 static int
@@ -1106,7 +1322,9 @@ int qemuDomainAssignAddresses(virDomainDefPtr def,
     if (rc)
         return rc;
 
-    qemuDomainAssignS390Addresses(def, qemuCaps);
+    rc = qemuDomainAssignS390Addresses(def, qemuCaps, obj);
+    if (rc)
+        return rc;
 
     return qemuDomainAssignPCIAddresses(def, qemuCaps, obj);
 }
@@ -1631,7 +1849,9 @@ qemuAssignDevicePCISlots(virDomainDefPtr def,
         /* don't touch s390 devices */
         if (def->disks[i]->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI ||
             def->disks[i]->info.type ==
-            VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390)
+            VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390 ||
+            def->disks[i]->info.type ==
+            VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW)
             continue;
 
         if (def->disks[i]->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE) {
@@ -1785,6 +2005,12 @@ qemuBuildDeviceAddressStr(virBufferPtr buf,
     } else if (info->type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_SPAPRVIO) {
         if (info->addr.spaprvio.has_reg)
             virBufferAsprintf(buf, ",reg=0x%llx", info->addr.spaprvio.reg);
+    } else if (info->type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW) {
+        if (info->addr.ccw.assigned)
+            virBufferAsprintf(buf, ",devno=%x.%x.%04x",
+                              info->addr.ccw.cssid,
+                              info->addr.ccw.ssid,
+                              info->addr.ccw.devno);
     }
 
     return 0;
@@ -2283,13 +2509,6 @@ qemuBuildDriveStr(virConnectPtr conn ATTRIBUTE_UNUSED,
         break;
 
     case VIR_DOMAIN_DISK_BUS_VIRTIO:
-        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_VIRTIO_S390) &&
-            (disk->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390)) {
-            /* Paranoia - leave in here for now */
-            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
-                           _("unexpected address type for s390-virtio disk"));
-            goto error;
-        }
         idx = -1;
         break;
 
@@ -2793,7 +3012,9 @@ qemuBuildDriveDevStr(virDomainDefPtr def,
                           disk->info.addr.drive.unit);
         break;
     case VIR_DOMAIN_DISK_BUS_VIRTIO:
-        if (disk->info.type ==
+        if (disk->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW) {
+            virBufferAddLit(&opt, "virtio-blk-ccw");
+        } else if (disk->info.type ==
                    VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390) {
             virBufferAddLit(&opt, "virtio-blk-s390");
         } else {
@@ -3075,6 +3296,9 @@ qemuBuildControllerDevStr(virDomainDefPtr domainDef,
     case VIR_DOMAIN_CONTROLLER_TYPE_VIRTIO_SERIAL:
         if (def->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) {
             virBufferAddLit(&buf, "virtio-serial-pci");
+        } else if (def->info.type ==
+                   VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW) {
+            virBufferAddLit(&buf, "virtio-serial-ccw");
         } else if (def->info.type ==
                    VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390) {
             virBufferAddLit(&buf, "virtio-serial-s390");
@@ -3173,7 +3397,9 @@ qemuBuildNicDevStr(virDomainNetDefPtr net,
     if (!net->model) {
         nic = "rtl8139";
     } else if (STREQ(net->model, "virtio")) {
-        if (net->info.type ==
+        if (net->info.type == VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW) {
+            nic = "virtio-net-ccw";
+        } else if (net->info.type ==
                    VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390) {
             nic = "virtio-net-s390";
         } else  {
@@ -3401,7 +3627,17 @@ qemuBuildMemballoonDevStr(virDomainMemballoonDefPtr dev,
 {
     virBuffer buf = VIR_BUFFER_INITIALIZER;
 
+    switch (dev->info.type) {
+        case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI:
             virBufferAddLit(&buf, "virtio-balloon-pci");
+            break;
+        case VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW:
+            virBufferAddLit(&buf, "virtio-balloon-ccw");
+            break;
+        default:
+            goto error;
+    }
+
     virBufferAsprintf(&buf, ",id=%s", dev->info.alias);
     if (qemuBuildDeviceAddressStr(&buf, &dev->info, qemuCaps) < 0)
         goto error;
@@ -4093,6 +4329,7 @@ qemuBuildVirtioSerialPortDevStr(virDomainChrDefPtr dev,
     }
 
     if (dev->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_NONE &&
+        dev->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW &&
         dev->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390) {
         /* Check it's a virtio-serial address */
         if (dev->info.type !=
@@ -7100,7 +7337,8 @@ qemuBuildCommandLine(virConnectPtr conn,
      * NB: Earlier we declared that VirtIO balloon will always be in
      * slot 0x3 on bus 0x0
      */
-    if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_VIRTIO_S390) && def->memballoon)
+    if (STREQLEN(def->os.machine, "s390-virtio", 10) &&
+        virQEMUCapsGet(qemuCaps, QEMU_CAPS_VIRTIO_S390) && def->memballoon)
         def->memballoon->model = VIR_DOMAIN_MEMBALLOON_MODEL_NONE;
 
     if (def->memballoon &&

src/qemu/qemu_command.h

@@ -215,6 +215,12 @@ int  qemuAssignDevicePCISlots(virDomainDefPtr def,
                               virQEMUCapsPtr qemuCaps,
                               qemuDomainPCIAddressSetPtr addrs);
 
+int qemuDomainCCWAddressReleaseAddr(qemuDomainCCWAddressSetPtr addrs,
+                                    virDomainDeviceInfoPtr dev);
+int qemuDomainCCWAddressAssign(virDomainDeviceInfoPtr dev, qemuDomainCCWAddressSetPtr addrs,
+                               bool autoassign);
+void qemuDomainCCWAddressSetFree(qemuDomainCCWAddressSetPtr addrs);
+
 int qemuAssignDeviceAliases(virDomainDefPtr def, virQEMUCapsPtr qemuCaps);
 int qemuDomainNetVLAN(virDomainNetDefPtr def);
 int qemuAssignDeviceNetAlias(virDomainDefPtr def, virDomainNetDefPtr net, int idx);

src/qemu/qemu_domain.c

@@ -236,6 +236,7 @@ qemuDomainObjPrivateFree(void *data)
     virObjectUnref(priv->qemuCaps);
 
     qemuDomainPCIAddressSetFree(priv->pciaddrs);
+    qemuDomainCCWAddressSetFree(priv->ccwaddrs);
     virDomainChrSourceDefFree(priv->monConfig);
     qemuDomainObjFreeJob(priv);
     VIR_FREE(priv->vcpupids);

src/qemu/qemu_domain.h

@@ -120,6 +120,8 @@ typedef qemuDomainPCIAddressSet *qemuDomainPCIAddressSetPtr;
 
 typedef void (*qemuDomainCleanupCallback)(virQEMUDriverPtr driver,
                                           virDomainObjPtr vm);
+typedef struct _qemuDomainCCWAddressSet qemuDomainCCWAddressSet;
+typedef qemuDomainCCWAddressSet *qemuDomainCCWAddressSetPtr;
 
 typedef struct _qemuDomainObjPrivate qemuDomainObjPrivate;
 typedef qemuDomainObjPrivate *qemuDomainObjPrivatePtr;
@@ -144,6 +146,7 @@ struct _qemuDomainObjPrivate {
     int *vcpupids;
 
     qemuDomainPCIAddressSetPtr pciaddrs;
+    qemuDomainCCWAddressSetPtr ccwaddrs;
     int persistentAddrs;
 
     virQEMUCapsPtr qemuCaps;

src/qemu/qemu_process.c

@@ -4186,6 +4186,9 @@ void qemuProcessStop(virQEMUDriverPtr driver,
         virDomainDefClearPCIAddresses(vm->def);
         qemuDomainPCIAddressSetFree(priv->pciaddrs);
         priv->pciaddrs = NULL;
+        virDomainDefClearCCWAddresses(vm->def);
+        qemuDomainCCWAddressSetFree(priv->ccwaddrs);
+        priv->ccwaddrs = NULL;
     }
 
     qemuDomainReAttachHostDevices(driver, vm->def);