qemu: Process DEVICE_DELETED event in a separate thread

Currently, we don not acquire any job when removing a device after
DEVICE_DELETED event was received from QEMU. This means that if there is
another API running at the time DEVICE_DELETED is delivered and the API
acquired a job, we may happily change the definition of the domain the
API is working with whenever it unlocks the domain object (e.g., to talk
with its monitor). That said, we have to acquire a job before finishing
device removal to make things safe. However, doing so in the main event
loop would cause a deadlock so we need to move most of the event handler
into a separate thread.

Another good reason for both acquiring a job and handling the event in a
separate thread is that we currently remove a device backend immediately
after removing its frontend while we should only remove the backend once
we already received DEVICE_DELETED event. That is, we will have to talk
to QEMU monitor from the event handler.

Signed-off-by: Jiri Denemark <jdenemar@redhat.com>

src/qemu/qemu_domain.h

@@ -181,6 +181,7 @@ struct _qemuDomainObjPrivate {
 typedef enum {
     QEMU_PROCESS_EVENT_WATCHDOG = 0,
     QEMU_PROCESS_EVENT_GUESTPANIC,
+    QEMU_PROCESS_EVENT_DEVICE_DELETED,
 
     QEMU_PROCESS_EVENT_LAST
 } qemuProcessEventType;
@@ -189,6 +190,7 @@ struct qemuProcessEvent {
     virDomainObjPtr vm;
     qemuProcessEventType eventType;
     int action;
+    void *data;
 };
 
 const char *qemuDomainAsyncJobPhaseToString(enum qemuDomainAsyncJob job,

src/qemu/qemu_driver.c

@@ -3955,6 +3955,47 @@ processGuestPanicEvent(virQEMUDriverPtr driver,
     virObjectUnref(cfg);
 }
 
+
+static void
+processDeviceDeletedEvent(virQEMUDriverPtr driver,
+                          virDomainObjPtr vm,
+                          char *devAlias)
+{
+    virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
+    virDomainDeviceDef dev;
+
+    VIR_DEBUG("Removing device %s from domain %p %s",
+              devAlias, vm, vm->def->name);
+
+    if (qemuDomainObjBeginJob(driver, vm, QEMU_JOB_MODIFY) < 0)
+        goto cleanup;
+
+    if (!virDomainObjIsActive(vm)) {
+        VIR_DEBUG("Domain is not running");
+        goto endjob;
+    }
+
+    if (virDomainDefFindDevice(vm->def, devAlias, &dev, true) < 0)
+        goto endjob;
+
+    qemuDomainRemoveDevice(driver, vm, &dev);
+
+    if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm) < 0)
+        VIR_WARN("unable to save domain status after removing device %s",
+                 devAlias);
+
+ endjob:
+    /* We had an extra reference to vm before starting a new job so ending the
+     * job is guaranteed not to remove the last reference.
+     */
+    ignore_value(qemuDomainObjEndJob(driver, vm));
+
+ cleanup:
+    VIR_FREE(devAlias);
+    virObjectUnref(cfg);
+}
+
+
 static void qemuProcessEventHandler(void *data, void *opaque)
 {
     struct qemuProcessEvent *processEvent = data;
@@ -3972,8 +4013,11 @@ static void qemuProcessEventHandler(void *data, void *opaque)
     case QEMU_PROCESS_EVENT_GUESTPANIC:
         processGuestPanicEvent(driver, vm, processEvent->action);
         break;
-    default:
+    case QEMU_PROCESS_EVENT_DEVICE_DELETED:
-       break;
+        processDeviceDeletedEvent(driver, vm, processEvent->data);
+        break;
+    case QEMU_PROCESS_EVENT_LAST:
+        break;
     }
 
     if (virObjectUnref(vm))

src/qemu/qemu_process.c

@@ -1373,8 +1373,8 @@ qemuProcessHandleDeviceDeleted(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
                                void *opaque)
 {
     virQEMUDriverPtr driver = opaque;
-    virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
+    struct qemuProcessEvent *processEvent = NULL;
-    virDomainDeviceDef dev;
+    char *data;
 
     virObjectLock(vm);
 
@@ -1384,18 +1384,29 @@ qemuProcessHandleDeviceDeleted(qemuMonitorPtr mon ATTRIBUTE_UNUSED,
     if (qemuDomainSignalDeviceRemoval(vm, devAlias))
         goto cleanup;
 
-    if (virDomainDefFindDevice(vm->def, devAlias, &dev, true) < 0)
+    if (VIR_ALLOC(processEvent) < 0)
-        goto cleanup;
+        goto error;
 
-    qemuDomainRemoveDevice(driver, vm, &dev);
+    processEvent->eventType = QEMU_PROCESS_EVENT_DEVICE_DELETED;
+    if (VIR_STRDUP(data, devAlias) < 0)
+        goto error;
+    processEvent->data = data;
+    processEvent->vm = vm;
 
-    if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm) < 0)
+    virObjectRef(vm);
-        VIR_WARN("unable to save domain status with balloon change");
+    if (virThreadPoolSendJob(driver->workerPool, 0, processEvent) < 0) {
+        ignore_value(virObjectUnref(vm));
+        goto error;
+    }
 
  cleanup:
     virObjectUnlock(vm);
-    virObjectUnref(cfg);
     return 0;
+ error:
+    if (processEvent)
+        VIR_FREE(processEvent->data);
+    VIR_FREE(processEvent);
+    goto cleanup;
 }