bhyve: reconnect to domains after libvirtd restart

Try to reconnect to the running domains after libvirtd restart. To
achieve that, do:

 * Save domain state
  - Modify virBhyveProcessStart() to save domain state to the state
    dir
  - Modify virBhyveProcessStop() to cleanup the pidfile and the state

 * Detect if the state information loaded from the driver's state
   dir matches the actual state. Consider domain active if:
    - PID it points to exist
    - Process title of this PID matches the expected one with the
      domain name

   Otherwise, mark the domain as shut off.

Note: earlier development bhyve versions before FreeBSD 10.0-RELEASE
didn't set proctitle we expect, so the current code will not detect
it. I don't plan adding support for this unless somebody requests
this.

src/bhyve/bhyve_driver.c

@@ -1145,6 +1145,8 @@ bhyveStateInitialize(bool priveleged ATTRIBUTE_UNUSED,
                      virStateInhibitCallback callback ATTRIBUTE_UNUSED,
                      void *opaque ATTRIBUTE_UNUSED)
 {
+    virConnectPtr conn = NULL;
+
     if (!priveleged) {
         VIR_INFO("Not running priveleged, disabling driver");
         return 0;
@@ -1192,6 +1194,15 @@ bhyveStateInitialize(bool priveleged ATTRIBUTE_UNUSED,
         goto cleanup;
     }
 
+    if (virDomainObjListLoadAllConfigs(bhyve_driver->domains,
+                                       BHYVE_STATE_DIR,
+                                       NULL, 1,
+                                       bhyve_driver->caps,
+                                       bhyve_driver->xmlopt,
+                                       1 << VIR_DOMAIN_VIRT_BHYVE,
+                                       NULL, NULL) < 0)
+        goto cleanup;
+
     if (virDomainObjListLoadAllConfigs(bhyve_driver->domains,
                                        BHYVE_CONFIG_DIR,
                                        BHYVE_AUTOSTART_DIR, 0,
@@ -1201,9 +1212,14 @@ bhyveStateInitialize(bool priveleged ATTRIBUTE_UNUSED,
                                        NULL, NULL) < 0)
         goto cleanup;
 
+    virBhyveProcessReconnectAll(bhyve_driver);
+
+    virObjectUnref(conn);
+
     return 0;
 
  cleanup:
+    virObjectUnref(conn);
     bhyveStateCleanup();
     return -1;
 }

src/bhyve/bhyve_process.c

@@ -185,6 +185,11 @@ virBhyveProcessStart(virConnectPtr conn,
     vm->def->id = vm->pid;
     virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason);
 
+    if (virDomainSaveStatus(driver->xmlopt,
+                            BHYVE_STATE_DIR,
+                            vm) < 0)
+        goto cleanup;
+
     ret = 0;
 
  cleanup:
@@ -257,6 +262,10 @@ virBhyveProcessStop(bhyveConnPtr driver,
 
  cleanup:
     virCommandFree(cmd);
+
+    virPidFileDelete(BHYVE_STATE_DIR, vm->def->name);
+    virDomainDeleteConfig(BHYVE_STATE_DIR, NULL, vm);
+
     return ret;
 }
 
@@ -295,3 +304,82 @@ virBhyveGetDomainTotalCpuStats(virDomainObjPtr vm,
 
     return ret;
 }
+
+struct bhyveProcessReconnectData {
+    bhyveConnPtr driver;
+    kvm_t *kd;
+};
+
+static int
+virBhyveProcessReconnect(virDomainObjPtr vm,
+                         void *opaque)
+{
+    struct bhyveProcessReconnectData *data = opaque;
+    struct kinfo_proc *kp;
+    int nprocs;
+    char **proc_argv;
+    char *expected_proctitle = NULL;
+    int ret = -1;
+
+    if (!virDomainObjIsActive(vm))
+        return 0;
+
+    if (!vm->pid)
+        return 0;
+
+    virObjectLock(vm);
+
+    kp = kvm_getprocs(data->kd, KERN_PROC_PID, vm->pid, &nprocs);
+    if (kp == NULL || nprocs != 1)
+        goto cleanup;
+
+    if (virAsprintf(&expected_proctitle, "bhyve: %s", vm->def->name) < 0)
+        goto cleanup;
+
+    proc_argv = kvm_getargv(data->kd, kp, 0);
+    if (proc_argv && proc_argv[0])
+         if (STREQ(expected_proctitle, proc_argv[0]))
+             ret = 0;
+
+ cleanup:
+    if (ret < 0) {
+        /* If VM is reported to be in active state, but we cannot find
+         * its PID, then we clear information about the PID and
+         * set state to 'shutdown' */
+        vm->pid = 0;
+        vm->def->id = -1;
+        virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF,
+                             VIR_DOMAIN_SHUTOFF_UNKNOWN);
+        ignore_value(virDomainSaveStatus(data->driver->xmlopt,
+                                         BHYVE_STATE_DIR,
+                                         vm));
+    }
+
+    virObjectUnlock(vm);
+    VIR_FREE(expected_proctitle);
+
+    return ret;
+}
+
+void
+virBhyveProcessReconnectAll(bhyveConnPtr driver)
+{
+    kvm_t *kd;
+    struct bhyveProcessReconnectData data;
+    char errbuf[_POSIX2_LINE_MAX];
+
+    if ((kd = kvm_openfiles(NULL, NULL, NULL, O_RDONLY, errbuf)) == NULL) {
+        virReportError(VIR_ERR_SYSTEM_ERROR,
+                       _("Unable to get kvm descriptor: %s"),
+                       errbuf);
+        return;
+
+    }
+
+    data.driver = driver;
+    data.kd = kd;
+
+    virDomainObjListForEach(driver->domains, virBhyveProcessReconnect, &data);
+
+    kvm_close(kd);
+}

src/bhyve/bhyve_process.h

@@ -37,6 +37,8 @@ int virBhyveProcessStop(bhyveConnPtr driver,
 int virBhyveGetDomainTotalCpuStats(virDomainObjPtr vm,
                                    unsigned long long *cpustats);
 
+void virBhyveProcessReconnectAll(bhyveConnPtr driver);
+
 typedef enum {
     VIR_BHYVE_PROCESS_START_AUTODESTROY = 1 << 0,
 } bhyveProcessStartFlags;