drvbhyve: Automatically tear down guest domains on shutdown

Reboot requires more sophistication and is left as a future work item --
but at least part of the plumbing is in place.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

po/POTFILES.in

@@ -11,6 +11,7 @@ src/access/viraccessmanager.c
 src/bhyve/bhyve_command.c
 src/bhyve/bhyve_device.c
 src/bhyve/bhyve_driver.c
+src/bhyve/bhyve_monitor.c
 src/bhyve/bhyve_process.c
 src/conf/capabilities.c
 src/conf/cpu_conf.c

src/Makefile.am

@@ -833,6 +833,8 @@ BHYVE_DRIVER_SOURCES =						\
 		bhyve/bhyve_domain.h				\
 		bhyve/bhyve_driver.h				\
 		bhyve/bhyve_driver.c				\
+		bhyve/bhyve_monitor.c				\
+		bhyve/bhyve_monitor.h				\
 		bhyve/bhyve_process.c				\
 		bhyve/bhyve_process.h				\
 		bhyve/bhyve_utils.h				\

src/bhyve/bhyve_monitor.c

@@ -0,0 +1,186 @@
+/*
+ * bhyve_monitor.c: Tear-down or reboot bhyve domains on guest shutdown
+ *
+ * Copyright (C) 2014 Conrad Meyer
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Conrad Meyer <cse.cem@gmail.com>
+ */
+
+#include <config.h>
+
+#include <sys/types.h>
+#include <sys/event.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+
+#include "bhyve_monitor.h"
+#include "bhyve_process.h"
+#include "viralloc.h"
+#include "virerror.h"
+#include "virfile.h"
+#include "virlog.h"
+
+#define VIR_FROM_THIS	VIR_FROM_BHYVE
+
+VIR_LOG_INIT("bhyve.bhyve_monitor");
+
+struct _bhyveMonitor {
+    int kq;
+    int watch;
+    virDomainObjPtr vm;
+    bhyveConnPtr driver;
+};
+
+static void
+bhyveMonitorIO(int watch, int kq, int events ATTRIBUTE_UNUSED, void *opaque)
+{
+    const struct timespec zerowait = {};
+    bhyveMonitorPtr mon = opaque;
+    struct kevent kev;
+    int rc, status;
+
+    if (watch != mon->watch || kq != mon->kq) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("event from unexpected fd %d!=%d / watch %d!=%d"),
+                       mon->kq, kq, mon->watch, watch);
+        return;
+    }
+
+    rc = kevent(kq, NULL, 0, &kev, 1, &zerowait);
+    if (rc < 0) {
+        virReportSystemError(errno, "%s", _("Unable to query kqueue"));
+        return;
+    }
+
+    if (rc == 0)
+        return;
+
+    if ((kev.flags & EV_ERROR) != 0) {
+        virReportSystemError(kev.data, "%s", _("Unable to query kqueue"));
+        return;
+    }
+
+    if (kev.filter == EVFILT_PROC && (kev.fflags & NOTE_EXIT) != 0) {
+        if ((pid_t)kev.ident != mon->vm->pid) {
+            virReportError(VIR_ERR_INTERNAL_ERROR,
+                        _("event from unexpected proc %ju!=%ju"),
+                        (uintmax_t)mon->vm->pid, (uintmax_t)kev.ident);
+            return;
+        }
+
+        status = kev.data;
+        if (WIFSIGNALED(status) && WCOREDUMP(status)) {
+            virReportError(VIR_ERR_INTERNAL_ERROR,
+                           _("Guest %s got signal %d and crashed"),
+                           mon->vm->def->name,
+                           WTERMSIG(status));
+            virBhyveProcessStop(mon->driver, mon->vm,
+                                VIR_DOMAIN_SHUTOFF_CRASHED);
+        } else if (WIFEXITED(status)) {
+            if (WEXITSTATUS(status) == 0) {
+                /* 0 - reboot */
+                /* TODO: Implementing reboot is a little more complicated. */
+                VIR_INFO("Guest %s rebooted; destroying domain.",
+                         mon->vm->def->name);
+                virBhyveProcessStop(mon->driver, mon->vm,
+                                    VIR_DOMAIN_SHUTOFF_SHUTDOWN);
+            } else if (WEXITSTATUS(status) < 3) {
+                /* 1 - shutdown, 2 - halt, 3 - triple fault. others - error */
+                VIR_INFO("Guest %s shut itself down; destroying domain.",
+                         mon->vm->def->name);
+                virBhyveProcessStop(mon->driver, mon->vm,
+                                    VIR_DOMAIN_SHUTOFF_SHUTDOWN);
+            } else {
+                VIR_INFO("Guest %s had an error and exited with status %d; destroying domain.",
+                         mon->vm->def->name, WEXITSTATUS(status));
+                virBhyveProcessStop(mon->driver, mon->vm,
+                                    VIR_DOMAIN_SHUTOFF_UNKNOWN);
+            }
+        }
+    }
+}
+
+static void
+bhyveMonitorRelease(void *opaque)
+{
+    bhyveMonitorPtr mon = opaque;
+
+    VIR_FORCE_CLOSE(mon->kq);
+    virObjectUnref(mon->vm);
+    VIR_FREE(mon);
+}
+
+bhyveMonitorPtr
+bhyveMonitorOpen(virDomainObjPtr vm, bhyveConnPtr driver)
+{
+    bhyveMonitorPtr mon;
+    struct kevent kev;
+    int rc;
+
+    if (VIR_ALLOC(mon) < 0)
+        return NULL;
+
+    mon->vm = virObjectRef(vm);
+    mon->driver = driver;
+
+    mon->kq = kqueue();
+    if (mon->kq < 0) {
+        virReportError(VIR_ERR_SYSTEM_ERROR, "%s",
+                       _("Unable to create kqueue"));
+        goto cleanup;
+    }
+
+    EV_SET(&kev, vm->pid, EVFILT_PROC, EV_ADD, NOTE_EXIT, 0, mon);
+    rc = kevent(mon->kq, &kev, 1, NULL, 0, NULL);
+    if (rc < 0) {
+        virReportError(VIR_ERR_SYSTEM_ERROR, "%s",
+                       _("Unable to register process kevent"));
+        goto cleanup;
+    }
+
+    mon->watch = virEventAddHandle(mon->kq,
+                                   VIR_EVENT_HANDLE_READABLE |
+                                   VIR_EVENT_HANDLE_ERROR |
+                                   VIR_EVENT_HANDLE_HANGUP,
+                                   bhyveMonitorIO,
+                                   mon,
+                                   bhyveMonitorRelease);
+    if (mon->watch < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("unable to register monitor events"));
+        goto cleanup;
+    }
+
+    return mon;
+
+ cleanup:
+    bhyveMonitorRelease(mon);
+    return NULL;
+}
+
+void
+bhyveMonitorClose(bhyveMonitorPtr mon)
+{
+
+    if (mon == NULL)
+        return;
+
+    if (mon->watch > 0)
+        virEventRemoveHandle(mon->watch);
+    else
+        bhyveMonitorRelease(mon);
+}

src/bhyve/bhyve_monitor.h

@@ -0,0 +1,36 @@
+/*
+ * bhyve_monitor.h: Tear-down or reboot bhyve domains on guest shutdown
+ *
+ * Copyright (C) 2014 Conrad Meyer
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Conrad Meyer <cse.cem@gmail.com>
+ */
+
+#ifndef BHYVE_MONITOR_H
+# define BHYVE_MONITOR_H
+
+# include "internal.h"
+# include "domain_conf.h"
+# include "bhyve_utils.h"
+
+typedef struct _bhyveMonitor bhyveMonitor;
+typedef bhyveMonitor *bhyveMonitorPtr;
+
+bhyveMonitorPtr bhyveMonitorOpen(virDomainObjPtr vm, bhyveConnPtr driver);
+void bhyveMonitorClose(bhyveMonitorPtr mon);
+
+#endif /* BHYVE_MONITOR_H */

src/bhyve/bhyve_process.c

@@ -32,8 +32,9 @@
 #include <net/if_tap.h>
 
 #include "bhyve_device.h"
-#include "bhyve_process.h"
 #include "bhyve_command.h"
+#include "bhyve_monitor.h"
+#include "bhyve_process.h"
 #include "datatypes.h"
 #include "virerror.h"
 #include "virlog.h"
@@ -209,6 +210,7 @@ virBhyveProcessStart(virConnectPtr conn,
 
     vm->def->id = vm->pid;
     virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason);
+    vm->privateData = bhyveMonitorOpen(vm, driver);
 
     if (virDomainSaveStatus(driver->xmlopt,
                             BHYVE_STATE_DIR,
@@ -268,6 +270,9 @@ virBhyveProcessStop(bhyveConnPtr driver,
         return -1;
     }
 
+    if (vm->privateData != NULL)
+        bhyveMonitorClose((bhyveMonitorPtr)vm->privateData);
+
     /* First, try to kill 'bhyve' process */
     if (virProcessKillPainfully(vm->pid, true) != 0)
         VIR_WARN("Failed to gracefully stop bhyve VM '%s' (pid: %d)",
@@ -371,9 +376,12 @@ virBhyveProcessReconnect(virDomainObjPtr vm,
         goto cleanup;
 
     proc_argv = kvm_getargv(data->kd, kp, 0);
-    if (proc_argv && proc_argv[0])
-         if (STREQ(expected_proctitle, proc_argv[0]))
+    if (proc_argv && proc_argv[0]) {
+         if (STREQ(expected_proctitle, proc_argv[0])) {
              ret = 0;
+             vm->privateData = bhyveMonitorOpen(vm, data->driver);
+         }
+    }
 
  cleanup:
     if (ret < 0) {