libvirt/qemud/qemud.c

/*
 * qemud.c: daemon start of day, guest process & i/o management
 *
 * Copyright (C) 2006, 2007 Red Hat, Inc.
 * Copyright (C) 2006 Daniel P. Berrange
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
 *
 * Author: Daniel P. Berrange <berrange@redhat.com>
 */

#include <config.h>

#define _GNU_SOURCE /* for asprintf */

#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <paths.h>
#include <limits.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/poll.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <netdb.h>
#include <stdlib.h>
#include <pwd.h>
#include <stdio.h>
#include <stdarg.h>
#include <syslog.h>
#include <string.h>
#include <errno.h>
#include <getopt.h>
#include <ctype.h>
#include <assert.h>
#include <fnmatch.h>
#include <gnutls/gnutls.h>
#include <gnutls/x509.h>

#include <libvirt/virterror.h>

#include "internal.h"
#include "../src/remote_internal.h"
#include "../src/conf.h"
#include "dispatch.h"
#include "driver.h"
#include "conf.h"
#include "iptables.h"

static int godaemon = 0;        /* -d: Be a daemon */
static int verbose = 0;         /* -v: Verbose mode */
static int remote = 0;          /* -r: Remote mode */
static int sys = 0;             /* -s: (QEMUD only) system mode */
static int timeout = -1;        /* -t: (QEMUD only) timeout */
static int sigwrite = -1;       /* Signal handler pipe */

/* Defaults for configuration file elements (remote only). */
static int listen_tls = 1;
static int listen_tcp = 0;
static const char *tls_port = LIBVIRTD_TLS_PORT;
static const char *tcp_port = LIBVIRTD_TCP_PORT;

static int tls_no_verify_certificate = 0;
static int tls_no_verify_address = 0;
static const char **tls_allowed_ip_list = 0;
static const char **tls_allowed_dn_list = 0;

static const char *key_file = LIBVIRT_SERVERKEY;
static const char *cert_file = LIBVIRT_SERVERCERT;
static const char *ca_file = LIBVIRT_CACERT;
static const char *crl_file = "";

static gnutls_certificate_credentials_t x509_cred;
static gnutls_dh_params_t dh_params;

#define DH_BITS 1024

static sig_atomic_t sig_errors = 0;
static int sig_lasterrno = 0;

static void sig_handler(int sig) {
    unsigned char sigc = sig;
    int origerrno;
    int r;

    if (sig == SIGCHLD) /* We explicitly waitpid the child later */
        return;

    origerrno = errno;
    r = write(sigwrite, &sigc, 1);
    if (r == -1) {
        sig_errors++;
        sig_lasterrno = errno;
    }
    errno = origerrno;
}

static int
remoteInitializeGnuTLS (void)
{
    int err;

    /* Initialise GnuTLS. */
    gnutls_global_init ();

    err = gnutls_certificate_allocate_credentials (&x509_cred);
    if (err) {
        qemudLog (QEMUD_ERR, "gnutls_certificate_allocate_credentials: %s",
                  gnutls_strerror (err));
        return -1;
    }

    if (ca_file && ca_file[0] != '\0') {
        qemudDebug ("loading CA cert from %s", ca_file);
        err = gnutls_certificate_set_x509_trust_file (x509_cred, ca_file,
                                                      GNUTLS_X509_FMT_PEM);
        if (err < 0) {
            qemudLog (QEMUD_ERR, "gnutls_certificate_set_x509_trust_file: %s",
                      gnutls_strerror (err));
            return -1;
        }
    }

    if (crl_file && crl_file[0] != '\0') {
        qemudDebug ("loading CRL from %s", crl_file);
        err = gnutls_certificate_set_x509_crl_file (x509_cred, crl_file,
                                                    GNUTLS_X509_FMT_PEM);
        if (err < 0) {
            qemudLog (QEMUD_ERR, "gnutls_certificate_set_x509_crl_file: %s",
                      gnutls_strerror (err));
            return -1;
        }
    }

    if (cert_file && cert_file[0] != '\0' && key_file && key_file[0] != '\0') {
        qemudDebug ("loading cert and key from %s and %s",
                    cert_file, key_file);
        err =
            gnutls_certificate_set_x509_key_file (x509_cred,
                                                  cert_file, key_file,
                                                  GNUTLS_X509_FMT_PEM);
        if (err < 0) {
            qemudLog (QEMUD_ERR, "gnutls_certificate_set_x509_key_file: %s",
                      gnutls_strerror (err));
            return -1;
        }
    }

    /* Generate Diffie Hellman parameters - for use with DHE
     * kx algorithms. These should be discarded and regenerated
     * once a day, once a week or once a month. Depending on the
     * security requirements.
     */
    err = gnutls_dh_params_init (&dh_params);
    if (err < 0) {
        qemudLog (QEMUD_ERR, "gnutls_dh_params_init: %s",
                  gnutls_strerror (err));
        return -1;
    }
    err = gnutls_dh_params_generate2 (dh_params, DH_BITS);
    if (err < 0) {
        qemudLog (QEMUD_ERR, "gnutls_dh_params_generate2: %s",
                  gnutls_strerror (err));
        return -1;
    }

    gnutls_certificate_set_dh_params (x509_cred, dh_params);

    return 0;
}

static int qemudDispatchSignal(struct qemud_server *server)
{
    unsigned char sigc;
    struct qemud_vm *vm;
    struct qemud_network *network;
    int ret;

    if (read(server->sigread, &sigc, 1) != 1) {
        qemudLog(QEMUD_ERR, "Failed to read from signal pipe: %s",
                 strerror(errno));
        return -1;
    }

    ret = 0;

    switch (sigc) {
    case SIGHUP:
        qemudLog(QEMUD_INFO, "Reloading configuration on SIGHUP");
        if (!remote) {
            ret = qemudScanConfigs(server);

            if (server->iptables) {
                qemudLog(QEMUD_INFO, "Reloading iptables rules");
                iptablesReloadRules(server->iptables);
            }
        }
        break;

    case SIGINT:
    case SIGQUIT:
    case SIGTERM:
        qemudLog(QEMUD_WARN, "Shutting down on signal %d", sigc);

        if (!remote) {
            /* shutdown active VMs */
            vm = server->vms;
            while (vm) {
                struct qemud_vm *next = vm->next;
                if (qemudIsActiveVM(vm))
                    qemudShutdownVMDaemon(server, vm);
                vm = next;
            }

            /* free inactive VMs */
            vm = server->vms;
            while (vm) {
                struct qemud_vm *next = vm->next;
                qemudFreeVM(vm);
                vm = next;
            }
            server->vms = NULL;
            server->nactivevms = 0;
            server->ninactivevms = 0;

            /* shutdown active networks */
            network = server->networks;
            while (network) {
                struct qemud_network *next = network->next;
                if (qemudIsActiveNetwork(network))
                    qemudShutdownNetworkDaemon(server, network);
                network = next;
            }

            /* free inactive networks */
            network = server->networks;
            while (network) {
                struct qemud_network *next = network->next;
                qemudFreeNetwork(network);
                network = next;
            }
            server->networks = NULL;
            server->nactivenetworks = 0;
            server->ninactivenetworks = 0;
        }

        server->shutdown = 1;
        break;

    default:
        break;
    }

    return ret;
}

static int qemudSetCloseExec(int fd) {
    int flags;
    if ((flags = fcntl(fd, F_GETFD)) < 0)
        goto error;
    flags |= FD_CLOEXEC;
    if ((fcntl(fd, F_SETFD, flags)) < 0)
        goto error;
    return 0;
 error:
    qemudLog(QEMUD_ERR, "Failed to set close-on-exec file descriptor flag");
    return -1;
}


static int qemudSetNonBlock(int fd) {
    int flags;
    if ((flags = fcntl(fd, F_GETFL)) < 0)
        goto error;
    flags |= O_NONBLOCK;
    if ((fcntl(fd, F_SETFL, flags)) < 0)
        goto error;
    return 0;
 error:
    qemudLog(QEMUD_ERR, "Failed to set non-blocking file descriptor flag");
    return -1;
}

void qemudLog(int priority, const char *fmt, ...)
{
    va_list args;

    va_start(args, fmt);

    if (godaemon) {
        int sysprio = -1;

        switch(priority) {
        case QEMUD_ERR:
            sysprio = LOG_ERR;
            break;
        case QEMUD_WARN:
            sysprio = LOG_WARNING;
            break;
        case QEMUD_INFO:
            if (verbose)
                sysprio = LOG_INFO;
            break;
#ifdef ENABLE_DEBUG
        case QEMUD_DEBUG:
            if (verbose)
                sysprio = LOG_DEBUG;
            break;
#endif
        default:
            break;
        }

        if (sysprio != -1)
            vsyslog(sysprio, fmt, args);
    } else {
        switch(priority) {
        case QEMUD_ERR:
        case QEMUD_WARN:
            vfprintf(stderr, fmt, args);
            fputc('\n', stderr);
            break;

        case QEMUD_INFO:
            if (verbose) {
                vprintf(fmt, args);
                fputc('\n', stdout);
            }
            break;

#ifdef ENABLE_DEBUG
        case QEMUD_DEBUG:
            if (verbose) {
                vprintf(fmt, args);
                fputc('\n', stdout);
            }
            break;
#endif
        default:
            break;
        }
    }

    va_end(args);
}

static int qemudGoDaemon(void) {
    int pid = fork();
    switch (pid) {
    case 0:
        {
            int stdinfd = -1;
            int stdoutfd = -1;
            int nextpid;

            if ((stdinfd = open(_PATH_DEVNULL, O_RDONLY)) < 0)
                goto cleanup;
            if ((stdoutfd = open(_PATH_DEVNULL, O_WRONLY)) < 0)
                goto cleanup;
            if (dup2(stdinfd, STDIN_FILENO) != STDIN_FILENO)
                goto cleanup;
            if (dup2(stdoutfd, STDOUT_FILENO) != STDOUT_FILENO)
                goto cleanup;
            if (dup2(stdoutfd, STDERR_FILENO) != STDERR_FILENO)
                goto cleanup;
            if (close(stdinfd) < 0)
                goto cleanup;
            stdinfd = -1;
            if (close(stdoutfd) < 0)
                goto cleanup;
            stdoutfd = -1;

            if (setsid() < 0)
                goto cleanup;

            nextpid = fork();
            switch (nextpid) {
            case 0:
                return 0;
            case -1:
                return -1;
            default:
                return nextpid;
            }

        cleanup:
            if (stdoutfd != -1)
                close(stdoutfd);
            if (stdinfd != -1)
                close(stdinfd);
            return -1;

        }

    case -1:
        return -1;

    default:
        {
            int got, status = 0;
            /* We wait to make sure the next child forked
               successfully */
            if ((got = waitpid(pid, &status, 0)) < 0 ||
                got != pid ||
                status != 0) {
                return -1;
            }
      
            return pid;
        }
    }
}

static int qemudWritePidFile(const char *pidFile) {
    int fd;
    FILE *fh;

    if (pidFile[0] == '\0')
        return 0;

    if ((fd = open(pidFile, O_WRONLY|O_CREAT|O_EXCL, 0644)) < 0) {
        qemudLog(QEMUD_ERR, "Failed to open pid file '%s' : %s",
                 pidFile, strerror(errno));
        return -1;
    }

    if (!(fh = fdopen(fd, "w"))) {
        qemudLog(QEMUD_ERR, "Failed to fdopen pid file '%s' : %s",
                 pidFile, strerror(errno));
        close(fd);
        return -1;
    }

    if (fprintf(fh, "%lu\n", (unsigned long)getpid()) < 0) {
        qemudLog(QEMUD_ERR, "Failed to write to pid file '%s' : %s",
                 pidFile, strerror(errno));
        close(fd);
        return -1;
    }

    if (fclose(fh) == EOF) {
        qemudLog(QEMUD_ERR, "Failed to close pid file '%s' : %s",
                 pidFile, strerror(errno));
        return -1;
    }

    return 0;
}

static int qemudListenUnix(struct qemud_server *server,
                           const char *path, int readonly) {
    struct qemud_socket *sock = calloc(1, sizeof(struct qemud_socket));
    struct sockaddr_un addr;
    mode_t oldmask;

    if (!sock) {
        qemudLog(QEMUD_ERR, "Failed to allocate memory for struct qemud_socket");
        return -1;
    }

    sock->readonly = readonly;
    sock->next = server->sockets;
    server->sockets = sock;
    server->nsockets++;

    if ((sock->fd = socket(PF_UNIX, SOCK_STREAM, 0)) < 0) {
        qemudLog(QEMUD_ERR, "Failed to create socket: %s",
                 strerror(errno));
        return -1;
    }

    if (qemudSetCloseExec(sock->fd) < 0 ||
        qemudSetNonBlock(sock->fd) < 0)
        return -1;

    memset(&addr, 0, sizeof(addr));
    addr.sun_family = AF_UNIX;
    strncpy(addr.sun_path, path, sizeof(addr.sun_path)-1);
    if (addr.sun_path[0] == '@')
        addr.sun_path[0] = '\0';


    if (readonly)
        oldmask = umask(~(S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH));
    else
        oldmask = umask(~(S_IRUSR | S_IWUSR));
    if (bind(sock->fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
        qemudLog(QEMUD_ERR, "Failed to bind socket to '%s': %s",
                 path, strerror(errno));
        return -1;
    }
    umask(oldmask);

    if (listen(sock->fd, 30) < 0) {
        qemudLog(QEMUD_ERR, "Failed to listen for connections on '%s': %s",
                 path, strerror(errno));
        return -1;
    }

    return 0;
}

// See: http://people.redhat.com/drepper/userapi-ipv6.html
static int
remoteMakeSockets (int *fds, int max_fds, int *nfds_r, const char *service)
{
    struct addrinfo *ai;
    struct addrinfo hints;
    memset (&hints, 0, sizeof hints);
    hints.ai_flags = AI_PASSIVE | AI_ADDRCONFIG;
    hints.ai_socktype = SOCK_STREAM;

    int e = getaddrinfo (NULL, service, &hints, &ai);
    if (e != 0) {
        qemudLog (QEMUD_ERR, "getaddrinfo: %s\n", gai_strerror (e));
        return -1;
    }

    struct addrinfo *runp = ai;
    while (runp && *nfds_r < max_fds) {
        fds[*nfds_r] = socket (runp->ai_family, runp->ai_socktype,
                               runp->ai_protocol);
        if (fds[*nfds_r] == -1) {
            qemudLog (QEMUD_ERR, "socket: %s", strerror (errno));
            return -1;
        }

        int opt = 1;
        setsockopt (fds[*nfds_r], SOL_SOCKET, SO_REUSEADDR, &opt, sizeof opt);

        if (bind (fds[*nfds_r], runp->ai_addr, runp->ai_addrlen) == -1) {
            if (errno != EADDRINUSE) {
                qemudLog (QEMUD_ERR, "bind: %s", strerror (errno));
                return -1;
            }
            close (fds[*nfds_r]);
        }
        else {
            if (listen (fds[*nfds_r], SOMAXCONN) == -1) {
                qemudLog (QEMUD_ERR, "listen: %s", strerror (errno));
                return -1;
            }
            ++*nfds_r;
        }
        runp = runp->ai_next;
    }

    freeaddrinfo (ai);
    return 0;
}

/* Listen on the named/numbered TCP port.  On a machine with IPv4 and
 * IPv6 interfaces this may generate several sockets.
 */
static int
remoteListenTCP (struct qemud_server *server,
                 const char *port,
                 int tls)
{
    int fds[2];
    int nfds = 0;
    int i;
    struct qemud_socket *sock;

    if (remoteMakeSockets (fds, 2, &nfds, port) == -1)
        return -1;

    for (i = 0; i < nfds; ++i) {
        sock = calloc (1, sizeof *sock);

        if (!sock) {
            qemudLog (QEMUD_ERR,
                      "remoteListenTCP: calloc: %s", strerror (errno));
            return -1;
        }

        sock->readonly = 0;
        sock->next = server->sockets;
        server->sockets = sock;
        server->nsockets++;

        sock->fd = fds[i];
        sock->tls = tls;

        if (qemudSetCloseExec(sock->fd) < 0 ||
            qemudSetNonBlock(sock->fd) < 0)
            return -1;

        if (listen (sock->fd, 30) < 0) {
            qemudLog (QEMUD_ERR,
                      "remoteListenTCP: listen: %s", strerror (errno));
            return -1;
        }
    }

    return 0;
}

static int qemudInitPaths(struct qemud_server *server,
                          char *sockname,
                          char *roSockname,
                          int maxlen) {
    char *base = 0;

    if (remote) {               /* Remote daemon */
        /* I'm not sure if it's meaningful to have a "session remote daemon"
         * so currently this code ignores the --system flag. - RWMJ.
         */

        if (snprintf (sockname, maxlen, "%s/run/libvirt/libvirt-sock",
                      LOCAL_STATE_DIR) >= maxlen)
            goto snprintf_error;

        unlink(sockname);

        if (snprintf (roSockname, maxlen, "%s/run/libvirt/libvirt-sock-ro",
                      LOCAL_STATE_DIR) >= maxlen)
            goto snprintf_error;

        unlink(roSockname);

        server->configDir =
            server->autostartDir =
            server->networkConfigDir =
            server->networkAutostartDir = NULL;

        if (snprintf(server->logDir, PATH_MAX, "%s/log/libvirt/qemu", LOCAL_STATE_DIR) >= PATH_MAX)
            goto snprintf_error;
    } else {
        uid_t uid = geteuid();
        struct passwd *pw;

        if (sys) {              /* QEMUD, system */
            if (uid != 0) {
                qemudLog (QEMUD_ERR,
                      "You must run the daemon as root to use system mode");
                return -1;
            }

            if (snprintf(sockname, maxlen, "%s/run/libvirt/qemud-sock", LOCAL_STATE_DIR) >= maxlen)
                goto snprintf_error;

            unlink(sockname);

            if (snprintf(roSockname, maxlen, "%s/run/libvirt/qemud-sock-ro", LOCAL_STATE_DIR) >= maxlen)
                goto snprintf_error;

            unlink(roSockname);

            if ((base = strdup (SYSCONF_DIR "/libvirt/qemu")) == NULL)
                goto out_of_memory;
        } else {                /* QEMUD, session */
            if (!(pw = getpwuid(uid))) {
                qemudLog(QEMUD_ERR, "Failed to find user record for uid '%d': %s",
                         uid, strerror(errno));
                return -1;
            }

            if (snprintf(sockname, maxlen, "@%s/.libvirt/qemud-sock", pw->pw_dir) >= maxlen)
                goto snprintf_error;

            if (asprintf (&base, "%s/.libvirt/qemu", pw->pw_dir) == -1) {
                qemudLog (QEMUD_ERR, "out of memory in asprintf");
                return -1;
            }
        }

        /* Configuration paths are either ~/.libvirt/qemu/... (session) or
         * /etc/libvirt/qemu/... (system).
         */
        if (asprintf (&server->configDir, "%s", base) == -1)
            goto out_of_memory;

        if (asprintf (&server->autostartDir, "%s/autostart", base) == -1)
            goto out_of_memory;

        if (asprintf (&server->networkConfigDir, "%s/networks", base) == -1)
            goto out_of_memory;

        if (asprintf (&server->networkAutostartDir, "%s/networks/autostart",
                      base) == -1)
            goto out_of_memory;

        if (snprintf(server->logDir, PATH_MAX, "%s/log", base) >= PATH_MAX)
            goto snprintf_error;
    } /* !remote */

    if (base) free (base);

    return 0;

 snprintf_error:
    qemudLog(QEMUD_ERR, "Resulting path to long for buffer in qemudInitPaths()");
    return -1;

 out_of_memory:
    qemudLog (QEMUD_ERR, "qemudInitPaths: out of memory");
    if (base) free (base);
    return -1;
}

static struct qemud_server *qemudInitialize(int sigread) {
    struct qemud_server *server;
    char sockname[PATH_MAX];
    char roSockname[PATH_MAX];

    if (!(server = calloc(1, sizeof(struct qemud_server)))) {
        qemudLog(QEMUD_ERR, "Failed to allocate struct qemud_server");
        return NULL;
    }

    /* We don't have a dom-0, so start from 1 */
    server->nextvmid = 1;
    server->sigread = sigread;

    roSockname[0] = '\0';

    if (qemudInitPaths(server, sockname, roSockname, PATH_MAX) < 0)
        goto cleanup;

    if (qemudListenUnix(server, sockname, 0) < 0)
        goto cleanup;

    if (roSockname[0] != '\0' && qemudListenUnix(server, roSockname, 1) < 0)
        goto cleanup;

    if (!remote) /* qemud only */ {
        if (qemudScanConfigs(server) < 0) {
            goto cleanup;
        }
    } else /* remote only */ {
        if (listen_tcp && remoteListenTCP (server, tcp_port, 0) < 0)
            goto cleanup;

        if (listen_tls) {
            if (remoteInitializeGnuTLS () < 0)
                goto cleanup;

            if (remoteListenTCP (server, tls_port, 1) < 0)
                goto cleanup;
        }
    }

    return server;

 cleanup:
    if (server) {
        struct qemud_socket *sock = server->sockets;
        while (sock) {
            close(sock->fd);
            sock = sock->next;
        }

        if (server->configDir) free (server->configDir);
        if (server->autostartDir) free (server->autostartDir);
        if (server->networkConfigDir) free (server->networkConfigDir);
        if (server->networkAutostartDir) free (server->networkAutostartDir);

        free(server);
    }
    return NULL;
}

static gnutls_session_t
remoteInitializeTLSSession (void)
{
  gnutls_session_t session;
  int err;

  err = gnutls_init (&session, GNUTLS_SERVER);
  if (err != 0) goto failed;

  /* avoid calling all the priority functions, since the defaults
   * are adequate.
   */
  err = gnutls_set_default_priority (session);
  if (err != 0) goto failed;

  err = gnutls_credentials_set (session, GNUTLS_CRD_CERTIFICATE, x509_cred);
  if (err != 0) goto failed;

  /* request client certificate if any.
   */
  gnutls_certificate_server_set_request (session, GNUTLS_CERT_REQUEST);

  gnutls_dh_set_prime_bits (session, DH_BITS);

  return session;

 failed:
  qemudLog (QEMUD_ERR, "remoteInitializeTLSSession: %s",
            gnutls_strerror (err));
  return NULL;
}

/* Check DN is on tls_allowed_dn_list. */
static int
remoteCheckDN (gnutls_x509_crt_t cert)
{
    char name[256];
    size_t namesize = sizeof name;
    const char **wildcards;
    int err;

    err = gnutls_x509_crt_get_dn (cert, name, &namesize);
    if (err != 0) {
        qemudLog (QEMUD_ERR,
                  "remoteCheckDN: gnutls_x509_cert_get_dn: %s",
                  gnutls_strerror (err));
        return 0;
    }

    /* If the list is not set, allow any DN. */
    wildcards = tls_allowed_dn_list;
    if (!wildcards)
        return 1;

    while (*wildcards) {
        if (fnmatch (*wildcards, name, 0) == 0)
            return 1;
        wildcards++;
    }

#ifdef ENABLE_DEBUG
    /* Print the client's DN. */
    qemudLog (QEMUD_DEBUG,
              "remoteCheckDN: failed: client DN is %s", name);
#endif

    return 0; // Not found.
}

static int
remoteCheckCertificate (gnutls_session_t session)
{
    int ret;
    unsigned int status;
    const gnutls_datum_t *certs;
    unsigned int nCerts, i;
    time_t now;

    if ((ret = gnutls_certificate_verify_peers2 (session, &status)) < 0){
        qemudLog (QEMUD_ERR, "remoteCheckCertificate: verify failed: %s",
                  gnutls_strerror (ret));
        return -1;
    }

    if (status != 0) {
        if (status & GNUTLS_CERT_INVALID)
            qemudLog (QEMUD_ERR, "remoteCheckCertificate: the client certificate is not trusted.");

        if (status & GNUTLS_CERT_SIGNER_NOT_FOUND)
            qemudLog (QEMUD_ERR, "remoteCheckCertificate: the client certificate hasn't got a known issuer.");

        if (status & GNUTLS_CERT_REVOKED)
            qemudLog (QEMUD_ERR, "remoteCheckCertificate: the client certificate has been revoked.");

        if (status & GNUTLS_CERT_INSECURE_ALGORITHM)
            qemudLog (QEMUD_ERR, "remoteCheckCertificate: the client certificate uses an insecure algorithm.");

        return -1;
    }

    if (gnutls_certificate_type_get (session) != GNUTLS_CRT_X509) {
        qemudLog (QEMUD_ERR, "remoteCheckCertificate: certificate is not X.509");
        return -1;
    }

    if (!(certs = gnutls_certificate_get_peers(session, &nCerts))) {
        qemudLog (QEMUD_ERR, "remoteCheckCertificate: no peers");
        return -1;
    }

    now = time (NULL);

    for (i = 0; i < nCerts; i++) {
        gnutls_x509_crt_t cert;

        if (gnutls_x509_crt_init (&cert) < 0) {
            qemudLog (QEMUD_ERR, "remoteCheckCertificate: gnutls_x509_crt_init failed");
            return -1;
        }

        if (gnutls_x509_crt_import(cert, &certs[i], GNUTLS_X509_FMT_DER) < 0) {
            gnutls_x509_crt_deinit (cert);
            return -1;
        }
    
        if (gnutls_x509_crt_get_expiration_time (cert) < now) {
            qemudLog (QEMUD_ERR, "remoteCheckCertificate: the client certificate has expired");
            gnutls_x509_crt_deinit (cert);
            return -1;
        }
    
        if (gnutls_x509_crt_get_activation_time (cert) > now) {
            qemudLog (QEMUD_ERR, "remoteCheckCertificate: the client certificate is not yet activated");
            gnutls_x509_crt_deinit (cert);
            return -1;
        }

        if (i == 0) {
            if (!remoteCheckDN (cert)) {
                /* This is the most common error: make it informative. */
                qemudLog (QEMUD_ERR, "remoteCheckCertificate: client's Distinguished Name is not on the list of allowed clients (tls_allowed_dn_list).  Use 'openssl x509 -in clientcert.pem -text' to view the Distinguished Name field in the client certificate, or run this daemon with --verbose option.");
                gnutls_x509_crt_deinit (cert);
                return -1;
            }
        }
    }

    return 0;
}

/* Check the client's access. */
static int
remoteCheckAccess (struct qemud_client *client)
{
    char addr[NI_MAXHOST];
    const char **wildcards;
    int found, err;

    /* Verify client certificate. */
    if (remoteCheckCertificate (client->session) == -1) {
        qemudLog (QEMUD_ERR, "remoteCheckCertificate: failed to verify client's certificate");
        if (!tls_no_verify_certificate) return -1;
        else qemudLog (QEMUD_INFO, "remoteCheckCertificate: tls_no_verify_certificate is set so the bad certificate is ignored");
    }

    /*----- IP address check, similar to tcp wrappers -----*/

    /* Convert IP address to printable string (eg. "127.0.0.1" or "::1"). */
    err = getnameinfo ((struct sockaddr *) &client->addr, client->addrlen,
                       addr, sizeof addr, NULL, 0,
                       NI_NUMERICHOST);
    if (err != 0) {
        qemudLog (QEMUD_ERR, "getnameinfo: %s", gai_strerror (err));
        return -1;
    }

    /* Verify the client is on the list of allowed clients.
     *
     * NB: No tls_allowed_ip_list in config file means anyone can access.
     * If tls_allowed_ip_list is in the config file but empty, means no
     * one can access (not particularly useful, but it's what the sysadmin
     * would expect).
     */
    wildcards = tls_allowed_ip_list;
    if (wildcards) {
        found = 0;

        while (*wildcards) {
            if (fnmatch (*wildcards, addr, 0) == 0) {
                found = 1;
                break;
            }
            wildcards++;
        }
    } else
        found = 1;

    if (!found) {
        qemudLog (QEMUD_ERR, "remoteCheckAccess: client's IP address (%s) is not on the list of allowed clients (tls_allowed_ip_list)", addr);
        if (!tls_no_verify_address) return -1;
        else qemudLog (QEMUD_INFO, "remoteCheckAccess: tls_no_verify_address is set so the client's IP address is ignored");
    }

    /* Checks have succeeded.  Write a '\1' byte back to the client to
     * indicate this (otherwise the socket is abruptly closed).
     * (NB. The '\1' byte is sent in an encrypted record).
     */
    client->bufferLength = 1;
    client->bufferOffset = 0;
    client->buffer[0] = '\1';
    client->mode = QEMUD_MODE_TX_PACKET;
    client->direction = QEMUD_TLS_DIRECTION_WRITE;
    return 0;
}

static int qemudDispatchServer(struct qemud_server *server, struct qemud_socket *sock) {
    int fd;
    struct sockaddr_storage addr;
    socklen_t addrlen = (socklen_t) (sizeof addr);
    struct qemud_client *client;
    int no_slow_start = 1;

    if ((fd = accept(sock->fd, (struct sockaddr *)&addr, &addrlen)) < 0) {
        if (errno == EAGAIN)
            return 0;
        qemudLog(QEMUD_ERR, "Failed to accept connection: %s", strerror(errno));
        return -1;
    }

    /* Disable Nagle.  Unix sockets will ignore this. */
    setsockopt (fd, IPPROTO_TCP, TCP_NODELAY, (void *)&no_slow_start,
                sizeof no_slow_start);

    if (qemudSetCloseExec(fd) < 0 ||
        qemudSetNonBlock(fd) < 0) {
        close(fd);
        return -1;
    }

    client = calloc(1, sizeof(struct qemud_client));
    client->magic = QEMUD_CLIENT_MAGIC;
    client->fd = fd;
    client->readonly = sock->readonly;
    client->tls = sock->tls;
    memcpy (&client->addr, &addr, sizeof addr);
    client->addrlen = addrlen;

    if (!client->tls) {
        client->mode = QEMUD_MODE_RX_HEADER;
        client->bufferLength = QEMUD_PKT_HEADER_XDR_LEN;
    } else {
        int ret;

        client->session = remoteInitializeTLSSession ();
        if (client->session == NULL) goto tls_failed;

        gnutls_transport_set_ptr (client->session,
                                  (gnutls_transport_ptr_t) (long) fd);

        /* Begin the TLS handshake. */
        ret = gnutls_handshake (client->session);
        if (ret == 0) {
            /* Unlikely, but ...  Next step is to check the certificate. */
            if (remoteCheckAccess (client) == -1)
                goto tls_failed;
        } else if (ret == GNUTLS_E_INTERRUPTED || ret == GNUTLS_E_AGAIN) {
            /* Most likely. */
            client->mode = QEMUD_MODE_TLS_HANDSHAKE;
            client->bufferLength = -1;
            client->direction = gnutls_record_get_direction (client->session);
        } else {
            qemudLog (QEMUD_ERR, "TLS handshake failed: %s",
                      gnutls_strerror (ret));
            goto tls_failed;
        }
    }

    client->next = server->clients;
    server->clients = client;
    server->nclients++;

    return 0;

 tls_failed:
    if (client->session) gnutls_deinit (client->session);
    close (fd);
    free (client);
    return -1;
}


static int
qemudExec(struct qemud_server *server, char **argv,
          int *retpid, int *outfd, int *errfd) {
    int pid, null;
    int pipeout[2] = {-1,-1};
    int pipeerr[2] = {-1,-1};

    if ((null = open(_PATH_DEVNULL, O_RDONLY)) < 0) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR, "cannot open %s : %s",
                         _PATH_DEVNULL, strerror(errno));
        goto cleanup;
    }

    if ((outfd != NULL && pipe(pipeout) < 0) ||
        (errfd != NULL && pipe(pipeerr) < 0)) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR, "cannot create pipe : %s",
                         strerror(errno));
        goto cleanup;
    }

    if ((pid = fork()) < 0) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR, "cannot fork child process : %s",
                         strerror(errno));
        goto cleanup;
    }

    if (pid) { /* parent */
        close(null);
        if (outfd) {
            close(pipeout[1]);
            qemudSetNonBlock(pipeout[0]);
            qemudSetCloseExec(pipeout[0]);
            *outfd = pipeout[0];
        }
        if (errfd) {
            close(pipeerr[1]);
            qemudSetNonBlock(pipeerr[0]);
            qemudSetCloseExec(pipeerr[0]);
            *errfd = pipeerr[0];
        }
        *retpid = pid;
        return 0;
    }

    /* child */

    if (pipeout[0] > 0 && close(pipeout[0]) < 0)
        _exit(1);
    if (pipeerr[0] > 0 && close(pipeerr[0]) < 0)
        _exit(1);

    if (dup2(null, STDIN_FILENO) < 0)
        _exit(1);
    if (dup2(pipeout[1] > 0 ? pipeout[1] : null, STDOUT_FILENO) < 0)
        _exit(1);
    if (dup2(pipeerr[1] > 0 ? pipeerr[1] : null, STDERR_FILENO) < 0)
        _exit(1);

    close(null);
    if (pipeout[1] > 0)
        close(pipeout[1]);
    if (pipeerr[1] > 0)
        close(pipeerr[1]);

    execvp(argv[0], argv);

    _exit(1);

    return 0;

 cleanup:
    if (pipeerr[0] > 0)
        close(pipeerr[0]);
    if (pipeerr[1] > 0)
        close(pipeerr[1]);
    if (pipeout[0] > 0)
        close(pipeout[0]);
    if (pipeout[1] > 0)
        close(pipeout[1]);
    if (null > 0)
        close(null);
    return -1;
}

/* Return -1 for error, 1 to continue reading and 0 for success */
typedef int qemudHandlerMonitorOutput(struct qemud_server *server,
                                      struct qemud_vm *vm,
                                      const char *output,
                                      int fd);

static int
qemudReadMonitorOutput(struct qemud_server *server,
                       struct qemud_vm *vm,
                       int fd,
                       char *buffer,
                       int buflen,
                       qemudHandlerMonitorOutput func,
                       const char *what)
{
#define MONITOR_TIMEOUT 3000

    int got = 0;
    buffer[0] = '\0';

   /* Consume & discard the initial greeting */
    while (got < (buflen-1)) {
        int ret;

        ret = read(fd, buffer+got, buflen-got-1);
        if (ret == 0) {
            qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                             "QEMU quit during %s startup\n%s", what, buffer);
            return -1;
        }
        if (ret < 0) {
            struct pollfd pfd = { .fd = fd, .events = POLLIN };
            if (errno == EINTR)
                continue;

            if (errno != EAGAIN) {
                qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                                 "Failure while reading %s startup output: %s",
                                 what, strerror(errno));
                return -1;
            }

            ret = poll(&pfd, 1, MONITOR_TIMEOUT);
            if (ret == 0) {
                qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                                 "Timed out while reading %s startup output", what);
                return -1;
            } else if (ret == -1) {
                if (errno != EINTR) {
                    qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                                     "Failure while reading %s startup output: %s",
                                     what, strerror(errno));
                    return -1;
                }
            } else {
                /* Make sure we continue loop & read any further data
                   available before dealing with EOF */
                if (pfd.revents & (POLLIN | POLLHUP))
                    continue;

                qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                                 "Failure while reading %s startup output", what);
                return -1;
            }
        } else {
            got += ret;
            buffer[got] = '\0';
            if ((ret = func(server, vm, buffer, fd)) != 1)
                return ret;
        }
    }

    qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                     "Out of space while reading %s startup output", what);
    return -1;

#undef MONITOR_TIMEOUT
}

static int
qemudCheckMonitorPrompt(struct qemud_server *server ATTRIBUTE_UNUSED,
                        struct qemud_vm *vm,
                        const char *output,
                        int fd)
{
    if (strstr(output, "(qemu) ") == NULL)
        return 1; /* keep reading */

    vm->monitor = fd;

    return 0;
}

static int qemudOpenMonitor(struct qemud_server *server, struct qemud_vm *vm, const char *monitor) {
    int monfd;
    char buffer[1024];
    int ret = -1;

    if (!(monfd = open(monitor, O_RDWR))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "Unable to open monitor path %s", monitor);
        return -1;
    }
    if (qemudSetCloseExec(monfd) < 0) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "Unable to set monitor close-on-exec flag");
        goto error;
    }
    if (qemudSetNonBlock(monfd) < 0) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "Unable to put monitor into non-blocking mode");
        goto error;
    }

    ret = qemudReadMonitorOutput(server, vm, monfd,
                                 buffer, sizeof(buffer),
                                 qemudCheckMonitorPrompt,
                                 "monitor");
 error:
    close(monfd);
    return ret;
}

static int qemudExtractMonitorPath(const char *haystack, char *path, int pathmax) {
    static const char needle[] = "char device redirected to";
    char *tmp;

    if (!(tmp = strstr(haystack, needle)))
        return -1;

    strncpy(path, tmp+sizeof(needle), pathmax-1);
    path[pathmax-1] = '\0';

    while (*path) {
        /*
         * The monitor path ends at first whitespace char
         * so lets search for it & NULL terminate it there
         */
        if (isspace(*path)) {
            *path = '\0';
            return 0;
        }
        path++;
    }

    /*
     * We found a path, but didn't find any whitespace,
     * so it must be still incomplete - we should at
     * least see a \n
     */
    return -1;
}

static int
qemudOpenMonitorPath(struct qemud_server *server,
                     struct qemud_vm *vm,
                     const char *output,
                     int fd ATTRIBUTE_UNUSED)
{
    char monitor[PATH_MAX];

    if (qemudExtractMonitorPath(output, monitor, sizeof(monitor)) < 0)
        return 1; /* keep reading */

    return qemudOpenMonitor(server, vm, monitor);
}

static int qemudWaitForMonitor(struct qemud_server *server, struct qemud_vm *vm) {
    char buffer[1024]; /* Plenty of space to get startup greeting */
    int ret = qemudReadMonitorOutput(server, vm, vm->stderr,
                                     buffer, sizeof(buffer),
                                     qemudOpenMonitorPath,
                                     "console");

    buffer[sizeof(buffer)-1] = '\0';
 retry:
    if (write(vm->logfile, buffer, strlen(buffer)) < 0) {
        /* Log, but ignore failures to write logfile for VM */
        if (errno == EINTR)
            goto retry;
        qemudLog(QEMUD_WARN, "Unable to log VM console data: %s",
                 strerror(errno));
    }

    return ret;
}

static int qemudNextFreeVNCPort(struct qemud_server *server ATTRIBUTE_UNUSED) {
    int i;

    for (i = 5900 ; i < 6000 ; i++) {
        int fd;
        int reuse = 1;
        struct sockaddr_in addr;
        addr.sin_family = AF_INET;
        addr.sin_port = htons(i);
        addr.sin_addr.s_addr = htonl(INADDR_ANY);
        fd = socket(PF_INET, SOCK_STREAM, 0);
        if (fd < 0)
            return -1;

        if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (void*)&reuse, sizeof(reuse)) < 0) {
            close(fd);
            break;
        }

        if (bind(fd, (struct sockaddr*)&addr, sizeof(addr)) == 0) {
            /* Not in use, lets grab it */
            close(fd);
            return i;
        }
        close(fd);

        if (errno == EADDRINUSE) {
            /* In use, try next */
            continue;
        }
        /* Some other bad failure, get out.. */
        break;
    }
    return -1;
}

int qemudStartVMDaemon(struct qemud_server *server,
                       struct qemud_vm *vm) {
    char **argv = NULL, **tmp;
    int i, ret = -1;
    char logfile[PATH_MAX];

    if (qemudIsActiveVM(vm)) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "VM is already active");
        return -1;
    }

    if (vm->def->vncPort < 0) {
        int port = qemudNextFreeVNCPort(server);
        if (port < 0) {
            qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                             "Unable to find an unused VNC port");
            return -1;
        }
        vm->def->vncActivePort = port;
    } else
        vm->def->vncActivePort = vm->def->vncPort;

    if ((strlen(server->logDir) + /* path */
         1 + /* Separator */
         strlen(vm->def->name) + /* basename */
         4 + /* suffix .log */
         1 /* NULL */) > PATH_MAX) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "config file path too long: %s/%s.log",
                         server->logDir, vm->def->name);
        return -1;
    }
    strcpy(logfile, server->logDir);
    strcat(logfile, "/");
    strcat(logfile, vm->def->name);
    strcat(logfile, ".log");

    if (qemudEnsureDir(server->logDir) < 0) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "cannot create log directory %s: %s",
                         server->logDir, strerror(errno));
        return -1;
    }

    if ((vm->logfile = open(logfile, O_CREAT | O_TRUNC | O_WRONLY,
                            S_IRUSR | S_IWUSR)) < 0) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to create logfile %s: %s",
                         logfile, strerror(errno));
        return -1;
    }

    if (qemudBuildCommandLine(server, vm, &argv) < 0) {
        close(vm->logfile);
        vm->logfile = -1;
        return -1;
    }

    tmp = argv;
    while (*tmp) {
        if (write(vm->logfile, *tmp, strlen(*tmp)) < 0)
            qemudLog(QEMUD_WARN, "Unable to write argv to logfile %d: %s",
                     errno, strerror(errno));
        if (write(vm->logfile, " ", 1) < 0)
            qemudLog(QEMUD_WARN, "Unable to write argv to logfile %d: %s",
                     errno, strerror(errno));
        tmp++;
    }
    if (write(vm->logfile, "\n", 1) < 0)
        qemudLog(QEMUD_WARN, "Unable to write argv to logfile %d: %s",
                 errno, strerror(errno));

    if (qemudExec(server, argv, &vm->pid, &vm->stdout, &vm->stderr) == 0) {
        vm->id = server->nextvmid++;
        vm->state = QEMUD_STATE_RUNNING;

        server->ninactivevms--;
        server->nactivevms++;
        server->nvmfds += 2;

        ret = 0;

        if (qemudWaitForMonitor(server, vm) < 0) {
            qemudShutdownVMDaemon(server, vm);
            ret = -1;
        }
    }

    if (vm->tapfds) {
        for (i = 0; vm->tapfds[i] != -1; i++) {
            close(vm->tapfds[i]);
            vm->tapfds[i] = -1;
        }
        free(vm->tapfds);
        vm->tapfds = NULL;
        vm->ntapfds = 0;
    }
  
    for (i = 0 ; argv[i] ; i++)
        free(argv[i]);
    free(argv);

    return ret;
}


static void qemudDispatchClientFailure(struct qemud_server *server, struct qemud_client *client) {
    struct qemud_client *tmp = server->clients;
    struct qemud_client *prev = NULL;
    while (tmp) {
        if (tmp == client) {
            if (prev == NULL)
                server->clients = client->next;
            else
                prev->next = client->next;
            server->nclients--;
            break;
        }
        prev = tmp;
        tmp = tmp->next;
    }

    if (client->tls && client->session) gnutls_deinit (client->session);
    close(client->fd);
    free(client);
}


static void qemudDispatchClientRequest(struct qemud_server *server,
                                       struct qemud_client *client,
                                       qemud_packet_client *req) {
    qemud_packet_server res;
    qemud_packet_header h;
    XDR x;

    assert (client->magic == QEMUD_CLIENT_MAGIC);

    if (req->serial != ++client->incomingSerial) {
        qemudDebug("Invalid serial number. Got %d expect %d",
                   req->serial, client->incomingSerial);
        qemudDispatchClientFailure(server, client);
        return;
    }

    if (qemudDispatch(server,
                      client,
                      &req->data,
                      &res.data) < 0) {
        qemudDispatchClientFailure(server, client);
        return;
    }

    res.serial = ++client->outgoingSerial;
    res.inReplyTo = req->serial;

    xdrmem_create(&x, client->buffer, sizeof client->buffer,
                  XDR_ENCODE);

    /* Encode a dummy header.  We'll come back to encode the real header. */
    if (!xdr_qemud_packet_header (&x, &h)) {
        qemudDebug ("failed to encode dummy header");
        qemudDispatchClientFailure (server, client);
        return;
    }

    /* Real payload. */
    if (!xdr_qemud_packet_server(&x, &res)) {
        qemudDebug("Failed to XDR encode reply payload");
        qemudDispatchClientFailure(server, client);
        return;
    }

    /* Go back and encode the real header. */
    h.length = xdr_getpos (&x);
    h.prog = QEMUD_PROGRAM;

    if (xdr_setpos (&x, 0) == 0) {
        qemudDebug("xdr_setpos failed");
        qemudDispatchClientFailure(server, client);
        return;
    }

    if (!xdr_qemud_packet_header(&x, &h)) {
        qemudDebug("Failed to XDR encode reply header");
        qemudDispatchClientFailure(server, client);
        return;
    }

    client->mode = QEMUD_MODE_TX_PACKET;
    client->bufferLength = h.length;
    client->bufferOffset = 0;
}

static int qemudClientRead(struct qemud_server *server,
                           struct qemud_client *client) {
    int ret, len;
    char *data;

    data = client->buffer + client->bufferOffset;
    len = client->bufferLength - client->bufferOffset;

    /*qemudDebug ("qemudClientRead: len = %d", len);*/

    if (!client->tls) {
        if ((ret = read (client->fd, data, len)) <= 0) {
            if (ret == 0 || errno != EAGAIN) {
                if (ret != 0)
                    qemudLog (QEMUD_ERR, "read: %s", strerror (errno));
                qemudDispatchClientFailure(server, client);
            }
            return -1;
        }
    } else {
        ret = gnutls_record_recv (client->session, data, len);
        client->direction = gnutls_record_get_direction (client->session);
        if (ret <= 0) {
            if (ret == 0 || (ret != GNUTLS_E_AGAIN &&
                             ret != GNUTLS_E_INTERRUPTED)) {
                if (ret != 0)
                    qemudLog (QEMUD_ERR, "gnutls_record_recv: %s",
                              gnutls_strerror (ret));
                qemudDispatchClientFailure (server, client);
            }
            return -1;
        }
    }

    client->bufferOffset += ret;
    return 0;
}

static void qemudDispatchClientRead(struct qemud_server *server, struct qemud_client *client) {

    /*qemudDebug ("qemudDispatchClientRead: mode = %d", client->mode);*/

    switch (client->mode) {
    case QEMUD_MODE_RX_HEADER: {
        XDR x;
        qemud_packet_header h;

        if (qemudClientRead(server, client) < 0)
            return; /* Error, or blocking */

        if (client->bufferOffset < client->bufferLength)
            return; /* Not read enough */

        xdrmem_create(&x, client->buffer, client->bufferLength, XDR_DECODE);

        if (!xdr_qemud_packet_header(&x, &h)) {
            qemudDebug("Failed to decode packet header");
            qemudDispatchClientFailure(server, client);
            return;
        }

        /* We're expecting either QEMUD_PROGRAM or REMOTE_PROGRAM,
         * corresponding to qemud or remote calls respectively.
         */
        if ((!remote && h.prog != QEMUD_PROGRAM)
            || (remote && h.prog != REMOTE_PROGRAM)) {
            qemudDebug("Header magic %x mismatch", h.prog);
            qemudDispatchClientFailure(server, client);
            return;
        }

        /* NB: h.length is unsigned. */
        if (h.length > REMOTE_MESSAGE_MAX) {
            qemudDebug("Packet length %u too large", h.length);
            qemudDispatchClientFailure(server, client);
            return;
        }

        client->mode = QEMUD_MODE_RX_PAYLOAD;
        client->bufferLength = h.length;
        if (client->tls) client->direction = QEMUD_TLS_DIRECTION_READ;
        /* Note that we don't reset bufferOffset here because we want
         * to retain the whole message, including header.
         */

        xdr_destroy (&x);

        /* Fall through */
    }

    case QEMUD_MODE_RX_PAYLOAD: {
        XDR x;
        qemud_packet_header h;

        if (qemudClientRead(server, client) < 0)
            return; /* Error, or blocking */

        if (client->bufferOffset < client->bufferLength)
            return; /* Not read enough */

        /* Reparse the header to decide if this is for qemud or remote. */
        xdrmem_create(&x, client->buffer, client->bufferLength, XDR_DECODE);

        if (!xdr_qemud_packet_header(&x, &h)) {
            qemudDebug("Failed to decode packet header");
            qemudDispatchClientFailure(server, client);
            return;
        }

        if (remote && h.prog == REMOTE_PROGRAM) {
            remoteDispatchClientRequest (server, client);
        } else if (!remote && h.prog == QEMUD_PROGRAM) {
            qemud_packet_client p;

            if (!xdr_qemud_packet_client(&x, &p)) {
                qemudDebug("Failed to decode client packet");
                qemudDispatchClientFailure(server, client);
                return;
            }

            qemudDispatchClientRequest(server, client, &p);
        } else {
            /* An internal error. */
            qemudDebug ("Not REMOTE_PROGRAM or QEMUD_PROGRAM");
            qemudDispatchClientFailure(server, client);
        }

        xdr_destroy (&x);

        break;
    }

    case QEMUD_MODE_TLS_HANDSHAKE: {
        int ret;

        /* Continue the handshake. */
        ret = gnutls_handshake (client->session);
        if (ret == 0) {
            /* Finished.  Next step is to check the certificate. */
            if (remoteCheckAccess (client) == -1)
                qemudDispatchClientFailure (server, client);
        } else if (ret != GNUTLS_E_AGAIN && ret != GNUTLS_E_INTERRUPTED) {
            qemudLog (QEMUD_ERR, "TLS handshake failed: %s",
                      gnutls_strerror (ret));
            qemudDispatchClientFailure (server, client);
        } else
            client->direction = gnutls_record_get_direction (client->session);

        break;
    }

    default:
        qemudDebug("Got unexpected data read while in %d mode", client->mode);
        qemudDispatchClientFailure(server, client);
    }
}


static int qemudClientWrite(struct qemud_server *server,
                            struct qemud_client *client) {
    int ret, len;
    char *data;

    data = client->buffer + client->bufferOffset;
    len = client->bufferLength - client->bufferOffset;

    if (!client->tls) {
        if ((ret = write(client->fd, data, len)) == -1) {
            if (errno != EAGAIN) {
                qemudLog (QEMUD_ERR, "write: %s", strerror (errno));
                qemudDispatchClientFailure(server, client);
            }
            return -1;
        }
    } else {
        ret = gnutls_record_send (client->session, data, len);
        client->direction = gnutls_record_get_direction (client->session);
        if (ret < 0) {
            if (ret != GNUTLS_E_INTERRUPTED && ret != GNUTLS_E_AGAIN) {
                qemudLog (QEMUD_ERR, "gnutls_record_send: %s",
                          gnutls_strerror (ret));
                qemudDispatchClientFailure (server, client);
            }
            return -1;
        }
    }

    client->bufferOffset += ret;
    return 0;
}


static void qemudDispatchClientWrite(struct qemud_server *server, struct qemud_client *client) {
    switch (client->mode) {
    case QEMUD_MODE_TX_PACKET: {
        if (qemudClientWrite(server, client) < 0)
            return;

        if (client->bufferOffset == client->bufferLength) {
            /* Done writing, switch back to receive */
            client->mode = QEMUD_MODE_RX_HEADER;
            client->bufferLength = QEMUD_PKT_HEADER_XDR_LEN;
            client->bufferOffset = 0;
            if (client->tls) client->direction = QEMUD_TLS_DIRECTION_READ;
        }
        /* Still writing */
        break;
    }

    case QEMUD_MODE_TLS_HANDSHAKE: {
        int ret;

        /* Continue the handshake. */
        ret = gnutls_handshake (client->session);
        if (ret == 0) {
            /* Finished.  Next step is to check the certificate. */
            if (remoteCheckAccess (client) == -1)
                qemudDispatchClientFailure (server, client);
        } else if (ret != GNUTLS_E_AGAIN && ret != GNUTLS_E_INTERRUPTED) {
            qemudLog (QEMUD_ERR, "TLS handshake failed: %s",
                      gnutls_strerror (ret));
            qemudDispatchClientFailure (server, client);
        } else
            client->direction = gnutls_record_get_direction (client->session);

        break;
    }

    default:
        qemudDebug("Got unexpected data write while in %d mode", client->mode);
        qemudDispatchClientFailure(server, client);
    }
}

static int qemudVMData(struct qemud_server *server ATTRIBUTE_UNUSED,
                       struct qemud_vm *vm, int fd) {
    char buf[4096];
    if (vm->pid < 0)
        return 0;

    for (;;) {
        int ret = read(fd, buf, sizeof(buf)-1);
        if (ret < 0) {
            if (errno == EAGAIN)
                return 0;
            return -1;
        }
        if (ret == 0) {
            return 0;
        }
        buf[ret] = '\0';

    retry:
        if (write(vm->logfile, buf, ret) < 0) {
            /* Log, but ignore failures to write logfile for VM */
            if (errno == EINTR)
                goto retry;
            qemudLog(QEMUD_WARN, "Unable to log VM console data: %s",
                     strerror(errno));
        }
    }
}


int qemudShutdownVMDaemon(struct qemud_server *server, struct qemud_vm *vm) {
    if (!qemudIsActiveVM(vm))
        return 0;

    qemudLog(QEMUD_INFO, "Shutting down VM '%s'", vm->def->name);

    kill(vm->pid, SIGTERM);

    qemudVMData(server, vm, vm->stdout);
    qemudVMData(server, vm, vm->stderr);
    if (close(vm->logfile) < 0)
        qemudLog(QEMUD_WARN, "Unable to close logfile %d: %s", errno, strerror(errno));
    close(vm->stdout);
    close(vm->stderr);
    if (vm->monitor != -1)
        close(vm->monitor);
    vm->logfile = -1;
    vm->stdout = -1;
    vm->stderr = -1;
    vm->monitor = -1;
    server->nvmfds -= 2;

    if (waitpid(vm->pid, NULL, WNOHANG) != vm->pid) {
        kill(vm->pid, SIGKILL);
        if (waitpid(vm->pid, NULL, 0) != vm->pid) {
            qemudLog(QEMUD_WARN, "Got unexpected pid, damn");
        }
    }

    vm->pid = -1;
    vm->id = -1;
    vm->state = QEMUD_STATE_STOPPED;

    if (vm->newDef) {
        qemudFreeVMDef(vm->def);
        vm->def = vm->newDef;
        vm->newDef = NULL;
    }

    server->nactivevms--;
    server->ninactivevms++;

    return 0;
}

static int qemudDispatchVMLog(struct qemud_server *server, struct qemud_vm *vm, int fd) {
    if (qemudVMData(server, vm, fd) < 0)
        if (qemudShutdownVMDaemon(server, vm) < 0)
            return -1;
    return 0;
}

static int qemudDispatchVMFailure(struct qemud_server *server, struct qemud_vm *vm,
                                  int fd ATTRIBUTE_UNUSED) {
    if (qemudShutdownVMDaemon(server, vm) < 0)
        return -1;
    return 0;
}

static int
qemudBuildDnsmasqArgv(struct qemud_server *server,
                      struct qemud_network *network,
                      char ***argv) {
    int i, len;
    char buf[PATH_MAX];
    struct qemud_dhcp_range_def *range;

    len =
        1 + /* dnsmasq */
        1 + /* --keep-in-foreground */
        1 + /* --strict-order */
        1 + /* --bind-interfaces */
        2 + /* --pid-file "" */
        2 + /* --conf-file "" */
        /*2 + *//* --interface virbr0 */
        2 + /* --except-interface lo */
        2 + /* --listen-address 10.0.0.1 */
        1 + /* --dhcp-leasefile=path */
        (2 * network->def->nranges) + /* --dhcp-range 10.0.0.2,10.0.0.254 */
        1;  /* NULL */

    if (!(*argv = calloc(len, sizeof(char *))))
        goto no_memory;

#define APPEND_ARG(v, n, s) do {     \
        if (!((v)[(n)] = strdup(s))) \
            goto no_memory;          \
    } while (0)

    i = 0;

    APPEND_ARG(*argv, i++, "dnsmasq");

    APPEND_ARG(*argv, i++, "--keep-in-foreground");
    /*
     * Needed to ensure dnsmasq uses same algorithm for processing
     * multiple nameserver entries in /etc/resolv.conf as GLibC.
     */
    APPEND_ARG(*argv, i++, "--strict-order");
    APPEND_ARG(*argv, i++, "--bind-interfaces");

    APPEND_ARG(*argv, i++, "--pid-file");
    APPEND_ARG(*argv, i++, "");

    APPEND_ARG(*argv, i++, "--conf-file");
    APPEND_ARG(*argv, i++, "");

    /*
     * XXX does not actually work, due to some kind of
     * race condition setting up ipv6 addresses on the
     * interface. A sleep(10) makes it work, but that's
     * clearly not practical
     *
     * APPEND_ARG(*argv, i++, "--interface");
     * APPEND_ARG(*argv, i++, network->def->bridge);
     */
    APPEND_ARG(*argv, i++, "--listen-address");
    APPEND_ARG(*argv, i++, network->def->ipAddress);

    APPEND_ARG(*argv, i++, "--except-interface");
    APPEND_ARG(*argv, i++, "lo");

    /*
     * NB, dnsmasq command line arg bug means we need to
     * use a single arg '--dhcp-leasefile=path' rather than
     * two separate args in '--dhcp-leasefile path' style
     */
    snprintf(buf, sizeof(buf), "--dhcp-leasefile=%s/lib/libvirt/dhcp-%s.leases",
             LOCAL_STATE_DIR, network->def->name);
    APPEND_ARG(*argv, i++, buf);

    range = network->def->ranges;
    while (range) {
        snprintf(buf, sizeof(buf), "%s,%s",
                 range->start, range->end);

        APPEND_ARG(*argv, i++, "--dhcp-range");
        APPEND_ARG(*argv, i++, buf);

        range = range->next;
    }

#undef APPEND_ARG

    return 0;

 no_memory:
    if (argv) {
        for (i = 0; (*argv)[i]; i++)
            free((*argv)[i]);
        free(*argv);
    }
    qemudReportError(server, VIR_ERR_NO_MEMORY, "dnsmasq argv");
    return -1;
}


static int
dhcpStartDhcpDaemon(struct qemud_server *server,
                    struct qemud_network *network)
{
    char **argv;
    int ret, i;

    if (network->def->ipAddress[0] == '\0') {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "cannot start dhcp daemon without IP address for server");
        return -1;
    }

    argv = NULL;
    if (qemudBuildDnsmasqArgv(server, network, &argv) < 0)
        return -1;

    ret = qemudExec(server, argv, &network->dnsmasqPid, NULL, NULL);

    for (i = 0; argv[i]; i++)
        free(argv[i]);
    free(argv);

    return ret;
}

static int
qemudAddIptablesRules(struct qemud_server *server,
                      struct qemud_network *network) {
    int err;

    if (!server->iptables && !(server->iptables = iptablesContextNew())) {
        qemudReportError(server, VIR_ERR_NO_MEMORY, "iptables support");
        return 1;
    }


    /* allow DHCP requests through to dnsmasq */
    if ((err = iptablesAddTcpInput(server->iptables, network->bridge, 67))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to add iptables rule to allow DHCP requests from '%s' : %s\n",
                         network->bridge, strerror(err));
        goto err1;
    }

    if ((err = iptablesAddUdpInput(server->iptables, network->bridge, 67))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to add iptables rule to allow DHCP requests from '%s' : %s\n",
                         network->bridge, strerror(err));
        goto err2;
    }

    /* allow DNS requests through to dnsmasq */
    if ((err = iptablesAddTcpInput(server->iptables, network->bridge, 53))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to add iptables rule to allow DNS requests from '%s' : %s\n",
                         network->bridge, strerror(err));
        goto err3;
    }

    if ((err = iptablesAddUdpInput(server->iptables, network->bridge, 53))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to add iptables rule to allow DNS requests from '%s' : %s\n",
                         network->bridge, strerror(err));
        goto err4;
    }


    /* Catch all rules to block forwarding to/from bridges */

    if ((err = iptablesAddForwardRejectOut(server->iptables, network->bridge))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to add iptables rule to block outbound traffic from '%s' : %s\n",
                         network->bridge, strerror(err));
        goto err5;
    }

    if ((err = iptablesAddForwardRejectIn(server->iptables, network->bridge))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to add iptables rule to block inbound traffic to '%s' : %s\n",
                         network->bridge, strerror(err));
        goto err6;
    }

    /* Allow traffic between guests on the same bridge */
    if ((err = iptablesAddForwardAllowCross(server->iptables, network->bridge))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to add iptables rule to allow cross bridge traffic on '%s' : %s\n",
                         network->bridge, strerror(err));
        goto err7;
    }


    /* The remaining rules are only needed for IP forwarding */
    if (!network->def->forward)
        return 1;

    /* allow forwarding packets from the bridge interface */
    if ((err = iptablesAddForwardAllowOut(server->iptables,
                                          network->def->network,
                                          network->bridge,
                                          network->def->forwardDev))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to add iptables rule to allow forwarding from '%s' : %s\n",
                         network->bridge, strerror(err));
        goto err8;
    }

    /* allow forwarding packets to the bridge interface if they are part of an existing connection */
    if ((err = iptablesAddForwardAllowIn(server->iptables,
                                         network->def->network,
                                         network->bridge,
                                         network->def->forwardDev))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to add iptables rule to allow forwarding to '%s' : %s\n",
                         network->bridge, strerror(err));
        goto err9;
    }

    /* enable masquerading */
    if ((err = iptablesAddForwardMasquerade(server->iptables,
                                            network->def->network,
                                            network->def->forwardDev))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to add iptables rule to enable masquerading : %s\n",
                         strerror(err));
        goto err10;
    }

    return 1;

 err10:
    iptablesRemoveForwardAllowIn(server->iptables,
                                 network->def->network,
                                 network->bridge,
                                 network->def->forwardDev);
 err9:
    iptablesRemoveForwardAllowOut(server->iptables,
                                  network->def->network,
                                  network->bridge,
                                  network->def->forwardDev);
 err8:
    iptablesRemoveForwardAllowCross(server->iptables,
                                    network->bridge);
 err7:
    iptablesRemoveForwardRejectIn(server->iptables,
                                  network->bridge);
 err6:
    iptablesRemoveForwardRejectOut(server->iptables,
                                   network->bridge);
 err5:
    iptablesRemoveUdpInput(server->iptables, network->bridge, 53);
 err4:
    iptablesRemoveTcpInput(server->iptables, network->bridge, 53);
 err3:
    iptablesRemoveUdpInput(server->iptables, network->bridge, 67);
 err2:
    iptablesRemoveTcpInput(server->iptables, network->bridge, 67);
 err1:
    return 0;
}

static void
qemudRemoveIptablesRules(struct qemud_server *server,
                         struct qemud_network *network) {
    if (network->def->forward) {
        iptablesRemoveForwardMasquerade(server->iptables,
                                     network->def->network,
                                     network->def->forwardDev);
        iptablesRemoveForwardAllowIn(server->iptables,
                                   network->def->network,
                                   network->bridge,
                                   network->def->forwardDev);
        iptablesRemoveForwardAllowOut(server->iptables,
                                      network->def->network,
                                      network->bridge,
                                      network->def->forwardDev);
    }
    iptablesRemoveForwardAllowCross(server->iptables, network->bridge);
    iptablesRemoveForwardRejectIn(server->iptables, network->bridge);
    iptablesRemoveForwardRejectOut(server->iptables, network->bridge);
    iptablesRemoveUdpInput(server->iptables, network->bridge, 53);
    iptablesRemoveTcpInput(server->iptables, network->bridge, 53);
    iptablesRemoveUdpInput(server->iptables, network->bridge, 67);
    iptablesRemoveTcpInput(server->iptables, network->bridge, 67);
}

static int
qemudEnableIpForwarding(void)
{
#define PROC_IP_FORWARD "/proc/sys/net/ipv4/ip_forward"

    int fd, ret;

    if ((fd = open(PROC_IP_FORWARD, O_WRONLY|O_TRUNC)) == -1)
        return 0;

    if (write(fd, "1\n", 2) < 0)
        ret = 0;

    close (fd);

    return 1;

#undef PROC_IP_FORWARD
}

int qemudStartNetworkDaemon(struct qemud_server *server,
                            struct qemud_network *network) {
    const char *name;
    int err;

    if (qemudIsActiveNetwork(network)) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "network is already active");
        return -1;
    }

    if (!server->brctl && (err = brInit(&server->brctl))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "cannot initialize bridge support: %s", strerror(err));
        return -1;
    }

    if (network->def->bridge[0] == '\0' ||
        strchr(network->def->bridge, '%')) {
        name = "vnet%d";
    } else {
        name = network->def->bridge;
    }

    if ((err = brAddBridge(server->brctl, name, network->bridge, sizeof(network->bridge)))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "cannot create bridge '%s' : %s", name, strerror(err));
        return -1;
    }

    if (network->def->ipAddress[0] &&
        (err = brSetInetAddress(server->brctl, network->bridge, network->def->ipAddress))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "cannot set IP address on bridge '%s' to '%s' : %s\n",
                         network->bridge, network->def->ipAddress, strerror(err));
        goto err_delbr;
    }

    if (network->def->netmask[0] &&
        (err = brSetInetNetmask(server->brctl, network->bridge, network->def->netmask))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "cannot set netmask on bridge '%s' to '%s' : %s\n",
                         network->bridge, network->def->netmask, strerror(err));
        goto err_delbr;
    }

    if (network->def->ipAddress[0] &&
        (err = brSetInterfaceUp(server->brctl, network->bridge, 1))) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to bring the bridge '%s' up : %s\n",
                         network->bridge, strerror(err));
        goto err_delbr;
    }

    if (!qemudAddIptablesRules(server, network))
        goto err_delbr1;

    if (network->def->forward &&
        !qemudEnableIpForwarding()) {
        qemudReportError(server, VIR_ERR_INTERNAL_ERROR,
                         "failed to enable IP forwarding : %s\n", strerror(err));
        goto err_delbr2;
    }

    if (network->def->ranges &&
        dhcpStartDhcpDaemon(server, network) < 0)
        goto err_delbr2;

    network->active = 1;

    server->ninactivenetworks--;
    server->nactivenetworks++;

    return 0;

 err_delbr2:
    qemudRemoveIptablesRules(server, network);

 err_delbr1:
    if (network->def->ipAddress[0] &&
        (err = brSetInterfaceUp(server->brctl, network->bridge, 0))) {
        qemudLog(QEMUD_WARN, "Failed to bring down bridge '%s' : %s",
                 network->bridge, strerror(err));
    }

 err_delbr:
    if ((err = brDeleteBridge(server->brctl, network->bridge))) {
        qemudLog(QEMUD_WARN, "Failed to delete bridge '%s' : %s\n",
                 network->bridge, strerror(err));
    }

    return -1;
}


int qemudShutdownNetworkDaemon(struct qemud_server *server,
                               struct qemud_network *network) {
    int err;

    qemudLog(QEMUD_INFO, "Shutting down network '%s'", network->def->name);

    if (!qemudIsActiveNetwork(network))
        return 0;

    if (network->dnsmasqPid > 0)
        kill(network->dnsmasqPid, SIGTERM);

    qemudRemoveIptablesRules(server, network);

    if (network->def->ipAddress[0] &&
        (err = brSetInterfaceUp(server->brctl, network->bridge, 0))) {
        qemudLog(QEMUD_WARN, "Failed to bring down bridge '%s' : %s\n",
                 network->bridge, strerror(err));
    }

    if ((err = brDeleteBridge(server->brctl, network->bridge))) {
        qemudLog(QEMUD_WARN, "Failed to delete bridge '%s' : %s\n",
                 network->bridge, strerror(err));
    }

    if (network->dnsmasqPid > 0 &&
        waitpid(network->dnsmasqPid, NULL, WNOHANG) != network->dnsmasqPid) {
        kill(network->dnsmasqPid, SIGKILL);
        if (waitpid(network->dnsmasqPid, NULL, 0) != network->dnsmasqPid)
            qemudLog(QEMUD_WARN, "Got unexpected pid for dnsmasq\n");
    }

    network->bridge[0] = '\0';
    network->dnsmasqPid = -1;
    network->active = 0;

    if (network->newDef) {
        qemudFreeNetworkDef(network->def);
        network->def = network->newDef;
        network->newDef = NULL;
    }

    server->nactivenetworks--;
    server->ninactivenetworks++;

    return 0;
}


static int qemudDispatchPoll(struct qemud_server *server, struct pollfd *fds) {
    struct qemud_socket *sock = server->sockets;
    struct qemud_client *client = server->clients;
    struct qemud_vm *vm;
    struct qemud_network *network;
    int ret = 0;
    int fd = 0;

    if (fds[fd++].revents && qemudDispatchSignal(server) < 0)
        return -1;

    if (server->shutdown)
        return 0;

    vm = server->vms;
    while (vm) {
        struct qemud_vm *next = vm->next;
        int failed = 0,
            stdoutfd = vm->stdout,
            stderrfd = vm->stderr;

        if (!qemudIsActiveVM(vm)) {
            vm = next;
            continue;
        }

        if (stdoutfd != -1) {
            if (fds[fd].revents) {
                if (fds[fd].revents == POLLIN) {
                    if (qemudDispatchVMLog(server, vm, fds[fd].fd) < 0)
                        failed = 1;
                } else {
                    if (qemudDispatchVMFailure(server, vm, fds[fd].fd) < 0)
                        failed = 1;
                }
            }
            fd++;
        }
        if (stderrfd != -1) {
            if (!failed) {
                if (fds[fd].revents) {
                    if (fds[fd].revents == POLLIN) {
                        if (qemudDispatchVMLog(server, vm, fds[fd].fd) < 0)
                            failed = 1;
                    } else {
                        if (qemudDispatchVMFailure(server, vm, fds[fd].fd) < 0)
                            failed = 1;
                    }
                }
            }
            fd++;
        }
        vm = next;
        if (failed)
            ret = -1; /* FIXME: the daemon shouldn't exit on failure here */
    }
    while (client) {
        struct qemud_client *next = client->next;

        assert (client->magic == QEMUD_CLIENT_MAGIC);

        if (fds[fd].revents) {
            qemudDebug("Poll data normal");
            if (fds[fd].revents == POLLOUT)
                qemudDispatchClientWrite(server, client);
            else if (fds[fd].revents == POLLIN)
                qemudDispatchClientRead(server, client);
            else
                qemudDispatchClientFailure(server, client);
        }
        fd++;
        client = next;
    }
    while (sock) {
        struct qemud_socket *next = sock->next;
        /* FIXME: the daemon shouldn't exit on error here */
        if (fds[fd].revents)
            if (qemudDispatchServer(server, sock) < 0)
                return -1;
        fd++;
        sock = next;
    }

    /* Cleanup any VMs which shutdown & dont have an associated
       config file */
    vm = server->vms;
    while (vm) {
        struct qemud_vm *next = vm->next;

        if (!qemudIsActiveVM(vm) && !vm->configFile[0])
            qemudRemoveInactiveVM(server, vm);

        vm = next;
    }

    /* Cleanup any networks too */
    network = server->networks;
    while (network) {
        struct qemud_network *next = network->next;

        if (!qemudIsActiveNetwork(network) && !network->configFile[0])
            qemudRemoveInactiveNetwork(server, network);

        network = next;
    }

    return ret;
}

static void qemudPreparePoll(struct qemud_server *server, struct pollfd *fds) {
    int  fd = 0;
    struct qemud_socket *sock;
    struct qemud_client *client;
    struct qemud_vm *vm;

    fds[fd].fd = server->sigread;
    fds[fd].events = POLLIN;
    fd++;

    for (vm = server->vms ; vm ; vm = vm->next) {
        if (!qemudIsActiveVM(vm))
            continue;
        if (vm->stdout != -1) {
            fds[fd].fd = vm->stdout;
            fds[fd].events = POLLIN | POLLERR | POLLHUP;
            fd++;
        }
        if (vm->stderr != -1) {
            fds[fd].fd = vm->stderr;
            fds[fd].events = POLLIN | POLLERR | POLLHUP;
            fd++;
        }
    }
    for (client = server->clients ; client ; client = client->next) {
        fds[fd].fd = client->fd;
        if (!client->tls) {
            /* Refuse to read more from client if tx is pending to
               rate limit */
            if (client->mode == QEMUD_MODE_TX_PACKET)
                fds[fd].events = POLLOUT | POLLERR | POLLHUP;
            else
                fds[fd].events = POLLIN | POLLERR | POLLHUP;
        } else {
            qemudDebug ("direction = %s",
                        client->direction ? "WRITE" : "READ");
            fds[fd].events = client->direction ? POLLOUT : POLLIN;
            fds[fd].events |= POLLERR | POLLHUP;
        }
        fd++;
    }
    for (sock = server->sockets ; sock ; sock = sock->next) {
        fds[fd].fd = sock->fd;
        fds[fd].events = POLLIN;
        fd++;
    }
}



static int qemudOneLoop(struct qemud_server *server) {
    int nfds = server->nsockets + server->nclients + server->nvmfds + 1; /* server->sigread */
    struct pollfd fds[nfds];
    int thistimeout = -1;
    int ret;
    sig_atomic_t errors;

    /* If we have no clients or vms, then timeout after
       30 seconds, letting daemon exit */
    if (timeout > 0 &&
        !server->nclients &&
        !server->nactivevms)
        thistimeout = timeout;

    qemudPreparePoll(server, fds);

 retry:

    if ((ret = poll(fds, nfds, thistimeout * 1000)) < 0) {
        if (errno == EINTR) {
            goto retry;
        }
        qemudLog(QEMUD_ERR, "Error polling on file descriptors: %s",
                 strerror(errno));
        return -1;
    }

    /* Must have timed out */
    if (ret == 0) {
        qemudLog(QEMUD_INFO, "Timed out while polling on file descriptors");
        return -1;
    }

    /* Check for any signal handling errors and log them. */
    errors = sig_errors;
    if (errors) {
        sig_errors -= errors;
        qemudLog (QEMUD_ERR,
                  "Signal handler reported %d errors: last error: %s",
                  errors, strerror (sig_lasterrno));
        return -1;
    }

    if (qemudDispatchPoll(server, fds) < 0)
        return -1;

    return 0;
}

static int qemudRunLoop(struct qemud_server *server) {
    int ret;

    while ((ret = qemudOneLoop(server)) == 0 && !server->shutdown)
        ;

    return ret == -1 ? -1 : 0;
}

static void qemudCleanup(struct qemud_server *server) {
    struct qemud_socket *sock;

    close(server->sigread);

    sock = server->sockets;
    while (sock) {
        struct qemud_socket *next = sock->next;
        close(sock->fd);
        free(sock);
        sock = next;
    }

    if (server->brctl)
        brShutdown(server->brctl);
    if (server->iptables)
        iptablesContextFree(server->iptables);

    if (server->configDir) free (server->configDir);
    if (server->autostartDir) free (server->autostartDir);
    if (server->networkConfigDir) free (server->networkConfigDir);
    if (server->networkAutostartDir) free (server->networkAutostartDir);

    free(server);
}

/* Read the config file if it exists.
 * Only used in the remote case, hence the name.
 */
static int
remoteReadConfigFile (const char *filename)
{
    virConfPtr conf;

    /* Just check the file is readable before opening it, otherwise
     * libvirt emits an error.
     */
    if (access (filename, R_OK) == -1) return 0;

    conf = virConfReadFile (filename);
    if (!conf) return 0;

    virConfValuePtr p;

#define CHECK_TYPE(name,typ) if (p && p->type != (typ)) {               \
        qemudLog (QEMUD_ERR,                                            \
                  "remoteReadConfigFile: %s: %s: expected type " #typ "\n", \
                  filename, (name));                                    \
        return -1;                                                      \
    }

    p = virConfGetValue (conf, "listen_tls");
    CHECK_TYPE ("listen_tls", VIR_CONF_LONG);
    listen_tls = p ? p->l : listen_tls;

    p = virConfGetValue (conf, "listen_tcp");
    CHECK_TYPE ("listen_tcp", VIR_CONF_LONG);
    listen_tcp = p ? p->l : listen_tcp;

    p = virConfGetValue (conf, "tls_port");
    CHECK_TYPE ("tls_port", VIR_CONF_STRING);
    tls_port = p ? strdup (p->str) : tls_port;

    p = virConfGetValue (conf, "tcp_port");
    CHECK_TYPE ("tcp_port", VIR_CONF_STRING);
    tcp_port = p ? strdup (p->str) : tcp_port;

    p = virConfGetValue (conf, "tls_no_verify_certificate");
    CHECK_TYPE ("tls_no_verify_certificate", VIR_CONF_LONG);
    tls_no_verify_certificate = p ? p->l : tls_no_verify_certificate;

    p = virConfGetValue (conf, "tls_no_verify_address");
    CHECK_TYPE ("tls_no_verify_address", VIR_CONF_LONG);
    tls_no_verify_address = p ? p->l : tls_no_verify_address;

    p = virConfGetValue (conf, "key_file");
    CHECK_TYPE ("key_file", VIR_CONF_STRING);
    key_file = p ? strdup (p->str) : key_file;

    p = virConfGetValue (conf, "cert_file");
    CHECK_TYPE ("cert_file", VIR_CONF_STRING);
    cert_file = p ? strdup (p->str) : cert_file;

    p = virConfGetValue (conf, "ca_file");
    CHECK_TYPE ("ca_file", VIR_CONF_STRING);
    ca_file = p ? strdup (p->str) : ca_file;

    p = virConfGetValue (conf, "crl_file");
    CHECK_TYPE ("crl_file", VIR_CONF_STRING);
    crl_file = p ? strdup (p->str) : crl_file;

    p = virConfGetValue (conf, "tls_allowed_dn_list");
    if (p) {
        switch (p->type) {
        case VIR_CONF_STRING:
            tls_allowed_dn_list = malloc (2 * sizeof (char *));
            tls_allowed_dn_list[0] = strdup (p->str);
            tls_allowed_dn_list[1] = 0;
            break;

        case VIR_CONF_LIST: {
            int i, len = 0;
            virConfValuePtr pp;
            for (pp = p->list; pp; pp = p->next)
                len++;
            tls_allowed_dn_list =
                malloc ((1+len) * sizeof (char *));
            for (i = 0, pp = p->list; pp; ++i, pp = p->next) {
                if (pp->type != VIR_CONF_STRING) {
                    qemudLog (QEMUD_ERR, "remoteReadConfigFile: %s: tls_allowed_dn_list: should be a string or list of strings\n", filename);
                    return -1;
                }
                tls_allowed_dn_list[i] = strdup (pp->str);
            }
            tls_allowed_dn_list[i] = 0;
            break;
        }

        default:
            qemudLog (QEMUD_ERR, "remoteReadConfigFile: %s: tls_allowed_dn_list: should be a string or list of strings\n", filename);
            return -1;
        }
    }

    p = virConfGetValue (conf, "tls_allowed_ip_list");
    if (p) {
        switch (p->type) {
        case VIR_CONF_STRING:
            tls_allowed_ip_list = malloc (2 * sizeof (char *));
            tls_allowed_ip_list[0] = strdup (p->str);
            tls_allowed_ip_list[1] = 0;
            break;

        case VIR_CONF_LIST: {
            int i, len = 0;
            virConfValuePtr pp;
            for (pp = p->list; pp; pp = p->next)
                len++;
            tls_allowed_ip_list =
                malloc ((1+len) * sizeof (char *));
            for (i = 0, pp = p->list; pp; ++i, pp = p->next) {
                if (pp->type != VIR_CONF_STRING) {
                    qemudLog (QEMUD_ERR, "remoteReadConfigFile: %s: tls_allowed_ip_list: should be a string or list of strings\n", filename);
                    return -1;
                }
                tls_allowed_ip_list[i] = strdup (pp->str);
            }
            tls_allowed_ip_list[i] = 0;
            break;
        }

        default:
            qemudLog (QEMUD_ERR, "remoteReadConfigFile: %s: tls_allowed_ip_list: should be a string or list of strings\n", filename);
            return -1;
        }
    }

    virConfFree (conf);
    return 0;
}

/* Print command-line usage. */
static void
usage (const char *argv0)
{
    fprintf (stderr,
             "\n\
Usage:\n\
  %s [options]\n\
\n\
Options:\n\
  -v | --verbose         Verbose messages.\n\
  -d | --daemon          Run as a daemon & write PID file.\n\
  -r | --remote          Act as remote server.\n\
  -s | --system          Run as system daemon (QEMUD only).\n\
  -t | --timeout <secs>  Exit after timeout period (QEMUD only).\n\
  -f | --config <file>   Configuration file (remote only).\n\
  -p | --pid-file <file> Change name of PID file.\n\
\n\
Remote and QEMU/network management:\n\
\n\
The '--remote' flag selects between running as a remote server\n\
for remote libvirt requests, versus running as a QEMU\n\
and network management daemon.\n\
\n\
Normally you need to have one daemon of each type.\n\
\n\
See also http://libvirt.org/remote.html\n\
\n\
For remote daemon:\n\
\n\
  Default paths:\n\
\n\
    Configuration file (unless overridden by -f):\n\
      " SYSCONF_DIR "/libvirt/libvirtd.conf\n\
\n\
    Sockets:\n\
      " LOCAL_STATE_DIR "/run/libvirt/libvirt-sock\n\
      " LOCAL_STATE_DIR "/run/libvirt/libvirt-sock-ro\n\
\n\
    TLS:\n\
      CA certificate:     " LIBVIRT_CACERT "\n\
      Server certificate: " LIBVIRT_SERVERCERT "\n\
      Server private key: " LIBVIRT_SERVERKEY "\n\
\n\
    PID file (unless overridden by --pid-file):\n\
      %s\n\
\n\
For QEMU and network management daemon:\n\
\n\
  For '--system' option you must be running this daemon as root.\n\
\n\
  The '--timeout' applies only when the daemon is not servicing\n\
  clients.\n\
\n\
  Default paths:\n\
\n\
    Configuration files (in system mode):\n\
      " SYSCONF_DIR "/libvirt/qemu\n\
      " SYSCONF_DIR "/libvirt/qemu/autostart\n\
      " SYSCONF_DIR "/libvirt/qemu/networkd\n\
      " SYSCONF_DIR "/libvirt/qemu/networks/autostart\n\
\n\
    Configuration files (not in system mode):\n\
      $HOME/.libvirt/qemu\n\
      $HOME/.libvirt/qemu/autostart\n\
      $HOME/.libvirt/qemu/networks\n\
      $HOME/.libvirt/qemu/networks/autostart\n\
\n\
    Sockets (in system mode):\n\
      " LOCAL_STATE_DIR "/run/libvirt/qemud-sock\n\
      " LOCAL_STATE_DIR "/run/libvirt/qemud-sock-ro\n\
\n\
    Sockets (not in system mode):\n\
      $HOME/.libvirt/qemud-sock (in Unix abstract namespace)\n\
\n\
    PID file (unless overridden by --pid-file):\n\
      %s\n\
\n",
             argv0,
             REMOTE_PID_FILE[0] != '\0'
               ? REMOTE_PID_FILE
               : "(disabled in ./configure)",
             QEMUD_PID_FILE[0] != '\0'
               ? QEMUD_PID_FILE
               : "(disabled in ./configure)");
}

#define MAX_LISTEN 5
int main(int argc, char **argv) {
    struct qemud_server *server;
    struct sigaction sig_action;
    int sigpipe[2];
    const char *pid_file = NULL;
    const char *remote_config_file = SYSCONF_DIR "/libvirt/libvirtd.conf";
    int ret = 1;

    struct option opts[] = {
        { "verbose", no_argument, &verbose, 1},
        { "daemon", no_argument, &godaemon, 1},
        { "remote", no_argument, &remote, 1},
        { "config", required_argument, NULL, 'f'},
        { "system", no_argument, &sys, 1},
        { "timeout", required_argument, NULL, 't'},
        { "pid-file", required_argument, NULL, 'p'},
        { "help", no_argument, NULL, '?' },
        {0, 0, 0, 0}
    };

    while (1) {
        int optidx = 0;
        int c;
        char *tmp;

        c = getopt_long(argc, argv, "dfp:s:t:v", opts, &optidx);

        if (c == -1) {
            break;
        }

        switch (c) {
        case 0:
            /* Got one of the flags */
            break;
        case 'v':
            verbose = 1;
            break;
        case 'd':
            godaemon = 1;
            break;
        case 'r':
            remote = 1;
            break;
        case 's':
            sys = 1;
            break;

        case 't':
            timeout = strtol(optarg, &tmp, 10);
            if (!tmp)
                timeout = -1;
            if (timeout <= 0)
                timeout = -1;
            break;

        case 'p':
            pid_file = optarg;
            break;

        case 'f':
            remote_config_file = optarg;
            break;

        case '?':
            usage (argv[0]);
            return 2;

        default:
            abort();
        }
    }

    /* In remote mode only, now read the config file (if it exists). */
    if (remote) {
        if (remoteReadConfigFile (remote_config_file) < 0)
            goto error1;
    }

    if (godaemon)
        openlog("libvirt-qemud", 0, 0);

    if (pipe(sigpipe) < 0 ||
        qemudSetNonBlock(sigpipe[0]) < 0 ||
        qemudSetNonBlock(sigpipe[1]) < 0) {
        qemudLog(QEMUD_ERR, "Failed to create pipe: %s",
                 strerror(errno));
        goto error1;
    }

    sigwrite = sigpipe[1];

    sig_action.sa_handler = sig_handler;
    sig_action.sa_flags = 0;
    sigemptyset(&sig_action.sa_mask);

    sigaction(SIGHUP, &sig_action, NULL);
    sigaction(SIGINT, &sig_action, NULL);
    sigaction(SIGQUIT, &sig_action, NULL);
    sigaction(SIGTERM, &sig_action, NULL);
    sigaction(SIGCHLD, &sig_action, NULL);

    sig_action.sa_handler = SIG_IGN;
    sigaction(SIGPIPE, &sig_action, NULL);

    if (godaemon) {
        int pid = qemudGoDaemon();
        if (pid < 0) {
            qemudLog(QEMUD_ERR, "Failed to fork as daemon: %s",
                     strerror(errno));
            goto error1;
        }
        if (pid > 0)
            goto out;

        /* Choose the name of the PID file. */
        if (!pid_file) {
            if (remote) {
                if (REMOTE_PID_FILE[0] != '\0')
                    pid_file = REMOTE_PID_FILE;
            } else {
                if (QEMUD_PID_FILE[0] != '\0')
                    pid_file = QEMUD_PID_FILE;
            }
        }

        if (pid_file && qemudWritePidFile (pid_file) < 0)
            goto error1;
    }

    if (!(server = qemudInitialize(sigpipe[0]))) {
        ret = 2;
        goto error2;
    }

    qemudRunLoop(server);

    qemudCleanup(server);

    close(sigwrite);

    if (godaemon)
        closelog();

 out:
    ret = 0;

 error2:
    if (godaemon && pid_file)
        unlink (pid_file);

 error1:
    return ret;
}

/*
 * Local variables:
 *  indent-tabs-mode: nil
 *  c-indent-level: 4
 *  c-basic-offset: 4
 *  tab-width: 4
 * End:
 */