libvirt/src/util/command.c

/*
 * command.c: Child command execution
 *
 * Copyright (C) 2010-2011 Red Hat, Inc.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
 *
 */

#include <config.h>

#include <poll.h>
#include <signal.h>
#include <stdarg.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <fcntl.h>

#if HAVE_CAPNG
# include <cap-ng.h>
#endif

#include "command.h"
#include "memory.h"
#include "virterror_internal.h"
#include "util.h"
#include "logging.h"
#include "files.h"
#include "buf.h"
#include "ignore-value.h"
#include "verify.h"

#define VIR_FROM_THIS VIR_FROM_NONE

#define virCommandError(code, ...)                                      \
    virReportErrorHelper(VIR_FROM_NONE, code, __FILE__,                 \
                         __FUNCTION__, __LINE__, __VA_ARGS__)

/* We have quite a bit of changes to make if this doesn't hold.  */
verify(sizeof(pid_t) <= sizeof(int));

/* Flags for virExecWithHook */
enum {
    VIR_EXEC_NONE   = 0,
    VIR_EXEC_NONBLOCK = (1 << 0),
    VIR_EXEC_DAEMON = (1 << 1),
    VIR_EXEC_CLEAR_CAPS = (1 << 2),
    VIR_EXEC_RUN_SYNC = (1 << 3),
};

struct _virCommand {
    int has_error; /* ENOMEM on allocation failure, -1 for anything else.  */

    char **args;
    size_t nargs;
    size_t maxargs;

    char **env;
    size_t nenv;
    size_t maxenv;

    char *pwd;

    /* XXX Use int[] if we ever need to support more than FD_SETSIZE fd's.  */
    fd_set preserve; /* FDs to pass to child. */
    fd_set transfer; /* FDs to close in parent. */

    unsigned int flags;

    char *inbuf;
    char **outbuf;
    char **errbuf;

    int infd;
    int inpipe;
    int outfd;
    int errfd;
    int *outfdptr;
    int *errfdptr;

    bool handshake;
    int handshakeWait[2];
    int handshakeNotify[2];

    virExecHook hook;
    void *opaque;

    pid_t pid;
    char *pidfile;
    bool reap;
};

#ifndef WIN32

# if HAVE_CAPNG
static int virClearCapabilities(void)
{
    int ret;

    capng_clear(CAPNG_SELECT_BOTH);

    if ((ret = capng_apply(CAPNG_SELECT_BOTH)) < 0) {
        virCommandError(VIR_ERR_INTERNAL_ERROR,
                        _("cannot clear process capabilities %d"), ret);
        return -1;
    }

    return 0;
}
# else
static int virClearCapabilities(void)
{
//    VIR_WARN("libcap-ng support not compiled in, unable to clear "
//             "capabilities");
    return 0;
}
# endif


/* virFork() - fork a new process while avoiding various race/deadlock
               conditions

   @pid - a pointer to a pid_t that will receive the return value from
          fork()

   on return from virFork(), if *pid < 0, the fork failed and there is
   no new process. Otherwise, just like fork(), if *pid == 0, it is the
   child process returning, and if *pid > 0, it is the parent.

   Even if *pid >= 0, if the return value from virFork() is < 0, it
   indicates a failure that occurred in the parent or child process
   after the fork. In this case, the child process should call
   _exit(EXIT_FAILURE) after doing any additional error reporting.

 */
int virFork(pid_t *pid) {
    sigset_t oldmask, newmask;
    struct sigaction sig_action;
    int saved_errno, ret = -1;

    *pid = -1;

    /*
     * Need to block signals now, so that child process can safely
     * kill off caller's signal handlers without a race.
     */
    sigfillset(&newmask);
    if (pthread_sigmask(SIG_SETMASK, &newmask, &oldmask) != 0) {
        saved_errno = errno;
        virReportSystemError(errno,
                             "%s", _("cannot block signals"));
        goto cleanup;
    }

    /* Ensure we hold the logging lock, to protect child processes
     * from deadlocking on another thread's inherited mutex state */
    virLogLock();

    *pid = fork();
    saved_errno = errno; /* save for caller */

    /* Unlock for both parent and child process */
    virLogUnlock();

    if (*pid < 0) {
        /* attempt to restore signal mask, but ignore failure, to
           avoid obscuring the fork failure */
        ignore_value (pthread_sigmask(SIG_SETMASK, &oldmask, NULL));
        virReportSystemError(saved_errno,
                             "%s", _("cannot fork child process"));
        goto cleanup;
    }

    if (*pid) {

        /* parent process */

        /* Restore our original signal mask now that the child is
           safely running */
        if (pthread_sigmask(SIG_SETMASK, &oldmask, NULL) != 0) {
            saved_errno = errno; /* save for caller */
            virReportSystemError(errno, "%s", _("cannot unblock signals"));
            goto cleanup;
        }
        ret = 0;

    } else {

        /* child process */

        int logprio;
        int i;

        /* Remove any error callback so errors in child now
           get sent to stderr where they stand a fighting chance
           of being seen / logged */
        virSetErrorFunc(NULL, NULL);
        virSetErrorLogPriorityFunc(NULL);

        /* Make sure any hook logging is sent to stderr, since child
         * process may close the logfile FDs */
        logprio = virLogGetDefaultPriority();
        virLogReset();
        virLogSetDefaultPriority(logprio);

        /* Clear out all signal handlers from parent so nothing
           unexpected can happen in our child once we unblock
           signals */
        sig_action.sa_handler = SIG_DFL;
        sig_action.sa_flags = 0;
        sigemptyset(&sig_action.sa_mask);

        for (i = 1; i < NSIG; i++) {
            /* Only possible errors are EFAULT or EINVAL
               The former wont happen, the latter we
               expect, so no need to check return value */

            sigaction(i, &sig_action, NULL);
        }

        /* Unmask all signals in child, since we've no idea
           what the caller's done with their signal mask
           and don't want to propagate that to children */
        sigemptyset(&newmask);
        if (pthread_sigmask(SIG_SETMASK, &newmask, NULL) != 0) {
            saved_errno = errno; /* save for caller */
            virReportSystemError(errno, "%s", _("cannot unblock signals"));
            goto cleanup;
        }
        ret = 0;
    }

cleanup:
    if (ret < 0)
        errno = saved_errno;
    return ret;
}

/*
 * Ensure that *null is an fd visiting /dev/null.  Return 0 on
 * success, -1 on failure.  Allows for lazy opening of shared
 * /dev/null fd only as required.
 */
static int
getDevNull(int *null)
{
    if (*null == -1 && (*null = open("/dev/null", O_RDWR)) < 0) {
        virReportSystemError(errno,
                             _("cannot open %s"),
                             "/dev/null");
        return -1;
    }
    return 0;
}

/*
 * @argv argv to exec
 * @envp optional environment to use for exec
 * @keepfd options fd_ret to keep open for child process
 * @retpid optional pointer to store child process pid
 * @infd optional file descriptor to use as child input, otherwise /dev/null
 * @outfd optional pointer to communicate output fd behavior
 *        outfd == NULL : Use /dev/null
 *        *outfd == -1  : Use a new fd
 *        *outfd != -1  : Use *outfd
 * @errfd optional pointer to communcate error fd behavior. See outfd
 * @flags possible combination of the following:
 *        VIR_EXEC_NONE     : Default function behavior
 *        VIR_EXEC_NONBLOCK : Set child process output fd's as non-blocking
 *        VIR_EXEC_DAEMON   : Daemonize the child process
 * @hook optional virExecHook function to call prior to exec
 * @data data to pass to the hook function
 * @pidfile path to use as pidfile for daemonized process (needs DAEMON flag)
 */
static int
virExecWithHook(const char *const*argv,
                const char *const*envp,
                const fd_set *keepfd,
                pid_t *retpid,
                int infd, int *outfd, int *errfd,
                unsigned int flags,
                virExecHook hook,
                void *data,
                char *pidfile)
{
    pid_t pid;
    int null = -1, i, openmax;
    int pipeout[2] = {-1,-1};
    int pipeerr[2] = {-1,-1};
    int childout = -1;
    int childerr = -1;
    int tmpfd;
    const char *binary = NULL;
    int forkRet;

    if (argv[0][0] != '/') {
        if (!(binary = virFindFileInPath(argv[0]))) {
            virReportSystemError(ENOENT,
                                 _("Cannot find '%s' in path"),
                                 argv[0]);
            return -1;
        }
    } else {
        binary = argv[0];
    }

    if (infd < 0) {
        if (getDevNull(&null) < 0)
            goto cleanup;
        infd = null;
    }

    if (outfd != NULL) {
        if (*outfd == -1) {
            if (pipe(pipeout) < 0) {
                virReportSystemError(errno,
                                     "%s", _("cannot create pipe"));
                goto cleanup;
            }

            if ((flags & VIR_EXEC_NONBLOCK) &&
                virSetNonBlock(pipeout[0]) == -1) {
                virReportSystemError(errno,
                                     "%s", _("Failed to set non-blocking file descriptor flag"));
                goto cleanup;
            }

            if (virSetCloseExec(pipeout[0]) == -1) {
                virReportSystemError(errno,
                                     "%s", _("Failed to set close-on-exec file descriptor flag"));
                goto cleanup;
            }

            childout = pipeout[1];
        } else {
            childout = *outfd;
        }
    } else {
        if (getDevNull(&null) < 0)
            goto cleanup;
        childout = null;
    }

    if (errfd != NULL) {
        if (*errfd == -1) {
            if (pipe(pipeerr) < 0) {
                virReportSystemError(errno,
                                     "%s", _("Failed to create pipe"));
                goto cleanup;
            }

            if ((flags & VIR_EXEC_NONBLOCK) &&
                virSetNonBlock(pipeerr[0]) == -1) {
                virReportSystemError(errno,
                                     "%s", _("Failed to set non-blocking file descriptor flag"));
                goto cleanup;
            }

            if (virSetCloseExec(pipeerr[0]) == -1) {
                virReportSystemError(errno,
                                     "%s", _("Failed to set close-on-exec file descriptor flag"));
                goto cleanup;
            }

            childerr = pipeerr[1];
        } else {
            childerr = *errfd;
        }
    } else {
        if (getDevNull(&null) < 0)
            goto cleanup;
        childerr = null;
    }

    forkRet = virFork(&pid);

    if (pid < 0) {
        goto cleanup;
    }

    if (pid) { /* parent */
        VIR_FORCE_CLOSE(null);
        if (outfd && *outfd == -1) {
            VIR_FORCE_CLOSE(pipeout[1]);
            *outfd = pipeout[0];
        }
        if (errfd && *errfd == -1) {
            VIR_FORCE_CLOSE(pipeerr[1]);
            *errfd = pipeerr[0];
        }

        if (forkRet < 0) {
            goto cleanup;
        }

        *retpid = pid;

        if (binary != argv[0])
            VIR_FREE(binary);

        return 0;
    }

    /* child */

    if (forkRet < 0) {
        /* The fork was successful, but after that there was an error
         * in the child (which was already logged).
        */
        goto fork_error;
    }

    openmax = sysconf(_SC_OPEN_MAX);
    for (i = 3; i < openmax; i++)
        if (i != infd &&
            i != childout &&
            i != childerr &&
            (!keepfd || i >= FD_SETSIZE || !FD_ISSET(i, keepfd))) {
            tmpfd = i;
            VIR_FORCE_CLOSE(tmpfd);
        }

    if (dup2(infd, STDIN_FILENO) < 0) {
        virReportSystemError(errno,
                             "%s", _("failed to setup stdin file handle"));
        goto fork_error;
    }
    if (childout > 0 &&
        dup2(childout, STDOUT_FILENO) < 0) {
        virReportSystemError(errno,
                             "%s", _("failed to setup stdout file handle"));
        goto fork_error;
    }
    if (childerr > 0 &&
        dup2(childerr, STDERR_FILENO) < 0) {
        virReportSystemError(errno,
                             "%s", _("failed to setup stderr file handle"));
        goto fork_error;
    }

    if (infd != STDIN_FILENO && infd != null)
        VIR_FORCE_CLOSE(infd);
    if (childout > STDERR_FILENO && childout != null) {
        tmpfd = childout;   /* preserve childout value */
        VIR_FORCE_CLOSE(tmpfd);
    }
    if (childerr > STDERR_FILENO &&
        childerr != childout &&
        childerr != null) {
        VIR_FORCE_CLOSE(childerr);
    }
    VIR_FORCE_CLOSE(null);

    /* Initialize full logging for a while */
    virLogSetFromEnv();

    /* Daemonize as late as possible, so the parent process can detect
     * the above errors with wait* */
    if (flags & VIR_EXEC_DAEMON) {
        if (setsid() < 0) {
            virReportSystemError(errno,
                                 "%s", _("cannot become session leader"));
            goto fork_error;
        }

        if (chdir("/") < 0) {
            virReportSystemError(errno,
                                 "%s", _("cannot change to root directory"));
            goto fork_error;
        }

        pid = fork();
        if (pid < 0) {
            virReportSystemError(errno,
                                 "%s", _("cannot fork child process"));
            goto fork_error;
        }

        if (pid > 0) {
            if (pidfile && virFileWritePidPath(pidfile,pid)) {
                kill(pid, SIGTERM);
                usleep(500*1000);
                kill(pid, SIGTERM);
                virReportSystemError(errno,
                                     _("could not write pidfile %s for %d"),
                                     pidfile, pid);
                goto fork_error;
            }
            _exit(0);
        }
    }

    if (hook) {
        /* virFork reset all signal handlers to the defaults.
         * This is good for the child process, but our hook
         * risks running something that generates SIGPIPE,
         * so we need to temporarily block that again
         */
        struct sigaction waxon, waxoff;
        memset(&waxoff, 0, sizeof(waxoff));
        waxoff.sa_handler = SIG_IGN;
        sigemptyset(&waxoff.sa_mask);
        memset(&waxon, 0, sizeof(waxon));
        if (sigaction(SIGPIPE, &waxoff, &waxon) < 0) {
            virReportSystemError(errno, "%s",
                                 _("Could not disable SIGPIPE"));
            goto fork_error;
        }

        if ((hook)(data) != 0) {
            VIR_DEBUG("Hook function failed.");
            goto fork_error;
        }

        if (sigaction(SIGPIPE, &waxon, NULL) < 0) {
            virReportSystemError(errno, "%s",
                                 _("Could not re-enable SIGPIPE"));
            goto fork_error;
        }
    }

    /* The steps above may need todo something privileged, so
     * we delay clearing capabilities until the last minute */
    if ((flags & VIR_EXEC_CLEAR_CAPS) &&
        virClearCapabilities() < 0)
        goto fork_error;

    /* Close logging again to ensure no FDs leak to child */
    virLogReset();

    if (envp)
        execve(binary, (char **) argv, (char**)envp);
    else
        execv(binary, (char **) argv);

    virReportSystemError(errno,
                         _("cannot execute binary %s"),
                         argv[0]);

 fork_error:
    virDispatchError(NULL);
    _exit(EXIT_FAILURE);

 cleanup:
    /* This is cleanup of parent process only - child
       should never jump here on error */

    if (binary != argv[0])
        VIR_FREE(binary);

    /* NB we don't virCommandError() on any failures here
       because the code which jumped hre already raised
       an error condition which we must not overwrite */
    VIR_FORCE_CLOSE(pipeerr[0]);
    VIR_FORCE_CLOSE(pipeerr[1]);
    VIR_FORCE_CLOSE(pipeout[0]);
    VIR_FORCE_CLOSE(pipeout[1]);
    VIR_FORCE_CLOSE(null);
    return -1;
}

/**
 * @argv NULL terminated argv to run
 * @status optional variable to return exit status in
 *
 * Run a command without using the shell.
 *
 * If status is NULL, then return 0 if the command run and
 * exited with 0 status; Otherwise return -1
 *
 * If status is not-NULL, then return 0 if the command ran.
 * The status variable is filled with the command exit status
 * and should be checked by caller for success. Return -1
 * only if the command could not be run.
 */
int
virRun(const char *const*argv, int *status)
{
    int ret;
    virCommandPtr cmd = virCommandNewArgs(argv);

    ret = virCommandRun(cmd, status);
    virCommandFree(cmd);
    return ret;
}

#else /* WIN32 */

int
virRun(const char *const *argv ATTRIBUTE_UNUSED,
       int *status)
{
    if (status)
        *status = ENOTSUP;
    else
        virCommandError(VIR_ERR_INTERNAL_ERROR,
                        "%s", _("virRun is not implemented for WIN32"));
    return -1;
}

static int
virExecWithHook(const char *const*argv ATTRIBUTE_UNUSED,
                const char *const*envp ATTRIBUTE_UNUSED,
                const fd_set *keepfd ATTRIBUTE_UNUSED,
                pid_t *retpid ATTRIBUTE_UNUSED,
                int infd ATTRIBUTE_UNUSED,
                int *outfd ATTRIBUTE_UNUSED,
                int *errfd ATTRIBUTE_UNUSED,
                int flags_unused ATTRIBUTE_UNUSED,
                virExecHook hook ATTRIBUTE_UNUSED,
                void *data ATTRIBUTE_UNUSED,
                char *pidfile ATTRIBUTE_UNUSED)
{
    /* XXX: Some day we can implement pieces of virCommand/virExec on
     * top of _spawn() or CreateProcess(), but we can't implement
     * everything, since mingw completely lacks fork(), so we cannot
     * run hook code in the child.  */
    virCommandError(VIR_ERR_INTERNAL_ERROR,
                    "%s", _("virExec is not implemented for WIN32"));
    return -1;
}

int
virFork(pid_t *pid)
{
    *pid = -1;
    errno = ENOTSUP;

    return -1;
}

#endif /* WIN32 */


/*
 * Create a new command for named binary
 */
virCommandPtr
virCommandNew(const char *binary)
{
    const char *const args[] = { binary, NULL };

    return virCommandNewArgs(args);
}

/*
 * Create a new command with a NULL terminated
 * set of args, taking binary from args[0]
 */
virCommandPtr
virCommandNewArgs(const char *const*args)
{
    virCommandPtr cmd;

    if (VIR_ALLOC(cmd) < 0)
        return NULL;

    cmd->handshakeWait[0] = -1;
    cmd->handshakeWait[1] = -1;
    cmd->handshakeNotify[0] = -1;
    cmd->handshakeNotify[1] = -1;

    FD_ZERO(&cmd->preserve);
    FD_ZERO(&cmd->transfer);
    cmd->infd = cmd->outfd = cmd->errfd = -1;
    cmd->inpipe = -1;
    cmd->pid = -1;

    virCommandAddArgSet(cmd, args);

    return cmd;
}

/*
 * Create a new command with a NULL terminated
 * list of args, starting with the binary to run
 */
virCommandPtr
virCommandNewArgList(const char *binary, ...)
{
    virCommandPtr cmd = virCommandNew(binary);
    va_list list;
    const char *arg;

    if (!cmd || cmd->has_error)
        return cmd;

    va_start(list, binary);
    while ((arg = va_arg(list, const char *)) != NULL)
        virCommandAddArg(cmd, arg);
    va_end(list);
    return cmd;
}


/*
 * Preserve the specified file descriptor in the child, instead of
 * closing it.  FD must not be one of the three standard streams.  If
 * transfer is true, then fd will be closed in the parent after a call
 * to Run/RunAsync/Free, otherwise caller is still responsible for fd.
 */
static void
virCommandKeepFD(virCommandPtr cmd, int fd, bool transfer)
{
    if (!cmd)
        return;

    if (fd <= STDERR_FILENO || FD_SETSIZE <= fd) {
        if (!cmd->has_error)
            cmd->has_error = -1;
        VIR_DEBUG("cannot preserve %d", fd);
        return;
    }

    FD_SET(fd, &cmd->preserve);
    if (transfer)
        FD_SET(fd, &cmd->transfer);
}

/*
 * Preserve the specified file descriptor
 * in the child, instead of closing it.
 * The parent is still responsible for managing fd.
 */
void
virCommandPreserveFD(virCommandPtr cmd, int fd)
{
    return virCommandKeepFD(cmd, fd, false);
}

/*
 * Transfer the specified file descriptor
 * to the child, instead of closing it.
 * Close the fd in the parent during Run/RunAsync/Free.
 */
void
virCommandTransferFD(virCommandPtr cmd, int fd)
{
    return virCommandKeepFD(cmd, fd, true);
}


/*
 * Save the child PID in a pidfile
 */
void
virCommandSetPidFile(virCommandPtr cmd, const char *pidfile)
{
    if (!cmd || cmd->has_error)
        return;

    VIR_FREE(cmd->pidfile);
    if (!(cmd->pidfile = strdup(pidfile))) {
        cmd->has_error = ENOMEM;
    }
}


/*
 * Remove all capabilities from the child
 */
void
virCommandClearCaps(virCommandPtr cmd)
{
    if (!cmd || cmd->has_error)
        return;

    cmd->flags |= VIR_EXEC_CLEAR_CAPS;
}

#if 0 /* XXX Enable if we have a need for capability management.  */

/*
 * Re-allow a specific capability
 */
void
virCommandAllowCap(virCommandPtr cmd,
                   int capability ATTRIBUTE_UNUSED)
{
    if (!cmd || cmd->has_error)
        return;

    /* XXX ? */
}

#endif /* 0 */


/*
 * Daemonize the child process
 */
void
virCommandDaemonize(virCommandPtr cmd)
{
    if (!cmd || cmd->has_error)
        return;

    cmd->flags |= VIR_EXEC_DAEMON;
}

/*
 * Set FDs created by virCommandSetOutputFD and virCommandSetErrorFD
 * as non-blocking in the parent.
 */
void
virCommandNonblockingFDs(virCommandPtr cmd)
{
    if (!cmd || cmd->has_error)
        return;

    cmd->flags |= VIR_EXEC_NONBLOCK;
}

/*
 * Add an environment variable to the child created by a printf-style format
 */
void
virCommandAddEnvFormat(virCommandPtr cmd, const char *format, ...)
{
    char *env;
    va_list list;

    if (!cmd || cmd->has_error)
        return;

    va_start(list, format);
    if (virVasprintf(&env, format, list) < 0) {
        cmd->has_error = ENOMEM;
        va_end(list);
        return;
    }
    va_end(list);

    /* Arg plus trailing NULL. */
    if (VIR_RESIZE_N(cmd->env, cmd->maxenv, cmd->nenv, 1 + 1) < 0) {
        VIR_FREE(env);
        cmd->has_error = ENOMEM;
        return;
    }

    cmd->env[cmd->nenv++] = env;
}

/*
 * Add an environment variable to the child
 * using separate name & value strings
 */
void
virCommandAddEnvPair(virCommandPtr cmd, const char *name, const char *value)
{
    virCommandAddEnvFormat(cmd, "%s=%s", name, value);
}


/*
 * Add an environment variable to the child
 * using a preformatted env string FOO=BAR
 */
void
virCommandAddEnvString(virCommandPtr cmd, const char *str)
{
    char *env;

    if (!cmd || cmd->has_error)
        return;

    if (!(env = strdup(str))) {
        cmd->has_error = ENOMEM;
        return;
    }

    /* env plus trailing NULL */
    if (VIR_RESIZE_N(cmd->env, cmd->maxenv, cmd->nenv, 1 + 1) < 0) {
        VIR_FREE(env);
        cmd->has_error = ENOMEM;
        return;
    }

    cmd->env[cmd->nenv++] = env;
}


/*
 * Convert a buffer containing preformatted name=value into an
 * environment variable of the child.
 * Correctly transfers memory errors or contents from buf to cmd.
 */
void
virCommandAddEnvBuffer(virCommandPtr cmd, virBufferPtr buf)
{
    if (!cmd || cmd->has_error) {
        virBufferFreeAndReset(buf);
        return;
    }

    /* env plus trailing NULL. */
    if (virBufferError(buf) ||
        VIR_RESIZE_N(cmd->env, cmd->maxenv, cmd->nenv, 1 + 1) < 0) {
        cmd->has_error = ENOMEM;
        virBufferFreeAndReset(buf);
        return;
    }

    cmd->env[cmd->nenv++] = virBufferContentAndReset(buf);
}


/*
 * Pass an environment variable to the child
 * using current process' value
 */
void
virCommandAddEnvPass(virCommandPtr cmd, const char *name)
{
    char *value;
    if (!cmd || cmd->has_error)
        return;

    value = getenv(name);
    if (value)
        virCommandAddEnvPair(cmd, name, value);
}


/*
 * Set LC_ALL to C, and propagate other essential environment
 * variables from the parent process.
 */
void
virCommandAddEnvPassCommon(virCommandPtr cmd)
{
    if (!cmd || cmd->has_error)
        return;

    /* Attempt to Pre-allocate; allocation failure will be detected
     * later during virCommandAdd*.  */
    ignore_value(VIR_RESIZE_N(cmd->env, cmd->maxenv, cmd->nenv, 9));

    virCommandAddEnvPair(cmd, "LC_ALL", "C");

    virCommandAddEnvPass(cmd, "LD_PRELOAD");
    virCommandAddEnvPass(cmd, "LD_LIBRARY_PATH");
    virCommandAddEnvPass(cmd, "PATH");
    virCommandAddEnvPass(cmd, "HOME");
    virCommandAddEnvPass(cmd, "USER");
    virCommandAddEnvPass(cmd, "LOGNAME");
    virCommandAddEnvPass(cmd, "TMPDIR");
}

/*
 * Add a command line argument to the child
 */
void
virCommandAddArg(virCommandPtr cmd, const char *val)
{
    char *arg;

    if (!cmd || cmd->has_error)
        return;

    if (!(arg = strdup(val))) {
        cmd->has_error = ENOMEM;
        return;
    }

    /* Arg plus trailing NULL. */
    if (VIR_RESIZE_N(cmd->args, cmd->maxargs, cmd->nargs, 1 + 1) < 0) {
        VIR_FREE(arg);
        cmd->has_error = ENOMEM;
        return;
    }

    cmd->args[cmd->nargs++] = arg;
}


/*
 * Convert a buffer into a command line argument to the child.
 * Correctly transfers memory errors or contents from buf to cmd.
 */
void
virCommandAddArgBuffer(virCommandPtr cmd, virBufferPtr buf)
{
    if (!cmd || cmd->has_error) {
        virBufferFreeAndReset(buf);
        return;
    }

    /* Arg plus trailing NULL. */
    if (virBufferError(buf) ||
        VIR_RESIZE_N(cmd->args, cmd->maxargs, cmd->nargs, 1 + 1) < 0) {
        cmd->has_error = ENOMEM;
        virBufferFreeAndReset(buf);
        return;
    }

    cmd->args[cmd->nargs++] = virBufferContentAndReset(buf);
}


/*
 * Add a command line argument created by a printf-style format
 */
void
virCommandAddArgFormat(virCommandPtr cmd, const char *format, ...)
{
    char *arg;
    va_list list;

    if (!cmd || cmd->has_error)
        return;

    va_start(list, format);
    if (virVasprintf(&arg, format, list) < 0) {
        cmd->has_error = ENOMEM;
        va_end(list);
        return;
    }
    va_end(list);

    /* Arg plus trailing NULL. */
    if (VIR_RESIZE_N(cmd->args, cmd->maxargs, cmd->nargs, 1 + 1) < 0) {
        VIR_FREE(arg);
        cmd->has_error = ENOMEM;
        return;
    }

    cmd->args[cmd->nargs++] = arg;
}

/*
 * Add "NAME=VAL" as a single command line argument to the child
 */
void
virCommandAddArgPair(virCommandPtr cmd, const char *name, const char *val)
{
    virCommandAddArgFormat(cmd, "%s=%s", name, val);
}

/*
 * Add a NULL terminated list of args
 */
void
virCommandAddArgSet(virCommandPtr cmd, const char *const*vals)
{
    int narg = 0;

    if (!cmd || cmd->has_error)
        return;

    if (vals[0] == NULL) {
        cmd->has_error = EINVAL;
        return;
    }

    while (vals[narg] != NULL)
        narg++;

    /* narg plus trailing NULL. */
    if (VIR_RESIZE_N(cmd->args, cmd->maxargs, cmd->nargs, narg + 1) < 0) {
        cmd->has_error = ENOMEM;
        return;
    }

    narg = 0;
    while (vals[narg] != NULL) {
        char *arg = strdup(vals[narg++]);
        if (!arg) {
            cmd->has_error = ENOMEM;
            return;
        }
        cmd->args[cmd->nargs++] = arg;
    }
}

/*
 * Add a NULL terminated list of args
 */
void
virCommandAddArgList(virCommandPtr cmd, ...)
{
    va_list list;
    int narg = 0;

    if (!cmd || cmd->has_error)
        return;

    va_start(list, cmd);
    while (va_arg(list, const char *) != NULL)
        narg++;
    va_end(list);

    /* narg plus trailing NULL. */
    if (VIR_RESIZE_N(cmd->args, cmd->maxargs, cmd->nargs, narg + 1) < 0) {
        cmd->has_error = ENOMEM;
        return;
    }

    va_start(list, cmd);
    while (1) {
        char *arg = va_arg(list, char *);
        if (!arg)
            break;
        arg = strdup(arg);
        if (!arg) {
            cmd->has_error = ENOMEM;
            va_end(list);
            return;
        }
        cmd->args[cmd->nargs++] = arg;
    }
    va_end(list);
}

/*
 * Set the working directory of a non-daemon child process, rather
 * than the parent's working directory.  Daemons automatically get /
 * without using this call.
 */
void
virCommandSetWorkingDirectory(virCommandPtr cmd, const char *pwd)
{
    if (!cmd || cmd->has_error)
        return;

    if (cmd->pwd) {
        cmd->has_error = -1;
        VIR_DEBUG("cannot set directory twice");
    } else {
        cmd->pwd = strdup(pwd);
        if (!cmd->pwd)
            cmd->has_error = ENOMEM;
    }
}


/*
 * Feed the child's stdin from a string buffer
 */
void
virCommandSetInputBuffer(virCommandPtr cmd, const char *inbuf)
{
    if (!cmd || cmd->has_error)
        return;

    if (cmd->infd != -1 || cmd->inbuf) {
        cmd->has_error = -1;
        VIR_DEBUG("cannot specify input twice");
        return;
    }

    cmd->inbuf = strdup(inbuf);
    if (!cmd->inbuf)
        cmd->has_error = ENOMEM;
}


/*
 * Capture the child's stdout to a string buffer.  *outbuf is
 * guaranteed to be allocated after successful virCommandRun or
 * virCommandWait, and is best-effort allocated after failed
 * virCommandRun; caller is responsible for freeing *outbuf.
 */
void
virCommandSetOutputBuffer(virCommandPtr cmd, char **outbuf)
{
    *outbuf = NULL;
    if (!cmd || cmd->has_error)
        return;

    if (cmd->outfdptr) {
        cmd->has_error = -1;
        VIR_DEBUG("cannot specify output twice");
        return;
    }

    cmd->outbuf = outbuf;
    cmd->outfdptr = &cmd->outfd;
}


/*
 * Capture the child's stderr to a string buffer.  *errbuf is
 * guaranteed to be allocated after successful virCommandRun or
 * virCommandWait, and is best-effort allocated after failed
 * virCommandRun; caller is responsible for freeing *errbuf.
 */
void
virCommandSetErrorBuffer(virCommandPtr cmd, char **errbuf)
{
    *errbuf = NULL;
    if (!cmd || cmd->has_error)
        return;

    if (cmd->errfdptr) {
        cmd->has_error = -1;
        VIR_DEBUG("cannot specify stderr twice");
        return;
    }

    cmd->errbuf = errbuf;
    cmd->errfdptr = &cmd->errfd;
}


/*
 * Attach a file descriptor to the child's stdin
 */
void
virCommandSetInputFD(virCommandPtr cmd, int infd)
{
    if (!cmd || cmd->has_error)
        return;

    if (cmd->infd != -1 || cmd->inbuf) {
        cmd->has_error = -1;
        VIR_DEBUG("cannot specify input twice");
        return;
    }
    if (infd < 0) {
        cmd->has_error = -1;
        VIR_DEBUG("cannot specify invalid input fd");
        return;
    }

    cmd->infd = infd;
}


/*
 * Attach a file descriptor to the child's stdout
 */
void
virCommandSetOutputFD(virCommandPtr cmd, int *outfd)
{
    if (!cmd || cmd->has_error)
        return;

    if (cmd->outfdptr) {
        cmd->has_error = -1;
        VIR_DEBUG("cannot specify output twice");
        return;
    }

    cmd->outfdptr = outfd;
}


/*
 * Attach a file descriptor to the child's stderr
 */
void
virCommandSetErrorFD(virCommandPtr cmd, int *errfd)
{
    if (!cmd || cmd->has_error)
        return;

    if (cmd->errfdptr) {
        cmd->has_error = -1;
        VIR_DEBUG("cannot specify stderr twice");
        return;
    }

    cmd->errfdptr = errfd;
}


/*
 * Run HOOK(OPAQUE) in the child as the last thing before changing
 * directories, dropping capabilities, and executing the new process.
 * Force the child to fail if HOOK does not return zero.
 */
void
virCommandSetPreExecHook(virCommandPtr cmd, virExecHook hook, void *opaque)
{
    if (!cmd || cmd->has_error)
        return;

    if (cmd->hook) {
        cmd->has_error = -1;
        VIR_DEBUG("cannot specify hook twice");
        return;
    }
    cmd->hook = hook;
    cmd->opaque = opaque;
}


/*
 * Call after adding all arguments and environment settings, but before
 * Run/RunAsync, to immediately output the environment and arguments of
 * cmd to logfd.  If virCommandRun cannot succeed (because of an
 * out-of-memory condition while building cmd), nothing will be logged.
 */
void
virCommandWriteArgLog(virCommandPtr cmd, int logfd)
{
    int ioError = 0;
    size_t i;

    /* Any errors will be reported later by virCommandRun, which means
     * no command will be run, so there is nothing to log. */
    if (!cmd || cmd->has_error)
        return;

    for (i = 0 ; i < cmd->nenv ; i++) {
        if (safewrite(logfd, cmd->env[i], strlen(cmd->env[i])) < 0)
            ioError = errno;
        if (safewrite(logfd, " ", 1) < 0)
            ioError = errno;
    }
    for (i = 0 ; i < cmd->nargs ; i++) {
        if (safewrite(logfd, cmd->args[i], strlen(cmd->args[i])) < 0)
            ioError = errno;
        if (safewrite(logfd, i == cmd->nargs - 1 ? "\n" : " ", 1) < 0)
            ioError = errno;
    }

    if (ioError) {
        char ebuf[1024];
        VIR_WARN("Unable to write command %s args to logfile: %s",
                 cmd->args[0], virStrerror(ioError, ebuf, sizeof ebuf));
    }
}


/*
 * Call after adding all arguments and environment settings, but before
 * Run/RunAsync, to return a string representation of the environment and
 * arguments of cmd.  If virCommandRun cannot succeed (because of an
 * out-of-memory condition while building cmd), NULL will be returned.
 * Caller is responsible for freeing the resulting string.
 */
char *
virCommandToString(virCommandPtr cmd)
{
    size_t i;
    virBuffer buf = VIR_BUFFER_INITIALIZER;

    /* Cannot assume virCommandRun will be called; so report the error
     * now.  If virCommandRun is called, it will report the same error. */
    if (!cmd ||cmd->has_error == ENOMEM) {
        virReportOOMError();
        return NULL;
    }
    if (cmd->has_error) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("invalid use of command API"));
        return NULL;
    }

    for (i = 0; i < cmd->nenv; i++) {
        virBufferAdd(&buf, cmd->env[i], strlen(cmd->env[i]));
        virBufferAddChar(&buf, ' ');
    }
    virBufferAdd(&buf, cmd->args[0], strlen(cmd->args[0]));
    for (i = 1; i < cmd->nargs; i++) {
        virBufferAddChar(&buf, ' ');
        virBufferAdd(&buf, cmd->args[i], strlen(cmd->args[i]));
    }

    if (virBufferError(&buf)) {
        virBufferFreeAndReset(&buf);
        virReportOOMError();
        return NULL;
    }

    return virBufferContentAndReset(&buf);
}


/*
 * Translate an exit status into a malloc'd string.  Generic helper
 * for virCommandRun, virCommandWait, and virPidWait status argument,
 * as well as raw waitpid and older virRun status.
 */
char *
virCommandTranslateStatus(int status)
{
    char *buf;
    if (WIFEXITED(status)) {
        virAsprintf(&buf, _("exit status %d"), WEXITSTATUS(status));
    } else if (WIFSIGNALED(status)) {
        virAsprintf(&buf, _("fatal signal %d"), WTERMSIG(status));
    } else {
        virAsprintf(&buf, _("invalid value %d"), status);
    }
    return buf;
}


/*
 * Manage input and output to the child process.
 */
static int
virCommandProcessIO(virCommandPtr cmd)
{
    int infd = -1, outfd = -1, errfd = -1;
    size_t inlen = 0, outlen = 0, errlen = 0;
    size_t inoff = 0;
    int ret = 0;

    /* With an input buffer, feed data to child
     * via pipe */
    if (cmd->inbuf) {
        inlen = strlen(cmd->inbuf);
        infd = cmd->inpipe;
    }

    /* With out/err buffer, the outfd/errfd have been filled with an
     * FD for us.  Guarantee an allocated string with partial results
     * even if we encounter a later failure, as well as freeing any
     * results accumulated over a prior run of the same command.  */
    if (cmd->outbuf) {
        outfd = cmd->outfd;
        if (VIR_REALLOC_N(*cmd->outbuf, 1) < 0) {
            virReportOOMError();
            ret = -1;
        }
    }
    if (cmd->errbuf) {
        errfd = cmd->errfd;
        if (VIR_REALLOC_N(*cmd->errbuf, 1) < 0) {
            virReportOOMError();
            ret = -1;
        }
    }
    if (ret == -1)
        goto cleanup;
    ret = -1;

    for (;;) {
        int i;
        struct pollfd fds[3];
        int nfds = 0;

        if (infd != -1) {
            fds[nfds].fd = infd;
            fds[nfds].events = POLLOUT;
            nfds++;
        }
        if (outfd != -1) {
            fds[nfds].fd = outfd;
            fds[nfds].events = POLLIN;
            nfds++;
        }
        if (errfd != -1) {
            fds[nfds].fd = errfd;
            fds[nfds].events = POLLIN;
            nfds++;
        }

        if (nfds == 0)
            break;

        if (poll(fds, nfds, -1) < 0) {
            if ((errno == EAGAIN) || (errno == EINTR))
                continue;
            virReportSystemError(errno, "%s",
                                 _("unable to poll on child"));
            goto cleanup;
        }

        for (i = 0; i < nfds ; i++) {
            if (fds[i].fd == errfd ||
                fds[i].fd == outfd) {
                char data[1024];
                char **buf;
                size_t *len;
                int done;
                if (fds[i].fd == outfd) {
                    buf = cmd->outbuf;
                    len = &outlen;
                } else {
                    buf = cmd->errbuf;
                    len = &errlen;
                }
                /* Silence a false positive from clang. */
                sa_assert(buf);

                done = read(fds[i].fd, data, sizeof(data));
                if (done < 0) {
                    if (errno != EINTR &&
                        errno != EAGAIN) {
                        virReportSystemError(errno, "%s",
                                             (fds[i].fd == outfd) ?
                                             _("unable to read child stdout") :
                                             _("unable to read child stderr"));
                        goto cleanup;
                    }
                } else if (done == 0) {
                    if (fds[i].fd == outfd)
                        outfd = -1;
                    else
                        errfd = -1;
                } else {
                    if (VIR_REALLOC_N(*buf, *len + done + 1) < 0) {
                        virReportOOMError();
                        goto cleanup;
                    }
                    memcpy(*buf + *len, data, done);
                    *len += done;
                }
            } else {
                int done;

                /* Coverity 5.3.0 can't see that we only get here if
                 * infd is in the set because it was non-negative.  */
                sa_assert(infd != -1);
                done = write(infd, cmd->inbuf + inoff,
                             inlen - inoff);
                if (done < 0) {
                    if (errno != EINTR &&
                        errno != EAGAIN) {
                        virReportSystemError(errno, "%s",
                                             _("unable to write to child input"));
                        goto cleanup;
                    }
                } else {
                    inoff += done;
                    if (inoff == inlen) {
                        int tmpfd = infd;
                        if (VIR_CLOSE(infd) < 0)
                            VIR_DEBUG("ignoring failed close on fd %d", tmpfd);
                    }
                }
            }

        }
    }

    ret = 0;
cleanup:
    if (cmd->outbuf && *cmd->outbuf)
        (*cmd->outbuf)[outlen] = '\0';
    if (cmd->errbuf && *cmd->errbuf)
        (*cmd->errbuf)[errlen] = '\0';
    return ret;
}

/*
 * Exec the command, replacing the current process. Meant to be called
 * after already forking / cloning, so does not attempt to daemonize or
 * preserve any FDs.
 *
 * Returns -1 on any error executing the command.
 * Will not return on success.
 */
#ifndef WIN32
int virCommandExec(virCommandPtr cmd)
{
    if (!cmd ||cmd->has_error == ENOMEM) {
        virReportOOMError();
        return -1;
    }
    if (cmd->has_error) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("invalid use of command API"));
        return -1;
    }

    return execve(cmd->args[0], cmd->args, cmd->env);
}
#else
int virCommandExec(virCommandPtr cmd ATTRIBUTE_UNUSED)
{
    /* Mingw execve() has a broken signature. Disable this
     * function until gnulib fixes the signature, since we
     * don't really need this on Win32 anyway.
     */
    virReportSystemError(ENOSYS, "%s",
                         _("Executing new processes is not supported on Win32 platform"));
    return -1;
}
#endif

/*
 * Run the command and wait for completion.
 * Returns -1 on any error executing the
 * command. Returns 0 if the command executed,
 * with the exit status set
 */
int
virCommandRun(virCommandPtr cmd, int *exitstatus)
{
    int ret = 0;
    char *outbuf = NULL;
    char *errbuf = NULL;
    int infd[2] = { -1, -1 };
    struct stat st;
    bool string_io;
    bool async_io = false;
    char *str;

    if (!cmd ||cmd->has_error == ENOMEM) {
        virReportOOMError();
        return -1;
    }
    if (cmd->has_error) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("invalid use of command API"));
        return -1;
    }

    /* Avoid deadlock, by requiring that any open fd not under our
     * control must be visiting a regular file, or that we are
     * daemonized and no string io is required.  */
    string_io = cmd->inbuf || cmd->outbuf || cmd->errbuf;
    if (cmd->infd != -1 &&
        (fstat(cmd->infd, &st) < 0 || !S_ISREG(st.st_mode)))
        async_io = true;
    if (cmd->outfdptr && cmd->outfdptr != &cmd->outfd &&
        (*cmd->outfdptr == -1 ||
         fstat(*cmd->outfdptr, &st) < 0 || !S_ISREG(st.st_mode)))
        async_io = true;
    if (cmd->errfdptr && cmd->errfdptr != &cmd->errfd &&
        (*cmd->errfdptr == -1 ||
         fstat(*cmd->errfdptr, &st) < 0 || !S_ISREG(st.st_mode)))
        async_io = true;
    if (async_io) {
        if (!(cmd->flags & VIR_EXEC_DAEMON) || string_io) {
            virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                            _("cannot mix caller fds with blocking execution"));
            return -1;
        }
    } else {
        if ((cmd->flags & VIR_EXEC_DAEMON) && string_io) {
            virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                            _("cannot mix string I/O with daemon"));
            return -1;
        }
    }

    /* If we have an input buffer, we need
     * a pipe to feed the data to the child */
    if (cmd->inbuf) {
        if (pipe(infd) < 0) {
            virReportSystemError(errno, "%s",
                                 _("unable to open pipe"));
            cmd->has_error = -1;
            return -1;
        }
        cmd->infd = infd[0];
        cmd->inpipe = infd[1];
    }

    /* If caller hasn't requested capture of stdout/err, then capture
     * it ourselves so we can log it.  But the intermediate child for
     * a daemon has no expected output, and we don't want our
     * capturing pipes passed on to the daemon grandchild.
     */
    if (!(cmd->flags & VIR_EXEC_DAEMON)) {
        if (!cmd->outfdptr) {
            cmd->outfdptr = &cmd->outfd;
            cmd->outbuf = &outbuf;
            string_io = true;
        }
        if (!cmd->errfdptr) {
            cmd->errfdptr = &cmd->errfd;
            cmd->errbuf = &errbuf;
            string_io = true;
        }
    }

    cmd->flags |= VIR_EXEC_RUN_SYNC;
    if (virCommandRunAsync(cmd, NULL) < 0) {
        if (cmd->inbuf) {
            int tmpfd = infd[0];
            if (VIR_CLOSE(infd[0]) < 0)
                VIR_DEBUG("ignoring failed close on fd %d", tmpfd);
            tmpfd = infd[1];
            if (VIR_CLOSE(infd[1]) < 0)
                VIR_DEBUG("ignoring failed close on fd %d", tmpfd);
        }
        cmd->has_error = -1;
        return -1;
    }

    if (string_io)
        ret = virCommandProcessIO(cmd);

    if (virCommandWait(cmd, exitstatus) < 0)
        ret = -1;

    str = (exitstatus ? virCommandTranslateStatus(*exitstatus)
           : (char *) "status 0");
    VIR_DEBUG("Result %s, stdout: '%s' stderr: '%s'",
              NULLSTR(str),
              cmd->outbuf ? NULLSTR(*cmd->outbuf) : "(null)",
              cmd->errbuf ? NULLSTR(*cmd->errbuf) : "(null)");
    if (exitstatus)
        VIR_FREE(str);

    /* Reset any capturing, in case caller runs
     * this identical command again */
    if (cmd->inbuf) {
        int tmpfd = infd[0];
        if (VIR_CLOSE(infd[0]) < 0)
            VIR_DEBUG("ignoring failed close on fd %d", tmpfd);
        tmpfd = infd[1];
        if (VIR_CLOSE(infd[1]) < 0)
            VIR_DEBUG("ignoring failed close on fd %d", tmpfd);
    }
    if (cmd->outbuf == &outbuf) {
        int tmpfd = cmd->outfd;
        if (VIR_CLOSE(cmd->outfd) < 0)
            VIR_DEBUG("ignoring failed close on fd %d", tmpfd);
        cmd->outfdptr = NULL;
        cmd->outbuf = NULL;
        VIR_FREE(outbuf);
    }
    if (cmd->errbuf == &errbuf) {
        int tmpfd = cmd->errfd;
        if (VIR_CLOSE(cmd->errfd) < 0)
            VIR_DEBUG("ignoring failed close on fd %d", tmpfd);
        cmd->errfdptr = NULL;
        cmd->errbuf = NULL;
        VIR_FREE(errbuf);
    }

    return ret;
}


/*
 * Perform all virCommand-specific actions, along with the user hook.
 */
static int
virCommandHook(void *data)
{
    virCommandPtr cmd = data;
    int res = 0;

    if (cmd->hook) {
        VIR_DEBUG("Run hook %p %p", cmd->hook, cmd->opaque);
        res = cmd->hook(cmd->opaque);
        VIR_DEBUG("Done hook %d", res);
    }
    if (res == 0 && cmd->pwd) {
        VIR_DEBUG("Running child in %s", cmd->pwd);
        res = chdir(cmd->pwd);
        if (res < 0) {
            virReportSystemError(errno,
                                 _("Unable to change to %s"), cmd->pwd);
        }
    }
    if (cmd->handshake) {
        char c = res < 0 ? '0' : '1';
        int rv;
        VIR_DEBUG("Notifying parent for handshake start on %d", cmd->handshakeWait[1]);
        if (safewrite(cmd->handshakeWait[1], &c, sizeof(c)) != sizeof(c)) {
            virReportSystemError(errno, "%s", _("Unable to notify parent process"));
            return -1;
        }

        /* On failure we pass the error message back to parent,
         * so they don't have to dig through stderr logs
         */
        if (res < 0) {
            virErrorPtr err = virGetLastError();
            const char *msg = err ? err->message :
                _("Unknown failure during hook execution");
            size_t len = strlen(msg) + 1;
            if (safewrite(cmd->handshakeWait[1], msg, len) != len) {
                virReportSystemError(errno, "%s", _("Unable to send error to parent process"));
                return -1;
            }
            return -1;
        }

        VIR_DEBUG("Waiting on parent for handshake complete on %d", cmd->handshakeNotify[0]);
        if ((rv = saferead(cmd->handshakeNotify[0], &c, sizeof(c))) != sizeof(c)) {
            if (rv < 0)
                virReportSystemError(errno, "%s", _("Unable to wait on parent process"));
            else
                virReportSystemError(EIO, "%s", _("libvirtd quit during handshake"));
            return -1;
        }
        if (c != '1') {
            virReportSystemError(EINVAL, _("Unexpected confirm code '%c' from parent process"), c);
            return -1;
        }
        VIR_FORCE_CLOSE(cmd->handshakeWait[1]);
        VIR_FORCE_CLOSE(cmd->handshakeNotify[0]);
    }

    VIR_DEBUG("Hook is done %d", res);

    return res;
}


/*
 * Run the command asynchronously
 * Returns -1 on any error executing the
 * command. Returns 0 if the command executed.
 *
 * There are two approaches to child process cleanup.
 * 1. Use auto-cleanup, by passing NULL for pid.  The child will be
 * auto-reaped by virCommandFree, unless you reap it earlier via
 * virCommandWait or virCommandAbort.  Good for where cmd is in
 * scope for the duration of the child process.
 * 2. Use manual cleanup, by passing the address of a pid_t variable
 * for pid.  While cmd is still in scope, you may reap the child via
 * virCommandWait or virCommandAbort.  But after virCommandFree, if
 * you have not yet reaped the child, then it continues to run until
 * you call virPidWait or virPidAbort.
 */
int
virCommandRunAsync(virCommandPtr cmd, pid_t *pid)
{
    int ret;
    char *str;
    int i;
    bool synchronous = false;

    if (!cmd || cmd->has_error == ENOMEM) {
        virReportOOMError();
        return -1;
    }
    if (cmd->has_error) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("invalid use of command API"));
        return -1;
    }

    synchronous = cmd->flags & VIR_EXEC_RUN_SYNC;
    cmd->flags &= ~VIR_EXEC_RUN_SYNC;

    /* Buffer management can only be requested via virCommandRun.  */
    if ((cmd->inbuf && cmd->infd == -1) ||
        (cmd->outbuf && cmd->outfdptr != &cmd->outfd) ||
        (cmd->errbuf && cmd->errfdptr != &cmd->errfd)) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("cannot mix string I/O with asynchronous command"));
        return -1;
    }

    if (cmd->pid != -1) {
        virCommandError(VIR_ERR_INTERNAL_ERROR,
                        _("command is already running as pid %d"),
                        cmd->pid);
        return -1;
    }

    if (!synchronous && (cmd->flags & VIR_EXEC_DAEMON)) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("daemonized command cannot use virCommandRunAsync"));
        return -1;
    }
    if (cmd->pwd && (cmd->flags & VIR_EXEC_DAEMON)) {
        virCommandError(VIR_ERR_INTERNAL_ERROR,
                        _("daemonized command cannot set working directory %s"),
                        cmd->pwd);
        return -1;
    }
    if (cmd->pidfile && !(cmd->flags & VIR_EXEC_DAEMON)) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("creation of pid file requires daemonized command"));
        return -1;
    }

    str = virCommandToString(cmd);
    VIR_DEBUG("About to run %s", str ? str : cmd->args[0]);
    VIR_FREE(str);

    ret = virExecWithHook((const char *const *)cmd->args,
                          (const char *const *)cmd->env,
                          &cmd->preserve,
                          &cmd->pid,
                          cmd->infd,
                          cmd->outfdptr,
                          cmd->errfdptr,
                          cmd->flags,
                          virCommandHook,
                          cmd,
                          cmd->pidfile);

    VIR_DEBUG("Command result %d, with PID %d",
              ret, (int)cmd->pid);

    for (i = STDERR_FILENO + 1; i < FD_SETSIZE; i++) {
        if (FD_ISSET(i, &cmd->transfer)) {
            int tmpfd = i;
            VIR_FORCE_CLOSE(tmpfd);
            FD_CLR(i, &cmd->transfer);
        }
    }

    if (ret == 0 && pid)
        *pid = cmd->pid;
    else
        cmd->reap = true;

    return ret;
}


/*
 * Wait for a child process to complete.
 * Return -1 on any error waiting for
 * completion. Returns 0 if the command
 * finished with the exit status set
 */
int
virPidWait(pid_t pid, int *exitstatus)
{
    int ret;
    int status;

    if (pid <= 0) {
        virReportSystemError(EINVAL, _("unable to wait for process %d"), pid);
        return -1;
    }

    /* Wait for intermediate process to exit */
    while ((ret = waitpid(pid, &status, 0)) == -1 &&
           errno == EINTR);

    if (ret == -1) {
        virReportSystemError(errno, _("unable to wait for process %d"), pid);
        return -1;
    }

    if (exitstatus == NULL) {
        if (status != 0) {
            char *st = virCommandTranslateStatus(status);
            virCommandError(VIR_ERR_INTERNAL_ERROR,
                            _("Child process (%d) status unexpected: %s"),
                            pid, NULLSTR(st));
            VIR_FREE(st);
            return -1;
        }
    } else {
        *exitstatus = status;
    }

    return 0;
}

/*
 * Wait for the async command to complete.
 * Return -1 on any error waiting for
 * completion. Returns 0 if the command
 * finished with the exit status set
 */
int
virCommandWait(virCommandPtr cmd, int *exitstatus)
{
    int ret;
    int status = 0;

    if (!cmd ||cmd->has_error == ENOMEM) {
        virReportOOMError();
        return -1;
    }
    if (cmd->has_error) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("invalid use of command API"));
        return -1;
    }

    if (cmd->pid == -1) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("command is not yet running"));
        return -1;
    }

    /* If virPidWait reaps pid but then returns failure because
     * exitstatus was NULL, then a second virCommandWait would risk
     * calling waitpid on an unrelated process.  Besides, that error
     * message is not as detailed as what we can provide.  So, we
     * guarantee that virPidWait only fails due to failure to wait,
     * and repeat the exitstatus check code ourselves.  */
    ret = virPidWait(cmd->pid, exitstatus ? exitstatus : &status);
    if (ret == 0) {
        cmd->pid = -1;
        cmd->reap = false;
        if (status) {
            char *str = virCommandToString(cmd);
            char *st = virCommandTranslateStatus(status);
            virCommandError(VIR_ERR_INTERNAL_ERROR,
                            _("Child process (%s) status unexpected: %s"),
                            str ? str : cmd->args[0], NULLSTR(st));
            VIR_FREE(str);
            VIR_FREE(st);
            return -1;
        }
    }

    return ret;
}


#ifndef WIN32
/*
 * Abort a child process if PID is positive and that child is still
 * running, without issuing any errors or affecting errno.  Designed
 * for error paths where some but not all paths to the cleanup code
 * might have started the child process.
 */
void
virPidAbort(pid_t pid)
{
    int saved_errno;
    int ret;
    int status;
    char *tmp = NULL;

    if (pid <= 0)
        return;

    /* See if intermediate process has exited; if not, try a nice
     * SIGTERM followed by a more severe SIGKILL.
     */
    saved_errno = errno;
    VIR_DEBUG("aborting child process %d", pid);
    while ((ret = waitpid(pid, &status, WNOHANG)) == -1 &&
           errno == EINTR);
    if (ret == pid) {
        tmp = virCommandTranslateStatus(status);
        VIR_DEBUG("process has ended: %s", tmp);
        goto cleanup;
    } else if (ret == 0) {
        VIR_DEBUG("trying SIGTERM to child process %d", pid);
        kill(pid, SIGTERM);
        usleep(10 * 1000);
        while ((ret = waitpid(pid, &status, WNOHANG)) == -1 &&
               errno == EINTR);
        if (ret == pid) {
            tmp = virCommandTranslateStatus(status);
            VIR_DEBUG("process has ended: %s", tmp);
            goto cleanup;
        } else if (ret == 0) {
            VIR_DEBUG("trying SIGKILL to child process %d", pid);
            kill(pid, SIGKILL);
            while ((ret = waitpid(pid, &status, 0)) == -1 &&
                   errno == EINTR);
            if (ret == pid) {
                tmp = virCommandTranslateStatus(status);
                VIR_DEBUG("process has ended: %s", tmp);
                goto cleanup;
            }
        }
    }
    VIR_DEBUG("failed to reap child %d, abandoning it", pid);

cleanup:
    VIR_FREE(tmp);
    errno = saved_errno;
}

/*
 * Abort an async command if it is running, without issuing
 * any errors or affecting errno.  Designed for error paths
 * where some but not all paths to the cleanup code might
 * have started the child process.
 */
void
virCommandAbort(virCommandPtr cmd)
{
    if (!cmd || cmd->pid == -1)
        return;
    virPidAbort(cmd->pid);
    cmd->pid = -1;
    cmd->reap = false;
}
#else /* WIN32 */
void
virPidAbort(pid_t pid)
{
    /* Not yet ported to mingw.  Any volunteers?  */
    VIR_DEBUG("failed to reap child %d, abandoning it", pid);
}

void
virCommandAbort(virCommandPtr cmd ATTRIBUTE_UNUSED)
{
    /* Mingw lacks WNOHANG and kill().  But since we haven't ported
     * virExecWithHook to mingw yet, there's no process to be killed,
     * making this implementation trivially correct for now :)  */
}
#endif


void virCommandRequireHandshake(virCommandPtr cmd)
{
    if (!cmd || cmd->has_error)
        return;

    if (cmd->handshake) {
        cmd->has_error = -1;
        VIR_DEBUG("Cannot require handshake twice");
        return;
    }

    if (pipe(cmd->handshakeWait) < 0) {
        cmd->has_error = errno;
        return;
    }
    if (pipe(cmd->handshakeNotify) < 0) {
        VIR_FORCE_CLOSE(cmd->handshakeWait[0]);
        VIR_FORCE_CLOSE(cmd->handshakeWait[1]);
        cmd->has_error = errno;
        return;
    }

    VIR_DEBUG("Transfer handshake wait=%d notify=%d",
              cmd->handshakeWait[1], cmd->handshakeNotify[0]);
    virCommandTransferFD(cmd, cmd->handshakeWait[1]);
    virCommandTransferFD(cmd, cmd->handshakeNotify[0]);
    cmd->handshake = true;
}

int virCommandHandshakeWait(virCommandPtr cmd)
{
    char c;
    int rv;
    if (!cmd ||cmd->has_error == ENOMEM) {
        virReportOOMError();
        return -1;
    }
    if (cmd->has_error || !cmd->handshake) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("invalid use of command API"));
        return -1;
    }

    if (cmd->handshakeWait[0] == -1) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("Handshake is already complete"));
        return -1;
    }

    VIR_DEBUG("Wait for handshake on %d", cmd->handshakeWait[0]);
    if ((rv = saferead(cmd->handshakeWait[0], &c, sizeof(c))) != sizeof(c)) {
        if (rv < 0)
            virReportSystemError(errno, "%s", _("Unable to wait for child process"));
        else
            virReportSystemError(EIO, "%s", _("Child process quit during startup handshake"));
        VIR_FORCE_CLOSE(cmd->handshakeWait[0]);
        return -1;
    }
    if (c != '1') {
        char *msg;
        ssize_t len;
        if (VIR_ALLOC_N(msg, 1024) < 0) {
            virReportOOMError();
            VIR_FORCE_CLOSE(cmd->handshakeWait[0]);
            return -1;
        }
        if ((len = saferead(cmd->handshakeWait[0], msg, 1024)) < 0) {
            VIR_FORCE_CLOSE(cmd->handshakeWait[0]);
            VIR_FREE(msg);
            virReportSystemError(errno, "%s", _("No error message from child failure"));
            return -1;
        }
        VIR_FORCE_CLOSE(cmd->handshakeWait[0]);
        msg[len-1] = '\0';
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s", msg);
        VIR_FREE(msg);
        return -1;
    }
    VIR_FORCE_CLOSE(cmd->handshakeWait[0]);
    return 0;
}

int virCommandHandshakeNotify(virCommandPtr cmd)
{
    char c = '1';
    if (!cmd ||cmd->has_error == ENOMEM) {
        virReportOOMError();
        return -1;
    }
    if (cmd->has_error || !cmd->handshake) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("invalid use of command API"));
        return -1;
    }

    if (cmd->handshakeNotify[1] == -1) {
        virCommandError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("Handshake is already complete"));
        return -1;
    }

    VIR_DEBUG("Notify handshake on %d", cmd->handshakeWait[0]);
    if (safewrite(cmd->handshakeNotify[1], &c, sizeof(c)) != sizeof(c)) {
        virReportSystemError(errno, "%s", _("Unable to notify child process"));
        VIR_FORCE_CLOSE(cmd->handshakeNotify[1]);
        return -1;
    }
    VIR_FORCE_CLOSE(cmd->handshakeNotify[1]);
    return 0;
}


/*
 * Release all resources.  The only exception is that if you called
 * virCommandRunAsync with a non-null pid, then the asynchronous child
 * is not reaped, and you must call virPidWait() yourself.
 */
void
virCommandFree(virCommandPtr cmd)
{
    int i;
    if (!cmd)
        return;

    for (i = STDERR_FILENO + 1; i < FD_SETSIZE; i++) {
        if (FD_ISSET(i, &cmd->transfer)) {
            int tmpfd = i;
            VIR_FORCE_CLOSE(tmpfd);
        }
    }

    VIR_FREE(cmd->inbuf);
    VIR_FORCE_CLOSE(cmd->outfd);
    VIR_FORCE_CLOSE(cmd->errfd);

    for (i = 0 ; i < cmd->nargs ; i++)
        VIR_FREE(cmd->args[i]);
    VIR_FREE(cmd->args);

    for (i = 0 ; i < cmd->nenv ; i++)
        VIR_FREE(cmd->env[i]);
    VIR_FREE(cmd->env);

    VIR_FREE(cmd->pwd);

    if (cmd->handshake) {
        /* The other 2 fds in these arrays are closed
         * due to use with virCommandTransferFD
         */
        VIR_FORCE_CLOSE(cmd->handshakeWait[0]);
        VIR_FORCE_CLOSE(cmd->handshakeNotify[1]);
    }

    VIR_FREE(cmd->pidfile);

    if (cmd->reap)
        virCommandAbort(cmd);

    VIR_FREE(cmd);
}