Replace polling for active VMs with signalling by drivers

Currently to deal with auto-shutdown libvirtd must periodically
poll all stateful drivers. Thus sucks because it requires
acquiring both the driver lock and locks on every single virtual
machine. Instead pass in a "inhibit" callback to virStateInitialize
which drivers can invoke whenever they want to inhibit shutdown
due to existance of active VMs.

Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
This commit is contained in:
Daniel P. Berrange 2012-10-31 19:03:55 +00:00
parent ae2163f852
commit 79b8a56995
26 changed files with 176 additions and 141 deletions

View File

@ -584,16 +584,6 @@ error:
}
static int daemonShutdownCheck(virNetServerPtr srv ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
if (virStateActive())
return 0;
return 1;
}
/*
* Set up the logging environment
* By default if daemonized all errors go to the logfile libvirtd.log,
@ -772,6 +762,18 @@ static int daemonSetupSignals(virNetServerPtr srv)
return 0;
}
static void daemonInhibitCallback(bool inhibit, void *opaque)
{
virNetServerPtr srv = opaque;
if (inhibit)
virNetServerAddShutdownInhibition(srv);
else
virNetServerRemoveShutdownInhibition(srv);
}
static void daemonRunStateInit(void *opaque)
{
virNetServerPtr srv = opaque;
@ -780,7 +782,9 @@ static void daemonRunStateInit(void *opaque)
* This is deliberately done after telling the parent process
* we're ready, since it can take a long time and this will
* seriously delay OS bootup process */
if (virStateInitialize(virNetServerIsPrivileged(srv)) < 0) {
if (virStateInitialize(virNetServerIsPrivileged(srv),
daemonInhibitCallback,
srv) < 0) {
VIR_ERROR(_("Driver state initialization failed"));
/* Ensure the main event loop quits */
kill(getpid(), SIGTERM);
@ -1270,9 +1274,7 @@ int main(int argc, char **argv) {
if (timeout != -1) {
VIR_DEBUG("Registering shutdown timeout %d", timeout);
virNetServerAutoShutdown(srv,
timeout,
daemonShutdownCheck,
NULL);
timeout);
}
if ((daemonSetupSignals(srv)) < 0) {

View File

@ -1603,12 +1603,14 @@ libvirt_net_rpc_server_la_SOURCES = \
rpc/virnetserver.h rpc/virnetserver.c
libvirt_net_rpc_server_la_CFLAGS = \
$(AVAHI_CFLAGS) \
$(DBUS_CFLAGS) \
$(XDR_CFLAGS) \
$(AM_CFLAGS) \
$(POLKIT_CFLAGS)
libvirt_net_rpc_server_la_LDFLAGS = \
$(AM_LDFLAGS) \
$(AVAHI_LIBS) \
$(DBUS_LIBS) \
$(POLKIT_LIBS) \
$(CYGWIN_EXTRA_LDFLAGS) \
$(MINGW_EXTRA_LDFLAGS)

View File

@ -1500,10 +1500,12 @@ struct _virStorageDriver {
};
# ifdef WITH_LIBVIRTD
typedef int (*virDrvStateInitialize) (bool privileged);
typedef int (*virDrvStateInitialize) (bool privileged,
virStateInhibitCallback callback,
void *opaque);
typedef int (*virDrvStateCleanup) (void);
typedef int (*virDrvStateReload) (void);
typedef int (*virDrvStateActive) (void);
typedef int (*virDrvStateStop) (void);
typedef struct _virStateDriver virStateDriver;
@ -1514,7 +1516,6 @@ struct _virStateDriver {
virDrvStateInitialize initialize;
virDrvStateCleanup cleanup;
virDrvStateReload reload;
virDrvStateActive active;
virDrvStateStop stop;
};
# endif

View File

@ -790,12 +790,17 @@ virRegisterStateDriver(virStateDriverPtr driver)
/**
* virStateInitialize:
* @privileged: set to true if running with root privilege, false otherwise
* @callback: callback to invoke to inhibit shutdown of the daemon
* @opaque: data to pass to @callback
*
* Initialize all virtualization drivers.
*
* Returns 0 if all succeed, -1 upon any failure.
*/
int virStateInitialize(bool privileged) {
int virStateInitialize(bool privileged,
virStateInhibitCallback callback,
void *opaque)
{
int i;
if (virInitialize() < 0)
@ -805,7 +810,9 @@ int virStateInitialize(bool privileged) {
if (virStateDriverTab[i]->initialize) {
VIR_DEBUG("Running global init for %s state driver",
virStateDriverTab[i]->name);
if (virStateDriverTab[i]->initialize(privileged) < 0) {
if (virStateDriverTab[i]->initialize(privileged,
callback,
opaque) < 0) {
VIR_ERROR(_("Initialization of %s state driver failed"),
virStateDriverTab[i]->name);
return -1;
@ -851,24 +858,6 @@ int virStateReload(void) {
return ret;
}
/**
* virStateActive:
*
* Run each virtualization driver's "active" method.
*
* Returns 0 if none are active, 1 if at least one is.
*/
int virStateActive(void) {
int i, ret = 0;
for (i = 0 ; i < virStateDriverTabCount ; i++) {
if (virStateDriverTab[i]->active &&
virStateDriverTab[i]->active())
ret = 1;
}
return ret;
}
/**
* virStateStop:
*

View File

@ -28,10 +28,14 @@
# include "internal.h"
# ifdef WITH_LIBVIRTD
int virStateInitialize(bool privileged);
typedef void (*virStateInhibitCallback)(bool inhibit,
void *opaque);
int virStateInitialize(bool privileged,
virStateInhibitCallback inhibit,
void *opaque);
int virStateCleanup(void);
int virStateReload(void);
int virStateActive(void);
int virStateStop(void);
# endif

View File

@ -1595,6 +1595,7 @@ xdr_virNetMessageError;
# virnetserver.h
virNetServerAddProgram;
virNetServerAddService;
virNetServerAddShutdownInhibition;
virNetServerAddSignalHandler;
virNetServerAutoShutdown;
virNetServerClose;
@ -1604,6 +1605,7 @@ virNetServerNew;
virNetServerNewPostExecRestart;
virNetServerPreExecRestart;
virNetServerQuit;
virNetServerRemoveShutdownInhibition;
virNetServerRun;
virNetServerSetTLSContext;
virNetServerUpdateServices;

View File

@ -61,6 +61,11 @@ struct _libxlDriverPrivate {
libxl_ctx ctx;
virBitmapPtr reservedVNCPorts;
size_t nactive;
virStateInhibitCallback inhibitCallback;
void *inhibitOpaque;
virDomainObjList domains;
virDomainEventStatePtr domainEventState;

View File

@ -312,6 +312,10 @@ libxlVmCleanup(libxlDriverPrivatePtr driver,
virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, reason);
}
driver->nactive--;
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(false, driver->inhibitOpaque);
if ((vm->def->ngraphics == 1) &&
vm->def->graphics[0]->type == VIR_DOMAIN_GRAPHICS_TYPE_VNC &&
vm->def->graphics[0]->data.vnc.autoport) {
@ -717,6 +721,10 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm,
if (virDomainSaveStatus(driver->caps, driver->stateDir, vm) < 0)
goto error;
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(true, driver->inhibitOpaque);
driver->nactive++;
event = virDomainEventNewFromObj(vm, VIR_DOMAIN_EVENT_STARTED,
restore_fd < 0 ?
VIR_DOMAIN_EVENT_STARTED_BOOTED :
@ -782,6 +790,10 @@ libxlReconnectDomain(void *payload,
vm->def->id = d_info.domid;
virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_UNKNOWN);
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(true, driver->inhibitOpaque);
driver->nactive++;
/* Recreate domain death et. al. events */
libxlCreateDomEvents(vm);
virDomainObjUnlock(vm);
@ -834,7 +846,10 @@ libxlShutdown(void)
}
static int
libxlStartup(bool privileged) {
libxlStartup(bool privileged,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
const libxl_version_info *ver_info;
char *log_file = NULL;
virCommandPtr cmd;
@ -1030,14 +1045,6 @@ libxlReload(void)
return 0;
}
static int
libxlActive(void)
{
if (!libxl_driver)
return 0;
return 1;
}
static virDrvOpenStatus
libxlOpen(virConnectPtr conn,
@ -3969,7 +3976,6 @@ static virStateDriver libxlStateDriver = {
.initialize = libxlStartup,
.cleanup = libxlShutdown,
.reload = libxlReload,
.active = libxlActive,
};

View File

@ -52,6 +52,11 @@ struct _virLXCDriver {
virCapsPtr caps;
virCgroupPtr cgroup;
size_t nactive;
virStateInhibitCallback inhibitCallback;
void *inhibitOpaque;
virDomainObjList domains;
char *configDir;
char *autostartDir;

View File

@ -70,7 +70,9 @@
#define LXC_NB_MEM_PARAM 3
static int lxcStartup(bool privileged);
static int lxcStartup(bool privileged,
virStateInhibitCallback callback,
void *opaque);
static int lxcShutdown(void);
virLXCDriverPtr lxc_driver = NULL;
@ -1398,7 +1400,9 @@ error:
}
static int lxcStartup(bool privileged)
static int lxcStartup(bool privileged,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
char *ld;
int rc;
@ -1564,26 +1568,6 @@ static int lxcShutdown(void)
return 0;
}
/**
* lxcActive:
*
* Checks if the LXC daemon is active, i.e. has an active domain
*
* Returns 1 if active, 0 otherwise
*/
static int
lxcActive(void) {
int active;
if (lxc_driver == NULL)
return 0;
lxcDriverLock(lxc_driver);
active = virDomainObjListNumOfDomains(&lxc_driver->domains, 1);
lxcDriverUnlock(lxc_driver);
return active;
}
static int lxcVersion(virConnectPtr conn ATTRIBUTE_UNUSED, unsigned long *version)
{
@ -3014,7 +2998,6 @@ static virStateDriver lxcStateDriver = {
.name = LXC_DRIVER_NAME,
.initialize = lxcStartup,
.cleanup = lxcShutdown,
.active = lxcActive,
.reload = lxcReload,
};

View File

@ -252,6 +252,10 @@ static void virLXCProcessCleanup(virLXCDriverPtr driver,
vm->pid = -1;
vm->def->id = -1;
driver->nactive--;
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(false, driver->inhibitOpaque);
for (i = 0 ; i < vm->def->nnets ; i++) {
virDomainNetDefPtr iface = vm->def->nets[i];
vport = virDomainNetGetActualVirtPortProfile(iface);
@ -1139,6 +1143,10 @@ int virLXCProcessStart(virConnectPtr conn,
virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason);
priv->doneStopEvent = false;
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(true, driver->inhibitOpaque);
driver->nactive++;
if (lxcContainerWaitForContinue(handshakefds[0]) < 0) {
char out[1024];
@ -1313,6 +1321,10 @@ virLXCProcessReconnectDomain(void *payload, const void *name ATTRIBUTE_UNUSED, v
virDomainObjSetState(vm, VIR_DOMAIN_RUNNING,
VIR_DOMAIN_RUNNING_UNKNOWN);
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(true, driver->inhibitOpaque);
driver->nactive++;
if (!(priv->monitor = virLXCProcessConnectMonitor(driver, vm)))
goto error;

View File

@ -332,7 +332,10 @@ firewalld_dbus_filter_bridge(DBusConnection *connection ATTRIBUTE_UNUSED,
* Initialization function for the QEmu daemon
*/
static int
networkStartup(bool privileged) {
networkStartup(bool privileged,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
char *base = NULL;
#ifdef HAVE_FIREWALLD
DBusConnection *sysbus = NULL;

View File

@ -586,9 +586,9 @@ static void device_prop_modified(LibHalContext *ctx ATTRIBUTE_UNUSED,
}
static int halDeviceMonitorStartup(bool privileged ATTRIBUTE_UNUSED)
static int halDeviceMonitorStartup(bool privileged ATTRIBUTE_UNUSED,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
LibHalContext *hal_ctx = NULL;
char **udi = NULL;

View File

@ -1604,7 +1604,9 @@ out:
return ret;
}
static int udevDeviceMonitorStartup(bool privileged ATTRIBUTE_UNUSED)
static int udevDeviceMonitorStartup(bool privileged ATTRIBUTE_UNUSED,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
udevPrivate *priv = NULL;
struct udev *udev = NULL;

View File

@ -165,7 +165,9 @@ nwfilterDriverInstallDBusMatches(DBusConnection *sysbus ATTRIBUTE_UNUSED)
* Initialization function for the QEmu daemon
*/
static int
nwfilterDriverStartup(bool privileged)
nwfilterDriverStartup(bool privileged ATTRIBUTE_UNUSED,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
char *base = NULL;
DBusConnection *sysbus = NULL;

View File

@ -73,6 +73,10 @@ struct _virQEMUDriver {
int cgroupControllers;
char **cgroupDeviceACL;
size_t nactive;
virStateInhibitCallback inhibitCallback;
void *inhibitOpaque;
virDomainObjList domains;
/* These four directories are ones libvirtd uses (so must be root:root

View File

@ -610,7 +610,10 @@ qemuDomainFindMaxID(void *payload,
* Initialization function for the QEmu daemon
*/
static int
qemuStartup(bool privileged) {
qemuStartup(bool privileged,
virStateInhibitCallback callback,
void *opaque)
{
char *base = NULL;
char *driverConf = NULL;
int rc;
@ -631,6 +634,8 @@ qemuStartup(bool privileged) {
qemu_driver->privileged = privileged;
qemu_driver->uri = privileged ? "qemu:///system" : "qemu:///session";
qemu_driver->inhibitCallback = callback;
qemu_driver->inhibitOpaque = opaque;
/* Don't have a dom0 so start from 1 */
qemu_driver->nextvmid = 1;
@ -974,28 +979,6 @@ qemuReload(void) {
return 0;
}
/**
* qemuActive:
*
* Checks if the QEmu daemon is active, i.e. has an active domain or
* an active network
*
* Returns 1 if active, 0 otherwise
*/
static int
qemuActive(void) {
int active = 0;
if (!qemu_driver)
return 0;
/* XXX having to iterate here is not great because it requires many locks */
qemuDriverLock(qemu_driver);
active = virDomainObjListNumOfDomains(&qemu_driver->domains, 1);
qemuDriverUnlock(qemu_driver);
return active;
}
/*
* qemuStop:
@ -15066,7 +15049,6 @@ static virStateDriver qemuStateDriver = {
.initialize = qemuStartup,
.cleanup = qemuShutdown,
.reload = qemuReload,
.active = qemuActive,
.stop = qemuStop,
};

View File

@ -3228,6 +3228,10 @@ qemuProcessReconnect(void *opaque)
goto error;
}
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(true, driver->inhibitOpaque);
driver->nactive++;
endjob:
if (!qemuDomainObjEndJob(driver, obj))
obj = NULL;
@ -3436,6 +3440,10 @@ int qemuProcessStart(virConnectPtr conn,
qemuDomainSetFakeReboot(driver, vm, false);
virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, VIR_DOMAIN_SHUTOFF_UNKNOWN);
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(true, driver->inhibitOpaque);
driver->nactive++;
/* Run an early hook to set-up missing devices */
if (virHookPresent(VIR_HOOK_DRIVER_QEMU)) {
char *xml = qemuDomainDefFormatXML(driver, vm->def, 0);
@ -4002,6 +4010,10 @@ void qemuProcessStop(virQEMUDriverPtr driver,
*/
vm->def->id = -1;
driver->nactive--;
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(false, driver->inhibitOpaque);
if ((logfile = qemuDomainCreateLog(driver, vm, true)) < 0) {
/* To not break the normal domain shutdown process, skip the
* timestamp log writing if failed on opening log file. */
@ -4233,6 +4245,10 @@ int qemuProcessAttach(virConnectPtr conn ATTRIBUTE_UNUSED,
vm->def->id = driver->nextvmid++;
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(true, driver->inhibitOpaque);
driver->nactive++;
if (virFileMakePath(driver->logDir) < 0) {
virReportSystemError(errno,
_("cannot create log directory %s"),

View File

@ -150,7 +150,9 @@ static char *get_transport_from_scheme(char *scheme);
#ifdef WITH_LIBVIRTD
static int
remoteStartup(bool privileged ATTRIBUTE_UNUSED)
remoteStartup(bool privileged ATTRIBUTE_UNUSED,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
/* Mark that we're inside the daemon so we can avoid
* re-entering ourselves

View File

@ -101,8 +101,7 @@ struct _virNetServer {
virNetTLSContextPtr tls;
unsigned int autoShutdownTimeout;
virNetServerAutoShutdownFunc autoShutdownFunc;
void *autoShutdownOpaque;
size_t autoShutdownInhibitions;
virNetServerClientPrivNew clientPrivNew;
virNetServerClientPrivPreExecRestart clientPrivPreExecRestart;
@ -707,19 +706,33 @@ bool virNetServerIsPrivileged(virNetServerPtr srv)
void virNetServerAutoShutdown(virNetServerPtr srv,
unsigned int timeout,
virNetServerAutoShutdownFunc func,
void *opaque)
unsigned int timeout)
{
virNetServerLock(srv);
srv->autoShutdownTimeout = timeout;
srv->autoShutdownFunc = func;
srv->autoShutdownOpaque = opaque;
virNetServerUnlock(srv);
}
void virNetServerAddShutdownInhibition(virNetServerPtr srv)
{
virNetServerLock(srv);
srv->autoShutdownInhibitions++;
virNetServerUnlock(srv);
}
void virNetServerRemoveShutdownInhibition(virNetServerPtr srv)
{
virNetServerLock(srv);
srv->autoShutdownInhibitions--;
virNetServerUnlock(srv);
}
static sig_atomic_t sigErrors = 0;
static int sigLastErrno = 0;
static int sigWrite = -1;
@ -932,7 +945,7 @@ static void virNetServerAutoShutdownTimer(int timerid ATTRIBUTE_UNUSED,
virNetServerLock(srv);
if (srv->autoShutdownFunc(srv, srv->autoShutdownOpaque)) {
if (!srv->autoShutdownInhibitions) {
VIR_DEBUG("Automatic shutdown triggered");
srv->quit = 1;
}

View File

@ -60,9 +60,10 @@ typedef int (*virNetServerAutoShutdownFunc)(virNetServerPtr srv, void *opaque);
bool virNetServerIsPrivileged(virNetServerPtr srv);
void virNetServerAutoShutdown(virNetServerPtr srv,
unsigned int timeout,
virNetServerAutoShutdownFunc func,
void *opaque);
unsigned int timeout);
void virNetServerAddShutdownInhibition(virNetServerPtr srv);
void virNetServerRemoveShutdownInhibition(virNetServerPtr srv);
typedef void (*virNetServerSignalFunc)(virNetServerPtr srv, siginfo_t *info, void *opaque);

View File

@ -1073,7 +1073,9 @@ secretDriverCleanup(void)
}
static int
secretDriverStartup(bool privileged)
secretDriverStartup(bool privileged,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
char *base = NULL;
@ -1166,7 +1168,6 @@ static virStateDriver stateDriver = {
.initialize = secretDriverStartup,
.cleanup = secretDriverCleanup,
.reload = secretDriverReload,
.active = NULL /* All persistent state is immediately saved to disk */
};
int

View File

@ -128,7 +128,9 @@ storageDriverAutostart(virStorageDriverStatePtr driver) {
* Initialization function for the QEmu daemon
*/
static int
storageDriverStartup(bool privileged)
storageDriverStartup(bool privileged,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
char *base = NULL;

View File

@ -45,11 +45,14 @@ struct uml_driver {
virMutex lock;
bool privileged;
virStateInhibitCallback inhibitCallback;
void *inhibitOpaque;
unsigned long umlVersion;
int nextvmid;
virDomainObjList domains;
size_t nactive;
char *configDir;
char *autostartDir;

View File

@ -372,6 +372,11 @@ reread:
}
dom->def->id = driver->nextvmid++;
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(true, driver->inhibitOpaque);
driver->nactive++;
virDomainObjSetState(dom, VIR_DOMAIN_RUNNING,
VIR_DOMAIN_RUNNING_BOOTED);
@ -419,7 +424,9 @@ cleanup:
* Initialization function for the Uml daemon
*/
static int
umlStartup(bool privileged)
umlStartup(bool privileged,
virStateInhibitCallback callback,
void *opaque)
{
char *base = NULL;
char *userdir = NULL;
@ -428,6 +435,8 @@ umlStartup(bool privileged)
return -1;
uml_driver->privileged = privileged;
uml_driver->inhibitCallback = callback;
uml_driver->inhibitOpaque = opaque;
if (virMutexInit(&uml_driver->lock) < 0) {
VIR_FREE(uml_driver);
@ -585,27 +594,6 @@ umlReload(void) {
return 0;
}
/**
* umlActive:
*
* Checks if the Uml daemon is active, i.e. has an active domain or
* an active network
*
* Returns 1 if active, 0 otherwise
*/
static int
umlActive(void) {
int active = 0;
if (!uml_driver)
return 0;
umlDriverLock(uml_driver);
active = virDomainObjListNumOfDomains(&uml_driver->domains, 1);
umlDriverUnlock(uml_driver);
return active;
}
static void
umlShutdownOneVM(void *payload, const void *name ATTRIBUTE_UNUSED, void *opaque)
@ -1154,6 +1142,10 @@ static void umlShutdownVMDaemon(struct uml_driver *driver,
vm->def->id = -1;
vm->newDef = NULL;
}
driver->nactive--;
if (!driver->nactive && driver->inhibitCallback)
driver->inhibitCallback(false, driver->inhibitOpaque);
}
@ -2634,7 +2626,6 @@ static virStateDriver umlStateDriver = {
.initialize = umlStartup,
.cleanup = umlShutdown,
.reload = umlReload,
.active = umlActive,
};
int umlRegister(void) {

View File

@ -201,7 +201,9 @@ done:
#ifdef WITH_LIBVIRTD
static int
xenInitialize(bool privileged ATTRIBUTE_UNUSED)
xenInitialize(bool privileged ATTRIBUTE_UNUSED,
virStateInhibitCallback callback ATTRIBUTE_UNUSED,
void *opaque ATTRIBUTE_UNUSED)
{
inside_daemon = true;
return 0;