libvirt/src/lxc/lxc_monitor.c

221 lines
6.1 KiB
C
Raw Normal View History

/*
* Copyright (C) 2010-2012 Red Hat, Inc.
*
* lxc_monitor.c: client for LXC controller monitor
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library. If not, see
* <http://www.gnu.org/licenses/>.
*/
#include <config.h>
#include "lxc_monitor.h"
#include "lxc_conf.h"
Run an RPC protocol over the LXC controller monitor This defines a new RPC protocol to be used between the LXC controller and the libvirtd LXC driver. There is only a single RPC message defined thus far, an asynchronous "EXIT" event that is emitted just before the LXC controller process exits. This provides the LXC driver with details about how the container shutdown - normally, or abnormally (crashed), thus allowing the driver to emit better libvirt events. Emitting the event in the LXC controller requires a few little tricks with the RPC service. Simply calling the virNetServiceClientSendMessage does not work, since this merely queues the message for asynchronous processing. In addition the main event loop is no longer running at the point the event is emitted, so no I/O is processed. Thus after invoking virNetServiceClientSendMessage it is necessary to mark the client as being in "delayed close" mode. Then the event loop is run again, until the client completes its close - this happens only after the queued message has been fully transmitted. The final complexity is that it is not safe to run virNetServerQuit() from the client close callback, since that is invoked from a context where the server is locked. Thus a zero-second timer is used to trigger shutdown of the event loop, causing the controller to finally exit. * src/Makefile.am: Add rules for generating RPC protocol files and dispatch methods * src/lxc/lxc_controller.c: Emit an RPC event immediately before exiting * src/lxc/lxc_domain.h: Record the shutdown reason given by the controller * src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register RPC program and event handler. Add callback to let driver receive EXIT event. * src/lxc/lxc_process.c: Use monitor exit event to decide what kind of domain event to emit * src/lxc/lxc_protocol.x: Define wire protocol for LXC controller monitor. Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-07-17 14:54:08 +00:00
#include "lxc_monitor_dispatch.h"
2012-12-12 18:06:53 +00:00
#include "viralloc.h"
#include "virerror.h"
2012-12-12 17:59:27 +00:00
#include "virlog.h"
#include "virthread.h"
#include "rpc/virnetclient.h"
#include "virstring.h"
#define VIR_FROM_THIS VIR_FROM_LXC
struct _virLXCMonitor {
virObjectLockable parent;
virDomainObjPtr vm;
Fix deadlock in handling EOF in LXC monitor Depending on the scenario in which LXC containers exit, it is possible for the EOF callback of the LXC monitor to deadlock the driver. #0 0x00000038a0a0de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00000038a0a09ca6 in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00000038a0a09ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007f4bd9579d55 in virMutexLock (m=<optimized out>) at util/threads-pthread.c:85 #4 0x00007f4bcacc7597 in lxcDriverLock (driver=0x7f4bc40c8290) at lxc/lxc_conf.h:81 #5 virLXCProcessMonitorEOFNotify (mon=<optimized out>, vm=0x7f4bb4000b00) at lxc/lxc_process.c:581 #6 0x00007f4bd9645c91 in virNetClientCloseLocked (client=client@entry=0x7f4bb4009e60) at rpc/virnetclient.c:554 #7 0x00007f4bd96460f8 in virNetClientIOEventLoopPassTheBuck (thiscall=0x0, client=0x7f4bb4009e60) at rpc/virnetclient.c:1306 #8 virNetClientIOEventLoopPassTheBuck (client=0x7f4bb4009e60, thiscall=0x0) at rpc/virnetclient.c:1287 #9 0x00007f4bd96467a2 in virNetClientCloseInternal (reason=3, client=0x7f4bb4009e60) at rpc/virnetclient.c:589 #10 virNetClientCloseInternal (client=0x7f4bb4009e60, reason=3) at rpc/virnetclient.c:561 #11 0x00007f4bcacc4a82 in virLXCMonitorClose (mon=0x7f4bb4000a00) at lxc/lxc_monitor.c:201 #12 0x00007f4bcacc55ac in virLXCProcessCleanup (reason=<optimized out>, vm=0x7f4bb4000b00, driver=0x7f4bc40c8290) at lxc/lxc_process.c:240 #13 virLXCProcessStop (driver=0x7f4bc40c8290, vm=vm@entry=0x7f4bb4000b00, reason=reason@entry=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:735 #14 0x00007f4bcacc5bd2 in virLXCProcessAutoDestroyDom (payload=<optimized out>, name=0x7f4bb4003c80, opaque=0x7fff41af2df0) at lxc/lxc_process.c:94 #15 0x00007f4bd9586649 in virHashForEach (table=0x7f4bc409b270, iter=iter@entry=0x7f4bcacc5ab0 <virLXCProcessAutoDestroyDom>, data=data@entry=0x7fff41af2df0) at util/virhash.c:514 #16 0x00007f4bcacc52d7 in virLXCProcessAutoDestroyRun (driver=driver@entry=0x7f4bc40c8290, conn=conn@entry=0x7f4bb8000ab0) at lxc/lxc_process.c:120 #17 0x00007f4bcacca628 in lxcClose (conn=0x7f4bb8000ab0) at lxc/lxc_driver.c:128 #18 0x00007f4bd95e67ab in virReleaseConnect (conn=conn@entry=0x7f4bb8000ab0) at datatypes.c:114 When the driver calls virLXCMonitorClose, there is really no need for the EOF callback to be invoked in this case, since the caller can easily handle events itself. In changing this, the monitor needs to take a deep copy of the callback list, not merely a reference. Also adds debug statements in various places to aid troubleshooting Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-09-26 14:08:20 +00:00
virLXCMonitorCallbacks cb;
virNetClientPtr client;
Run an RPC protocol over the LXC controller monitor This defines a new RPC protocol to be used between the LXC controller and the libvirtd LXC driver. There is only a single RPC message defined thus far, an asynchronous "EXIT" event that is emitted just before the LXC controller process exits. This provides the LXC driver with details about how the container shutdown - normally, or abnormally (crashed), thus allowing the driver to emit better libvirt events. Emitting the event in the LXC controller requires a few little tricks with the RPC service. Simply calling the virNetServiceClientSendMessage does not work, since this merely queues the message for asynchronous processing. In addition the main event loop is no longer running at the point the event is emitted, so no I/O is processed. Thus after invoking virNetServiceClientSendMessage it is necessary to mark the client as being in "delayed close" mode. Then the event loop is run again, until the client completes its close - this happens only after the queued message has been fully transmitted. The final complexity is that it is not safe to run virNetServerQuit() from the client close callback, since that is invoked from a context where the server is locked. Thus a zero-second timer is used to trigger shutdown of the event loop, causing the controller to finally exit. * src/Makefile.am: Add rules for generating RPC protocol files and dispatch methods * src/lxc/lxc_controller.c: Emit an RPC event immediately before exiting * src/lxc/lxc_domain.h: Record the shutdown reason given by the controller * src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register RPC program and event handler. Add callback to let driver receive EXIT event. * src/lxc/lxc_process.c: Use monitor exit event to decide what kind of domain event to emit * src/lxc/lxc_protocol.x: Define wire protocol for LXC controller monitor. Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-07-17 14:54:08 +00:00
virNetClientProgramPtr program;
};
static virClassPtr virLXCMonitorClass;
static void virLXCMonitorDispose(void *obj);
static int virLXCMonitorOnceInit(void)
{
if (!(virLXCMonitorClass = virClassNew(virClassForObjectLockable(),
"virLXCMonitor",
sizeof(virLXCMonitor),
virLXCMonitorDispose)))
return -1;
return 0;
}
VIR_ONCE_GLOBAL_INIT(virLXCMonitor)
Run an RPC protocol over the LXC controller monitor This defines a new RPC protocol to be used between the LXC controller and the libvirtd LXC driver. There is only a single RPC message defined thus far, an asynchronous "EXIT" event that is emitted just before the LXC controller process exits. This provides the LXC driver with details about how the container shutdown - normally, or abnormally (crashed), thus allowing the driver to emit better libvirt events. Emitting the event in the LXC controller requires a few little tricks with the RPC service. Simply calling the virNetServiceClientSendMessage does not work, since this merely queues the message for asynchronous processing. In addition the main event loop is no longer running at the point the event is emitted, so no I/O is processed. Thus after invoking virNetServiceClientSendMessage it is necessary to mark the client as being in "delayed close" mode. Then the event loop is run again, until the client completes its close - this happens only after the queued message has been fully transmitted. The final complexity is that it is not safe to run virNetServerQuit() from the client close callback, since that is invoked from a context where the server is locked. Thus a zero-second timer is used to trigger shutdown of the event loop, causing the controller to finally exit. * src/Makefile.am: Add rules for generating RPC protocol files and dispatch methods * src/lxc/lxc_controller.c: Emit an RPC event immediately before exiting * src/lxc/lxc_domain.h: Record the shutdown reason given by the controller * src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register RPC program and event handler. Add callback to let driver receive EXIT event. * src/lxc/lxc_process.c: Use monitor exit event to decide what kind of domain event to emit * src/lxc/lxc_protocol.x: Define wire protocol for LXC controller monitor. Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-07-17 14:54:08 +00:00
static void
virLXCMonitorHandleEventExit(virNetClientProgramPtr prog,
virNetClientPtr client,
void *evdata, void *opaque);
static void
virLXCMonitorHandleEventInit(virNetClientProgramPtr prog,
virNetClientPtr client,
void *evdata, void *opaque);
Run an RPC protocol over the LXC controller monitor This defines a new RPC protocol to be used between the LXC controller and the libvirtd LXC driver. There is only a single RPC message defined thus far, an asynchronous "EXIT" event that is emitted just before the LXC controller process exits. This provides the LXC driver with details about how the container shutdown - normally, or abnormally (crashed), thus allowing the driver to emit better libvirt events. Emitting the event in the LXC controller requires a few little tricks with the RPC service. Simply calling the virNetServiceClientSendMessage does not work, since this merely queues the message for asynchronous processing. In addition the main event loop is no longer running at the point the event is emitted, so no I/O is processed. Thus after invoking virNetServiceClientSendMessage it is necessary to mark the client as being in "delayed close" mode. Then the event loop is run again, until the client completes its close - this happens only after the queued message has been fully transmitted. The final complexity is that it is not safe to run virNetServerQuit() from the client close callback, since that is invoked from a context where the server is locked. Thus a zero-second timer is used to trigger shutdown of the event loop, causing the controller to finally exit. * src/Makefile.am: Add rules for generating RPC protocol files and dispatch methods * src/lxc/lxc_controller.c: Emit an RPC event immediately before exiting * src/lxc/lxc_domain.h: Record the shutdown reason given by the controller * src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register RPC program and event handler. Add callback to let driver receive EXIT event. * src/lxc/lxc_process.c: Use monitor exit event to decide what kind of domain event to emit * src/lxc/lxc_protocol.x: Define wire protocol for LXC controller monitor. Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-07-17 14:54:08 +00:00
static virNetClientProgramEvent virLXCMonitorEvents[] = {
{ VIR_LXC_MONITOR_PROC_EXIT_EVENT,
Run an RPC protocol over the LXC controller monitor This defines a new RPC protocol to be used between the LXC controller and the libvirtd LXC driver. There is only a single RPC message defined thus far, an asynchronous "EXIT" event that is emitted just before the LXC controller process exits. This provides the LXC driver with details about how the container shutdown - normally, or abnormally (crashed), thus allowing the driver to emit better libvirt events. Emitting the event in the LXC controller requires a few little tricks with the RPC service. Simply calling the virNetServiceClientSendMessage does not work, since this merely queues the message for asynchronous processing. In addition the main event loop is no longer running at the point the event is emitted, so no I/O is processed. Thus after invoking virNetServiceClientSendMessage it is necessary to mark the client as being in "delayed close" mode. Then the event loop is run again, until the client completes its close - this happens only after the queued message has been fully transmitted. The final complexity is that it is not safe to run virNetServerQuit() from the client close callback, since that is invoked from a context where the server is locked. Thus a zero-second timer is used to trigger shutdown of the event loop, causing the controller to finally exit. * src/Makefile.am: Add rules for generating RPC protocol files and dispatch methods * src/lxc/lxc_controller.c: Emit an RPC event immediately before exiting * src/lxc/lxc_domain.h: Record the shutdown reason given by the controller * src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register RPC program and event handler. Add callback to let driver receive EXIT event. * src/lxc/lxc_process.c: Use monitor exit event to decide what kind of domain event to emit * src/lxc/lxc_protocol.x: Define wire protocol for LXC controller monitor. Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-07-17 14:54:08 +00:00
virLXCMonitorHandleEventExit,
sizeof(virLXCMonitorExitEventMsg),
(xdrproc_t)xdr_virLXCMonitorExitEventMsg },
{ VIR_LXC_MONITOR_PROC_INIT_EVENT,
virLXCMonitorHandleEventInit,
sizeof(virLXCMonitorInitEventMsg),
(xdrproc_t)xdr_virLXCMonitorInitEventMsg },
Run an RPC protocol over the LXC controller monitor This defines a new RPC protocol to be used between the LXC controller and the libvirtd LXC driver. There is only a single RPC message defined thus far, an asynchronous "EXIT" event that is emitted just before the LXC controller process exits. This provides the LXC driver with details about how the container shutdown - normally, or abnormally (crashed), thus allowing the driver to emit better libvirt events. Emitting the event in the LXC controller requires a few little tricks with the RPC service. Simply calling the virNetServiceClientSendMessage does not work, since this merely queues the message for asynchronous processing. In addition the main event loop is no longer running at the point the event is emitted, so no I/O is processed. Thus after invoking virNetServiceClientSendMessage it is necessary to mark the client as being in "delayed close" mode. Then the event loop is run again, until the client completes its close - this happens only after the queued message has been fully transmitted. The final complexity is that it is not safe to run virNetServerQuit() from the client close callback, since that is invoked from a context where the server is locked. Thus a zero-second timer is used to trigger shutdown of the event loop, causing the controller to finally exit. * src/Makefile.am: Add rules for generating RPC protocol files and dispatch methods * src/lxc/lxc_controller.c: Emit an RPC event immediately before exiting * src/lxc/lxc_domain.h: Record the shutdown reason given by the controller * src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register RPC program and event handler. Add callback to let driver receive EXIT event. * src/lxc/lxc_process.c: Use monitor exit event to decide what kind of domain event to emit * src/lxc/lxc_protocol.x: Define wire protocol for LXC controller monitor. Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-07-17 14:54:08 +00:00
};
static void
virLXCMonitorHandleEventExit(virNetClientProgramPtr prog ATTRIBUTE_UNUSED,
virNetClientPtr client ATTRIBUTE_UNUSED,
void *evdata, void *opaque)
{
virLXCMonitorPtr mon = opaque;
virLXCMonitorExitEventMsg *msg = evdata;
Run an RPC protocol over the LXC controller monitor This defines a new RPC protocol to be used between the LXC controller and the libvirtd LXC driver. There is only a single RPC message defined thus far, an asynchronous "EXIT" event that is emitted just before the LXC controller process exits. This provides the LXC driver with details about how the container shutdown - normally, or abnormally (crashed), thus allowing the driver to emit better libvirt events. Emitting the event in the LXC controller requires a few little tricks with the RPC service. Simply calling the virNetServiceClientSendMessage does not work, since this merely queues the message for asynchronous processing. In addition the main event loop is no longer running at the point the event is emitted, so no I/O is processed. Thus after invoking virNetServiceClientSendMessage it is necessary to mark the client as being in "delayed close" mode. Then the event loop is run again, until the client completes its close - this happens only after the queued message has been fully transmitted. The final complexity is that it is not safe to run virNetServerQuit() from the client close callback, since that is invoked from a context where the server is locked. Thus a zero-second timer is used to trigger shutdown of the event loop, causing the controller to finally exit. * src/Makefile.am: Add rules for generating RPC protocol files and dispatch methods * src/lxc/lxc_controller.c: Emit an RPC event immediately before exiting * src/lxc/lxc_domain.h: Record the shutdown reason given by the controller * src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register RPC program and event handler. Add callback to let driver receive EXIT event. * src/lxc/lxc_process.c: Use monitor exit event to decide what kind of domain event to emit * src/lxc/lxc_protocol.x: Define wire protocol for LXC controller monitor. Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-07-17 14:54:08 +00:00
VIR_DEBUG("Event exit %d", msg->status);
Fix deadlock in handling EOF in LXC monitor Depending on the scenario in which LXC containers exit, it is possible for the EOF callback of the LXC monitor to deadlock the driver. #0 0x00000038a0a0de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00000038a0a09ca6 in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00000038a0a09ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007f4bd9579d55 in virMutexLock (m=<optimized out>) at util/threads-pthread.c:85 #4 0x00007f4bcacc7597 in lxcDriverLock (driver=0x7f4bc40c8290) at lxc/lxc_conf.h:81 #5 virLXCProcessMonitorEOFNotify (mon=<optimized out>, vm=0x7f4bb4000b00) at lxc/lxc_process.c:581 #6 0x00007f4bd9645c91 in virNetClientCloseLocked (client=client@entry=0x7f4bb4009e60) at rpc/virnetclient.c:554 #7 0x00007f4bd96460f8 in virNetClientIOEventLoopPassTheBuck (thiscall=0x0, client=0x7f4bb4009e60) at rpc/virnetclient.c:1306 #8 virNetClientIOEventLoopPassTheBuck (client=0x7f4bb4009e60, thiscall=0x0) at rpc/virnetclient.c:1287 #9 0x00007f4bd96467a2 in virNetClientCloseInternal (reason=3, client=0x7f4bb4009e60) at rpc/virnetclient.c:589 #10 virNetClientCloseInternal (client=0x7f4bb4009e60, reason=3) at rpc/virnetclient.c:561 #11 0x00007f4bcacc4a82 in virLXCMonitorClose (mon=0x7f4bb4000a00) at lxc/lxc_monitor.c:201 #12 0x00007f4bcacc55ac in virLXCProcessCleanup (reason=<optimized out>, vm=0x7f4bb4000b00, driver=0x7f4bc40c8290) at lxc/lxc_process.c:240 #13 virLXCProcessStop (driver=0x7f4bc40c8290, vm=vm@entry=0x7f4bb4000b00, reason=reason@entry=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:735 #14 0x00007f4bcacc5bd2 in virLXCProcessAutoDestroyDom (payload=<optimized out>, name=0x7f4bb4003c80, opaque=0x7fff41af2df0) at lxc/lxc_process.c:94 #15 0x00007f4bd9586649 in virHashForEach (table=0x7f4bc409b270, iter=iter@entry=0x7f4bcacc5ab0 <virLXCProcessAutoDestroyDom>, data=data@entry=0x7fff41af2df0) at util/virhash.c:514 #16 0x00007f4bcacc52d7 in virLXCProcessAutoDestroyRun (driver=driver@entry=0x7f4bc40c8290, conn=conn@entry=0x7f4bb8000ab0) at lxc/lxc_process.c:120 #17 0x00007f4bcacca628 in lxcClose (conn=0x7f4bb8000ab0) at lxc/lxc_driver.c:128 #18 0x00007f4bd95e67ab in virReleaseConnect (conn=conn@entry=0x7f4bb8000ab0) at datatypes.c:114 When the driver calls virLXCMonitorClose, there is really no need for the EOF callback to be invoked in this case, since the caller can easily handle events itself. In changing this, the monitor needs to take a deep copy of the callback list, not merely a reference. Also adds debug statements in various places to aid troubleshooting Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-09-26 14:08:20 +00:00
if (mon->cb.exitNotify)
mon->cb.exitNotify(mon, msg->status, mon->vm);
Run an RPC protocol over the LXC controller monitor This defines a new RPC protocol to be used between the LXC controller and the libvirtd LXC driver. There is only a single RPC message defined thus far, an asynchronous "EXIT" event that is emitted just before the LXC controller process exits. This provides the LXC driver with details about how the container shutdown - normally, or abnormally (crashed), thus allowing the driver to emit better libvirt events. Emitting the event in the LXC controller requires a few little tricks with the RPC service. Simply calling the virNetServiceClientSendMessage does not work, since this merely queues the message for asynchronous processing. In addition the main event loop is no longer running at the point the event is emitted, so no I/O is processed. Thus after invoking virNetServiceClientSendMessage it is necessary to mark the client as being in "delayed close" mode. Then the event loop is run again, until the client completes its close - this happens only after the queued message has been fully transmitted. The final complexity is that it is not safe to run virNetServerQuit() from the client close callback, since that is invoked from a context where the server is locked. Thus a zero-second timer is used to trigger shutdown of the event loop, causing the controller to finally exit. * src/Makefile.am: Add rules for generating RPC protocol files and dispatch methods * src/lxc/lxc_controller.c: Emit an RPC event immediately before exiting * src/lxc/lxc_domain.h: Record the shutdown reason given by the controller * src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register RPC program and event handler. Add callback to let driver receive EXIT event. * src/lxc/lxc_process.c: Use monitor exit event to decide what kind of domain event to emit * src/lxc/lxc_protocol.x: Define wire protocol for LXC controller monitor. Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-07-17 14:54:08 +00:00
}
static void
virLXCMonitorHandleEventInit(virNetClientProgramPtr prog ATTRIBUTE_UNUSED,
virNetClientPtr client ATTRIBUTE_UNUSED,
void *evdata, void *opaque)
{
virLXCMonitorPtr mon = opaque;
virLXCMonitorInitEventMsg *msg = evdata;
VIR_DEBUG("Event init %llu",
(unsigned long long)msg->initpid);
if (mon->cb.initNotify)
mon->cb.initNotify(mon, (pid_t)msg->initpid, mon->vm);
}
static void virLXCMonitorEOFNotify(virNetClientPtr client ATTRIBUTE_UNUSED,
int reason ATTRIBUTE_UNUSED,
void *opaque)
{
virLXCMonitorPtr mon = opaque;
virLXCMonitorCallbackEOFNotify eofNotify;
virDomainObjPtr vm;
Fix deadlock in handling EOF in LXC monitor Depending on the scenario in which LXC containers exit, it is possible for the EOF callback of the LXC monitor to deadlock the driver. #0 0x00000038a0a0de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00000038a0a09ca6 in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00000038a0a09ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007f4bd9579d55 in virMutexLock (m=<optimized out>) at util/threads-pthread.c:85 #4 0x00007f4bcacc7597 in lxcDriverLock (driver=0x7f4bc40c8290) at lxc/lxc_conf.h:81 #5 virLXCProcessMonitorEOFNotify (mon=<optimized out>, vm=0x7f4bb4000b00) at lxc/lxc_process.c:581 #6 0x00007f4bd9645c91 in virNetClientCloseLocked (client=client@entry=0x7f4bb4009e60) at rpc/virnetclient.c:554 #7 0x00007f4bd96460f8 in virNetClientIOEventLoopPassTheBuck (thiscall=0x0, client=0x7f4bb4009e60) at rpc/virnetclient.c:1306 #8 virNetClientIOEventLoopPassTheBuck (client=0x7f4bb4009e60, thiscall=0x0) at rpc/virnetclient.c:1287 #9 0x00007f4bd96467a2 in virNetClientCloseInternal (reason=3, client=0x7f4bb4009e60) at rpc/virnetclient.c:589 #10 virNetClientCloseInternal (client=0x7f4bb4009e60, reason=3) at rpc/virnetclient.c:561 #11 0x00007f4bcacc4a82 in virLXCMonitorClose (mon=0x7f4bb4000a00) at lxc/lxc_monitor.c:201 #12 0x00007f4bcacc55ac in virLXCProcessCleanup (reason=<optimized out>, vm=0x7f4bb4000b00, driver=0x7f4bc40c8290) at lxc/lxc_process.c:240 #13 virLXCProcessStop (driver=0x7f4bc40c8290, vm=vm@entry=0x7f4bb4000b00, reason=reason@entry=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:735 #14 0x00007f4bcacc5bd2 in virLXCProcessAutoDestroyDom (payload=<optimized out>, name=0x7f4bb4003c80, opaque=0x7fff41af2df0) at lxc/lxc_process.c:94 #15 0x00007f4bd9586649 in virHashForEach (table=0x7f4bc409b270, iter=iter@entry=0x7f4bcacc5ab0 <virLXCProcessAutoDestroyDom>, data=data@entry=0x7fff41af2df0) at util/virhash.c:514 #16 0x00007f4bcacc52d7 in virLXCProcessAutoDestroyRun (driver=driver@entry=0x7f4bc40c8290, conn=conn@entry=0x7f4bb8000ab0) at lxc/lxc_process.c:120 #17 0x00007f4bcacca628 in lxcClose (conn=0x7f4bb8000ab0) at lxc/lxc_driver.c:128 #18 0x00007f4bd95e67ab in virReleaseConnect (conn=conn@entry=0x7f4bb8000ab0) at datatypes.c:114 When the driver calls virLXCMonitorClose, there is really no need for the EOF callback to be invoked in this case, since the caller can easily handle events itself. In changing this, the monitor needs to take a deep copy of the callback list, not merely a reference. Also adds debug statements in various places to aid troubleshooting Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-09-26 14:08:20 +00:00
VIR_DEBUG("EOF notify mon=%p", mon);
virObjectLock(mon);
Fix deadlock in handling EOF in LXC monitor Depending on the scenario in which LXC containers exit, it is possible for the EOF callback of the LXC monitor to deadlock the driver. #0 0x00000038a0a0de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00000038a0a09ca6 in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00000038a0a09ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007f4bd9579d55 in virMutexLock (m=<optimized out>) at util/threads-pthread.c:85 #4 0x00007f4bcacc7597 in lxcDriverLock (driver=0x7f4bc40c8290) at lxc/lxc_conf.h:81 #5 virLXCProcessMonitorEOFNotify (mon=<optimized out>, vm=0x7f4bb4000b00) at lxc/lxc_process.c:581 #6 0x00007f4bd9645c91 in virNetClientCloseLocked (client=client@entry=0x7f4bb4009e60) at rpc/virnetclient.c:554 #7 0x00007f4bd96460f8 in virNetClientIOEventLoopPassTheBuck (thiscall=0x0, client=0x7f4bb4009e60) at rpc/virnetclient.c:1306 #8 virNetClientIOEventLoopPassTheBuck (client=0x7f4bb4009e60, thiscall=0x0) at rpc/virnetclient.c:1287 #9 0x00007f4bd96467a2 in virNetClientCloseInternal (reason=3, client=0x7f4bb4009e60) at rpc/virnetclient.c:589 #10 virNetClientCloseInternal (client=0x7f4bb4009e60, reason=3) at rpc/virnetclient.c:561 #11 0x00007f4bcacc4a82 in virLXCMonitorClose (mon=0x7f4bb4000a00) at lxc/lxc_monitor.c:201 #12 0x00007f4bcacc55ac in virLXCProcessCleanup (reason=<optimized out>, vm=0x7f4bb4000b00, driver=0x7f4bc40c8290) at lxc/lxc_process.c:240 #13 virLXCProcessStop (driver=0x7f4bc40c8290, vm=vm@entry=0x7f4bb4000b00, reason=reason@entry=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:735 #14 0x00007f4bcacc5bd2 in virLXCProcessAutoDestroyDom (payload=<optimized out>, name=0x7f4bb4003c80, opaque=0x7fff41af2df0) at lxc/lxc_process.c:94 #15 0x00007f4bd9586649 in virHashForEach (table=0x7f4bc409b270, iter=iter@entry=0x7f4bcacc5ab0 <virLXCProcessAutoDestroyDom>, data=data@entry=0x7fff41af2df0) at util/virhash.c:514 #16 0x00007f4bcacc52d7 in virLXCProcessAutoDestroyRun (driver=driver@entry=0x7f4bc40c8290, conn=conn@entry=0x7f4bb8000ab0) at lxc/lxc_process.c:120 #17 0x00007f4bcacca628 in lxcClose (conn=0x7f4bb8000ab0) at lxc/lxc_driver.c:128 #18 0x00007f4bd95e67ab in virReleaseConnect (conn=conn@entry=0x7f4bb8000ab0) at datatypes.c:114 When the driver calls virLXCMonitorClose, there is really no need for the EOF callback to be invoked in this case, since the caller can easily handle events itself. In changing this, the monitor needs to take a deep copy of the callback list, not merely a reference. Also adds debug statements in various places to aid troubleshooting Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-09-26 14:08:20 +00:00
eofNotify = mon->cb.eofNotify;
vm = mon->vm;
virObjectUnlock(mon);
Fix deadlock in handling EOF in LXC monitor Depending on the scenario in which LXC containers exit, it is possible for the EOF callback of the LXC monitor to deadlock the driver. #0 0x00000038a0a0de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00000038a0a09ca6 in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00000038a0a09ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007f4bd9579d55 in virMutexLock (m=<optimized out>) at util/threads-pthread.c:85 #4 0x00007f4bcacc7597 in lxcDriverLock (driver=0x7f4bc40c8290) at lxc/lxc_conf.h:81 #5 virLXCProcessMonitorEOFNotify (mon=<optimized out>, vm=0x7f4bb4000b00) at lxc/lxc_process.c:581 #6 0x00007f4bd9645c91 in virNetClientCloseLocked (client=client@entry=0x7f4bb4009e60) at rpc/virnetclient.c:554 #7 0x00007f4bd96460f8 in virNetClientIOEventLoopPassTheBuck (thiscall=0x0, client=0x7f4bb4009e60) at rpc/virnetclient.c:1306 #8 virNetClientIOEventLoopPassTheBuck (client=0x7f4bb4009e60, thiscall=0x0) at rpc/virnetclient.c:1287 #9 0x00007f4bd96467a2 in virNetClientCloseInternal (reason=3, client=0x7f4bb4009e60) at rpc/virnetclient.c:589 #10 virNetClientCloseInternal (client=0x7f4bb4009e60, reason=3) at rpc/virnetclient.c:561 #11 0x00007f4bcacc4a82 in virLXCMonitorClose (mon=0x7f4bb4000a00) at lxc/lxc_monitor.c:201 #12 0x00007f4bcacc55ac in virLXCProcessCleanup (reason=<optimized out>, vm=0x7f4bb4000b00, driver=0x7f4bc40c8290) at lxc/lxc_process.c:240 #13 virLXCProcessStop (driver=0x7f4bc40c8290, vm=vm@entry=0x7f4bb4000b00, reason=reason@entry=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:735 #14 0x00007f4bcacc5bd2 in virLXCProcessAutoDestroyDom (payload=<optimized out>, name=0x7f4bb4003c80, opaque=0x7fff41af2df0) at lxc/lxc_process.c:94 #15 0x00007f4bd9586649 in virHashForEach (table=0x7f4bc409b270, iter=iter@entry=0x7f4bcacc5ab0 <virLXCProcessAutoDestroyDom>, data=data@entry=0x7fff41af2df0) at util/virhash.c:514 #16 0x00007f4bcacc52d7 in virLXCProcessAutoDestroyRun (driver=driver@entry=0x7f4bc40c8290, conn=conn@entry=0x7f4bb8000ab0) at lxc/lxc_process.c:120 #17 0x00007f4bcacca628 in lxcClose (conn=0x7f4bb8000ab0) at lxc/lxc_driver.c:128 #18 0x00007f4bd95e67ab in virReleaseConnect (conn=conn@entry=0x7f4bb8000ab0) at datatypes.c:114 When the driver calls virLXCMonitorClose, there is really no need for the EOF callback to be invoked in this case, since the caller can easily handle events itself. In changing this, the monitor needs to take a deep copy of the callback list, not merely a reference. Also adds debug statements in various places to aid troubleshooting Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-09-26 14:08:20 +00:00
if (eofNotify) {
VIR_DEBUG("EOF callback mon=%p vm=%p", mon, vm);
eofNotify(mon, vm);
} else {
VIR_DEBUG("No EOF callback");
}
}
static void virLXCMonitorCloseFreeCallback(void *opaque)
{
virLXCMonitorPtr mon = opaque;
Fix deadlock in handling EOF in LXC monitor Depending on the scenario in which LXC containers exit, it is possible for the EOF callback of the LXC monitor to deadlock the driver. #0 0x00000038a0a0de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00000038a0a09ca6 in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00000038a0a09ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007f4bd9579d55 in virMutexLock (m=<optimized out>) at util/threads-pthread.c:85 #4 0x00007f4bcacc7597 in lxcDriverLock (driver=0x7f4bc40c8290) at lxc/lxc_conf.h:81 #5 virLXCProcessMonitorEOFNotify (mon=<optimized out>, vm=0x7f4bb4000b00) at lxc/lxc_process.c:581 #6 0x00007f4bd9645c91 in virNetClientCloseLocked (client=client@entry=0x7f4bb4009e60) at rpc/virnetclient.c:554 #7 0x00007f4bd96460f8 in virNetClientIOEventLoopPassTheBuck (thiscall=0x0, client=0x7f4bb4009e60) at rpc/virnetclient.c:1306 #8 virNetClientIOEventLoopPassTheBuck (client=0x7f4bb4009e60, thiscall=0x0) at rpc/virnetclient.c:1287 #9 0x00007f4bd96467a2 in virNetClientCloseInternal (reason=3, client=0x7f4bb4009e60) at rpc/virnetclient.c:589 #10 virNetClientCloseInternal (client=0x7f4bb4009e60, reason=3) at rpc/virnetclient.c:561 #11 0x00007f4bcacc4a82 in virLXCMonitorClose (mon=0x7f4bb4000a00) at lxc/lxc_monitor.c:201 #12 0x00007f4bcacc55ac in virLXCProcessCleanup (reason=<optimized out>, vm=0x7f4bb4000b00, driver=0x7f4bc40c8290) at lxc/lxc_process.c:240 #13 virLXCProcessStop (driver=0x7f4bc40c8290, vm=vm@entry=0x7f4bb4000b00, reason=reason@entry=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:735 #14 0x00007f4bcacc5bd2 in virLXCProcessAutoDestroyDom (payload=<optimized out>, name=0x7f4bb4003c80, opaque=0x7fff41af2df0) at lxc/lxc_process.c:94 #15 0x00007f4bd9586649 in virHashForEach (table=0x7f4bc409b270, iter=iter@entry=0x7f4bcacc5ab0 <virLXCProcessAutoDestroyDom>, data=data@entry=0x7fff41af2df0) at util/virhash.c:514 #16 0x00007f4bcacc52d7 in virLXCProcessAutoDestroyRun (driver=driver@entry=0x7f4bc40c8290, conn=conn@entry=0x7f4bb8000ab0) at lxc/lxc_process.c:120 #17 0x00007f4bcacca628 in lxcClose (conn=0x7f4bb8000ab0) at lxc/lxc_driver.c:128 #18 0x00007f4bd95e67ab in virReleaseConnect (conn=conn@entry=0x7f4bb8000ab0) at datatypes.c:114 When the driver calls virLXCMonitorClose, there is really no need for the EOF callback to be invoked in this case, since the caller can easily handle events itself. In changing this, the monitor needs to take a deep copy of the callback list, not merely a reference. Also adds debug statements in various places to aid troubleshooting Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-09-26 14:08:20 +00:00
virObjectUnref(mon);
}
virLXCMonitorPtr virLXCMonitorNew(virDomainObjPtr vm,
const char *socketdir,
virLXCMonitorCallbacksPtr cb)
{
virLXCMonitorPtr mon;
char *sockpath = NULL;
if (virLXCMonitorInitialize() < 0)
return NULL;
if (!(mon = virObjectLockableNew(virLXCMonitorClass)))
return NULL;
if (virAsprintf(&sockpath, "%s/%s.sock",
socketdir, vm->def->name) < 0)
goto error;
if (!(mon->client = virNetClientNewUNIX(sockpath, false, NULL)))
goto error;
if (virNetClientRegisterAsyncIO(mon->client) < 0)
goto error;
if (!(mon->program = virNetClientProgramNew(VIR_LXC_MONITOR_PROGRAM,
VIR_LXC_MONITOR_PROGRAM_VERSION,
virLXCMonitorEvents,
ARRAY_CARDINALITY(virLXCMonitorEvents),
Run an RPC protocol over the LXC controller monitor This defines a new RPC protocol to be used between the LXC controller and the libvirtd LXC driver. There is only a single RPC message defined thus far, an asynchronous "EXIT" event that is emitted just before the LXC controller process exits. This provides the LXC driver with details about how the container shutdown - normally, or abnormally (crashed), thus allowing the driver to emit better libvirt events. Emitting the event in the LXC controller requires a few little tricks with the RPC service. Simply calling the virNetServiceClientSendMessage does not work, since this merely queues the message for asynchronous processing. In addition the main event loop is no longer running at the point the event is emitted, so no I/O is processed. Thus after invoking virNetServiceClientSendMessage it is necessary to mark the client as being in "delayed close" mode. Then the event loop is run again, until the client completes its close - this happens only after the queued message has been fully transmitted. The final complexity is that it is not safe to run virNetServerQuit() from the client close callback, since that is invoked from a context where the server is locked. Thus a zero-second timer is used to trigger shutdown of the event loop, causing the controller to finally exit. * src/Makefile.am: Add rules for generating RPC protocol files and dispatch methods * src/lxc/lxc_controller.c: Emit an RPC event immediately before exiting * src/lxc/lxc_domain.h: Record the shutdown reason given by the controller * src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register RPC program and event handler. Add callback to let driver receive EXIT event. * src/lxc/lxc_process.c: Use monitor exit event to decide what kind of domain event to emit * src/lxc/lxc_protocol.x: Define wire protocol for LXC controller monitor. Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-07-17 14:54:08 +00:00
mon)))
goto error;
if (virNetClientAddProgram(mon->client,
mon->program) < 0)
goto error;
mon->vm = vm;
Fix deadlock in handling EOF in LXC monitor Depending on the scenario in which LXC containers exit, it is possible for the EOF callback of the LXC monitor to deadlock the driver. #0 0x00000038a0a0de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00000038a0a09ca6 in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00000038a0a09ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007f4bd9579d55 in virMutexLock (m=<optimized out>) at util/threads-pthread.c:85 #4 0x00007f4bcacc7597 in lxcDriverLock (driver=0x7f4bc40c8290) at lxc/lxc_conf.h:81 #5 virLXCProcessMonitorEOFNotify (mon=<optimized out>, vm=0x7f4bb4000b00) at lxc/lxc_process.c:581 #6 0x00007f4bd9645c91 in virNetClientCloseLocked (client=client@entry=0x7f4bb4009e60) at rpc/virnetclient.c:554 #7 0x00007f4bd96460f8 in virNetClientIOEventLoopPassTheBuck (thiscall=0x0, client=0x7f4bb4009e60) at rpc/virnetclient.c:1306 #8 virNetClientIOEventLoopPassTheBuck (client=0x7f4bb4009e60, thiscall=0x0) at rpc/virnetclient.c:1287 #9 0x00007f4bd96467a2 in virNetClientCloseInternal (reason=3, client=0x7f4bb4009e60) at rpc/virnetclient.c:589 #10 virNetClientCloseInternal (client=0x7f4bb4009e60, reason=3) at rpc/virnetclient.c:561 #11 0x00007f4bcacc4a82 in virLXCMonitorClose (mon=0x7f4bb4000a00) at lxc/lxc_monitor.c:201 #12 0x00007f4bcacc55ac in virLXCProcessCleanup (reason=<optimized out>, vm=0x7f4bb4000b00, driver=0x7f4bc40c8290) at lxc/lxc_process.c:240 #13 virLXCProcessStop (driver=0x7f4bc40c8290, vm=vm@entry=0x7f4bb4000b00, reason=reason@entry=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:735 #14 0x00007f4bcacc5bd2 in virLXCProcessAutoDestroyDom (payload=<optimized out>, name=0x7f4bb4003c80, opaque=0x7fff41af2df0) at lxc/lxc_process.c:94 #15 0x00007f4bd9586649 in virHashForEach (table=0x7f4bc409b270, iter=iter@entry=0x7f4bcacc5ab0 <virLXCProcessAutoDestroyDom>, data=data@entry=0x7fff41af2df0) at util/virhash.c:514 #16 0x00007f4bcacc52d7 in virLXCProcessAutoDestroyRun (driver=driver@entry=0x7f4bc40c8290, conn=conn@entry=0x7f4bb8000ab0) at lxc/lxc_process.c:120 #17 0x00007f4bcacca628 in lxcClose (conn=0x7f4bb8000ab0) at lxc/lxc_driver.c:128 #18 0x00007f4bd95e67ab in virReleaseConnect (conn=conn@entry=0x7f4bb8000ab0) at datatypes.c:114 When the driver calls virLXCMonitorClose, there is really no need for the EOF callback to be invoked in this case, since the caller can easily handle events itself. In changing this, the monitor needs to take a deep copy of the callback list, not merely a reference. Also adds debug statements in various places to aid troubleshooting Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-09-26 14:08:20 +00:00
memcpy(&mon->cb, cb, sizeof(mon->cb));
virObjectRef(mon);
virNetClientSetCloseCallback(mon->client, virLXCMonitorEOFNotify, mon,
virLXCMonitorCloseFreeCallback);
cleanup:
VIR_FREE(sockpath);
return mon;
error:
virObjectUnref(mon);
mon = NULL;
goto cleanup;
}
static void virLXCMonitorDispose(void *opaque)
{
virLXCMonitorPtr mon = opaque;
VIR_DEBUG("mon=%p", mon);
Fix deadlock in handling EOF in LXC monitor Depending on the scenario in which LXC containers exit, it is possible for the EOF callback of the LXC monitor to deadlock the driver. #0 0x00000038a0a0de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00000038a0a09ca6 in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00000038a0a09ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007f4bd9579d55 in virMutexLock (m=<optimized out>) at util/threads-pthread.c:85 #4 0x00007f4bcacc7597 in lxcDriverLock (driver=0x7f4bc40c8290) at lxc/lxc_conf.h:81 #5 virLXCProcessMonitorEOFNotify (mon=<optimized out>, vm=0x7f4bb4000b00) at lxc/lxc_process.c:581 #6 0x00007f4bd9645c91 in virNetClientCloseLocked (client=client@entry=0x7f4bb4009e60) at rpc/virnetclient.c:554 #7 0x00007f4bd96460f8 in virNetClientIOEventLoopPassTheBuck (thiscall=0x0, client=0x7f4bb4009e60) at rpc/virnetclient.c:1306 #8 virNetClientIOEventLoopPassTheBuck (client=0x7f4bb4009e60, thiscall=0x0) at rpc/virnetclient.c:1287 #9 0x00007f4bd96467a2 in virNetClientCloseInternal (reason=3, client=0x7f4bb4009e60) at rpc/virnetclient.c:589 #10 virNetClientCloseInternal (client=0x7f4bb4009e60, reason=3) at rpc/virnetclient.c:561 #11 0x00007f4bcacc4a82 in virLXCMonitorClose (mon=0x7f4bb4000a00) at lxc/lxc_monitor.c:201 #12 0x00007f4bcacc55ac in virLXCProcessCleanup (reason=<optimized out>, vm=0x7f4bb4000b00, driver=0x7f4bc40c8290) at lxc/lxc_process.c:240 #13 virLXCProcessStop (driver=0x7f4bc40c8290, vm=vm@entry=0x7f4bb4000b00, reason=reason@entry=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:735 #14 0x00007f4bcacc5bd2 in virLXCProcessAutoDestroyDom (payload=<optimized out>, name=0x7f4bb4003c80, opaque=0x7fff41af2df0) at lxc/lxc_process.c:94 #15 0x00007f4bd9586649 in virHashForEach (table=0x7f4bc409b270, iter=iter@entry=0x7f4bcacc5ab0 <virLXCProcessAutoDestroyDom>, data=data@entry=0x7fff41af2df0) at util/virhash.c:514 #16 0x00007f4bcacc52d7 in virLXCProcessAutoDestroyRun (driver=driver@entry=0x7f4bc40c8290, conn=conn@entry=0x7f4bb8000ab0) at lxc/lxc_process.c:120 #17 0x00007f4bcacca628 in lxcClose (conn=0x7f4bb8000ab0) at lxc/lxc_driver.c:128 #18 0x00007f4bd95e67ab in virReleaseConnect (conn=conn@entry=0x7f4bb8000ab0) at datatypes.c:114 When the driver calls virLXCMonitorClose, there is really no need for the EOF callback to be invoked in this case, since the caller can easily handle events itself. In changing this, the monitor needs to take a deep copy of the callback list, not merely a reference. Also adds debug statements in various places to aid troubleshooting Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-09-26 14:08:20 +00:00
if (mon->cb.destroy)
(mon->cb.destroy)(mon, mon->vm);
virObjectUnref(mon->program);
}
void virLXCMonitorClose(virLXCMonitorPtr mon)
{
Fix deadlock in handling EOF in LXC monitor Depending on the scenario in which LXC containers exit, it is possible for the EOF callback of the LXC monitor to deadlock the driver. #0 0x00000038a0a0de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00000038a0a09ca6 in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00000038a0a09ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007f4bd9579d55 in virMutexLock (m=<optimized out>) at util/threads-pthread.c:85 #4 0x00007f4bcacc7597 in lxcDriverLock (driver=0x7f4bc40c8290) at lxc/lxc_conf.h:81 #5 virLXCProcessMonitorEOFNotify (mon=<optimized out>, vm=0x7f4bb4000b00) at lxc/lxc_process.c:581 #6 0x00007f4bd9645c91 in virNetClientCloseLocked (client=client@entry=0x7f4bb4009e60) at rpc/virnetclient.c:554 #7 0x00007f4bd96460f8 in virNetClientIOEventLoopPassTheBuck (thiscall=0x0, client=0x7f4bb4009e60) at rpc/virnetclient.c:1306 #8 virNetClientIOEventLoopPassTheBuck (client=0x7f4bb4009e60, thiscall=0x0) at rpc/virnetclient.c:1287 #9 0x00007f4bd96467a2 in virNetClientCloseInternal (reason=3, client=0x7f4bb4009e60) at rpc/virnetclient.c:589 #10 virNetClientCloseInternal (client=0x7f4bb4009e60, reason=3) at rpc/virnetclient.c:561 #11 0x00007f4bcacc4a82 in virLXCMonitorClose (mon=0x7f4bb4000a00) at lxc/lxc_monitor.c:201 #12 0x00007f4bcacc55ac in virLXCProcessCleanup (reason=<optimized out>, vm=0x7f4bb4000b00, driver=0x7f4bc40c8290) at lxc/lxc_process.c:240 #13 virLXCProcessStop (driver=0x7f4bc40c8290, vm=vm@entry=0x7f4bb4000b00, reason=reason@entry=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:735 #14 0x00007f4bcacc5bd2 in virLXCProcessAutoDestroyDom (payload=<optimized out>, name=0x7f4bb4003c80, opaque=0x7fff41af2df0) at lxc/lxc_process.c:94 #15 0x00007f4bd9586649 in virHashForEach (table=0x7f4bc409b270, iter=iter@entry=0x7f4bcacc5ab0 <virLXCProcessAutoDestroyDom>, data=data@entry=0x7fff41af2df0) at util/virhash.c:514 #16 0x00007f4bcacc52d7 in virLXCProcessAutoDestroyRun (driver=driver@entry=0x7f4bc40c8290, conn=conn@entry=0x7f4bb8000ab0) at lxc/lxc_process.c:120 #17 0x00007f4bcacca628 in lxcClose (conn=0x7f4bb8000ab0) at lxc/lxc_driver.c:128 #18 0x00007f4bd95e67ab in virReleaseConnect (conn=conn@entry=0x7f4bb8000ab0) at datatypes.c:114 When the driver calls virLXCMonitorClose, there is really no need for the EOF callback to be invoked in this case, since the caller can easily handle events itself. In changing this, the monitor needs to take a deep copy of the callback list, not merely a reference. Also adds debug statements in various places to aid troubleshooting Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-09-26 14:08:20 +00:00
VIR_DEBUG("mon=%p", mon);
if (mon->client) {
Fix deadlock in handling EOF in LXC monitor Depending on the scenario in which LXC containers exit, it is possible for the EOF callback of the LXC monitor to deadlock the driver. #0 0x00000038a0a0de4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00000038a0a09ca6 in _L_lock_840 () from /lib64/libpthread.so.0 #2 0x00000038a0a09ba8 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007f4bd9579d55 in virMutexLock (m=<optimized out>) at util/threads-pthread.c:85 #4 0x00007f4bcacc7597 in lxcDriverLock (driver=0x7f4bc40c8290) at lxc/lxc_conf.h:81 #5 virLXCProcessMonitorEOFNotify (mon=<optimized out>, vm=0x7f4bb4000b00) at lxc/lxc_process.c:581 #6 0x00007f4bd9645c91 in virNetClientCloseLocked (client=client@entry=0x7f4bb4009e60) at rpc/virnetclient.c:554 #7 0x00007f4bd96460f8 in virNetClientIOEventLoopPassTheBuck (thiscall=0x0, client=0x7f4bb4009e60) at rpc/virnetclient.c:1306 #8 virNetClientIOEventLoopPassTheBuck (client=0x7f4bb4009e60, thiscall=0x0) at rpc/virnetclient.c:1287 #9 0x00007f4bd96467a2 in virNetClientCloseInternal (reason=3, client=0x7f4bb4009e60) at rpc/virnetclient.c:589 #10 virNetClientCloseInternal (client=0x7f4bb4009e60, reason=3) at rpc/virnetclient.c:561 #11 0x00007f4bcacc4a82 in virLXCMonitorClose (mon=0x7f4bb4000a00) at lxc/lxc_monitor.c:201 #12 0x00007f4bcacc55ac in virLXCProcessCleanup (reason=<optimized out>, vm=0x7f4bb4000b00, driver=0x7f4bc40c8290) at lxc/lxc_process.c:240 #13 virLXCProcessStop (driver=0x7f4bc40c8290, vm=vm@entry=0x7f4bb4000b00, reason=reason@entry=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:735 #14 0x00007f4bcacc5bd2 in virLXCProcessAutoDestroyDom (payload=<optimized out>, name=0x7f4bb4003c80, opaque=0x7fff41af2df0) at lxc/lxc_process.c:94 #15 0x00007f4bd9586649 in virHashForEach (table=0x7f4bc409b270, iter=iter@entry=0x7f4bcacc5ab0 <virLXCProcessAutoDestroyDom>, data=data@entry=0x7fff41af2df0) at util/virhash.c:514 #16 0x00007f4bcacc52d7 in virLXCProcessAutoDestroyRun (driver=driver@entry=0x7f4bc40c8290, conn=conn@entry=0x7f4bb8000ab0) at lxc/lxc_process.c:120 #17 0x00007f4bcacca628 in lxcClose (conn=0x7f4bb8000ab0) at lxc/lxc_driver.c:128 #18 0x00007f4bd95e67ab in virReleaseConnect (conn=conn@entry=0x7f4bb8000ab0) at datatypes.c:114 When the driver calls virLXCMonitorClose, there is really no need for the EOF callback to be invoked in this case, since the caller can easily handle events itself. In changing this, the monitor needs to take a deep copy of the callback list, not merely a reference. Also adds debug statements in various places to aid troubleshooting Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
2012-09-26 14:08:20 +00:00
/* When manually closing the monitor, we don't
* want to have callbacks back into us, since
* the caller is not re-entrant safe
*/
VIR_DEBUG("Clear EOF callback mon=%p", mon);
mon->cb.eofNotify = NULL;
virNetClientClose(mon->client);
virObjectUnref(mon->client);
mon->client = NULL;
}
}