mirror of
https://gitlab.com/libvirt/libvirt.git
synced 2024-11-02 19:31:18 +00:00
4e5f0dd2d3
There's a race in lxc driver causing a deadlock. If a domain is destroyed immediately after started, the deadlock can occur. When domain is started, the even loop tries to connect to the monitor. If the connecting succeeds, virLXCProcessMonitorInitNotify() is called with @mon->client locked. The first thing that callee does, is virObjectLock(vm). So the order of locking is: 1) @mon->client, 2) @vm. However, if there's another thread executing virDomainDestroy on the very same domain, the first thing done here is locking the @vm. Then, the corresponding libvirt_lxc process is killed and monitor is closed via calling virLXCMonitorClose(). This callee tries to lock @mon->client too. So the order is reversed to the first case. This situation results in deadlock and unresponsive libvirtd (since the eventloop is involved). The proper solution is to unlock the @vm in virLXCMonitorClose prior entering virNetClientClose(). See the backtrace as follows: Thread 25 (Thread 0x7f1b7c9b8700 (LWP 16312)): 0 0x00007f1b80539714 in __lll_lock_wait () from /lib64/libpthread.so.0 1 0x00007f1b8053516c in _L_lock_516 () from /lib64/libpthread.so.0 2 0x00007f1b80534fbb in pthread_mutex_lock () from /lib64/libpthread.so.0 3 0x00007f1b82a637cf in virMutexLock (m=0x7f1b3c0038d0) at util/virthreadpthread.c:85 4 0x00007f1b82a4ccf2 in virObjectLock (anyobj=0x7f1b3c0038c0) at util/virobject.c:320 5 0x00007f1b82b861f6 in virNetClientCloseInternal (client=0x7f1b3c0038c0, reason=3) at rpc/virnetclient.c:696 6 0x00007f1b82b862f5 in virNetClientClose (client=0x7f1b3c0038c0) at rpc/virnetclient.c:721 7 0x00007f1b6ee12500 in virLXCMonitorClose (mon=0x7f1b3c007210) at lxc/lxc_monitor.c:216 8 0x00007f1b6ee129f0 in virLXCProcessCleanup (driver=0x7f1b68100240, vm=0x7f1b680ceb70, reason=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:174 9 0x00007f1b6ee14106 in virLXCProcessStop (driver=0x7f1b68100240, vm=0x7f1b680ceb70, reason=VIR_DOMAIN_SHUTOFF_DESTROYED) at lxc/lxc_process.c:710 10 0x00007f1b6ee1aa36 in lxcDomainDestroyFlags (dom=0x7f1b5c002560, flags=0) at lxc/lxc_driver.c:1291 11 0x00007f1b6ee1ab1a in lxcDomainDestroy (dom=0x7f1b5c002560) at lxc/lxc_driver.c:1321 12 0x00007f1b82b05be5 in virDomainDestroy (domain=0x7f1b5c002560) at libvirt.c:2303 13 0x00007f1b835a7e85 in remoteDispatchDomainDestroy (server=0x7f1b857419d0, client=0x7f1b8574ae40, msg=0x7f1b8574acf0, rerr=0x7f1b7c9b7c30, args=0x7f1b5c004a50) at remote_dispatch.h:3143 14 0x00007f1b835a7d78 in remoteDispatchDomainDestroyHelper (server=0x7f1b857419d0, client=0x7f1b8574ae40, msg=0x7f1b8574acf0, rerr=0x7f1b7c9b7c30, args=0x7f1b5c004a50, ret=0x7f1b5c0029e0) at remote_dispatch.h:3121 15 0x00007f1b82b93704 in virNetServerProgramDispatchCall (prog=0x7f1b8573af90, server=0x7f1b857419d0, client=0x7f1b8574ae40, msg=0x7f1b8574acf0) at rpc/virnetserverprogram.c:435 16 0x00007f1b82b93263 in virNetServerProgramDispatch (prog=0x7f1b8573af90, server=0x7f1b857419d0, client=0x7f1b8574ae40, msg=0x7f1b8574acf0) at rpc/virnetserverprogram.c:305 17 0x00007f1b82b8c0f6 in virNetServerProcessMsg (srv=0x7f1b857419d0, client=0x7f1b8574ae40, prog=0x7f1b8573af90, msg=0x7f1b8574acf0) at rpc/virnetserver.c:163 18 0x00007f1b82b8c1da in virNetServerHandleJob (jobOpaque=0x7f1b8574dca0, opaque=0x7f1b857419d0) at rpc/virnetserver.c:184 19 0x00007f1b82a64158 in virThreadPoolWorker (opaque=0x7f1b8573cb10) at util/virthreadpool.c:144 20 0x00007f1b82a63ae5 in virThreadHelper (data=0x7f1b8574b9f0) at util/virthreadpthread.c:161 21 0x00007f1b80532f4a in start_thread () from /lib64/libpthread.so.0 22 0x00007f1b7fc4f20d in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7f1b83546740 (LWP 16297)): 0 0x00007f1b80539714 in __lll_lock_wait () from /lib64/libpthread.so.0 1 0x00007f1b8053516c in _L_lock_516 () from /lib64/libpthread.so.0 2 0x00007f1b80534fbb in pthread_mutex_lock () from /lib64/libpthread.so.0 3 0x00007f1b82a637cf in virMutexLock (m=0x7f1b680ceb80) at util/virthreadpthread.c:85 4 0x00007f1b82a4ccf2 in virObjectLock (anyobj=0x7f1b680ceb70) at util/virobject.c:320 5 0x00007f1b6ee13bd7 in virLXCProcessMonitorInitNotify (mon=0x7f1b3c007210, initpid=4832, vm=0x7f1b680ceb70) at lxc/lxc_process.c:601 6 0x00007f1b6ee11fd3 in virLXCMonitorHandleEventInit (prog=0x7f1b3c001f10, client=0x7f1b3c0038c0, evdata=0x7f1b8574a7d0, opaque=0x7f1b3c007210) at lxc/lxc_monitor.c:109 7 0x00007f1b82b8a196 in virNetClientProgramDispatch (prog=0x7f1b3c001f10, client=0x7f1b3c0038c0, msg=0x7f1b3c003928) at rpc/virnetclientprogram.c:259 8 0x00007f1b82b87030 in virNetClientCallDispatchMessage (client=0x7f1b3c0038c0) at rpc/virnetclient.c:1019 9 0x00007f1b82b876bb in virNetClientCallDispatch (client=0x7f1b3c0038c0) at rpc/virnetclient.c:1140 10 0x00007f1b82b87d41 in virNetClientIOHandleInput (client=0x7f1b3c0038c0) at rpc/virnetclient.c:1312 11 0x00007f1b82b88f51 in virNetClientIncomingEvent (sock=0x7f1b3c0044e0, events=1, opaque=0x7f1b3c0038c0) at rpc/virnetclient.c:1832 12 0x00007f1b82b9e1c8 in virNetSocketEventHandle (watch=3321, fd=54, events=1, opaque=0x7f1b3c0044e0) at rpc/virnetsocket.c:1695 13 0x00007f1b82a272cf in virEventPollDispatchHandles (nfds=21, fds=0x7f1b8574ded0) at util/vireventpoll.c:498 14 0x00007f1b82a27af2 in virEventPollRunOnce () at util/vireventpoll.c:645 15 0x00007f1b82a25a61 in virEventRunDefaultImpl () at util/virevent.c:273 16 0x00007f1b82b8e97e in virNetServerRun (srv=0x7f1b857419d0) at rpc/virnetserver.c:1097 17 0x00007f1b8359db6b in main (argc=2, argv=0x7ffff98dbaa8) at libvirtd.c:1512
233 lines
6.3 KiB
C
233 lines
6.3 KiB
C
/*
|
|
* Copyright (C) 2010-2012 Red Hat, Inc.
|
|
*
|
|
* lxc_monitor.c: client for LXC controller monitor
|
|
*
|
|
* This library is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
*
|
|
* This library is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
* License along with this library. If not, see
|
|
* <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include <config.h>
|
|
|
|
#include "lxc_monitor.h"
|
|
#include "lxc_conf.h"
|
|
#include "lxc_monitor_dispatch.h"
|
|
|
|
#include "viralloc.h"
|
|
|
|
#include "virerror.h"
|
|
#include "virlog.h"
|
|
#include "virthread.h"
|
|
#include "rpc/virnetclient.h"
|
|
#include "virstring.h"
|
|
|
|
#define VIR_FROM_THIS VIR_FROM_LXC
|
|
|
|
struct _virLXCMonitor {
|
|
virObjectLockable parent;
|
|
|
|
virDomainObjPtr vm;
|
|
virLXCMonitorCallbacks cb;
|
|
|
|
virNetClientPtr client;
|
|
virNetClientProgramPtr program;
|
|
};
|
|
|
|
static virClassPtr virLXCMonitorClass;
|
|
static void virLXCMonitorDispose(void *obj);
|
|
|
|
static int virLXCMonitorOnceInit(void)
|
|
{
|
|
if (!(virLXCMonitorClass = virClassNew(virClassForObjectLockable(),
|
|
"virLXCMonitor",
|
|
sizeof(virLXCMonitor),
|
|
virLXCMonitorDispose)))
|
|
return -1;
|
|
|
|
return 0;
|
|
}
|
|
|
|
VIR_ONCE_GLOBAL_INIT(virLXCMonitor)
|
|
|
|
static void
|
|
virLXCMonitorHandleEventExit(virNetClientProgramPtr prog,
|
|
virNetClientPtr client,
|
|
void *evdata, void *opaque);
|
|
static void
|
|
virLXCMonitorHandleEventInit(virNetClientProgramPtr prog,
|
|
virNetClientPtr client,
|
|
void *evdata, void *opaque);
|
|
|
|
static virNetClientProgramEvent virLXCMonitorEvents[] = {
|
|
{ VIR_LXC_MONITOR_PROC_EXIT_EVENT,
|
|
virLXCMonitorHandleEventExit,
|
|
sizeof(virLXCMonitorExitEventMsg),
|
|
(xdrproc_t)xdr_virLXCMonitorExitEventMsg },
|
|
{ VIR_LXC_MONITOR_PROC_INIT_EVENT,
|
|
virLXCMonitorHandleEventInit,
|
|
sizeof(virLXCMonitorInitEventMsg),
|
|
(xdrproc_t)xdr_virLXCMonitorInitEventMsg },
|
|
};
|
|
|
|
|
|
static void
|
|
virLXCMonitorHandleEventExit(virNetClientProgramPtr prog ATTRIBUTE_UNUSED,
|
|
virNetClientPtr client ATTRIBUTE_UNUSED,
|
|
void *evdata, void *opaque)
|
|
{
|
|
virLXCMonitorPtr mon = opaque;
|
|
virLXCMonitorExitEventMsg *msg = evdata;
|
|
|
|
VIR_DEBUG("Event exit %d", msg->status);
|
|
if (mon->cb.exitNotify)
|
|
mon->cb.exitNotify(mon, msg->status, mon->vm);
|
|
}
|
|
|
|
|
|
static void
|
|
virLXCMonitorHandleEventInit(virNetClientProgramPtr prog ATTRIBUTE_UNUSED,
|
|
virNetClientPtr client ATTRIBUTE_UNUSED,
|
|
void *evdata, void *opaque)
|
|
{
|
|
virLXCMonitorPtr mon = opaque;
|
|
virLXCMonitorInitEventMsg *msg = evdata;
|
|
|
|
VIR_DEBUG("Event init %llu",
|
|
(unsigned long long)msg->initpid);
|
|
if (mon->cb.initNotify)
|
|
mon->cb.initNotify(mon, (pid_t)msg->initpid, mon->vm);
|
|
}
|
|
|
|
|
|
static void virLXCMonitorEOFNotify(virNetClientPtr client ATTRIBUTE_UNUSED,
|
|
int reason ATTRIBUTE_UNUSED,
|
|
void *opaque)
|
|
{
|
|
virLXCMonitorPtr mon = opaque;
|
|
virLXCMonitorCallbackEOFNotify eofNotify;
|
|
virDomainObjPtr vm;
|
|
|
|
VIR_DEBUG("EOF notify mon=%p", mon);
|
|
virObjectLock(mon);
|
|
eofNotify = mon->cb.eofNotify;
|
|
vm = mon->vm;
|
|
virObjectUnlock(mon);
|
|
|
|
if (eofNotify) {
|
|
VIR_DEBUG("EOF callback mon=%p vm=%p", mon, vm);
|
|
eofNotify(mon, vm);
|
|
} else {
|
|
VIR_DEBUG("No EOF callback");
|
|
}
|
|
}
|
|
|
|
|
|
static void virLXCMonitorCloseFreeCallback(void *opaque)
|
|
{
|
|
virLXCMonitorPtr mon = opaque;
|
|
virObjectUnref(mon);
|
|
}
|
|
|
|
|
|
virLXCMonitorPtr virLXCMonitorNew(virDomainObjPtr vm,
|
|
const char *socketdir,
|
|
virLXCMonitorCallbacksPtr cb)
|
|
{
|
|
virLXCMonitorPtr mon;
|
|
char *sockpath = NULL;
|
|
|
|
if (virLXCMonitorInitialize() < 0)
|
|
return NULL;
|
|
|
|
if (!(mon = virObjectLockableNew(virLXCMonitorClass)))
|
|
return NULL;
|
|
|
|
if (virAsprintf(&sockpath, "%s/%s.sock",
|
|
socketdir, vm->def->name) < 0)
|
|
goto error;
|
|
|
|
if (!(mon->client = virNetClientNewUNIX(sockpath, false, NULL)))
|
|
goto error;
|
|
|
|
if (virNetClientRegisterAsyncIO(mon->client) < 0)
|
|
goto error;
|
|
|
|
if (!(mon->program = virNetClientProgramNew(VIR_LXC_MONITOR_PROGRAM,
|
|
VIR_LXC_MONITOR_PROGRAM_VERSION,
|
|
virLXCMonitorEvents,
|
|
ARRAY_CARDINALITY(virLXCMonitorEvents),
|
|
mon)))
|
|
goto error;
|
|
|
|
if (virNetClientAddProgram(mon->client,
|
|
mon->program) < 0)
|
|
goto error;
|
|
|
|
mon->vm = vm;
|
|
memcpy(&mon->cb, cb, sizeof(mon->cb));
|
|
|
|
virObjectRef(mon);
|
|
virNetClientSetCloseCallback(mon->client, virLXCMonitorEOFNotify, mon,
|
|
virLXCMonitorCloseFreeCallback);
|
|
|
|
cleanup:
|
|
VIR_FREE(sockpath);
|
|
return mon;
|
|
|
|
error:
|
|
virObjectUnref(mon);
|
|
mon = NULL;
|
|
goto cleanup;
|
|
}
|
|
|
|
|
|
static void virLXCMonitorDispose(void *opaque)
|
|
{
|
|
virLXCMonitorPtr mon = opaque;
|
|
|
|
VIR_DEBUG("mon=%p", mon);
|
|
if (mon->cb.destroy)
|
|
(mon->cb.destroy)(mon, mon->vm);
|
|
virObjectUnref(mon->program);
|
|
}
|
|
|
|
|
|
void virLXCMonitorClose(virLXCMonitorPtr mon)
|
|
{
|
|
virDomainObjPtr vm;
|
|
virNetClientPtr client;
|
|
|
|
VIR_DEBUG("mon=%p", mon);
|
|
if (mon->client) {
|
|
/* When manually closing the monitor, we don't
|
|
* want to have callbacks back into us, since
|
|
* the caller is not re-entrant safe
|
|
*/
|
|
VIR_DEBUG("Clear EOF callback mon=%p", mon);
|
|
vm = mon->vm;
|
|
client = mon->client;
|
|
mon->client = NULL;
|
|
mon->cb.eofNotify = NULL;
|
|
|
|
virObjectRef(vm);
|
|
virObjectUnlock(vm);
|
|
|
|
virNetClientClose(client);
|
|
virObjectUnref(client);
|
|
|
|
virObjectLock(vm);
|
|
virObjectUnref(vm);
|
|
}
|
|
}
|