libvirt/src/rpc/virnetserver.c

1259 lines
37 KiB
C
Raw Normal View History

/*
* virnetserver.c: generic network RPC server
*
* Copyright (C) 2006-2015 Red Hat, Inc.
* Copyright (C) 2006 Daniel P. Berrange
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library. If not, see
* <http://www.gnu.org/licenses/>.
*/
#include <config.h>
#include "virnetserver.h"
2012-12-12 17:59:27 +00:00
#include "virlog.h"
2012-12-12 18:06:53 +00:00
#include "viralloc.h"
#include "virerror.h"
#include "virthread.h"
#include "virthreadpool.h"
#include "virstring.h"
#include "virutil.h"
#define VIR_FROM_THIS VIR_FROM_RPC
VIR_LOG_INIT("rpc.netserver");
typedef struct _virNetServerJob virNetServerJob;
typedef virNetServerJob *virNetServerJobPtr;
struct _virNetServerJob {
virNetServerClientPtr client;
virNetMessagePtr msg;
virNetServerProgramPtr prog;
};
struct _virNetServer {
virObjectLockable parent;
char *name;
/* Immutable pointer, self-locking APIs */
virThreadPoolPtr workers;
size_t nservices;
virNetServerServicePtr *services;
size_t nprograms;
virNetServerProgramPtr *programs;
size_t nclients; /* Current clients count */
virNetServerClientPtr *clients; /* Clients */
unsigned long long next_client_id; /* next client ID */
size_t nclients_max; /* Max allowed clients count */
size_t nclients_unauth; /* Unauthenticated clients count */
size_t nclients_unauth_max; /* Max allowed unauth clients count */
int keepaliveInterval;
unsigned int keepaliveCount;
virNetTLSContextPtr tls;
virNetServerClientPrivNew clientPrivNew;
virNetServerClientPrivPreExecRestart clientPrivPreExecRestart;
virFreeCallback clientPrivFree;
void *clientPrivOpaque;
};
static virClassPtr virNetServerClass;
static void virNetServerDispose(void *obj);
static void virNetServerUpdateServicesLocked(virNetServerPtr srv,
bool enabled);
static inline size_t virNetServerTrackPendingAuthLocked(virNetServerPtr srv);
static inline size_t virNetServerTrackCompletedAuthLocked(virNetServerPtr srv);
static int virNetServerOnceInit(void)
{
if (!VIR_CLASS_NEW(virNetServer, virClassForObjectLockable()))
return -1;
return 0;
}
VIR_ONCE_GLOBAL_INIT(virNetServer);
unsigned long long virNetServerNextClientID(virNetServerPtr srv)
{
unsigned long long val;
virObjectLock(srv);
val = srv->next_client_id++;
virObjectUnlock(srv);
return val;
}
static int virNetServerProcessMsg(virNetServerPtr srv,
virNetServerClientPtr client,
virNetServerProgramPtr prog,
virNetMessagePtr msg)
{
if (!prog) {
/* Only send back an error for type == CALL. Other
* message types are not expecting replies, so we
* must just log it & drop them
*/
if (msg->header.type == VIR_NET_CALL ||
msg->header.type == VIR_NET_CALL_WITH_FDS) {
if (virNetServerProgramUnknownError(client,
msg,
&msg->header) < 0)
return -1;
} else {
VIR_INFO("Dropping client message, unknown program %d version %d type %d proc %d",
msg->header.prog, msg->header.vers,
msg->header.type, msg->header.proc);
/* Send a dummy reply to free up 'msg' & unblock client rx */
virNetMessageClear(msg);
msg->header.type = VIR_NET_REPLY;
if (virNetServerClientSendMessage(client, msg) < 0)
return -1;
}
return 0;
}
if (virNetServerProgramDispatch(prog,
srv,
client,
msg) < 0)
return -1;
return 0;
}
static void virNetServerHandleJob(void *jobOpaque, void *opaque)
{
virNetServerPtr srv = opaque;
virNetServerJobPtr job = jobOpaque;
VIR_DEBUG("server=%p client=%p message=%p prog=%p",
srv, job->client, job->msg, job->prog);
if (virNetServerProcessMsg(srv, job->client, job->prog, job->msg) < 0)
goto error;
virObjectUnref(job->prog);
virObjectUnref(job->client);
VIR_FREE(job);
return;
error:
virObjectUnref(job->prog);
virNetMessageFree(job->msg);
virNetServerClientClose(job->client);
virObjectUnref(job->client);
VIR_FREE(job);
}
/**
* virNetServerGetProgramLocked:
* @srv: server (must be locked by the caller)
* @msg: message
*
* Searches @srv for the right program for a given message @msg.
*
* Returns a pointer to the server program or NULL if not found.
*/
static virNetServerProgramPtr
virNetServerGetProgramLocked(virNetServerPtr srv,
virNetMessagePtr msg)
{
size_t i;
for (i = 0; i < srv->nprograms; i++) {
if (virNetServerProgramMatches(srv->programs[i], msg))
return srv->programs[i];
}
return NULL;
}
static void
virNetServerDispatchNewMessage(virNetServerClientPtr client,
virNetMessagePtr msg,
void *opaque)
{
virNetServerPtr srv = opaque;
virNetServerProgramPtr prog = NULL;
unsigned int priority = 0;
VIR_DEBUG("server=%p client=%p message=%p",
srv, client, msg);
virObjectLock(srv);
prog = virNetServerGetProgramLocked(srv, msg);
/* we can unlock @srv since @prog can only become invalid in case
* of disposing @srv, but let's grab a ref first to ensure nothing
* disposes of it before we use it. */
virObjectRef(srv);
virObjectUnlock(srv);
if (virThreadPoolGetMaxWorkers(srv->workers) > 0) {
virNetServerJobPtr job;
if (VIR_ALLOC(job) < 0)
goto error;
job->client = virObjectRef(client);
job->msg = msg;
if (prog) {
job->prog = virObjectRef(prog);
priority = virNetServerProgramGetPriority(prog, msg->header.proc);
}
if (virThreadPoolSendJob(srv->workers, priority, job) < 0) {
virObjectUnref(client);
VIR_FREE(job);
virObjectUnref(prog);
goto error;
}
} else {
if (virNetServerProcessMsg(srv, client, prog, msg) < 0)
goto error;
}
virObjectUnref(srv);
return;
error:
virNetMessageFree(msg);
virNetServerClientClose(client);
virObjectUnref(srv);
}
/**
* virNetServerCheckLimits:
* @srv: server to check limits on
*
* Check if limits like max_clients or max_anonymous_clients
* are satisfied. If so, re-enable accepting new clients. If these are violated
* however, temporarily disable accepting new clients.
* The @srv must be locked when this function is called.
*/
static void
virNetServerCheckLimits(virNetServerPtr srv)
{
VIR_DEBUG("Checking client-related limits to re-enable or temporarily "
"suspend services: nclients=%zu nclients_max=%zu "
"nclients_unauth=%zu nclients_unauth_max=%zu",
srv->nclients, srv->nclients_max,
srv->nclients_unauth, srv->nclients_unauth_max);
/* Check the max_anonymous_clients and max_clients limits so that we can
* decide whether the services should be temporarily suspended, thus not
* accepting any more clients for a while or re-enabling the previously
* suspended services in order to accept new clients again.
* A new client can only be accepted if both max_clients and
* max_anonymous_clients wouldn't get overcommitted by accepting it.
*/
if (srv->nclients >= srv->nclients_max ||
(srv->nclients_unauth_max &&
srv->nclients_unauth >= srv->nclients_unauth_max)) {
/* Temporarily stop accepting new clients */
VIR_INFO("Temporarily suspending services");
virNetServerUpdateServicesLocked(srv, false);
} else if (srv->nclients < srv->nclients_max &&
(!srv->nclients_unauth_max ||
srv->nclients_unauth < srv->nclients_unauth_max)) {
/* Now it makes sense to accept() a new client. */
VIR_INFO("Re-enabling services");
virNetServerUpdateServicesLocked(srv, true);
}
}
int virNetServerAddClient(virNetServerPtr srv,
virNetServerClientPtr client)
{
virObjectLock(srv);
if (virNetServerClientInit(client) < 0)
goto error;
if (VIR_EXPAND_N(srv->clients, srv->nclients, 1) < 0)
goto error;
srv->clients[srv->nclients-1] = virObjectRef(client);
virObjectLock(client);
rpc: virnetserver: Fix race on srv->nclients_unauth There is a race between virNetServerProcessClients (main thread) and remoteDispatchAuthList/remoteDispatchAuthPolkit/remoteSASLFinish (worker thread) that can lead to decrementing srv->nclients_unauth when it's zero. Since virNetServerCheckLimits relies on the value srv->nclients_unauth the underrun causes libvirtd to stop accepting new connections forever. Example race scenario (assuming libvirtd is using policykit and the client is privileged): 1. The client calls the RPC remoteDispatchAuthList => remoteDispatchAuthList is executed on a worker thread (Thread T1). We're assuming now the execution stops for some time before the line 'virNetServerClientSetAuth(client, 0)' 2. The client closes the connection irregularly. This causes the event loop to wake up and virNetServerProcessClient to be called (on the main thread T0). During the virNetServerProcessClients the srv lock is hold. The condition virNetServerClientNeedAuth(client) will be checked and as the authentication is not finished right now virNetServerTrackCompletedAuthLocked(srv) will be called => --srv->nclients_unauth => 0 3. The Thread T1 continues, marks the client as authenticated, and calls virNetServerTrackCompletedAuthLocked(srv) => --srv->nclients_unauth => --0 => wrap around as nclient_unauth is unsigned 4. virNetServerCheckLimits(srv) will disable the services forever To fix it, add an auth_pending field to the client struct so that it is now possible to determine if the authentication process has already been handled for this client. Setting the authentication method to none for the client in virNetServerProcessClients is not a proper way to indicate that the counter has been decremented, as this would imply that the client is authenticated. Additionally, adjust the existing test cases for this new field. Signed-off-by: Marc Hartmayer <mhartmay@linux.vnet.ibm.com> Reviewed-by: Boris Fiuczynski <fiuczy@linux.vnet.ibm.com>
2017-12-21 14:29:05 +00:00
if (virNetServerClientIsAuthPendingLocked(client))
virNetServerTrackPendingAuthLocked(srv);
virObjectUnlock(client);
virNetServerCheckLimits(srv);
virNetServerClientSetDispatcher(client,
virNetServerDispatchNewMessage,
srv);
if (virNetServerClientInitKeepAlive(client, srv->keepaliveInterval,
srv->keepaliveCount) < 0)
goto error;
virObjectUnlock(srv);
return 0;
error:
virObjectUnlock(srv);
return -1;
}
static int virNetServerDispatchNewClient(virNetServerServicePtr svc,
virNetSocketPtr clientsock,
void *opaque)
{
virNetServerPtr srv = opaque;
virNetServerClientPtr client;
if (!(client = virNetServerClientNew(virNetServerNextClientID(srv),
clientsock,
virNetServerServiceGetAuth(svc),
virNetServerServiceIsReadonly(svc),
virNetServerServiceGetMaxRequests(svc),
virNetServerServiceGetTLSContext(svc),
srv->clientPrivNew,
srv->clientPrivPreExecRestart,
srv->clientPrivFree,
srv->clientPrivOpaque)))
return -1;
if (virNetServerAddClient(srv, client) < 0) {
virNetServerClientClose(client);
virObjectUnref(client);
return -1;
}
virObjectUnref(client);
return 0;
}
virNetServerPtr virNetServerNew(const char *name,
unsigned long long next_client_id,
size_t min_workers,
size_t max_workers,
size_t priority_workers,
size_t max_clients,
size_t max_anonymous_clients,
int keepaliveInterval,
unsigned int keepaliveCount,
virNetServerClientPrivNew clientPrivNew,
virNetServerClientPrivPreExecRestart clientPrivPreExecRestart,
virFreeCallback clientPrivFree,
void *clientPrivOpaque)
{
virNetServerPtr srv;
if (virNetServerInitialize() < 0)
return NULL;
if (!(srv = virObjectLockableNew(virNetServerClass)))
return NULL;
if (!(srv->workers = virThreadPoolNewFull(min_workers, max_workers,
priority_workers,
virNetServerHandleJob,
"rpc-worker",
srv)))
goto error;
srv->name = g_strdup(name);
srv->next_client_id = next_client_id;
srv->nclients_max = max_clients;
srv->nclients_unauth_max = max_anonymous_clients;
srv->keepaliveInterval = keepaliveInterval;
srv->keepaliveCount = keepaliveCount;
srv->clientPrivNew = clientPrivNew;
srv->clientPrivPreExecRestart = clientPrivPreExecRestart;
srv->clientPrivFree = clientPrivFree;
srv->clientPrivOpaque = clientPrivOpaque;
return srv;
error:
virObjectUnref(srv);
return NULL;
}
virNetServerPtr virNetServerNewPostExecRestart(virJSONValuePtr object,
const char *name,
virNetServerClientPrivNew clientPrivNew,
virNetServerClientPrivNewPostExecRestart clientPrivNewPostExecRestart,
virNetServerClientPrivPreExecRestart clientPrivPreExecRestart,
virFreeCallback clientPrivFree,
void *clientPrivOpaque)
{
virNetServerPtr srv = NULL;
virJSONValuePtr clients;
virJSONValuePtr services;
size_t i;
unsigned int min_workers;
unsigned int max_workers;
unsigned int priority_workers;
unsigned int max_clients;
unsigned int max_anonymous_clients;
unsigned int keepaliveInterval;
unsigned int keepaliveCount;
unsigned long long next_client_id;
if (virJSONValueObjectGetNumberUint(object, "min_workers", &min_workers) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Missing min_workers data in JSON document"));
goto error;
}
if (virJSONValueObjectGetNumberUint(object, "max_workers", &max_workers) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Missing max_workers data in JSON document"));
goto error;
}
if (virJSONValueObjectGetNumberUint(object, "priority_workers", &priority_workers) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Missing priority_workers data in JSON document"));
goto error;
}
if (virJSONValueObjectGetNumberUint(object, "max_clients", &max_clients) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Missing max_clients data in JSON document"));
goto error;
}
if (virJSONValueObjectHasKey(object, "max_anonymous_clients")) {
if (virJSONValueObjectGetNumberUint(object, "max_anonymous_clients",
&max_anonymous_clients) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Malformed max_anonymous_clients data in JSON document"));
goto error;
}
} else {
max_anonymous_clients = max_clients;
}
if (virJSONValueObjectGetNumberUint(object, "keepaliveInterval", &keepaliveInterval) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Missing keepaliveInterval data in JSON document"));
goto error;
}
if (virJSONValueObjectGetNumberUint(object, "keepaliveCount", &keepaliveCount) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Missing keepaliveCount data in JSON document"));
goto error;
}
if (virJSONValueObjectGetNumberUlong(object, "next_client_id",
&next_client_id) < 0) {
VIR_WARN("Missing next_client_id data in JSON document");
next_client_id = 1;
}
if (!(srv = virNetServerNew(name, next_client_id,
min_workers, max_workers,
priority_workers, max_clients,
max_anonymous_clients,
keepaliveInterval, keepaliveCount,
clientPrivNew, clientPrivPreExecRestart,
clientPrivFree, clientPrivOpaque)))
goto error;
if (!(services = virJSONValueObjectGet(object, "services"))) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Missing services data in JSON document"));
goto error;
}
if (!virJSONValueIsArray(services)) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Malformed services array"));
goto error;
}
for (i = 0; i < virJSONValueArraySize(services); i++) {
virNetServerServicePtr service;
virJSONValuePtr child = virJSONValueArrayGet(services, i);
if (!child) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Missing service data in JSON document"));
goto error;
}
if (!(service = virNetServerServiceNewPostExecRestart(child)))
goto error;
if (virNetServerAddService(srv, service) < 0) {
virObjectUnref(service);
goto error;
}
}
if (!(clients = virJSONValueObjectGet(object, "clients"))) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Missing clients data in JSON document"));
goto error;
}
if (!virJSONValueIsArray(clients)) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Malformed clients array"));
goto error;
}
for (i = 0; i < virJSONValueArraySize(clients); i++) {
virNetServerClientPtr client;
virJSONValuePtr child = virJSONValueArrayGet(clients, i);
if (!child) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Missing client data in JSON document"));
goto error;
}
if (!(client = virNetServerClientNewPostExecRestart(srv,
child,
clientPrivNewPostExecRestart,
clientPrivPreExecRestart,
clientPrivFree,
clientPrivOpaque)))
goto error;
if (virNetServerAddClient(srv, client) < 0) {
virObjectUnref(client);
goto error;
}
virObjectUnref(client);
}
return srv;
error:
virObjectUnref(srv);
return NULL;
}
virJSONValuePtr virNetServerPreExecRestart(virNetServerPtr srv)
{
virJSONValuePtr object = virJSONValueNewObject();
virJSONValuePtr clients;
virJSONValuePtr services;
size_t i;
virObjectLock(srv);
if (virJSONValueObjectAppendNumberUint(object, "min_workers",
virThreadPoolGetMinWorkers(srv->workers)) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Cannot set min_workers data in JSON document"));
goto error;
}
if (virJSONValueObjectAppendNumberUint(object, "max_workers",
virThreadPoolGetMaxWorkers(srv->workers)) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Cannot set max_workers data in JSON document"));
goto error;
}
if (virJSONValueObjectAppendNumberUint(object, "priority_workers",
virThreadPoolGetPriorityWorkers(srv->workers)) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Cannot set priority_workers data in JSON document"));
goto error;
}
if (virJSONValueObjectAppendNumberUint(object, "max_clients", srv->nclients_max) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Cannot set max_clients data in JSON document"));
goto error;
}
if (virJSONValueObjectAppendNumberUint(object, "max_anonymous_clients",
srv->nclients_unauth_max) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Cannot set max_anonymous_clients data in JSON document"));
goto error;
}
if (virJSONValueObjectAppendNumberUint(object, "keepaliveInterval", srv->keepaliveInterval) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Cannot set keepaliveInterval data in JSON document"));
goto error;
}
if (virJSONValueObjectAppendNumberUint(object, "keepaliveCount", srv->keepaliveCount) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Cannot set keepaliveCount data in JSON document"));
goto error;
}
if (virJSONValueObjectAppendNumberUlong(object, "next_client_id",
srv->next_client_id) < 0) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("Cannot set next_client_id data in JSON document"));
goto error;
}
services = virJSONValueNewArray();
if (virJSONValueObjectAppend(object, "services", services) < 0) {
virJSONValueFree(services);
goto error;
}
for (i = 0; i < srv->nservices; i++) {
virJSONValuePtr child;
if (!(child = virNetServerServicePreExecRestart(srv->services[i])))
goto error;
if (virJSONValueArrayAppend(services, child) < 0) {
virJSONValueFree(child);
goto error;
}
}
clients = virJSONValueNewArray();
if (virJSONValueObjectAppend(object, "clients", clients) < 0) {
virJSONValueFree(clients);
goto error;
}
for (i = 0; i < srv->nclients; i++) {
virJSONValuePtr child;
if (!(child = virNetServerClientPreExecRestart(srv->clients[i])))
goto error;
if (virJSONValueArrayAppend(clients, child) < 0) {
virJSONValueFree(child);
goto error;
}
}
virObjectUnlock(srv);
return object;
error:
virJSONValueFree(object);
virObjectUnlock(srv);
return NULL;
}
int virNetServerAddService(virNetServerPtr srv,
virNetServerServicePtr svc)
{
virObjectLock(srv);
if (VIR_EXPAND_N(srv->services, srv->nservices, 1) < 0)
goto error;
srv->services[srv->nservices-1] = virObjectRef(svc);
virNetServerServiceSetDispatcher(svc,
virNetServerDispatchNewClient,
srv);
virObjectUnlock(srv);
return 0;
error:
virObjectUnlock(srv);
return -1;
}
static int
virNetServerAddServiceActivation(virNetServerPtr srv,
virSystemdActivationPtr act,
const char *actname,
int auth,
virNetTLSContextPtr tls,
bool readonly,
size_t max_queued_clients,
size_t nrequests_client_max)
{
int *fds;
size_t nfds;
if (act == NULL)
return 0;
virSystemdActivationClaimFDs(act, actname, &fds, &nfds);
if (nfds) {
virNetServerServicePtr svc;
svc = virNetServerServiceNewFDs(fds,
nfds,
false,
auth,
tls,
readonly,
max_queued_clients,
nrequests_client_max);
if (!svc)
return -1;
if (virNetServerAddService(srv, svc) < 0) {
virObjectUnref(svc);
return -1;
}
}
/* Intentionally return 1 any time activation is present,
* even if we didn't find any sockets with the matching
* name. The user needs to be free to disable some of the
* services via unit files without causing us to fallback
* to creating the service manually.
*/
return 1;
}
int virNetServerAddServiceTCP(virNetServerPtr srv,
virSystemdActivationPtr act,
const char *actname,
const char *nodename,
const char *service,
int family,
int auth,
virNetTLSContextPtr tls,
bool readonly,
size_t max_queued_clients,
size_t nrequests_client_max)
{
virNetServerServicePtr svc = NULL;
int ret;
ret = virNetServerAddServiceActivation(srv, act, actname,
auth,
tls,
readonly,
max_queued_clients,
nrequests_client_max);
if (ret < 0)
return -1;
if (ret == 1)
return 0;
if (!(svc = virNetServerServiceNewTCP(nodename,
service,
family,
auth,
tls,
readonly,
max_queued_clients,
nrequests_client_max)))
return -1;
if (virNetServerAddService(srv, svc) < 0) {
virObjectUnref(svc);
return -1;
}
virObjectUnref(svc);
return 0;
}
int virNetServerAddServiceUNIX(virNetServerPtr srv,
virSystemdActivationPtr act,
const char *actname,
const char *path,
mode_t mask,
gid_t grp,
int auth,
virNetTLSContextPtr tls,
bool readonly,
size_t max_queued_clients,
size_t nrequests_client_max)
{
virNetServerServicePtr svc = NULL;
int ret;
ret = virNetServerAddServiceActivation(srv, act, actname,
auth,
tls,
readonly,
max_queued_clients,
nrequests_client_max);
if (ret < 0)
return -1;
if (ret == 1)
return 0;
if (!(svc = virNetServerServiceNewUNIX(path,
mask,
grp,
auth,
tls,
readonly,
max_queued_clients,
nrequests_client_max)))
return -1;
if (virNetServerAddService(srv, svc) < 0) {
virObjectUnref(svc);
return -1;
}
virObjectUnref(svc);
return 0;
}
int virNetServerAddProgram(virNetServerPtr srv,
virNetServerProgramPtr prog)
{
virObjectLock(srv);
if (VIR_EXPAND_N(srv->programs, srv->nprograms, 1) < 0)
goto error;
srv->programs[srv->nprograms-1] = virObjectRef(prog);
virObjectUnlock(srv);
return 0;
error:
virObjectUnlock(srv);
return -1;
}
int virNetServerSetTLSContext(virNetServerPtr srv,
virNetTLSContextPtr tls)
{
srv->tls = virObjectRef(tls);
return 0;
}
rpc: virnetserver: Fix race on srv->nclients_unauth There is a race between virNetServerProcessClients (main thread) and remoteDispatchAuthList/remoteDispatchAuthPolkit/remoteSASLFinish (worker thread) that can lead to decrementing srv->nclients_unauth when it's zero. Since virNetServerCheckLimits relies on the value srv->nclients_unauth the underrun causes libvirtd to stop accepting new connections forever. Example race scenario (assuming libvirtd is using policykit and the client is privileged): 1. The client calls the RPC remoteDispatchAuthList => remoteDispatchAuthList is executed on a worker thread (Thread T1). We're assuming now the execution stops for some time before the line 'virNetServerClientSetAuth(client, 0)' 2. The client closes the connection irregularly. This causes the event loop to wake up and virNetServerProcessClient to be called (on the main thread T0). During the virNetServerProcessClients the srv lock is hold. The condition virNetServerClientNeedAuth(client) will be checked and as the authentication is not finished right now virNetServerTrackCompletedAuthLocked(srv) will be called => --srv->nclients_unauth => 0 3. The Thread T1 continues, marks the client as authenticated, and calls virNetServerTrackCompletedAuthLocked(srv) => --srv->nclients_unauth => --0 => wrap around as nclient_unauth is unsigned 4. virNetServerCheckLimits(srv) will disable the services forever To fix it, add an auth_pending field to the client struct so that it is now possible to determine if the authentication process has already been handled for this client. Setting the authentication method to none for the client in virNetServerProcessClients is not a proper way to indicate that the counter has been decremented, as this would imply that the client is authenticated. Additionally, adjust the existing test cases for this new field. Signed-off-by: Marc Hartmayer <mhartmay@linux.vnet.ibm.com> Reviewed-by: Boris Fiuczynski <fiuczy@linux.vnet.ibm.com>
2017-12-21 14:29:05 +00:00
/**
* virNetServerSetClientAuthCompletedLocked:
* @srv: server must be locked by the caller
* @client: client must be locked by the caller
*
* If the client authentication was pending, clear that pending and
* update the server tracking.
*/
static void
virNetServerSetClientAuthCompletedLocked(virNetServerPtr srv,
virNetServerClientPtr client)
{
if (virNetServerClientIsAuthPendingLocked(client)) {
virNetServerClientSetAuthPendingLocked(client, false);
virNetServerTrackCompletedAuthLocked(srv);
}
}
/**
* virNetServerSetClientAuthenticated:
* @srv: server must be unlocked
* @client: client must be unlocked
*
* Mark @client as authenticated and tracks on @srv that the
* authentication of this @client has been completed. Also it checks
* the limits of @srv.
*/
void
virNetServerSetClientAuthenticated(virNetServerPtr srv,
virNetServerClientPtr client)
{
virObjectLock(srv);
virObjectLock(client);
virNetServerClientSetAuthLocked(client, VIR_NET_SERVER_SERVICE_AUTH_NONE);
rpc: virnetserver: Fix race on srv->nclients_unauth There is a race between virNetServerProcessClients (main thread) and remoteDispatchAuthList/remoteDispatchAuthPolkit/remoteSASLFinish (worker thread) that can lead to decrementing srv->nclients_unauth when it's zero. Since virNetServerCheckLimits relies on the value srv->nclients_unauth the underrun causes libvirtd to stop accepting new connections forever. Example race scenario (assuming libvirtd is using policykit and the client is privileged): 1. The client calls the RPC remoteDispatchAuthList => remoteDispatchAuthList is executed on a worker thread (Thread T1). We're assuming now the execution stops for some time before the line 'virNetServerClientSetAuth(client, 0)' 2. The client closes the connection irregularly. This causes the event loop to wake up and virNetServerProcessClient to be called (on the main thread T0). During the virNetServerProcessClients the srv lock is hold. The condition virNetServerClientNeedAuth(client) will be checked and as the authentication is not finished right now virNetServerTrackCompletedAuthLocked(srv) will be called => --srv->nclients_unauth => 0 3. The Thread T1 continues, marks the client as authenticated, and calls virNetServerTrackCompletedAuthLocked(srv) => --srv->nclients_unauth => --0 => wrap around as nclient_unauth is unsigned 4. virNetServerCheckLimits(srv) will disable the services forever To fix it, add an auth_pending field to the client struct so that it is now possible to determine if the authentication process has already been handled for this client. Setting the authentication method to none for the client in virNetServerProcessClients is not a proper way to indicate that the counter has been decremented, as this would imply that the client is authenticated. Additionally, adjust the existing test cases for this new field. Signed-off-by: Marc Hartmayer <mhartmay@linux.vnet.ibm.com> Reviewed-by: Boris Fiuczynski <fiuczy@linux.vnet.ibm.com>
2017-12-21 14:29:05 +00:00
virNetServerSetClientAuthCompletedLocked(srv, client);
virNetServerCheckLimits(srv);
virObjectUnlock(client);
virObjectUnlock(srv);
}
static void
virNetServerUpdateServicesLocked(virNetServerPtr srv,
bool enabled)
{
size_t i;
for (i = 0; i < srv->nservices; i++)
virNetServerServiceToggle(srv->services[i], enabled);
}
void virNetServerUpdateServices(virNetServerPtr srv,
bool enabled)
{
virObjectLock(srv);
virNetServerUpdateServicesLocked(srv, enabled);
virObjectUnlock(srv);
}
void virNetServerDispose(void *obj)
{
virNetServerPtr srv = obj;
size_t i;
VIR_FREE(srv->name);
virThreadPoolFree(srv->workers);
for (i = 0; i < srv->nservices; i++)
virObjectUnref(srv->services[i]);
VIR_FREE(srv->services);
for (i = 0; i < srv->nprograms; i++)
virObjectUnref(srv->programs[i]);
VIR_FREE(srv->programs);
for (i = 0; i < srv->nclients; i++)
virObjectUnref(srv->clients[i]);
VIR_FREE(srv->clients);
}
void virNetServerClose(virNetServerPtr srv)
{
size_t i;
if (!srv)
return;
virObjectLock(srv);
for (i = 0; i < srv->nservices; i++)
virNetServerServiceClose(srv->services[i]);
for (i = 0; i < srv->nclients; i++)
virNetServerClientClose(srv->clients[i]);
virObjectUnlock(srv);
}
static inline size_t
virNetServerTrackPendingAuthLocked(virNetServerPtr srv)
{
return ++srv->nclients_unauth;
}
static inline size_t
virNetServerTrackCompletedAuthLocked(virNetServerPtr srv)
{
return --srv->nclients_unauth;
}
bool
virNetServerHasClients(virNetServerPtr srv)
{
bool ret;
virObjectLock(srv);
ret = !!srv->nclients;
virObjectUnlock(srv);
return ret;
}
void
virNetServerProcessClients(virNetServerPtr srv)
{
size_t i;
virNetServerClientPtr client;
virObjectLock(srv);
reprocess:
for (i = 0; i < srv->nclients; i++) {
/* Coverity 5.3.0 couldn't see that srv->clients is non-NULL
* if srv->nclients is non-zero. */
sa_assert(srv->clients);
client = srv->clients[i];
virObjectLock(client);
if (virNetServerClientWantCloseLocked(client))
virNetServerClientCloseLocked(client);
if (virNetServerClientIsClosedLocked(client)) {
VIR_DELETE_ELEMENT(srv->clients, i, srv->nclients);
rpc: virnetserver: Fix race on srv->nclients_unauth There is a race between virNetServerProcessClients (main thread) and remoteDispatchAuthList/remoteDispatchAuthPolkit/remoteSASLFinish (worker thread) that can lead to decrementing srv->nclients_unauth when it's zero. Since virNetServerCheckLimits relies on the value srv->nclients_unauth the underrun causes libvirtd to stop accepting new connections forever. Example race scenario (assuming libvirtd is using policykit and the client is privileged): 1. The client calls the RPC remoteDispatchAuthList => remoteDispatchAuthList is executed on a worker thread (Thread T1). We're assuming now the execution stops for some time before the line 'virNetServerClientSetAuth(client, 0)' 2. The client closes the connection irregularly. This causes the event loop to wake up and virNetServerProcessClient to be called (on the main thread T0). During the virNetServerProcessClients the srv lock is hold. The condition virNetServerClientNeedAuth(client) will be checked and as the authentication is not finished right now virNetServerTrackCompletedAuthLocked(srv) will be called => --srv->nclients_unauth => 0 3. The Thread T1 continues, marks the client as authenticated, and calls virNetServerTrackCompletedAuthLocked(srv) => --srv->nclients_unauth => --0 => wrap around as nclient_unauth is unsigned 4. virNetServerCheckLimits(srv) will disable the services forever To fix it, add an auth_pending field to the client struct so that it is now possible to determine if the authentication process has already been handled for this client. Setting the authentication method to none for the client in virNetServerProcessClients is not a proper way to indicate that the counter has been decremented, as this would imply that the client is authenticated. Additionally, adjust the existing test cases for this new field. Signed-off-by: Marc Hartmayer <mhartmay@linux.vnet.ibm.com> Reviewed-by: Boris Fiuczynski <fiuczy@linux.vnet.ibm.com>
2017-12-21 14:29:05 +00:00
/* Update server authentication tracking */
virNetServerSetClientAuthCompletedLocked(srv, client);
virObjectUnlock(client);
virNetServerCheckLimits(srv);
virObjectUnlock(srv);
virObjectUnref(client);
virObjectLock(srv);
goto reprocess;
} else {
virObjectUnlock(client);
}
}
virObjectUnlock(srv);
}
const char *
virNetServerGetName(virNetServerPtr srv)
{
return srv->name;
}
int
virNetServerGetThreadPoolParameters(virNetServerPtr srv,
size_t *minWorkers,
size_t *maxWorkers,
size_t *nWorkers,
size_t *freeWorkers,
size_t *nPrioWorkers,
size_t *jobQueueDepth)
{
virObjectLock(srv);
*minWorkers = virThreadPoolGetMinWorkers(srv->workers);
*maxWorkers = virThreadPoolGetMaxWorkers(srv->workers);
*freeWorkers = virThreadPoolGetFreeWorkers(srv->workers);
*nWorkers = virThreadPoolGetCurrentWorkers(srv->workers);
*nPrioWorkers = virThreadPoolGetPriorityWorkers(srv->workers);
*jobQueueDepth = virThreadPoolGetJobQueueDepth(srv->workers);
virObjectUnlock(srv);
return 0;
}
admin: Introduce virAdmServerSetThreadPoolParameters Since threadpool increments the current number of threads according to current load, i.e. how many jobs are waiting in the queue. The count however, is constrained by max and min limits of workers. The logic of this new API works like this: 1) setting the minimum a) When the limit is increased, depending on the current number of threads, new threads are possibly spawned if the current number of threads is less than the new minimum limit b) Decreasing the minimum limit has no possible effect on the current number of threads 2) setting the maximum a) Icreasing the maximum limit has no immediate effect on the current number of threads, it only allows the threadpool to spawn more threads when new jobs, that would otherwise end up queued, arrive. b) Decreasing the maximum limit may affect the current number of threads, if the current number of threads is less than the new maximum limit. Since there may be some ongoing time-consuming jobs that would effectively block this API from killing any threads. Therefore, this API is asynchronous with best-effort execution, i.e. the necessary number of workers will be terminated once they finish their previous job, unless other workers had already terminated, decreasing the limit to the requested value. 3) setting priority workers - both increase and decrease in count of these workers have an immediate impact on the current number of workers, new ones will be spawned or some of them get terminated respectively. Signed-off-by: Erik Skultety <eskultet@redhat.com>
2016-02-22 13:24:04 +00:00
int
virNetServerSetThreadPoolParameters(virNetServerPtr srv,
long long int minWorkers,
long long int maxWorkers,
long long int prioWorkers)
{
int ret;
virObjectLock(srv);
ret = virThreadPoolSetParameters(srv->workers, minWorkers,
maxWorkers, prioWorkers);
virObjectUnlock(srv);
return ret;
}
size_t
virNetServerGetMaxClients(virNetServerPtr srv)
{
size_t ret;
virObjectLock(srv);
ret = srv->nclients_max;
virObjectUnlock(srv);
return ret;
}
size_t
virNetServerGetCurrentClients(virNetServerPtr srv)
{
size_t ret;
virObjectLock(srv);
ret = srv->nclients;
virObjectUnlock(srv);
return ret;
}
size_t
virNetServerGetMaxUnauthClients(virNetServerPtr srv)
{
size_t ret;
virObjectLock(srv);
ret = srv->nclients_unauth_max;
virObjectUnlock(srv);
return ret;
}
size_t
virNetServerGetCurrentUnauthClients(virNetServerPtr srv)
{
size_t ret;
virObjectLock(srv);
ret = srv->nclients_unauth;
virObjectUnlock(srv);
admin: Introduce virAdmServerSetThreadPoolParameters Since threadpool increments the current number of threads according to current load, i.e. how many jobs are waiting in the queue. The count however, is constrained by max and min limits of workers. The logic of this new API works like this: 1) setting the minimum a) When the limit is increased, depending on the current number of threads, new threads are possibly spawned if the current number of threads is less than the new minimum limit b) Decreasing the minimum limit has no possible effect on the current number of threads 2) setting the maximum a) Icreasing the maximum limit has no immediate effect on the current number of threads, it only allows the threadpool to spawn more threads when new jobs, that would otherwise end up queued, arrive. b) Decreasing the maximum limit may affect the current number of threads, if the current number of threads is less than the new maximum limit. Since there may be some ongoing time-consuming jobs that would effectively block this API from killing any threads. Therefore, this API is asynchronous with best-effort execution, i.e. the necessary number of workers will be terminated once they finish their previous job, unless other workers had already terminated, decreasing the limit to the requested value. 3) setting priority workers - both increase and decrease in count of these workers have an immediate impact on the current number of workers, new ones will be spawned or some of them get terminated respectively. Signed-off-by: Erik Skultety <eskultet@redhat.com>
2016-02-22 13:24:04 +00:00
return ret;
}
bool virNetServerNeedsAuth(virNetServerPtr srv,
int auth)
{
bool ret = false;
size_t i;
virObjectLock(srv);
for (i = 0; i < srv->nservices; i++) {
if (virNetServerServiceGetAuth(srv->services[i]) == auth)
ret = true;
}
virObjectUnlock(srv);
return ret;
}
int
virNetServerGetClients(virNetServerPtr srv,
virNetServerClientPtr **clts)
{
int ret = -1;
size_t i;
size_t nclients = 0;
virNetServerClientPtr *list = NULL;
virObjectLock(srv);
for (i = 0; i < srv->nclients; i++) {
virNetServerClientPtr client = virObjectRef(srv->clients[i]);
if (VIR_APPEND_ELEMENT(list, nclients, client) < 0) {
virObjectUnref(client);
goto cleanup;
}
}
*clts = list;
list = NULL;
ret = nclients;
cleanup:
virObjectListFreeCount(list, nclients);
virObjectUnlock(srv);
return ret;
}
virNetServerClientPtr
virNetServerGetClient(virNetServerPtr srv,
unsigned long long id)
{
size_t i;
virNetServerClientPtr ret = NULL;
virObjectLock(srv);
for (i = 0; i < srv->nclients; i++) {
virNetServerClientPtr client = srv->clients[i];
if (virNetServerClientGetID(client) == id)
ret = virObjectRef(client);
}
virObjectUnlock(srv);
if (!ret)
virReportError(VIR_ERR_NO_CLIENT,
_("No client with matching ID '%llu'"), id);
return ret;
}
int
virNetServerSetClientLimits(virNetServerPtr srv,
long long int maxClients,
long long int maxClientsUnauth)
{
int ret = -1;
size_t max, max_unauth;
virObjectLock(srv);
max = maxClients >= 0 ? maxClients : srv->nclients_max;
max_unauth = maxClientsUnauth >= 0 ?
maxClientsUnauth : srv->nclients_unauth_max;
if (max < max_unauth) {
virReportError(VIR_ERR_INVALID_ARG, "%s",
_("The overall maximum number of clients must be "
"greater than the maximum number of clients waiting "
"for authentication"));
goto cleanup;
}
if (maxClients >= 0)
srv->nclients_max = maxClients;
if (maxClientsUnauth >= 0)
srv->nclients_unauth_max = maxClientsUnauth;
virNetServerCheckLimits(srv);
ret = 0;
cleanup:
virObjectUnlock(srv);
return ret;
}
static virNetTLSContextPtr
virNetServerGetTLSContext(virNetServerPtr srv)
{
size_t i;
virNetTLSContextPtr ctxt = NULL;
virNetServerServicePtr svc = NULL;
/* find svcTLS from srv, get svcTLS->tls */
for (i = 0; i < srv->nservices; i++) {
svc = srv->services[i];
ctxt = virNetServerServiceGetTLSContext(svc);
if (ctxt != NULL)
break;
}
return ctxt;
}
int
virNetServerUpdateTlsFiles(virNetServerPtr srv)
{
int ret = -1;
virNetTLSContextPtr ctxt = NULL;
bool privileged = geteuid() == 0;
ctxt = virNetServerGetTLSContext(srv);
if (!ctxt) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
_("no tls service found, unable to update tls files"));
return -1;
}
virObjectLock(srv);
virObjectLock(ctxt);
if (virNetTLSContextReloadForServer(ctxt, !privileged)) {
VIR_DEBUG("failed to reload server's tls context");
goto cleanup;
}
VIR_DEBUG("update tls files success");
ret = 0;
cleanup:
virObjectUnlock(ctxt);
virObjectUnlock(srv);
return ret;
}