diff --git a/src/rpc/virnetserver.c b/src/rpc/virnetserver.c
index 2b9dd4d695..c93c3f8a39 100644
--- a/src/rpc/virnetserver.c
+++ b/src/rpc/virnetserver.c
@@ -131,8 +131,6 @@ static void virNetServerHandleJob(void *jobOpaque, void *opaque)
     virNetServerProgramPtr prog = NULL;
     size_t i;
 
-    virNetServerClientRef(job->client);
-
     virNetServerLock(srv);
     VIR_DEBUG("server=%p client=%p message=%p",
               srv, job->client, job->msg);
diff --git a/src/rpc/virnetserverclient.c b/src/rpc/virnetserverclient.c
index 317d59cf55..3c0dba8d65 100644
--- a/src/rpc/virnetserverclient.c
+++ b/src/rpc/virnetserverclient.c
@@ -763,10 +763,12 @@ readmore:
 
         /* Send off to for normal dispatch to workers */
         if (msg) {
+            client->refs++;
             if (!client->dispatchFunc ||
                 client->dispatchFunc(client, msg, client->dispatchOpaque) < 0) {
                 virNetMessageFree(msg);
                 client->wantClose = true;
+                client->refs--;
                 return;
             }
         }