diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c
index af46384441..0f8d6071f3 100644
--- a/src/remote/remote_driver.c
+++ b/src/remote/remote_driver.c
@@ -3882,12 +3882,14 @@ remoteStreamSend(virStreamPtr st,
     VIR_DEBUG("st=%p data=%p nbytes=%zu", st, data, nbytes);
     struct private_data *priv = st->conn->privateData;
     virNetClientStreamPtr privst = st->privateData;
-    int rv = -1;
-
-    remoteDriverLock(priv);
+    int rv;
 
     if (virNetClientStreamRaiseError(privst))
-        goto cleanup;
+        return -1;
+
+    remoteDriverLock(priv);
+    priv->localUses++;
+    remoteDriverUnlock(priv);
 
     rv = virNetClientStreamSendPacket(privst,
                                       priv->client,
@@ -3895,9 +3897,9 @@ remoteStreamSend(virStreamPtr st,
                                       data,
                                       nbytes);
 
-cleanup:
+    remoteDriverLock(priv);
+    priv->localUses--;
     remoteDriverUnlock(priv);
-
     return rv;
 }
 
@@ -3910,12 +3912,14 @@ remoteStreamRecv(virStreamPtr st,
     VIR_DEBUG("st=%p data=%p nbytes=%zu", st, data, nbytes);
     struct private_data *priv = st->conn->privateData;
     virNetClientStreamPtr privst = st->privateData;
-    int rv = -1;
-
-    remoteDriverLock(priv);
+    int rv;
 
     if (virNetClientStreamRaiseError(privst))
-        goto cleanup;
+        return -1;
+
+    remoteDriverLock(priv);
+    priv->localUses++;
+    remoteDriverUnlock(priv);
 
     rv = virNetClientStreamRecvPacket(privst,
                                       priv->client,
@@ -3925,9 +3929,9 @@ remoteStreamRecv(virStreamPtr st,
 
     VIR_DEBUG("Done %d", rv);
 
-cleanup:
+    remoteDriverLock(priv);
+    priv->localUses--;
     remoteDriverUnlock(priv);
-
     return rv;
 }
 
@@ -4044,12 +4048,18 @@ remoteStreamFinish(virStreamPtr st)
     if (virNetClientStreamRaiseError(privst))
         goto cleanup;
 
+    priv->localUses++;
+    remoteDriverUnlock(priv);
+
     ret = virNetClientStreamSendPacket(privst,
                                        priv->client,
                                        VIR_NET_OK,
                                        NULL,
                                        0);
 
+    remoteDriverLock(priv);
+    priv->localUses--;
+
 cleanup:
     virNetClientRemoveStream(priv->client, privst);
     virNetClientStreamFree(privst);
@@ -4073,12 +4083,18 @@ remoteStreamAbort(virStreamPtr st)
     if (virNetClientStreamRaiseError(privst))
         goto cleanup;
 
+    priv->localUses++;
+    remoteDriverUnlock(priv);
+
     ret = virNetClientStreamSendPacket(privst,
                                        priv->client,
                                        VIR_NET_ERROR,
                                        NULL,
                                        0);
 
+    remoteDriverLock(priv);
+    priv->localUses--;
+
 cleanup:
     virNetClientRemoveStream(priv->client, privst);
     virNetClientStreamFree(privst);