diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
index 2705e65cf9..9245a24c2a 100644
--- a/src/libxl/libxl_conf.c
+++ b/src/libxl/libxl_conf.c
@@ -64,60 +64,6 @@ static const char *xen_cap_re = "(xen|hvm)-[[:digit:]]+\\.[[:digit:]]+-(x86_32|x
 static regex_t xen_cap_rec;
 
 
-static int
-libxlNextFreeVncPort(libxlDriverPrivatePtr driver, int startPort)
-{
-    int i;
-
-    for (i = startPort ; i < LIBXL_VNC_PORT_MAX; i++) {
-        int fd;
-        int reuse = 1;
-        struct sockaddr_in addr;
-        bool used = false;
-
-        if (virBitmapGetBit(driver->reservedVNCPorts,
-                            i - LIBXL_VNC_PORT_MIN, &used) < 0)
-            VIR_DEBUG("virBitmapGetBit failed on bit %d", i - LIBXL_VNC_PORT_MIN);
-
-        if (used)
-            continue;
-
-        addr.sin_family = AF_INET;
-        addr.sin_port = htons(i);
-        addr.sin_addr.s_addr = htonl(INADDR_ANY);
-        fd = socket(PF_INET, SOCK_STREAM, 0);
-        if (fd < 0)
-            return -1;
-
-        if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (void*)&reuse, sizeof(reuse)) < 0) {
-            VIR_FORCE_CLOSE(fd);
-            break;
-        }
-
-        if (bind(fd, (struct sockaddr*)&addr, sizeof(addr)) == 0) {
-            /* Not in use, lets grab it */
-            VIR_FORCE_CLOSE(fd);
-            /* Add port to bitmap of reserved ports */
-            if (virBitmapSetBit(driver->reservedVNCPorts,
-                                i - LIBXL_VNC_PORT_MIN) < 0) {
-                VIR_DEBUG("virBitmapSetBit failed on bit %d",
-                          i - LIBXL_VNC_PORT_MIN);
-            }
-            return i;
-        }
-        VIR_FORCE_CLOSE(fd);
-
-        if (errno == EADDRINUSE) {
-            /* In use, try next */
-            continue;
-        }
-        /* Some other bad failure, get out.. */
-        break;
-    }
-    return -1;
-}
-
-
 static int libxlDefaultConsoleType(const char *ostype,
                                    virArch arch ATTRIBUTE_UNUSED)
 {
@@ -712,7 +658,7 @@ libxlMakeVfb(libxlDriverPrivatePtr driver,
              virDomainGraphicsDefPtr l_vfb,
              libxl_device_vfb *x_vfb)
 {
-    int port;
+    unsigned short port;
     const char *listenAddr;
 
     switch (l_vfb->type) {
@@ -735,8 +681,10 @@ libxlMakeVfb(libxlDriverPrivatePtr driver,
             /* driver handles selection of free port */
             libxl_defbool_set(&x_vfb->vnc.findunused, 0);
             if (l_vfb->data.vnc.autoport) {
-                port = libxlNextFreeVncPort(driver, LIBXL_VNC_PORT_MIN);
-                if (port < 0) {
+
+                if (virPortAllocatorAcquire(driver->reservedVNCPorts, &port) < 0)
+                    return -1;
+                if (port == 0) {
                     virReportError(VIR_ERR_INTERNAL_ERROR,
                                    "%s", _("Unable to find an unused VNC port"));
                     return -1;
diff --git a/src/libxl/libxl_conf.h b/src/libxl/libxl_conf.h
index c8808a199d..a3cce080a1 100644
--- a/src/libxl/libxl_conf.h
+++ b/src/libxl/libxl_conf.h
@@ -34,7 +34,7 @@
 # include "domain_event.h"
 # include "capabilities.h"
 # include "configmake.h"
-# include "virbitmap.h"
+# include "virportallocator.h"
 
 
 # define LIBXL_VNC_PORT_MIN  5900
@@ -60,7 +60,7 @@ struct _libxlDriverPrivate {
     /* libxl ctx for driver wide ops; getVersion, getNodeInfo, ... */
     libxl_ctx *ctx;
 
-    virBitmapPtr reservedVNCPorts;
+    virPortAllocatorPtr reservedVNCPorts;
 
     size_t nactive;
     virStateInhibitCallback inhibitCallback;
diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c
index 8017a4a4ab..64a52fba79 100644
--- a/src/libxl/libxl_driver.c
+++ b/src/libxl/libxl_driver.c
@@ -466,8 +466,8 @@ libxlVmCleanup(libxlDriverPrivatePtr driver,
         vm->def->graphics[0]->data.vnc.autoport) {
         vnc_port = vm->def->graphics[0]->data.vnc.port;
         if (vnc_port >= LIBXL_VNC_PORT_MIN) {
-            if (virBitmapClearBit(driver->reservedVNCPorts,
-                                  vnc_port - LIBXL_VNC_PORT_MIN) < 0)
+            if (virPortAllocatorRelease(driver->reservedVNCPorts,
+                                        vnc_port) < 0)
                 VIR_DEBUG("Could not mark port %d as unused", vnc_port);
         }
     }
@@ -923,7 +923,7 @@ libxlShutdown(void)
     if (libxl_driver->logger_file)
         VIR_FORCE_FCLOSE(libxl_driver->logger_file);
 
-    virBitmapFree(libxl_driver->reservedVNCPorts);
+    virObjectUnref(libxl_driver->reservedVNCPorts);
 
     VIR_FREE(libxl_driver->configDir);
     VIR_FREE(libxl_driver->autostartDir);
@@ -979,9 +979,10 @@ libxlStartup(bool privileged,
     libxlDriverLock(libxl_driver);
 
     /* Allocate bitmap for vnc port reservation */
-    if ((libxl_driver->reservedVNCPorts =
-         virBitmapNew(LIBXL_VNC_PORT_MAX - LIBXL_VNC_PORT_MIN)) == NULL)
-        goto out_of_memory;
+    if (!(libxl_driver->reservedVNCPorts =
+          virPortAllocatorNew(LIBXL_VNC_PORT_MIN,
+                              LIBXL_VNC_PORT_MAX)))
+        goto error;
 
     if (virDomainObjListInit(&libxl_driver->domains) < 0)
         goto out_of_memory;