vbox: Read runtime RDP port and handle autoport

VirutalBox has a IVRDEServerInfo structure available that
gives the effective runtime port that the VM is using when it's
running. This is useful when the "TCP/Ports" VBox property was set to
port range (e.g. via autoport = "yes" or via VBoxManage) in which
case it would be impossible to get the "active" port otherwise.

src/vbox/vbox_common.c

@@ -3326,7 +3326,7 @@ vboxDumpDisplay(virDomainDefPtr def, vboxDriverPtr data, IMachine *machine)
         if (VIR_ALLOC(graphics) < 0)
             goto cleanup;
 
-        gVBoxAPI.UIVRDEServer.GetPorts(data, VRDEServer, graphics);
+        gVBoxAPI.UIVRDEServer.GetPorts(data, VRDEServer, machine, graphics);
 
         graphics->type = VIR_DOMAIN_GRAPHICS_TYPE_RDP;
 

src/vbox/vbox_tmpl.c

@@ -221,29 +221,6 @@ _vboxIIDFromArrayItem(vboxDriverPtr data, vboxIID *iid,
     _vboxIIDFromArrayItem(data, iid, array, idx)
 #define DEBUGIID(msg, strUtf16) DEBUGPRUnichar(msg, strUtf16)
 
-/**
- * Converts Utf-16 string to int
- */
-static int PRUnicharToInt(PCVBOXXPCOM pFuncs, PRUnichar *strUtf16)
-{
-    char *strUtf8 = NULL;
-    int ret = 0;
-
-    if (!strUtf16)
-        return -1;
-
-    pFuncs->pfnUtf16ToUtf8(strUtf16, &strUtf8);
-    if (!strUtf8)
-        return -1;
-
-    if (virStrToLong_i(strUtf8, NULL, 10, &ret) < 0)
-        ret = -1;
-
-    pFuncs->pfnUtf8Free(strUtf8);
-
-    return ret;
-}
-
 /**
  * Converts int to Utf-16 string
  */
@@ -280,6 +257,54 @@ static virDomainState _vboxConvertState(PRUint32 state)
     }
 }
 
+
+static int
+vboxGetActiveVRDEServerPort(ISession *session, IMachine *machine)
+{
+    nsresult rc;
+    PRInt32 port = -1;
+    IVRDEServerInfo *vrdeInfo = NULL;
+    IConsole *console = NULL;
+
+    rc = machine->vtbl->LockMachine(machine, session, LockType_Shared);
+    if (NS_FAILED(rc)) {
+        VIR_WARN("Could not obtain shared lock on VBox VM, rc=%08x", rc);
+        return -1;
+    }
+
+    rc = session->vtbl->GetConsole(session, &console);
+    if (NS_FAILED(rc)) {
+        VIR_WARN("Could not get VBox session console, rc=%08x", rc);
+        goto cleanup;
+    }
+
+    /* it may be null if VM is not running */
+    if (!console)
+        goto cleanup;
+
+    rc = console->vtbl->GetVRDEServerInfo(console, &vrdeInfo);
+
+    if (NS_FAILED(rc) || !vrdeInfo) {
+        VIR_WARN("Could not get VBox VM VRDEServerInfo, rc=%08x", rc);
+        goto cleanup;
+    }
+
+    rc = vrdeInfo->vtbl->GetPort(vrdeInfo, &port);
+
+    if (NS_FAILED(rc)) {
+        VIR_WARN("Could not read port from VRDEServerInfo, rc=%08x", rc);
+        goto cleanup;
+    }
+
+ cleanup:
+    VBOX_RELEASE(console);
+    VBOX_RELEASE(vrdeInfo);
+    session->vtbl->UnlockMachine(session);
+
+    return port;
+}
+
+
 static int
 _vboxDomainSnapshotRestore(virDomainPtr dom,
                           IMachine *machine,
@@ -1576,24 +1601,67 @@ _vrdeServerSetEnabled(IVRDEServer *VRDEServer, PRBool enabled)
 }
 
 static nsresult
-_vrdeServerGetPorts(vboxDriverPtr data ATTRIBUTE_UNUSED,
-                    IVRDEServer *VRDEServer, virDomainGraphicsDefPtr graphics)
+_vrdeServerGetPorts(vboxDriverPtr data, IVRDEServer *VRDEServer,
+                    IMachine *machine, virDomainGraphicsDefPtr graphics)
 {
     nsresult rc;
     PRUnichar *VRDEPortsKey = NULL;
     PRUnichar *VRDEPortsValue = NULL;
+    PRInt32 port = -1;
+    ssize_t nmatches = 0;
+    char **matches = NULL;
+    char *portUtf8 = NULL;
 
+    /* get active (effective) port - available only when VM is running and has
+     * the VBOX extensions installed (without extenstions RDP server
+     * functionality is disabled)
+     */
+    port = vboxGetActiveVRDEServerPort(data->vboxSession, machine);
+
+    if (port > 0)
+        graphics->data.rdp.port = port;
+
+    /* get the port (or port range) set in VM properties, this info will
+     * be used to determine whether to set autoport flag
+     */
     VBOX_UTF8_TO_UTF16("TCP/Ports", &VRDEPortsKey);
-    rc = VRDEServer->vtbl->GetVRDEProperty(VRDEServer, VRDEPortsKey, &VRDEPortsValue);
-    VBOX_UTF16_FREE(VRDEPortsKey);
-    if (VRDEPortsValue) {
-        /* even if vbox supports mutilpe ports, single port for now here */
-        graphics->data.rdp.port = PRUnicharToInt(data->pFuncs, VRDEPortsValue);
-        VBOX_UTF16_FREE(VRDEPortsValue);
-    } else {
-        graphics->data.rdp.autoport = true;
+    rc = VRDEServer->vtbl->GetVRDEProperty(VRDEServer, VRDEPortsKey,
+                                           &VRDEPortsValue);
+
+    if (NS_FAILED(rc)) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("Failed to read RDP port value, rc=%08x"),
+                       (unsigned) rc);
+       goto cleanup;
     }
 
+    VBOX_UTF16_TO_UTF8(VRDEPortsValue, &portUtf8);
+
+    if (portUtf8) {
+        /* does the string contain digits only */
+        nmatches = virStringSearch(portUtf8, "(^[[:digit:]]+$)", 1, &matches);
+
+        /* the port property is not numeric, then it must be a port range or
+         * port list or combination of the two, either way it's an autoport
+         */
+        if (nmatches != 1)
+            graphics->data.rdp.autoport = true;
+
+        /* no active port available, e.g. VM is powered off, try to get it from
+         * the property string
+         */
+        if (port < 0) {
+            if (nmatches == 1 && virStrToLong_i(portUtf8, NULL, 10, &port) == 0)
+                graphics->data.rdp.port = port;
+        }
+    }
+
+ cleanup:
+    virStringListFree(matches);
+    VBOX_UTF8_FREE(portUtf8);
+    VBOX_UTF16_FREE(VRDEPortsValue);
+    VBOX_UTF16_FREE(VRDEPortsKey);
+
     return rc;
 }
 

src/vbox/vbox_uniformed_api.h

@@ -341,7 +341,7 @@ typedef struct {
     nsresult (*GetEnabled)(IVRDEServer *VRDEServer, PRBool *enabled);
     nsresult (*SetEnabled)(IVRDEServer *VRDEServer, PRBool enabled);
     nsresult (*GetPorts)(vboxDriverPtr driver, IVRDEServer *VRDEServer,
-                         virDomainGraphicsDefPtr graphics);
+                         IMachine *machine, virDomainGraphicsDefPtr graphics);
     nsresult (*SetPorts)(vboxDriverPtr driver, IVRDEServer *VRDEServer,
                          virDomainGraphicsDefPtr graphics);
     nsresult (*GetReuseSingleConnection)(IVRDEServer *VRDEServer, PRBool *enabled);