virCaps: Expose distance between host NUMA nodes

If user or management application wants to create a guest,
it may be useful to know the cost of internode latencies
before the guest resources are pinned. For example:

<capabilities>

  <host>
    ...
    <topology>
      <cells num='2'>
        <cell id='0'>
          <memory unit='KiB'>4004132</memory>
          <distances>
            <sibling id='0' value='10'/>
            <sibling id='1' value='20'/>
          </distances>
          <cpus num='2'>
            <cpu id='0' socket_id='0' core_id='0' siblings='0'/>
            <cpu id='2' socket_id='0' core_id='2' siblings='2'/>
          </cpus>
        </cell>
        <cell id='1'>
          <memory unit='KiB'>4030064</memory>
          <distances>
            <sibling id='0' value='20'/>
            <sibling id='1' value='10'/>
          </distances>
          <cpus num='2'>
            <cpu id='1' socket_id='0' core_id='0' siblings='1'/>
            <cpu id='3' socket_id='0' core_id='2' siblings='3'/>
          </cpus>
        </cell>
      </cells>
    </topology>
    ...
  </host>
  ...
</capabilities>

We can see the distance from node1 to node0 is 20 and within nodes 10.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

docs/schemas/capability.rng

@@ -188,6 +188,21 @@
         <ref name='memory'/>
       </optional>
 
+      <optional>
+        <element name='distances'>
+          <zeroOrMore>
+            <element name='sibling'>
+              <attribute name='id'>
+                <ref name='unsignedInt'/>
+              </attribute>
+              <attribute name='value'>
+                <ref name='unsignedInt'/>
+              </attribute>
+            </element>
+          </zeroOrMore>
+        </element>
+      </optional>
+
       <optional>
         <element name='cpus'>
           <attribute name='num'>

src/conf/capabilities.c

@@ -107,6 +107,7 @@ virCapabilitiesFreeHostNUMACell(virCapsHostNUMACellPtr cell)
     virCapabilitiesClearHostNUMACellCPUTopology(cell->cpus, cell->ncpus);
 
     VIR_FREE(cell->cpus);
+    VIR_FREE(cell->siblings);
     VIR_FREE(cell);
 }
 
@@ -275,9 +276,11 @@ virCapabilitiesAddHostMigrateTransport(virCapsPtr caps,
  * virCapabilitiesAddHostNUMACell:
  * @caps: capabilities to extend
  * @num: ID number of NUMA cell
- * @ncpus: number of CPUs in cell
  * @mem: Total size of memory in the NUMA node (in KiB)
+ * @ncpus: number of CPUs in cell
  * @cpus: array of CPU definition structures, the pointer is stolen
+ * @nsiblings: number of sibling NUMA nodes
+ * @siblings: info on sibling NUMA nodes
  *
  * Registers a new NUMA cell for a host, passing in a
  * array of CPU IDs belonging to the cell
@@ -285,9 +288,11 @@ virCapabilitiesAddHostMigrateTransport(virCapsPtr caps,
 int
 virCapabilitiesAddHostNUMACell(virCapsPtr caps,
                                int num,
-                               int ncpus,
                                unsigned long long mem,
-                               virCapsHostNUMACellCPUPtr cpus)
+                               int ncpus,
+                               virCapsHostNUMACellCPUPtr cpus,
+                               int nsiblings,
+                               virCapsHostNUMACellSiblingInfoPtr siblings)
 {
     virCapsHostNUMACellPtr cell;
 
@@ -302,6 +307,8 @@ virCapabilitiesAddHostNUMACell(virCapsPtr caps,
     cell->num = num;
     cell->mem = mem;
     cell->cpus = cpus;
+    cell->siblings = siblings;
+    cell->nsiblings = nsiblings;
 
     caps->host.numaCell[caps->host.nnumaCell++] = cell;
 
@@ -766,6 +773,18 @@ virCapabilitiesFormatNUMATopology(virBufferPtr buf,
             virBufferAsprintf(buf, "<memory unit='KiB'>%llu</memory>\n",
                               cells[i]->mem);
 
+        if (cells[i]->nsiblings) {
+            virBufferAddLit(buf, "<distances>\n");
+            virBufferAdjustIndent(buf, 2);
+            for (j = 0; j < cells[i]->nsiblings; j++) {
+                virBufferAsprintf(buf, "<sibling id='%d' value='%d'/>\n",
+                                  cells[i]->siblings[j].node,
+                                  cells[i]->siblings[j].distance);
+            }
+            virBufferAdjustIndent(buf, -2);
+            virBufferAddLit(buf, "</distances>\n");
+        }
+
         virBufferAsprintf(buf, "<cpus num='%d'>\n", cells[i]->ncpus);
         virBufferAdjustIndent(buf, 2);
         for (j = 0; j < cells[i]->ncpus; j++) {

src/conf/capabilities.h

@@ -95,6 +95,13 @@ struct _virCapsHostNUMACellCPU {
     virBitmapPtr siblings;
 };
 
+typedef struct _virCapsHostNUMACellSiblingInfo virCapsHostNUMACellSiblingInfo;
+typedef virCapsHostNUMACellSiblingInfo *virCapsHostNUMACellSiblingInfoPtr;
+struct _virCapsHostNUMACellSiblingInfo {
+    int node;               /* foreign NUMA node */
+    unsigned int distance;  /* distance to the node */
+};
+
 typedef struct _virCapsHostNUMACell virCapsHostNUMACell;
 typedef virCapsHostNUMACell *virCapsHostNUMACellPtr;
 struct _virCapsHostNUMACell {
@@ -102,6 +109,8 @@ struct _virCapsHostNUMACell {
     int ncpus;
     unsigned long long mem; /* in kibibytes */
     virCapsHostNUMACellCPUPtr cpus;
+    int nsiblings;
+    virCapsHostNUMACellSiblingInfoPtr siblings;
 };
 
 typedef struct _virCapsHostSecModelLabel virCapsHostSecModelLabel;
@@ -193,9 +202,11 @@ virCapabilitiesAddHostMigrateTransport(virCapsPtr caps,
 extern int
 virCapabilitiesAddHostNUMACell(virCapsPtr caps,
                                int num,
-                               int ncpus,
                                unsigned long long mem,
-                               virCapsHostNUMACellCPUPtr cpus);
+                               int ncpus,
+                               virCapsHostNUMACellCPUPtr cpus,
+                               int nsiblings,
+                               virCapsHostNUMACellSiblingInfoPtr siblings);
 
 
 extern int

src/libxl/libxl_conf.c

@@ -207,9 +207,10 @@ libxlCapsInitNuma(libxl_ctx *ctx, virCapsPtr caps)
         if (numa_info[i].size == LIBXL_NUMAINFO_INVALID_ENTRY)
             continue;
 
-        if (virCapabilitiesAddHostNUMACell(caps, i, nr_cpus_node[i],
+        if (virCapabilitiesAddHostNUMACell(caps, i,
                                            numa_info[i].size / 1024,
-                                           cpus[i]) < 0) {
+                                           nr_cpus_node[i], cpus[i],
+                                           0, NULL) < 0) {
             virCapabilitiesClearHostNUMACellCPUTopology(cpus[i],
                                                         nr_cpus_node[i]);
             goto cleanup;

src/nodeinfo.c

@@ -1644,9 +1644,9 @@ nodeCapsInitNUMAFake(virCapsPtr caps ATTRIBUTE_UNUSED)
     }
 
     if (virCapabilitiesAddHostNUMACell(caps, 0,
-                                       ncpus,
                                        nodeinfo.memory,
-                                       cpus) < 0)
+                                       ncpus, cpus,
+                                       0, NULL) < 0)
         goto error;
 
     return 0;
@@ -1748,6 +1748,53 @@ virNodeCapsFillCPUInfo(int cpu_id ATTRIBUTE_UNUSED,
 #endif
 }
 
+static int
+virNodeCapsGetSiblingInfo(int node,
+                          virCapsHostNUMACellSiblingInfoPtr *siblings,
+                          int *nsiblings)
+{
+    virCapsHostNUMACellSiblingInfoPtr tmp = NULL;
+    int tmp_size = 0;
+    int ret = -1;
+    int *distances = NULL;
+    int ndistances = 0;
+    size_t i;
+
+    if (virNumaGetDistances(node, &distances, &ndistances) < 0)
+        goto cleanup;
+
+    if (!distances) {
+        *siblings = NULL;
+        *nsiblings = 0;
+        return 0;
+    }
+
+    if (VIR_ALLOC_N(tmp, ndistances) < 0)
+        goto cleanup;
+
+    for (i = 0; i < ndistances; i++) {
+        if (!distances[i])
+            continue;
+
+        tmp[tmp_size].node = i;
+        tmp[tmp_size].distance = distances[i];
+        tmp_size++;
+    }
+
+    if (VIR_REALLOC_N(tmp, tmp_size) < 0)
+        goto cleanup;
+
+    *siblings = tmp;
+    *nsiblings = tmp_size;
+    tmp = NULL;
+    tmp_size = 0;
+    ret = 0;
+ cleanup:
+    VIR_FREE(distances);
+    VIR_FREE(tmp);
+    return ret;
+}
+
 int
 nodeCapsInitNUMA(virCapsPtr caps)
 {
@@ -1755,6 +1802,8 @@ nodeCapsInitNUMA(virCapsPtr caps)
     unsigned long long memory;
     virCapsHostNUMACellCPUPtr cpus = NULL;
     virBitmapPtr cpumap = NULL;
+    virCapsHostNUMACellSiblingInfoPtr siblings = NULL;
+    int nsiblings;
     int ret = -1;
     int ncpus = 0;
     int cpu;
@@ -1794,14 +1843,20 @@ nodeCapsInitNUMA(virCapsPtr caps)
             }
         }
 
+        if (virNodeCapsGetSiblingInfo(n, &siblings, &nsiblings) < 0)
+            goto cleanup;
+
         /* Detect the amount of memory in the numa cell in KiB */
         virNumaGetNodeMemory(n, &memory, NULL);
         memory >>= 10;
 
-        if (virCapabilitiesAddHostNUMACell(caps, n, ncpus, memory, cpus) < 0)
+        if (virCapabilitiesAddHostNUMACell(caps, n, memory,
+                                           ncpus, cpus,
+                                           nsiblings, siblings) < 0)
             goto cleanup;
 
         cpus = NULL;
+        siblings = NULL;
     }
 
     ret = 0;
@@ -1812,6 +1867,7 @@ nodeCapsInitNUMA(virCapsPtr caps)
 
     virBitmapFree(cpumap);
     VIR_FREE(cpus);
+    VIR_FREE(siblings);
 
     if (ret < 0)
         VIR_FREE(cpus);

src/test/test_driver.c

@@ -336,8 +336,9 @@ testBuildCapabilities(virConnectPtr conn)
                sizeof(*cpu_cells) * privconn->cells[i].numCpus);
 
 
-        if (virCapabilitiesAddHostNUMACell(caps, i, privconn->cells[i].numCpus,
+        if (virCapabilitiesAddHostNUMACell(caps, i, 0,
-                                           0, cpu_cells) < 0)
+                                           privconn->cells[i].numCpus,
+                                           cpu_cells, 0, NULL) < 0)
             goto error;
     }
 

src/xen/xend_internal.c

@@ -1100,7 +1100,9 @@ sexpr_to_xend_topology(const struct sexpr *root, virCapsPtr caps)
         }
         virBitmapFree(cpuset);
 
-        if (virCapabilitiesAddHostNUMACell(caps, cell, nb_cpus, 0, cpuInfo) < 0)
+        if (virCapabilitiesAddHostNUMACell(caps, cell, 0,
+                                           nb_cpus, cpuInfo,
+                                           0, NULL) < 0)
             goto error;
         cpuInfo = NULL;
     }

tests/vircapstest.c

@@ -64,9 +64,9 @@ buildNUMATopology(int seq)
         id++;
 
         if (virCapabilitiesAddHostNUMACell(caps, cell_id + seq,
-                                           MAX_CPUS_IN_CELL,
                                            MAX_MEM_IN_CELL,
-                                           cell_cpus) < 0)
+                                           MAX_CPUS_IN_CELL, cell_cpus,
+                                           0, NULL) < 0)
            goto error;
 
         cell_cpus = NULL;