Allow NUMA nodes without vCPUs

QEMU allows creating NUMA nodes that have memory only.
These are somehow important for HMAT.

With check done in qemuValidateDomainDef() for QEMU 2.7 or newer
(checked via QEMU_CAPS_NUMA), we can be sure that the vCPUs are
fully assigned to NUMA nodes in domain XML.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>

docs/formatdomain.html.in

@@ -1840,6 +1840,8 @@
       consistent across qemu and libvirt versions.
       <code>memory</code> specifies the node memory
       in kibibytes (i.e. blocks of 1024 bytes).
+      <span class="since">Since 6.6.0</span> the <code>cpus</code> attribute
+      is optional and if omitted a CPU-less NUMA node is created.
       <span class="since">Since 1.2.11</span> one can use an additional <a
           href="#elementsMemoryAllocation"><code>unit</code></a> attribute to
       define units in which <code>memory</code> is specified.

docs/schemas/cputypes.rng

@@ -115,9 +115,11 @@
           <ref name="unsignedInt"/>
         </attribute>
       </optional>
-      <attribute name="cpus">
-        <ref name="cpuset"/>
-      </attribute>
+      <optional>
+        <attribute name="cpus">
+          <ref name="cpuset"/>
+        </attribute>
+      </optional>
       <attribute name="memory">
         <ref name="memoryKB"/>
       </attribute>

src/conf/numa_conf.c

@@ -889,32 +889,28 @@ virDomainNumaDefParseXML(virDomainNumaPtr def,
         }
         VIR_FREE(tmp);
 
-        if (def->mem_nodes[cur_cell].cpumask) {
+        if (def->mem_nodes[cur_cell].mem) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("Duplicate NUMA cell info for cell id '%u'"),
                            cur_cell);
             goto cleanup;
         }
 
-        if (!(tmp = virXMLPropString(nodes[i], "cpus"))) {
-            virReportError(VIR_ERR_XML_ERROR, "%s",
-                           _("Missing 'cpus' attribute in NUMA cell"));
-            goto cleanup;
-        }
+        if ((tmp = virXMLPropString(nodes[i], "cpus"))) {
+            g_autoptr(virBitmap) cpumask = NULL;
 
-        if (virBitmapParse(tmp, &def->mem_nodes[cur_cell].cpumask,
-                           VIR_DOMAIN_CPUMASK_LEN) < 0)
-            goto cleanup;
+            if (virBitmapParse(tmp, &cpumask, VIR_DOMAIN_CPUMASK_LEN) < 0)
+                goto cleanup;
 
-        if (virBitmapIsAllClear(def->mem_nodes[cur_cell].cpumask)) {
-            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-                          _("NUMA cell %d has no vCPUs assigned"), cur_cell);
-            goto cleanup;
+            if (!virBitmapIsAllClear(cpumask))
+                def->mem_nodes[cur_cell].cpumask = g_steal_pointer(&cpumask);
+            VIR_FREE(tmp);
         }
-        VIR_FREE(tmp);
 
         for (j = 0; j < n; j++) {
-            if (j == cur_cell || !def->mem_nodes[j].cpumask)
+            if (j == cur_cell ||
+                !def->mem_nodes[j].cpumask ||
+                !def->mem_nodes[cur_cell].cpumask)
                 continue;
 
             if (virBitmapOverlaps(def->mem_nodes[j].cpumask,
@@ -975,7 +971,6 @@ virDomainNumaDefFormatXML(virBufferPtr buf,
 {
     virDomainMemoryAccess memAccess;
     virTristateBool discard;
-    char *cpustr;
     size_t ncells = virDomainNumaGetNodeCount(def);
     size_t i;
 
@@ -985,17 +980,22 @@ virDomainNumaDefFormatXML(virBufferPtr buf,
     virBufferAddLit(buf, "<numa>\n");
     virBufferAdjustIndent(buf, 2);
     for (i = 0; i < ncells; i++) {
+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def, i);
         int ndistances;
 
         memAccess = virDomainNumaGetNodeMemoryAccessMode(def, i);
         discard = virDomainNumaGetNodeDiscard(def, i);
 
-        if (!(cpustr = virBitmapFormat(virDomainNumaGetNodeCpumask(def, i))))
-            return -1;
-
         virBufferAddLit(buf, "<cell");
         virBufferAsprintf(buf, " id='%zu'", i);
-        virBufferAsprintf(buf, " cpus='%s'", cpustr);
+
+        if (cpumask) {
+            g_autofree char *cpustr = virBitmapFormat(cpumask);
+
+            if (!cpustr)
+                return -1;
+            virBufferAsprintf(buf, " cpus='%s'", cpustr);
+        }
         virBufferAsprintf(buf, " memory='%llu'",
                           virDomainNumaGetNodeMemorySize(def, i));
         virBufferAddLit(buf, " unit='KiB'");
@@ -1031,8 +1031,6 @@ virDomainNumaDefFormatXML(virBufferPtr buf,
             virBufferAdjustIndent(buf, -2);
             virBufferAddLit(buf, "</cell>\n");
         }
-
-        VIR_FREE(cpustr);
     }
     virBufferAdjustIndent(buf, -2);
     virBufferAddLit(buf, "</numa>\n");
@@ -1047,8 +1045,12 @@ virDomainNumaGetCPUCountTotal(virDomainNumaPtr numa)
     size_t i;
     unsigned int ret = 0;
 
-    for (i = 0; i < numa->nmem_nodes; i++)
-        ret += virBitmapCountBits(virDomainNumaGetNodeCpumask(numa, i));
+    for (i = 0; i < numa->nmem_nodes; i++) {
+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, i);
+
+        if (cpumask)
+            ret += virBitmapCountBits(cpumask);
+    }
 
     return ret;
 }
@@ -1060,11 +1062,14 @@ virDomainNumaGetMaxCPUID(virDomainNumaPtr numa)
     unsigned int ret = 0;
 
     for (i = 0; i < numa->nmem_nodes; i++) {
+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, i);
         int bit;
 
-        bit = virBitmapLastSetBit(virDomainNumaGetNodeCpumask(numa, i));
-        if (bit > ret)
-            ret = bit;
+        if (cpumask) {
+            bit = virBitmapLastSetBit(cpumask);
+            if (bit > ret)
+                ret = bit;
+        }
     }
 
     return ret;

src/libxl/xen_xl.c

@@ -1444,19 +1444,21 @@ xenFormatXLVnuma(virConfValuePtr list,
 {
     int ret = -1;
     size_t i;
-
     virBuffer buf = VIR_BUFFER_INITIALIZER;
     virConfValuePtr numaVnode, tmp;
-
+    virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, node);
     size_t nodeSize = virDomainNumaGetNodeMemorySize(numa, node) / 1024;
-    char *nodeVcpus = virBitmapFormat(virDomainNumaGetNodeCpumask(numa, node));
+    g_autofree char *nodeVcpus = NULL;
 
-    if (VIR_ALLOC(numaVnode) < 0)
+    if (!cpumask ||
+        VIR_ALLOC(numaVnode) < 0)
         goto cleanup;
 
     numaVnode->type = VIR_CONF_LIST;
     numaVnode->list = NULL;
 
+    nodeVcpus = virBitmapFormat(cpumask);
+
     /* pnode */
     virBufferAsprintf(&buf, "pnode=%zu", node);
     xenFormatXLVnode(numaVnode, &buf);

src/qemu/qemu_command.c

@@ -7124,8 +7124,6 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
     size_t i, j;
     virQEMUCapsPtr qemuCaps = priv->qemuCaps;
     g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
-    char *cpumask = NULL;
-    char *tmpmask = NULL;
     char *next = NULL;
     virBufferPtr nodeBackends = NULL;
     bool needBackend = false;
@@ -7170,9 +7168,7 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
         goto cleanup;
 
     for (i = 0; i < ncells; i++) {
-        VIR_FREE(cpumask);
-        if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i))))
-            goto cleanup;
+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);
 
         if (needBackend) {
             virCommandAddArg(cmd, "-object");
@@ -7182,11 +7178,19 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
         virCommandAddArg(cmd, "-numa");
         virBufferAsprintf(&buf, "node,nodeid=%zu", i);
 
-        for (tmpmask = cpumask; tmpmask; tmpmask = next) {
-            if ((next = strchr(tmpmask, ',')))
-                *(next++) = '\0';
-            virBufferAddLit(&buf, ",cpus=");
-            virBufferAdd(&buf, tmpmask, -1);
+        if (cpumask) {
+            g_autofree char *cpumaskStr = NULL;
+            char *tmpmask;
+
+            if (!(cpumaskStr = virBitmapFormat(cpumask)))
+                goto cleanup;
+
+            for (tmpmask = cpumaskStr; tmpmask; tmpmask = next) {
+                if ((next = strchr(tmpmask, ',')))
+                    *(next++) = '\0';
+                virBufferAddLit(&buf, ",cpus=");
+                virBufferAdd(&buf, tmpmask, -1);
+            }
         }
 
         if (needBackend)
@@ -7217,8 +7221,6 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
     ret = 0;
 
  cleanup:
-    VIR_FREE(cpumask);
-
     if (nodeBackends) {
         for (i = 0; i < ncells; i++)
             virBufferFreeAndReset(&nodeBackends[i]);

src/qemu/qemu_validate.c

@@ -654,7 +654,7 @@ qemuValidateDomainDefNuma(const virDomainDef *def,
     }
 
     for (i = 0; i < ncells; i++) {
-        g_autofree char * cpumask = NULL;
+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);
 
         if (!hasMemoryCap &&
             virDomainNumaGetNodeMemoryAccessMode(def->numa, i)) {
@@ -664,17 +664,19 @@ qemuValidateDomainDefNuma(const virDomainDef *def,
             return -1;
         }
 
-        if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i))))
-            return -1;
+        if (cpumask) {
+            g_autofree char * cpumaskStr = NULL;
+            if (!(cpumaskStr = virBitmapFormat(cpumask)))
+                return -1;
 
-        if (strchr(cpumask, ',') &&
-            !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA)) {
-            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                           _("disjoint NUMA cpu ranges are not supported "
-                             "with this QEMU"));
-            return -1;
+            if (strchr(cpumaskStr, ',') &&
+                !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA)) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                               _("disjoint NUMA cpu ranges are not supported "
+                                 "with this QEMU"));
+                return -1;
+            }
         }
-
     }
 
     if (virDomainNumaNodesDistancesAreBeingSet(def->numa) &&

tests/qemuxml2argvdata/numatune-no-vcpu.args

@@ -0,0 +1,33 @@
+LC_ALL=C \
+PATH=/bin \
+HOME=/tmp/lib/domain--1-QEMUGuest \
+USER=test \
+LOGNAME=test \
+XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \
+XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \
+XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \
+QEMU_AUDIO_DRV=none \
+/usr/bin/qemu-system-x86_64 \
+-name QEMUGuest \
+-S \
+-machine pc,accel=tcg,usb=off,dump-guest-core=off \
+-m 12288 \
+-realtime mlock=off \
+-smp 12,sockets=12,cores=1,threads=1 \
+-numa node,nodeid=0,cpus=0-3,mem=2048 \
+-numa node,nodeid=1,cpus=4-7,mem=2048 \
+-numa node,nodeid=2,cpus=8-11,mem=2048 \
+-numa node,nodeid=3,mem=2048 \
+-numa node,nodeid=4,mem=2048 \
+-numa node,nodeid=5,mem=2048 \
+-uuid c7a5fdb2-cdaf-9455-926a-d65c16db1809 \
+-display none \
+-no-user-config \
+-nodefaults \
+-chardev socket,id=charmonitor,path=/tmp/lib/domain--1-QEMUGuest/monitor.sock,\
+server,nowait \
+-mon chardev=charmonitor,id=monitor,mode=control \
+-rtc base=utc \
+-no-shutdown \
+-usb \
+-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3

tests/qemuxml2argvdata/numatune-no-vcpu.xml

@@ -0,0 +1,42 @@
+<domain type='qemu'>
+  <name>QEMUGuest</name>
+  <uuid>c7a5fdb2-cdaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>12582912</memory>
+  <currentMemory unit='KiB'>12582912</currentMemory>
+  <vcpu placement='static'>12</vcpu>
+  <os>
+    <type arch='x86_64' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+  </features>
+  <cpu>
+    <numa>
+      <cell id='0' cpus='0-3' memory='2097152' unit='KiB'/>
+      <cell id='1' cpus='4-7' memory='2097152' unit='KiB'/>
+      <cell id='2' cpus='8-11' memory='2097152' unit='KiB'/>
+      <cell id='3' memory='2097152' unit='KiB'/>
+      <cell id='4' memory='2097152' unit='KiB'/>
+      <cell id='5' memory='2097152' unit='KiB'/>
+    </numa>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>restart</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu-system-x86_64</emulator>
+    <controller type='usb' index='0'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
+    </controller>
+    <controller type='pci' index='0' model='pci-root'/>
+    <input type='mouse' bus='ps2'/>
+    <input type='keyboard' bus='ps2'/>
+    <memballoon model='virtio'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </memballoon>
+  </devices>
+</domain>

tests/qemuxml2argvtest.c

@@ -1945,6 +1945,7 @@ mymain(void)
     DO_TEST_PARSE_ERROR("numatune-memnode-no-memory", NONE);
 
     DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);
+    DO_TEST("numatune-no-vcpu", NONE);
 
     DO_TEST("numatune-auto-nodeset-invalid", NONE);
     DO_TEST("numatune-auto-prefer", QEMU_CAPS_OBJECT_MEMORY_RAM,

tests/qemuxml2xmloutdata/numatune-no-vcpu.xml

@@ -0,0 +1 @@
+../qemuxml2argvdata/numatune-no-vcpu.xml

tests/qemuxml2xmltest.c

@@ -1126,6 +1126,7 @@ mymain(void)
     DO_TEST("numatune-memnode", QEMU_CAPS_NUMA, QEMU_CAPS_OBJECT_MEMORY_FILE);
     DO_TEST("numatune-memnode-no-memory", QEMU_CAPS_OBJECT_MEMORY_FILE);
     DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);
+    DO_TEST("numatune-no-vcpu", QEMU_CAPS_NUMA);
 
     DO_TEST("bios-nvram", NONE);
     DO_TEST("bios-nvram-os-interleave", NONE);