numatune: add support for per-node memory bindings in private APIs

Signed-off-by: Martin Kletzander <mkletzan@redhat.com>

src/conf/numatune_conf.c

@@ -63,6 +63,18 @@ struct _virDomainNumatune {
 };
 
 
+static inline bool
+virDomainNumatuneNodeSpecified(virDomainNumatunePtr numatune,
+                               int cellid)
+{
+    if (numatune &&
+        cellid >= 0 &&
+        cellid < numatune->nmem_nodes)
+        return numatune->mem_nodes[cellid].nodeset;
+
+    return false;
+}
+
 static int
 virDomainNumatuneNodeParseXML(virDomainDefPtr def,
                               xmlXPathContextPtr ctxt)
@@ -330,26 +342,37 @@ virDomainNumatuneFree(virDomainNumatunePtr numatune)
 }
 
 virDomainNumatuneMemMode
-virDomainNumatuneGetMode(virDomainNumatunePtr numatune)
+virDomainNumatuneGetMode(virDomainNumatunePtr numatune,
+                         int cellid)
 {
-    return (numatune && numatune->memory.specified) ? numatune->memory.mode : 0;
+    if (!numatune)
+        return 0;
+
+    if (virDomainNumatuneNodeSpecified(numatune, cellid))
+        return numatune->mem_nodes[cellid].mode;
+
+    if (numatune->memory.specified)
+        return numatune->memory.mode;
+
+    return 0;
 }
 
 virBitmapPtr
 virDomainNumatuneGetNodeset(virDomainNumatunePtr numatune,
-                            virBitmapPtr auto_nodeset)
+                            virBitmapPtr auto_nodeset,
+                            int cellid)
 {
     if (!numatune)
         return NULL;
 
-    if (numatune->memory.placement == VIR_DOMAIN_NUMATUNE_PLACEMENT_AUTO)
+    if (numatune->memory.specified &&
+        numatune->memory.placement == VIR_DOMAIN_NUMATUNE_PLACEMENT_AUTO)
         return auto_nodeset;
 
-    /*
+    if (virDomainNumatuneNodeSpecified(numatune, cellid))
-     * This weird logic has the same meaning as switch with
+        return numatune->mem_nodes[cellid].nodeset;
-     * auto/static/default, but can be more readably changed later.
+
-     */
+    if (!numatune->memory.specified)
-    if (numatune->memory.placement != VIR_DOMAIN_NUMATUNE_PLACEMENT_STATIC)
         return NULL;
 
     return numatune->memory.nodeset;
@@ -357,23 +380,31 @@ virDomainNumatuneGetNodeset(virDomainNumatunePtr numatune,
 
 char *
 virDomainNumatuneFormatNodeset(virDomainNumatunePtr numatune,
-                               virBitmapPtr auto_nodeset)
+                               virBitmapPtr auto_nodeset,
+                               int cellid)
 {
     return virBitmapFormat(virDomainNumatuneGetNodeset(numatune,
-                                                       auto_nodeset));
+                                                       auto_nodeset,
+                                                       cellid));
 }
 
 int
 virDomainNumatuneMaybeFormatNodeset(virDomainNumatunePtr numatune,
                                     virBitmapPtr auto_nodeset,
-                                    char **mask)
+                                    char **mask,
+                                    int cellid)
 {
     *mask = NULL;
 
     if (!numatune)
         return 0;
 
-    if (numatune->memory.placement == VIR_DOMAIN_NUMATUNE_PLACEMENT_AUTO &&
+    if (!virDomainNumatuneNodeSpecified(numatune, cellid) &&
+        !numatune->memory.specified)
+        return 0;
+
+    if (numatune->memory.specified &&
+        numatune->memory.placement == VIR_DOMAIN_NUMATUNE_PLACEMENT_AUTO &&
         !auto_nodeset) {
         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("Advice from numad is needed in case of "
@@ -381,7 +412,7 @@ virDomainNumatuneMaybeFormatNodeset(virDomainNumatunePtr numatune,
         return -1;
     }
 
-    *mask = virDomainNumatuneFormatNodeset(numatune, auto_nodeset);
+    *mask = virDomainNumatuneFormatNodeset(numatune, auto_nodeset, cellid);
     if (!*mask)
         return -1;
 
@@ -475,6 +506,35 @@ virDomainNumatuneSet(virDomainDefPtr def,
     return ret;
 }
 
+static bool
+virDomainNumatuneNodesEqual(virDomainNumatunePtr n1,
+                            virDomainNumatunePtr n2)
+{
+    size_t i = 0;
+
+    if (n1->nmem_nodes != n2->nmem_nodes)
+        return false;
+
+    for (i = 0; i < n1->nmem_nodes; i++) {
+        virDomainNumatuneNodePtr nd1 = &n1->mem_nodes[i];
+        virDomainNumatuneNodePtr nd2 = &n2->mem_nodes[i];
+
+        if (!nd1->nodeset && !nd2->nodeset)
+            continue;
+
+        if (!nd1->nodeset || !nd2->nodeset)
+            return false;
+
+        if (nd1->mode != nd2->mode)
+            return false;
+
+        if (!virBitmapEqual(nd1->nodeset, nd2->nodeset))
+            return false;
+    }
+
+    return true;
+}
+
 bool
 virDomainNumatuneEquals(virDomainNumatunePtr n1,
                         virDomainNumatunePtr n2)
@@ -486,7 +546,7 @@ virDomainNumatuneEquals(virDomainNumatunePtr n1,
         return false;
 
     if (!n1->memory.specified && !n2->memory.specified)
-        return true;
+        return virDomainNumatuneNodesEqual(n1, n2);
 
     if (!n1->memory.specified || !n2->memory.specified)
         return false;
@@ -497,7 +557,10 @@ virDomainNumatuneEquals(virDomainNumatunePtr n1,
     if (n1->memory.placement != n2->memory.placement)
         return false;
 
-    return virBitmapEqual(n1->memory.nodeset, n2->memory.nodeset);
+    if (!virBitmapEqual(n1->memory.nodeset, n2->memory.nodeset))
+        return false;
+
+    return virDomainNumatuneNodesEqual(n1, n2);
 }
 
 bool
@@ -514,3 +577,19 @@ virDomainNumatuneHasPlacementAuto(virDomainNumatunePtr numatune)
 
     return false;
 }
+
+bool
+virDomainNumatuneHasPerNodeBinding(virDomainNumatunePtr numatune)
+{
+    size_t i = 0;
+
+    if (!numatune)
+        return false;
+
+    for (i = 0; i < numatune->nmem_nodes; i++) {
+        if (numatune->mem_nodes[i].nodeset)
+            return true;
+    }
+
+    return false;
+}

src/conf/numatune_conf.h

@@ -69,20 +69,24 @@ int virDomainNumatuneFormatXML(virBufferPtr buf, virDomainNumatunePtr numatune)
 /*
  * Getters
  */
-virDomainNumatuneMemMode virDomainNumatuneGetMode(virDomainNumatunePtr numatune);
+virDomainNumatuneMemMode virDomainNumatuneGetMode(virDomainNumatunePtr numatune,
+                                                  int cellid);
 
 virBitmapPtr virDomainNumatuneGetNodeset(virDomainNumatunePtr numatune,
-                                         virBitmapPtr auto_nodeset);
+                                         virBitmapPtr auto_nodeset,
+                                         int cellid);
 
 /*
  * Formatters
  */
 char *virDomainNumatuneFormatNodeset(virDomainNumatunePtr numatune,
-                                     virBitmapPtr auto_nodeset);
+                                     virBitmapPtr auto_nodeset,
+                                     int cellid);
 
 int virDomainNumatuneMaybeFormatNodeset(virDomainNumatunePtr numatune,
                                         virBitmapPtr auto_nodeset,
-                                        char **mask);
+                                        char **mask,
+                                        int cellid);
 
 /*
  * Setters
@@ -99,4 +103,6 @@ bool virDomainNumatuneEquals(virDomainNumatunePtr n1,
 
 bool virDomainNumatuneHasPlacementAuto(virDomainNumatunePtr numatune);
 
+bool virDomainNumatuneHasPerNodeBinding(virDomainNumatunePtr numatune);
+
 #endif /* __NUMATUNE_CONF_H__ */

src/libvirt_private.syms

@@ -614,6 +614,7 @@ virDomainNumatuneFormatXML;
 virDomainNumatuneFree;
 virDomainNumatuneGetMode;
 virDomainNumatuneGetNodeset;
+virDomainNumatuneHasPerNodeBinding;
 virDomainNumatuneHasPlacementAuto;
 virDomainNumatuneMaybeFormatNodeset;
 virDomainNumatuneMemModeTypeFromString;

src/lxc/lxc_cgroup.c

@@ -79,7 +79,8 @@ static int virLXCCgroupSetupCpusetTune(virDomainDefPtr def,
             goto cleanup;
     }
 
-    if (virDomainNumatuneMaybeFormatNodeset(def->numatune, nodemask, &mask) < 0)
+    if (virDomainNumatuneMaybeFormatNodeset(def->numatune, nodemask,
+                                            &mask, -1) < 0)
         goto cleanup;
 
     if (mask && virCgroupSetCpusetMems(cgroup, mask) < 0)

src/qemu/qemu_cgroup.c

@@ -603,7 +603,7 @@ qemuSetupCpusetCgroup(virDomainObjPtr vm,
 
     if (virDomainNumatuneMaybeFormatNodeset(vm->def->numatune,
                                             nodemask,
-                                            &mem_mask) < 0)
+                                            &mem_mask, -1) < 0)
         goto cleanup;
 
     if (mem_mask &&

src/qemu/qemu_driver.c

@@ -8637,7 +8637,7 @@ qemuDomainSetNumaParamsLive(virDomainObjPtr vm,
     size_t i = 0;
     int ret = -1;
 
-    if (virDomainNumatuneGetMode(vm->def->numatune) !=
+    if (virDomainNumatuneGetMode(vm->def->numatune, -1) !=
         VIR_DOMAIN_NUMATUNE_MEM_STRICT) {
         virReportError(VIR_ERR_OPERATION_INVALID, "%s",
                        _("change of nodeset for running domain "
@@ -8778,7 +8778,7 @@ qemuDomainSetNumaParameters(virDomainPtr dom,
 
     if (flags & VIR_DOMAIN_AFFECT_LIVE) {
         if (mode != -1 &&
-            virDomainNumatuneGetMode(vm->def->numatune) != mode) {
+            virDomainNumatuneGetMode(vm->def->numatune, -1) != mode) {
             virReportError(VIR_ERR_OPERATION_INVALID, "%s",
                            _("can't change numatune mode for running domain"));
             goto cleanup;
@@ -8874,15 +8874,15 @@ qemuDomainGetNumaParameters(virDomainPtr dom,
                 goto cleanup;
 
             if (flags & VIR_DOMAIN_AFFECT_CONFIG)
-                param->value.i = virDomainNumatuneGetMode(persistentDef->numatune);
+                param->value.i = virDomainNumatuneGetMode(persistentDef->numatune, -1);
             else
-                param->value.i = virDomainNumatuneGetMode(vm->def->numatune);
+                param->value.i = virDomainNumatuneGetMode(vm->def->numatune, -1);
             break;
 
         case 1: /* fill numa nodeset here */
             if (flags & VIR_DOMAIN_AFFECT_CONFIG) {
                 nodeset = virDomainNumatuneFormatNodeset(persistentDef->numatune,
-                                                         NULL);
+                                                         NULL, -1);
                 if (!nodeset)
                     goto cleanup;
             } else {

src/util/virnuma.c

@@ -98,7 +98,7 @@ virNumaSetupMemoryPolicy(virDomainNumatunePtr numatune,
     int maxnode = 0;
     virBitmapPtr tmp_nodemask = NULL;
 
-    tmp_nodemask = virDomainNumatuneGetNodeset(numatune, nodemask);
+    tmp_nodemask = virDomainNumatuneGetNodeset(numatune, nodemask, -1);
     if (!tmp_nodemask)
         return 0;
 
@@ -123,7 +123,7 @@ virNumaSetupMemoryPolicy(virDomainNumatunePtr numatune,
         nodemask_set(&mask, bit);
     }
 
-    switch (virDomainNumatuneGetMode(numatune)) {
+    switch (virDomainNumatuneGetMode(numatune, -1)) {
     case VIR_DOMAIN_NUMATUNE_MEM_STRICT:
         numa_set_bind_policy(1);
         numa_set_membind(&mask);
@@ -320,7 +320,7 @@ int
 virNumaSetupMemoryPolicy(virDomainNumatunePtr numatune,
                          virBitmapPtr nodemask ATTRIBUTE_UNUSED)
 {
-    if (virDomainNumatuneGetNodeset(numatune, NULL)) {
+    if (virDomainNumatuneGetNodeset(numatune, NULL, -1)) {
         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                        _("libvirt is compiled without NUMA tuning support"));