Implement NUMA info/apis in QEMU driver

ChangeLog

@@ -1,3 +1,12 @@
+Thu May 22 11:24:29 EST 2008 Daniel P. Berrange <berrange@redhat.com>
+
+	Support for NUMA info in the QEMU driver
+	* configure.in: check for libnuma
+	* libvirt.spec.in: add requirement on libnuma-devel at build
+	* src/Makefile.am: add NUMA compiler / linker flags
+	* src/qemu_conf.c: populate capabilities data with NUMA topology
+	* src/qemu_driver.c: implement free memory APIs
+
 Thu May 22 11:15:29 EST 2008 Daniel P. Berrange <berrange@redhat.com>
 
 	Support the free memory API calls in the remote driver/daemon

configure.in

@@ -534,6 +534,40 @@ AM_CONDITIONAL(HAVE_SELINUX, [test "$with_selinux" != "no"])
 AC_SUBST(SELINUX_CFLAGS)
 AC_SUBST(SELINUX_LIBS)
 
+dnl NUMA lib
+AC_ARG_WITH(numactl,
+  [  --with-numactl         use numactl for host topology info],
+  [],
+  [with_numactl=check])
+
+NUMACTL_CFLAGS=
+NUMACTL_LIBS=
+if test "$with_qemu" = "yes" -a "$with_numactl" != "no"; then
+  old_cflags="$CFLAGS"
+  old_libs="$LIBS"
+  if test "$with_numactl" = "check"; then
+    AC_CHECK_HEADER([numa.h],[],[with_numactl=no])
+    AC_CHECK_LIB(numa, numa_available,[],[with_numactl=no])
+    if test "$with_numactl" != "no"; then
+      with_numactl="yes"
+    fi
+  else
+    AC_CHECK_HEADER([numa.h],[],
+       [AC_MSG_ERROR([You must install the numactl development package in order to compile libvirt])])
+    AC_CHECK_LIB(numa, numa_available,[],
+       [AC_MSG_ERROR([You must install the numactl development package in order to compile and run libvirt])])
+  fi
+  CFLAGS="$old_cflags"
+  LIBS="$old_libs"
+fi
+if test "$with_numactl" = "yes"; then
+  NUMACTL_LIBS="-lnuma"
+  AC_DEFINE_UNQUOTED(HAVE_NUMACTL, 1, [whether Numactl is available for security])
+fi
+AM_CONDITIONAL(HAVE_NUMACTL, [test "$with_numactl" != "no"])
+AC_SUBST(NUMACTL_CFLAGS)
+AC_SUBST(NUMACTL_LIBS)
+
 dnl virsh libraries
 AC_CHECK_HEADERS([readline/readline.h])
 
@@ -1001,6 +1035,11 @@ AC_MSG_NOTICE([  selinux: $SELINUX_CFLAGS $SELINUX_LIBS])
 else
 AC_MSG_NOTICE([  selinux: no])
 fi
+if test "$with_numactl" = "yes" ; then
+AC_MSG_NOTICE([  numactl: $NUMACTL_CFLAGS $NUMACTL_LIBS])
+else
+AC_MSG_NOTICE([  numactl: no])
+fi
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([Miscellaneous])
 AC_MSG_NOTICE([])

libvirt.spec.in

@@ -67,6 +67,9 @@ BuildRequires: dnsmasq
 BuildRequires: bridge-utils
 BuildRequires: qemu
 BuildRequires: cyrus-sasl-devel
+%if %{with_qemu}
+BuildRequires: numactl-devel
+%endif
 %if %{with_polkit}
 BuildRequires: PolicyKit-devel >= 0.6
 %endif

src/Makefile.am

@@ -9,6 +9,7 @@ INCLUDES = \
 	   $(GNUTLS_CFLAGS) \
 	   $(SASL_CFLAGS) \
 	   $(SELINUX_CFLAGS) \
+	   $(NUMACTL_CFLAGS) \
 	   -DBINDIR=\""$(libexecdir)"\" \
 	   -DSBINDIR=\""$(sbindir)"\" \
 	   -DSYSCONF_DIR="\"$(sysconfdir)\"" \
@@ -100,6 +101,7 @@ endif
 
 libvirt_la_SOURCES = $(CLIENT_SOURCES) $(SERVER_SOURCES)
 libvirt_la_LIBADD = $(LIBXML_LIBS) $(GNUTLS_LIBS) $(SASL_LIBS) $(SELINUX_LIBS) \
+                    $(NUMACTL_LIBS) \
 		    @CYGWIN_EXTRA_LIBADD@ ../gnulib/lib/libgnu.la
 libvirt_la_LDFLAGS = -Wl,--version-script=$(srcdir)/libvirt_sym.version \
                      -version-info @LIBVIRT_VERSION_INFO@ \

src/qemu_conf.c

@@ -42,6 +42,10 @@
 #include <libxml/xpath.h>
 #include <libxml/uri.h>
 
+#if HAVE_NUMACTL
+#include <numa.h>
+#endif
+
 #include "libvirt/virterror.h"
 
 #include "qemu_conf.h"
@@ -49,6 +53,7 @@
 #include "buf.h"
 #include "conf.h"
 #include "util.h"
+#include "memory.h"
 #include "verify.h"
 #include "c-ctype.h"
 
@@ -390,6 +395,65 @@ qemudCapsInitGuest(virCapsPtr caps,
     return 0;
 }
 
+#if HAVE_NUMACTL
+#define MAX_CPUS 4096
+#define MAX_CPUS_MASK_SIZE (sizeof(unsigned long))
+#define MAX_CPUS_MASK_LEN (MAX_CPUS / MAX_CPUS_MASK_SIZE)
+#define MAX_CPUS_MASK_BYTES (MAX_CPUS / 8)
+
+#define MASK_CPU_ISSET(mask, cpu) \
+    (((mask)[((cpu) / MAX_CPUS_MASK_SIZE)] >> ((cpu) % MAX_CPUS_MASK_SIZE)) & 1)
+
+static int
+qemudCapsInitNUMA(virCapsPtr caps)
+{
+    int n, i;
+    unsigned long *mask = NULL;
+    int ncpus;
+    int *cpus = NULL;
+    int ret = -1;
+
+    if (numa_available() < 0)
+        return 0;
+
+    if (VIR_ALLOC_N(mask, MAX_CPUS_MASK_LEN) < 0)
+        goto cleanup;
+
+    for (n = 0 ; n <= numa_max_node() ; n++) {
+        if (numa_node_to_cpus(n, mask, MAX_CPUS_MASK_BYTES) < 0)
+            goto cleanup;
+
+        for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++)
+            if (MASK_CPU_ISSET(mask, i))
+                ncpus++;
+
+        if (VIR_ALLOC_N(cpus, ncpus) < 0)
+            goto cleanup;
+
+        for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++)
+            if (MASK_CPU_ISSET(mask, i))
+                cpus[ncpus++] = i;
+
+        if (virCapabilitiesAddHostNUMACell(caps,
+                                           n,
+                                           ncpus,
+                                           cpus) < 0)
+            goto cleanup;
+
+        VIR_FREE(cpus);
+    }
+
+    ret = 0;
+
+cleanup:
+    VIR_FREE(cpus);
+    VIR_FREE(mask);
+    return ret;
+}
+#else
+static int qemudCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; }
+#endif
+
 virCapsPtr qemudCapsInit(void) {
     struct utsname utsname;
     virCapsPtr caps;
@@ -402,6 +466,9 @@ virCapsPtr qemudCapsInit(void) {
                                    0, 0)) == NULL)
         goto no_memory;
 
+    if (qemudCapsInitNUMA(caps) < 0)
+        goto no_memory;
+
     for (i = 0 ; i < (sizeof(arch_info_hvm)/sizeof(arch_info_hvm[0])) ; i++)
         if (qemudCapsInitGuest(caps,
                                utsname.machine,

src/qemu_driver.c

@@ -46,6 +46,10 @@
 #include <sys/wait.h>
 #include <libxml/uri.h>
 
+#if HAVE_NUMACTL
+#include <numa.h>
+#endif
+
 #include "libvirt/virterror.h"
 
 #include "c-ctype.h"
@@ -1619,6 +1623,61 @@ static char *qemudGetCapabilities(virConnectPtr conn) {
 }
 
 
+#if HAVE_NUMACTL
+static int
+qemudNodeGetCellsFreeMemory(virConnectPtr conn,
+                            unsigned long long *freeMems,
+                            int startCell,
+                            int maxCells)
+{
+    int n, lastCell, numCells;
+
+    if (numa_available() < 0) {
+        qemudReportError(conn, NULL, NULL, VIR_ERR_NO_SUPPORT,
+                         "%s", _("NUMA not supported on this host"));
+        return -1;
+    }
+    lastCell = startCell + maxCells - 1;
+    if (lastCell > numa_max_node())
+        lastCell = numa_max_node();
+
+    for (numCells = 0, n = startCell ; n <= lastCell ; n++) {
+        long long mem;
+        if (numa_node_size64(n, &mem) < 0) {
+            qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                             "%s", _("Failed to query NUMA free memory"));
+            return -1;
+        }
+        freeMems[numCells++] = mem;
+    }
+    return numCells;
+}
+
+static unsigned long long
+qemudNodeGetFreeMemory (virConnectPtr conn)
+{
+    unsigned long long freeMem = 0;
+    int n;
+    if (numa_available() < 0) {
+        qemudReportError(conn, NULL, NULL, VIR_ERR_NO_SUPPORT,
+                         "%s", _("NUMA not supported on this host"));
+        return -1;
+    }
+
+    for (n = 0 ; n <= numa_max_node() ; n++) {
+        long long mem;
+        if (numa_node_size64(n, &mem) < 0) {
+            qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                             "%s", _("Failed to query NUMA free memory"));
+            return -1;
+        }
+        freeMem += mem;
+    }
+
+    return freeMem;
+}
+
+#endif
 
 static int qemudGetProcessInfo(unsigned long long *cpuTime, int pid) {
     char proc[PATH_MAX];
@@ -3182,8 +3241,13 @@ static virDriver qemuDriver = {
     NULL, /* domainMigrateFinish */
     qemudDomainBlockStats, /* domainBlockStats */
     qemudDomainInterfaceStats, /* domainInterfaceStats */
+#if HAVE_NUMACTL
+    qemudNodeGetCellsFreeMemory, /* nodeGetCellsFreeMemory */
+    qemudNodeGetFreeMemory,  /* getFreeMemory */
+#else
     NULL, /* nodeGetCellsFreeMemory */
     NULL, /* getFreeMemory */
+#endif
 };
 
 static virNetworkDriver qemuNetworkDriver = {