From d010b68962ec2e8101d2d8f3060fab704c9969fb Mon Sep 17 00:00:00 2001 From: Jim Meyering Date: Sun, 21 Dec 2008 18:55:09 +0000 Subject: [PATCH] make NUMA-initialization code more portable and more robust qemudCapsInitNUMA and umlCapsInitNUMA were identical, so this change factors them into a new function, virCapsInitNUMA, and puts it in nodeinfo.c. In addition to factoring out the duplicates, this change also adjusts that function definition (along with its macros) so that it works with Fedora 9's numactl version 1, and makes it so the code will work even if someone builds the kernel with CONFIG_NR_CPUS > 4096. Finally, also perform this NUMA initialization for the lxc and openvz drivers. * src/nodeinfo.c: Include , and "memory.h". (virCapsInitNUMA): Rename from qemudCapsInitNUMA and umlCapsInitNUMA. (NUMA_MAX_N_CPUS): Define depending on NUMA API version. (n_bits, MASK_CPU_ISSET): Define, adjust, use uint64 rather than long. * src/nodeinfo.h: Include "capabilities.h". (virCapsInitNUMA): Declare it. * examples/domain-events/events-c/Makefile.am: * src/Makefile.am: Add $(NUMACTL_CFLAGS) and $(NUMACTL_LIBS) to various compile/link-related variables. * src/qemu_conf.c: Include "nodeinfo.h". (qemudCapsInitNUMA): Remove duplicate code. Adjust caller. * src/uml_conf.c (umlCapsInitNUMA): Likewise. Include "nodeinfo.h". * src/lxc_conf.c: Include "nodeinfo.h". (lxcCapsInit): Perform NUMA initialization here, too. * src/openvz_conf.c (openvzCapsInit): And here. Include "nodeinfo.h". * src/libvirt_sym.version.in: Add virCapsInitNUMA so that libvirtd can link to this function. --- ChangeLog | 36 ++++++++++++++++++ docs/examples/.cvsignore | 1 - src/Makefile.am | 21 ++++++----- src/libvirt_sym.version.in | 1 + src/lxc_conf.c | 6 ++- src/nodeinfo.c | 75 +++++++++++++++++++++++++++++++++++++- src/nodeinfo.h | 4 +- src/openvz_conf.c | 3 ++ src/qemu_conf.c | 68 +--------------------------------- src/uml_conf.c | 70 +---------------------------------- 10 files changed, 136 insertions(+), 149 deletions(-) diff --git a/ChangeLog b/ChangeLog index 193117179c..fb3f8876c5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,39 @@ +Sun Dec 21 19:50:16 +0100 2008 Jim Meyering + + make NUMA-initialization code more portable and more robust + qemudCapsInitNUMA and umlCapsInitNUMA were identical, so this change + factors them into a new function, virCapsInitNUMA, and puts it in + nodeinfo.c. + + In addition to factoring out the duplicates, this change also + adjusts that function definition (along with its macros) so + that it works with Fedora 9's numactl version 1, and makes it + so the code will work even if someone builds the kernel with + CONFIG_NR_CPUS > 4096. + + Finally, also perform this NUMA initialization for the lxc + and openvz drivers. + + * src/nodeinfo.c: Include , and "memory.h". + (virCapsInitNUMA): Rename from qemudCapsInitNUMA and umlCapsInitNUMA. + (NUMA_MAX_N_CPUS): Define depending on NUMA API version. + (n_bits, MASK_CPU_ISSET): Define, adjust, use uint64 rather than long. + * src/nodeinfo.h: Include "capabilities.h". + (virCapsInitNUMA): Declare it. + * examples/domain-events/events-c/Makefile.am: + * src/Makefile.am: Add $(NUMACTL_CFLAGS) and $(NUMACTL_LIBS) to various + compile/link-related variables. + * src/qemu_conf.c: Include "nodeinfo.h". + (qemudCapsInitNUMA): Remove duplicate code. Adjust caller. + * src/uml_conf.c (umlCapsInitNUMA): Likewise. + Include "nodeinfo.h". + * src/lxc_conf.c: Include "nodeinfo.h". + (lxcCapsInit): Perform NUMA initialization here, too. + * src/openvz_conf.c (openvzCapsInit): And here. + Include "nodeinfo.h". + * src/libvirt_sym.version.in: Add virCapsInitNUMA so that libvirtd + can link to this function. + Sun Dec 21 19:46:35 +0100 2008 Jim Meyering * src/node_device_hal.c: Include before everything else. diff --git a/docs/examples/.cvsignore b/docs/examples/.cvsignore index ad3227c51c..5f0e25166a 100644 --- a/docs/examples/.cvsignore +++ b/docs/examples/.cvsignore @@ -1,4 +1,3 @@ -*.exe .memdump Makefile.in Makefile diff --git a/src/Makefile.am b/src/Makefile.am index 68f0c017d4..6c65f91585 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -55,7 +55,7 @@ UTIL_SOURCES = \ xml.c xml.h # Internal generic driver infrastructure -DRIVER_SOURCES = \ +DRIVER_SOURCES = \ driver.c driver.h \ internal.h \ datatypes.c datatypes.h \ @@ -147,7 +147,7 @@ STORAGE_DRIVER_FS_SOURCES = \ storage_backend_fs.h storage_backend_fs.c STORAGE_DRIVER_LVM_SOURCES = \ - storage_backend_logical.h \ + storage_backend_logical.h \ storage_backend_logical.c STORAGE_DRIVER_ISCSI_SOURCES = \ @@ -189,8 +189,8 @@ libvirt_driver_la_SOURCES = \ $(STORAGE_CONF_SOURCES) \ $(NODE_DEVICE_CONF_SOURCES) -libvirt_driver_la_CFLAGS = $(XEN_CFLAGS) -libvirt_driver_la_LDFLAGS = $(XEN_LIBS) +libvirt_driver_la_CFLAGS = $(XEN_CFLAGS) $(NUMACTL_CFLAGS) +libvirt_driver_la_LDFLAGS = $(XEN_LIBS) $(NUMACTL_LIBS) if WITH_TEST if WITH_DRIVER_MODULES @@ -430,13 +430,14 @@ virsh_SOURCES = \ virsh.c virsh_LDFLAGS = $(WARN_CFLAGS) $(COVERAGE_LDFLAGS) -virsh_LDADD = \ - $(STATIC_BINARIES) \ +virsh_LDADD = \ + $(STATIC_BINARIES) \ $(WARN_CFLAGS) \ + $(NUMACTL_LIBS) \ libvirt.la \ ../gnulib/lib/libgnu.la \ $(VIRSH_LIBS) -virsh_CFLAGS = $(COVERAGE_CFLAGS) $(READLINE_CFLAGS) +virsh_CFLAGS = $(COVERAGE_CFLAGS) $(READLINE_CFLAGS) $(NUMACTL_CFLAGS) BUILT_SOURCES = virsh-net-edit.c virsh-pool-edit.c virsh-net-edit.c: virsh.c Makefile.am @@ -518,11 +519,11 @@ libexec_PROGRAMS += libvirt_lxc libvirt_lxc_SOURCES = \ $(LXC_CONTROLLER_SOURCES) \ - $(UTIL_SOURCES) \ + $(UTIL_SOURCES) \ $(DOMAIN_CONF_SOURCES) libvirt_lxc_LDFLAGS = $(WARN_CFLAGS) $(COVERAGE_LDCFLAGS) -libvirt_lxc_LDADD = $(LIBXML_LIBS) ../gnulib/lib/libgnu.la -libvirt_lxc_CFLAGS = $(LIBPARTED_CFLAGS) +libvirt_lxc_LDADD = $(LIBXML_LIBS) $(NUMACTL_LIBS) ../gnulib/lib/libgnu.la +libvirt_lxc_CFLAGS = $(LIBPARTED_CFLAGS) $(NUMACTL_CFLAGS) endif endif EXTRA_DIST += $(LXC_CONTROLLER_SOURCES) diff --git a/src/libvirt_sym.version.in b/src/libvirt_sym.version.in index b3812b6fe1..91b08010ec 100644 --- a/src/libvirt_sym.version.in +++ b/src/libvirt_sym.version.in @@ -491,6 +491,7 @@ LIBVIRT_PRIVATE_@VERSION@ { # nodeinfo.h virNodeInfoPopulate; + virCapsInitNUMA; # node_device_conf.h diff --git a/src/lxc_conf.c b/src/lxc_conf.c index 0db9bb59b3..fe964c2cd8 100644 --- a/src/lxc_conf.c +++ b/src/lxc_conf.c @@ -27,8 +27,9 @@ #include -#include "virterror_internal.h" #include "lxc_conf.h" +#include "nodeinfo.h" +#include "virterror_internal.h" /* Functions */ virCapsPtr lxcCapsInit(void) @@ -43,6 +44,9 @@ virCapsPtr lxcCapsInit(void) 0, 0)) == NULL) goto no_memory; + if (virCapsInitNUMA(caps) < 0) + goto no_memory; + /* XXX shouldn't 'borrow' KVM's prefix */ virCapabilitiesSetMacPrefix(caps, (unsigned char []){ 0x52, 0x54, 0x00 }); diff --git a/src/nodeinfo.c b/src/nodeinfo.c index fd58c2bb6c..d9f2b1d701 100644 --- a/src/nodeinfo.c +++ b/src/nodeinfo.c @@ -26,17 +26,24 @@ #include #include #include +#include #include -#include "c-ctype.h" + +#if HAVE_NUMACTL +# define NUMA_VERSION1_COMPATIBILITY 1 +# include +#endif #ifdef HAVE_SYS_UTSNAME_H #include #endif -#include "virterror_internal.h" +#include "c-ctype.h" +#include "memory.h" #include "nodeinfo.h" #include "physmem.h" #include "util.h" +#include "virterror_internal.h" #ifdef __linux__ #define CPUINFO_PATH "/proc/cpuinfo" @@ -171,3 +178,67 @@ int virNodeInfoPopulate(virConnectPtr conn, return -1; #endif } + +#if HAVE_NUMACTL +# if LIBNUMA_API_VERSION <= 1 +# define NUMA_MAX_N_CPUS 4096 +# else +# define NUMA_MAX_N_CPUS (numa_all_cpus_ptr->size) +# endif + +# define n_bits(var) (8 * sizeof(var)) +# define MASK_CPU_ISSET(mask, cpu) \ + (((mask)[((cpu) / n_bits(*(mask)))] >> ((cpu) % n_bits(*(mask)))) & 1) + +int +virCapsInitNUMA(virCapsPtr caps) +{ + int n; + uint64_t *mask = NULL; + int *cpus = NULL; + int ret = -1; + int max_n_cpus = NUMA_MAX_N_CPUS; + + if (numa_available() < 0) + return 0; + + int mask_n_bytes = max_n_cpus / 8; + if (VIR_ALLOC_N(mask, mask_n_bytes / sizeof *mask) < 0) + goto cleanup; + + for (n = 0 ; n <= numa_max_node() ; n++) { + int i; + int ncpus; + if (numa_node_to_cpus(n, mask, mask_n_bytes) < 0) + goto cleanup; + + for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) + if (MASK_CPU_ISSET(mask, i)) + ncpus++; + + if (VIR_ALLOC_N(cpus, ncpus) < 0) + goto cleanup; + + for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) + if (MASK_CPU_ISSET(mask, i)) + cpus[ncpus++] = i; + + if (virCapabilitiesAddHostNUMACell(caps, + n, + ncpus, + cpus) < 0) + goto cleanup; + + VIR_FREE(cpus); + } + + ret = 0; + +cleanup: + VIR_FREE(cpus); + VIR_FREE(mask); + return ret; +} +#else +int virCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; } +#endif diff --git a/src/nodeinfo.h b/src/nodeinfo.h index e7c78eb0bf..030f0ee2b3 100644 --- a/src/nodeinfo.h +++ b/src/nodeinfo.h @@ -1,7 +1,7 @@ /* * nodeinfo.c: Helper routines for OS specific node information * - * Copyright (C) 2006, 2007 Red Hat, Inc. + * Copyright (C) 2006-2008 Red Hat, Inc. * Copyright (C) 2006 Daniel P. Berrange * * This library is free software; you can redistribute it and/or @@ -25,7 +25,9 @@ #define __VIR_NODEINFO_H__ #include "libvirt/libvirt.h" +#include "capabilities.h" int virNodeInfoPopulate(virConnectPtr conn, virNodeInfoPtr nodeinfo); +int virCapsInitNUMA(virCapsPtr caps); #endif /* __VIR_NODEINFO_H__*/ diff --git a/src/openvz_conf.c b/src/openvz_conf.c index 44a243bfe1..49cc183334 100644 --- a/src/openvz_conf.c +++ b/src/openvz_conf.c @@ -146,6 +146,9 @@ virCapsPtr openvzCapsInit(void) 0, 0)) == NULL) goto no_memory; + if (virCapsInitNUMA(caps) < 0) + goto no_memory; + virCapabilitiesSetMacPrefix(caps, (unsigned char[]){ 0x52, 0x54, 0x00 }); if ((guest = virCapabilitiesAddGuest(caps, diff --git a/src/qemu_conf.c b/src/qemu_conf.c index 469bf1feb5..f78678ee84 100644 --- a/src/qemu_conf.c +++ b/src/qemu_conf.c @@ -36,11 +36,6 @@ #include #include -#if HAVE_NUMACTL -#define NUMA_VERSION1_COMPATIBILITY 1 -#include -#endif - #include "virterror_internal.h" #include "qemu_conf.h" #include "uuid.h" @@ -51,6 +46,7 @@ #include "verify.h" #include "datatypes.h" #include "xml.h" +#include "nodeinfo.h" VIR_ENUM_DECL(virDomainDiskQEMUBus) VIR_ENUM_IMPL(virDomainDiskQEMUBus, VIR_DOMAIN_DISK_BUS_LAST, @@ -300,66 +296,6 @@ qemudCapsInitGuest(virCapsPtr caps, return 0; } -#if HAVE_NUMACTL -#define MAX_CPUS 4096 -#define MAX_CPUS_MASK_SIZE (sizeof(unsigned long)) -#define MAX_CPUS_MASK_BITS (MAX_CPUS_MASK_SIZE * 8) -#define MAX_CPUS_MASK_LEN (MAX_CPUS / (MAX_CPUS_MASK_BITS)) - -#define MASK_CPU_ISSET(mask, cpu) \ - (((mask)[((cpu) / MAX_CPUS_MASK_BITS)] >> ((cpu) % MAX_CPUS_MASK_BITS)) & 1) - -static int -qemudCapsInitNUMA(virCapsPtr caps) -{ - int n, i; - unsigned long *mask = NULL; - int ncpus; - int *cpus = NULL; - int ret = -1; - - if (numa_available() < 0) - return 0; - - if (VIR_ALLOC_N(mask, MAX_CPUS_MASK_LEN) < 0) - goto cleanup; - - for (n = 0 ; n <= numa_max_node() ; n++) { - int mask_n_bytes = numa_all_cpus_ptr->size / 8; - if (numa_node_to_cpus(n, mask, mask_n_bytes) < 0) - goto cleanup; - - for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++) - if (MASK_CPU_ISSET(mask, i)) - ncpus++; - - if (VIR_ALLOC_N(cpus, ncpus) < 0) - goto cleanup; - - for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++) - if (MASK_CPU_ISSET(mask, i)) - cpus[ncpus++] = i; - - if (virCapabilitiesAddHostNUMACell(caps, - n, - ncpus, - cpus) < 0) - goto cleanup; - - VIR_FREE(cpus); - } - - ret = 0; - -cleanup: - VIR_FREE(cpus); - VIR_FREE(mask); - return ret; -} -#else -static int qemudCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; } -#endif - virCapsPtr qemudCapsInit(void) { struct utsname utsname; virCapsPtr caps; @@ -375,7 +311,7 @@ virCapsPtr qemudCapsInit(void) { /* Using KVM's mac prefix for QEMU too */ virCapabilitiesSetMacPrefix(caps, (unsigned char[]){ 0x52, 0x54, 0x00 }); - if (qemudCapsInitNUMA(caps) < 0) + if (virCapsInitNUMA(caps) < 0) goto no_memory; for (i = 0 ; i < ARRAY_CARDINALITY(arch_info_hvm) ; i++) diff --git a/src/uml_conf.c b/src/uml_conf.c index 00adf27357..8b564be056 100644 --- a/src/uml_conf.c +++ b/src/uml_conf.c @@ -36,84 +36,18 @@ #include #include -#if HAVE_NUMACTL -#define NUMA_VERSION1_COMPATIBILITY 1 -#include -#endif - #include "uml_conf.h" #include "uuid.h" #include "buf.h" #include "conf.h" #include "util.h" #include "memory.h" +#include "nodeinfo.h" #include "verify.h" #define umlLog(level, msg...) fprintf(stderr, msg) - - -#if HAVE_NUMACTL -#define MAX_CPUS 4096 -#define MAX_CPUS_MASK_SIZE (sizeof(unsigned long)) -#define MAX_CPUS_MASK_BITS (MAX_CPUS_MASK_SIZE * 8) -#define MAX_CPUS_MASK_LEN (MAX_CPUS / (MAX_CPUS_MASK_BITS)) - -#define MASK_CPU_ISSET(mask, cpu) \ - (((mask)[((cpu) / MAX_CPUS_MASK_BITS)] >> ((cpu) % MAX_CPUS_MASK_BITS)) & 1) - -static int -umlCapsInitNUMA(virCapsPtr caps) -{ - int n, i; - unsigned long *mask = NULL; - int ncpus; - int *cpus = NULL; - int ret = -1; - - if (numa_available() < 0) - return 0; - - if (VIR_ALLOC_N(mask, MAX_CPUS_MASK_LEN) < 0) - goto cleanup; - - for (n = 0 ; n <= numa_max_node() ; n++) { - int mask_n_bytes = numa_all_cpus_ptr->size / 8; - if (numa_node_to_cpus(n, mask, mask_n_bytes) < 0) - goto cleanup; - - for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++) - if (MASK_CPU_ISSET(mask, i)) - ncpus++; - - if (VIR_ALLOC_N(cpus, ncpus) < 0) - goto cleanup; - - for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++) - if (MASK_CPU_ISSET(mask, i)) - cpus[ncpus++] = i; - - if (virCapabilitiesAddHostNUMACell(caps, - n, - ncpus, - cpus) < 0) - goto cleanup; - - VIR_FREE(cpus); - } - - ret = 0; - -cleanup: - VIR_FREE(cpus); - VIR_FREE(mask); - return ret; -} -#else -static int umlCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; } -#endif - virCapsPtr umlCapsInit(void) { struct utsname utsname; virCapsPtr caps; @@ -126,7 +60,7 @@ virCapsPtr umlCapsInit(void) { 0, 0)) == NULL) goto no_memory; - if (umlCapsInitNUMA(caps) < 0) + if (virCapsInitNUMA(caps) < 0) goto no_memory; if ((guest = virCapabilitiesAddGuest(caps,