diff --git a/ChangeLog b/ChangeLog index 193117179c..fb3f8876c5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,39 @@ +Sun Dec 21 19:50:16 +0100 2008 Jim Meyering + + make NUMA-initialization code more portable and more robust + qemudCapsInitNUMA and umlCapsInitNUMA were identical, so this change + factors them into a new function, virCapsInitNUMA, and puts it in + nodeinfo.c. + + In addition to factoring out the duplicates, this change also + adjusts that function definition (along with its macros) so + that it works with Fedora 9's numactl version 1, and makes it + so the code will work even if someone builds the kernel with + CONFIG_NR_CPUS > 4096. + + Finally, also perform this NUMA initialization for the lxc + and openvz drivers. + + * src/nodeinfo.c: Include , and "memory.h". + (virCapsInitNUMA): Rename from qemudCapsInitNUMA and umlCapsInitNUMA. + (NUMA_MAX_N_CPUS): Define depending on NUMA API version. + (n_bits, MASK_CPU_ISSET): Define, adjust, use uint64 rather than long. + * src/nodeinfo.h: Include "capabilities.h". + (virCapsInitNUMA): Declare it. + * examples/domain-events/events-c/Makefile.am: + * src/Makefile.am: Add $(NUMACTL_CFLAGS) and $(NUMACTL_LIBS) to various + compile/link-related variables. + * src/qemu_conf.c: Include "nodeinfo.h". + (qemudCapsInitNUMA): Remove duplicate code. Adjust caller. + * src/uml_conf.c (umlCapsInitNUMA): Likewise. + Include "nodeinfo.h". + * src/lxc_conf.c: Include "nodeinfo.h". + (lxcCapsInit): Perform NUMA initialization here, too. + * src/openvz_conf.c (openvzCapsInit): And here. + Include "nodeinfo.h". + * src/libvirt_sym.version.in: Add virCapsInitNUMA so that libvirtd + can link to this function. + Sun Dec 21 19:46:35 +0100 2008 Jim Meyering * src/node_device_hal.c: Include before everything else. diff --git a/docs/examples/.cvsignore b/docs/examples/.cvsignore index ad3227c51c..5f0e25166a 100644 --- a/docs/examples/.cvsignore +++ b/docs/examples/.cvsignore @@ -1,4 +1,3 @@ -*.exe .memdump Makefile.in Makefile diff --git a/src/Makefile.am b/src/Makefile.am index 68f0c017d4..6c65f91585 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -55,7 +55,7 @@ UTIL_SOURCES = \ xml.c xml.h # Internal generic driver infrastructure -DRIVER_SOURCES = \ +DRIVER_SOURCES = \ driver.c driver.h \ internal.h \ datatypes.c datatypes.h \ @@ -147,7 +147,7 @@ STORAGE_DRIVER_FS_SOURCES = \ storage_backend_fs.h storage_backend_fs.c STORAGE_DRIVER_LVM_SOURCES = \ - storage_backend_logical.h \ + storage_backend_logical.h \ storage_backend_logical.c STORAGE_DRIVER_ISCSI_SOURCES = \ @@ -189,8 +189,8 @@ libvirt_driver_la_SOURCES = \ $(STORAGE_CONF_SOURCES) \ $(NODE_DEVICE_CONF_SOURCES) -libvirt_driver_la_CFLAGS = $(XEN_CFLAGS) -libvirt_driver_la_LDFLAGS = $(XEN_LIBS) +libvirt_driver_la_CFLAGS = $(XEN_CFLAGS) $(NUMACTL_CFLAGS) +libvirt_driver_la_LDFLAGS = $(XEN_LIBS) $(NUMACTL_LIBS) if WITH_TEST if WITH_DRIVER_MODULES @@ -430,13 +430,14 @@ virsh_SOURCES = \ virsh.c virsh_LDFLAGS = $(WARN_CFLAGS) $(COVERAGE_LDFLAGS) -virsh_LDADD = \ - $(STATIC_BINARIES) \ +virsh_LDADD = \ + $(STATIC_BINARIES) \ $(WARN_CFLAGS) \ + $(NUMACTL_LIBS) \ libvirt.la \ ../gnulib/lib/libgnu.la \ $(VIRSH_LIBS) -virsh_CFLAGS = $(COVERAGE_CFLAGS) $(READLINE_CFLAGS) +virsh_CFLAGS = $(COVERAGE_CFLAGS) $(READLINE_CFLAGS) $(NUMACTL_CFLAGS) BUILT_SOURCES = virsh-net-edit.c virsh-pool-edit.c virsh-net-edit.c: virsh.c Makefile.am @@ -518,11 +519,11 @@ libexec_PROGRAMS += libvirt_lxc libvirt_lxc_SOURCES = \ $(LXC_CONTROLLER_SOURCES) \ - $(UTIL_SOURCES) \ + $(UTIL_SOURCES) \ $(DOMAIN_CONF_SOURCES) libvirt_lxc_LDFLAGS = $(WARN_CFLAGS) $(COVERAGE_LDCFLAGS) -libvirt_lxc_LDADD = $(LIBXML_LIBS) ../gnulib/lib/libgnu.la -libvirt_lxc_CFLAGS = $(LIBPARTED_CFLAGS) +libvirt_lxc_LDADD = $(LIBXML_LIBS) $(NUMACTL_LIBS) ../gnulib/lib/libgnu.la +libvirt_lxc_CFLAGS = $(LIBPARTED_CFLAGS) $(NUMACTL_CFLAGS) endif endif EXTRA_DIST += $(LXC_CONTROLLER_SOURCES) diff --git a/src/libvirt_sym.version.in b/src/libvirt_sym.version.in index b3812b6fe1..91b08010ec 100644 --- a/src/libvirt_sym.version.in +++ b/src/libvirt_sym.version.in @@ -491,6 +491,7 @@ LIBVIRT_PRIVATE_@VERSION@ { # nodeinfo.h virNodeInfoPopulate; + virCapsInitNUMA; # node_device_conf.h diff --git a/src/lxc_conf.c b/src/lxc_conf.c index 0db9bb59b3..fe964c2cd8 100644 --- a/src/lxc_conf.c +++ b/src/lxc_conf.c @@ -27,8 +27,9 @@ #include -#include "virterror_internal.h" #include "lxc_conf.h" +#include "nodeinfo.h" +#include "virterror_internal.h" /* Functions */ virCapsPtr lxcCapsInit(void) @@ -43,6 +44,9 @@ virCapsPtr lxcCapsInit(void) 0, 0)) == NULL) goto no_memory; + if (virCapsInitNUMA(caps) < 0) + goto no_memory; + /* XXX shouldn't 'borrow' KVM's prefix */ virCapabilitiesSetMacPrefix(caps, (unsigned char []){ 0x52, 0x54, 0x00 }); diff --git a/src/nodeinfo.c b/src/nodeinfo.c index fd58c2bb6c..d9f2b1d701 100644 --- a/src/nodeinfo.c +++ b/src/nodeinfo.c @@ -26,17 +26,24 @@ #include #include #include +#include #include -#include "c-ctype.h" + +#if HAVE_NUMACTL +# define NUMA_VERSION1_COMPATIBILITY 1 +# include +#endif #ifdef HAVE_SYS_UTSNAME_H #include #endif -#include "virterror_internal.h" +#include "c-ctype.h" +#include "memory.h" #include "nodeinfo.h" #include "physmem.h" #include "util.h" +#include "virterror_internal.h" #ifdef __linux__ #define CPUINFO_PATH "/proc/cpuinfo" @@ -171,3 +178,67 @@ int virNodeInfoPopulate(virConnectPtr conn, return -1; #endif } + +#if HAVE_NUMACTL +# if LIBNUMA_API_VERSION <= 1 +# define NUMA_MAX_N_CPUS 4096 +# else +# define NUMA_MAX_N_CPUS (numa_all_cpus_ptr->size) +# endif + +# define n_bits(var) (8 * sizeof(var)) +# define MASK_CPU_ISSET(mask, cpu) \ + (((mask)[((cpu) / n_bits(*(mask)))] >> ((cpu) % n_bits(*(mask)))) & 1) + +int +virCapsInitNUMA(virCapsPtr caps) +{ + int n; + uint64_t *mask = NULL; + int *cpus = NULL; + int ret = -1; + int max_n_cpus = NUMA_MAX_N_CPUS; + + if (numa_available() < 0) + return 0; + + int mask_n_bytes = max_n_cpus / 8; + if (VIR_ALLOC_N(mask, mask_n_bytes / sizeof *mask) < 0) + goto cleanup; + + for (n = 0 ; n <= numa_max_node() ; n++) { + int i; + int ncpus; + if (numa_node_to_cpus(n, mask, mask_n_bytes) < 0) + goto cleanup; + + for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) + if (MASK_CPU_ISSET(mask, i)) + ncpus++; + + if (VIR_ALLOC_N(cpus, ncpus) < 0) + goto cleanup; + + for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) + if (MASK_CPU_ISSET(mask, i)) + cpus[ncpus++] = i; + + if (virCapabilitiesAddHostNUMACell(caps, + n, + ncpus, + cpus) < 0) + goto cleanup; + + VIR_FREE(cpus); + } + + ret = 0; + +cleanup: + VIR_FREE(cpus); + VIR_FREE(mask); + return ret; +} +#else +int virCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; } +#endif diff --git a/src/nodeinfo.h b/src/nodeinfo.h index e7c78eb0bf..030f0ee2b3 100644 --- a/src/nodeinfo.h +++ b/src/nodeinfo.h @@ -1,7 +1,7 @@ /* * nodeinfo.c: Helper routines for OS specific node information * - * Copyright (C) 2006, 2007 Red Hat, Inc. + * Copyright (C) 2006-2008 Red Hat, Inc. * Copyright (C) 2006 Daniel P. Berrange * * This library is free software; you can redistribute it and/or @@ -25,7 +25,9 @@ #define __VIR_NODEINFO_H__ #include "libvirt/libvirt.h" +#include "capabilities.h" int virNodeInfoPopulate(virConnectPtr conn, virNodeInfoPtr nodeinfo); +int virCapsInitNUMA(virCapsPtr caps); #endif /* __VIR_NODEINFO_H__*/ diff --git a/src/openvz_conf.c b/src/openvz_conf.c index 44a243bfe1..49cc183334 100644 --- a/src/openvz_conf.c +++ b/src/openvz_conf.c @@ -146,6 +146,9 @@ virCapsPtr openvzCapsInit(void) 0, 0)) == NULL) goto no_memory; + if (virCapsInitNUMA(caps) < 0) + goto no_memory; + virCapabilitiesSetMacPrefix(caps, (unsigned char[]){ 0x52, 0x54, 0x00 }); if ((guest = virCapabilitiesAddGuest(caps, diff --git a/src/qemu_conf.c b/src/qemu_conf.c index 469bf1feb5..f78678ee84 100644 --- a/src/qemu_conf.c +++ b/src/qemu_conf.c @@ -36,11 +36,6 @@ #include #include -#if HAVE_NUMACTL -#define NUMA_VERSION1_COMPATIBILITY 1 -#include -#endif - #include "virterror_internal.h" #include "qemu_conf.h" #include "uuid.h" @@ -51,6 +46,7 @@ #include "verify.h" #include "datatypes.h" #include "xml.h" +#include "nodeinfo.h" VIR_ENUM_DECL(virDomainDiskQEMUBus) VIR_ENUM_IMPL(virDomainDiskQEMUBus, VIR_DOMAIN_DISK_BUS_LAST, @@ -300,66 +296,6 @@ qemudCapsInitGuest(virCapsPtr caps, return 0; } -#if HAVE_NUMACTL -#define MAX_CPUS 4096 -#define MAX_CPUS_MASK_SIZE (sizeof(unsigned long)) -#define MAX_CPUS_MASK_BITS (MAX_CPUS_MASK_SIZE * 8) -#define MAX_CPUS_MASK_LEN (MAX_CPUS / (MAX_CPUS_MASK_BITS)) - -#define MASK_CPU_ISSET(mask, cpu) \ - (((mask)[((cpu) / MAX_CPUS_MASK_BITS)] >> ((cpu) % MAX_CPUS_MASK_BITS)) & 1) - -static int -qemudCapsInitNUMA(virCapsPtr caps) -{ - int n, i; - unsigned long *mask = NULL; - int ncpus; - int *cpus = NULL; - int ret = -1; - - if (numa_available() < 0) - return 0; - - if (VIR_ALLOC_N(mask, MAX_CPUS_MASK_LEN) < 0) - goto cleanup; - - for (n = 0 ; n <= numa_max_node() ; n++) { - int mask_n_bytes = numa_all_cpus_ptr->size / 8; - if (numa_node_to_cpus(n, mask, mask_n_bytes) < 0) - goto cleanup; - - for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++) - if (MASK_CPU_ISSET(mask, i)) - ncpus++; - - if (VIR_ALLOC_N(cpus, ncpus) < 0) - goto cleanup; - - for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++) - if (MASK_CPU_ISSET(mask, i)) - cpus[ncpus++] = i; - - if (virCapabilitiesAddHostNUMACell(caps, - n, - ncpus, - cpus) < 0) - goto cleanup; - - VIR_FREE(cpus); - } - - ret = 0; - -cleanup: - VIR_FREE(cpus); - VIR_FREE(mask); - return ret; -} -#else -static int qemudCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; } -#endif - virCapsPtr qemudCapsInit(void) { struct utsname utsname; virCapsPtr caps; @@ -375,7 +311,7 @@ virCapsPtr qemudCapsInit(void) { /* Using KVM's mac prefix for QEMU too */ virCapabilitiesSetMacPrefix(caps, (unsigned char[]){ 0x52, 0x54, 0x00 }); - if (qemudCapsInitNUMA(caps) < 0) + if (virCapsInitNUMA(caps) < 0) goto no_memory; for (i = 0 ; i < ARRAY_CARDINALITY(arch_info_hvm) ; i++) diff --git a/src/uml_conf.c b/src/uml_conf.c index 00adf27357..8b564be056 100644 --- a/src/uml_conf.c +++ b/src/uml_conf.c @@ -36,84 +36,18 @@ #include #include -#if HAVE_NUMACTL -#define NUMA_VERSION1_COMPATIBILITY 1 -#include -#endif - #include "uml_conf.h" #include "uuid.h" #include "buf.h" #include "conf.h" #include "util.h" #include "memory.h" +#include "nodeinfo.h" #include "verify.h" #define umlLog(level, msg...) fprintf(stderr, msg) - - -#if HAVE_NUMACTL -#define MAX_CPUS 4096 -#define MAX_CPUS_MASK_SIZE (sizeof(unsigned long)) -#define MAX_CPUS_MASK_BITS (MAX_CPUS_MASK_SIZE * 8) -#define MAX_CPUS_MASK_LEN (MAX_CPUS / (MAX_CPUS_MASK_BITS)) - -#define MASK_CPU_ISSET(mask, cpu) \ - (((mask)[((cpu) / MAX_CPUS_MASK_BITS)] >> ((cpu) % MAX_CPUS_MASK_BITS)) & 1) - -static int -umlCapsInitNUMA(virCapsPtr caps) -{ - int n, i; - unsigned long *mask = NULL; - int ncpus; - int *cpus = NULL; - int ret = -1; - - if (numa_available() < 0) - return 0; - - if (VIR_ALLOC_N(mask, MAX_CPUS_MASK_LEN) < 0) - goto cleanup; - - for (n = 0 ; n <= numa_max_node() ; n++) { - int mask_n_bytes = numa_all_cpus_ptr->size / 8; - if (numa_node_to_cpus(n, mask, mask_n_bytes) < 0) - goto cleanup; - - for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++) - if (MASK_CPU_ISSET(mask, i)) - ncpus++; - - if (VIR_ALLOC_N(cpus, ncpus) < 0) - goto cleanup; - - for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++) - if (MASK_CPU_ISSET(mask, i)) - cpus[ncpus++] = i; - - if (virCapabilitiesAddHostNUMACell(caps, - n, - ncpus, - cpus) < 0) - goto cleanup; - - VIR_FREE(cpus); - } - - ret = 0; - -cleanup: - VIR_FREE(cpus); - VIR_FREE(mask); - return ret; -} -#else -static int umlCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; } -#endif - virCapsPtr umlCapsInit(void) { struct utsname utsname; virCapsPtr caps; @@ -126,7 +60,7 @@ virCapsPtr umlCapsInit(void) { 0, 0)) == NULL) goto no_memory; - if (umlCapsInitNUMA(caps) < 0) + if (virCapsInitNUMA(caps) < 0) goto no_memory; if ((guest = virCapabilitiesAddGuest(caps,