Added code coverage testing enabled via --enable-test-coverage

ChangeLog

@@ -1,3 +1,16 @@
+Tue Feb 13 19:46:35 EST 2007 Daniel Berrange <berrange@redhat.com>
+
+	* autobuild.sh: Run coverage tests during automated build
+	* autogen.sh: Point aclocal at m4 directory explicitly
+	* configure.in: Added command line arg to enable coverage
+	* Makefile.am: Added scripts directory
+	* src/Makefile.am: Build with coverage testing if requested
+	and add target for extracting coverage data
+	* scripts/*: Helper tools for coverage testing
+	* tests/Makefile.am: Make tests link to object files instad
+	of the static library to ensure single output dest for
+	coverage data when running tests
+
 Tue Feb 13 19:29:35 EST 2007 Daniel Berrange <berrange@redhat.com>
 
 	* src/qemu_internal.h, src/qemu_internal.c, src/Makefile.am,

Makefile.am

@@ -1,6 +1,6 @@
 ## Process this file with automake to produce Makefile.in
 
-SUBDIRS = src qemud proxy include docs @PYTHON_SUBDIR@ tests po
+SUBDIRS = src qemud proxy include docs @PYTHON_SUBDIR@ tests po scripts
 
 ACLOCAL_AMFLAGS = -I m4
 
@@ -21,4 +21,23 @@ check-local: all tests
 tests:
 	@(cd docs/examples ; $(MAKE) MAKEFLAGS+=--silent tests)
 	@(if [ "@PYTHON_SUBDIR@" != "" ] ; then cd python ; \
-	  $(MAKE) MAKEFLAGS+=--silent tests ; fi)	
+	  $(MAKE) MAKEFLAGS+=--silent tests ; fi)
+
+cov: cov-recursive cov-am
+
+clean-cov:
+	rm -rf $(top_builddir)/coverage
+	cd src && $(MAKE) $(AM_MAKEFLAGS) clean-cov
+
+cov-recursive:
+	cd src && $(MAKE) $(AM_MAKEFLAGS) cov
+
+cov-am:
+	rm -rf $(top_builddir)/coverage
+	mkdir $(top_builddir)/coverage
+	perl $(srcdir)/scripts/coverage-report.pl src/*.cov > $(top_builddir)/coverage/index.xml
+	xsltproc $(srcdir)/scripts/coverage-report.xsl \
+	  $(top_builddir)/coverage/index.xml \
+	  > $(top_builddir)/coverage/index.html
+	for i in $(top_builddir)/src/*.gcov ; do o=`echo $$i | sed -e 's,$(top_builddir)/src,coverage,'` ; \
+	  perl $(srcdir)/scripts/coverage-report-entry.pl $$i > $$o.html ; done

autobuild.sh

@@ -1,25 +1,26 @@
 #!/bin/sh
 
 set -e
+set -v
 
 # Make things clean.
 
 test -n "$1" && RESULTS="$1" || RESULTS="results.log"
 
 test -f Makefile && make -k distclean || :
-rm -rf MANIFEST blib
+rm -rf coverage
 
 #rm -rf build
 #mkdir build
 #cd build
 
-./autogen.sh --prefix=$AUTOBUILD_INSTALL_ROOT
+./autogen.sh --prefix=$AUTOBUILD_INSTALL_ROOT --enable-test-coverage
 
 make
 make install
 
-make check 1>$RESULTS 2>&1
+make check 2>&1 | tee $RESULTS
-#make cov
+make cov
 
 rm -f *.tar.gz
 make dist

autogen.sh

@@ -57,7 +57,7 @@ fi
 autopoint --force
 #rm -rf m4
 libtoolize --copy --force
-aclocal $ACLOCAL_FLAGS
+aclocal -I m4
 automake --add-missing
 autoconf
 

configure.in

@@ -268,6 +268,23 @@ AC_MSG_RESULT($RUNNING_XEND)
 
 AM_CONDITIONAL(ENABLE_XEN_TESTS, [test "$RUNNING_XEN" != "no" -a "$RUNNING_XEND" != "no"])
 
+AC_ARG_ENABLE(test-coverage,
+[  --enable-test-coverage   turn on code coverage instrumentation],
+[case "${enableval}" in
+   yes|no) ;;
+   *)      AC_MSG_ERROR([bad value ${enableval} for test-coverage option]) ;;
+ esac],
+              [enableval=no])
+
+if test "${enableval}" = yes; then
+  gl_COMPILER_FLAGS(-fprofile-arcs)
+  gl_COMPILER_FLAGS(-ftest-coverage)
+  AC_SUBST([COVERAGE_CFLAGS], [$COMPILER_FLAGS])
+  AC_SUBST([COVERAGE_LDFLAGS], [$COMPILER_FLAGS])
+  COMPILER_FLAGS=
+fi
+
+
 AM_GNU_GETTEXT_VERSION([0.14.1])
 AM_GNU_GETTEXT([external])
 if test -d po
@@ -285,7 +302,7 @@ AC_OUTPUT(Makefile src/Makefile include/Makefile docs/Makefile \
           docs/examples/Makefile docs/devhelp/Makefile \
 	  docs/examples/python/Makefile \
           libvirt.pc libvirt.spec \
-          po/Makefile.in \
+          po/Makefile.in scripts/Makefile \
 	  include/libvirt/Makefile include/libvirt/libvirt.h \
 	  python/Makefile python/tests/Makefile \
           qemud/Makefile \

scripts/.cvsignore

@@ -0,0 +1,2 @@
+Makefile
+Makefile.in

scripts/Makefile.am

@@ -0,0 +1,4 @@
+
+EXTRA_DIST = coverage-report.pl \
+             coverage-report-entry.pl \
+             coverage-report.xsl

scripts/README

@@ -0,0 +1,3 @@
+This directory provides a collection of tools used in the
+build / test process. They are not installed / used after
+deployment.

scripts/coverage-report-entry.pl

@@ -0,0 +1,77 @@
+#!/usr/bin/perl
+#
+# Copyright (C) 2006-2007 Daniel P. Berrange
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+#
+# Author: Daniel P. Berrange <berrange@redhat.com>
+#
+# coverage-report-entry.pl: convert gcov annotated source into HTML
+#
+# This script takes a gcov annotated source code files on STDIN
+# converts it to HTML, coloured according to coverage, and sends
+# it to STDOUT
+
+print <<EOF;
+<html>
+<head>
+<title>Coverage report for $ARGV[0]</title>
+<style type="text/css">
+          span.perfect {
+            background: rgb(0,255,0);
+          }
+          span.terrible {
+            background: rgb(255,0,0);
+          }
+</style>
+</head>
+<body>
+<h1>Coverage report for $ARGV[0]</h1>
+
+<pre>
+EOF
+
+
+while (<>) {
+    s/&/&amp;/g;
+    s/</&lt;/g;
+    s/>/&gt;/g;
+
+    if (/^\s*function (\S+) called (\d+) returned \d+% blocks executed \d+%/) {
+	my $class = $2 > 0 ? "perfect" : "terrible";
+	$_ = "<span class=\"$class\" id=\"" . $1 . "\">$_</span>";
+    } elsif (/^\s*branch\s+\d+\s+taken\s+(\d+)%\s+.*$/) {
+	my $class = $1 > 0 ? "perfect" : "terrible";
+	$_ = "<span class=\"$class\">$_</span>";
+    } elsif (/^\s*branch\s+\d+\s+never executed.*$/) {
+	my $class = "terrible";
+	$_ = "<span class=\"$class\">$_</span>";
+    } elsif (/^\s*call\s+\d+\s+never executed.*$/) {
+	my $class = "terrible";
+	$_ = "<span class=\"$class\">$_</span>";
+    } elsif (/^\s*call\s+\d+\s+returned\s+(\d+)%.*$/) {
+	my $class = $1 > 0 ? "perfect" : "terrible";
+	$_ = "<span class=\"$class\">$_</span>";
+    }
+
+
+    print;
+}
+
+print <<EOF;
+</pre>
+</body>
+</html>
+EOF

scripts/coverage-report.pl

@@ -0,0 +1,137 @@
+#!/usr/bin/perl
+#
+# Copyright (C) 2006-2007 Daniel P. Berrange
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+#
+# Author: Daniel P. Berrange <berrange@redhat.com>
+#
+# coverage-report.pl: generate XML coverage summary report
+#
+# This script takes a listof gcov .cov files as args, and generates
+# an XML document summarizing the coverage per function and per
+# source file.
+
+use warnings;
+use strict;
+
+my %coverage = ( functions => {}, files => {} );
+
+my %filemap;
+
+my $type;
+my $name;
+
+my @functions;
+
+while (<>) {
+    if (/^Function '(.*)'\s*$/) {
+	$type = "function";
+	$name = $1;
+	$coverage{$type}->{$name} = {};
+	push @functions, $name;
+    } elsif (/^File '(.*?)'\s*$/) {
+	$type = "file";
+	$name = $1;
+	$coverage{$type}->{$name} = {};
+
+	foreach my $func (@functions) {
+	    $coverage{"function"}->{$func}->{file} = $name;
+	}
+	@functions = ();
+    } elsif (/^Lines executed:(.*)%\s*of\s*(\d+)\s*$/) {
+	$coverage{$type}->{$name}->{lines} = $2;
+	$coverage{$type}->{$name}->{linesCoverage} = $1;
+    } elsif (/^Branches executed:(.*)%\s*of\s*(\d+)\s*$/) {
+	$coverage{$type}->{$name}->{branches} = $2;
+	$coverage{$type}->{$name}->{branchesCoverage} = $1;
+    } elsif (/^Taken at least once:(.*)%\s*of\s*(\d+)\s*$/) {
+	$coverage{$type}->{$name}->{conds} = $2;
+	$coverage{$type}->{$name}->{condsCoverage} = $1;
+    } elsif (/^Calls executed:(.*)%\s*of\s*(\d+)\s*$/) {
+	$coverage{$type}->{$name}->{calls} = $2;
+	$coverage{$type}->{$name}->{callsCoverage} = $1;
+    } elsif (/^No branches$/) {
+	$coverage{$type}->{$name}->{branches} = 0;
+	$coverage{$type}->{$name}->{branchesCoverage} = "100.00";
+	$coverage{$type}->{$name}->{conds} = 0;
+	$coverage{$type}->{$name}->{condsCoverage} = "100.00";
+    } elsif (/^No calls$/) {
+	$coverage{$type}->{$name}->{calls} = 0;
+	$coverage{$type}->{$name}->{callsCoverage} = "100.00";
+    } elsif (/^\s*(.*):creating '(.*)'\s*$/) {
+	$filemap{$1} = $2;
+    } elsif (/^\s*$/) {
+	# nada
+    } else {
+	warn "unexpected input [$_]\n";
+    }
+}
+
+my %summary;
+foreach my $type ("function", "file") {
+    $summary{$type} = {};
+    foreach my $m ("lines", "branches", "conds", "calls") {
+	my $totalGot = 0;
+	my $totalMiss = 0;
+	my $count = 0;
+	foreach my $func (keys %{$coverage{function}}) {
+	    $count++;
+	    my $got = $coverage{function}->{$func}->{$m};
+	    $totalGot += $got;
+	    my $miss = $got * $coverage{function}->{$func}->{$m ."Coverage"} / 100;
+	    $totalMiss += $miss;
+	}
+	$summary{$type}->{$m} = sprintf("%d", $totalGot);
+	if ($totalGot == 0) {
+	    $summary{$type}->{$m . "Coverage"} = "100.00";
+	} else {
+	    $summary{$type}->{$m . "Coverage"} = sprintf("%.2f", $totalMiss / $totalGot * 100);
+	}
+    }
+}
+
+
+
+print "<coverage>\n";
+
+foreach my $type ("function", "file") {
+    printf "<%ss>\n", $type;
+    foreach my $name (sort { $a cmp $b } keys %{$coverage{$type}}) {
+	my $rec = $coverage{$type}->{$name};
+	printf "  <entry name=\"%s\" details=\"%s\">\n", $name, ($type eq "file" ? $filemap{$name} : $filemap{$rec->{file}});
+	printf "    <lines count=\"%s\" coverage=\"%s\"/>\n", $rec->{lines}, $rec->{linesCoverage};
+	if (exists $rec->{branches}) {
+	    printf "    <branches count=\"%s\" coverage=\"%s\"/>\n", $rec->{branches}, $rec->{branchesCoverage};
+	}
+	if (exists $rec->{conds}) {
+	    printf "    <conditions count=\"%s\" coverage=\"%s\"/>\n", $rec->{conds}, $rec->{condsCoverage};
+	}
+	if (exists $rec->{calls}) {
+	    printf "    <calls count=\"%s\" coverage=\"%s\"/>\n", $rec->{calls}, $rec->{callsCoverage};
+	}
+	print  "  </entry>\n";
+    }
+
+    printf "  <summary>\n";
+    printf "    <lines count=\"%s\" coverage=\"%s\"/>\n", $summary{$type}->{lines}, $summary{$type}->{linesCoverage};
+    printf "    <branches count=\"%s\" coverage=\"%s\"/>\n", $summary{$type}->{branches}, $summary{$type}->{branchesCoverage};
+    printf "    <conditions count=\"%s\" coverage=\"%s\"/>\n", $summary{$type}->{conds}, $summary{$type}->{condsCoverage};
+    printf "    <calls count=\"%s\" coverage=\"%s\"/>\n", $summary{$type}->{calls}, $summary{$type}->{callsCoverage};
+    printf  "  </summary>\n";
+    printf "</%ss>\n", $type;
+}
+
+print "</coverage>\n";

scripts/coverage-report.xsl

@@ -0,0 +1,217 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                version="1.0">
+
+  <xsl:output method="html"/>
+
+  <xsl:template match="coverage">
+    <html>
+      <head>
+        <title>Coverage report</title>
+        <style type="text/css">
+          tbody tr.odd td.label {
+            border-top: 1px solid rgb(128,128,128);
+            border-bottom: 1px solid rgb(128,128,128);
+          }
+          tbody tr.odd td.label {
+            background: rgb(200,200,200);
+          }
+          
+          thead, tfoot {
+            background: rgb(60,60,60);
+            color: white;
+            font-weight: bold;
+          }
+
+          tr td.perfect {
+            background: rgb(0,255,0);
+            color: black;
+          }
+          tr td.excellant {
+            background: rgb(140,255,140);
+            color: black;
+          }
+          tr td.good {
+            background: rgb(160,255,0);
+            color: black;
+          }
+          tr td.poor {
+            background: rgb(255,160,0);
+            color: black;
+          }
+          tr td.bad {
+            background: rgb(255,140,140);
+            color: black;
+          }
+          tr td.terrible {
+            background: rgb(255,0,0);
+            color: black;
+          }
+        </style>
+      </head>
+      <body>
+        <h1>Coverage report</h1>
+        <xsl:apply-templates/>
+      </body>
+    </html>
+  </xsl:template>
+
+  <xsl:template match="functions">
+    <h2>Function coverage</h2>
+    <xsl:call-template name="content">
+      <xsl:with-param name="type" select="'function'"/>
+    </xsl:call-template>
+  </xsl:template>
+  
+
+  <xsl:template match="files">
+    <h2>File coverage</h2>
+    <xsl:call-template name="content">
+      <xsl:with-param name="type" select="'file'"/>
+    </xsl:call-template>
+  </xsl:template>
+
+  <xsl:template name="content">
+    <xsl:param name="type"/>
+    <table>
+      <thead>
+        <tr>
+          <th>Name</th>
+          <th>Lines</th>
+          <th>Branches</th>
+          <th>Conditions</th>
+          <th>Calls</th>
+        </tr>
+      </thead>
+      <tbody>
+        <xsl:for-each select="entry">
+          <xsl:call-template name="entry">
+            <xsl:with-param name="type" select="$type"/>
+            <xsl:with-param name="class">
+              <xsl:choose>
+                <xsl:when test="position() mod 2">
+                  <xsl:text>odd</xsl:text>
+                </xsl:when>
+                <xsl:otherwise>
+                  <xsl:text>even</xsl:text>
+                </xsl:otherwise>
+              </xsl:choose>
+            </xsl:with-param>
+          </xsl:call-template>
+        </xsl:for-each>
+      </tbody>
+      <tfoot>
+        <xsl:for-each select="summary">
+          <xsl:call-template name="entry">
+            <xsl:with-param name="type" select="'summary'"/>
+            <xsl:with-param name="class">
+              <xsl:choose>
+                <xsl:when test="position() mod 2">
+                  <xsl:text>odd</xsl:text>
+                </xsl:when>
+                <xsl:otherwise>
+                  <xsl:text>even</xsl:text>
+                </xsl:otherwise>
+              </xsl:choose>
+            </xsl:with-param>
+          </xsl:call-template>
+        </xsl:for-each>
+      </tfoot>
+    </table>
+  </xsl:template>
+  
+  <xsl:template name="entry">
+    <xsl:param name="type"/>
+    <xsl:param name="class"/>
+    <tr class="{$class}">
+      <xsl:choose>
+        <xsl:when test="$type = 'function'">
+          <td class="label"><a href="{@details}.html#{@name}"><xsl:value-of select="@name"/></a></td>
+        </xsl:when>
+        <xsl:when test="$type = 'file'">
+          <td class="label"><a href="{@details}.html"><xsl:value-of select="@name"/></a></td>
+        </xsl:when>
+        <xsl:otherwise>
+          <td class="label">Summary</td>
+        </xsl:otherwise>
+      </xsl:choose>
+
+      <xsl:if test="count(lines)">
+        <xsl:apply-templates select="lines"/>
+      </xsl:if>
+      <xsl:if test="not(count(lines))">
+        <xsl:call-template name="missing"/>
+      </xsl:if>
+
+      <xsl:if test="count(branches)">
+        <xsl:apply-templates select="branches"/>
+      </xsl:if>
+      <xsl:if test="not(count(branches))">
+        <xsl:call-template name="missing"/>
+      </xsl:if>
+
+      <xsl:if test="count(conditions)">
+        <xsl:apply-templates select="conditions"/>
+      </xsl:if>
+      <xsl:if test="not(count(conditions))">
+        <xsl:call-template name="missing"/>
+      </xsl:if>
+
+      <xsl:if test="count(calls)">
+        <xsl:apply-templates select="calls"/>
+      </xsl:if>
+      <xsl:if test="not(count(calls))">
+        <xsl:call-template name="missing"/>
+      </xsl:if>
+
+    </tr>
+  </xsl:template>
+  
+  <xsl:template match="lines">
+    <xsl:call-template name="row"/>
+  </xsl:template>
+
+  <xsl:template match="branches">
+    <xsl:call-template name="row"/>
+  </xsl:template>
+
+  <xsl:template match="conditions">
+    <xsl:call-template name="row"/>
+  </xsl:template>
+
+  <xsl:template match="calls">
+    <xsl:call-template name="row"/>
+  </xsl:template>
+
+  <xsl:template name="missing">
+    <td></td>
+  </xsl:template>
+
+  <xsl:template name="row">
+    <xsl:variable name="quality">
+      <xsl:choose>
+        <xsl:when test="@coverage = 100">
+          <xsl:text>perfect</xsl:text>
+        </xsl:when>
+        <xsl:when test="@coverage >= 80.0">
+          <xsl:text>excellant</xsl:text>
+        </xsl:when>
+        <xsl:when test="@coverage >= 60.0">
+          <xsl:text>good</xsl:text>
+        </xsl:when>
+        <xsl:when test="@coverage >= 40.0">
+          <xsl:text>poor</xsl:text>
+        </xsl:when>
+        <xsl:when test="@coverage >= 20.0">
+          <xsl:text>bad</xsl:text>
+        </xsl:when>
+        <xsl:otherwise>
+          <xsl:text>terrible</xsl:text>
+        </xsl:otherwise>
+      </xsl:choose>
+    </xsl:variable>
+    
+    <td class="{$quality}"><xsl:value-of select="@coverage"/>% of <xsl:value-of select="@count"/></td>
+  </xsl:template>
+
+</xsl:stylesheet>

src/Makefile.am

@@ -13,8 +13,9 @@ EXTRA_DIST = libvirt_sym.version
 lib_LTLIBRARIES = libvirt.la
 libvirt_la_LIBADD = @LIBXML_LIBS@
 libvirt_la_LDFLAGS = -Wl,--version-script=$(srcdir)/libvirt_sym.version \
-                    -version-info @LIBVIRT_VERSION_INFO@
+                    -version-info @LIBVIRT_VERSION_INFO@ \
-
+                    $(COVERAGE_CFLAGS:-f%=-Wc,-f%)
+libvirt_la_CFLAGS = $(COVERAGE_CFLAGS)
 libvirt_la_SOURCES =						\
 		libvirt.c internal.h				\
 		hash.c hash.h					\
@@ -34,12 +35,25 @@ libvirt_la_SOURCES =						\
 bin_PROGRAMS = virsh
 
 virsh_SOURCES = virsh.c console.c console.h
-virsh_LDFLAGS =
+virsh_LDFLAGS = $(COVERAGE_LDFLAGS)
 virsh_DEPENDENCIES = $(DEPS)
 virsh_LDADD = $(LDADDS) $(VIRSH_LIBS)
+virsh_CFLAGS = $(COVERAGE_CFLAGS)
 
 #
 # target to ease building test programs
 #
 tst: tst.c
 	$(CC) $(CFLAGS) -I../include -o tst tst.c .libs/libvirt.a -lxml2 -lxenstore -lpthread
+
+COVERAGE_FILES = $(libvirt_la_SOURCES:%.c=libvirt_la-%.cov)
+
+cov: clean-cov $(COVERAGE_FILES)
+
+clean-cov:
+	rm -f *.cov *.gcov
+
+%.cov: .libs/%.o
+	gcov -b -f -o .libs $< > $@
+
+CLEANFILES = *.cov *.gcov .libs/*.gcda .libs/*.gcno *.gcno *.gcda

tests/Makefile.am

@@ -2,7 +2,13 @@
 
 SUBDIRS = virshdata confdata sexpr2xmldata xml2sexprdata xmconfigdata
 
-LIBVIRT = $(top_builddir)/src/.libs/libvirt.a
+# Wierd libtool related juju...
+#
+# We explicitly want wildcard here instead of just linking
+# to the libvirt.a file. This ensures that when coverage
+# tests are run, all the output data ends up in the correct
+# location. ie, src/ instead of src/.libs.
+LIBVIRT = $(wildcard $(top_builddir)/src/.libs/*.o)
 
 INCLUDES = \
 	-I$(top_builddir)/include \
@@ -11,11 +17,14 @@ INCLUDES = \
 	-I$(top_srcdir)/src \
 	@LIBXML_CFLAGS@ \
         -D_XOPEN_SOURCE=600 -D_POSIX_C_SOURCE=199506L \
-        -DGETTEXT_PACKAGE=\"$(PACKAGE)\"
+        -DGETTEXT_PACKAGE=\"$(PACKAGE)\" \
+         $(COVERAGE_CFLAGS)
+
 LDADDS = \
 	@STATIC_BINARIES@ \
 	@LIBXML_LIBS@ \
-	$(LIBVIRT)
+	$(LIBVIRT) \
+        $(COVERAGE_LDFLAGS)
 
 EXTRA_DIST = xmlrpcserver.py test_conf.sh