[vala/wip/valadate: 80/101] side-ported Valadate-2.0 to Vala test harness
- From: Rico Tzschichholz <ricotz src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [vala/wip/valadate: 80/101] side-ported Valadate-2.0 to Vala test harness
- Date: Sat, 7 Oct 2017 10:56:42 +0000 (UTC)
commit d5f5e737ae6c1b0bb66140c0b6f5c6d0c4f8c7b7
Author: chebizarro gmail com <chebizarro gmail com>
Date: Thu Apr 27 00:02:53 2017 -0700
side-ported Valadate-2.0 to Vala test harness
configure.ac | 5 +
tests/Makefile.am | 8 +-
tests/valadatetests.vala | 2 +-
valadate/Makefile.am | 19 ++-
valadate/assembly.vala | 73 +++++++++
valadate/assemblyerror.vala | 27 ++++
valadate/gnutestreportprinter.vala | 79 ++++++++++
valadate/taptestprinter.vala | 108 ++++++++++++++
valadate/taptestreportprinter.vala | 108 ++++++++++++++
valadate/test.vala | 42 +++--
valadate/testadapter.vala | 107 ++++++++++++++
valadate/testassembly.vala | 93 ++++++++++++
valadate/testcase.vala | 130 ++++++++---------
valadate/testconfig.vala | 128 +++++------------
valadate/testerror.vala | 27 ++++
valadate/testgatherer.vala | 57 +++++++
valadate/testmodule.vala | 43 +++---
valadate/testoptions.vala | 135 +++++++++++++++++
valadate/testplan.vala | 283 ++++++++++++++++++++++++++++++++++++
valadate/testreport.vala | 278 +++++++++++++++++++++++++++++++++++
valadate/testreportprinter.vala | 46 ++++++
valadate/testresult.vala | 220 +++++++++++-----------------
valadate/testrunner.vala | 194 ++++++++++++++-----------
valadate/teststatus.vala | 30 ++++
valadate/testsuite.vala | 72 +++++++---
valadate/xmlfile.vala | 108 ++++++++++++++
valadate/xmltestreportprinter.vala | 69 +++++++++
27 files changed, 2046 insertions(+), 445 deletions(-)
---
diff --git a/configure.ac b/configure.ac
index 3ba2387..3a04131 100644
--- a/configure.ac
+++ b/configure.ac
@@ -103,6 +103,11 @@ PKG_CHECK_MODULES(GMODULE, gmodule-2.0 >= $GLIB_REQUIRED)
AC_SUBST(GMODULE_CFLAGS)
AC_SUBST(GMODULE_LIBS)
+PKG_CHECK_MODULES(LIBXML, libxml-2.0)
+
+AC_SUBST(LIBXML_CFLAGS)
+AC_SUBST(LIBXML_LIBS)
+
PKG_CHECK_MODULES(LIBGVC, libgvc >= $LIBGVC_REQUIRED)
AC_MSG_CHECKING([for CGRAPH])
cgraph_tmp_LIBADD="$LIBADD"
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 5790044..978bc17 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,9 +1,7 @@
include $(top_srcdir)/Makefile.common
include $(top_srcdir)/build-aux/glib-tap.mk
-noinst_PROGRAMS = valadatetests valactests@PACKAGE_SUFFIX@
-
-test_programs = $(noinst_PROGRAMS)
+test_programs = valadatetests #valactests@PACKAGE_SUFFIX@
# Valadate tests
valadatetests_VALAFLAGS = \
@@ -33,11 +31,14 @@ valadatetests_CPPFLAGS = \
$(GLIB_CPPFLAGS) \
$(GIO_CFLAGS) \
$(GMODULE_CPPFLAGS) \
+ $(XML_CFLAGS) \
-fPIE \
$(NULL)
valadatetests_CFLAGS = \
-I$(top_srcdir)/valadate \
+ -I$(top_srcdir)/vala \
+ -I$(top_srcdir)/gee \
$(GLIB_CFLAGS) \
$(GIO_CFLAGS) \
$(GMODULE_CFLAGS) \
@@ -70,6 +71,7 @@ valactests@PACKAGE_SUFFIX@_LDADD = \
valactests@PACKAGE_SUFFIX@_CPPFLAGS = \
-I$(top_srcdir)/valadate \
+ -I$(top_srcdir)/vala \
$(GLIB_CPPFLAGS) \
$(GIO_CFLAGS) \
$(GMODULE_CPPFLAGS) \
diff --git a/tests/valadatetests.vala b/tests/valadatetests.vala
index 95c28fb..4fc4de7 100644
--- a/tests/valadatetests.vala
+++ b/tests/valadatetests.vala
@@ -37,7 +37,7 @@ public class Valadate.Tests.TestFixture : Valadate.TestCase {
}
public void test_testcase_2 () {
- message (Valadate.get_current_test_path ());
+ message (TestOptions.get_current_test_path ());
skip ("No reason");
debug ("This is a second test of the system");
}
diff --git a/valadate/Makefile.am b/valadate/Makefile.am
index 7d1cc4b..2a73a4a 100644
--- a/valadate/Makefile.am
+++ b/valadate/Makefile.am
@@ -11,6 +11,7 @@ AM_CPPFLAGS = \
$(GLIB_CFLAGS) \
$(GIO_CFLAGS) \
$(GMODULE_CFLAGS) \
+ $(LIBXML_CFLAGS) \
-g \
$(NULL)
@@ -21,15 +22,27 @@ lib_LTLIBRARIES = \
$(NULL)
libvaladate_la_VALASOURCES = \
+ assembly.vala \
+ assemblyerror.vala \
+ gnutestreportprinter.vala \
+ taptestreportprinter.vala \
+ testreportprinter.vala \
test.vala \
+ testadapter.vala \
+ testassembly.vala \
testcase.vala \
testconfig.vala \
- testexplorer.vala \
- testiterator.vala \
testmodule.vala \
+ testoptions.vala \
+ testgatherer.vala \
+ testplan.vala \
testresult.vala \
+ testreport.vala \
testrunner.vala \
+ teststatus.vala \
testsuite.vala \
+ xmlfile.vala \
+ xmltestreportprinter.vala \
$(NULL)
libvaladate_la_SOURCES = \
@@ -46,6 +59,7 @@ valadate.vapi valadate.vala.stamp: $(libvaladate_la_VALASOURCES)
--vapidir $(top_srcdir)/vapi --pkg gobject-2.0 \
--pkg gio-2.0 \
--pkg gmodule-2.0 \
+ --pkg libxml-2.0 \
--pkg config \
--pkg libvala@PACKAGE_SUFFIX@ \
-H valadate.h \
@@ -61,6 +75,7 @@ libvaladate_la_LIBADD = \
$(GLIB_LIBS) \
$(GIO_LIBS) \
$(GMODULE_LIBS) \
+ $(LIBXML_LIBS) \
$(NULL)
EXTRA_DIST = $(libvaladate_la_VALASOURCES) valadate.vapi valadate.vala.stamp
diff --git a/valadate/assembly.vala b/valadate/assembly.vala
new file mode 100644
index 0000000..8536460
--- /dev/null
+++ b/valadate/assembly.vala
@@ -0,0 +1,73 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2016 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+
+public abstract class Valadate.Assembly {
+
+ protected static SubprocessLauncher launcher;
+
+ private static void init_launcher() {
+ if (launcher == null) {
+ launcher = new SubprocessLauncher(
+ GLib.SubprocessFlags.STDIN_PIPE |
+ GLib.SubprocessFlags.STDOUT_PIPE |
+ GLib.SubprocessFlags.STDERR_PIPE);
+ launcher.setenv("G_MESSAGES_DEBUG","all",true);
+ launcher.setenv("G_DEBUG","fatal-criticals fatal-warnings gc-friendly",true);
+ launcher.setenv("G_SLICE","always-malloc debug-blocks",true);
+ }
+ }
+
+ public File binary {get;set;}
+ public InputStream stderr {get;set;}
+ public OutputStream stdin {get;set;}
+ public InputStream stdout {get;set;}
+
+ protected Subprocess process;
+
+ public Assembly(File binary) throws Error {
+ init_launcher();
+ if(!binary.query_exists())
+ throw new FileError.NOENT("The file %s does not exist", binary.get_path());
+ if(!GLib.FileUtils.test(binary.get_path(), FileTest.IS_EXECUTABLE))
+ throw new FileError.PERM("The file %s is not executable", binary.get_path());
+ this.binary = binary;
+ }
+
+ public abstract Assembly clone() throws Error;
+
+ public virtual Assembly run(string? command = null, Cancellable? cancellable = null) throws Error {
+ string[] args;
+ Shell.parse_argv("%s %s".printf(binary.get_path(), command ?? ""), out args);
+ process = launcher.spawnv(args);
+ stdout = new DataInputStream (process.get_stdout_pipe());
+ stderr = new DataInputStream (process.get_stderr_pipe());
+ stdin = new DataOutputStream (process.get_stdin_pipe());
+ process.wait_check(cancellable);
+ cancellable.set_error_if_cancelled();
+ return this;
+ }
+
+ public virtual void quit() {
+ if(process != null)
+ process.force_exit();
+ }
+}
diff --git a/valadate/assemblyerror.vala b/valadate/assemblyerror.vala
new file mode 100644
index 0000000..aa1b57e
--- /dev/null
+++ b/valadate/assemblyerror.vala
@@ -0,0 +1,27 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2016 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+
+public errordomain Valadate.AssemblyError {
+ NOT_FOUND,
+ LOAD,
+ METHOD
+}
diff --git a/valadate/gnutestreportprinter.vala b/valadate/gnutestreportprinter.vala
new file mode 100644
index 0000000..87ce5e5
--- /dev/null
+++ b/valadate/gnutestreportprinter.vala
@@ -0,0 +1,79 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2016 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+public class Valadate.GnuTestReportPrinter : TestReportPrinter {
+
+ private const string TAP_VERSION = "13";
+
+ private List<TestCase> testcases = new List<TestCase>();
+
+ public GnuTestReportPrinter(TestConfig config) throws Error {
+ base(config);
+ }
+
+ public override void print(TestReport report) {
+
+ if(report.test is TestSuite) {
+ testcases = new List<TestCase>();
+ } else if(report.test is TestCase) {
+ testcases.append(report.test as TestCase);
+ } else if(report.test is TestAdapter) {
+ var test = report.test as TestAdapter;
+ var idx = testcases.index(test.parent as TestCase);
+ int index = 1;
+
+ if(idx > 0)
+ for(int i=0; i<idx;i++)
+ index += testcases.nth_data(i).count;
+
+ for(int i=0;i<test.parent.count; i++) {
+ if(test.parent[i] == test) {
+ index += i;
+ }
+ }
+
+ switch(report.test.status) {
+ case TestStatus.PASSED:
+ stdout.printf("PASS: %s %d - %s\n", test.parent.name, index,
test.name);
+ break;
+ case TestStatus.SKIPPED:
+ stdout.printf("SKIP: %s %d - %s # SKIP %s \n",
+ test.parent.name, index, test.label, test.status_message ??
"");
+ break;
+ case TestStatus.TODO:
+ var errs = report.xml.eval("//failure | //error");
+ if(errs.size > 0)
+ stdout.printf("XFAIL: %s %d - %s # TODO %s \n",
+ test.parent.name, index, test.label,
test.status_message ?? "");
+ else
+ stdout.printf("PASS: %s %d - %s # TODO %s \n",
+ test.parent.name, index, test.label,
test.status_message ?? "");
+ break;
+ case TestStatus.FAILED:
+ case TestStatus.ERROR:
+ stdout.printf("FAIL: %s %d - %s\n", test.parent.name, index,
test.label);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+}
diff --git a/valadate/taptestprinter.vala b/valadate/taptestprinter.vala
new file mode 100644
index 0000000..c1f9d77
--- /dev/null
+++ b/valadate/taptestprinter.vala
@@ -0,0 +1,108 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2016 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+
+public class Valadate.TapTestReportPrinter : TestReportPrinter {
+
+ private const string TAP_VERSION = "13";
+
+ private List<TestCase> testcases = new List<TestCase>();
+
+ public TapTestReportPrinter(TestConfig config) throws Error {
+ base(config);
+ if(!config.list_only) {
+ stdout.printf("TAP version %s\n", TAP_VERSION);
+ stdout.printf("# random seed: %s\n", config.seed);
+ }
+ }
+
+ public override void print(TestReport report) {
+
+ if(report.test is TestSuite && report.test.parent.name == "/") {
+ stdout.printf("1..%d\n", report.test.count);
+
+ var props = report.xml.eval("//properties/property");
+ stdout.puts("# Environment\n");
+ foreach(Xml.Node* prop in props) {
+ stdout.printf("# %s : %s\n",
+ prop->get_prop("name"), prop->get_prop("value"));
+ }
+
+ } else if(report.test is TestCase) {
+ testcases.append(report.test as TestCase);
+ stdout.printf("# Start of %s tests\n", report.test.label);
+
+ } else if(report.test is TestAdapter) {
+ var test = report.test as TestAdapter;
+ var idx = testcases.index(test.parent as TestCase);
+ int index = 1;
+ bool lasttest = false;
+
+ if(idx > 0)
+ for(int i=0; i<idx;i++)
+ index += testcases.nth_data(i).count;
+
+ for(int i=0;i<test.parent.count; i++) {
+ if(test.parent[i] == test) {
+ index += i;
+ lasttest = (i == test.parent.count-1);
+ }
+ }
+
+ switch(report.test.status) {
+ case TestStatus.PASSED:
+ stdout.printf("ok %d - %s\n", index, test.label);
+ break;
+ case TestStatus.SKIPPED:
+ stdout.printf("ok %d - %s # SKIP %s \n", index, test.label,
test.status_message ?? "Skipping");
+ break;
+ case TestStatus.TODO:
+ var errs = report.xml.eval("//failure | //error");
+ if(errs.size > 0)
+ stdout.printf("not ok %d - %s # TODO %s \n", index,
test.label, test.status_message ?? "Todo");
+ else
+ stdout.printf("ok %d - %s # TODO %s \n", index, test.label,
test.status_message ?? "Todo");
+ break;
+ case TestStatus.FAILED:
+ case TestStatus.ERROR:
+ stdout.printf("not ok %d - %s\n", index, test.label);
+ break;
+ default:
+ stdout.printf("Bail out! %s\n", "There was an unexpected error");
+ break;
+ }
+ stdout.puts(" ---\n");
+ stdout.printf(" duration_ms: %.4f\n", test.time);
+ var messages = report.xml.eval("//failure | //error | //info");
+ foreach(Xml.Node* mess in messages) {
+ stdout.printf(" message: >\n %s\n", mess->get_prop("message"));
+ stdout.printf(" severity: %s\n", mess->name);
+ }
+ stdout.puts(" ...\n");
+ messages = report.xml.eval("//system-out | //system-err");
+ foreach(Xml.Node* mess in messages)
+ stdout.printf("# %s\n", string.joinv("\n# ",
mess->get_content().split("\n")));
+ if(lasttest)
+ stdout.printf("# End of %s tests\n", test.parent.label);
+ stdout.flush();
+ }
+ }
+}
diff --git a/valadate/taptestreportprinter.vala b/valadate/taptestreportprinter.vala
new file mode 100644
index 0000000..c1f9d77
--- /dev/null
+++ b/valadate/taptestreportprinter.vala
@@ -0,0 +1,108 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2016 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+
+public class Valadate.TapTestReportPrinter : TestReportPrinter {
+
+ private const string TAP_VERSION = "13";
+
+ private List<TestCase> testcases = new List<TestCase>();
+
+ public TapTestReportPrinter(TestConfig config) throws Error {
+ base(config);
+ if(!config.list_only) {
+ stdout.printf("TAP version %s\n", TAP_VERSION);
+ stdout.printf("# random seed: %s\n", config.seed);
+ }
+ }
+
+ public override void print(TestReport report) {
+
+ if(report.test is TestSuite && report.test.parent.name == "/") {
+ stdout.printf("1..%d\n", report.test.count);
+
+ var props = report.xml.eval("//properties/property");
+ stdout.puts("# Environment\n");
+ foreach(Xml.Node* prop in props) {
+ stdout.printf("# %s : %s\n",
+ prop->get_prop("name"), prop->get_prop("value"));
+ }
+
+ } else if(report.test is TestCase) {
+ testcases.append(report.test as TestCase);
+ stdout.printf("# Start of %s tests\n", report.test.label);
+
+ } else if(report.test is TestAdapter) {
+ var test = report.test as TestAdapter;
+ var idx = testcases.index(test.parent as TestCase);
+ int index = 1;
+ bool lasttest = false;
+
+ if(idx > 0)
+ for(int i=0; i<idx;i++)
+ index += testcases.nth_data(i).count;
+
+ for(int i=0;i<test.parent.count; i++) {
+ if(test.parent[i] == test) {
+ index += i;
+ lasttest = (i == test.parent.count-1);
+ }
+ }
+
+ switch(report.test.status) {
+ case TestStatus.PASSED:
+ stdout.printf("ok %d - %s\n", index, test.label);
+ break;
+ case TestStatus.SKIPPED:
+ stdout.printf("ok %d - %s # SKIP %s \n", index, test.label,
test.status_message ?? "Skipping");
+ break;
+ case TestStatus.TODO:
+ var errs = report.xml.eval("//failure | //error");
+ if(errs.size > 0)
+ stdout.printf("not ok %d - %s # TODO %s \n", index,
test.label, test.status_message ?? "Todo");
+ else
+ stdout.printf("ok %d - %s # TODO %s \n", index, test.label,
test.status_message ?? "Todo");
+ break;
+ case TestStatus.FAILED:
+ case TestStatus.ERROR:
+ stdout.printf("not ok %d - %s\n", index, test.label);
+ break;
+ default:
+ stdout.printf("Bail out! %s\n", "There was an unexpected error");
+ break;
+ }
+ stdout.puts(" ---\n");
+ stdout.printf(" duration_ms: %.4f\n", test.time);
+ var messages = report.xml.eval("//failure | //error | //info");
+ foreach(Xml.Node* mess in messages) {
+ stdout.printf(" message: >\n %s\n", mess->get_prop("message"));
+ stdout.printf(" severity: %s\n", mess->name);
+ }
+ stdout.puts(" ...\n");
+ messages = report.xml.eval("//system-out | //system-err");
+ foreach(Xml.Node* mess in messages)
+ stdout.printf("# %s\n", string.joinv("\n# ",
mess->get_content().split("\n")));
+ if(lasttest)
+ stdout.printf("# End of %s tests\n", test.parent.label);
+ stdout.flush();
+ }
+ }
+}
diff --git a/valadate/test.vala b/valadate/test.vala
index 51c24ec..0d853eb 100644
--- a/valadate/test.vala
+++ b/valadate/test.vala
@@ -1,6 +1,6 @@
/*
* Valadate - Unit testing library for GObject-based libraries.
- * Copyright (C) 2016 Chris Daley <chebizarro gmail com>
+ * Copyright (C) 2017 Chris Daley <chebizarro gmail com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -25,33 +25,41 @@
* It is the base interface for all runnable Tests.
*/
public interface Valadate.Test : Object {
-
+ /**
+ * Runs the Tests and collects the results in a TestResult
+ *
+ * @param result the TestResult object used to store the results of the Test
+ */
+ public abstract void run (TestResult result);
/**
* The name of the test
*/
public abstract string name { get; set; }
-
+ /**
+ * The label of the test
+ */
+ public abstract string label { get; set; }
/**
* Returns the number of tests that will be run by this test
+ * TestSuites should return the total number of tests that will
+ * be run.
*/
- public abstract int count { get; }
-
+ public abstract int count {get;}
/**
- * Runs the Tests and collects the results in a TestResult
- *
- * @param result the TestResult object used to store the results of the Test
+ * This is used for the iterator and does not return the number of
+ * tests that will be run
*/
- public abstract void run (TestResult result);
+ public abstract int size {get;}
+ /**
+ * The #TestStatus of the test
+ */
+ public abstract TestStatus status {get;set;default=TestStatus.NOT_RUN;}
- public abstract Test get_test (int index);
+ public abstract double time {get;set;}
- public virtual TestIterator iterator() {
- return new TestIterator (this);
- }
+ public abstract Test? parent {get;set;}
- public virtual void set_up () {
- }
+ public abstract Test get(int index);
- public virtual void tear_down () {
- }
+ public abstract void set(int index, Test test);
}
diff --git a/valadate/testadapter.vala b/valadate/testadapter.vala
new file mode 100644
index 0000000..195d1a2
--- /dev/null
+++ b/valadate/testadapter.vala
@@ -0,0 +1,107 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2017 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+
+private class Valadate.TestAdapter : Object, Test {
+
+ public string name {get;set;}
+ public string label {get;set;}
+ public double time {get;set;}
+
+ public int timeout {get;set;}
+
+ public TestStatus status {get;set;default=TestStatus.NOT_RUN;}
+ public string status_message {get;set;}
+
+ public int count {
+ get {
+ return 1;
+ }
+ }
+
+ public int size {
+ get {
+ return count;
+ }
+ }
+
+ private TestCase.TestMethod test;
+ public Test? parent {get;set;}
+
+ public new Test get(int index) {
+ return this;
+ }
+
+ public TestAdapter(string name, int timeout) {
+ this.name = name;
+ this.timeout = timeout;
+ }
+
+ public void add_test(owned TestPlan.TestMethod testmethod) {
+ this.test = () => {
+ testmethod(parent as TestCase);
+ };
+ }
+
+ public void add_async_test (
+ TestPlan.AsyncTestMethod async_begin,
+ TestPlan.AsyncTestMethodResult async_finish)
+ {
+ var p = parent as TestCase;
+ this.test = () => {
+ AsyncResult? result = null;
+ var loop = new MainLoop();
+ var thread = new Thread<void*>.try(name, () => {
+ async_begin(p, (o, r) => { result = r; loop.quit();});
+ return null;
+ });
+ Timeout.add(timeout, () => {
+ loop.quit();
+ return false;
+ },
+ Priority.HIGH);
+ loop.run();
+ if(result == null)
+ throw new IOError.TIMED_OUT(
+ "The test timed out after %d milliseconds",timeout);
+ async_finish(p, result);
+ };
+ }
+
+ public void add_test_method(owned TestCase.TestMethod testmethod) {
+ this.test = (owned)testmethod;
+ }
+
+ public void run(TestResult result) {
+ if(status == TestStatus.SKIPPED)
+ return;
+ var p = parent as TestCase;
+ result.add_test(this);
+ p.set_up();
+ try {
+ test();
+ } catch (Error e) {
+ result.add_failure(this, e.message);
+ }
+ p.tear_down();
+ result.add_success(this);
+ }
+}
diff --git a/valadate/testassembly.vala b/valadate/testassembly.vala
new file mode 100644
index 0000000..bc8e5aa
--- /dev/null
+++ b/valadate/testassembly.vala
@@ -0,0 +1,93 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2016 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+
+public class Valadate.TestAssembly : TestModule {
+
+ public File srcdir {get;set;}
+ public File builddir {get;set;}
+
+ public TestOptions options {get;set;}
+
+ public TestAssembly(string[] args) throws Error {
+ base(File.new_for_path(args[0]));
+ options = new TestOptions(args);
+ setup_dirs();
+ }
+
+ private TestAssembly.copy(TestAssembly other) throws Error {
+ base(other.binary);
+ options = other.options;
+ }
+
+ private void setup_dirs() throws Error {
+ var buildstr = Environment.get_variable("G_TEST_BUILDDIR");
+
+ if(buildstr == null) {
+ builddir = binary.get_parent();
+ if(builddir.get_basename() == ".libs")
+ builddir = builddir.get_parent();
+ } else {
+ builddir = File.new_for_path(buildstr);
+ }
+
+ var srcstr = Environment.get_variable("G_TEST_SRCDIR");
+
+ if(srcstr == null) {
+ // we're running outside the test harness
+ // check for buildir!=srcdir
+ // this currently on checks for autotools
+ if(!builddir.get_child("Makefile.in").query_exists()) {
+ // check for Makefile in builddir and extract VPATH
+ var makefile = builddir.get_child("Makefile");
+ if(makefile.query_exists()) {
+ var reader = new DataInputStream(makefile.read());
+ var line = reader.read_line();
+ while(line!= null) {
+ if(line.has_prefix("VPATH = ")) {
+ srcstr = Path.build_path(Path.DIR_SEPARATOR_S,
builddir.get_path(), line.split(" = ")[1]);
+ break;
+ }
+ line = reader.read_line();
+ }
+ }
+ }
+ }
+
+ if(srcstr == null)
+ srcdir = builddir;
+ else
+ srcdir = File.new_for_path(srcstr);
+
+ var mesondir = srcdir.get_child(Path.get_basename(binary.get_path()) + "@exe");
+
+ if(mesondir.query_exists())
+ srcdir = mesondir;
+
+ Environment.set_variable("G_TEST_BUILDDIR", builddir.get_path(), true);
+ Environment.set_variable("G_TEST_SRCDIR", srcdir.get_path(), true);
+
+ }
+
+ public override Assembly clone() throws Error {
+ return new TestAssembly.copy(this);
+ }
+}
diff --git a/valadate/testcase.vala b/valadate/testcase.vala
index 2a92377..fc2e52d 100644
--- a/valadate/testcase.vala
+++ b/valadate/testcase.vala
@@ -24,108 +24,106 @@
* Julien Peeters <contact julienpeeters fr>
*/
-public errordomain Valadate.TestError {
- NOT_FOUND
-}
-
-/**
- * The TestMethod delegate represents a {@link Valadate.Test} method
- * that can be added to a TestCase and run
- */
-public delegate void Valadate.TestMethod ();
-
public abstract class Valadate.TestCase : Object, Test {
-
+ /**
+ * The TestMethod delegate represents a {@link Valadate.Test} method
+ * that can be added to a TestCase and run
+ */
+ public delegate void TestMethod () throws Error;
/**
* the name of the TestCase
*/
public string name { get; set; }
-
+ /**
+ * the label of the TestCase
+ */
+ public string label { get; set; }
/**
* Returns the number of {@link Valadate.Test}s that will be run by this TestCase
*/
public int count {
get {
int testcount = 0;
- tests.foreach ((t) => {
+ _tests.foreach((t) => {
+ testcount += t.count;
+ });
+ return testcount;
+ }
+ }
+
+ public int size {
+ get {
+ int testcount = 0;
+ _tests.foreach((t) => {
testcount += t.count;
});
return testcount;
}
}
- public string bug_base { get; set; }
+ public Test? parent {get;set;}
- private List<Test> tests = new List<Test>();
+ public TestStatus status {get;set;default=TestStatus.NOT_RUN;}
+ public string status_message {get;set;}
+ public double time {get;set;}
- /**
- * The public constructor takes an optional string parameter for the
- * TestCase's name
- */
- public TestCase (string? name = null) {
- Object (name : name);
- }
+ public string bug_base {get;set;}
+
+ private List<Test> _tests = new List<Test>();
- construct {
- if (name == null)
- name = get_type ().name ();
- }
+ private Test current_test;
+ private TestResult current_result;
- public void add_test (string testname, owned TestMethod test) {
- var adaptor = new TestAdaptor (testname, (owned) test, this);
- tests.append (adaptor);
+ public new Test get(int index) {
+ return _tests.nth_data((uint)index);
}
- public Test get_test (int index) {
- return tests.nth_data (index);
+ public new void set(int index, Test test) {
+ test.parent = this;
+ _tests.insert_before(_tests.nth(index), test);
+ var t = _tests.nth_data((uint)index++);
+ _tests.remove(t);
}
- public void bug (string reference)
- requires (bug_base != null)
- {
- stdout.printf ("MSG Bug Reference: %s%s", bug_base, reference);
- stdout.flush ();
+ public void add_test(Test test) {
+ test.parent = this;
+ _tests.append(test);
}
- public void skip (string message) {
- stderr.printf ("SKIP %s", message);
- stdout.flush ();
+ public void add_test_method(string testname, owned TestMethod test, int timeout, string? label =
null) {
+ var adapter = new TestAdapter (testname, timeout);
+ adapter.add_test_method((owned)test);
+ adapter.label = label;
+ adapter.parent = this;
+ _tests.append(adapter);
}
-
- public void fail (string? message = null) {
- error ("FAIL %s", message ?? "");
+
+ public virtual void run(TestResult result) {
+ if(status != TestStatus.NOT_RUN)
+ return;
+ current_result = result;
+ _tests.foreach((t) => {
+ current_test = t;
+ t.run(result);
+ });
}
- public virtual void run (TestResult result) {
+ public void bug(string reference)
+ requires(bug_base != null)
+ {
+ info("Bug Reference: %s%s",bug_base, reference);
}
-}
-private class Valadate.TestAdaptor : Object, Test {
+ public void skip(string message) {
+ current_result.add_skip(current_test, message);
- public string name { get; set; }
-
- public int count {
- get {
- return 1;
- }
}
- private TestMethod test;
- private unowned TestCase testcase;
-
- public TestAdaptor (string name, owned TestMethod test, TestCase testcase) {
- this.test = (owned) test;
- this.name = name;
- this.testcase = testcase;
+ public void fail(string? message = null) {
+ current_result.add_failure(current_test, message);
}
- public Test get_test (int index) {
- return this;
- }
+ public virtual void set_up() {}
- public void run (TestResult result) {
- this.testcase.set_up ();
- this.test ();
- this.testcase.tear_down ();
- }
+ public virtual void tear_down() {}
}
diff --git a/valadate/testconfig.vala b/valadate/testconfig.vala
index 2a32523..0c772a4 100644
--- a/valadate/testconfig.vala
+++ b/valadate/testconfig.vala
@@ -20,130 +20,72 @@
* Chris Daley <chebizarro gmail com>
*/
-namespace Valadate {
- public static string? get_current_test_path () {
- return TestConfig._runtest;
- }
-}
-
public errordomain Valadate.TestConfigError {
MODULE,
- TESTPLAN
+ TESTPLAN,
+ METHOD,
+ TEST_PRINTER
}
public class Valadate.TestConfig : Object {
- private static string _seed;
- private static string testplan;
- internal static string _runtest;
- private static string format = "tap";
- private static bool list;
- private static bool _keepgoing = true;
- private static bool quiet;
- private static bool timed;
- private static bool verbose;
- private static bool version;
- private static bool vala_version;
-
- [CCode (array_length = false, array_null_terminated = true)]
- private static string[] paths;
- [CCode (array_length = false, array_null_terminated = true)]
- private static string[] skip;
-
+ public TestOptions options {get;construct set;}
- public string seed {
+ public virtual string format {
get {
- return _seed;
+ return options.format;
}
}
- public string runtest {
+ public virtual string seed {
get {
- return _runtest;
+ return options.seed;
}
}
- public bool list_only {
+ public string? running_test {
get {
- return list;
+ return options.running_test;
}
}
- public bool keep_going {
+ public bool in_subprocess {
get {
- return _keepgoing;
+ return options.running_test != null;
}
}
- public TestSuite root {get;set;}
-
- public OptionContext opt_context;
-
- public const OptionEntry[] options = {
- { "seed", 0, 0, OptionArg.STRING, ref _seed, "Start tests with random seed", "SEEDSTRING" },
- { "format", 'f', 0, OptionArg.STRING, ref format, "Output test results using format",
"FORMAT" },
- { "list", 'l', 0, OptionArg.NONE, ref list, "List test cases available in a test executable",
null },
- { "", 'k', 0, OptionArg.NONE, ref _keepgoing, "Skip failed tests and continue running", null
},
- { "skip", 's', 0, OptionArg.STRING_ARRAY, ref skip, "Skip all tests matching", "TESTPATH..."
},
- { "quiet", 'q', 0, OptionArg.NONE, ref quiet, "Run tests quietly", null },
- { "timed", 0, 0, OptionArg.NONE, ref timed, "Run timed tests", null },
- { "testplan", 0, 0, OptionArg.STRING, ref testplan, "Run the specified TestPlan", "FILE" },
- { "", 'r', 0, OptionArg.STRING, ref _runtest, null, null },
- { "verbose", 0, 0, OptionArg.NONE, ref verbose, "Run tests verbosely", null },
- { "version", 0, 0, OptionArg.NONE, ref version, "Display version number", null },
- { "vala-version", 0, 0, OptionArg.NONE, ref vala_version, "Display Vala version number", null
},
- { "", 0, 0, OptionArg.STRING_ARRAY, ref paths, "Only start test cases matching",
"TESTPATH..." },
- { null }
- };
-
-
- public TestConfig() {
- opt_context = new OptionContext ("- Valadate Testing Framework");
- opt_context.set_help_enabled (true);
- opt_context.add_main_entries (options, null);
+ public virtual bool run_async {
+ get {
+ return options.run_async;
+ }
}
- public int parse(string[] args) {
- var binary = args[0];
- GLib.Environment.set_prgname(binary);
+ public virtual bool list_only {
+ get {
+ return options.list;
+ }
+ }
- try {
- opt_context.parse (ref args);
- } catch (OptionError e) {
- stdout.printf ("%s\n", e.message);
- stdout.printf ("Run '%s --help' to see a full list of available command line
options.\n", args[0]);
- return 1;
+ public virtual bool keep_going {
+ get {
+ return options.keepgoing;
}
+ }
- if (version) {
- stdout.printf ("Valadate %s\n", "1.0");
- return 0;
- } else if (vala_version) {
- stdout.printf ("Vala %s\n", Config.PACKAGE_SUFFIX.substring (1));
- return 0;
+ public virtual int timeout {
+ get {
+ return options.timeout;
}
-
- if(_seed == null)
- _seed = "R02S%08x%08x%08x%08x".printf(
- GLib.Random.next_int(),
- GLib.Random.next_int(),
- GLib.Random.next_int(),
- GLib.Random.next_int());
-
- root = new TestSuite("/");
-
- try {
- load(binary);
- } catch (TestConfigError e) {
- stdout.printf ("%s\n", e.message);
- return 1;
+ }
+
+ public virtual bool timed {
+ get {
+ return options.timed;
}
-
- return -1;
}
- private void load(string binary) throws TestConfigError {
- var testexplorer = new TestExplorer(binary, root);
- testexplorer.load();
+ public TestConfig(TestOptions options) {
+ Object(options : options);
}
}
diff --git a/valadate/testerror.vala b/valadate/testerror.vala
new file mode 100644
index 0000000..b173f71
--- /dev/null
+++ b/valadate/testerror.vala
@@ -0,0 +1,27 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright 2017 Chris Daley <bizarro@localhost.localdomain>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ *
+ *
+ */
+
+public errordomain Valadate.TestError {
+ NOT_FOUND,
+ MODULE,
+ METHOD
+}
diff --git a/valadate/testgatherer.vala b/valadate/testgatherer.vala
new file mode 100644
index 0000000..3d13934
--- /dev/null
+++ b/valadate/testgatherer.vala
@@ -0,0 +1,57 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2016 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+
+public class Valadate.TestGatherer : Vala.CodeVisitor {
+
+ public HashTable<Type, Vala.Class> classes =
+ new HashTable<Type, Vala.Class>(direct_hash, direct_equal);
+
+ private TestAssembly assembly;
+
+ public TestGatherer(TestAssembly assembly) {
+ this.assembly = assembly;
+ }
+
+ public override void visit_namespace(Vala.Namespace ns) {
+ ns.accept_children(this);
+ }
+
+ public override void visit_class(Vala.Class cls) {
+ try {
+ var classtype = find_type(cls);
+ if (classtype.is_a(typeof(TestCase)))
+ classes.insert(classtype, cls);
+ } catch (Error e) {
+ warning(e.message);
+ }
+ cls.accept_children(this);
+ }
+
+ private Type find_type(Vala.Class cls) throws Error {
+ var attr = new Vala.CCodeAttribute (cls);
+ unowned TestPlan.GetType node_get_type =
+ (TestPlan.GetType)assembly.get_method(
+ "%sget_type".printf(attr.lower_case_prefix));
+ var ctype = node_get_type();
+ return ctype;
+ }
+}
diff --git a/valadate/testmodule.vala b/valadate/testmodule.vala
index eda6ac5..c06ae65 100644
--- a/valadate/testmodule.vala
+++ b/valadate/testmodule.vala
@@ -20,39 +20,36 @@
* Chris Daley <chebizarro gmail com>
*/
-public errordomain Valadate.TestModuleError {
- NOT_FOUND,
- LOAD,
- METHOD
-}
-
/**
* Represents a loadable module containing {@link Valadate.Test}s
*/
-public class Valadate.TestModule : Object {
-
- private string lib_path;
+public class Valadate.TestModule : Assembly {
+
private GLib.Module module;
-
- public TestModule (string libpath) {
- lib_path = libpath;
+
+ public TestModule(File binary) throws Error {
+ base(binary);
}
- public void load_module () throws TestModuleError {
- if (!File.new_for_path (lib_path).query_exists ())
- throw new TestModuleError.NOT_FOUND ("Module: %s does not exist", lib_path);
-
- module = GLib.Module.open (lib_path, ModuleFlags.BIND_LOCAL);
+ private void load_module() throws AssemblyError {
+ module = GLib.Module.open (binary.get_path(), ModuleFlags.BIND_LAZY);
if (module == null)
- throw new TestModuleError.LOAD (GLib.Module.error ());
- module.make_resident ();
+ throw new AssemblyError.LOAD(GLib.Module.error());
+ module.make_resident();
}
-
- internal void* get_method (string method_name) throws TestModuleError {
+
+ public virtual void* get_method(string method_name) throws AssemblyError {
+ if(module == null)
+ load_module();
void* function;
- if (module.symbol (method_name, out function))
+ if(module.symbol (method_name, out function))
if (function != null)
return function;
- throw new TestModuleError.METHOD (GLib.Module.error ());
+ throw new AssemblyError.METHOD(GLib.Module.error());
}
+
+ public override Assembly clone() throws Error {
+ return new TestModule(binary);
+ }
+
}
diff --git a/valadate/testoptions.vala b/valadate/testoptions.vala
new file mode 100644
index 0000000..39e8abe
--- /dev/null
+++ b/valadate/testoptions.vala
@@ -0,0 +1,135 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2016 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+namespace Valadate {
+
+ public class TestOptions {
+
+ private static bool _async = true;
+ private static bool _tap;
+ private static string _format = "tap";
+ private static bool _keepgoing = false;
+ private static bool _list;
+ private static bool _quiet;
+ private static string _runtest = null;
+ [CCode (array_length = false, array_null_terminated = true)]
+ private static string[] _skip;
+ private static int _timeout = 60000;
+ private static string _seed;
+ private static bool _timed = true;
+ private static string _testplan;
+ private static bool _verbose;
+ private static bool _version;
+ [CCode (array_length = false, array_null_terminated = true)]
+ private static string[] _paths;
+
+ public const OptionEntry[] options = {
+ { "async", 'a', 0, OptionArg.NONE, ref _async, "Run tests asynchronously in a
separate subprocess [Experimental]", null },
+ { "format", 'f', 0, OptionArg.STRING, ref _format, "Output test results using
format", "FORMAT" },
+ { "", 'k', 0, OptionArg.NONE, ref _keepgoing, "Skip failed tests and continue
running", null },
+ { "list", 'l', 0, OptionArg.NONE, ref _list, "List test cases available in a test
executable", null },
+ { "quiet", 'q', 0, OptionArg.NONE, ref _quiet, "Run tests quietly", null },
+ { "", 'r', 0, OptionArg.STRING, ref _runtest, null, null },
+ { "skip", 's', 0, OptionArg.STRING_ARRAY, ref _skip, "Skip all tests matching",
"TESTPATH..." },
+ { "timeout", 't', 0, OptionArg.INT, ref _timeout, "Default timeout for tests",
"MILLISECONDS" },
+ { "seed", 0, 0, OptionArg.STRING, ref _seed, "Start tests with random seed",
"SEEDSTRING" },
+ { "timed", 0, 0, OptionArg.NONE, ref _timed, "Run timed tests", null },
{ "tap", 0, 0, OptionArg.NONE, ref _tap, "Output test results using TAP format" },
+ { "tap", 0, 0, OptionArg.NONE, ref _tap, "Output test results using TAP format" },
+ { "testplan", 0, 0, OptionArg.STRING, ref _testplan, "Run the specified TestPlan",
"FILE" },
+ { "verbose", 0, 0, OptionArg.NONE, ref _verbose, "Run tests verbosely", null },
+ { "version", 0, 0, OptionArg.NONE, ref _version, "Display version number", null },
+ { "", 0, 0, OptionArg.STRING_ARRAY, ref _paths, "Only start test cases matching",
"TESTPATH..." },
+ { null }
+ };
+
+ public OptionContext opt_context;
+
+ public static string? get_current_test_path() {
+ return _runtest;
+ }
+
+ public string format {
+ get {
+ return _format;
+ }
+ }
+
+ public string seed {
+ get {
+ return _seed;
+ }
+ }
+
+ public string? running_test {
+ get {
+ return _runtest;
+ }
+ }
+
+ public bool run_async {
+ get {
+ return _async;
+ }
+ }
+
+ public bool list {
+ get {
+ return _list;
+ }
+ }
+
+ public bool keepgoing {
+ get {
+ return _keepgoing;
+ }
+ }
+
+ public int timeout {
+ get {
+ return _timeout;
+ }
+ }
+
+ public bool timed {
+ get {
+ return _timed;
+ }
+ }
+
+ public TestOptions(string[] args) throws OptionError {
+ _runtest = null;
+
+ opt_context = new OptionContext ("- Valadate Testing Framework");
+ opt_context.set_help_enabled (true);
+ opt_context.add_main_entries (options, null);
+ opt_context.parse (ref args);
+
+ if(_seed == null)
+ _seed = "R02S%08x%08x%08x%08x".printf(
+ GLib.Random.next_int(),
+ GLib.Random.next_int(),
+ GLib.Random.next_int(),
+ GLib.Random.next_int());
+
+ }
+
+ }
+}
diff --git a/valadate/testplan.vala b/valadate/testplan.vala
new file mode 100644
index 0000000..fc8928e
--- /dev/null
+++ b/valadate/testplan.vala
@@ -0,0 +1,283 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2016 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+
+public class Valadate.TestPlan : Vala.CodeVisitor {
+
+ [CCode (has_target = false)]
+ public delegate void TestMethod(TestCase self) throws Error;
+
+ public delegate void AsyncTestMethod(TestCase self, AsyncReadyCallback cb);
+ public delegate void AsyncTestMethodResult(TestCase self, AsyncResult res) throws Error;
+
+ public delegate Type GetType();
+
+ public File plan {get;set;}
+
+ public TestAssembly assembly {get;set;}
+
+ public TestOptions options {get;set;}
+
+ public TestConfig config {get;set;}
+
+ public TestResult result {get;set;}
+
+ public TestRunner runner {get;set;}
+
+ public TestSuite root {get;protected set;}
+
+ private Vala.CodeContext context;
+ private TestGatherer gatherer;
+ private delegate TestCase Constructor();
+ private TestSuite testsuite;
+ private TestCase testcase;
+
+ public TestPlan(TestAssembly assembly) throws Error {
+
+ this.assembly = assembly;
+ options = assembly.options;
+
+ var plan_name = Path.get_basename(assembly.binary.get_path());
+ if(plan_name.has_prefix("lt-"))
+ plan_name = plan_name.substring(3);
+
+ plan = assembly.srcdir.get_child(plan_name + ".vapi");
+ if(!plan.query_exists()) {
+ plan = assembly.builddir.get_child(plan_name + ".vapi");
+ if(!plan.query_exists()) {
+ throw new TestConfigError.TESTPLAN(
+ "Test Plan %s Not Found in %s or %s", plan_name,
assembly.srcdir.get_path(), assembly.builddir.get_path());
+ }
+ }
+ config = new TestConfig(options);
+ runner = new TestRunner();
+ result = new TestResult(config);
+ testsuite = root = new TestSuite("/");
+ setup_context();
+ load_test_plan();
+ }
+
+ public int run() throws Error {
+ return runner.run_all(this);
+ }
+
+ private void setup_context() {
+ context = new Vala.CodeContext ();
+ Vala.CodeContext.push (context);
+ context.report.enable_warnings = false;
+ context.report.set_verbose_errors (false);
+ context.verbose_mode = false;
+ }
+
+ public void load_test_plan() throws Error {
+ context.add_source_file (new Vala.SourceFile (
+ context, Vala.SourceFileType.PACKAGE, plan.get_path()));
+ var parser = new Vala.Parser ();
+ parser.parse (context);
+ gatherer = new TestGatherer(assembly);
+ context.accept(gatherer);
+ context.accept(this);
+ }
+
+ public override void visit_namespace(Vala.Namespace ns) {
+ if (ns.name != null) {
+ var currpath = "/" + ns.get_full_name().replace(".","/");
+ if(config.in_subprocess)
+ if(!options.running_test.has_prefix(currpath))
+ return;
+
+ if(currpath.last_index_of("/") == 0)
+ testsuite = root;
+
+ var ts = new TestSuite(ns.name);
+ testsuite.add_test(ts);
+ testsuite = ts;
+ }
+ ns.accept_children(this);
+ }
+
+ public override void visit_class(Vala.Class cls) {
+
+ try {
+ if (is_subtype_of(cls, typeof(TestCase)) && !cls.is_abstract) {
+ unowned Constructor ctor = get_constructor(cls);
+ testcase = ctor();
+ testcase.name = cls.name;
+ testcase.label = "/%s".printf(cls.get_full_name().replace(".","/"));
+ testsuite.add_test(testcase);
+ visit_testcase(cls);
+
+ } else if (is_subtype_of(cls,typeof(TestSuite))) {
+ visit_testsuite(cls);
+ }
+ } catch (Error e) {
+ error(e.message);
+ }
+ cls.accept_children(this);
+ }
+
+ private bool is_subtype_of(Vala.Class cls, Type type) {
+ var t = Type.from_name(cls.get_full_name().replace(".",""));
+ if(t.is_a(type))
+ return true;
+ return false;
+ }
+
+ private unowned Constructor get_constructor(Vala.Class cls) throws Error {
+ var attr = new Vala.CCodeAttribute (cls.default_construction_method);
+ return (Constructor)assembly.get_method(attr.name);
+ }
+
+ public void visit_testcase(Vala.Class cls) {
+
+ var t = Type.from_name(cls.get_full_name().replace(".",""));
+ var p = t.parent();
+ if(p != typeof(TestCase)) {
+ var basecls = gatherer.classes.get(p);
+ if(basecls != null)
+ visit_testcase(basecls);
+ }
+
+ foreach(var method in cls.get_methods()) {
+
+ if(config.in_subprocess)
+ if (options.running_test != "%s/%s".printf(
+ testcase.label, method.name))
+ continue;
+
+ if(!is_test(method))
+ continue;
+
+ var added = false;
+ foreach(var test in testcase)
+ if(test.name == method.name)
+ added=true;
+ if(added)
+ continue;
+
+ var adapter = new TestAdapter(method.name, config.timeout);
+ annotate_label(adapter);
+ annotate(adapter, method);
+
+ if(config.in_subprocess && adapter.status != TestStatus.SKIPPED) {
+ var attr = new Vala.CCodeAttribute (method);
+
+ if(method.coroutine) {
+ try {
+ unowned TestPlan.AsyncTestMethod beginmethod =
+
(TestPlan.AsyncTestMethod)assembly.get_method(attr.name);
+ unowned TestPlan.AsyncTestMethodResult testmethod =
+
(TestPlan.AsyncTestMethodResult)assembly.get_method(attr.finish_real_name);
+ adapter.add_async_test(beginmethod, testmethod);
+ } catch (Error e) {
+ var message = e.message;
+ adapter.add_test_method(()=> {debug(message);});
+ }
+ } else {
+ try {
+ TestPlan.TestMethod testmethod =
+ (TestPlan.TestMethod)assembly.get_method(attr.name);
+ adapter.add_test((owned)testmethod);
+ } catch (Error e) {
+ var message = e.message;
+ adapter.add_test_method(()=> {debug(message);});
+ }
+ }
+ } else {
+ adapter.add_test_method(()=> {assert_not_reached();});
+ }
+
+ adapter.label = "%s/%s".printf(
+ testcase.label,
+ adapter.label);
+
+ testcase.add_test(adapter);
+ }
+
+ }
+
+ private void annotate_label(Test test) {
+ if(test.name.has_prefix("test_")) {
+ test.label = test.name.substring(5);
+ } else if(test.name.has_prefix("_test_")) {
+ test.label = test.name.substring(6);
+ test.status = TestStatus.SKIPPED;
+ } else if(test.name.has_prefix("todo_test_")) {
+ test.label = test.name.substring(10);
+ test.status = TestStatus.TODO;
+ } else {
+ test.label = test.name;
+ }
+ test.label = test.label.replace("_", " ");
+ }
+
+ private void annotate(TestAdapter adapter, Vala.Method method) {
+
+ foreach(var attr in method.attributes) {
+ if(attr.name == "Test") {
+ if(attr.has_argument("name"))
+ adapter.label = attr.get_string("name");
+ if(attr.has_argument("skip")) {
+ adapter.status = TestStatus.SKIPPED;
+ adapter.status_message = attr.get_string("skip");
+ } else if(attr.has_argument("todo")) {
+ adapter.status = TestStatus.SKIPPED;
+ adapter.status_message = attr.get_string("todo");
+ } else if(attr.has_argument("timeout")) {
+ adapter.timeout = int.parse(attr.get_string("timeout"));
+ }
+ }
+ }
+ }
+
+ private bool is_test(Vala.Method method) {
+ bool istest = false;
+
+ if(method.is_virtual)
+ foreach(var test in testcase)
+ if(test.name == method.name)
+ return false;
+
+ if (method.name.has_prefix("test_") ||
+ method.name.has_prefix("_test_") ||
+ method.name.has_prefix("todo_test_"))
+ istest = true;
+
+ foreach(var attr in method.attributes)
+ if(attr.name == "Test")
+ istest = true;
+
+ if(method.has_result)
+ istest = false;
+
+ if(method.get_parameters().size > 0)
+ istest = false;
+
+ return istest;
+ }
+
+ public TestSuite visit_testsuite(Vala.Class testclass) throws Error {
+ unowned Constructor meth = get_constructor(testclass);
+ var testcase_test = meth() as TestSuite;
+ testcase_test.name = testclass.name;
+ return testcase_test;
+ }
+}
diff --git a/valadate/testreport.vala b/valadate/testreport.vala
new file mode 100644
index 0000000..8e4dc73
--- /dev/null
+++ b/valadate/testreport.vala
@@ -0,0 +1,278 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2017 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+
+public class Valadate.TestReport {
+
+ private const string XML_DECL ="<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
+ private const string TESTSUITE_XML =
+ """<testsuite disabled="" errors="" failures="" hostname="" id="" """ +
+ """name="" package="" skipped="" tests="" time="" timestamp="" >"""+
+ """<properties/></testsuite>""";
+ private const string TESTCASE_XML =
+ """<testcase assertions="" classname="" name="" status="" time="" />""";
+ private const string MESSAGE_XML = "<%s message=\"%s\" type=\"%s\">%s</%s>";
+ private const string TESTCASE_START =
+ "<testcase assertions=\"\" classname=\"%s\" name=\"%s\" status=\"\" time=\"\">";
+ private const string VDX_NS = "xmlns:vdx=\"https://www.valadate.org/vdx\"";;
+ private const string TESTCASE_TAG = "testcase";
+ private const string ROOT_TAG = "root";
+ private const string SKIP_TAG = "skipped";
+ private const string ERROR_TAG = "error";
+ private const string FAILURE_TAG = "failure";
+ private const string INFO_TAG = "info";
+ private const string TIMER_TAG = "timer";
+ private const string SYSTEM_OUT_TAG = "system-out";
+ private const string SYSTEM_ERR_TAG = "system-err";
+
+ public Test test {get;set;}
+ public bool subprocess {get;set;}
+
+ public XmlFile xml {get;set;}
+
+ private static int64 start_time;
+ private static int64 end_time;
+
+ private static Regex regex_err;
+ private const string regex_err_string =
+ """(\*{2}\n([A-Z]*):([\S]*) ([\S ]*)\n)""";
+
+ public TestReport(Test test, bool subprocess) throws Error {
+ this.test = test;
+ this.subprocess = subprocess;
+
+ if(test.status == TestStatus.NOT_RUN)
+ test.status = TestStatus.RUNNING;
+
+ if(subprocess) {
+ Log.set_default_handler (log_func);
+ GLib.set_printerr_handler (printerr_func);
+ regex_err = new Regex(regex_err_string);
+ }
+
+ if(test is TestSuite || test is TestCase)
+ new_testsuite();
+ else if (test is TestAdapter)
+ new_testcase();
+ }
+
+ private void new_testsuite() throws Error {
+ if(subprocess)
+ return;
+
+ var decl = "%s<%s>%s</%s>".printf(XML_DECL, ROOT_TAG, TESTSUITE_XML, ROOT_TAG);
+ var doc = Xml.Parser.read_memory(decl, decl.length);
+ var root = doc->get_root_element()->children;
+ root->set_prop("tests", test.count.to_string());
+ root->set_prop("name",test.label);
+ xml = new XmlFile.from_doc(doc);
+
+ if(test.parent != null && test.parent.name != "/")
+ return;
+
+ var props = root->children;
+
+ foreach(var key in Environment.list_variables()) {
+ Xml.Node* node = new Xml.Node(null, "property");
+ node->set_prop("name", key);
+ node->set_prop("value", Markup.escape_text(Environment.get_variable(key)));
+ props->add_child(node);
+ }
+ }
+
+ private void new_testcase() throws Error {
+ if(subprocess) {
+ stderr.printf("%s<%s>",XML_DECL,ROOT_TAG);
+ stderr.printf(TESTCASE_START,test.parent.get_type().name(), test.label);
+ start_time = get_monotonic_time();
+ } else {
+ var decl = "%s<%s>%s</%s>".printf(XML_DECL, ROOT_TAG, TESTCASE_XML, ROOT_TAG);
+ var doc = Xml.Parser.read_memory(decl, decl.length);
+ var root = doc->get_root_element()->children;
+ root->set_prop("classname",((TestAdapter)test).parent.name);
+ root->set_prop("status",test.status.to_string().substring(21));
+ root->set_prop("name",test.label);
+ xml = new XmlFile.from_doc(doc);
+ }
+ }
+
+ public void add_error(string message) {
+ if (test.status != TestStatus.SKIPPED &&
+ test.status != TestStatus.TODO)
+ test.status = TestStatus.ERROR;
+
+ add_message(ERROR_TAG, message);
+
+ if(subprocess) {
+ emit_timer();
+ stderr.printf("</%s></%s>",TESTCASE_TAG, ROOT_TAG);
+ stderr.putc(0);
+ }
+ update_status();
+ }
+
+ public void add_failure(string message) {
+ if (test.status != TestStatus.SKIPPED &&
+ test.status != TestStatus.TODO)
+ test.status = TestStatus.FAILED;
+
+ add_message(FAILURE_TAG, message);
+
+ if(subprocess) {
+ emit_timer();
+ stderr.printf("</%s></%s>",TESTCASE_TAG, ROOT_TAG);
+ stderr.putc(0);
+ }
+ update_status();
+ }
+
+ public void add_skip(string message) {
+ test.status = TestStatus.SKIPPED;
+ add_message(SKIP_TAG, message);
+ update_status();
+ }
+
+ public void add_success() {
+ if (test.status != TestStatus.SKIPPED &&
+ test.status != TestStatus.TODO)
+ test.status = TestStatus.PASSED;
+ if(subprocess) {
+ emit_timer();
+ stderr.printf("</%s></%s>",TESTCASE_TAG, ROOT_TAG);
+ stderr.putc(0);
+ }
+ update_status();
+ }
+
+ private void add_message(string tag, string message) {
+ var escaped = Markup.escape_text(message);
+ if(subprocess) {
+ stderr.printf(MESSAGE_XML, tag, escaped, tag.up(), message, tag);
+ } else {
+ Xml.Node* child = new Xml.Node(null, tag);
+ child->set_content(escaped);
+
+ string[] tags = {ERROR_TAG, FAILURE_TAG, INFO_TAG};
+
+ if(tag in tags) {
+ child->new_prop("message", escaped);
+ child->new_prop("type", tag.up());
+ }
+
+ Xml.Node* root = xml.eval("//testcase | //testsuite")[0];
+ root->add_child(child);
+ }
+ }
+ /**
+ * Adds arbitrary text to the TestReport. In the xml output this
+ * text will be encapsulated in <system-out/> or <system-err/> tag
+ *
+ * @param text The text to be added to the {@link TestReport}.
+ * the text will be escaped before being added.
+ * @param tag The tag to use for adding the text
+ */
+ public void add_text(string text, string tag) {
+ var markup = Markup.escape_text(text);
+ Xml.Node* child = new Xml.Node(null, tag);
+ child->set_content(markup);
+
+ string[] tags = {ERROR_TAG, FAILURE_TAG, INFO_TAG};
+
+ if(tag in tags) {
+ child->new_prop("message", markup);
+ child->new_prop("type", tag.up());
+ }
+
+ Xml.Node* root = xml.eval("//testcase | //testsuite")[0];
+ root->add_child(child);
+ }
+
+ public void update_status() {
+ if(test is TestAdapter && !subprocess) {
+ Xml.Node* root = xml.eval("//testcase")[0];
+ root->set_prop("status",test.status.to_string().substring(21));
+ root->set_prop("time",test.time.to_string());
+ }
+ }
+
+ private static void emit_timer() {
+ end_time = get_monotonic_time();
+ var ms = "%f".printf(((double)(end_time-start_time))/1000);
+ stderr.printf(MESSAGE_XML, TIMER_TAG, ms, TIMER_TAG, ms, TIMER_TAG);
+ }
+
+ private static void printerr_func (string? text) {
+ if(text == null)
+ return;
+ MatchInfo info;
+ if(regex_err.match(text, 0, out info)) {
+ var escaped = Markup.escape_text(info.fetch(4));
+ stderr.printf(MESSAGE_XML, ERROR_TAG, escaped, ERROR_TAG, text, ERROR_TAG);
+ emit_timer();
+ stderr.printf("</%s></%s>",TESTCASE_TAG, ROOT_TAG);
+ stderr.putc(0);
+ }
+ }
+
+ private void log_func (
+ string? log_domain,
+ LogLevelFlags log_levels,
+ string? message) {
+
+ if (((log_levels & LogLevelFlags.LEVEL_INFO) != 0) ||
+ ((log_levels & LogLevelFlags.LEVEL_MESSAGE) != 0) ||
+ ((log_levels & LogLevelFlags.LEVEL_DEBUG) != 0)) {
+ add_message(INFO_TAG, message);
+ } else {
+ add_error(message);
+ }
+ }
+
+ public void process_buffer(string buffer) throws Error {
+
+ xml = new XmlFile.from_string(buffer);
+
+ var bits = xml.eval("//testcase/text()");
+
+ if(bits.size != 0) {
+ Xml.Node* textnode = bits[0];
+ add_message(SYSTEM_ERR_TAG, textnode->get_content());
+ textnode->unlink();
+ }
+
+ var errs = xml.eval("//failure | //error");
+ if (errs.size > 0 &&
+ test.status != TestStatus.SKIPPED &&
+ test.status != TestStatus.TODO)
+ test.status = TestStatus.FAILED;
+
+ bits = xml.eval("//timer");
+ Xml.Node* timer = bits[0];
+ test.time = double.parse(timer->get_content());
+ timer->unlink();
+
+ update_status();
+ }
+
+ public void add_stdout(string text) {
+ add_message(SYSTEM_OUT_TAG, text);
+ }
+}
diff --git a/valadate/testreportprinter.vala b/valadate/testreportprinter.vala
new file mode 100644
index 0000000..5ddaaf0
--- /dev/null
+++ b/valadate/testreportprinter.vala
@@ -0,0 +1,46 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2017 Chris Daley <chebizarro gmail com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ *
+ *
+ */
+
+public abstract class Valadate.TestReportPrinter {
+
+ public static TestReportPrinter @new(TestConfig config) throws Error {
+ switch(config.format) {
+ case "tap" :
+ return new TapTestReportPrinter(config);
+ case "xml" :
+ return new XmlTestReportPrinter(config);
+ case "gnu" :
+ return new GnuTestReportPrinter(config);
+ default:
+ throw new TestConfigError.TEST_PRINTER("TestReportPrinter %s does not exist",
config.format);
+ }
+ }
+
+ public TestConfig config {get; set;}
+
+ public TestReportPrinter(TestConfig config) throws Error {
+ this.config = config;
+ }
+
+ public abstract void print(TestReport report);
+
+}
diff --git a/valadate/testresult.vala b/valadate/testresult.vala
index e95b703..d04660d 100644
--- a/valadate/testresult.vala
+++ b/valadate/testresult.vala
@@ -20,165 +20,117 @@
* Chris Daley <chebizarro gmail com>
*/
-public class Valadate.TestResult : Object {
-
- private enum TestStatus {
- NOT_RUN,
- RUNNING,
- PASSED,
- SKIPPED,
- ERROR,
- FAILED
- }
-
- private class TestReport {
-
- public signal void report (TestStatus status);
-
- public Test test { get; set; }
+public class Valadate.TestResult {
- public TestStatus status { get; set; }
+ public TestConfig config {get;set;}
+ public TestReportPrinter printer {get;set;}
- public int index { get; set; }
+ private Queue<TestReport> reports = new Queue<TestReport>();
+ private HashTable<Test, TestReport> tests = new HashTable<Test, TestReport>(direct_hash,
direct_equal);
- public string message { get; set; }
+ public TestResult(TestConfig config) throws Error {
+ this.config = config;
+ if(!config.in_subprocess)
+ printer = TestReportPrinter.new(config);
+ }
- public TestReport (Test test, TestStatus status, int index, string? message = null) {
- this.test = test;
- this.status = status;
- this.index = index;
- this.message = message;
+ public bool report() {
+ if (reports.is_empty())
+ return false;
+
+ var rpt = reports.peek_head();
+
+ if (rpt.test.status == TestStatus.PASSED ||
+ rpt.test.status == TestStatus.SKIPPED ||
+ rpt.test.status == TestStatus.TODO ||
+ rpt.test.status == TestStatus.FAILED ||
+ rpt.test.status == TestStatus.ERROR) {
+
+ printer.print(rpt);
+ reports.pop_head();
+ report();
}
+ return true;
}
- private Queue<TestReport> reports = new Queue<TestReport> ();
- private HashTable<Test, TestReport> tests = new HashTable<Test, TestReport> (direct_hash,
direct_equal);
-
- private TestConfig config;
- private MainLoop loop;
-
- public TestResult (TestConfig config) {
- this.config = config;
- }
-
- public void report () {
+ private void update_status(Test test) {
+ var rept = tests.get(test);
+ if(rept == null)
+ return;
- if (reports.is_empty ()) {
- loop.quit ();
+ var parent = test.parent;
+ if(parent == null)
return;
+
+ TestStatus status = TestStatus.PASSED;
+ foreach(var t in parent) {
+ if(t.status == TestStatus.RUNNING)
+ return;
+ else if(t.status == TestStatus.ERROR)
+ status = TestStatus.ERROR;
+ else if(t.status == TestStatus.FAILED)
+ status = TestStatus.FAILED;
}
-
- var rpt = reports.peek_head ();
-
- if (rpt.status == TestStatus.PASSED ||
- rpt.status == TestStatus.SKIPPED ||
- rpt.status == TestStatus.FAILED ||
- rpt.status == TestStatus.ERROR) {
- if (rpt.message != null)
- stdout.puts (rpt.message);
- stdout.flush ();
- rpt.report (rpt.status);
- reports.pop_head ();
- report ();
+ parent.status = status;
+ update_status(parent);
+ }
+
+ public void add_test(Test test) {
+ try {
+ reports.push_tail(new TestReport(test, config.in_subprocess));
+ tests.insert(test, reports.peek_tail());
+ } catch (Error e) {
+ error(e.message);
}
}
-
- public void add_error (Test test, string error) {
- update_test (test, TestStatus.ERROR, "# %s\nnot ok %s %s\n".printf (error, "%d", test.name));
- }
- public void add_failure (Test test, string failure) {
- update_test (test, TestStatus.FAILED, "# %s\nnot ok %s %s\n".printf (failure, "%d",
test.name));
+ public void add_error(Test test, string error) {
+ tests.get(test).add_error(error);
+ update_status(test);
}
- public void add_success (Test test, string message) {
- update_test (test, TestStatus.PASSED, "# %s\nok %s %s\n".printf (message, "%d", test.name));
+ public void add_failure(Test test, string failure) {
+ tests.get(test).add_failure(failure);
+ update_status(test);
}
- public void add_skip (Test test, string reason, string message) {
- update_test (test, TestStatus.SKIPPED, "# %s\nok %s %s # %s\n".printf (message, "%d",
test.name, reason));
+ public void add_success(Test test) {
+ tests.get(test).add_success();
+ update_status(test);
}
- private void update_test (Test test, TestStatus status, string message) {
- var rept = tests.get (test);
- rept.status = status;
- rept.message = message.printf (rept.index);
+ public void add_skip(Test test, string message) {
+ tests.get(test).add_skip(message);
+ update_status(test);
}
- /**
- * Runs a the {@link Valadate.Test}s using the supplied
- * {@link Valadate.TestRunner}.
- *
- * @param runner
- */
- public void run (TestRunner runner) {
-
- var testcount = count_tests (config.root);
-
- if (!config.list_only && config.runtest == null) {
- stdout.printf ("# random seed: %s\n", config.seed);
- stdout.printf ("1..%d\n", testcount);
- }
+ public void process_buffers(Test test, Assembly assembly) throws Error {
+
+ var rept = tests.get(test);
+ if(rept == null)
+ return;
- run_test (runner, config.root, "");
-
- if (config.runtest == null) {
- loop = new MainLoop ();
- var time = new TimeoutSource (15);
- time.set_callback (() => {
- report ();
- return true;
- });
- time.attach (loop.get_context ());
- loop.run ();
- }
- }
+ var bis = new BufferedInputStream (assembly.stderr);
+ bis.fill(-1);
+ var xml = (string)bis.peek_buffer();
+ if(xml.length < 8)
+ return;
- private int count_tests (Test test) {
- var testcount = 0;
+ rept.process_buffer(xml);
- if (test is TestSuite)
- foreach (var subtest in test)
- testcount += count_tests (subtest);
- else
- testcount += test.count;
+ update_status(test);
- return testcount;
- }
+ uint8 outbuffer[4096] = {};
+ assembly.stdout.read_all(outbuffer, null);
+ xml = ((string)outbuffer).strip();
+ int i;
+ for(i=xml.length-1;i==0;i--)
+ if(xml.get_char(i).isgraph())
+ break;
- private int testno = 0;
-
- private void run_test (TestRunner runner, Test test, string path) {
- foreach (var subtest in test) {
- string testpath = "%s/%s".printf (path, subtest.name);
- if (subtest is TestCase) {
- if (config.runtest == null && !config.list_only) {
- reports.push_tail (new TestReport (subtest, TestStatus.PASSED, -1, "#
Start of %s tests\n".printf (testpath)));
- run_test (runner, subtest, testpath);
- reports.push_tail (new TestReport (subtest, TestStatus.PASSED, -1, "#
End of %s tests\n".printf (testpath)));
- } else {
- run_test (runner, subtest, testpath);
- }
- } else if (subtest is TestSuite) {
- run_test (runner, subtest, testpath);
- if (config.runtest == null) {
- var rpt = new TestReport (subtest, TestStatus.PASSED, -1);
- rpt.report.connect ((s)=> ((TestSuite)subtest).tear_down ());
- reports.push_tail (rpt);
- }
- } else if (config.list_only) {
- stdout.printf ("%s\n", testpath);
- } else if (config.runtest != null) {
- if (config.runtest == testpath)
- runner.run_test (subtest, this);
- } else {
- testno++;
- subtest.name = testpath;
- var rept = new TestReport (subtest, TestStatus.RUNNING, testno);
- reports.push_tail (rept);
- tests.insert (subtest, rept);
- runner.run.begin (subtest, this);
- }
- }
+ xml = xml.substring(0, i+1);
+ if(xml.length < 1 || xml == "\n")
+ return;
+ rept.add_stdout(xml);
}
}
diff --git a/valadate/testrunner.vala b/valadate/testrunner.vala
index 205966b..b029539 100644
--- a/valadate/testrunner.vala
+++ b/valadate/testrunner.vala
@@ -21,123 +21,145 @@
*/
public class Valadate.TestRunner : Object {
-
- private uint _n_ongoing_tests = 0;
- private Queue<DelegateWrapper> _pending_tests = new Queue<DelegateWrapper> ();
-
- /* Change this to change the cap on the number of concurrent operations. */
- private static uint _max_n_ongoing_tests = GLib.get_num_processors ();
-
private class DelegateWrapper {
public SourceFunc cb;
}
- private SubprocessLauncher launcher =
- new SubprocessLauncher (GLib.SubprocessFlags.STDOUT_PIPE | GLib.SubprocessFlags.STDERR_MERGE);
-
- private string binary;
+ private uint _n_ongoing_tests = 0;
+ private Queue<DelegateWrapper> _pending_tests = new Queue<DelegateWrapper> ();
+ private static uint _max_n_ongoing_tests = GLib.get_num_processors();
+ private MainLoop loop;
+ private TestPlan plan;
- public TestRunner (string binary) {
- this.binary = binary;
- this.launcher.setenv("G_MESSAGES_DEBUG","all", true);
- this.launcher.setenv("G_DEBUG","fatal-criticals fatal-warnings gc-friendly", true);
- this.launcher.setenv("G_SLICE","always-malloc debug-blocks", true);
- GLib.set_printerr_handler (printerr_func_stack_trace);
- Log.set_default_handler (log_func_stack_trace);
+ public void run(Test test, TestResult result) {
+ result.add_test(test);
+ test.run(result);
+ result.report();
}
- private static void printerr_func_stack_trace (string? text) {
- if (text == null || str_equal (text, ""))
- return;
- stderr.printf (text);
-
- /* Print a stack trace since we've hit some major issue */
- GLib.on_error_stack_trace ("libtool --mode=execute gdb");
+ public int run_all(TestPlan plan) throws Error {
+ this.plan = plan;
+
+ if (plan.config.list_only) {
+ list_tests(plan.root, "");
+ return 0;
+ } else if (plan.root.count == 0) {
+ return 0;
+ } else if (!plan.config.in_subprocess) {
+ loop = new MainLoop();
+ Timeout.add(
+ 10,
+ () => {
+ bool res = plan.result.report();
+ if(!res)
+ loop.quit();
+ return res;
+
+ },
+ Priority.HIGH_IDLE);
+ run_test_internal(plan.root, plan.result, "");
+ loop.run();
+ } else {
+ run_test_internal(plan.root, plan.result, "");
+ }
+ return 0;
}
-
- private void log_func_stack_trace (
- string? log_domain,
- LogLevelFlags log_levels,
- string message) {
- Log.default_handler (log_domain, log_levels, message);
-
- /* Print a stack trace for any message at the warning level or above */
- if ((log_levels & (
- LogLevelFlags.LEVEL_WARNING |
- LogLevelFlags.LEVEL_ERROR |
- LogLevelFlags.LEVEL_CRITICAL)) != 0) {
- GLib.on_error_stack_trace ("libtool --mode=execute gdb");
+
+ private void list_tests(Test test, string path) {
+ foreach(var subtest in test) {
+ string testpath = "%s/%s".printf(path, subtest.name);
+ if(subtest is TestAdapter)
+ stdout.printf("%s\n", testpath);
+ else
+ list_tests(subtest, testpath);
}
}
- public void run_test (Test test, TestResult result) {
- test.run (result);
+ public void run_test(Test test, TestResult result) {
+ test.run(result);
}
- public async void run (Test test, TestResult result) {
-
- string command = "%s -r %s".printf(binary, test.name);
- string[] args;
- string buffer = null;
+ private void run_test_internal(Test test, TestResult result, string path) throws Error {
+
+ foreach(var subtest in test) {
+
+ var testpath = "%s/%s".printf(path, subtest.name);
+
+ if(subtest is TestCase) {
+ if(!plan.config.in_subprocess)
+ result.add_test(subtest);
+ run_test_internal(subtest, result, testpath);
+ } else if (subtest is TestSuite) {
+ result.add_test(subtest);
+ run_test_internal(subtest, result, testpath);
+ } else if (plan.config.in_subprocess) {
+ if(plan.config.running_test == testpath)
+ test.run(result);
+ } else if (subtest is TestAdapter) {
+ subtest.name = testpath;
+ result.add_test(subtest);
+ run_async.begin(subtest, result);
+ }
+ }
+ }
+ private async void run_async(Test test, TestResult result) throws Error
+ requires(plan.config.in_subprocess != true) {
+ var timeout = plan.config.timeout;
+ var testprog = plan.assembly.clone();
if (_n_ongoing_tests > _max_n_ongoing_tests) {
var wrapper = new DelegateWrapper();
- wrapper.cb = run.callback;
+ wrapper.cb = run_async.callback;
_pending_tests.push_tail((owned)wrapper);
yield;
}
try {
_n_ongoing_tests++;
+ var cancellable = new Cancellable ();
+ var tcase = test as TestAdapter;
+ if(timeout != tcase.timeout)
+ timeout = tcase.timeout;
+
+ var time = new TimeoutSource (timeout);
- Shell.parse_argv(command, out args);
- var process = launcher.spawnv(args);
- yield process.communicate_utf8_async(null, null, out buffer, null);
-
- if(process.wait_check())
- process_buffer(test, result, buffer);
-
+ cancellable.cancelled.connect (() => {
+ testprog.quit();
+ });
+
+ time.set_callback (() => {
+ if (tcase.status == TestStatus.RUNNING)
+ cancellable.cancel();
+ return false;
+ });
+ time.attach (loop.get_context());
+
+ testprog.run("-r %s".printf(test.name), cancellable);
+ result.add_success(test);
+ result.process_buffers(test, testprog);
+ } catch (IOError e) {
+ result.add_error(test, "The test timed out after %d milliseconds".printf(timeout));
+ result.process_buffers(test, testprog);
} catch (Error e) {
- process_buffer(test, result, buffer, true);
+ result.add_error(test, e.message);
+ result.process_buffers(test, testprog);
} finally {
+ result.report();
_n_ongoing_tests--;
var wrapper = _pending_tests.pop_head ();
if(wrapper != null)
wrapper.cb();
}
}
-
- public void process_buffer (Test test, TestResult result, string buffer, bool failed = false) {
- string skip = null;
- string[] message = {};
-
- foreach(string line in buffer.split("\n"))
- if (line.has_prefix("SKIP "))
- skip = line;
- else
- message += line;
-
- if (skip != null)
- result.add_skip(test, skip, string.joinv("\n",message));
- else
- if(failed)
- result.add_failure(test, string.joinv("\n",message));
- else
- result.add_success(test, string.joinv("\n",message));
- }
-
+
public static int main (string[] args) {
- var bin = args[0];
- var config = new TestConfig();
- int result = config.parse(args);
-
- if(result >= 0)
- return result;
-
- var runner = new TestRunner(bin);
- var testresult = new TestResult(config);
- testresult.run(runner);
-
- return 0;
+ try {
+ var assembly = new TestAssembly(args);
+ var testplan = new TestPlan(assembly);
+ return testplan.run();
+ } catch (Error e) {
+ error(e.message);
+ }
}
+
}
diff --git a/valadate/teststatus.vala b/valadate/teststatus.vala
new file mode 100644
index 0000000..db9e4bf
--- /dev/null
+++ b/valadate/teststatus.vala
@@ -0,0 +1,30 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2017 Chris Daley <chebizarro gmail com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Authors:
+ * Chris Daley <chebizarro gmail com>
+ */
+public enum Valadate.TestStatus {
+ NOT_RUN,
+ RUNNING,
+ PASSED,
+ SKIPPED,
+ TODO,
+ ERROR,
+ FAILED
+}
diff --git a/valadate/testsuite.vala b/valadate/testsuite.vala
index 37762ad..6b0239c 100644
--- a/valadate/testsuite.vala
+++ b/valadate/testsuite.vala
@@ -22,58 +22,90 @@
public class Valadate.TestSuite : Object, Test {
+ private List<Test> _tests = new List<Test>();
/**
* the name of the TestSuite
*/
public string name { get; set; }
-
+ /**
+ * the label of the TestSuite
+ */
+ public string label { get; set; }
+ /**
+ * Iterator (not the actual number of Tests that will be run)
+ */
+ public int size {
+ get {
+ return (int)_tests.length();
+ }
+ }
/**
* Returns the number of {@link Valadate.Test}s that will be run by
* this TestSuite
*/
public int count {
get {
- return (int) _tests.length ();
+ int testcount = 0;
+ _tests.foreach((t) => {
+ testcount += t.count;
+ });
+ return testcount;
}
}
-
+ public Test? parent {get;set;}
/**
* Returns a {@link GLib.List} of {@link Valadate.Test}s that will be
* run by this TestSuite
*/
- public GLib.List<Test> tests {
+ public List<Test> tests {
get {
return _tests;
}
}
-
- private GLib.List<Test> _tests = new GLib.List<Test> ();
-
+ public TestStatus status {get;set;default=TestStatus.NOT_RUN;}
+ public double time {get;set;}
+ public int skipped {get;set;}
+ public int errors {get;set;}
+ public int failures {get;set;}
/**
* The public constructor takes an optional string parameter for the
* TestSuite's name
*/
- public TestSuite (string? name = null) {
- Object (name : name);
+ public TestSuite(string? name = null) {
+ this.name = name ?? this.get_type().name();
+ this.label = name;
}
-
- construct {
- if (name == null)
- name = get_type ().name ();
- }
-
/**
* Adds a test to the suite.
*/
- public void add_test (Test test) {
- _tests.append (test);
+ public void add_test(Test test) {
+ test.parent = this;
+ _tests.append(test);
}
-
+ /**
+ * Runs all of the tests in the Suite
+ */
public void run (TestResult result) {
- _tests.foreach ((t) => { t.run (result); });
+
+ if(status != TestStatus.NOT_RUN)
+ return;
+
+ _tests.foreach((t) => {
+ t.run(result);
+ });
}
- public Test get_test (int index) {
+ public new Test get(int index) {
return _tests.nth_data((uint)index);
}
+
+ public new void set(int index, Test test) {
+ test.parent = this;
+ _tests.insert_before(_tests.nth(index), test);
+ var t = _tests.nth_data((uint)index++);
+ _tests.remove(t);
+ }
+
+ public virtual void set_up() {}
+ public virtual void tear_down() {}
}
diff --git a/valadate/xmlfile.vala b/valadate/xmlfile.vala
new file mode 100644
index 0000000..ca76e17
--- /dev/null
+++ b/valadate/xmlfile.vala
@@ -0,0 +1,108 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright 2016 Chris Daley <chebizarro gmail com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ *
+ */
+
+public errordomain Valadate.XmlFileError {
+ ERROR
+}
+
+public class Valadate.XmlSearchResults {
+
+ private Xml.XPath.Object* result;
+
+ public int size {
+ get {
+ if(result == null || result->type != Xml.XPath.ObjectType.NODESET ||
result->nodesetval == null)
+ return 0;
+ return result->nodesetval->length();
+ }
+ }
+
+ public void* get(int i)
+ requires(size > 0)
+ requires(i < size)
+ requires(i >= 0)
+ {
+ return result->nodesetval->item (i);
+ }
+
+
+ internal XmlSearchResults(Xml.XPath.Object* result) {
+ this.result = result;
+ }
+
+ ~XmlSearchResults() {
+ if(result != null) delete result;
+ }
+
+}
+
+public class Valadate.XmlFile {
+
+ private Xml.Doc* document;
+ private Xml.XPath.Context context;
+ private bool owns_doc = false;
+
+ public XmlFile(File path) throws Error {
+ this.from_doc(Xml.Parser.parse_file(path.get_path()));
+ owns_doc = true;
+ }
+
+ internal XmlFile.from_doc(Xml.Doc* xmldoc) throws Error {
+ document = xmldoc;
+ owns_doc = true;
+
+ if (document == null)
+ throw new XmlFileError.ERROR(
+ "There was an error parsing the Xml.Doc");
+
+ set_context();
+ }
+
+ public XmlFile.from_string(string xml) throws Error {
+ document = Xml.Parser.read_memory(xml, xml.length, null, null,
+ Xml.ParserOption.RECOVER | Xml.ParserOption.NOERROR |
+ Xml.ParserOption.NOWARNING | Xml.ParserOption.NOBLANKS);
+ owns_doc = true;
+
+ if (document == null)
+ throw new XmlFileError.ERROR(
+ "There was an error parsing the string %s", xml);
+ set_context();
+ }
+
+ private void set_context() {
+ context = new Xml.XPath.Context (document);
+ }
+
+ ~XmlFile() {
+ if (owns_doc)
+ delete document;
+ }
+
+ public void register_ns(string prefix, string ns) {
+ context.register_ns(prefix, ns);
+ }
+
+ public XmlSearchResults eval(string expression) {
+ return new XmlSearchResults(context.eval_expression (expression));
+ }
+
+}
diff --git a/valadate/xmltestreportprinter.vala b/valadate/xmltestreportprinter.vala
new file mode 100644
index 0000000..3ae7c3a
--- /dev/null
+++ b/valadate/xmltestreportprinter.vala
@@ -0,0 +1,69 @@
+/*
+ * Valadate - Unit testing library for GObject-based libraries.
+ * Copyright (C) 2017 Chris Daley <chebizarro gmail com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ *
+ *
+ */
+
+public class Valadate.XmlTestReportPrinter : TestReportPrinter {
+
+ private const string XML_DECL ="<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
+ private const string TESTSUITES_XML =
+ """<testsuites disabled="" errors="" failures="" name="" """ +
+ """tests="" time=""></testsuites>""";
+
+ public XmlFile xml {get;set;}
+
+ private Xml.Node* testsuite;
+ private Xml.Node* oldtestsuite;
+ private int testcount = -1;
+ private int casecount = -1;
+
+ public XmlTestReportPrinter(TestConfig config) throws Error {
+ base(config);
+ this.config = config;
+ xml = new XmlFile.from_string(XML_DECL + TESTSUITES_XML);
+ }
+
+ public override void print(TestReport report) {
+ Xml.Node* root = xml.eval("//testsuites")[0];
+ Xml.Node* node = report.xml.eval("//testsuite | //testcase")[0];
+
+ if(report.test is TestSuite) {
+ if(testsuite == null) {
+ testcount = report.test.count;
+ testsuite = root->add_child(node->copy_list());
+ } else {
+ oldtestsuite = testsuite;
+ testsuite = testsuite->add_child(node->copy_list());
+ }
+ } else if (report.test is TestCase) {
+ oldtestsuite = testsuite;
+ testsuite = testsuite->add_child(node->copy_list());
+ casecount = report.test.count;
+ } else if(report.test is TestAdapter) {
+ testsuite->add_child(node->copy_list());
+ testcount--;
+ casecount--;
+ if(casecount == 0)
+ testsuite = oldtestsuite;
+ }
+ if(testcount == 0)
+ root->doc->dump_format(stdout);
+ }
+}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
