[vala/wip/valadate: 126/137] now runs in separate processes



commit 18d1f8aa4e09f3c4f1a1f4701a9a09dfe86b1f51
Author: chebizarro gmail com <chebizarro gmail com>
Date:   Wed Jul 27 08:34:51 2016 -0700

    now runs in separate processes

 tests/Makefile.am          |    2 +-
 tests/valadatetests.vala   |   19 ++++-
 valadate/testcase.vala     |  205 ++++---------------------------------------
 valadate/testconfig.vala   |   28 ++++---
 valadate/testexplorer.vala |    8 ++-
 valadate/testresult.vala   |  107 ++++++++++++++---------
 valadate/testrunner.vala   |   45 ++++++----
 7 files changed, 154 insertions(+), 260 deletions(-)
---
diff --git a/tests/Makefile.am b/tests/Makefile.am
index ab4a6ab..b37e2b3 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,7 +1,7 @@
 include $(top_srcdir)/Makefile.common
 include $(top_srcdir)/build-aux/glib-tap.mk
 
-test_programs = valadatetests valactests@PACKAGE_SUFFIX@
+test_programs = valadatetests #valactests@PACKAGE_SUFFIX@
 
 # Valadate tests
 valadatetests_VALAFLAGS = \
diff --git a/tests/valadatetests.vala b/tests/valadatetests.vala
index 231b32a..abaa7bb 100644
--- a/tests/valadatetests.vala
+++ b/tests/valadatetests.vala
@@ -24,10 +24,27 @@ public class Valadate.Tests.TestFixture : Valadate.TestCase {
        
        public void test_testcase() {
                
+               bug_base = "http://bugzilla.gnome.org/";;
+               
+               bug("555666");
+               
+               stdout.puts("This is a test of the system");
+       }
+
+       public void test_testcase_1() {
+               message("This is a test of the system");
+       }
+
+       public void test_testcase_2() {
+               debug("This is a second test of the system");
+               skip("No reason");
        }
 
-       public void test_testcase_two() {
+       public void test_testcase_3() {
                
+               //stdout.puts("Before");
+               //assert(false);
+               fail("after");
        }
        
 }
diff --git a/valadate/testcase.vala b/valadate/testcase.vala
index 7a68e7c..121f0d7 100644
--- a/valadate/testcase.vala
+++ b/valadate/testcase.vala
@@ -62,6 +62,9 @@ public abstract class Valadate.TestCase : Object, Test, TestFixture {
                }
        }
 
+       public string bug_base {get;set;}
+       
+
        private List<Test> _tests = new List<Test>();
 
        public void add_test(string testname, owned TestMethod test) {
@@ -70,11 +73,7 @@ public abstract class Valadate.TestCase : Object, Test, TestFixture {
        }
 
        
-       public void run(TestResult result) {
-
-               //result.run(this);
-               
-       }
+       public virtual void run(TestResult result) { }
 
        public Test get_test(int index) {
 
@@ -82,6 +81,20 @@ public abstract class Valadate.TestCase : Object, Test, TestFixture {
 
        }
 
+       public void bug(string reference)
+               requires(bug_base != null)
+       {
+               stdout.printf("MSG: Bug Reference: %s%s\n",bug_base, reference);
+               
+       }
+
+       public void skip(string message) {
+               stdout.printf("SKIP Skipping Test %s\n", message);
+       }
+
+       public void fail(string message) {
+               error("FAIL Test %s\n", message);
+       }
 
        public virtual void set_up() {}
 
@@ -111,7 +124,7 @@ public abstract class Valadate.TestCase : Object, Test, TestFixture {
                        this.testcase = testcase;
                }
 
-               public void run(TestResult test) {
+               public void run(TestResult result) {
                        this.testcase.set_up();
                        this.test();
                        this.testcase.tear_down();
@@ -119,184 +132,4 @@ public abstract class Valadate.TestCase : Object, Test, TestFixture {
 
        }
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-       
-
-
-
-       public delegate void AsyncBegin(AsyncReadyCallback callback);
-       public delegate void AsyncFinish(AsyncResult result) throws GLib.Error;
-
-       public GLib.TestSuite _suite;
-
-       public GLib.TestSuite suite {
-               get {
-                       if (_suite == null)
-                               _suite = new GLib.TestSuite (this.name);
-                       return _suite;
-               }
-       }
-
-
-       private Adaptor[] adaptors = new Adaptor[0];
-
-
-       
-       construct {
-               //name = this.get_type().name();
-       }
-
-       public void add_testb (string name, owned TestMethod test)
-               requires (name.contains("/") != true)
-       {
-               var adaptor = new Adaptor (name, (owned)test, this);
-               this.adaptors += adaptor;
-
-               //this._tests.insert(name, adaptor);
-
-               this.suite.add (new GLib.TestCase (adaptor.name,
-                                                                                  adaptor.set_up,
-                                                                                  adaptor.run,
-                                                                                  adaptor.tear_down ));
-       }
-
-
-       public void add_async_test (
-               string name,
-               owned AsyncBegin async_begin,
-               owned AsyncFinish async_finish,
-               int timeout = 200)
-       {
-               var adaptor = new Adaptor (name, () => { }, this);
-               adaptor.is_async = true;
-               adaptor.async_begin = (owned)async_begin;
-               adaptor.async_finish = (owned)async_finish;
-               adaptor.async_timeout = timeout;
-               this.adaptors += adaptor;
-
-               //this._tests.insert(name, adaptor);
-
-               this.suite.add (new GLib.TestCase (
-                       adaptor.name,
-                       adaptor.set_up,
-                       adaptor.run,
-                       adaptor.tear_down,
-                       sizeof(Adaptor)));
-
-       }
-
-
-
-
-
-       private class Adaptor {
-               [CCode (notify = false)]
-               public string name { get; private set; }
-               public int async_timeout { get; set; }
-
-               private TestMethod test;
-               private TestCase test_case;
-
-               public bool is_async = false;
-               public AsyncBegin async_begin;
-               public AsyncFinish async_finish;
-
-               public Adaptor (string name,
-                                               owned TestMethod test,
-                                               TestCase test_case) {
-                       this.name = name;
-                       this.test = (owned)test;
-                       this.test_case = test_case;
-               }
-
-               public void set_up (void* fixture) {
-                       GLib.set_printerr_handler (printerr_func_stack_trace);
-                       Log.set_default_handler (log_func_stack_trace);
-                       this.test_case.set_up ();
-               }
-
-               private static void printerr_func_stack_trace (string? text) {
-                       if (text == null || str_equal (text, ""))
-                               return;
-
-                       stderr.printf (text);
-
-                       /* Print a stack trace since we've hit some major issue */
-                       GLib.on_error_stack_trace ("libtool --mode=execute gdb");
-               }
-
-               private void log_func_stack_trace (
-                       string? log_domain,
-                       LogLevelFlags log_levels,
-                       string message) {
-                       Log.default_handler (log_domain, log_levels, message);
-
-                       /* Print a stack trace for any message at the warning level or above */
-                       if ((log_levels & (
-                               LogLevelFlags.LEVEL_WARNING |
-                               LogLevelFlags.LEVEL_ERROR |
-                               LogLevelFlags.LEVEL_CRITICAL)) != 0) {
-                               GLib.on_error_stack_trace ("libtool --mode=execute gdb");
-                       }
-               }
-
-               public void run (void* fixture) {
-                       if (this.is_async) {
-                               try     {
-                                       assert( wait_for_async (async_timeout, this.async_begin, 
this.async_finish) );
-                               }
-                               catch (GLib.Error err) {
-                                       message(@"Got exception while excuting asynchronous test: 
$(err.message)");
-                                       GLib.Test.fail();
-                               }
-                       } else {
-                               this.test();
-                       }
-               }
-
-
-               public void tear_down (void* fixture) {
-                       this.test_case.tear_down ();
-               }
-               
-               public bool wait_for_async(int timeout, AsyncBegin async_function, AsyncFinish async_finish) 
throws GLib.Error {
-                       var loop = new MainLoop(MainContext.default(), true);
-                       AsyncResult? result = null;
-                       // Plan the async function
-                       async_function((o, r) => { result = r; loop.quit(); });
-                       // Plan timeout
-                       var t1 = Timeout.add(timeout, () => { loop.quit(); return false; });
-                       // Run the loop if it was not quit yet.
-                       if(loop.is_running())
-                               loop.run();
-                       // Cancel timer
-                       Source.remove(t1);
-                       // Check the outcome
-                       if(result == null)
-                               return false;
-                       async_finish(result);
-                       return true;
-               }
-       }
 }
diff --git a/valadate/testconfig.vala b/valadate/testconfig.vala
index 11990df..6fcfaab 100644
--- a/valadate/testconfig.vala
+++ b/valadate/testconfig.vala
@@ -26,8 +26,8 @@ public class Valadate.TestConfig : Object {
        private static string testplan;
        private static string _runtest;
        private static string format = "tap";
-       private static bool fatal_warnings;
        private static bool list;
+       private static bool _keepgoing;
        private static bool quiet;
        private static bool timed;
        private static bool verbose;
@@ -58,6 +58,12 @@ public class Valadate.TestConfig : Object {
                }
        }
 
+       public bool keep_going {
+               get {
+                       return _keepgoing;
+               }
+       }
+
        public string binary {get;set;}
 
        public TestSuite root {get;set;}
@@ -73,8 +79,8 @@ public class Valadate.TestConfig : Object {
        public const OptionEntry[] options = {
                { "seed", 0, 0, OptionArg.STRING, ref _seed, "Start tests with random seed", "SEEDSTRING" },
                { "format", 'f', 0, OptionArg.STRING, ref format, "Output test results using format", 
"FORMAT" },
-               { "g-fatal-warnings", 0, 0, OptionArg.NONE, ref fatal_warnings, "Make all warnings fatal", 
null },
                { "list", 'l', 0, OptionArg.NONE, ref list, "List test cases available in a test executable", 
null },
+               { "", 'k', 0, OptionArg.NONE, ref _keepgoing, "Skip failed tests and continue running", null 
},
                { "skip", 's', 0, OptionArg.STRING_ARRAY, ref skip, "Skip all tests matching", "TESTPATH..." 
},
                { "quiet", 'q', 0, OptionArg.NONE, ref quiet, "Run tests quietly", null },
                { "timed", 0, 0, OptionArg.NONE, ref timed, "Run timed tests", null },
@@ -94,14 +100,6 @@ public class Valadate.TestConfig : Object {
                opt_context.add_main_entries (options, null);
        }
 
-       private void setup_context() {
-               context = new Vala.CodeContext ();
-               Vala.CodeContext.push (context);
-               context.report.enable_warnings = false;
-               context.report.set_verbose_errors (false);
-               context.verbose_mode = false;
-       }
-
        public int parse(string[] args) {
                binary = args[0];
                GLib.Environment.set_prgname(binary);
@@ -162,7 +160,6 @@ public class Valadate.TestConfig : Object {
                }
        }
 
-
        internal void load_test_plan(string path) throws ConfigError {
                setup_context();
 
@@ -174,6 +171,15 @@ public class Valadate.TestConfig : Object {
                var testexplorer = new TestExplorer(this);
                context.accept(testexplorer);
        }
+       
+       private void setup_context() {
+               context = new Vala.CodeContext ();
+               Vala.CodeContext.push (context);
+               context.report.enable_warnings = false;
+               context.report.set_verbose_errors (false);
+               context.verbose_mode = false;
+       }
+
 
 }
 
diff --git a/valadate/testexplorer.vala b/valadate/testexplorer.vala
index 974340e..fd34840 100644
--- a/valadate/testexplorer.vala
+++ b/valadate/testexplorer.vala
@@ -76,12 +76,18 @@ internal class Valadate.TestExplorer : Vala.CodeVisitor {
                                method.has_result != true &&
                                method.get_parameters().size == 0
                        ) {
+
+                               if (config.runtest != null &&
+                                       config.runtest != "/" + method.get_full_name().replace(".","/"))
+                                       continue;
+
+
                                unowned TestMethod testmethod = null;
                                var attr = new Vala.CCodeAttribute(method);
                                testmethod = (TestMethod)config.module.get_method(attr.name);
 
                                if (testmethod != null) {
-                                       current_test.add_test(method.name.substring(5), ()=> {
+                                       current_test.add_test(method.name, ()=> {
                                                testmethod(current_test);
                                        });
                                }
diff --git a/valadate/testresult.vala b/valadate/testresult.vala
index f0e162d..f083816 100644
--- a/valadate/testresult.vala
+++ b/valadate/testresult.vala
@@ -21,16 +21,24 @@
  */
 public class Valadate.TestResult : Object {
 
+       private enum TestStatus {
+               NOT_RUN,
+               RUNNING,
+               PASSED,
+               FAILED
+       }
+
        public signal void test_error(Test test, string error);
        public signal void test_failure(Test test, string error);
        public signal void test_complete(Test test);
        public signal void test_start(Test test);
        
-       internal List<TestFailure> errors = new List<TestFailure>();
+       /*internal List<TestFailure> errors = new List<TestFailure>();
        internal List<TestFailure> failures = new List<TestFailure>();
 
        public bool should_stop {get;set;default=false;}
        
+       
        public int error_count {
                get {
                        return (int)errors.length();
@@ -50,80 +58,95 @@ public class Valadate.TestResult : Object {
                        return (failure_count == 0 && error_count == 0);
                }
        }
+       */
        
-       internal TestConfig config;
-       private TestRunner runner;
+       public string binary {
+               get {
+                       return config.binary;
+               }
+       }
        
-       public TestResult(TestConfig config) {
-               this.config = config;
+       private TestConfig config;
+       private TestRunner runner;
+
+       /*
+       private HashTable<Test, TestRecord> _tests = new HashTable<Test, TestRecord>(direct_hash, 
direct_equal);
+
+       private class TestRecord : Object {
                
+               public string path {get;set;}
+               public int index {get;set;}
+               public TestStatus status {get;set;}
                
+               public TestRecord(string path, int index, TestStatus status) {
+                       this.path = path;
+                       this.index = index;
+                       this.status = status;
+               }
                
+       }*/
+       
+       
+       public TestResult(TestConfig config) {
+               this.config = config;
        }
        
        public void add_error(Test test, string error) {
-               errors.append(new TestFailure(test, error));
-               test_error(test, error);
+               stdout.printf("%s\n", error);
+               stdout.printf("not ok %d %s\n", testno, test.name);
+               //errors.append(new TestFailure(test, error));
+               //test_error(test, error);
        }
 
        public void add_failure(Test test, string failure) {
-               failures.append(new TestFailure(test, failure));
-               test_failure(test, failure);
+               stdout.printf("%s\n", failure);
+               stdout.printf("not ok %d %s\n", testno, test.name);
+               //failures.append(new TestFailure(test, failure));
+               //test_failure(test, failure);
        }
 
-       public void start_test(Test test) {
-               run_count += test.count;
-               test_start(test);
+       public void add_success(Test test, string message) {
+               stdout.printf("%s\n", message);
+               stdout.printf("ok %d %s\n", testno, test.name);
+               //failures.append(new TestFailure(test, failure));
+               //test_failure(test, failure);
        }
+
        
        /**
         * Runs a {@link Valadate.Test}
         */
        public void run(TestRunner runner) {
                this.runner = runner;
-               stdout.printf("# random seed: %s\n", config.seed);
-               stdout.printf("%d..%d\n", (config.test_count > 0) ? 1 : 0, config.test_count);
+               if (!config.list_only && config.runtest == null) {
+                       stdout.printf("# random seed: %s\n", config.seed);
+                       stdout.printf("1..%d\n", config.test_count);
+               }
                run_test(config.root, "");
        }
 
-       private static int testno = 0;
+       private int testno = 0;
 
        private void run_test(Test test, string path) {
                foreach(var subtest in test) {
                        string testpath = "%s/%s".printf(path, subtest.name);
                        if(subtest is TestCase) {
-
-                               stdout.printf("# Start of %s tests\n", testpath);
-                               
+                               if(config.runtest == null)
+                                       stdout.printf("# Start of %s tests\n", testpath);
                                run_test(subtest, testpath);
-                               
-                               stdout.printf("# End of %s tests\n", testpath);
-
+                               if(config.runtest == null)
+                                       stdout.printf("# End of %s tests\n", testpath);
                        } else if (subtest is TestSuite) {
-
                                run_test(subtest, testpath);
-
+                       } else if (config.list_only) {
+                               stdout.printf("%s\n", testpath);
+                       } else if (config.runtest != null) {
+                               if(config.runtest == testpath)
+                                       runner.run_test(subtest, this);
                        } else {
-
                                testno++;
-
-                               if (config.list_only) {
-
-                                       stdout.printf("%s\n", testpath);
-
-                               } else if (config.runtest != null && config.runtest == testpath) {
-
-                                       runner.run(subtest, testpath, this);
-
-                                       stdout.printf("ok %d %s/%s\n", testno, path, subtest.name);
-
-                               } else {
-
-                                       runner.run_test(subtest, testpath, this);
-
-                                       stdout.printf("ok %d %s/%s\n", testno, path, subtest.name);
-
-                               }
+                               subtest.name = testpath;
+                               runner.run(subtest, this);
                        }
                }
        }
diff --git a/valadate/testrunner.vala b/valadate/testrunner.vala
index 04f279d..25b9947 100644
--- a/valadate/testrunner.vala
+++ b/valadate/testrunner.vala
@@ -22,34 +22,43 @@
 
 public class Valadate.TestRunner : Object {
 
-       private TestResult result;
-
        private SubprocessLauncher launcher =
-               new SubprocessLauncher(GLib.SubprocessFlags.STDOUT_PIPE | GLib.SubprocessFlags.STDERR_PIPE);
+               new SubprocessLauncher(GLib.SubprocessFlags.STDOUT_PIPE | GLib.SubprocessFlags.STDERR_MERGE);
        
+       private string binary;
        
-       public void run_test(Test test, string testpath, TestResult result) {
-
+       public TestRunner(string binary) {
+               this.binary = binary;
+               this.launcher.setenv("G_MESSAGES_DEBUG","all", true);
+               this.launcher.setenv("G_DEBUG","fatal-criticals fatal-warnings gc-friendly", true);
+               this.launcher.setenv("G_SLICE","always-malloc debug-blocks", true);
+       }
+       
+       public void run_test(Test test, TestResult result) {
                test.run(result);
-
        }
 
-       public void run(Test test, string testpath, TestResult result) {
+       public void run(Test test, TestResult result) {
                
-               string command = "%s -r %s".printf(result.config.binary, testpath);
+               string command = "%s -r %s".printf(binary, test.name);
 
                string[] args;
-               Shell.parse_argv(command, out args);
-
-               var process = launcher.spawnv(args);
-               var stderr_pipe = process.get_stderr_pipe();
 
-               uint8 buffer[1028];
-               var err = stderr_pipe.read(buffer);
+               string buffer = null;
+               
+               try {
+                       Shell.parse_argv(command, out args);
+
+                       var process = launcher.spawnv(args);
+                       process.communicate_utf8(null, null, out buffer, null);
+                       
+                       if(process.wait_check()) {
+                               result.add_success(test, buffer);
+                       }
+               } catch (Error e) {
+                       result.add_error(test, buffer);
+               }
                
-               if (err > 0) {
-                       result.add_failure(test, (string)buffer);
-               }               
 
        }
 
@@ -62,7 +71,7 @@ public class Valadate.TestRunner : Object {
                if(result >= 0)
                        return result;
 
-               var runner = new TestRunner();
+               var runner = new TestRunner(config.binary);
                var testresult = new TestResult(config);
                
                testresult.run(runner);


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]