[libgxps] Add regressions test program
- From: Carlos Garcia Campos <carlosgc src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [libgxps] Add regressions test program
- Date: Tue, 1 Nov 2011 11:54:30 +0000 (UTC)
commit 2b9c3259c5bce182d423c207e5c7786e5a13d301
Author: Carlos Garcia Campos <carlosgc gnome org>
Date: Tue Nov 1 12:42:17 2011 +0100
Add regressions test program
It's the same program I wrote for poppler, but adapted to libgxps and
without the backends part.
TODO | 1 -
regtest/Config.py | 32 ++++++
regtest/Test.py | 205 +++++++++++++++++++++++++++++++++++++++
regtest/TestReferences.py | 70 +++++++++++++
regtest/TestRun.py | 155 +++++++++++++++++++++++++++++
regtest/Timer.py | 73 ++++++++++++++
regtest/Utils.py | 55 +++++++++++
regtest/commands/__init__.py | 93 ++++++++++++++++++
regtest/commands/create-refs.py | 65 ++++++++++++
regtest/commands/run-tests.py | 73 ++++++++++++++
regtest/gxps-regtest | 6 +
regtest/main.py | 73 ++++++++++++++
12 files changed, 900 insertions(+), 1 deletions(-)
---
diff --git a/TODO b/TODO
index ef71635..8ce338c 100644
--- a/TODO
+++ b/TODO
@@ -1,7 +1,6 @@
General
-------
-- Regression tests
- API tests
- Demo program
diff --git a/regtest/Config.py b/regtest/Config.py
new file mode 100644
index 0000000..ff03e3c
--- /dev/null
+++ b/regtest/Config.py
@@ -0,0 +1,32 @@
+# Config.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc gnome org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+class Config:
+
+ shared_state = {}
+
+ def __init__(self, config = None):
+ if config is not None:
+ self.__class__.shared_state = config
+ self.__dict__ = self.__class__.shared_state
+
+if __name__ == '__main__':
+ c = Config({'foo' : 25})
+ print(c.foo)
+ cc = Config()
+ print(cc.foo)
diff --git a/regtest/Test.py b/regtest/Test.py
new file mode 100644
index 0000000..51f98d3
--- /dev/null
+++ b/regtest/Test.py
@@ -0,0 +1,205 @@
+# Test
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc gnome org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+from hashlib import md5
+import os
+import subprocess
+import shutil
+import errno
+from Config import Config
+
+class Test:
+
+ def __init__(self):
+ self._xpstopng = os.path.join(Config().tools_dir, 'xpstopng')
+
+ def __should_have_checksum(self, entry):
+ return entry not in ('md5', 'crashed', 'failed', 'stderr');
+
+ def create_checksums(self, refs_path, delete_refs = False):
+ path = os.path.join(refs_path, 'md5')
+ md5_file = open(path, 'w')
+
+ for entry in os.listdir(refs_path):
+ if not self.__should_have_checksum(entry):
+ continue
+
+ ref_path = os.path.join(refs_path, entry)
+ f = open(ref_path, 'rb')
+ md5_file.write("%s %s\n" % (md5(f.read()).hexdigest(), ref_path))
+ f.close()
+ if delete_refs:
+ os.remove(ref_path)
+
+ md5_file.close()
+
+ def compare_checksums(self, refs_path, out_path, remove_results = True, create_diffs = True, update_refs = False):
+ retval = True
+
+ md5_path = os.path.join(refs_path, 'md5')
+ md5_file = open(md5_path, 'r')
+ tests = os.listdir(out_path)
+ result_md5 = []
+
+ for line in md5_file.readlines():
+ md5sum, ref_path = line.strip('\n').split(' ', 1)
+ basename = os.path.basename(ref_path)
+ if not self.__should_have_checksum(basename):
+ continue
+
+ if not basename in tests:
+ retval = False
+ print("%s found in md5 ref file but missing in output dir %s" % (basename, out_path))
+ continue
+
+ result_path = os.path.join(out_path, basename)
+ f = open(result_path, 'rb')
+ result_md5sum = md5(f.read()).hexdigest()
+ matched = md5sum == result_md5sum
+ f.close()
+
+ if update_refs:
+ result_md5.append("%s %s\n" % (result_md5sum, ref_path))
+
+ if matched:
+ if remove_results:
+ os.remove(result_path)
+ else:
+ print("Differences found in %s" % (basename))
+ if create_diffs:
+ if not os.path.exists(ref_path):
+ print("Reference file %s not found, skipping diff for %s" % (ref_path, result_path))
+ else:
+ self._create_diff(ref_path, result_path)
+
+ if update_refs:
+ if os.path.exists(ref_path):
+ print("Updating image reference %s" % (ref_path))
+ shutil.copyfile(result_path, ref_path)
+ retval = False
+ md5_file.close()
+
+ if update_refs and not retval:
+ print("Updating md5 reference %s" % (md5_path))
+ f = open(md5_path + '.tmp', 'wb')
+ f.writelines(result_md5)
+ f.close()
+ os.rename(md5_path + '.tmp', md5_path)
+
+ for ref in ('crashed', 'failed', 'stderr'):
+ src = os.path.join(out_path, ref)
+ dest = os.path.join(refs_path, ref)
+ try:
+ shutil.copyfile(src, dest)
+ except IOError as e:
+ if e.errno != errno.ENOENT:
+ raise
+
+ return retval
+
+ def has_md5(self, test_path):
+ return os.path.exists(os.path.join(test_path, 'md5'))
+
+ def is_crashed(self, test_path):
+ return os.path.exists(os.path.join(test_path, 'crashed'))
+
+ def is_failed(self, test_path):
+ failed_path = os.path.join(test_path, 'failed')
+ if not os.path.exists(failed_path):
+ return 0
+
+ f = open(failed_path, 'r')
+ status = int(f.read())
+ f.close()
+
+ return status
+
+ def has_stderr(self, test_path):
+ return os.path.exists(os.path.join(test_path, 'stderr'))
+
+ def __create_stderr_file(self, stderr, out_path):
+ if not stderr:
+ return
+
+ stderr_file = open(os.path.join(out_path, 'stderr'), 'wb')
+ stderr_file.write(stderr)
+ stderr_file.close()
+
+ def __create_failed_file_if_needed(self, status, out_path):
+ if os.WIFEXITED(status) or os.WEXITSTATUS(status) == 0:
+ return False
+
+ failed_file = open(os.path.join(out_path, 'failed'), 'w')
+ failed_file.write("%d" % (os.WEXITSTATUS(status)))
+ failed_file.close()
+
+ return True
+
+ def _check_exit_status(self, p, out_path):
+ status = p.wait()
+
+ stderr = p.stderr.read()
+ self.__create_stderr_file(stderr, out_path)
+
+ if not os.WIFEXITED(status):
+ open(os.path.join(out_path, 'crashed'), 'w').close()
+ return False
+
+ if self.__create_failed_file_if_needed(status, out_path):
+ return False
+
+ return True
+
+ def _check_exit_status2(self, p1, p2, out_path):
+ status1 = p1.wait()
+ status2 = p2.wait()
+
+ p1_stderr = p1.stderr.read()
+ p2_stderr = p2.stderr.read()
+ if p1_stderr or p2_stderr:
+ self.__create_stderr_file(p1_stderr + p2_stderr, out_path)
+
+ if not os.WIFEXITED(status1) or not os.WIFEXITED(status2):
+ open(os.path.join(out_path, 'crashed'), 'w').close()
+ return False
+
+ if self.__create_failed_file_if_needed(status1, out_path):
+ return False
+ if self.__create_failed_file_if_needed(status2, out_path):
+ return False
+
+ return True
+
+ def _create_diff(self, ref_path, result_path):
+ try:
+ import Image, ImageChops
+ except ImportError:
+ raise NotImplementedError
+
+ ref = Image.open(ref_path)
+ result = Image.open(result_path)
+ diff = ImageChops.difference(ref, result)
+ diff.save(result_path + '.diff', 'png')
+
+ def create_refs(self, doc_path, refs_path):
+ out_path = os.path.join(refs_path, 'page')
+ p1 = subprocess.Popen([self._xpstopng, '-r', '72', '-e', doc_path, out_path], stderr = subprocess.PIPE)
+ p2 = subprocess.Popen([self._xpstopng, '-r', '72', '-o', doc_path, out_path], stderr = subprocess.PIPE)
+
+ return self._check_exit_status2(p1, p2, refs_path)
+
diff --git a/regtest/TestReferences.py b/regtest/TestReferences.py
new file mode 100644
index 0000000..ea8b306
--- /dev/null
+++ b/regtest/TestReferences.py
@@ -0,0 +1,70 @@
+# TestReferences.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc gnome org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os
+import errno
+from Test import Test
+from Config import Config
+from Utils import get_document_paths_from_dir, get_skipped_tests
+
+class TestReferences:
+
+ def __init__(self, docsdir, refsdir):
+ self._docsdir = docsdir
+ self._refsdir = refsdir
+ self._skipped = get_skipped_tests(docsdir)
+ self._test = Test()
+ self.config = Config()
+
+ try:
+ os.makedirs(self._refsdir)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ except:
+ raise
+
+ def create_refs_for_file(self, filename, n_doc = 1, total_docs = 1):
+ if filename in self._skipped:
+ print("Skipping test '%s' (%d/%d)" % (os.path.join(self._docsdir, filename), n_doc, total_docs))
+ return
+
+ refs_path = os.path.join(self._refsdir, filename)
+ try:
+ os.makedirs(refs_path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ except:
+ raise
+ doc_path = os.path.join(self._docsdir, filename)
+
+ if not self.config.force and self._test.has_md5(refs_path):
+ print("Checksum file found, skipping '%s' (%d/%d)" % (doc_path, n_doc, total_docs))
+ return
+
+ print("Creating refs for '%s' (%d/%d)" % (doc_path, n_doc, total_docs))
+ if self._test.create_refs(doc_path, refs_path):
+ self._test.create_checksums(refs_path, self.config.checksums_only)
+
+ def create_refs(self):
+ docs, total_docs = get_document_paths_from_dir(self._docsdir)
+ n_doc = 0
+ for doc in docs:
+ n_doc += 1
+ self.create_refs_for_file(doc, n_doc, total_docs)
diff --git a/regtest/TestRun.py b/regtest/TestRun.py
new file mode 100644
index 0000000..05c8789
--- /dev/null
+++ b/regtest/TestRun.py
@@ -0,0 +1,155 @@
+# TestRun.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc gnome org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+from Test import Test
+from Config import Config
+from Utils import get_document_paths_from_dir, get_skipped_tests
+import sys
+import os
+import errno
+
+class TestRun:
+
+ def __init__(self, docsdir, refsdir, outdir):
+ self._docsdir = docsdir
+ self._refsdir = refsdir
+ self._outdir = outdir
+ self._skipped = get_skipped_tests(docsdir)
+ self._test = Test()
+ self.config = Config()
+
+ # Results
+ self._n_tests = 0
+ self._n_passed = 0
+ self._failed = []
+ self._crashed = []
+ self._failed_status_error = []
+ self._stderr = []
+
+ try:
+ os.makedirs(self._outdir);
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ except:
+ raise
+
+ def test(self, refs_path, doc_path, test_path, n_doc, total_docs):
+ # First check whether there are test results
+ ref_has_md5 = self._test.has_md5(refs_path)
+ ref_is_crashed = self._test.is_crashed(refs_path)
+ ref_is_failed = self._test.is_failed(refs_path)
+ if not ref_has_md5 and not ref_is_crashed and not ref_is_failed:
+ print("Reference files not found, skipping '%s'" % (doc_path))
+ return
+
+ self._n_tests += 1
+ sys.stdout.write("Testing '%s' (%d/%d): " % (doc_path, n_doc, total_docs))
+ sys.stdout.flush()
+ test_has_md5 = self._test.create_refs(doc_path, test_path)
+
+ if self._test.has_stderr(test_path):
+ self._stderr.append(doc_path)
+
+ if ref_has_md5 and test_has_md5:
+ if self._test.compare_checksums(refs_path, test_path, not self.config.keep_results, self.config.create_diffs, self.config.update_refs):
+ # FIXME: remove dir if it's empty?
+ print("PASS")
+ self._n_passed += 1
+ else:
+ print("FAIL")
+ self._failed.append(doc_path)
+ return
+ elif test_has_md5:
+ if ref_is_crashed:
+ print("DOES NOT CRASH")
+ elif ref_is_failed:
+ print("DOES NOT FAIL")
+
+ return
+
+ test_is_crashed = self._test.is_crashed(test_path)
+ if ref_is_crashed and test_is_crashed:
+ print("PASS (Expected crash)")
+ self._n_passed += 1
+ return
+
+ test_is_failed = self._test.is_failed(test_path)
+ if ref_is_failed and test_is_failed:
+ # FIXME: compare status errors
+ print("PASS (Expected fail with status error %d)" % (test_is_failed))
+ self._n_passed += 1
+ return
+
+ if test_is_crashed:
+ print("CRASH")
+ self._crashed.append(doc_path)
+ return
+
+ if test_is_failed:
+ print("FAIL (status error %d)" % (test_is_failed))
+ self._failed_status_error(doc_path)
+ return
+
+ def run_test(self, filename, n_doc = 1, total_docs = 1):
+ if filename in self._skipped:
+ print("Skipping test '%s' (%d/%d)" % (os.path.join(self._docsdir, filename), n_doc, total_docs))
+ return
+
+ out_path = os.path.join(self._outdir, filename)
+ try:
+ os.makedirs(out_path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ except:
+ raise
+ doc_path = os.path.join(self._docsdir, filename)
+ refs_path = os.path.join(self._refsdir, filename)
+
+ if not os.path.isdir(refs_path):
+ print("Reference dir not found for %s, skipping (%d/%d)" % (doc_path, n_doc, total_docs))
+ return
+
+ self.test(refs_path, doc_path, out_path, n_doc, total_docs)
+
+ def run_tests(self):
+ docs, total_docs = get_document_paths_from_dir(self._docsdir)
+ n_doc = 0
+ for doc in docs:
+ n_doc += 1
+ self.run_test(doc, n_doc, total_docs)
+
+ def summary(self):
+ if not self._n_tests:
+ print("No tests run")
+ return
+
+ print("Total %d tests" % (self._n_tests))
+ print("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_tests))
+ def report_tests(test_list, test_type):
+ n_tests = len(test_list)
+ if not n_tests:
+ return
+ print("%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_tests, ", ".join(test_list)))
+ report_tests(self._failed, "failed")
+ report_tests(self._crashed, "crashed")
+ report_tests(self._failed_status_error, "failed to run")
+ report_tests(self._stderr, "have stderr output")
+
+
diff --git a/regtest/Timer.py b/regtest/Timer.py
new file mode 100644
index 0000000..e86f8c5
--- /dev/null
+++ b/regtest/Timer.py
@@ -0,0 +1,73 @@
+# Timer.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc gnome org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+from time import time, strftime, gmtime
+
+class Timer:
+
+ def __init__(self, start = True):
+ self._stop = None
+ if start:
+ self.start()
+ else:
+ self._start = None
+
+ def start(self):
+ self._start = time()
+
+ def stop(self):
+ self._stop = time()
+
+ def elapsed(self):
+ if self._start is None:
+ return 0
+
+ if self._stop is None:
+ return time() - self._start
+
+ return self._stop - self._start
+
+ def elapsed_str(self):
+ h, m, s = [int(i) for i in strftime('%H:%M:%S', gmtime(self.elapsed())).split(':')]
+ retval = "%d seconds" % (s)
+ if h == 0 and m == 0:
+ return retval
+
+ retval = "%d minutes and %s" % (m, retval)
+ if h == 0:
+ return retval
+
+ retval = "%d hours, %s" % (h, retval)
+ return retval
+
+
+if __name__ == '__main__':
+ from time import sleep
+
+ t = Timer()
+ sleep(5)
+ print("Elapsed: %s" % (t.elapsed_str()))
+ sleep(1)
+ print("Elapsed: %s" % (t.elapsed_str()))
+
+ t.start()
+ sleep(2)
+ t.stop()
+ print("Elapsed: %s" % (t.elapsed_str()))
+ sleep(2)
+ print("Elapsed: %s" % (t.elapsed_str()))
diff --git a/regtest/Utils.py b/regtest/Utils.py
new file mode 100644
index 0000000..e95ae8a
--- /dev/null
+++ b/regtest/Utils.py
@@ -0,0 +1,55 @@
+# Utils.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc gnome org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os
+
+def get_document_paths_from_dir(docsdir):
+ paths = []
+ n_paths = 0
+ for root, dirs, files in os.walk(docsdir, False):
+ for entry in files:
+ if not entry.lower().endswith('.xps'):
+ continue
+
+ test_path = os.path.join(root[len(docsdir):], entry)
+ paths.append(test_path.lstrip(os.path.sep))
+ n_paths += 1
+ paths.sort()
+ return paths, n_paths
+
+def get_skipped_tests(docsdir):
+ from Config import Config
+ config = Config()
+ if config.skipped_file:
+ skipped_file = config.skipped_file
+ elif os.path.exists(os.path.join(docsdir, 'Skipped')):
+ skipped_file = os.path.join(docsdir, 'Skipped')
+ else:
+ return []
+
+ skipped = []
+ f = open(skipped_file, 'r')
+ for line in f.readlines():
+ line = line.rstrip('\n \t\b\r')
+ if not line or line[0] == '#':
+ continue
+ skipped.append(line)
+ f.close()
+ return skipped
+
+
diff --git a/regtest/commands/__init__.py b/regtest/commands/__init__.py
new file mode 100644
index 0000000..1ea49ef
--- /dev/null
+++ b/regtest/commands/__init__.py
@@ -0,0 +1,93 @@
+# commands
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc gnome org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import argparse
+
+__all__ = [ 'register_command',
+ 'print_help',
+ 'run',
+ 'UnknownCommandError',
+ 'Command' ]
+
+class UnknownCommandError(Exception):
+ '''Unknown command'''
+
+class Command:
+
+ name = None
+ usage_args = '[ options ... ]'
+ description = None
+
+ def __init__(self):
+ self._parser = argparse.ArgumentParser(
+ description = self.description,
+ prog = 'gxps-regtest %s' % (self.name),
+ usage = 'gxps-regtest %s %s' % (self.name, self.usage_args))
+
+ def _get_args_parser(self):
+ return self._parser
+
+ def execute(self, args):
+ ns = self._parser.parse_args(args)
+ self.run(vars(ns))
+
+ def run(self, options):
+ raise NotImplementedError
+
+_commands = {}
+def register_command(command_name, command_class):
+ _commands[command_name] = command_class
+
+def _get_command(command_name):
+ if command_name not in _commands:
+ try:
+ __import__('commands.%s' % command_name)
+ except ImportError:
+ pass
+
+ if command_name not in _commands:
+ raise UnknownCommandError('Invalid %s command' % command_name)
+
+ return _commands[command_name]
+
+def run(args):
+ command_class = _get_command(args[0])
+ command = command_class()
+ command.execute(args[1:])
+
+def print_help():
+ import os
+ thisdir = os.path.abspath(os.path.dirname(__file__))
+
+ for fname in os.listdir(os.path.join(thisdir)):
+ name, ext = os.path.splitext(fname)
+ if not ext == '.py':
+ continue
+ try:
+ __import__('commands.%s' % name)
+ except ImportError:
+ pass
+
+ print("Commands are:")
+ commands = [(x.name, x.description) for x in _commands.values()]
+ commands.sort()
+ for name, description in commands:
+ print(" %-15s %s" % (name, description))
+
+ print
+ print("For more information run 'gxps-regtest --help-command <command>'")
diff --git a/regtest/commands/create-refs.py b/regtest/commands/create-refs.py
new file mode 100644
index 0000000..b055703
--- /dev/null
+++ b/regtest/commands/create-refs.py
@@ -0,0 +1,65 @@
+# create-refs.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc gnome org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+from commands import Command, register_command
+from TestReferences import TestReferences
+from Timer import Timer
+from Config import Config
+import os
+import tempfile
+
+class CreateRefs(Command):
+
+ name = 'create-refs'
+ usage_args = '[ options ... ] tests '
+ description = 'Create references for tests'
+
+ def __init__(self):
+ Command.__init__(self)
+ parser = self._get_args_parser()
+ parser.add_argument('--refs-dir',
+ action = 'store', dest = 'refs_dir', default = os.path.join(tempfile.gettempdir(), 'refs'),
+ help = 'Directory where the references will be created')
+ parser.add_argument('-f', '--force',
+ action = 'store_true', dest = 'force', default = False,
+ help = 'Create references again for tests that already have references')
+ parser.add_argument('-c', '--checksums-only',
+ action = 'store_true', dest = 'checksums_only', default = False,
+ help = 'Leave only checksum files in references dir, other files will be deleted')
+ parser.add_argument('tests')
+
+ def run(self, options):
+ config = Config()
+ config.force = options['force']
+ config.checksums_only = options['checksums_only']
+
+ t = Timer()
+ doc = options['tests']
+ if os.path.isdir(doc):
+ docs_dir = doc
+ else:
+ docs_dir = os.path.dirname(doc)
+
+ refs = TestReferences(docs_dir, options['refs_dir'])
+ if doc == docs_dir:
+ refs.create_refs()
+ else:
+ refs.create_refs_for_file(os.path.basename(doc))
+ print("Refs created in %s" % (t.elapsed_str()))
+
+register_command('create-refs', CreateRefs)
diff --git a/regtest/commands/run-tests.py b/regtest/commands/run-tests.py
new file mode 100644
index 0000000..d05d815
--- /dev/null
+++ b/regtest/commands/run-tests.py
@@ -0,0 +1,73 @@
+# run-tests.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc gnome org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+from commands import Command, register_command
+from TestRun import TestRun
+from Timer import Timer
+from Config import Config
+import os
+import tempfile
+
+class RunTests(Command):
+
+ name = 'run-tests'
+ usage_args = '[ options ... ] tests '
+ description = 'Run tests for documents'
+
+ def __init__(self):
+ Command.__init__(self)
+ parser = self._get_args_parser()
+ parser.add_argument('--refs-dir',
+ action = 'store', dest = 'refs_dir', default = os.path.join(tempfile.gettempdir(), 'refs'),
+ help = 'Directory containing the references')
+ parser.add_argument('-o', '--out-dir',
+ action = 'store', dest = 'out_dir', default = os.path.join(tempfile.gettempdir(), 'out'),
+ help = 'Directory where test results will be created')
+ parser.add_argument('--keep-results',
+ action = 'store_true', dest = 'keep_results', default = False,
+ help = 'Do not remove result files for passing tests')
+ parser.add_argument('--create-diffs',
+ action = 'store_true', dest = 'create_diffs', default = False,
+ help = 'Create diff files for failed tests')
+ parser.add_argument('--update-refs',
+ action = 'store_true', dest = 'update_refs', default = False,
+ help = 'Update references for failed tests')
+ parser.add_argument('tests')
+
+ def run(self, options):
+ config = Config()
+ config.keep_results = options['keep_results']
+ config.create_diffs = options['create_diffs']
+ config.update_refs = options['update_refs']
+
+ t = Timer()
+ doc = options['tests']
+ if os.path.isdir(doc):
+ docs_dir = doc
+ else:
+ docs_dir = os.path.dirname(doc)
+
+ tests = TestRun(docs_dir, options['refs_dir'], options['out_dir'])
+ if doc == docs_dir:
+ tests.run_tests()
+ else:
+ tests.run_test(os.path.basename(doc))
+ tests.summary()
+ print("Tests run in %s" % (t.elapsed_str()))
+
+register_command('run-tests', RunTests)
diff --git a/regtest/gxps-regtest b/regtest/gxps-regtest
new file mode 100755
index 0000000..fb1e126
--- /dev/null
+++ b/regtest/gxps-regtest
@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import sys
+import main
+
+main.main(sys.argv[1:])
diff --git a/regtest/main.py b/regtest/main.py
new file mode 100644
index 0000000..c97ec74
--- /dev/null
+++ b/regtest/main.py
@@ -0,0 +1,73 @@
+# main.py
+#
+# Copyright (C) 2011 Carlos Garcia Campos <carlosgc gnome org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import sys
+import argparse
+import commands
+import os
+from Config import Config
+
+class ListAction(argparse.Action):
+ def __call__(self, parser, namespace, values, option_string = None):
+ setattr(namespace, self.dest, values.split(','))
+
+class HelpAction(argparse.Action):
+ def __call__(self, parser, namespace, values, option_string = None):
+ if option_string == '--help-command':
+ commands.run([values, '--help'])
+ sys.exit(0)
+
+ parser.print_help()
+ commands.print_help()
+
+ sys.exit(0)
+
+def main(args):
+ parser = argparse.ArgumentParser(
+ description = 'libgxps regression tests',
+ prog = 'gxps-regtest',
+ usage = '%(prog)s [options ...] command [command-options ...] tests',
+ add_help = False)
+ parser.add_argument('-h', '--help',
+ action = HelpAction, nargs = 0)
+ parser.add_argument('--help-command', metavar = 'COMMAND',
+ action = HelpAction,
+ help = 'Show help for a given command')
+ parser.add_argument('--tools-dir',
+ action = 'store', dest = 'tools_dir', default = os.path.abspath("../tools"),
+ help = 'Directory of gxps tools used for the tests')
+ parser.add_argument('--skip', metavar = 'FILE',
+ action = 'store', dest = 'skipped_file',
+ help = 'File containing tests to skip')
+
+ ns, args = parser.parse_known_args(args)
+ if not args:
+ parser.print_help()
+ sys.exit(0)
+
+ Config(vars(ns))
+ try:
+ commands.run(args)
+ except commands.UnknownCommandError:
+ sys.stderr.write("Unknown command: %s\n" % (args[0]))
+ commands.print_help()
+ sys.exit(1)
+
+if __name__ == '__main__':
+ import sys
+ main(sys.argv[1:])
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
