[glom: 1/4] Refactoring Dialog_Import_CSV: first parser tests created

[Michael Hasselmann <mikhas src gnome org>]

commit 2d0286c6f48a47e243d4addaa864e481390c2103
Author: Michael Hasselmann <michaelh openismus com>
Date:   Fri Sep 11 20:14:06 2009 +0200

    Refactoring Dialog_Import_CSV: first parser tests created
    
    * Makefile_tests.am, tests/import.sh,
    tests/import/test_parsing.cc: The first tests for the CsvParser are ready.
    Basic input is tested but no real world examples yet. Run with 'make check
    TESTS=tests/import.sh' to run tests with valgrind.
    
    Renamed regression_tests folder to tests

 ChangeLog                                          |   13 ++
 Makefile_tests.am                                  |   32 ++--
 {regression_tests => tests}/.gitignore             |    0
 tests/import.sh                                    |    8 +
 tests/import/test_parsing.cc                       |  191 ++++++++++++++++++++
 .../test_load_python_library.cc                    |    0
 {regression_tests => tests}/test_parsing_time.cc   |    0
 {regression_tests => tests}/test_signal_reemit.cc  |    0
 8 files changed, 232 insertions(+), 12 deletions(-)
---
diff --git a/ChangeLog b/ChangeLog
index 9571b2d..cccbbd0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+2009-09-11  Michael Hasselmann <michaelh openismus com>
+
+	Refactoring Glom::Dialog_Import_CSV: first parser tests created
+
+	* Makefile_tests.am, tests/import.sh,
+	tests/import/test_parsing.cc: The first tests for the CsvParser
+	are ready. Basic input is tested but no real world examples yet. Run with 'make
+	check TESTS=tests/import.sh' to run tests with valgrind.
+
+	Renamed regression_tests folder to tests
+
+	* tests, Makefile_tests.am: Changed paths accordingly.
+
 2009-09-09  Michael Hasselmann <michaelh openismus com>
 
 	Refactoring Glom::Dialog_Import_CSV
diff --git a/Makefile_tests.am b/Makefile_tests.am
index 6e95bd0..f5760a2 100644
--- a/Makefile_tests.am
+++ b/Makefile_tests.am
@@ -22,13 +22,16 @@ check_PROGRAMS =						\
 	glom/utility_widgets/test_flowtable			\
 	glom/utility_widgets/egg/toolpalette/testtoolpalette	\
 	glom/test_pyembed					\
-	regression_tests/test_parsing_time			\
-	regression_tests/test_signal_reemit			\
-	regression_tests/test_load_python_library
+	tests/test_parsing_time			\
+	tests/test_signal_reemit			\
+	tests/test_load_python_library\
+	tests/import/test_parsing
 
-TESTS =	regression_tests/test_parsing_time	\
-	regression_tests/test_signal_reemit	\
-	regression_tests/test_load_python_library
+TESTS =	tests/test_parsing_time	\
+	tests/test_signal_reemit	\
+	tests/test_load_python_library\
+	tests/import/test_parsing\
+	tests/import.sh
 
 tests_ldadd = glom/libglom/libglom-$(GLOM_ABI_VERSION).la $(LIBGLOM_LIBS)
 
@@ -36,9 +39,13 @@ glom_libglom_test_connectionpool_SOURCES = glom/libglom/test_connectionpool.cc
 glom_libglom_test_document_SOURCES = glom/libglom/test_document.cc
 glom_libglom_test_sharedptr_layoutitem_SOURCES = glom/libglom/test_sharedptr_layoutitem.cc
 glom_test_pyembed_SOURCES = glom/test_pyembed.cc
-regression_tests_test_parsing_time_SOURCES = regression_tests/test_parsing_time.cc
-regression_tests_test_signal_reemit_SOURCES = regression_tests/test_signal_reemit.cc
-regression_tests_test_load_python_library_SOURCES = regression_tests/test_load_python_library.cc
+tests_test_parsing_time_SOURCES = tests/test_parsing_time.cc
+tests_test_signal_reemit_SOURCES = tests/test_signal_reemit.cc
+tests_test_load_python_library_SOURCES = tests/test_load_python_library.cc
+tests_import_test_parsing_SOURCES =	\
+	glom/import_csv.cc	\
+	glom/import_csv.h	\
+	tests/import/test_parsing.cc
 
 glom_libglom_test_connectionpool_LDADD = $(tests_ldadd)
 glom_libglom_test_document_LDADD = $(tests_ldadd)
@@ -60,6 +67,7 @@ glom_utility_widgets_egg_toolpalette_testtoolpalette_LDADD =	\
 
 glom_test_pyembed_LDADD = $(LIBGLOM_LIBS) $(PYTHON_LIBS)
 
-regression_tests_test_parsing_time_LDADD = $(tests_ldadd)
-regression_tests_test_signal_reemit_LDADD = $(LIBGLOM_LIBS)
-regression_tests_test_load_python_library_LDADD = $(LIBGLOM_LIBS)
+tests_test_parsing_time_LDADD = $(tests_ldadd)
+tests_test_signal_reemit_LDADD = $(LIBGLOM_LIBS)
+tests_test_load_python_library_LDADD = $(LIBGLOM_LIBS)
+tests_import_test_parsing_LDADD = $(LIBGLOM_LIBS)
diff --git a/regression_tests/.gitignore b/tests/.gitignore
similarity index 100%
rename from regression_tests/.gitignore
rename to tests/.gitignore
diff --git a/tests/import.sh b/tests/import.sh
new file mode 100755
index 0000000..ce6d6ac
--- /dev/null
+++ b/tests/import.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# hm, can someone please put me in the test makefile?
+
+export G_DEBUG=gc-friendly
+export G_SLICE=always-malloc
+
+which valgrind && valgrind --tool=memcheck --leak-check=full --leak-resolution=high --trace-children=yes --num-callers=30 tests/import/test_parsing
diff --git a/tests/import/test_parsing.cc b/tests/import/test_parsing.cc
new file mode 100644
index 0000000..7e784c3
--- /dev/null
+++ b/tests/import/test_parsing.cc
@@ -0,0 +1,191 @@
+#include <glom/import_csv.h>
+#include <glibmm/regex.h>
+#include <iostream>
+#include <cstdlib>
+
+namespace
+{
+
+typedef std::vector<std::string> Tokens;
+
+Tokens& get_tokens_instance()
+{
+  static Tokens tokens;
+  return tokens;
+}
+
+
+void on_line_scanned(const Glib::ustring& line, guint line_number);
+
+void print_tokens()
+{
+  for(Tokens::const_iterator iter = get_tokens_instance().begin();
+       iter != get_tokens_instance().end();
+       ++iter)
+  {
+    std::cout << " [" << *iter << "] ";
+  }
+
+  std::cout << std::endl;
+}
+
+bool check_tokens(Glib::RefPtr<Glib::Regex> check)
+{
+  for(Tokens::const_iterator iter = get_tokens_instance().begin();
+       iter != get_tokens_instance().end();
+       ++iter)
+  {
+    if(!check->match(*iter)) return false;
+  }
+
+  return true;
+}
+
+void set_parser_contents(Glom::CsvParser& parser, const char* input, guint size)
+{
+
+  parser.m_raw = std::vector<char>(input, input + size);
+}
+
+void on_line_scanned(const Glib::ustring& line, guint /*line_number*/)
+{
+  Glib::ustring field;
+  Glib::ustring::const_iterator line_iter(line.begin());
+
+  while (line_iter != line.end())
+  {
+    line_iter = Glom::CsvParser::advance_field(line_iter, line.end(), field);
+    get_tokens_instance().push_back(field);
+
+    // Manually have to skip separators.
+    if (',' == *line_iter)
+    {
+      get_tokens_instance().push_back(",");
+      ++line_iter;
+    }
+  }
+}
+
+} // namespace
+
+// Testcases
+int main()
+{
+  Glom::CsvParser parser("UTF-8");
+  parser.signal_line_scanned().connect(sigc::ptr_fun(&on_line_scanned));
+
+  bool test_dquoted_string = false;
+  bool test_skip_on_no_ending_newline = false;
+  bool test_skip_on_no_quotes_around_token = false;
+  bool test_skip_spaces_around_separators = false;
+  bool test_fail_on_non_comma_separators = false;
+  bool test_parse_newline_inside_quotes = false;
+
+  std::stringstream results;
+
+  // test_dquoted_string
+  {
+    const char raw_line[] = "\"a \"\"quoted\"\" token\",\"sans quotes\"\n";
+    set_parser_contents(parser, raw_line, sizeof(raw_line));
+
+    while(parser.on_idle_parse())
+    {}
+
+    results << "test_dquoted_string: "
+            << (test_dquoted_string = check_tokens(Glib::Regex::create("^(a \"quoted\" token|,|sans quotes)$")))
+            << std::endl;
+
+    get_tokens_instance().clear();
+    parser.clear();
+  }
+
+  // test_skip_on_no_ending_newline
+  {
+    const char raw_line[] = "\"this\",\"line\",\"will\",\"be\",\"skipped\"";
+    set_parser_contents(parser, raw_line, sizeof(raw_line));
+
+    while(parser.on_idle_parse())
+    {}
+
+    results << "test_skip_on_no_ending_newline: "
+            << (test_skip_on_no_ending_newline = (get_tokens_instance().size() == 0))
+            << std::endl;
+
+    get_tokens_instance().clear();
+    parser.clear();
+  }
+
+  // test_skip_on_no_quotes_around_token
+  {
+    const char raw_line[] = "this,line,contains,only,empty,tokens\n";
+    set_parser_contents(parser, raw_line, sizeof(raw_line));
+
+    while(parser.on_idle_parse())
+    {}
+
+    results << "test_skip_on_no_quotes_around_token: "
+            << (test_skip_on_no_quotes_around_token = check_tokens(Glib::Regex::create("^(|,)$")))
+            << std::endl;
+
+    get_tokens_instance().clear();
+    parser.clear();
+  }
+
+  // test_skip_spaces_around_separators
+  {
+    const char raw_line[] = "\"spaces\" , \"around\", \"separators\"\n";
+    set_parser_contents(parser, raw_line, sizeof(raw_line));
+
+    while(parser.on_idle_parse())
+    {}
+
+    results << "test_skip_spaces_around_separators: "
+            << (test_skip_spaces_around_separators = (get_tokens_instance().size() == 5))
+            << std::endl;
+
+    get_tokens_instance().clear();
+    parser.clear();
+  }
+
+  // test_fail_on_non_comma_separators
+  {
+    const char raw_line[] = "\"cannot\"\t\"tokenize\"\t\"this\"\n";
+    set_parser_contents(parser, raw_line, sizeof(raw_line));
+
+    while(parser.on_idle_parse())
+    {}
+
+    results << "test_fail_on_non_comma_separators: "
+            << (test_fail_on_non_comma_separators = check_tokens(Glib::Regex::create("^cannottokenizethis$")))
+            << std::endl;
+
+    get_tokens_instance().clear();
+    parser.clear();
+  }
+
+  // test_parse_newline_inside_quotes
+  {
+    const char raw_line[] = "\"cell with\nnewline\"\n\"token on next line\"";
+    set_parser_contents(parser, raw_line, sizeof(raw_line));
+
+    while(parser.on_idle_parse())
+    {}
+
+    results << "test_parse_newline_inside_quotes: "
+            << (test_parse_newline_inside_quotes = check_tokens(Glib::Regex::create("^(cell with\nnewline|token on next line)$")))
+            << std::endl;
+
+    get_tokens_instance().clear();
+    parser.clear();
+  }
+
+  std::cout << results.rdbuf();
+
+  return (test_dquoted_string &&
+          test_skip_on_no_ending_newline &&
+          test_skip_on_no_quotes_around_token &&
+          test_skip_spaces_around_separators &&
+          test_fail_on_non_comma_separators &&
+          test_parse_newline_inside_quotes) ? EXIT_SUCCESS
+                                            : EXIT_FAILURE;
+}
diff --git a/regression_tests/test_load_python_library.cc b/tests/test_load_python_library.cc
similarity index 100%
rename from regression_tests/test_load_python_library.cc
rename to tests/test_load_python_library.cc
diff --git a/regression_tests/test_parsing_time.cc b/tests/test_parsing_time.cc
similarity index 100%
rename from regression_tests/test_parsing_time.cc
rename to tests/test_parsing_time.cc
diff --git a/regression_tests/test_signal_reemit.cc b/tests/test_signal_reemit.cc
similarity index 100%
rename from regression_tests/test_signal_reemit.cc
rename to tests/test_signal_reemit.cc