[gobject-introspection: 1/2] sourcescanner: collect error messages and expose them

[Christoph Reiter <creiter src gnome org>]

commit f9a873a0881e189a2d20e4904d40bb0c5cbf0d94
Author: Christoph Reiter <reiter christoph gmail com>
Date:   Sat Dec 8 18:26:47 2018 +0100

    sourcescanner: collect error messages and expose them
    
    It just printed errors to stderr and always returns success even if parsing
    fails. This prevents us to write any tests for it.
    
    As a first step collect all lexing/parsing error messages and print them to stderr after
    the scanner is done. This allows us to add some regression tests for !78.
    
    In the future we probably want to raise an exception with those errors if parsing
    fails.

 giscanner/giscannermodule.c         | 23 +++++++++++++-
 giscanner/scannerlexer.l            |  3 +-
 giscanner/scannermain.py            |  8 +++--
 giscanner/scannerparser.y           | 11 ++++---
 giscanner/sourcescanner.c           | 14 +++++++++
 giscanner/sourcescanner.h           |  2 ++
 giscanner/sourcescanner.py          |  3 ++
 tests/scanner/test_sourcescanner.py | 61 +++++++++++++++++++++++++------------
 8 files changed, 98 insertions(+), 27 deletions(-)
---
diff --git a/giscanner/giscannermodule.c b/giscanner/giscannermodule.c
index 45701d17..24b84050 100644
--- a/giscanner/giscannermodule.c
+++ b/giscanner/giscannermodule.c
@@ -81,7 +81,7 @@ typedef struct {
 
 NEW_CLASS (PyGISourceSymbol, "SourceSymbol", GISourceSymbol, 10);
 NEW_CLASS (PyGISourceType, "SourceType", GISourceType, 9);
-NEW_CLASS (PyGISourceScanner, "SourceScanner", GISourceScanner, 8);
+NEW_CLASS (PyGISourceScanner, "SourceScanner", GISourceScanner, 9);
 
 
 /* Symbol */
@@ -507,6 +507,26 @@ pygi_source_scanner_get_symbols (PyGISourceScanner *self, G_GNUC_UNUSED PyObject
   return list;
 }
 
+static PyObject *
+pygi_source_scanner_get_errors (PyGISourceScanner *self, G_GNUC_UNUSED PyObject *unused)
+{
+  GSList *l, *errors;
+  PyObject *list;
+  int i = 0;
+
+  errors = gi_source_scanner_get_errors (self->scanner);
+  list = PyList_New (g_slist_length (errors));
+
+  for (l = errors; l; l = l->next)
+    {
+      PyObject *item = PyUnicode_FromString (l->data);
+      PyList_SetItem (list, i++, item);
+    }
+
+  g_slist_free (errors);
+  return list;
+}
+
 static PyObject *
 pygi_source_scanner_get_comments (PyGISourceScanner *self, G_GNUC_UNUSED PyObject *unused)
 {
@@ -563,6 +583,7 @@ pygi_source_scanner_get_comments (PyGISourceScanner *self, G_GNUC_UNUSED PyObjec
 }
 
 static const PyMethodDef _PyGISourceScanner_methods[] = {
+  { "get_errors", (PyCFunction) pygi_source_scanner_get_errors, METH_NOARGS },
   { "get_comments", (PyCFunction) pygi_source_scanner_get_comments, METH_NOARGS },
   { "get_symbols", (PyCFunction) pygi_source_scanner_get_symbols, METH_NOARGS },
   { "append_filename", (PyCFunction) pygi_source_scanner_append_filename, METH_VARARGS },
diff --git a/giscanner/scannerlexer.l b/giscanner/scannerlexer.l
index 971b3a73..c2fb7234 100644
--- a/giscanner/scannerlexer.l
+++ b/giscanner/scannerlexer.l
@@ -513,7 +513,8 @@ print_error (GISourceScanner *scanner)
 {
   if (yytext[0]) {
     char *filename = g_file_get_parse_name (scanner->current_file);
-    fprintf(stderr, "%s:%d: unexpected character `%c'\n", filename, lineno, yytext[0]);
+    gchar *error = g_strdup_printf ("%s:%d: unexpected character `%c'", filename, lineno, yytext[0]);
+    scanner->errors = g_slist_prepend (scanner->errors, error);
     g_free (filename);
   }
 }
diff --git a/giscanner/scannermain.py b/giscanner/scannermain.py
index ceca66f4..c2c1ac3d 100644
--- a/giscanner/scannermain.py
+++ b/giscanner/scannermain.py
@@ -444,8 +444,12 @@ def create_source_scanner(options, args):
                        options.cpp_defines,
                        options.cpp_undefines,
                        cflags=options.cflags)
-    ss.parse_files(filenames)
-    ss.parse_macros(filenames)
+    try:
+        ss.parse_files(filenames)
+        ss.parse_macros(filenames)
+    finally:
+        for error in ss.get_errors():
+            print(error, file=sys.stderr)
     return ss, filenames
 
 
diff --git a/giscanner/scannerparser.y b/giscanner/scannerparser.y
index bf7bb37f..72c17ec3 100644
--- a/giscanner/scannerparser.y
+++ b/giscanner/scannerparser.y
@@ -165,7 +165,8 @@ pop_conditional (GISourceScanner *scanner)
   if (type == 0)
     {
       gchar *filename = g_file_get_path (scanner->current_file);
-      fprintf (stderr, "%s:%d: mismatched %s", filename, lineno, yytext);
+      gchar *error = g_strdup_printf ("%s:%d: mismatched %s", filename, lineno, yytext);
+      scanner->errors = g_slist_prepend (scanner->errors, error);
       g_free (filename);
     }
 
@@ -180,8 +181,9 @@ warn_if_cond_has_gi_scanner (GISourceScanner *scanner,
   if (strstr (text, "__GI_SCANNER__"))
     {
       gchar *filename = g_file_get_path (scanner->current_file);
-      fprintf (stderr, "%s:%d: the __GI_SCANNER__ constant should only be used with simple #ifdef or #endif: 
%s",
+      gchar *error = g_strdup_printf ("%s:%d: the __GI_SCANNER__ constant should only be used with simple 
#ifdef or #endif: %s",
                filename, lineno, text);
+      scanner->errors = g_slist_prepend (scanner->errors, error);
       g_free (filename);
     }
 }
@@ -1567,8 +1569,9 @@ yyerror (GISourceScanner *scanner, const char *s)
    * have valid expressions */
   if (!scanner->macro_scan)
     {
-      fprintf(stderr, "%s:%d: %s in '%s' at '%s'\n",
-             g_file_get_parse_name (scanner->current_file), lineno, s, linebuf, yytext);
+      gchar *error = g_strdup_printf ("%s:%d: %s in '%s' at '%s'",
+          g_file_get_parse_name (scanner->current_file), lineno, s, linebuf, yytext);
+      scanner->errors = g_slist_prepend (scanner->errors, error);
     }
 }
 
diff --git a/giscanner/sourcescanner.c b/giscanner/sourcescanner.c
index 464e4695..4d10c88b 100644
--- a/giscanner/sourcescanner.c
+++ b/giscanner/sourcescanner.c
@@ -246,6 +246,7 @@ gi_source_scanner_free (GISourceScanner *scanner)
   g_slist_free (scanner->comments);
   g_slist_foreach (scanner->symbols, (GFunc)(void *)gi_source_symbol_unref, NULL);
   g_slist_free (scanner->symbols);
+  g_slist_free_full (scanner->errors, g_free);
 
   g_hash_table_unref (scanner->files);
 
@@ -325,6 +326,19 @@ gi_source_scanner_get_symbols (GISourceScanner  *scanner)
   return g_slist_reverse (g_slist_copy (scanner->symbols));
 }
 
+/**
+ * gi_source_scanner_get_errors:
+ * @scanner: scanner instance
+ *
+ * Returns: (transfer container): List of strings.
+ *   Free resulting list with g_slist_free().
+ */
+GSList *
+gi_source_scanner_get_errors (GISourceScanner  *scanner)
+{
+  return g_slist_reverse (g_slist_copy (scanner->errors));
+}
+
 /**
  * gi_source_scanner_get_comments:
  * @scanner: scanner instance
diff --git a/giscanner/sourcescanner.h b/giscanner/sourcescanner.h
index e9fa5421..bcf1afc4 100644
--- a/giscanner/sourcescanner.h
+++ b/giscanner/sourcescanner.h
@@ -118,6 +118,7 @@ struct _GISourceScanner
   GHashTable *const_table;
   gboolean skipping;
   GQueue conditionals;
+  GSList *errors;
 };
 
 struct _GISourceSymbol
@@ -162,6 +163,7 @@ void                gi_source_scanner_set_macro_scan   (GISourceScanner  *scanne
                                                        gboolean          macro_scan);
 GSList *            gi_source_scanner_get_symbols      (GISourceScanner  *scanner);
 GSList *            gi_source_scanner_get_comments     (GISourceScanner  *scanner);
+GSList *            gi_source_scanner_get_errors       (GISourceScanner  *scanner);
 void                gi_source_scanner_free             (GISourceScanner  *scanner);
 
 GISourceSymbol *    gi_source_symbol_new               (GISourceSymbolType  type, GFile *file, int line);
diff --git a/giscanner/sourcescanner.py b/giscanner/sourcescanner.py
index d867a4e9..6a0bace7 100644
--- a/giscanner/sourcescanner.py
+++ b/giscanner/sourcescanner.py
@@ -274,6 +274,9 @@ class SourceScanner(object):
     def get_comments(self):
         return self._scanner.get_comments()
 
+    def get_errors(self):
+        return self._scanner.get_errors()
+
     def dump(self):
         print('-' * 30)
         for symbol in self._scanner.get_symbols():
diff --git a/tests/scanner/test_sourcescanner.py b/tests/scanner/test_sourcescanner.py
index 831af486..624d9df9 100644
--- a/tests/scanner/test_sourcescanner.py
+++ b/tests/scanner/test_sourcescanner.py
@@ -10,7 +10,20 @@ import os
 from giscanner.sourcescanner import SourceScanner
 
 
-two_typedefs_source = """
+class Test(unittest.TestCase):
+
+    def _parse_files(self, code, header=True):
+        scanner = SourceScanner()
+        tmp_fd, tmp_name = tempfile.mkstemp(suffix=".h" if header else ".c")
+        fileobj = os.fdopen(tmp_fd, 'wb')
+        with fileobj:
+            fileobj.write(code.encode("utf-8"))
+        scanner.parse_files([tmp_name])
+        os.unlink(tmp_name)
+        return scanner
+
+    def test_length_consistency(self):
+        scanner = self._parse_files("""
 /**
  * Spam:
  */
@@ -20,26 +33,36 @@ typedef struct _spam Spam;
  * Eggs:
  */
 typedef struct _eggs Eggs;
-"""
+""")
 
+        self.assertEqual(len(list(scanner.get_symbols())), 2)
+        self.assertEqual(len(list(scanner.get_symbols())), 2)
+        self.assertEqual(len(list(scanner.get_comments())), 2)
+        self.assertEqual(len(list(scanner.get_comments())), 2)
+        self.assertFalse(scanner.get_errors())
 
-class Test(unittest.TestCase):
-    def setUp(self):
-        self.ss = SourceScanner()
-        tmp_fd, tmp_name = tempfile.mkstemp()
-        file = os.fdopen(tmp_fd, 'wt')
-        file.write(two_typedefs_source)
-        file.close()
-
-        self.ss.parse_files([tmp_name])
-
-    def test_get_symbols_length_consistency(self):
-        self.assertEqual(len(list(self.ss.get_symbols())), 2)
-        self.assertEqual(len(list(self.ss.get_symbols())), 2)
-
-    def test_get_comments_length_consistency(self):
-        self.assertEqual(len(list(self.ss.get_comments())), 2)
-        self.assertEqual(len(list(self.ss.get_comments())), 2)
+    def test_parser_error(self):
+        scanner = self._parse_files("""
+void foo() {
+    a =
+}""")
+
+        errors = scanner.get_errors()
+        self.assertEqual(len(errors), 1)
+        self.assertTrue("syntax error" in errors[0])
+
+    def test_ignore_typeof(self):
+        # https://gitlab.gnome.org/GNOME/gobject-introspection/merge_requests/71
+        scanner = self._parse_files("""
+/**
+* foo:
+*/
+void foo(int bar) {
+    bar = ((__typeof__(bar)) (foo) (bar));
+}
+""")
+        self.assertEqual(len(list(scanner.get_comments())), 1)
+        self.assertFalse(scanner.get_errors())
 
 
 if __name__ == '__main__':