[gjs/iwyu: 30/34] build: Postprocess IWYU output

[Philip Chimento <pchimento src gnome org>]

commit c5f24a7cf4a2f2d23d519d2651aa68488f6899e1
Author: Philip Chimento <philip chimento gmail com>
Date:   Fri May 15 23:26:39 2020 -0700

    build: Postprocess IWYU output
    
    When running IWYU, the output is really hard to read. Ideally we want
    only a diff of which lines to add or remove for each file, and no output
    for a file that needs no changes.
    
    Also take this opportunity to implement the "fwd-decls" header
    js/TypeDecls.h in the postprocessor so that we don't get requests to
    forward-declare JSContext everywhere; and make the postprocessor
    suppress false positives.

 tools/process_iwyu.py | 145 ++++++++++++++++++++++++++++++++++++++++++++++++++
 tools/run_iwyu.sh     |  17 +++---
 2 files changed, 154 insertions(+), 8 deletions(-)
---
diff --git a/tools/process_iwyu.py b/tools/process_iwyu.py
new file mode 100755
index 00000000..f91778a2
--- /dev/null
+++ b/tools/process_iwyu.py
@@ -0,0 +1,145 @@
+# coding: utf8
+
+# IWYU is missing the feature to designate a certain header as a "forward-decls
+# header". In the case of SpiderMonkey, there are certain commonly used forward
+# declarations that are all gathered in js/TypeDecls.h.
+# We postprocess the IWYU output to fix this, and also fix the output format
+# which is quite verbose, making it tedious to scroll through for 60 files.
+
+import re
+import sys
+
+
+class Colors:
+    NORMAL = '\33[0m'
+    RED = '\33[31m'
+    GREEN = '\33[32m'
+
+
+ADD, REMOVE, FULL = range(3)
+state = None
+file = None
+add = {}
+remove = {}
+all_includes = {}
+
+# When encountering one of these lines, move to a different state
+MATCHERS = {
+    r'\.\./(.*) should add these lines:': ADD,
+    r'\.\./(.*) should remove these lines:': REMOVE,
+    r'The full include-list for \.\./(.*):': FULL,
+    r'\(\.\./(.*) has correct #includes/fwd-decls\)': None
+}
+
+FWD_HEADER = '#include <js/TypeDecls.h>'
+FWD_DECLS_IN_HEADER = (
+    'class JSAtom;',
+    'struct JSContext;',
+    'class JSFunction;',
+    'class JSObject;',
+    'struct JSRuntime;',
+    'class JSScript;',
+    'class JSString;',
+    'class JSTracer;',
+    'struct JSFreeOp;',
+    'namespace js { class TempAllocPolicy; }'
+    'namespace JS { struct PropertyKey; }',
+    'namespace JS { class Symbol; }',
+    'namespace JS { class BigInt; }',
+    'namespace JS { union Value; }',
+    'namespace JS { class Compartment; }',
+    'namespace JS { class Realm; }',
+    'namespace JS { struct Runtime; }',
+    'namespace JS { class Zone; }',
+)
+add_fwd_header = False
+
+FALSE_POSITIVES = (
+    # The bodies of these structs already come before their usage,
+    # we don't need to have forward declarations of them as well
+    ('gjs/atoms.h', 'class GjsAtoms;', ''),
+    ('gjs/atoms.h', 'struct GjsSymbolAtom;', ''),
+
+    # IWYU weird false positive when using std::vector::emplace_back() or
+    # std::vector::push_back()
+    ('gi/private.cpp', '#include <algorithm>', 'for max'),
+    ('gjs/importer.cpp', '#include <algorithm>', 'for max'),
+    ('modules/cairo-context.cpp', '#include <algorithm>', 'for max'),
+)
+
+
+def output():
+    global file, state, add_fwd_header
+
+    if add_fwd_header:
+        if FWD_HEADER not in all_includes:
+            if FWD_HEADER in remove:
+                remove.pop(FWD_HEADER, None)
+            else:
+                add[FWD_HEADER] = ''
+
+    if add or remove:
+        print(f'\n== {file} ==')
+        for line, why in add.items():
+            if why:
+                why = '  // ' + why
+            print(f'{Colors.GREEN}+{line}{Colors.NORMAL}{why}')
+        for line, why in remove.items():
+            if why:
+                why = '  // ' + why
+            print(f'{Colors.RED}-{line}{Colors.NORMAL}{why}')
+
+    state = None
+    file = None
+    add.clear()
+    remove.clear()
+    all_includes.clear()
+    add_fwd_header = False
+
+
+for line in sys.stdin:
+    line = line.strip()
+    if not line:
+        continue
+
+    # filter out errors having to do with compiler arguments unknown to IWYU
+    if line.startswith('error:'):
+        continue
+
+    if line == '---':
+        output()
+        continue
+
+    state_changed = False
+    file_changed = False
+    for matcher, newstate in MATCHERS.items():
+        match = re.match(matcher, line)
+        if match:
+            state = newstate
+            if match.group(1) != file:
+                if file is not None:
+                    file_changed = True
+                file = match.group(1)
+            state_changed = True
+            break
+    if file_changed:
+        output()
+        continue
+    if state_changed:
+        continue
+
+    line, _, why = line.partition(' // ')
+    line = line.strip()
+    if state == ADD:
+        if line in FWD_DECLS_IN_HEADER:
+            add_fwd_header = True
+            continue
+        if (file, line, why) in FALSE_POSITIVES:
+            continue
+        add[line] = why
+    elif state == REMOVE:
+        if line.startswith('- '):
+            line = line[2:]
+        remove[line] = why
+    elif state == FULL:
+        all_includes[line] = why
diff --git a/tools/run_iwyu.sh b/tools/run_iwyu.sh
index 57b7bc5e..ff3b9996 100755
--- a/tools/run_iwyu.sh
+++ b/tools/run_iwyu.sh
@@ -8,6 +8,7 @@ ninja
 IWYU="iwyu_tool -p ."
 PRIVATE_MAPPING="-Xiwyu --mapping_file=$SRCDIR/tools/gjs-private-iwyu.imp -Xiwyu --keep=config.h"
 PUBLIC_MAPPING="-Xiwyu --mapping_file=$SRCDIR/tools/gjs-public-iwyu.imp"
+POSTPROCESS="python3 $SRCDIR/tools/process_iwyu.py"
 
 for FILE in $SRCDIR/gi/*.cpp $SRCDIR/gjs/atoms.cpp $SRCDIR/gjs/byteArray.cpp \
     $SRCDIR/gjs/coverage.cpp $SRCDIR/gjs/debugger.cpp \
@@ -19,24 +20,24 @@ for FILE in $SRCDIR/gi/*.cpp $SRCDIR/gjs/atoms.cpp $SRCDIR/gjs/byteArray.cpp \
     $SRCDIR/modules/system.cpp $SRCDIR/test/*.cpp $SRCDIR/util/*.cpp \
     $SRCDIR/libgjs-private/*.c
 do
-    $IWYU $FILE -- $PRIVATE_MAPPING
+    $IWYU $FILE -- $PRIVATE_MAPPING | $POSTPROCESS
 done
 $IWYU $SRCDIR/gjs/context.cpp -- $PRIVATE_MAPPING \
-    -Xiwyu --check_also=$SRCDIR/gjs/context-private.h
+    -Xiwyu --check_also=$SRCDIR/gjs/context-private.h | $POSTPROCESS
 $IWYU $SRCDIR/gjs/jsapi-dynamic-class.cpp -- $PRIVATE_MAPPING \
-    -Xiwyu --check_also=$SRCDIR/gjs/jsapi-class.h
+    -Xiwyu --check_also=$SRCDIR/gjs/jsapi-class.h | $POSTPROCESS
 $IWYU $SRCDIR/gjs/jsapi-util.cpp -- $PRIVATE_MAPPING \
     -Xiwyu --check_also=$SRCDIR/gjs/jsapi-util-args.h \
-    -Xiwyu --check_also=$SRCDIR/gjs/jsapi-util-root.h
+    -Xiwyu --check_also=$SRCDIR/gjs/jsapi-util-root.h | $POSTPROCESS
 $IWYU $SRCDIR/gjs/mem.cpp -- $PRIVATE_MAPPING \
-    -Xiwyu --check_also=$SRCDIR/gjs/mem-private.h
+    -Xiwyu --check_also=$SRCDIR/gjs/mem-private.h | $POSTPROCESS
 $IWYU $SRCDIR/gjs/profiler.cpp -- $PRIVATE_MAPPING \
-    -Xiwyu --check_also=$SRCDIR/gjs/profiler-private.h
+    -Xiwyu --check_also=$SRCDIR/gjs/profiler-private.h | $POSTPROCESS
 $IWYU $SRCDIR/modules/cairo.cpp -- $PRIVATE_MAPPING \
     -Xiwyu --check_also=$SRCDIR/modules/cairo-module.h \
-    -Xiwyu --check_also=$SRCDIR/modules/cairo-private.h
+    -Xiwyu --check_also=$SRCDIR/modules/cairo-private.h | $POSTPROCESS
 
 for FILE in $SRCDIR/gjs/console.cpp $SRCDIR/installed-tests/minijasmine.cpp
 do
-    $IWYU $FILE -- $PUBLIC_MAPPING
+    $IWYU $FILE -- $PUBLIC_MAPPING | $POSTPROCESS
 done