[gnome-continuous-yocto/gnomeostree-3.28-rocko: 4426/8267] verify-bashisms: check scripts only once, include original file and line

[Emmanuele Bassi <ebassi src gnome org>]

commit 0b95e47180588306ad72f6ee0a6a65937a2c74b0
Author: Patrick Ohly <patrick ohly intel com>
Date:   Tue Jan 31 13:50:31 2017 +0100

    verify-bashisms: check scripts only once, include original file and line
    
    Several scripts that are defined in .bbclass files end up in multiple
    different recipes. It's better (faster, less repetitive error reports)
    to check them only once.
    
    In addition, the real information for the developer is where he can
    find the script, not which recipe file uses it. verify-bashisms now
    prints the original file instead of the recipe whenever possible
    (i.e. 'filename' is set) and also bumps the line number so that it is
    relative to the file and not the script.
    
    Example with one real error and one added just for testing:
    
      $ verify-bashisms core-image-minimal core-image-sato
      Loading cache: 100% |#################################################################################| 
Time: 0:00:00
      Loaded 2935 entries from dependency cache.
      Parsing recipes: 100% |###############################################################################| 
Time: 0:00:01
      Parsing of 2137 .bb files complete (2101 cached, 36 parsed). 2935 targets, 412 skipped, 0 masked, 0 
errors.
      Generating scripts...
      Scanning scripts...
    
      /.../openembedded-core/meta/classes/populate_sdk_ext.bbclass
       possible bashism in install_tools line 515 (should be 'b = a'):
        if [ "${SDK_INCLUDE_TOOLCHAIN}" == "1" -a ! -e $unfsd_path ] ; then
       possible bashism in install_tools line 521 (type):
                type fixme
    
    (From OE-Core rev: ca4932b60f464430266cc43e34122b2973e8a200)
    
    Signed-off-by: Patrick Ohly <patrick ohly intel com>
    Signed-off-by: Richard Purdie <richard purdie linuxfoundation org>

 scripts/verify-bashisms |   56 ++++++++++++++++++++++++++++++----------------
 1 files changed, 36 insertions(+), 20 deletions(-)
---
diff --git a/scripts/verify-bashisms b/scripts/verify-bashisms
index df071e3..7283980 100755
--- a/scripts/verify-bashisms
+++ b/scripts/verify-bashisms
@@ -22,7 +22,9 @@ def is_whitelisted(s):
             return True
     return False
 
-def process(recipe, function, script):
+SCRIPT_LINENO_RE = re.compile(r' line (\d+) ')
+
+def process(filename, function, lineno, script):
     import tempfile
 
     if not script.startswith("#!"):
@@ -45,13 +47,25 @@ def process(recipe, function, script):
             print("Unexpected output from checkbashism: %s" % str(output))
             return
 
-        # Turn the output into a list of (message, source) values
+        # Turn the output into a single string like this:
+        # /.../foobar.bb
+        #  possible bashism in updatercd_postrm line 2 (type):
+        #   if ${@use_updatercd(d)} && type update-rc.d >/dev/null 2>/dev/null; then
+        #  ...
+        #   ...
         result = []
         # Check the results against the whitelist
         for message, source in zip(output[0::2], output[1::2]):
             if not is_whitelisted(source):
-                result.append((message, source))
-        return result
+                if lineno is not None:
+                    message = SCRIPT_LINENO_RE.sub(lambda m: ' line %d ' % (int(m.group(1)) + int(lineno) - 
1),
+                                                   message)
+                result.extend([' ' + message, '  ' + source])
+        if result:
+            result.insert(0, filename)
+            return '\n'.join(result)
+        else:
+            return None
 
 def get_tinfoil():
     scripts_path = os.path.dirname(os.path.realpath(__file__))
@@ -75,12 +89,8 @@ if __name__=='__main__':
     # initializing the pool and connecting to the
     # bitbake server is crucial, don't change it.
     def func(item):
-        fn, scripts = item
-        result = []
-        for key, script in scripts:
-            r = process(fn, key, script)
-            if r: result.extend(r)
-        return fn, result
+        (filename, key, lineno), script = item
+        return process(filename, key, lineno, script)
 
     import multiprocessing
     pool = multiprocessing.Pool()
@@ -97,27 +107,33 @@ if __name__=='__main__':
     else:
         initial_pns = sorted(pkg_pn)
 
-    pns = {}
+    pns = set()
+    scripts = {}
     print("Generating scripts...")
     for pn in initial_pns:
         for fn in pkg_pn[pn]:
             # There's no point checking multiple BBCLASSEXTENDed variants of the same recipe
+            # (at least in general - there is some risk that the variants contain different scripts)
             realfn, _, _ = bb.cache.virtualfn2realfn(fn)
             if realfn not in pns:
+                pns.add(realfn)
                 data = tinfoil.parse_recipe_file(realfn)
-                scripts = []
                 for key in data.keys():
                     if data.getVarFlag(key, "func") and not data.getVarFlag(key, "python"):
                         script = data.getVar(key, False)
                         if script:
-                            scripts.append((key, script))
-                pns[realfn] = scripts
+                            filename = data.getVarFlag(key, "filename")
+                            lineno = data.getVarFlag(key, "lineno")
+                            # There's no point in checking a function multiple
+                            # times just because different recipes include it.
+                            # We identify unique scripts by file, name, and (just in case)
+                            # line number.
+                            attributes = (filename or realfn, key, lineno)
+                            scripts.setdefault(attributes, script)
+
 
     print("Scanning scripts...\n")
-    for pn, results in pool.imap(func, pns.items()):
-        if results:
-            print(pn)
-            for message,source in results:
-                print(" %s\n  %s" % (message, source))
-            print()
+    for result in pool.imap(func, scripts.items()):
+        if result:
+            print(result)
     tinfoil.shutdown()