GNOME.org
 

[gnome-build-meta/mcatanzaro/cve-report] Update generate-cve-report from freedesktop-sdk


commit 57f3699282d91d2c1ede42a519d346e36d8ef0ed
Author: Michael Catanzaro <mcatanzaro redhat com>
Date:   Wed Sep 28 10:26:00 2022 -0500

    Update generate-cve-report from freedesktop-sdk

 .gitlab-ci.yml               |   4 +-
 utils/generate-cve-report.py | 217 ++++++++++++++++++++++++++++++++-----------
 2 files changed, 164 insertions(+), 57 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3e410df24..fce07b7b8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -442,8 +442,8 @@ cve_report:
   - ../utils/update-local-cve-database.py
 
   - mkdir -p ../cve-reports
-  - ../utils/generate-cve-report.py ../sdk-manifest/usr/manifest.json ../cve-reports/sdk.html
-  - ../utils/generate-cve-report.py ../platform-manifest/usr/manifest.json ../cve-reports/platform.html
+  - python3 ../utils/generate-cve-report.py ../sdk-manifest/usr/manifest.json ../cve-reports/sdk.html
+  - python3 ../utils/generate-cve-report.py ../platform-manifest/usr/manifest.json 
../cve-reports/platform.html
   cache:
     key: cve
     paths:
diff --git a/utils/generate-cve-report.py b/utils/generate-cve-report.py
index 0c498ed67..1a09385d6 100755
--- a/utils/generate-cve-report.py
+++ b/utils/generate-cve-report.py
@@ -1,74 +1,181 @@
-#!/usr/bin/env python3
-"""Generate and HTML output with all current CVEs for a given manifest.
+"""
+Downloads CVE database and generate HTML output with all current CVEs for a given manifest.
 
 Usage:
   python3 generate-cve-report.py path/to/manifest.json output.html
 
-This requires you to run update-local-cve-database.py first in the
-same current directory.
+This tool will create files in the current
+directory:
+ - nvdcve-2.0-*.xml.gz: The cached raw XML databases from the CVE database.
+ - nvdcve-2.0-*.xml.gz.etag: Related etags for downloaded files
+
+Files are not downloaded if not modified. But we still verify with the
+remote database we have the latest version of the files.
 """
 
 import json
 import sys
-import sqlite3
+import gzip
+import glob
+import os
 
-conn = sqlite3.connect('data-2.db')
-c = conn.cursor()
+import requests
+import packaging.version
 
-with open(sys.argv[1], 'rb') as f:
-    manifest = json.load(f)
 
-with open(sys.argv[2], 'w') as out:
-    out.write('<!DOCTYPE html>\n')
-    out.write('<html><head><title>Report</title></head><body><table>\n')
+LOOKUP_TABLE = {}
 
-    entries = []
+with open(sys.argv[1], 'rb') as f:
+    manifest = json.load(f)
     for module in manifest["modules"]:
-        name = module["name"]
-        sources = module["sources"]
         cpe = module["x-cpe"]
-        product = cpe["product"]
         version = cpe.get("version")
-        vendor = cpe.get("vendor")
-        patches = cpe.get("patches", [])
-        ignored = cpe.get("ignored", [])
         if not version:
-            print("{} is missing a version".format(name))
             continue
+        vendor = cpe.get("vendor")
+        vendor_dict = LOOKUP_TABLE.setdefault(cpe.get("vendor"), {})
+        vendor_dict[cpe["product"]] = {
+            "name": module["name"],
+            "version": version,
+            "patches": cpe.get("patches", []),
+            "ignored": cpe.get("ignored", [])
+        }
+
+
+def extract_product_vulns_sub(node):
+    if "cpe_match" in node:
+        for cpe_match in node["cpe_match"]:
+            if cpe_match["vulnerable"]:
+                yield cpe_match
+    else:
+        for child in node.get("children", []):
+            yield from extract_product_vulns_sub(child)
+
+
+def extract_product_vulns(tree):
+    for item in tree['CVE_Items']:
+        summary = item['cve']['description']['description_data'][0]['value']
+        scorev2 = item['impact'].get('baseMetricV2', {}).get('cvssV2', {}).get('baseScore')
+        scorev3 = item['impact'].get('baseMetricV3', {}).get('cvssV3', {}).get('baseScore')
+
+        cve_id = item['cve']['CVE_data_meta']['ID']
+        for node in item['configurations']["nodes"]:
+            for cpe_match in extract_product_vulns_sub(node):
+                yield cve_id, summary, scorev2, scorev3, cpe_match
+
+api = os.environ.get("CI_API_V4_URL")
+project_id = os.environ.get("CI_PROJECT_ID")
+token = os.environ.get("GITLAB_TOKEN")
+
+def get_entries(entry_char, entry_type, cveid):
+    resp = requests.get(
+        f'{api}/projects/{project_id}/{entry_type}?search={cveid}',
+        headers={'Authorization': f'Bearer {token}'},
+        timeout=30*60,
+    )
+    if resp.ok:
+        for entry in resp.json():
+            iid = entry.get('iid')
+            yield f'{entry_char}{iid}', entry.get('web_url')
+    else:
+        print(resp.status_code, resp.text)
+
+def get_issues_and_mrs(cveid):
+    if not api or not project_id or not token:
+        return
+    for entry_name, url in get_entries('!', 'merge_requests', cveid):
+        yield entry_name, url
+    for entry_name, url in get_entries('#', 'issues', cveid):
+        yield entry_name, url
 
-        if vendor:
-            c.execute("""SELECT cve.id, cve.summary, cve.score FROM cve, product_vuln
-                             WHERE cve.id=product_vuln.cve_id
-                               AND product_vuln.name=?
-                               AND product_vuln.version=?
-                               AND product_vuln.vendor=?""",
-                      (product, version, vendor))
-        else:
-            c.execute("""SELECT cve.id, cve.summary, cve.score FROM cve, product_vuln
-                             WHERE cve.id=product_vuln.cve_id
-                               AND product_vuln.name=?
-                               AND product_vuln.version=?""",
-                      (product, version))
-        while True:
-            row = c.fetchone()
-            if row is None:
-                break
-            if row[0] in patches or row[0] in ignored:
+
+def extract_vulnerabilities(filename):
+    print(f"Processing {filename}")
+    with gzip.open(filename) as file:
+        tree = json.load(file)
+        for cve_id, summary, scorev2, scorev3, cpe_match in extract_product_vulns(tree):
+            product_name = cpe_match["cpe23Uri"]
+            vendor, name, version = product_name.split(':')[3:6]
+
+            module = LOOKUP_TABLE.get(vendor, {}).get(name)
+            if not module:
+                module = LOOKUP_TABLE.get(None, {}).get(name)
+            if not module:
                 continue
-            entries.append((row[0], name, version, row[1], row[2]))
-
-    def by_score(entry, ):
-        ID, name, version, summary, score = entry
-        try:
-            return float(score)
-        except ValueError:
-            return float("inf")
-
-    for ID, name, version, summary, score in sorted(entries, key=by_score, reverse=True):
-        out.write('<tr>')
-        out.write('<td><a href="https://nvd.nist.gov/vuln/detail/{ID}";>{ID}</a></td>'.format(ID=ID))
-        for d in [name, version, summary, score]:
-            out.write('<td>{}</td>'.format(d))
-        out.write('</tr>\n')
-
-    out.write('</table></html>\n')
+
+            if cve_id in module["patches"] or cve_id in module["ignored"]:
+                vulnerable = False
+            elif module["version"] == version:
+                vulnerable = True
+            elif version == "*":
+                version_object = packaging.version.LegacyVersion(module["version"])
+                vulnerable = True
+                if "versionStartIncluding" in cpe_match:
+                    start = packaging.version.LegacyVersion(cpe_match["versionStartIncluding"])
+                    if version_object < start:
+                        vulnerable = False
+                elif "versionStartExcluding" in cpe_match:
+                    start = packaging.version.LegacyVersion(cpe_match["versionStartExcluding"])
+                    if version_object <= start:
+                        vulnerable = False
+                if "versionEndIncluding" in cpe_match:
+                    end = packaging.version.LegacyVersion(cpe_match["versionEndIncluding"])
+                    if version_object > end:
+                        vulnerable = False
+                elif "versionEndExcluding" in cpe_match:
+                    end = packaging.version.LegacyVersion(cpe_match["versionEndExcluding"])
+                    if version_object >= end:
+                        vulnerable = False
+            else:
+                vulnerable = False
+
+            yield cve_id, module["name"], module["version"], summary, scorev2, scorev3, vulnerable
+
+
+def maybe_score(item):
+    try:
+        return float(item)
+    except (ValueError, TypeError):
+        return -1
+
+
+def by_score(entry):
+    scorev2 = maybe_score(entry[4])
+    scorev3 = maybe_score(entry[5])
+    return scorev3, scorev2
+
+def format_score(score):
+    if score is None:
+        return ""
+    return score
+
+
+if __name__ == "__main__":
+    vuln_map = {}
+    for filename in sorted(glob.glob("nvdcve-1.1-*.json.gz")):
+        for cve_id, name, version, summary, scorev2, scorev3, vulnerable in 
extract_vulnerabilities(filename):
+            if not vulnerable:
+                try:
+                    del vuln_map[cve_id]
+                except KeyError:
+                    pass
+            else:
+                vuln_map[cve_id] = cve_id, name, version, summary, scorev2, scorev3
+
+    entries = list(vuln_map.values())
+
+    entries.sort(key=by_score, reverse=True)
+
+    with open(sys.argv[2], 'w', encoding="utf-8") as out:
+        out.write("|Vulnerability|Element|Version|Summary|CVSS V3.x|CVSS V2.0|WIP|\n")
+        out.write("|---|---|---|---|---|---|---|\n")
+
+        for ID, name, version, summary, scorev2, scorev3 in entries:
+            issues_mrs = ", ".join(f"[{id}]({link})" for id, link in get_issues_and_mrs(ID)) or "None"
+            
out.write(f"|[{ID}](https://nvd.nist.gov/vuln/detail/{ID})|{name}|{version}|{summary}|{format_score(scorev3)}|{format_score(scorev2)}|{issues_mrs}|\n")
+
+        out.write('<!-- Markdeep: -->'
+                  '<style 
class="fallback">body{visibility:hidden;white-space:pre;font-family:monospace}</style>'
+                  '<script src="markdeep.min.js" charset="utf-8"></script>'
+                  '<script src="https://morgan3d.github.io/markdeep/latest/markdeep.min.js"; 
charset="utf-8"></script>'
+                  
'<script>window.alreadyProcessedMarkdeep||(document.body.style.visibility="visible")</script>')
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]