[gimp-web] content, tools: new mirrors.json meant to replace mirrors2.json.

[Jehan <jehanp src gnome org>]

commit ee16b0892c8d5fa263228c69dd7dc2372c238e06
Author: Jehan <jehan girinstud io>
Date:   Fri Aug 27 23:46:22 2021 +0200

    content, tools: new mirrors.json meant to replace mirrors2.json.
    
    This file is manually-curated as it needs to contain humanly-displayed
    text, such as the names of the entities which sponsor mirrors, and
    possibly an info URL, as well as their official location.
    It will eventually replace mirrors2.json when we will have moved the
    mirror list to a dedicated page with nicer render.
    See also #236.
    
    This will allow to display much nicer "sponsor" page, with names of
    entities (university and others sponsoring mirrors) rather than a list
    of URLs.
    Also let's not list all possible URLs ever, but at least the official
    one (which will be https). In particular, we don't want to list unsafe
    protocol URL (http, ftp…) anymore.
    
    Nevertheless we will want to make sure this manually-curated list still
    match with the real mirrors configured in GNOME servers (i.e. from
    entities which made the process to get official rsync credentials).
    This is what the new script `cmp-mirrors.py` is for. Also
    `update-mirrors.py` has been improved to run this new script after
    updating the mirror list.
    
    Why I still made it as a separate script is that we might want to run
    `cmp-mirrors.py` as a git-hook too (at least user side, possibly even
    server side), in order to validate our mirrors.json and make sure it is
    updated as soon as needed.

 content/downloads/mirrors.json    | 121 ++++++++++++++++++++++++++++++++++++++
 tools/downloads/cmp-mirrors.py    |  70 ++++++++++++++++++++++
 tools/downloads/update-mirrors.py |  16 ++++-
 3 files changed, 206 insertions(+), 1 deletion(-)
---
diff --git a/content/downloads/mirrors.json b/content/downloads/mirrors.json
new file mode 100644
index 00000000..66afbc56
--- /dev/null
+++ b/content/downloads/mirrors.json
@@ -0,0 +1,121 @@
+{
+    "Artfiles New Media GmbH": {
+      "location": "Hamburg, Germany",
+      "mirrors": [ "https://artfiles.org/gimp.org/pub/gimp/"; ],
+      "about": "https://artfiles.org/";,
+      "more": [
+        "https://bugzilla.gnome.org/show_bug.cgi?id=764576";
+      ]
+    },
+    "Academic Computer Club, Umeå University": {
+      "location": "Sweden",
+      "mirrors": [ "https://ftp.acc.umu.se/pub/gimp/gimp/"; ],
+      "about": "https://ftp.acc.umu.se/";,
+      "more": [
+        "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/46";
+      ]
+    },
+    "University of Crete": {
+      "location": "Greece",
+      "mirrors": [ "https://ftp.cc.uoc.gr/mirrors/gimp/"; ],
+      "about": "https://ftp.cc.uoc.gr/";,
+      "more": [ ]
+    },
+    "Friedrich-Alexander-Universität Erlangen-Nürnberg (FAU)": {
+      "location": "Erlangen, Germany",
+      "mirrors": [ "https://ftp.fau.de/gimp/gimp/"; ],
+      "about": "https://ftp.fau.de/";,
+      "more": [ ]
+    },
+    "Göttingen University and Max Planck Society": {
+      "location": "Göttingen, Germany",
+      "mirrors": [ "https://ftp.gwdg.de/pub/misc/grafik/gimp/"; ],
+      "about": "https://ftp.gwdg.de";,
+      "more": [ ]
+    },
+    "Interdisciplinary Centre for Mathematical and Computational Modelling UW, University of Warsaw": {
+      "location": "Warsaw, Poland",
+      "mirrors": [ "https://ftp.icm.edu.pl/pub/graphics/gimp/"; ],
+      "about": "https://ftp.icm.edu.pl/";,
+      "more": [
+        "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/45";
+      ]
+    },
+    "Lysator ACS": {
+      "location": "Sweden",
+      "mirrors": [ "https://ftp.lysator.liu.se/pub/gimp/"; ],
+      "about": "https://www.lysator.liu.se/";,
+      "more": [ "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/125"; ]
+    },
+    "Studenten Net Twente (SNT), the University of Twente": {
+      "location": "Netherlands",
+      "mirrors": [ "https://ftp.snt.utwente.nl/pub/software/gimp/gimp/"; ],
+      "about": "https://ftp.snt.utwente.nl/";,
+      "more": [ ]
+    },
+    "IP-Connect": {
+      "location": "Vinnytsia, Ukraine",
+      "mirrors": [ "https://gimp.ip-connect.info/gimp/"; ],
+      "about": "https://ip-connect.info/";,
+      "more": [ "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/152"; ]
+    },
+    "Jaleco": {
+      "location": "Washington D.C., USA",
+      "mirrors": [ "https://mirror.jaleco.com/gimp/gimp/"; ],
+      "about": "https://mirror.jaleco.com";,
+      "more": [ "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/63"; ]
+    },
+    "Klaus-Uwe Mitterer": {
+      "location": "Vienna, Austria",
+      "mirrors": [ "https://mirror.klaus-uwe.me/gimp/gimp/"; ],
+      "about": "https://mirror.klaus-uwe.me/";,
+      "more": [ "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/31";,
+                "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/79"; ]
+    },
+    "Aalborg University": {
+      "location": "Aalborg, Denmark",
+      "mirrors": [ "https://mirrors.dotsrc.org/gimp/gimp/"; ],
+      "about": "https://dotsrc.org/";,
+      "more": [ "https://gitlab.gnome.org/Infrastructure/gimp-web/-/issues/68"; ]
+    },
+    "University of Maryland, College Park": {
+      "location": "College Park, MD, USA",
+      "mirrors": [ "https://mirror.umd.edu/gimp/gimp/"; ],
+      "about": "https://mirror.umd.edu/";,
+      "more": [
+        "https://gitlab.gnome.org/Infrastructure/Infrastructure/-/issues/617";
+      ]
+    },
+    "CSC - IT Center for Science / FUNET": {
+      "location": "Manchester, UK, Europe",
+      "mirrors": [ "https://www.nic.funet.fi/pub/mirrors/ftp.gimp.org/"; ],
+      "about": "https://www.nic.funet.fi/README";,
+      "more": [
+        "https://gitlab.gnome.org/Infrastructure/Infrastructure/-/issues/618";
+      ]
+    },
+    "Korea FreeBSD Users Group": {
+      "location": "Seoul, South Korea, Asia",
+      "mirrors": [ "https://ftp.kr.freebsd.org/pub/gimp/pub/gimp/"; ],
+      "about": "https://www.kr.freebsd.org/mirrors/";,
+      "more": [
+        "https://gitlab.gnome.org/Infrastructure/Infrastructure/-/issues/620";
+      ]
+    },
+    "XMission": {
+      "location": "Salt Lake City, Utah, United States, North America",
+      "mirrors": [ "https://mirrors.xmission.com/gimp/gimp/"; ],
+      "about": "https://mirrors.xmission.com/";,
+      "more": [
+        "https://gitlab.gnome.org/Infrastructure/Infrastructure/-/issues/624";
+      ]
+    },
+    "UKFast": {
+      "location": "Manchester, UK, Europe",
+      "mirrors": [ "https://mirrors.ukfast.co.uk/sites/gimp.org/pub/gimp/"; ],
+      "about": "https://mirrors.ukfast.co.uk/";,
+      "more": [
+        "https://gitlab.gnome.org/Infrastructure/Infrastructure/-/issues/646";
+      ]
+    }
+}
diff --git a/tools/downloads/cmp-mirrors.py b/tools/downloads/cmp-mirrors.py
new file mode 100755
index 00000000..4d5f19a6
--- /dev/null
+++ b/tools/downloads/cmp-mirrors.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import sys
+
+def cmp_mirror_files(mirrorsfile, mirrorsjson):
+  '''
+  mirrorsfile is a file containing each mirrors as a https URL, one per
+  line. This file is usually updated automatically from remote webserver
+  configuration files through tools/downloads/update-mirrors.py
+
+  mirrorsjson is a manually curated json file listing all the mirrors
+  with their official entity names, an info URL and the location
+  information.
+
+  This function simply compare the files to make sure that every mirror
+  from one is in the other, no more, no less.
+  '''
+  with open(mirrorsjson, 'r') as f:
+    mirrors = json.load(f)
+
+  missing_in_json = []
+
+  with open(mirrorsfile, 'r') as f:
+    for line in f:
+      line = line.strip()
+      if line == 'https://download.gimp.org/pub/gimp/':
+        # Not a mirror, the source server.
+        continue
+      for name in mirrors:
+        if line in mirrors[name]['mirrors']:
+          del mirrors[name]
+          break
+      else:
+        missing_in_json += [line]
+
+  if len(missing_in_json) > 0:
+    sys.stderr.write('The following {} mirrors are not listed in {}:\n'.format(len(missing_in_json),
+                                                                               mirrorsjson))
+    for missing in missing_in_json:
+      sys.stderr.write('\t{}\n'.format(missing))
+    sys.stderr.write('\n')
+
+  if len(mirrors) > 0:
+    sys.stderr.write('The following {} mirrors should be removed from {}:\n'.format(len(mirrors),
+                                                                                    mirrorsjson))
+    for name in mirrors:
+      sys.stderr.write('\t{}\n'.format(name))
+
+  if len(missing_in_json) > 0 or len(mirrors) > 0:
+    return False
+  else:
+    return True
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description='Update the list of mirrors as per download.gimp.org 
configuration.')
+  parser.add_argument('--mirrorsfile', metavar='<file>', default=os.path.dirname(__file__) + 
'/downloads.http.txt',
+                      help='A file with one download mirror per line.')
+  parser.add_argument('--mirrorsjson', metavar='<file>',
+                      default=os.path.dirname(__file__) + '/../../content/downloads/mirrors.json',
+                      help='Json file with additional data per mirror (manually curated).')
+
+  args = parser.parse_args()
+
+  if cmp_mirror_files(args.mirrorsfile, args.mirrorsjson):
+    sys.exit(os.EX_OK)
+  else:
+    sys.exit(os.EX_DATAERR)
diff --git a/tools/downloads/update-mirrors.py b/tools/downloads/update-mirrors.py
index a9713d04..77a166dd 100755
--- a/tools/downloads/update-mirrors.py
+++ b/tools/downloads/update-mirrors.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import importlib
 import os
 import paramiko
 import sys
@@ -34,4 +35,17 @@ with open(args.mirrorsfile, 'wb') as f:
   f.write(mirrors)
   f.write(b'https://download.gimp.org/pub/gimp/\n')
 
-print("Mirrors updated in {}. Please double-check before committing the change.".format(args.mirrorsfile))
+print("Mirrors updated in {}.".format(args.mirrorsfile))
+
+mirrorsjson = os.path.dirname(__file__) + '/../../content/downloads/mirrors.json'
+print("Now comparing to {} ...".format(mirrorsjson))
+
+cmp = importlib.import_module("cmp-mirrors")
+same = cmp.cmp_mirror_files(args.mirrorsfile, mirrorsjson)
+
+if same:
+  print("Both files look to match. Please double-check before committing the change.")
+  sys.exit(os.EX_OK)
+else:
+  # cmp.cmp_mirror_files() will already have outputted the errors.
+  sys.exit(os.EX_DATAERR)