[releng/abderrahim/downloadsites: 2/3] tarball-conversion: add a specific implementation for download.gnome.org

[Abderrahim Kitouni <akitouni src gnome org>]

commit 8649be76657a263f187c5884de42014021dc97b2
Author: Abderrahim Kitouni <akitouni gnome org>
Date:   Thu Apr 30 11:04:36 2020 +0100

    tarball-conversion: add a specific implementation for download.gnome.org

 tools/smoketesting/downloadsites.py               | 48 +++++++++++++++++++++++
 tools/smoketesting/tarball-conversion-3-34.config |  2 +-
 tools/smoketesting/tarball-conversion-3-36.config |  2 +-
 tools/smoketesting/tarball-conversion.config      |  2 +-
 4 files changed, 51 insertions(+), 3 deletions(-)
---
diff --git a/tools/smoketesting/downloadsites.py b/tools/smoketesting/downloadsites.py
index 68309d7..52cb0b8 100644
--- a/tools/smoketesting/downloadsites.py
+++ b/tools/smoketesting/downloadsites.py
@@ -89,9 +89,57 @@ class Tarballs(DownloadSite):
 
         return location, version, None
 
+class GNOME(DownloadSite):
+    def __init__(self, baseurl):
+        super().__init__(baseurl)
+
+        resp = requests.get(self.baseurl)
+        resp.raise_for_status()
+
+        moduleregex = re.compile('([^/]+)/')
+
+        for link in sorted(get_links(resp.text)):
+            m = moduleregex.match(link)
+            if m:
+                self.modules.add(m.group(1))
+
+    def find_tarball(self, modulename, max_version, wantchecksum):
+        if modulename not in self.modules:
+            return None, None, None
+
+        resp = requests.get(posixjoin(self.baseurl, modulename, 'cache.json'))
+        resp.raise_for_status()
+
+        versions = resp.json()[1][modulename]
+        latest = get_latest_version(versions.keys(), max_version)
+
+        extensions = ['tar.xz', 'tar.bz2', 'tar.gz']
+        for ext in extensions:
+            if ext in versions[latest]:
+                tarball = versions[latest][ext]
+                break
+        else:
+            # unknown extension
+            return None, None, None
+
+        checksum = None
+        if wantchecksum and 'sha256sum' in versions[latest]:
+            resp = requests.get(posixjoin(self.baseurl, modulename, versions[latest]['sha256sum']))
+            resp.raise_for_status()
+
+            basename = os.path.basename(tarball)
+            for l in resp.text.splitlines():
+                l = l.split()
+                if basename == l[1]:
+                    checksum = l[0]
+                    break
+
+        return posixjoin(self.baseurl, modulename, tarball), latest, checksum
+
 # mapping from name to DownloadSite subclasses
 SITE_KINDS = {
     'tarballs': Tarballs,
+    'gnome': GNOME,
 }
 
 # utility functions
diff --git a/tools/smoketesting/tarball-conversion-3-34.config 
b/tools/smoketesting/tarball-conversion-3-34.config
index 34e5b06..74353a4 100644
--- a/tools/smoketesting/tarball-conversion-3-34.config
+++ b/tools/smoketesting/tarball-conversion-3-34.config
@@ -1,7 +1,7 @@
 <?xml version="1.0" ?>
 <conversioninfo>
   <locations>  FIXME: get a better name
-    <site default="true" location="https://download.gnome.org/sources/{module}/"; />
+    <site kind="gnome" default="true" location='https://download.gnome.org/sources/' />
 
     <site module="ModemManager"              location="https://www.freedesktop.org/software/ModemManager/"/>
     <site module="PackageKit"                
location="https://www.freedesktop.org/software/PackageKit/releases/"/>
diff --git a/tools/smoketesting/tarball-conversion-3-36.config 
b/tools/smoketesting/tarball-conversion-3-36.config
index e3fefa9..adbf8fd 100644
--- a/tools/smoketesting/tarball-conversion-3-36.config
+++ b/tools/smoketesting/tarball-conversion-3-36.config
@@ -1,7 +1,7 @@
 <?xml version="1.0" ?>
 <conversioninfo>
   <locations>  FIXME: get a better name
-    <site default="true" location="https://download.gnome.org/sources/{module}/"; />
+    <site kind="gnome" default="true" location='https://download.gnome.org/sources/' />
 
     <site module="ModemManager"              location="https://www.freedesktop.org/software/ModemManager/"/>
     <site module="accountsservice"           
location="https://www.freedesktop.org/software/accountsservice/"/>
diff --git a/tools/smoketesting/tarball-conversion.config b/tools/smoketesting/tarball-conversion.config
index 3564dd9..ccee7be 100644
--- a/tools/smoketesting/tarball-conversion.config
+++ b/tools/smoketesting/tarball-conversion.config
@@ -1,7 +1,7 @@
 <?xml version="1.0" ?>
 <conversioninfo>
   <locations>  FIXME: get a better name
-    <site default="true" location="https://download.gnome.org/sources/{module}/"; />
+    <site kind="gnome" default="true" location='https://download.gnome.org/sources/' />
 
     <site module="ModemManager"              location="https://www.freedesktop.org/software/ModemManager/"/>
     <site module="accountsservice"           
location="https://www.freedesktop.org/software/accountsservice/"/>