[releng/mcatanzaro/429] Sleep one second when receiving HTTP 429 error

[Michael Catanzaro <mcatanzaro src gnome org>]

commit 819c104f9067bd25b72cce013764ce68bb9ddbd1
Author: Michael Catanzaro <mcatanzaro gnome org>
Date:   Wed Dec 23 12:42:06 2020 -0600

    Sleep one second when receiving HTTP 429 error
    
    One of our mirrors has started rate-limiting us. It seems this mirror is
    selected for me 100% of the time, and tarball conversion fails 100% of
    the time. Ouch. We can avoid this by sleeping for one second and
    retrying when it occurs.
    
    Note this is called on a thread, so it theoretically only affects the
    thread that actaully received the 429 response. This means that other
    threads are likely to hit the same error, but that's OK.
    
    Fixes #23

 tools/smoketesting/downloadsites.py |  26 +++++---
 tools/smoketesting/versions         | 130 ++++++++++++++++++++++++++++++++++++
 2 files changed, 147 insertions(+), 9 deletions(-)
---
diff --git a/tools/smoketesting/downloadsites.py b/tools/smoketesting/downloadsites.py
index 593a9b9..bb8ff32 100644
--- a/tools/smoketesting/downloadsites.py
+++ b/tools/smoketesting/downloadsites.py
@@ -22,9 +22,10 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
 # USA
 
+import os
 import re
 import requests
-import os
+import time
 
 from html.parser import HTMLParser
 from posixpath import join as posixjoin # Handy for URLs
@@ -38,6 +39,17 @@ class DownloadSite:
     def find_tarball(self, modulename, max_version, wantchecksum):
         raise NotImplementedError
 
+def perform_request(location):
+    req = None
+    while True:
+        req = requests.get(location)
+        if req.status_code == 429:
+            # Too Many Requests: we hit a rate limit
+            time.sleep(1)
+            continue
+        req.raise_for_status()
+        return req
+
 class Tarballs(DownloadSite):
     def __init__(self, baseurl):
         super().__init__(baseurl)
@@ -50,8 +62,7 @@ class Tarballs(DownloadSite):
         location = self.baseurl.format(module=modulename)
 
         while True:
-            req = requests.get(location)
-            req.raise_for_status()
+            req = perform_request(location)
             files = get_links(req.text)
 
             # Check to see if we need to descend to a subdirectory
@@ -94,8 +105,7 @@ class GNOME(DownloadSite):
     def __init__(self, baseurl):
         super().__init__(baseurl)
 
-        resp = requests.get(self.baseurl)
-        resp.raise_for_status()
+        resp = perform_request(self.baseurl)
 
         moduleregex = re.compile('([^/]+)/')
 
@@ -108,8 +118,7 @@ class GNOME(DownloadSite):
         if modulename not in self.modules:
             return None, None, None
 
-        resp = requests.get(posixjoin(self.baseurl, modulename, 'cache.json'))
-        resp.raise_for_status()
+        resp = perform_request(posixjoin(self.baseurl, modulename, 'cache.json'))
 
         versions = resp.json()[1][modulename]
         latest = get_latest_version(versions.keys(), max_version)
@@ -128,8 +137,7 @@ class GNOME(DownloadSite):
 
         checksum = None
         if wantchecksum and 'sha256sum' in versions[latest]:
-            resp = requests.get(posixjoin(self.baseurl, modulename, versions[latest]['sha256sum']))
-            resp.raise_for_status()
+            resp = perform_request(posixjoin(self.baseurl, modulename, versions[latest]['sha256sum']))
 
             basename = os.path.basename(tarball)
             for l in resp.text.splitlines():
diff --git a/tools/smoketesting/versions b/tools/smoketesting/versions
new file mode 100644
index 0000000..cf79705
--- /dev/null
+++ b/tools/smoketesting/versions
@@ -0,0 +1,130 @@
+## CORE
+core:adwaita-icon-theme:3.38.0:
+core:at-spi2-atk:2.38.0:
+core:at-spi2-core:2.38.0:
+core:atk:2.36.0:
+core:atkmm:2.28.1:
+core:baobab:3.38.0:
+core:cantarell-fonts:0.301:
+core:cheese:3.38.0:
+core:clutter:1.26.4:
+core:clutter-gst:3.0.27:
+core:clutter-gtk:1.8.4:
+core:cogl:1.22.8:
+core:dconf:0.38.0:
+core:eog:3.38.1:
+core:epiphany:3.38.2:
+core:evince:3.38.0:
+core:evolution-data-server:3.38.2:
+core:file-roller:3.38.0:
+core:folks:0.14.0:
+core:gcab:1.4:
+core:gcr:3.38.0:
+core:gdk-pixbuf:2.42.2:
+core:gdm:3.38.2.1:
+core:gedit:3.38.1:
+core:geocode-glib:3.26.2:
+core:gjs:1.66.1:
+core:glib:2.67.1:
+core:glib-networking:2.66.0:
+core:glibmm:2.64.5:
+core:gmime:3.2.7:
+core:gnome-autoar:0.2.4:
+core:gnome-backgrounds:3.38.0:
+core:gnome-bluetooth:3.34.3:
+core:gnome-boxes:3.38.2:
+core:gnome-calculator:3.38.2:
+core:gnome-calendar:3.38.2:
+core:gnome-characters:3.34.0:
+core:gnome-clocks:3.38.0:
+core:gnome-color-manager:3.36.0:
+core:gnome-contacts:3.38.1:
+core:gnome-control-center:3.38.2:
+core:gnome-desktop:3.38.2:
+core:gnome-disk-utility:3.38.1:
+core:gnome-font-viewer:3.34.0:
+core:gnome-getting-started-docs:3.38.0:
+core:gnome-initial-setup:3.38.2:
+core:gnome-keyring:3.36.0:
+core:gnome-logs:3.36.0:
+core:gnome-maps:3.38.2:
+core:gnome-menus:3.36.0:
+core:gnome-music:3.38.2:
+core:gnome-online-accounts:3.38.0:
+core:gnome-online-miners:3.34.0:
+core:gnome-photos:3.38.0:
+core:gnome-screenshot:3.38.0:
+core:gnome-session:3.38.0:
+core:gnome-settings-daemon:3.38.1:
+core:gnome-shell:3.38.2:
+core:gnome-shell-extensions:3.38.2:
+core:gnome-software:3.38.0:
+core:gnome-system-monitor:3.38.0:
+core:gnome-terminal:3.38.1:
+core:gnome-user-docs:3.38.2:
+core:gnome-user-share:3.34.0:
+core:gnome-video-effects:0.5.0:
+core:gnome-weather:3.36.1:
+core:gobject-introspection:1.66.1:
+core:gom:0.4:
+core:grilo:0.3.13:
+core:grilo-plugins:0.3.12:
+core:gsettings-desktop-schemas:3.38.0:
+core:gsound:1.0.2:
+core:gspell:1.9.1:
+core:gssdp:1.2.3:
+core:gtk:4.0.0:
+core:gtk+:3.24.24:
+core:gtk-doc:1.33.1:
+core:gtk-vnc:1.0.0:
+core:gtkmm:3.24.3:
+core:gtksourceview:4.8.0:
+core:gupnp:1.2.4:
+core:gupnp-av:0.12.11:
+core:gupnp-dlna:0.10.5:
+core:gvfs:1.46.1:
+core:json-glib:1.6.0:
+core:libchamplain:0.12.20:
+core:libdazzle:3.38.0:
+core:libgdata:0.17.13:
+core:libgee:0.20.3:
+core:gfbgraph:0.2.4:
+core:libgnomekbd:3.26.1:
+core:libgsf:1.14.47:
+core:libgtop:2.40.0:
+core:libgweather:3.36.1:
+core:libgxps:0.3.1:
+core:libhandy:1.0.2:
+core:libmediaart:1.9.4:
+core:libnma:1.8.30:
+core:libnotify:0.7.9:
+core:libpeas:1.28.0:
+core:rest:0.8.1:
+core:librsvg:2.50.2:
+core:libsecret:0.20.4:
+core:libsigc++:2.10.6:
+core:libsoup:2.72.0:
+core:mm-common:1.0.2:
+core:mutter:3.38.2:
+core:nautilus:3.38.2:
+core:orca:3.38.2:
+core:pango:1.48.0:
+core:pangomm:2.42.2:
+core:phodav:2.5:
+core:pyatspi:2.38.0:
+core:pygobject:3.38.0:
+core:rygel:0.40.0:
+core:simple-scan:3.38.2:
+core:sushi:3.38.0:
+core:totem:3.38.0:
+core:totem-pl-parser:3.26.5:
+core:tracker:3.0.2:
+core:tracker:2.3.6:
+core:tracker-miners:3.0.3:
+core:tracker-miners:2.3.5:
+core:vala:0.50.2:
+core:vte:0.62.1:
+core:yelp:3.38.2:
+core:yelp-tools:3.38.0:
+core:yelp-xsl:3.38.2:
+core:zenity:3.32.0: