[releng/mcatanzaro/429] Sleep one second when receiving HTTP 429 error

[Michael Catanzaro <mcatanzaro src gnome org>]

commit 717862116d357521151fb59a61e018b277c300c3
Author: Michael Catanzaro <mcatanzaro gnome org>
Date:   Wed Dec 23 12:42:06 2020 -0600

    Sleep one second when receiving HTTP 429 error
    
    One of our mirrors has started rate-limiting us. It seems this mirror is
    selected for me 100% of the time, and tarball conversion fails 100% of
    the time. Ouch. We can avoid this by sleeping for one second and
    retrying when it occurs.
    
    Note this is called on a thread, so it theoretically only affects the
    thread that actaully received the 429 response. This means that other
    threads are likely to hit the same error, but that's OK.
    
    Fixes #23

 tools/smoketesting/downloadsites.py | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)
---
diff --git a/tools/smoketesting/downloadsites.py b/tools/smoketesting/downloadsites.py
index 593a9b9..fff3335 100644
--- a/tools/smoketesting/downloadsites.py
+++ b/tools/smoketesting/downloadsites.py
@@ -22,9 +22,10 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
 # USA
 
+import os
 import re
 import requests
-import os
+import time
 
 from html.parser import HTMLParser
 from posixpath import join as posixjoin # Handy for URLs
@@ -38,6 +39,17 @@ class DownloadSite:
     def find_tarball(self, modulename, max_version, wantchecksum):
         raise NotImplementedError
 
+def performRequest(location):
+    req = None
+    while True:
+        req = requests.get(location)
+        if req.status_code == 429:
+            # Too Many Requests: we hit a rate limit
+            time.sleep(1)
+            continue
+        req.raise_for_status()
+        return req
+
 class Tarballs(DownloadSite):
     def __init__(self, baseurl):
         super().__init__(baseurl)
@@ -50,8 +62,7 @@ class Tarballs(DownloadSite):
         location = self.baseurl.format(module=modulename)
 
         while True:
-            req = requests.get(location)
-            req.raise_for_status()
+            req = performRequest(location)
             files = get_links(req.text)
 
             # Check to see if we need to descend to a subdirectory
@@ -94,8 +105,7 @@ class GNOME(DownloadSite):
     def __init__(self, baseurl):
         super().__init__(baseurl)
 
-        resp = requests.get(self.baseurl)
-        resp.raise_for_status()
+        resp = performRequest(self.baseurl)
 
         moduleregex = re.compile('([^/]+)/')
 
@@ -108,8 +118,7 @@ class GNOME(DownloadSite):
         if modulename not in self.modules:
             return None, None, None
 
-        resp = requests.get(posixjoin(self.baseurl, modulename, 'cache.json'))
-        resp.raise_for_status()
+        resp = performRequest(posixjoin(self.baseurl, modulename, 'cache.json'))
 
         versions = resp.json()[1][modulename]
         latest = get_latest_version(versions.keys(), max_version)
@@ -128,8 +137,7 @@ class GNOME(DownloadSite):
 
         checksum = None
         if wantchecksum and 'sha256sum' in versions[latest]:
-            resp = requests.get(posixjoin(self.baseurl, modulename, versions[latest]['sha256sum']))
-            resp.raise_for_status()
+            resp = performRequest(posixjoin(self.baseurl, modulename, versions[latest]['sha256sum']))
 
             basename = os.path.basename(tarball)
             for l in resp.text.splitlines():