[releng] use requests module, avoid some redirects
- From: Olav Vitters <ovitters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [releng] use requests module, avoid some redirects
- Date: Wed, 30 Jul 2014 23:34:53 +0000 (UTC)
commit c32e50c9e0df43913ef4a9ce2594870ff48c9048
Author: Olav Vitters <olav vitters nl>
Date: Thu Jul 31 01:34:41 2014 +0200
use requests module, avoid some redirects
tools/smoketesting/convert-to-tarballs.py | 32 ++++++++++----------
.../smoketesting/tarball-conversion-stable.config | 2 +-
tools/smoketesting/tarball-conversion.config | 2 +-
3 files changed, 18 insertions(+), 18 deletions(-)
---
diff --git a/tools/smoketesting/convert-to-tarballs.py b/tools/smoketesting/convert-to-tarballs.py
index c09881c..f0d4e1b 100755
--- a/tools/smoketesting/convert-to-tarballs.py
+++ b/tools/smoketesting/convert-to-tarballs.py
@@ -76,7 +76,7 @@ import subprocess
from ftplib import FTP
from xml.dom import minidom, Node
from sgmllib import SGMLParser
-import urllib2
+import requests
import urlparse
if not hasattr(__builtins__, 'set'):
from sets import Set as set
@@ -318,7 +318,6 @@ class urllister(SGMLParser):
class TarballLocator:
def __init__(self, tarballdir, mirrors, determine_stats=True, local_only=False):
self.tarballdir = tarballdir
- self.urlopen = urllib2.build_opener()
self.have_sftp = self._test_sftp()
self.get_stats = determine_stats
self.local_only = local_only
@@ -584,7 +583,6 @@ class TarballLocator:
return location, files
def _get_files_from_http(self, parsed_url, max_version):
- obj = self.urlopen
good_dir = re.compile('^([0-9]+\.)*[0-9]+/?$')
def hasdirs(x): return good_dir.search(x)
def fixdirs(x): return re.sub(r'^([0-9]+\.[0-9]+)/?$', r'\1', x)
@@ -593,10 +591,12 @@ class TarballLocator:
redirect_location = location
while True:
# Get the files
- usock = obj.open(redirect_location)
+ req = requests.get(redirect_location)
+ if req.status_code != requests.codes.ok:
+ files = None
+ break
parser = urllister()
- parser.feed(usock.read())
- usock.close()
+ parser.feed(req.content)
parser.close()
files = parser.urls
@@ -605,7 +605,7 @@ class TarballLocator:
newdirs = map(fixdirs, newdirs)
if newdirs:
newdir = self._get_latest_version(newdirs, max_version)
- redirect_location = posixjoin(usock.url, newdir)
+ redirect_location = posixjoin(req.url, newdir)
location = posixjoin(location, newdir)
else:
break
@@ -623,7 +623,7 @@ class TarballLocator:
u = urlparse.urlparse(baselocation)
if u.scheme != 'file' and self.local_only:
- return 'http://somewhere.in.france/', '', 'blablablaihavenorealclue', 'HUGE'
+ return None, '', None, None
# Determine which function handles the actual retrieving
locator = getattr(self, '_get_files_from_%s' % u.scheme, None)
@@ -685,8 +685,8 @@ class TarballLocator:
if self.get_stats:
hash, size = self._get_tarball_stats(location, tarballs[index])
else:
- hash = 'md5:blablablaihavenorealclue'
- size = 'HUGE'
+ hash = None
+ size = None
return location, version, hash, size
class ConvertToTarballs:
@@ -756,7 +756,7 @@ class ConvertToTarballs:
max_version = self.options.get_version_limit(name)
location, version, hash, size = \
self.locator.find_tarball(baselocation, real_name, max_version)
- print ' ', location, version, hash, size
+ print ' ', location, version, hash if hash else "", size if size else ""
branch_node.setAttribute('version', version)
branch_node.setAttribute('repo', repo)
branch_node.setAttribute('module', location[len(repo):])
@@ -771,11 +771,11 @@ class ConvertToTarballs:
print ''
if not id in self.not_found:
self.not_found.append(id)
- branch_node.setAttribute('version', 'EAT-YOUR-BRAAAAAANE')
- branch_node.setAttribute('repo', 'http://somewhere.over.the.rainbow/')
- branch_node.setAttribute('module', 'where/bluebirds/die')
- branch_node.setAttribute('size', 'HUGE')
- branch_node.setAttribute('hash', 'md5:blablablaihavenorealclue')
+ #branch_node.setAttribute('version', 'EAT-YOUR-BRAAAAAANE')
+ #branch_node.setAttribute('repo', 'http://somewhere.over.the.rainbow/')
+ #branch_node.setAttribute('module', 'where/bluebirds/die')
+ #branch_node.setAttribute('size', 'HUGE')
+ #branch_node.setAttribute('hash', 'md5:blablablaihavenorealclue')
if revision and not max_version:
self.no_max_version.append(id)
return tarball
diff --git a/tools/smoketesting/tarball-conversion-stable.config
b/tools/smoketesting/tarball-conversion-stable.config
index 38fe523..f27f73c 100644
--- a/tools/smoketesting/tarball-conversion-stable.config
+++ b/tools/smoketesting/tarball-conversion-stable.config
@@ -3,7 +3,7 @@
<locations> FIXME: get a better name
<site cvs="gnome.org"
location="https://download.gnome.org/sources/"
- subdir="$module"
+ subdir="$module/"
/>
<site cvs="freedesktop.org"
location="http://www.freedesktop.org/software/"
diff --git a/tools/smoketesting/tarball-conversion.config b/tools/smoketesting/tarball-conversion.config
index d412321..30880bd 100644
--- a/tools/smoketesting/tarball-conversion.config
+++ b/tools/smoketesting/tarball-conversion.config
@@ -3,7 +3,7 @@
<locations> FIXME: get a better name
<site cvs="gnome.org"
location="https://download.gnome.org/sources/"
- subdir="$module"
+ subdir="$module/"
/>
<site cvs="freedesktop.org"
location="http://www.freedesktop.org/software/"
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]