Re: Who does 2.12 Beta 2

[Elijah Newren <newren gmail com>]

On 7/28/05, Murray Cumming <murrayc murrayc com> wrote:
> On Thu, 2005-07-28 at 12:29 -0600, Elijah Newren wrote:
> > On 7/28/05, Murray Cumming <murrayc murrayc com> wrote:
> > > Would someone else like to do the 2.12 Beta 2 release on August 10th
> > > (tarballs due August 8th)? I'll be away from home at that time, and I'd
> > > like us to take turns anyway.
<snip>
> > My understanding of the steps so far:
<snip>
> Yeah, and run the signal-ftp-sync script.

Okay, I'll take this next one.  And I'll write all this stuff into
some kind of documentation to make it easier for others.

> > [1] I'm really messed up on the versions thing.  Why does
> > releng/tools/versions include some freedesktop.org stuff (e.g.
> > libxklavier and startup-notification)--aren't those external tarballs?
> >  (If not, why are they included when cairo isn't?)  Follow-up
> > question--why is startup-notification on the list of modules on the
> > wiki (note: libxklavier and cairo are not)?
> 
> Not sure.

Anyone else?

> > [2] Any hints here in the steps needed to modify garnome would be appreciated.
> 
> I figure out what's new by looking at ftp-release lists, but your new
> update-versions scripts is much better for that. If you're lucky, the
> latest garnome is almost up-to-date anyway.
> 
> Then I change the version numbers in the garnome Makefile files. Then I
> run "make makesum" to update all the checksums from the downloaded files
> Then I run make from inside desktop/.

Ah, cool.  So you don't build the bindings just to make sure they compile?

Also, I have a working but incomplete script that takes a jhbuild
moduleset, and transforms it into a new moduleset that uses tarballs
(I thought it'd be cool to do it this way, because then you don't have
to worry about changing dependencies, you get all the module patches
that jhbuild has, etc.); it also searches for the latest tarballs
(well, at least on ftp.gnome.org).  It probably sucks because I don't
know python, and because it isn't complete enough to use (though you
can have it generate the basic moduleset for you and then you just
fill in blank fields like md5sums and sizes and tarball locations that
weren't found and such).  But if anyone has some time to play with it,
let me know.  I'll attach it for the curious.  I doubt I'll have
enough time to get it to work before the next release...

#!/usr/bin/python

# ./testme5.py -s ~/cvs/gnome/jhbuild/modulesets -v 2.11.90 gnome-2.12.modules
#    reads in ~/cvs/gnome/jhbuild/modulesets/gnome-2.12.modules and
#    creates a ./gnome-2.11.90.modules file (and
#    freedesktop-2.11.90.modules file) based off of it but which uses
#    tarballs

import sys, string
import re
import getopt
import os
from ftplib import FTP
from xml.dom import minidom, Node

usage = ' -s sourcedir -v version filename-to-convert'
help = \
'''Get a psychiatrist.'''

# Some TODOs
#   - Make a useful help message
#   - After finding the tarball location on the internet, download it
#     and get its size and md5sum
#   - Make a little class that stores cvsroot->ftp/http-site
#     information, provides easy accessors for it, etc.  Then use that
#     instead of my hardcoded stuff that only works with ftp.gnome.org
#     right now.  Should probably be slurped in from some xml file
#   - Make some kind of code that works for http sites, if necessary
#     (I currently only have ftp hardcoded and with ftp.gnome.org in
#     particular)
#   - Make an easy cvsname->tarball name conversion
#     (e.g. gtkhtml2->libgtkhtml) instead of my hacks; probably also
#     uses an xml file (or the same one)
#   - Make it easy to make an exclude list, or maybe even just a
#     whitelist (jhbuild includes all kinds of crap that; Get rid of
#     my nasty hacks for doing this.  Would probably also make use
#     an/the xml file
#   - Get this script to provide an updated versions file
#     automatically (for use with the release-suites file) instead of
#     having to run releng/tools/update-versions manually and hoping
#     the 'latest tarball' still means the same one

def bigger_version(a, b):
    a_nums = re.split(r'\.', a)
    b_nums = re.split(r'\.', b)
    num_fields = min(len(a_nums), len(b_nums))
    for i in range(0,num_fields):
        if   int(a_nums[i]) > int(b_nums[i]):
            return a
        elif int(a_nums[i]) < int(b_nums[i]):
            return b
    if len(a_nums) > len(b_nums):
        return a
    else:
        return b

def get_latest_version(versions):
    max_version = versions[0]
    for version in versions[1:]:
        max_version = bigger_version(max_version, version)
    return max_version

def get_files_in_tarball_dir(ftp):
    location = ''
    good_dir = re.compile('^([0-9]+\.)*[0-9]+$')
    def hasdirs(x): return good_dir.search(x)
    while True:
        files = ftp.nlst()
        newdirs = filter(hasdirs, files)
        if newdirs:
            newdir = get_latest_version(newdirs)
            location += '/' + newdir
            ftp.cwd(newdir)
        else:
            break
    return location, files

class ConvertToTarball:
    def __init__(self, sourcedir, version):
        self.sourcedir = sourcedir
        self.version = version

    def find_tarball(self, name, cvsroot):
        print "Trying to find tarball for " + name + "...",

        # FIXME: EW, WHAT AN UGLY HACK
        translate = { 'gtkhtml2'             : 'libgtkhtml',
                      'gconf'                : 'GConf',
                      'gnome-control-center' : 'control-center',
                      'gdm2'                 : 'gdm',
                      'libsigc++2'           : 'libsigc++'
                    }

        try:
            newname = translate[name]
            name = newname
        except KeyError:
            name = name  # How do you no-op an exception?

        # FIXME: EW, WHAT AN UGLIER HACK
        name = re.sub(r'^gnome-python/(.*)$', r'\1', name)
        name = re.sub(r'^gnomemm/(.*)$', r'\1', name)

        proto = 'ftp://'
        site = 'ftp.gnome.org'
        baseloc = 'pub/GNOME/sources/' + name
        
        ftp = FTP(site)
        ftp.login('anonymous', '')
        ftp.cwd(baseloc)
        subdir, files = get_files_in_tarball_dir(ftp)

        def isbz2tarball(x): return x.endswith('.tar.bz2')
        def isgztarball(x):  return x.endswith('.tar.gz')
        tarballs = filter(isbz2tarball, files)
        if not tarballs:
            tarballs = filter(isgztarball, files)

        def getversion(x): return re.sub(r'^.*-(([0-9]+\.)*[0-9]+)\.tar.*$', r'\1', x)
        versions = map(getversion, tarballs)
        version = get_latest_version(versions)
        index = versions.index(version)

        location = proto+site+'/'+baseloc+subdir+'/'+tarballs[index]
        print location
        ftp.quit()
        return location, version, 'idonotknow', 'ProbablyBig'

    def create_tarball_node(self, document, node):
        tarball = document.createElement('tarball')
        attrs = node.attributes
        cvsroot = None
        for attrName in attrs.keys():
            if attrName == 'cvsroot':
                cvsroot = attrs.get(attrName).nodeValue
                continue
            if attrName == 'id':
                id = attrs.get(attrName).nodeValue
                # FIXME: EW, WHAT A NASTY UGLY HACK
                if id == 'nautilus-open-terminal' or \
                   id == 'nautilus-vcs' or \
                   id == 'glade2c' or \
                   id == 'pan' or \
                   id == 'gnet' or \
                   id == 'fast-user-switch-applet' or \
                   id == 'gnome-power-manager' or \
                   id == 'libgircclient' or \
                   id == 'gnomechat' or \
                   id == 'conglomerate' or \
                   id == 'anjuta' or \
                   id == 'OpenApplet' or \
                   re.search(r'^storage/.*', id) or \
                   id == 'monkey-bubble' or \
                   id == 'gnome-schedule':
                    return None
            attrNode = attrs.get(attrName)
            attrValue = attrNode.nodeValue
            tarball.setAttribute(attrName, attrValue)
        source_node = document.createElement('source')
        tarball.appendChild(source_node)

        if cvsroot == None:  # gnome cvs
            location, version, md5sum, size = self.find_tarball(id, cvsroot)
            tarball.setAttribute('version', version)
            source_node.setAttribute('href', location)
            source_node.setAttribute('size', size)
            source_node.setAttribute('md5sum', md5sum)
        else:  # Here's where we'd go searching other places, like freedesktop.org
            tarball.setAttribute('version', 'EAT-YOUR-BRAAAAAANE')
            source_node.setAttribute('href', 'http://somewhere.over.the.rainbow/where/bluebirds/die')
            source_node.setAttribute('size', 'HUGE')
            source_node.setAttribute('md5sum', 'blablablaihavenorealclue')
#        else:  # Just some of Jamesh's code that I'm keeping for comparison
#            filename, version = self._find_tarball(pkg)
#            tarball.setAttribute('version', version)
#
#            source_node.setAttribute('href',
#                                     urlparse.urljoin(self.uribase, filename))
#            info = os.stat(os.path.join(self.sourcedir, filename))
#            size = info[stat.ST_SIZE]
#            source_node.setAttribute('size', str(info[stat.ST_SIZE]))
#
#            sum = md5.new()
#            fp = open(os.path.join(self.sourcedir, filename), 'rb')
#            data = fp.read(4096)
#            while data:
#                sum.update(data)
#                data = fp.read(4096)
#            fp.close()
#            source_node.setAttribute('md5sum', sum.hexdigest())
        return tarball

    def walk(self, oldRoot, newRoot, document):
        for node in oldRoot.childNodes:
            if node.nodeType == Node.ELEMENT_NODE:
                if node.nodeName == 'cvsroot' or \
                   node.nodeName == 'svnroot' or \
                   node.nodeName == 'arch-archive':
                    continue
                elif node.nodeName == 'cvsmodule' or node.nodeName == 'mozillamodule':
                    entry = self.create_tarball_node(document, node)
                elif node.nodeName == 'include':
                    self.fix_file(node.attributes.get('href').nodeValue)
                else:
                    # Write out the element name.
                    entry = document.createElement(node.nodeName)
                    # Write out the attributes.
                    attrs = node.attributes
                    for attrName in attrs.keys():
                        attrNode = attrs.get(attrName)
                        attrValue = attrNode.nodeValue
                        entry.setAttribute(attrName, attrValue)
                if entry:   # entry can be None if we want to skip this tarball
                    # Walk the child nodes.
                    self.walk(node, entry, document)
                    # Append the new node to the newRoot
                    newRoot.appendChild(entry)

    def fix_file(self, input_filename):
        newname = re.sub(r'^([a-z]+).*(.modules)$',
                         r'\1-' + self.version + r'\2',
                         input_filename)
        if (os.path.isfile(newname)):
            sys.stderr.write('Can''t proceed; would overwrite '+newname+'\n')
            sys.exit(1)

        old_document = minidom.parse(os.path.join(self.sourcedir, input_filename))
        oldRoot = old_document.documentElement

        new_document = minidom.Document()
        newRoot = new_document.createElement(oldRoot.nodeName)
        new_document.appendChild(newRoot)

        self.walk(oldRoot, newRoot, new_document)

        newfile = file(newname, 'w+')
        new_document.writexml(newfile, "", "  ", '\n')

        old_document.unlink()
        new_document.unlink()

def main(args):
    program_name = args[0]
    try:
        opts, args = getopt.getopt(args[1:], 's:v:h',
                                   ['sourcedir=', 'version=', 'help'])
    except getopt.error, exc:
        sys.stderr.write(program_name % ': %s\n' % str(exc))
        sys.stderr.write(program_name + usage + '\n')
        sys.exit(1)

    sourcedir = None
    version = None
    for opt, arg in opts:
        if opt in ('-h', '--help'):
            print program_name + usage
            print help
            sys.exit(0)
        elif opt in ('-s', '--sourcedir'):
            sourcedir = arg
        elif opt in ('-v', '--version'):
            version = arg
    if not sourcedir or not version:
        sys.stderr.write(program_name + usage + '\n')
        sys.exit(1)

    convert = ConvertToTarball(sourcedir, version)
    convert.fix_file(args[-1])

if __name__ == '__main__':
    main(sys.argv)