[jhbuild: 4/6] Add new partial_build key, "sysdeps" command
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [jhbuild: 4/6] Add new partial_build key, "sysdeps" command
- Date: Fri, 15 Jul 2011 20:51:57 +0000 (UTC)
commit 3a2d06bc8c2e0ac8b64b10a0fd84995884d88992
Author: Colin Walters <walters verbum org>
Date: Thu Jun 23 16:43:51 2011 -0400
Add new partial_build key, "sysdeps" command
Basically, when the partial_build key is set (which it is by default),
when we're gathering a module list (for say "jhbuild build", or just
"jhbuild list"), we look at the installed pkg-config files. If there
is a local module which matches the id of the tarball, we skip it.
We should still be comparing versions.
But, this allows me to skip building e.g. cairo and nspr if I have
them on the system.
To allow developers to fill in using the system if available, add a
new "sysdeps" command. This command parses the moduleset and finds
all .pc files that are needed by tarballs from the given moduleset,
and installs them.
https://bugzilla.gnome.org/show_bug.cgi?id=564373
jhbuild/commands/Makefile.am | 1 +
jhbuild/commands/sysdeps.py | 102 +++++++++++++++++++++++++++
jhbuild/config.py | 2 +-
jhbuild/defaults.jhbuildrc | 5 +-
jhbuild/moduleset.py | 40 ++++++++++-
jhbuild/utils/Makefile.am | 1 +
jhbuild/utils/cmds.py | 22 +++---
jhbuild/utils/systeminstall.py | 148 ++++++++++++++++++++++++++++++++++++++++
8 files changed, 306 insertions(+), 15 deletions(-)
---
diff --git a/jhbuild/commands/Makefile.am b/jhbuild/commands/Makefile.am
index d5f3bc9..704fb7d 100644
--- a/jhbuild/commands/Makefile.am
+++ b/jhbuild/commands/Makefile.am
@@ -14,6 +14,7 @@ app_PYTHON = \
rdepends.py \
sanitycheck.py \
snapshot.py \
+ sysdeps.py \
tinderbox.py \
uninstall.py
diff --git a/jhbuild/commands/sysdeps.py b/jhbuild/commands/sysdeps.py
new file mode 100644
index 0000000..fc4642f
--- /dev/null
+++ b/jhbuild/commands/sysdeps.py
@@ -0,0 +1,102 @@
+# jhbuild - a build script for GNOME 1.x and 2.x
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# sysdeps.py: Install system dependencies
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os
+import sys
+import urllib
+from optparse import make_option
+import logging
+
+import jhbuild.moduleset
+import jhbuild.frontends
+from jhbuild.errors import UsageError, FatalError
+from jhbuild.commands import Command, register_command
+import jhbuild.commands.base
+from jhbuild.commands.base import cmd_build
+from jhbuild.utils.cmds import check_version
+from jhbuild.utils.systeminstall import SystemInstall
+from jhbuild.versioncontrol.tarball import TarballBranch
+
+class cmd_sysdeps(cmd_build):
+ doc = N_('Check and install tarball dependencies using system packages')
+
+ name = 'sysdeps'
+
+ def __init__(self):
+ Command.__init__(self, [
+ make_option('--install',
+ action='store_true', default = False,
+ help=_('Install pkg-config modules via system'))])
+
+ def run(self, config, options, args, help=None):
+ config.set_from_cmdline_options(options)
+
+ installer = SystemInstall.find_best()
+ if installer is None:
+ raise FatalError(_("Don't know how to install packages on this system"))
+
+ if not config.partial_build:
+ raise FatalError(_("Partial build is not enabled; add partial_build = True to ~/.jhbuildrc"))
+
+ module_set = jhbuild.moduleset.load(config)
+ modules = args or config.modules
+ module_list = module_set.get_module_list(modules, process_sysdeps=False)
+ module_state = module_set.get_system_modules(module_list)
+
+ have_new_enough = False
+ have_too_old = False
+
+ print _('System installed packages which are new enough:')
+ for pkg_config,(module, req_version, installed_version, new_enough) in module_state.iteritems():
+ if (installed_version is not None) and new_enough:
+ have_new_enough = True
+ print (_(" %(pkg)s (required=%(req)s, installed=%(installed)s)" % {'pkg': pkg_config,
+ 'req': req_version,
+ 'installed': installed_version}))
+ if not have_new_enough:
+ print _(' (none)')
+
+ print _('System installed packages which are too old:')
+ for pkg_config,(module, req_version, installed_version, new_enough) in module_state.iteritems():
+ if (installed_version is not None) and (not new_enough):
+ have_too_old = False
+ print (_(" %(pkg)s (required=%(req)s, installed=%(installed)s)" % {'pkg': pkg_config,
+ 'req': req_version,
+ 'installed': installed_version}))
+ if not have_too_old:
+ print _(' (none)')
+
+ print _('No matching system package installed:')
+ uninstalled = []
+ for pkg_config,(module, req_version, installed_version, new_enough) in module_state.iteritems():
+ if installed_version is None:
+ print (_(" %(pkg)s (required=%(req)s)") % {'pkg': pkg_config,
+ 'req': req_version})
+ uninstalled.append(pkg_config)
+ if len(uninstalled) == 0:
+ print _(' (none)')
+
+ if options.install:
+ if len(uninstalled) == 0:
+ logging.info(_("No uninstalled system dependencies to install for modules: %r" % (modules, )))
+ else:
+ logging.info(_("Installing dependencies on system: %s" % (' '.join(uninstalled), )))
+ installer.install(uninstalled)
+
+register_command(cmd_sysdeps)
diff --git a/jhbuild/config.py b/jhbuild/config.py
index bcf5a9a..0488a16 100644
--- a/jhbuild/config.py
+++ b/jhbuild/config.py
@@ -40,7 +40,7 @@ _defaults_file = os.path.join(os.path.dirname(__file__), 'defaults.jhbuildrc')
_default_jhbuildrc = os.path.join(os.environ['HOME'], '.jhbuildrc')
_known_keys = [ 'moduleset', 'modules', 'skip', 'tags', 'prefix',
- 'checkoutroot', 'buildroot', 'top_builddir',
+ 'partial_build', 'checkoutroot', 'buildroot', 'top_builddir',
'autogenargs', 'makeargs',
'installprog', 'repos', 'branches', 'noxvfb', 'xvfbargs',
'builddir_pattern', 'module_autogenargs', 'module_makeargs',
diff --git a/jhbuild/defaults.jhbuildrc b/jhbuild/defaults.jhbuildrc
index 9041028..9381d26 100644
--- a/jhbuild/defaults.jhbuildrc
+++ b/jhbuild/defaults.jhbuildrc
@@ -21,6 +21,9 @@ modules = [ 'meta-gnome-core', 'meta-gnome-apps-tested' ]
# have changed.
build_policy = 'updated-deps'
+# If True, ignore tarball modules already installed while building
+partial_build = True
+
# Skip modules installed more recently than the specified relative time.
# min_age can only be specified via the command-line. Setting min_age within
# the configuration file has no effect.
@@ -172,4 +175,4 @@ dvcs_mirror_dir = None
# A string displayed before JHBuild executes a command. String may contain the
# variables %(command)s, %(cwd)s
-print_command_pattern = '%(command)s'
\ No newline at end of file
+print_command_pattern = '%(command)s'
diff --git a/jhbuild/moduleset.py b/jhbuild/moduleset.py
index aacd4a9..98803cb 100644
--- a/jhbuild/moduleset.py
+++ b/jhbuild/moduleset.py
@@ -36,8 +36,10 @@ except ImportError:
from jhbuild import modtypes
from jhbuild.versioncontrol import get_repo_type
from jhbuild.utils import httpcache
-from jhbuild.utils.cmds import get_output
+from jhbuild.utils.cmds import compare_version, get_output
from jhbuild.modtypes.testmodule import TestModule
+from jhbuild.versioncontrol.tarball import TarballBranch
+from jhbuild.utils.systeminstall import SystemInstall
__all__ = ['load', 'load_tests', 'get_default_repo']
@@ -67,7 +69,7 @@ class ModuleSet:
def get_module_list(self, seed, skip=[], tags=[], ignore_cycles=False,
ignore_suggests=False, include_optional_modules=False,
- ignore_missing=False):
+ ignore_missing=False, process_sysdeps=True):
'''gets a list of module objects (in correct dependency order)
needed to build the modules in the seed list'''
@@ -99,6 +101,7 @@ class ModuleSet:
'invalid': modname})
dep_missing = True
continue
+
if not depmod in all_modules:
all_modules.append(depmod)
@@ -125,6 +128,14 @@ class ModuleSet:
# mark skipped modules as already processed
self._state[self.modules.get(modname)] = 'processed'
+ # process_sysdeps lets us avoid repeatedly checking system module state when
+ # handling recursive dependencies.
+ if self.config.partial_build and process_sysdeps:
+ system_module_state = self.get_system_modules(all_modules)
+ for pkg_config,(module, req_version, installed_version, new_enough) in system_module_state.iteritems():
+ if new_enough:
+ self._state[module] = 'processed'
+
if tags:
for modname in self.modules:
for tag in tags:
@@ -186,7 +197,7 @@ class ModuleSet:
# <http://bugzilla.gnome.org/show_bug.cgi?id=546640>
t_ms = ModuleSet(self.config)
t_ms.modules = self.modules.copy()
- dep_modules = t_ms.get_module_list(seed=[depmod.name])
+ dep_modules = t_ms.get_module_list(seed=[depmod.name], process_sysdeps=False)
for m in dep_modules[:-1]:
if m in all_modules:
extra_afters.append(m)
@@ -228,6 +239,29 @@ class ModuleSet:
if test_app in mod.tested_pkgs:
test_modules.append(mod)
return test_modules
+
+ def get_system_modules(self, modules):
+ assert self.config.partial_build
+
+ installed_pkgconfig = SystemInstall.get_installed_pkgconfigs()
+
+ # pkgconfig -> (required_version, installed_verison)
+ module_state = {}
+ for module in modules:
+ if module.pkg_config is None:
+ continue
+ if not isinstance(module.branch, TarballBranch):
+ continue
+ # Strip off the .pc
+ module_pkg = module.pkg_config[:-3]
+ required_version = module.branch.version
+ if not module_pkg in installed_pkgconfig:
+ module_state[module_pkg] = (module, required_version, None, False)
+ else:
+ installed_version = installed_pkgconfig[module_pkg]
+ new_enough = compare_version(installed_version, required_version)
+ module_state[module_pkg] = (module, required_version, installed_version, new_enough)
+ return module_state
def write_dot(self, modules=None, fp=sys.stdout, suggests=False, clusters=False):
from jhbuild.modtypes import MetaModule
diff --git a/jhbuild/utils/Makefile.am b/jhbuild/utils/Makefile.am
index 01a23e3..6e0a7fb 100644
--- a/jhbuild/utils/Makefile.am
+++ b/jhbuild/utils/Makefile.am
@@ -7,6 +7,7 @@ app_PYTHON = \
notify.py \
packagedb.py \
sxml.py \
+ systeminstall.py \
trigger.py \
trayicon.py \
unpack.py
diff --git a/jhbuild/utils/cmds.py b/jhbuild/utils/cmds.py
index 7a62f51..2aa2f5e 100644
--- a/jhbuild/utils/cmds.py
+++ b/jhbuild/utils/cmds.py
@@ -240,16 +240,7 @@ def has_command(cmd):
return True
return False
-def check_version(cmd, regexp, minver, extra_env=None):
- try:
- data = get_output(cmd, extra_env=extra_env)
- except:
- return False
- match = re.match(regexp, data, re.MULTILINE)
- if not match:
- return False
- version = match.group(1)
-
+def compare_version(version, minver):
version = version.split('.')
for i, ver in enumerate(version):
part = re.sub(r'^[^\d]*(\d+).*$', r'\1', ver)
@@ -265,3 +256,14 @@ def check_version(cmd, regexp, minver, extra_env=None):
else:
minver[i] = int(part)
return version >= minver
+
+def check_version(cmd, regexp, minver, extra_env=None):
+ try:
+ data = get_output(cmd, extra_env=extra_env)
+ except:
+ return False
+ match = re.match(regexp, data, re.MULTILINE)
+ if not match:
+ return False
+ version = match.group(1)
+ return compare_version(version, minver)
diff --git a/jhbuild/utils/systeminstall.py b/jhbuild/utils/systeminstall.py
new file mode 100644
index 0000000..684ca0d
--- /dev/null
+++ b/jhbuild/utils/systeminstall.py
@@ -0,0 +1,148 @@
+# jhbuild - a build script for GNOME 1.x and 2.x
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# systeminstall.py - Use system-specific means to acquire dependencies
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import os
+import sys
+import logging
+import subprocess
+import pipes
+from StringIO import StringIO
+
+import cmds
+
+class SystemInstall(object):
+ def __init__(self):
+ pass
+
+ def install(self, pkgconfig_ids):
+ """Takes a list of pkg-config identifiers and uses a system-specific method to install them."""
+ raise NotImplementedError()
+
+ @classmethod
+ def get_installed_pkgconfigs(cls):
+ """Returns a dictionary mapping pkg-config names to their current versions on the system."""
+ env = dict(os.environ)
+ if 'PKG_CONFIG_PATH' in env:
+ del env['PKG_CONFIG_PATH']
+ proc = subprocess.Popen(['pkg-config', '--list-all'], stdout=subprocess.PIPE, env=env, close_fds=True)
+ stdout = proc.communicate()[0]
+ proc.wait()
+ pkgs = []
+ for line in StringIO(stdout):
+ pkg, rest = line.split(None, 1)
+ pkgs.append(pkg)
+ args = ['pkg-config', '--modversion']
+ args.extend(pkgs)
+ proc = subprocess.Popen(args, stdout=subprocess.PIPE, close_fds=True)
+ stdout = proc.communicate()[0]
+ proc.wait()
+ pkgversions = {}
+ for pkg,verline in zip(pkgs, StringIO(stdout)):
+ pkgversions[pkg] = verline.strip()
+ return pkgversions
+
+ @classmethod
+ def find_best(cls):
+ global _classes
+ for possible_cls in _classes:
+ if possible_cls.detect():
+ return possible_cls()
+
+# NOTE: This class is unfinished
+class PKSystemInstall(SystemInstall):
+ def __init__(self):
+ SystemInstall.__init__(self)
+
+ def _get_package_for(self, pkg_config):
+ assert pkg_config.endswith('.pc')
+ pkg_config = pkg_config[:-3]
+ proc = subprocess.Popen(['pkcon', '-p', 'what-provides', 'pkgconfig(%s)' % (pkg_config, ),
+ '--filter=arch;newest'], stdout=subprocess.PIPE, close_fds=True)
+ devnull.close()
+ stdout = proc.communicate()[0]
+ if proc.ecode != 0:
+ return None
+ pkg = None
+ for line in StringIO(stdout):
+ if line.startswith('Package:'):
+ pkg = line[line.find(':') + 1:].strip()
+ break
+ return pkg
+
+ def _on_pk_message(self, msgtype, msg):
+ logging.info(_('PackageKit: %s' % (msg,)))
+
+ def _on_pk_error(self, msgtype, msg):
+ logging.error(_('PackageKit: %s' % (msg,)))
+
+ def install(self, pkgconfig_ids):
+ import dbus
+ import dbus.glib
+ import glib
+
+ loop = glib.MainLoop()
+
+ sysbus = dbus.SystemBus()
+ pk = dbus.Interface(sysbus.get_object('org.freedesktop.PackageKit',
+ '/org/freedesktop/PackageKit'),
+ 'org.freedesktop.PackageKit')
+ tid = pk.GetTid()
+ txn = sysbus.get_object('org.freedesktop.PackageKit', tid)
+ txn_tx = dbus.Interface(txn, 'org.freedesktop.PackageKit.Transaction')
+ txn.connect_to_signal('Message', self._on_pk_message)
+ txn.connect_to_signal('ErrorCode', self._on_pk_error)
+ txn.connect_to_signal('Destroy', lambda *args: loop.quit())
+
+ pk_package_ids = set()
+ txn.connect_to_signal('Package', lambda info, pkid, summary: pk_package_ids.add(pkid))
+ txn_tx.WhatProvides("arch;newest;~installed", "any", map(lambda x: 'pkgconfig(%s)' % (x, ), pkgconfig_ids))
+ loop.run()
+
+ del txn
+
+ if len(pk_package_ids) == 0:
+ logging.info(_('Nothing available to install'))
+ return
+
+ logging.info(_('Installing: %s' % (' '.join(pk_package_ids, ))))
+
+ tid = pk.GetTid()
+ txn = sysbus.get_object('org.freedesktop.PackageKit', tid)
+ txn_tx = dbus.Interface(txn, 'org.freedesktop.PackageKit.Transaction')
+ txn.connect_to_signal('Message', self._on_pk_message)
+ txn.connect_to_signal('ErrorCode', self._on_pk_error)
+ txn.connect_to_signal('Destroy', lambda *args: loop.quit())
+
+ txn_tx.InstallPackages(True, pk_package_ids)
+ loop.run()
+
+ logging.info(_('Complete!'))
+
+ @classmethod
+ def detect(cls):
+ return cmds.has_command('pkcon')
+
+# Ordered from best to worst
+_classes = [PKSystemInstall]
+
+if __name__ == '__main__':
+ logging.basicConfig(level=logging.INFO)
+ installer = SystemInstall.find_best()
+ print "Using %r" % (installer, )
+ installer.install(sys.argv[1:])
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
