[gnome-ostree/wip/integrated-build: 2/2] wip submodules
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree/wip/integrated-build: 2/2] wip submodules
- Date: Fri, 21 Sep 2012 23:29:06 +0000 (UTC)
commit ae527cf7d641a86179912379a14a6f9980daf701
Author: Colin Walters <walters verbum org>
Date: Fri Sep 21 17:05:08 2012 -0400
wip submodules
Makefile-integration.am | 43 ------------
Makefile-ostbuild.am | 1 +
Makefile.am | 1 -
configure.ac | 5 --
deleted-components/README | 2 +
gnomeos-3.6.json => manifest.json | 13 +---
src/integration/lib-group | 46 ------------
src/integration/lib-passwd | 25 -------
src/integration/os-release.in | 5 --
src/ostbuild/pyostbuild/builtin_build.py | 53 ++-------------
src/ostbuild/pyostbuild/builtin_fetch.py | 110 ++++++++++++++++++++++++++++++
src/ostbuild/pyostbuild/builtins.py | 79 +++++++--------------
src/ostbuild/pyostbuild/main.py | 1 +
13 files changed, 148 insertions(+), 236 deletions(-)
---
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..e69de29
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index b8035c4..246040c 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -39,6 +39,7 @@ pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/builtin_checkout.py \
src/ostbuild/pyostbuild/builtin_deploy_qemu.py \
src/ostbuild/pyostbuild/builtin_deploy_root.py \
+ src/ostbuild/pyostbuild/builtin_fetch.py \
src/ostbuild/pyostbuild/builtin_run_qemu.py \
src/ostbuild/pyostbuild/builtin_import_tree.py \
src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py \
diff --git a/Makefile.am b/Makefile.am
index 65e4203..963bff0 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -35,7 +35,6 @@ privlib_LTLIBRARIES =
INSTALL_DATA_HOOKS =
include Makefile-ostbuild.am
-include Makefile-integration.am
install-data-hook: $(INSTALL_DATA_HOOKS)
diff --git a/configure.ac b/configure.ac
index 8e5a01b..67db4a7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -25,11 +25,6 @@ AC_ARG_ENABLE(buildsystem,
enable_buildsystem=yes)
AM_CONDITIONAL(BUILDSYSTEM, test x$enable_buildsystem != xno)
-AC_ARG_ENABLE(system-integration,
- AS_HELP_STRING([--enable-system-integration], [Install integration for target system]),,
- enable_system_integration=no)
-AM_CONDITIONAL(SYSTEM_INTEGRATION, test x$enable_system_integration != xno)
-
AC_CONFIG_FILES([
Makefile
])
diff --git a/deleted-components/README b/deleted-components/README
new file mode 100644
index 0000000..0efb3f5
--- /dev/null
+++ b/deleted-components/README
@@ -0,0 +1,2 @@
+Components that used to exist but are no longer listed in the manifest
+are moved here for safety.
diff --git a/gnomeos-3.6.json b/manifest.json
similarity index 98%
rename from gnomeos-3.6.json
rename to manifest.json
index 2526d23..edb9dcd 100644
--- a/gnomeos-3.6.json
+++ b/manifest.json
@@ -1,5 +1,5 @@
{
- "00ostbuild-manifest-version": 0,
+ "00ostbuild-manifest-version": 1,
"prefix": "gnomeos-3.6",
"architectures": ["i686"],
@@ -657,16 +657,7 @@
"config-opts": ["--disable-documentation",
"--with-soup-gnome"]},
- {"src": "gnome:gnome-ostree",
- "name": "gnome-ostree-integration",
- "config-opts": ["--disable-buildsystem",
- "--enable-system-integration"]},
-
- {"src": "gnome:gnome-ostree",
- "name": "gnome-ostree-buildsystem",
- "component": "devel",
- "config-opts": ["--enable-buildsystem",
- "--disable-system-integration"]},
+ {"src": "gnome:gnome-ostree-integration"},
{"src": "fd-gstreamer:gstreamer",
"name": "gstreamer-0.10",
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 71e8504..ded2729 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -244,16 +244,7 @@ class OstbuildBuild(builtins.Builtin):
if 'patches' in expanded_component:
patches_revision = expanded_component['patches']['revision']
- if self.args.patches_path:
- patchdir = self.args.patches_path
- elif self.cached_patchdir_revision == patches_revision:
- patchdir = self.patchdir
- else:
- patchdir = vcs.checkout_patches(self.mirrordir,
- self.patchdir,
- expanded_component,
- patches_path=self.args.patches_path)
- self.cached_patchdir_revision = patches_revision
+ patchdir = os.path.join(self.srcdir, 'patches')
if ((previous_metadata is not None) and
'patches' in previous_metadata and
previous_metadata['patches']['revision'] == patches_revision):
@@ -285,22 +276,7 @@ class OstbuildBuild(builtins.Builtin):
json.dump(expanded_component, f, indent=4, sort_keys=True)
f.close()
- checkoutdir = os.path.join(self.workdir, 'checkouts')
- component_src = os.path.join(checkoutdir, buildname)
- fileutil.ensure_parent_dir(component_src)
- child_args = ['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
- '--checkoutdir=' + component_src,
- '--metadata-path=' + temp_metadata_path,
- '--clean', '--overwrite', basename]
- if self.args.patches_path:
- child_args.append('--patches-path=' + self.args.patches_path)
- elif patchdir is not None:
- child_args.append('--patches-path=' + patchdir)
- run_sync(child_args)
-
- os.unlink(temp_metadata_path)
-
- artifact_meta = dict(component)
+ component_src = self._component_abspath(expanded_component)
logdir = os.path.join(self.workdir, 'logs', buildname)
fileutil.ensure_dir(logdir)
@@ -374,7 +350,7 @@ class OstbuildBuild(builtins.Builtin):
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--skip-if-unchanged']
- setuid_files = artifact_meta.get('setuid', [])
+ setuid_files = component.get('setuid', [])
statoverride_path = None
if len(setuid_files) > 0:
(fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
@@ -390,10 +366,6 @@ class OstbuildBuild(builtins.Builtin):
os.unlink(statoverride_path)
if not self.args.no_clean_results:
- if os.path.islink(component_src):
- os.unlink(component_src)
- else:
- shutil.rmtree(component_src)
shutil.rmtree(component_resultdir)
return run_sync_get_output(['ostree', '--repo=' + self.repo,
@@ -498,23 +470,16 @@ and the manifest input."""
def _build_base(self):
"""Build the Yocto base system."""
basemeta = self.snapshot['base']
- checkoutdir = os.path.join(self.workdir, 'checkouts', basemeta['name'])
- fileutil.ensure_parent_dir(checkoutdir)
-
- (keytype, uri) = buildutil.parse_src_key(basemeta['src'])
- vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
- basemeta['revision'],
- overwrite=False)
-
builddir = os.path.join(self.workdir, 'build-' + basemeta['name'])
image_deploy_dir = os.path.join(builddir, 'tmp-eglibc', 'deploy', 'images')
repo_link = os.path.join(image_deploy_dir, 'repo')
if not os.path.islink(repo_link):
os.symlink(self.repo, repo_link)
+ src = self._component_abspath(basemeta)
cmd = ['linux-user-chroot', '--unshare-pid', '/',
os.path.join(LIBDIR, 'ostbuild', 'ostree-build-yocto'),
- checkoutdir, builddir]
+ src, builddir]
# We specifically want to kill off any environment variables jhbuild
# may have set.
run_sync(cmd, env=buildutil.BUILD_ENV)
@@ -522,8 +487,6 @@ and the manifest input."""
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
- parser.add_argument('--src-snapshot')
- parser.add_argument('--patches-path')
parser.add_argument('--status-json-path',
help="Write data to this JSON file as build progresses")
parser.add_argument('--force-rebuild', action='store_true')
@@ -538,9 +501,7 @@ and the manifest input."""
self.args = args
self.parse_config()
- self.parse_snapshot(args.prefix, args.src_snapshot)
-
- log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
+ self.parse_snapshot()
self._write_status({'state': 'build-starting'})
@@ -551,8 +512,6 @@ and the manifest input."""
self.force_build_components = set()
- self.cached_patchdir_revision = None
-
self._initialize_repo()
self._build_base()
diff --git a/src/ostbuild/pyostbuild/builtin_fetch.py b/src/ostbuild/pyostbuild/builtin_fetch.py
new file mode 100755
index 0000000..63c994e
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_fetch.py
@@ -0,0 +1,110 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import copy
+import argparse
+import json
+import time
+import urlparse
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import ostbuildrc
+from . import vcs
+from . import jsondb
+from . import buildutil
+from . import kvfile
+from . import odict
+
+class OstbuildFetch(builtins.Builtin):
+ name = "fetch"
+ short_description = "Update components tracking git branches"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _fetch_submodule(self, component):
+ run_sync(['git', 'fetch'], cwd=self._component_path(component))
+
+ def _ensure_submodule(self, component):
+ component_path = self._component_path(component)
+ branch = component.get('branch')
+ tag = component.get('tag')
+ if (branch is None) and (tag is not None):
+ branch = 'master' # Hack - git submodule bails trying to check out non-branches
+ (keytype, uri) = vcs.parse_src_key(component['src'])
+ if not os.path.isdir(component_path):
+ run_sync(['git', 'submodule', 'add', '-b', branch, uri, component_path])
+
+ def _sync_submodule(self, component):
+ component_path = self._component_path(component)
+ branch = component.get('branch')
+ tag = component.get('tag')
+ tag_or_branch = tag or branch
+ log("Syncing %s to %s" % (component['name'], tag_or_branch))
+ run_sync(['git', 'checkout', '-q', tag_or_branch], cwd=component_path,
+ log_initiation=False)
+
+ def _clean_submodules(self):
+ all_components = {}
+ for component in self.snapshot['components']:
+ all_components[component['name']] = component
+ for name in os.listdir(self.componentdir):
+ fullpath = os.path.join(self.componentdir, name)
+ relpath = os.path.relpath(fullpath, self.srcdir)
+ if all_components.has_key(name):
+ continue
+ deleted_name = os.path.join(self.srcdir, 'deleted-components')
+ if os.path.isdir(deleted_name):
+ shutil.rmtree(deleted_name)
+ run_sync(['git', 'rm', '--cached', fullpath])
+ run_sync(['git', 'config', '-f', '.git/config', '--remove-section', 'submodule.' + relpath])
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--keep-going', action='store_true',
+ help="Don't exit on fetch failures")
+ parser.add_argument('components', nargs='*',
+ help="List of component names to git fetch")
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ self.parse_config()
+ self.parse_snapshot()
+
+ self._ensure_submodule(self.snapshot['base'])
+ self._sync_submodule(self.snapshot['base'])
+ for component in self.snapshot['components']:
+ self._ensure_submodule(component)
+ self._sync_submodule(component)
+
+ self._clean_submodules()
+
+ if len(args.components) > 0:
+ fetch_components = args.components
+ else:
+ fetch_components = map(lambda x: x['name'], self.snapshot['components'])
+ fetch_components.append(base_meta['name'])
+
+ for component in fetch_components:
+ self._fetch_submodule(component)
+
+builtins.register(OstbuildFetch)
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index a4d5cd4..ef1ae63 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -25,7 +25,7 @@ import json
from . import ostbuildrc
from . import fileutil
-from . import jsondb
+from . import buildutil
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
@@ -63,6 +63,14 @@ class Builtin(object):
else:
return (None, None)
+ def _component_path(self, component):
+ component_path = os.path.join(self.componentdir, component['name'])
+ return os.path.relpath(component_path, self.srcdir)
+
+ def _component_abspath(self, component):
+ component_path = os.path.join(self.componentdir, component['name'])
+ return os.path.abspath(component_path)
+
def get_component_from_cwd(self):
cwd = os.getcwd()
parent = os.path.dirname(cwd)
@@ -72,16 +80,10 @@ class Builtin(object):
def parse_config(self):
self.ostbuildrc = ostbuildrc
- self.mirrordir = os.path.expanduser(ostbuildrc.get_key('mirrordir'))
- if not os.path.isdir(self.mirrordir):
- fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
self.workdir = os.path.expanduser(ostbuildrc.get_key('workdir'))
if not os.path.isdir(self.workdir):
fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
- self.snapshot_dir = os.path.join(self.workdir, 'snapshots')
- self.patchdir = os.path.join(self.workdir, 'patches')
-
def get_component_snapshot(self, name):
found = False
for content in self.active_branch_contents['contents']:
@@ -136,35 +138,6 @@ class Builtin(object):
def get_expanded_component(self, name):
return self.expand_component(self.get_component(name))
- def get_prefix(self):
- if self.prefix is None:
- path = os.path.expanduser('~/.config/ostbuild-prefix')
- if not os.path.exists(path):
- fatal("No prefix set; use \"ostbuild prefix\" to set one")
- f = open(path)
- self.prefix = f.read().strip()
- f.close()
- return self.prefix
-
- def create_db(self, dbsuffix, prefix=None):
- if prefix is None:
- target_prefix = self.get_prefix()
- else:
- target_prefix = prefix
- name = '%s-%s' % (target_prefix, dbsuffix)
- fileutil.ensure_dir(self.snapshot_dir)
- return jsondb.JsonDB(self.snapshot_dir, prefix=name)
-
- def get_src_snapshot_db(self):
- if self._src_snapshots is None:
- self._src_snapshots = self.create_db('src-snapshot')
- return self._src_snapshots
-
- def get_bin_snapshot_db(self):
- if self._bin_snapshots is None:
- self._bin_snapshots = self.create_db('bin-snapshot')
- return self._bin_snapshots
-
def init_repo(self):
if self.repo is not None:
return self.repo
@@ -174,27 +147,27 @@ class Builtin(object):
else:
self.repo = os.path.join(self.workdir, 'repo')
- def parse_prefix(self, prefix):
- if prefix is not None:
- self.prefix = prefix
-
- def parse_snapshot(self, prefix, path):
- self.parse_prefix(prefix)
+ def parse_snapshot(self):
self.init_repo()
- if path is None:
- latest_path = self.get_src_snapshot_db().get_latest_path()
- if latest_path is None:
- raise Exception("No source snapshot found for prefix %r" % (self.prefix, ))
- self.snapshot_path = latest_path
- else:
- self.snapshot_path = path
+ cwd_path = os.path.join(os.getcwd(), 'manifest.json')
+ if not os.path.isfile(cwd_path):
+ raise Exception("No source snapshot found here; looking for %s" % (cwd_path, ))
+ self.snapshot_path = cwd_path
self.snapshot = json.load(open(self.snapshot_path))
key = '00ostbuild-manifest-version'
src_ver = self.snapshot[key]
- if src_ver != 0:
- fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, ))
- if self.prefix is None:
- self.prefix = self.snapshot['prefix']
+ if src_ver != 1:
+ fatal("Unhandled %s version \"%d\", expected 1" % (key, src_ver, ))
+ self.srcdir = os.getcwd()
+ self.prefix = self.snapshot['prefix']
+ self.componentdir = os.path.join(os.getcwd(), 'components')
+
+ components = map(lambda x: buildutil.resolve_component_meta(self.snapshot, x), self.snapshot['components'])
+ components = map(lambda x: self.expand_component(x), components)
+ self.snapshot['components'] = components
+ base_meta = buildutil.resolve_component_meta(self.snapshot, self.snapshot['base'])
+ self.snapshot['base'] = base_meta
+
def parse_snapshot_from_current(self):
if self.ostree_dir is None:
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 8a0bf27..0ab26e2 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -26,6 +26,7 @@ from . import builtin_build
from . import builtin_checkout
from . import builtin_deploy_root
from . import builtin_deploy_qemu
+from . import builtin_fetch
from . import builtin_git_mirror
from . import builtin_import_tree
from . import builtin_init
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]