GNOME.org
 

[gnome-ostree/wip/integrated-build: 2/4] Move all sources into git submodules

commit 606d66e0b3d978afa19020cd3b10f46e7b7bd52e
Author: Colin Walters <walters verbum org>
Date:   Fri Sep 21 17:05:08 2012 -0400

    Move all sources into git submodules
    
    Git submodules were designed to do exactly what we're doing.  While
    they have some major drawbacks, I think the benefits outweigh the
    drawbacks.
    
    For example, we can speak of "git revision a19e3 of gnome-ostree"
    which in turn points to exact revisions of every component. This makes
    it far easier to do coordinated changes; you can send a patch that
    touches three components, plus the manifest.

 Makefile-ostbuild.am                     |    1 +
 deleted-components/README                |    2 +
 gnomeos-3.6.json => manifest.json        |    2 +-
 src/ostbuild/pyostbuild/builtin_build.py |   51 ++------------
 src/ostbuild/pyostbuild/builtin_fetch.py |  110 ++++++++++++++++++++++++++++++
 src/ostbuild/pyostbuild/builtins.py      |   79 +++++++--------------
 src/ostbuild/pyostbuild/main.py          |    1 +
 7 files changed, 147 insertions(+), 99 deletions(-)
---
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..e69de29
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index b8035c4..246040c 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -39,6 +39,7 @@ pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/builtin_checkout.py	\
 	src/ostbuild/pyostbuild/builtin_deploy_qemu.py	\
 	src/ostbuild/pyostbuild/builtin_deploy_root.py	\
+	src/ostbuild/pyostbuild/builtin_fetch.py	\
 	src/ostbuild/pyostbuild/builtin_run_qemu.py	\
 	src/ostbuild/pyostbuild/builtin_import_tree.py	\
 	src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py	\
diff --git a/deleted-components/README b/deleted-components/README
new file mode 100644
index 0000000..0efb3f5
--- /dev/null
+++ b/deleted-components/README
@@ -0,0 +1,2 @@
+Components that used to exist but are no longer listed in the manifest
+are moved here for safety.
diff --git a/gnomeos-3.6.json b/manifest.json
similarity index 99%
rename from gnomeos-3.6.json
rename to manifest.json
index dea57a3..f97bed4 100644
--- a/gnomeos-3.6.json
+++ b/manifest.json
@@ -1,5 +1,5 @@
 {
-  "00ostbuild-manifest-version": 0,
+  "00ostbuild-manifest-version": 1,
 
   "prefix": "gnomeos-3.6",
   "architectures": ["i686"],
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 7737f00..8928892 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -245,16 +245,7 @@ class OstbuildBuild(builtins.Builtin):
 
         if 'patches' in expanded_component:
             patches_revision = expanded_component['patches']['revision']
-            if self.args.patches_path:
-                patchdir = self.args.patches_path
-            elif self.cached_patchdir_revision == patches_revision:
-                patchdir = self.patchdir
-            else:
-                patchdir = vcs.checkout_patches(self.mirrordir,
-                                                self.patchdir,
-                                                expanded_component,
-                                                patches_path=self.args.patches_path)
-                self.cached_patchdir_revision = patches_revision
+            patchdir = os.path.join(self.srcdir, 'patches')
             if ((previous_metadata is not None) and
                 'patches' in previous_metadata and
                 previous_metadata['patches']['revision'] == patches_revision):
@@ -286,22 +277,9 @@ class OstbuildBuild(builtins.Builtin):
         json.dump(expanded_component, f, indent=4, sort_keys=True)
         f.close()
 
-        checkoutdir = os.path.join(self.workdir, 'checkouts')
-        component_src = os.path.join(checkoutdir, buildname)
-        fileutil.ensure_parent_dir(component_src)
-        child_args = ['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
-                      '--checkoutdir=' + component_src,
-                      '--metadata-path=' + temp_metadata_path]
+        component_src = self._component_abspath(expanded_component)
         if not self.buildopts.no_clean:
-            child_args.append('--clean')
-        child_args.extend(['--overwrite', basename])
-        if self.args.patches_path:
-            child_args.append('--patches-path=' + self.args.patches_path)
-        elif patchdir is not None:
-            child_args.append('--patches-path=' + patchdir)
-        run_sync(child_args)
-
-        os.unlink(temp_metadata_path)
+            run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=component_src)
 
         logdir = os.path.join(self.workdir, 'logs', buildname)
         fileutil.ensure_dir(logdir)
@@ -392,10 +370,6 @@ class OstbuildBuild(builtins.Builtin):
             os.unlink(statoverride_path)
 
         if not self.args.no_clean_results:
-            if os.path.islink(component_src):
-                os.unlink(component_src)
-            else:
-                shutil.rmtree(component_src)
             shutil.rmtree(component_resultdir)
 
         return run_sync_get_output(['ostree', '--repo=' + self.repo,
@@ -500,23 +474,16 @@ and the manifest input."""
     def _build_base(self):
         """Build the Yocto base system."""
         basemeta = self.snapshot['base']
-        checkoutdir = os.path.join(self.workdir, 'checkouts', basemeta['name'])
-        fileutil.ensure_parent_dir(checkoutdir)
-
-        (keytype, uri) = buildutil.parse_src_key(basemeta['src'])
-        vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
-                             basemeta['revision'],
-                             overwrite=False)
-
         builddir = os.path.join(self.workdir, 'build-' + basemeta['name'])
         image_deploy_dir = os.path.join(builddir, 'tmp-eglibc', 'deploy', 'images')
         repo_link = os.path.join(image_deploy_dir, 'repo')
         if not os.path.islink(repo_link):
             os.symlink(self.repo, repo_link)
 
+        src = self._component_abspath(basemeta)
         cmd = ['linux-user-chroot', '--unshare-pid', '/',
                os.path.join(LIBDIR, 'ostbuild', 'ostree-build-yocto'),
-               checkoutdir, builddir]
+               src, builddir]
         # We specifically want to kill off any environment variables jhbuild
         # may have set.
         run_sync(cmd, env=buildutil.BUILD_ENV)
@@ -524,8 +491,6 @@ and the manifest input."""
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--prefix')
-        parser.add_argument('--src-snapshot')
-        parser.add_argument('--patches-path')
         parser.add_argument('--status-json-path',
                             help="Write data to this JSON file as build progresses")
         parser.add_argument('--force-rebuild', action='store_true')
@@ -541,9 +506,7 @@ and the manifest input."""
         self.args = args
         
         self.parse_config()
-        self.parse_snapshot(args.prefix, args.src_snapshot)
-
-        log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
+        self.parse_snapshot()
 
         self._write_status({'state': 'build-starting'})
 
@@ -555,8 +518,6 @@ and the manifest input."""
 
         self.force_build_components = set()
 
-        self.cached_patchdir_revision = None
-
         self._initialize_repo()
         self._build_base()
 
diff --git a/src/ostbuild/pyostbuild/builtin_fetch.py b/src/ostbuild/pyostbuild/builtin_fetch.py
new file mode 100755
index 0000000..4c39a6d
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_fetch.py
@@ -0,0 +1,110 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import copy
+import argparse
+import json
+import time
+import urlparse
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import ostbuildrc
+from . import vcs
+from . import jsondb
+from . import buildutil
+from . import kvfile
+from . import odict
+
+class OstbuildFetch(builtins.Builtin):
+    name = "fetch"
+    short_description = "Update components tracking git branches" 
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def _fetch_submodule(self, component):
+        run_sync(['git', 'fetch'], cwd=self._component_path(component))
+
+    def _ensure_submodule(self, component):
+        component_path = self._component_path(component)
+        branch = component.get('branch')
+        tag = component.get('tag')
+        if (branch is None) and (tag is not None):
+            branch = 'master' # Hack - git submodule bails trying to check out non-branches
+        (keytype, uri) = vcs.parse_src_key(component['src'])
+        if not os.path.isdir(component_path):
+            run_sync(['git', 'submodule', 'add', '-b', branch, uri, component_path])
+
+    def _sync_submodule(self, component):
+        component_path = self._component_path(component)
+        branch = component.get('branch')
+        tag = component.get('tag')
+        tag_or_branch = tag or branch
+        log("Syncing %s to %s" % (component['name'], tag_or_branch))
+        run_sync(['git', 'checkout', '-q', tag_or_branch], cwd=component_path,
+                 log_initiation=False)
+
+    def _clean_submodules(self):
+        all_components = {self.snapshot['base']['name']: self.snapshot['base']}
+        for component in self.snapshot['components']:
+            all_components[component['name']] = component
+        for name in os.listdir(self.componentdir):
+            fullpath = os.path.join(self.componentdir, name)
+            relpath = os.path.relpath(fullpath, self.srcdir)
+            if all_components.has_key(name):
+                continue
+            deleted_name = os.path.join(self.srcdir, 'deleted-components')
+            if os.path.isdir(deleted_name):
+                shutil.rmtree(deleted_name)
+            run_sync(['git', 'rm', '--cached', fullpath])
+            run_sync(['git', 'config', '-f', '.git/config', '--remove-section', 'submodule.' + relpath])
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('--keep-going', action='store_true',
+                            help="Don't exit on fetch failures")
+        parser.add_argument('components', nargs='*',
+                            help="List of component names to git fetch")
+
+        args = parser.parse_args(argv)
+        self.args = args
+
+        self.parse_config()
+        self.parse_snapshot()
+
+        self._ensure_submodule(self.snapshot['base'])
+        self._sync_submodule(self.snapshot['base'])
+        for component in self.snapshot['components']:
+            self._ensure_submodule(component)
+            self._sync_submodule(component)
+
+        self._clean_submodules()
+
+        if len(args.components) > 0:
+            fetch_components = args.components
+        else:
+            fetch_components = map(lambda x: x['name'], self.snapshot['components'])
+            fetch_components.append(self.snapshot['base']['name'])
+         
+        for component_name in fetch_components:
+            self._fetch_submodule(self.get_component(component_name))
+        
+builtins.register(OstbuildFetch)
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index a4d5cd4..ef1ae63 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -25,7 +25,7 @@ import json
 
 from . import ostbuildrc
 from . import fileutil
-from . import jsondb
+from . import buildutil
 from .ostbuildlog import log, fatal
 from .subprocess_helpers import run_sync, run_sync_get_output
 
@@ -63,6 +63,14 @@ class Builtin(object):
         else:
             return (None, None)
 
+    def _component_path(self, component):
+        component_path = os.path.join(self.componentdir, component['name'])
+        return os.path.relpath(component_path, self.srcdir)
+
+    def _component_abspath(self, component):
+        component_path = os.path.join(self.componentdir, component['name'])
+        return os.path.abspath(component_path)
+
     def get_component_from_cwd(self):
         cwd = os.getcwd()
         parent = os.path.dirname(cwd)
@@ -72,16 +80,10 @@ class Builtin(object):
     def parse_config(self):
         self.ostbuildrc = ostbuildrc
 
-        self.mirrordir = os.path.expanduser(ostbuildrc.get_key('mirrordir'))
-        if not os.path.isdir(self.mirrordir):
-            fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
         self.workdir = os.path.expanduser(ostbuildrc.get_key('workdir'))
         if not os.path.isdir(self.workdir):
             fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
 
-        self.snapshot_dir = os.path.join(self.workdir, 'snapshots')
-        self.patchdir = os.path.join(self.workdir, 'patches')
-
     def get_component_snapshot(self, name):
         found = False
         for content in self.active_branch_contents['contents']:
@@ -136,35 +138,6 @@ class Builtin(object):
     def get_expanded_component(self, name):
         return self.expand_component(self.get_component(name))
 
-    def get_prefix(self):
-        if self.prefix is None:
-            path = os.path.expanduser('~/.config/ostbuild-prefix')
-            if not os.path.exists(path):
-                fatal("No prefix set; use \"ostbuild prefix\" to set one")
-            f = open(path)
-            self.prefix = f.read().strip()
-            f.close()
-        return self.prefix
-
-    def create_db(self, dbsuffix, prefix=None):
-        if prefix is None:
-            target_prefix = self.get_prefix()
-        else:
-            target_prefix = prefix
-        name = '%s-%s' % (target_prefix, dbsuffix)
-        fileutil.ensure_dir(self.snapshot_dir)
-        return jsondb.JsonDB(self.snapshot_dir, prefix=name)
-
-    def get_src_snapshot_db(self):
-        if self._src_snapshots is None:
-            self._src_snapshots = self.create_db('src-snapshot')
-        return self._src_snapshots
-
-    def get_bin_snapshot_db(self):
-        if self._bin_snapshots is None:
-            self._bin_snapshots = self.create_db('bin-snapshot')
-        return self._bin_snapshots
-
     def init_repo(self):
         if self.repo is not None:
             return self.repo
@@ -174,27 +147,27 @@ class Builtin(object):
         else:
             self.repo = os.path.join(self.workdir, 'repo')
 
-    def parse_prefix(self, prefix):
-        if prefix is not None:
-            self.prefix = prefix
-
-    def parse_snapshot(self, prefix, path):
-        self.parse_prefix(prefix)
+    def parse_snapshot(self):
         self.init_repo()
-        if path is None:
-            latest_path = self.get_src_snapshot_db().get_latest_path()
-            if latest_path is None:
-                raise Exception("No source snapshot found for prefix %r" % (self.prefix, ))
-            self.snapshot_path = latest_path
-        else:
-            self.snapshot_path = path
+        cwd_path = os.path.join(os.getcwd(), 'manifest.json')
+        if not os.path.isfile(cwd_path):
+            raise Exception("No source snapshot found here; looking for %s" % (cwd_path, ))
+        self.snapshot_path = cwd_path
         self.snapshot = json.load(open(self.snapshot_path))
         key = '00ostbuild-manifest-version'
         src_ver = self.snapshot[key]
-        if src_ver != 0:
-            fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, ))
-        if self.prefix is None:
-            self.prefix = self.snapshot['prefix']
+        if src_ver != 1:
+            fatal("Unhandled %s version \"%d\", expected 1" % (key, src_ver, ))
+        self.srcdir = os.getcwd()
+        self.prefix = self.snapshot['prefix']
+        self.componentdir = os.path.join(os.getcwd(), 'components')
+
+        components = map(lambda x: buildutil.resolve_component_meta(self.snapshot, x), self.snapshot['components'])
+        components = map(lambda x: self.expand_component(x), components)
+        self.snapshot['components'] = components
+        base_meta = buildutil.resolve_component_meta(self.snapshot, self.snapshot['base'])
+        self.snapshot['base'] = base_meta
+
 
     def parse_snapshot_from_current(self):
         if self.ostree_dir is None:
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 8a0bf27..0ab26e2 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -26,6 +26,7 @@ from . import builtin_build
 from . import builtin_checkout
 from . import builtin_deploy_root
 from . import builtin_deploy_qemu
+from . import builtin_fetch
 from . import builtin_git_mirror
 from . import builtin_import_tree
 from . import builtin_init
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]