GNOME.org
 

[ostree/wip/ostbuild-v3: 9/10] ostbuild: Add import-tree

commit be5acd7b7c4ab654d85cd58f44397ac5a73d1ed4
Author: Colin Walters <walters verbum org>
Date:   Mon May 14 20:47:14 2012 -0400

    ostbuild: Add import-tree
    
    This is the new workflow:
    
    $ ostbuild import-tree wip-gnome-menus-3.6
    $ $EDITOR ~/build/ostbuild/snapshots/wip-gnome-menus-3.6-2012.1-abcd.json
    $ ostbuild build

 src/ostbuild/pyostbuild/builtin_build.py           |  104 +++++++++++++-------
 .../pyostbuild/builtin_chroot_compile_one.py       |   12 +-
 src/ostbuild/pyostbuild/builtin_import_tree.py     |   50 ++++++++--
 src/ostbuild/pyostbuild/builtins.py                |   50 +++-------
 4 files changed, 128 insertions(+), 88 deletions(-)
---
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 1dd69dc..b4b932b 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -66,8 +66,8 @@ class OstbuildBuild(builtins.Builtin):
         basename = component['name']
         branch = component['branch']
 
-        archname = '%s/%s' % (basename, architecture)
-        buildname = 'components/%s' % (archname, )
+        buildname = '%s/%s/%s' % (self.snapshot['prefix'], basename, architecture)
+        build_ref = 'components/%s' % (buildname, )
 
         current_vcs_version = component.get('revision')
 
@@ -86,12 +86,12 @@ class OstbuildBuild(builtins.Builtin):
             previous_build_version = None
         else:
             previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                          'rev-parse', buildname],
+                                                          'rev-parse', build_ref],
                                                          stderr=open('/dev/null', 'w'),
                                                          none_on_error=True)
         if (current_vcs_version is not None
             and previous_build_version is not None):
-            log("Previous build of '%s' is %s" % (archname, previous_build_version))
+            log("Previous build of '%s' is %s" % (buildname, previous_build_version))
 
             previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                                           'cat', previous_build_version,
@@ -118,10 +118,10 @@ class OstbuildBuild(builtins.Builtin):
                 else:
                     log("Metadata differs; note vcs version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
         else:
-            log("No previous build for '%s' found" % (archname, ))
+            log("No previous build for '%s' found" % (buildname, ))
 
         checkoutdir = os.path.join(self.workdir, 'checkouts')
-        component_src = os.path.join(checkoutdir, archname)
+        component_src = os.path.join(checkoutdir, buildname)
         fileutil.ensure_parent_dir(component_src)
         run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
                   '--checkoutdir=' + component_src,
@@ -129,7 +129,7 @@ class OstbuildBuild(builtins.Builtin):
 
         artifact_meta = dict(component)
 
-        logdir = os.path.join(self.workdir, 'logs', archname)
+        logdir = os.path.join(self.workdir, 'logs', buildname)
         fileutil.ensure_dir(logdir)
         log_path = os.path.join(logdir, 'compile.log')
         if os.path.isfile(log_path):
@@ -137,10 +137,16 @@ class OstbuildBuild(builtins.Builtin):
             saved_name = os.path.join(logdir, 'compile-prev.log')
             os.rename(log_path, saved_name)
 
+        component_resultdir = os.path.join(self.workdir, 'results', buildname)
+        if os.path.isdir(component_resultdir):
+            shutil.rmtree(component_resultdir)
+        fileutil.ensure_dir(component_resultdir)
+
         log("Logging to %s" % (log_path, ))
         f = open(log_path, 'w')
         chroot_args = self._get_ostbuild_chroot_args(architecture)
-        chroot_args.extend(['--name=' + basename, '--arch=' + architecture])
+        chroot_args.extend(['--name=' + basename, '--arch=' + architecture,
+                            '--resultdir=' + component_resultdir])
         if self.buildopts.shell_on_failure:
             ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
             if ecode != 0:
@@ -149,7 +155,7 @@ class OstbuildBuild(builtins.Builtin):
             run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src)
 
         args = ['ostree', '--repo=' + self.repo,
-                'commit', '-b', buildname, '-s', 'Build',
+                'commit', '-b', build_ref, '-s', 'Build',
                 '--owner-uid=0', '--owner-gid=0', '--no-xattrs', 
                 '--skip-if-unchanged']
 
@@ -163,50 +169,66 @@ class OstbuildBuild(builtins.Builtin):
             f.close()
             args.append('--statoverride=' + statoverride_path)
 
-        component_resultdir = os.path.join(self.workdir, 'results', archname)
-            
         run_sync(args, cwd=component_resultdir)
         if statoverride_path is not None:
             os.unlink(statoverride_path)
 
+        if os.path.islink(component_src):
+            os.unlink(component_src)
+        else:
+            shutil.rmtree(component_src)
+        shutil.rmtree(component_resultdir)
+
         return run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                    'rev-parse', buildname])
+                                    'rev-parse', build_ref])
 
     def _compose_one_target(self, target, component_build_revs):
         base = target['base']
         base_name = 'bases/%s' % (base['name'], )
+        runtime_name = 'bases/%s' % (base['runtime'], )
+        devel_name = 'bases/%s' % (base['devel'], )
 
         compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
         if os.path.isdir(compose_rootdir):
             shutil.rmtree(compose_rootdir)
         os.mkdir(compose_rootdir)
 
+        related_refs = {}
+
         base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                              'rev-parse', base_name])
 
-        compose_contents = [(base_name, base_revision, '/')]
-        for tree_content in target['contents']:
-            name = tree_content['name']
-            rev = component_build_revs[name]
-            subtrees = tree_content['trees']
-            for subpath in subtrees:
-                compose_contents.append((name, rev, subpath))
+        runtime_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                                'rev-parse', runtime_name])
+        related_refs[runtime_name] = runtime_revision
+        devel_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                              'rev-parse', devel_name])
+        related_refs[devel_name] = devel_revision
+
+        for name,rev in component_build_revs.iteritems():
+            build_ref = 'components/%s/%s' % (self.snapshot['prefix'], name)
+            related_refs[build_ref] = rev
 
         (related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
         related_f = os.fdopen(related_fd, 'w')
-        resolved_refs = {}
-        for (name, branch, subpath) in compose_contents:
-            resolved_refs[name] = branch
-        for (name, rev) in resolved_refs.iteritems():
+        for (name, rev) in related_refs.iteritems():
             related_f.write(name) 
             related_f.write(' ') 
             related_f.write(rev) 
             related_f.write('\n') 
         related_f.close()
 
+        compose_contents = [(base_revision, '/')]
+        for tree_content in target['contents']:
+            name = tree_content['name']
+            rev = component_build_revs[name]
+            subtrees = tree_content['trees']
+            for subpath in subtrees:
+                compose_contents.append((rev, subpath))
+
         (contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
         contents_f = os.fdopen(contents_fd, 'w')
-        for (name, branch, subpath) in compose_contents:
+        for (branch, subpath) in compose_contents:
             contents_f.write(branch)
             contents_f.write('\0')
             contents_f.write(subpath)
@@ -224,12 +246,15 @@ class OstbuildBuild(builtins.Builtin):
         f.close()
 
         treename = 'trees/%s' % (target['name'], )
-
-        run_sync(['ostree', '--repo=' + self.repo,
-                  'commit', '-b', treename, '-s', 'Compose',
-                  '--owner-uid=0', '--owner-gid=0', '--no-xattrs', 
-                  '--related-objects-file=' + related_tmppath,
-                  '--skip-if-unchanged'], cwd=compose_rootdir)
+        
+        child_args = ['ostree', '--repo=' + self.repo,
+                      'commit', '-b', treename, '-s', 'Compose',
+                      '--owner-uid=0', '--owner-gid=0', '--no-xattrs', 
+                      '--related-objects-file=' + related_tmppath,
+                      ]
+        if not self.buildopts.no_skip_if_unchanged:
+            child_args.append('--skip-if-unchanged')
+        run_sync(child_args, cwd=compose_rootdir)
         os.unlink(related_tmppath)
         shutil.rmtree(compose_rootdir)
 
@@ -240,6 +265,7 @@ class OstbuildBuild(builtins.Builtin):
         parser.add_argument('--force-rebuild', action='store_true')
         parser.add_argument('--skip-vcs-matches', action='store_true')
         parser.add_argument('--no-compose', action='store_true')
+        parser.add_argument('--no-skip-if-unchanged', action='store_true')
         parser.add_argument('--compose-only', action='store_true')
         parser.add_argument('--shell-on-failure', action='store_true')
         parser.add_argument('--debug-shell', action='store_true')
@@ -257,6 +283,7 @@ class OstbuildBuild(builtins.Builtin):
         self.buildopts.shell_on_failure = args.shell_on_failure
         self.buildopts.force_rebuild = args.force_rebuild
         self.buildopts.skip_vcs_matches = args.skip_vcs_matches
+        self.buildopts.no_skip_if_unchanged = args.no_skip_if_unchanged
 
         self.force_build_components = set()
 
@@ -290,10 +317,8 @@ class OstbuildBuild(builtins.Builtin):
             component_to_arches[name] = component_arches
 
         for name in args.components:
-            component = components.get(name)
-            if component is None:
-                fatal("Unknown component %r" % (name, ))
-            self.force_build_components.add(name)
+            component = self.get_component(name)
+            self.force_build_components.add(component['name'])
 
         components_to_build = []
         component_skipped_count = 0
@@ -318,8 +343,15 @@ class OstbuildBuild(builtins.Builtin):
                 targets_list.append(target)
                 target['name'] = '%s-%s-%s' % (prefix, architecture, target_component_type)
 
-                base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
-                target['base'] = {'name': base_ref}
+                runtime_ref = '%s-%s-runtime' % (base_prefix, architecture)
+                buildroot_ref = '%s-%s-devel' % (base_prefix, architecture)
+                if target_component_type == 'runtime':
+                    base_ref = runtime_ref
+                else:
+                    base_ref = buildroot_ref
+                target['base'] = {'name': base_ref,
+                                  'runtime': runtime_ref,
+                                  'devel': buildroot_ref}
 
                 if target_component_type == 'runtime':
                     target_components = runtime_components
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index 9edab0c..9ff1122 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -63,8 +63,10 @@ class OstbuildChrootCompileOne(builtins.Builtin):
 
         ref_to_rev = {}
 
+        prefix = self.snapshot['prefix']
+
         arch_buildroot_name = 'bases/%s/%s-%s-devel' % (self.snapshot['base']['name'],
-                                                        self.snapshot['prefix'],
+                                                        prefix,
                                                         architecture)
 
         arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse',
@@ -74,7 +76,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         checkout_trees = [(arch_buildroot_name, '/')]
         refs_to_resolve = []
         for dependency in build_dependencies:
-            buildname = 'components/%s/%s' % (dependency['name'], architecture)
+            buildname = 'components/%s/%s/%s' % (prefix, dependency['name'], architecture)
             refs_to_resolve.append(buildname)
             checkout_trees.append((buildname, '/runtime'))
             checkout_trees.append((buildname, '/devel'))
@@ -152,6 +154,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         parser.add_argument('--prefix')
         parser.add_argument('--snapshot', required=True)
         parser.add_argument('--name')
+        parser.add_argument('--resultdir')
         parser.add_argument('--arch', required=True)
         parser.add_argument('--debug-shell', action='store_true')
         
@@ -177,10 +180,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
             shutil.rmtree(child_tmpdir)
         fileutil.ensure_dir(child_tmpdir)
 
-        resultdir = os.path.join(self.workdir, 'results', component_name, args.arch)
-        if os.path.isdir(resultdir):
-            shutil.rmtree(resultdir)
-        fileutil.ensure_dir(resultdir)
+        resultdir = args.resultdir
         
         rootdir = self._compose_buildroot(component_name, args.arch)
 
diff --git a/src/ostbuild/pyostbuild/builtin_import_tree.py b/src/ostbuild/pyostbuild/builtin_import_tree.py
index b24b0ae..e83d59a 100755
--- a/src/ostbuild/pyostbuild/builtin_import_tree.py
+++ b/src/ostbuild/pyostbuild/builtin_import_tree.py
@@ -30,7 +30,7 @@ from . import buildutil
 
 class OstbuildImportTree(builtins.Builtin):
     name = "import-tree"
-    short_description = "Extract source data from tree"
+    short_description = "Extract source data from tree into new prefix"
 
     def __init__(self):
         builtins.Builtin.__init__(self)
@@ -45,22 +45,52 @@ class OstbuildImportTree(builtins.Builtin):
 
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
-        parser.add_argument('--prefix')
         parser.add_argument('--tree')
+        parser.add_argument('new_prefix')
 
         args = parser.parse_args(argv)
         self.parse_config()
-        if args.prefix:
-            self.prefix = args.prefix
+        self.parse_snapshot_from_current()
 
-        if args.tree:
-            self.load_bin_snapshot_from_path(args.tree)
-        else:
-            self.load_bin_snapshot_from_current()
+        log("Loading source from tree %r" % (self.snapshot_path, ))
+
+        related_objects = run_sync_get_output(['ostree', '--repo='+ self.repo,
+                                               'show', '--print-related',
+                                               self.active_branch_checksum])
+        ref_to_revision = {}
+        for line in StringIO(related_objects):
+            line = line.strip()
+            (ref, revision) = line.split(' ', 1)
+            ref_to_revision[ref] = revision
+
+        (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-import-tree-')
+        f = os.fdopen(fd, 'w')
+        for (ref, rev) in ref_to_revision.iteritems():
+            if ref.startswith('components/'):
+                ref = ref[len('components/'):]
+                (prefix, subref) = ref.split('/', 1)
+                newref = 'components/%s/%s' % (args.new_prefix, subref)
+            elif ref.startswith('bases/'):
+                # hack
+                base_key = '/' + self.snapshot['prefix'] + '-'
+                replace_key = '/' + args.new_prefix + '-'
+                newref = ref.replace(base_key, replace_key)
+            else:
+                fatal("Unhandled ref %r; expected components/ or bases/" % (ref, ))
+                
+            f.write('%s %s\n' % (newref, rev))
+        f.close()
+
+        run_sync(['ostree', '--repo=' + self.repo,
+                  'write-refs'], stdin=open(tmppath))
+
+        self.snapshot['prefix'] = args.new_prefix
+
+        run_sync(['ostbuild', 'prefix', args.new_prefix])
+        self.prefix = args.new_prefix
 
-        snapshot = self.bin_snapshot_to_src(self.bin_snapshot)
         db = self.get_src_snapshot_db()
-        path = db.store(snapshot)
+        path = db.store(self.snapshot)
         log("Source snapshot: %s" % (path, ))
 
 builtins.register(OstbuildImportTree)
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index 7d6dfb4..62a20b3 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -92,27 +92,6 @@ class Builtin(object):
         self.snapshot_dir = os.path.join(self.workdir, 'snapshots')
         self.patchdir = os.path.join(self.workdir, 'patches')
 
-    def load_bin_snapshot_from_path(self, path):
-        self.bin_snapshot_path = os.path.join(path, 'ostree-meta', 'contents.json')
-        if not os.path.exists(self.bin_snapshot_path):
-            self.bin_snapshot_path = os.path.join(path, 'contents.json')
-        self.bin_snapshot = json.load(open(self.bin_snapshot_path))
-        bin_ver = self.bin_snapshot['00ostree-bin-snapshot-version']
-        if bin_ver != 1:
-            fatal("Unhandled 00ostree-bin-snapshot-version %r, expected 1" % (bin_ver, ))
-
-    def load_bin_snapshot_from_current(self):
-        if self.ostree_dir is None:
-            fatal("/ostree directory not found")
-        repo_path = os.path.join(self.ostree_dir, 'repo')
-        if not os.path.isdir(repo_path):
-            fatal("Repository '%s' doesn't exist" % (repo_path, ))
-        self.repo = repo_path
-        if self.active_branch is None:
-            fatal("No \"current\" link found")
-        tree_path = os.path.join(self.ostree_dir, self.active_branch)
-        self.load_bin_snapshot_from_path(tree_path)
-
     def get_component_snapshot(self, name):
         found = False
         for content in self.active_branch_contents['contents']:
@@ -138,7 +117,7 @@ class Builtin(object):
                 patches = dict(global_patchmeta)
                 patches['files'] = component_patch_files
                 meta['patches'] = patches
-        config_opts = self.snapshot.get('config-opts', [])
+        config_opts = list(self.snapshot.get('config-opts', []))
         config_opts.extend(component.get('config-opts', []))
         meta['config-opts'] = config_opts
         return meta
@@ -213,21 +192,20 @@ class Builtin(object):
         src_ver = self.snapshot[key]
         if src_ver != 0:
             fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, ))
+        if self.prefix is None:
+            self.prefix = self.snapshot['prefix']
 
-    def parse_bin_snapshot(self, prefix, path):
-        self.parse_prefix(prefix)
-        self._init_repo()
-        if path is None:
-            latest_path = self.get_bin_snapshot_db().get_latest_path()
-            if latest_path is None:
-                raise Exception("No binary snapshot found for prefix %r" % (self.prefix, ))
-            self.bin_snapshot_path = latest_path
-        else:
-            self.bin_snapshot_path = path
-        self.bin_snapshot = json.load(open(self.bin_snapshot_path))
-        bin_ver = self.bin_snapshot['00ostree-bin-snapshot-version']
-        if bin_ver != 1:
-            fatal("Unhandled 00ostree-bin-snapshot-version %r, expected 1" % (bin_ver, ))
+    def parse_snapshot_from_current(self):
+        if self.ostree_dir is None:
+            fatal("/ostree directory not found")
+        repo_path = os.path.join(self.ostree_dir, 'repo')
+        if not os.path.isdir(repo_path):
+            fatal("Repository '%s' doesn't exist" % (repo_path, ))
+        self.repo = repo_path
+        if self.active_branch is None:
+            fatal("No \"current\" link found")
+        tree_path = os.path.join(self.ostree_dir, self.active_branch)
+        self.parse_snapshot(None, os.path.join(tree_path, 'contents.json'))
 
     def execute(self, args):
         raise NotImplementedError()
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]