[Notes] [Git][BuildStream/buildstream][phil/source-checkout-options] 6 c

Phil Dawson pushed to branch phil/source-checkout-options at BuildStream / buildstream

Commits:

b0290026

by Phil Dawson at 2018-12-06T16:12:41Z

Add --tar option to source-checkout command

This commit is part of the work towards #672

ee96f007

by Phil Dawson at 2018-12-06T16:12:41Z

Add --force / -f option to source-checkout command

7a3c8353

by Phil Dawson at 2018-12-06T16:12:41Z

Add option to source-checkout command to generate build scripts

051042bf

by Phil Dawson at 2018-12-06T16:12:42Z

Remove source bundle command

This is part of the work towards #672

29d2b8a3

by Phil Dawson at 2018-12-06T16:12:42Z

NEWS: Add entry for new source-checkout command

ff9458c7

by Phil Dawson at 2018-12-06T16:12:42Z

NEWS: Add entry about the removal of source-bundle command

Changes:

NEWS

@@ -2,6 +2,17 @@
  buildstream 1.3.1
  =================
 +  o BREAKING CHANGE: The bst source-bundle command has been removed. The
 +    functionality it provided has been replaced by the `--include-build-scripts`
 +    option of the `bst source-checkout` command. To produce a tarball containing
 +    an element's sources and generated build scripts you can do the command
 +    `bst source-checkout --include-build-scripts --tar foo.bst some-file.tar`
++
 +  o A `bst source-checkout` command has been added. This command allows an
 +    element's sources to be checkout out into a user specified location. For
 +    example, `bst source-checkout foo.bst some/path/` will collect the sources
 +    of `foo.bst` and place them in the directory `some/path`.
++
    o All elements must now be suffixed with `.bst`
      Attempting to use an element that does not have the `.bst` extension,
      will result in a warning.

buildstream/_frontend/cli.py

@@ -684,6 +684,8 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
  #                  Source Checkout Command                      #
  ##################################################################
  @cli.command(name='source-checkout', short_help='Checkout sources for an element')
 +@click.option('--force', '-f', default=False, is_flag=True,
 +              help="Allow files to be overwritten")
  @click.option('--except', 'except_', multiple=True,
                type=click.Path(readable=False),
                help="Except certain dependencies")
@@ -692,19 +694,26 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
                help='The dependencies whose sources to checkout (default: none)')
  @click.option('--fetch', 'fetch_', default=False, is_flag=True,
                help='Fetch elements if they are not fetched')
 -@click.argument('element',
 -                type=click.Path(readable=False))
 +@click.option('--tar', 'tar', default=False, is_flag=True,
 +              help='Create a tarball from the element\'s sources instead of a '
 +                   'file tree.')
 +@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
 +@click.argument('element', type=click.Path(readable=False))
  @click.argument('location', type=click.Path())
  @click.pass_obj
 -def source_checkout(app, element, location, deps, fetch_, except_):
 +def source_checkout(app, element, location, force, deps, fetch_, except_,
 +                    tar, build_scripts):
      """Checkout sources of an element to the specified location
      """
      with app.initialized():
          app.stream.source_checkout(element,
                                     location=location,
 +                                   force=force,
                                     deps=deps,
                                     fetch=fetch_,
 -                                   except_targets=except_)
 +                                   except_targets=except_,
 +                                   tar=tar,
 +                                   include_build_scripts=build_scripts)
  ##################################################################
@@ -835,34 +844,3 @@ def workspace_list(app):
      with app.initialized():
          app.stream.workspace_list()
+-
+-
 -##################################################################
 -#                     Source Bundle Command                      #
 -##################################################################
 -@cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
 -@click.option('--except', 'except_', multiple=True,
 -              type=click.Path(readable=False),
 -              help="Elements to except from the tarball")
 -@click.option('--compression', default='gz',
 -              type=click.Choice(['none', 'gz', 'bz2', 'xz']),
 -              help="Compress the tar file using the given algorithm.")
 -@click.option('--track', 'track_', default=False, is_flag=True,
 -              help="Track new source references before bundling")
 -@click.option('--force', '-f', default=False, is_flag=True,
 -              help="Overwrite an existing tarball")
 -@click.option('--directory', default=os.getcwd(),
 -              help="The directory to write the tarball to")
 -@click.argument('element',
 -                type=click.Path(readable=False))
 -@click.pass_obj
 -def source_bundle(app, element, force, directory,
 -                  track_, compression, except_):
 -    """Produce a source bundle to be manually executed
 -    """
 -    with app.initialized():
 -        app.stream.source_bundle(element, directory,
 -                                 track_first=track_,
 -                                 force=force,
 -                                 compression=compression,
 -                                 except_targets=except_)

buildstream/_stream.py

@@ -25,8 +25,9 @@ import stat
  import shlex
  import shutil
  import tarfile
 -from contextlib import contextmanager
 -from tempfile import TemporaryDirectory
 +import tempfile
 +from contextlib import contextmanager, suppress
 +from pathlib import Path
  from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
  from ._message import Message, MessageType
@@ -449,11 +450,14 @@ class Stream():
+     #
      def source_checkout(self, target, *,
                          location=None,
 +                        force=False,
                          deps='none',
                          fetch=False,
 -                        except_targets=()):
 +                        except_targets=(),
 +                        tar=False,
 +                        include_build_scripts=False):
 -        self._check_location_writable(location)
 +        self._check_location_writable(location, force=force, tar=tar)
          elements, _ = self._load((target,), (),
                                   selection=deps,
@@ -467,7 +471,8 @@ class Stream():
          # Stage all sources determined by scope
          try:
 -            self._write_element_sources(location, elements)
 +            self._source_checkout(elements, location, force, deps,
 +                                  fetch, tar, include_build_scripts)
          except BstError as e:
              raise StreamError("Error while writing sources"
                                ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
@@ -724,87 +729,6 @@ class Stream():
              'workspaces': workspaces
          })
 -    # source_bundle()
 -    #
 -    # Create a host buildable tarball bundle for the given target.
 -    #
 -    # Args:
 -    #    target (str): The target element to bundle
 -    #    directory (str): The directory to output the tarball
 -    #    track_first (bool): Track new source references before bundling
 -    #    compression (str): The compression type to use
 -    #    force (bool): Overwrite an existing tarball
 -    #
 -    def source_bundle(self, target, directory, *,
 -                      track_first=False,
 -                      force=False,
 -                      compression="gz",
 -                      except_targets=()):
+-
 -        if track_first:
 -            track_targets = (target,)
 -        else:
 -            track_targets = ()
+-
 -        elements, track_elements = self._load((target,), track_targets,
 -                                              selection=PipelineSelection.ALL,
 -                                              except_targets=except_targets,
 -                                              track_selection=PipelineSelection.ALL,
 -                                              fetch_subprojects=True)
+-
 -        # source-bundle only supports one target
 -        target = self.targets[0]
+-
 -        self._message(MessageType.INFO, "Bundling sources for target {}".format(target.name))
+-
 -        # Find the correct filename for the compression algorithm
 -        tar_location = os.path.join(directory, target.normal_name + ".tar")
 -        if compression != "none":
 -            tar_location += "." + compression
+-
 -        # Attempt writing a file to generate a good error message
 -        # early
 -        #
 -        # FIXME: A bit hackish
 -        try:
 -            open(tar_location, mode="x")
 -            os.remove(tar_location)
 -        except IOError as e:
 -            raise StreamError("Cannot write to {0}: {1}"
 -                              .format(tar_location, e)) from e
+-
 -        # Fetch and possibly track first
 -        #
 -        self._fetch(elements, track_elements=track_elements)
+-
 -        # We don't use the scheduler for this as it is almost entirely IO
 -        # bound.
+-
 -        # Create a temporary directory to build the source tree in
 -        builddir = self._context.builddir
 -        os.makedirs(builddir, exist_ok=True)
 -        prefix = "{}-".format(target.normal_name)
+-
 -        with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir:
 -            source_directory = os.path.join(tempdir, 'source')
 -            try:
 -                os.makedirs(source_directory)
 -            except OSError as e:
 -                raise StreamError("Failed to create directory: {}"
 -                                  .format(e)) from e
+-
 -            # Any elements that don't implement _write_script
 -            # should not be included in the later stages.
 -            elements = [
 -                element for element in elements
 -                if self._write_element_script(source_directory, element)
 -            ]
+-
 -            self._write_element_sources(os.path.join(tempdir, "source"), elements)
 -            self._write_build_script(tempdir, elements)
 -            self._collect_sources(tempdir, tar_location,
 -                                  target.normal_name, compression)
+-
      # redirect_element_names()
+     #
      # Takes a list of element names and returns a list where elements have been
@@ -1185,6 +1109,54 @@ class Stream():
          sandbox_vroot.export_files(directory, can_link=True, can_destroy=True)
 +    # Helper function for source_checkout()
 +    def _source_checkout(self, elements,
 +                         location=None,
 +                         force=False,
 +                         deps='none',
 +                         fetch=False,
 +                         tar=False,
 +                         include_build_scripts=False):
 +        location = os.path.abspath(location)
 +        location_parent = os.path.abspath(os.path.join(location, ".."))
++
 +        # Stage all our sources in a temporary directory. The this
 +        # directory can be used to either construct a tarball or moved
 +        # to the final desired location.
 +        temp_source_dir = tempfile.TemporaryDirectory(dir=location_parent)
 +        self._write_element_sources(temp_source_dir.name, elements)
 +        if include_build_scripts:
 +                self._write_build_scripts(temp_source_dir.name, elements)
 +        try:
 +            if tar:
 +                self._create_tarball(temp_source_dir.name, location)
 +            else:
 +                self._move_directory(temp_source_dir.name, location, force)
 +        except OSError as e:
 +            raise StreamError("Failed to checkout sources to {}: {}"
 +                              .format(location, e)) from e
 +        finally:
 +            with suppress(FileNotFoundError):
 +                temp_source_dir.cleanup()
++
 +    # Move a directory src to dest. This will work across devices and
 +    # may optionaly overwrite existing files.
 +    def _move_directory(self, src, dest, force=False):
 +        def is_empty_dir(path):
 +            return(os.path.isdir(dest) and not os.listdir(dest))
++
 +        try:
 +            os.renames(src, dest)
 +            return
 +        except OSError:
 +            pass
++
 +        if force or is_empty_dir(dest):
 +            try:
 +                utils.link_files(src, dest)
 +            except utils.UtilError as e:
 +                raise StreamError("Failed to move directory: {}".format(e)) from e
++
      # Write the element build script to the given directory
      def _write_element_script(self, directory, element):
          try:
@@ -1201,8 +1173,28 @@ class Stream():
                  os.makedirs(element_source_dir)
                  element._stage_sources_at(element_source_dir)
 +    # Create a tarball from the content of directory
 +    def _create_tarball(self, directory, tar_name):
 +        try:
 +            with utils.save_file_atomic(tar_name, mode='wb') as f:
 +                # This TarFile does not need to be explicitly closed
 +                # as the underlying file object will be closed be the
 +                # save_file_atomic contect manager
 +                tarball = tarfile.open(fileobj=f, mode='w')
 +                for item in os.listdir(str(directory)):
 +                    file_to_add = os.path.join(directory, item)
 +                    tarball.add(file_to_add, arcname=item)
 +        except OSError as e:
 +            raise StreamError("Failed to create tar archive: {}".format(e)) from e
++
 +    # Write all the build_scripts for elements in the directory location
 +    def _write_build_scripts(self, location, elements):
 +        for element in elements:
 +            self._write_element_script(location, element)
 +        self._write_master_build_script(location, elements)
++
      # Write a master build script to the sandbox
 -    def _write_build_script(self, directory, elements):
 +    def _write_master_build_script(self, directory, elements):
          module_string = ""
          for element in elements:

doc/source/using_commands.rst

@@ -86,13 +86,6 @@ project's main directory.
  ----
 -.. _invoking_source_bundle:
+-
 -.. click:: buildstream._frontend.cli:source_bundle
 -   :prog: bst source bundle
+-
 -----
+-
  .. _invoking_workspace:
  .. click:: buildstream._frontend.cli:workspace

tests/completions/completions.py

@@ -16,7 +16,6 @@ MAIN_COMMANDS = [
      'shell ',
      'show ',
      'source-checkout ',
 -    'source-bundle ',
      'track ',
      'workspace '
+ ]

tests/frontend/help.py

@@ -25,7 +25,6 @@ def test_help_main(cli):
      ('push'),
      ('shell'),
      ('show'),
 -    ('source-bundle'),
      ('track'),
      ('workspace')
  ])

tests/frontend/source_bundle.py deleted

 -#
 -#  Copyright (C) 2018 Bloomberg Finance LP
 -#
 -#  This program is free software; you can redistribute it and/or
 -#  modify it under the terms of the GNU Lesser General Public
 -#  License as published by the Free Software Foundation; either
 -#  version 2 of the License, or (at your option) any later version.
 -#
 -#  This library is distributed in the hope that it will be useful,
 -#  but WITHOUT ANY WARRANTY; without even the implied warranty of
 -#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 -#  Lesser General Public License for more details.
 -#
 -#  You should have received a copy of the GNU Lesser General Public
 -#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 -#
 -#  Authors: Chandan Singh <csingh43 bloomberg net>
 -#
+-
 -import os
 -import tarfile
+-
 -import pytest
+-
 -from tests.testutils import cli
+-
 -# Project directory
 -DATA_DIR = os.path.join(
 -    os.path.dirname(os.path.realpath(__file__)),
 -    "project",
 -)
+-
+-
 -@pytest.mark.datafiles(DATA_DIR)
 -def test_source_bundle(cli, tmpdir, datafiles):
 -    project_path = os.path.join(datafiles.dirname, datafiles.basename)
 -    element_name = 'source-bundle/source-bundle-hello.bst'
 -    normal_name = 'source-bundle-source-bundle-hello'
+-
 -    # Verify that we can correctly produce a source-bundle
 -    args = ['source-bundle', element_name, '--directory', str(tmpdir)]
 -    result = cli.run(project=project_path, args=args)
 -    result.assert_success()
+-
 -    # Verify that the source-bundle contains our sources and a build script
 -    with tarfile.open(os.path.join(str(tmpdir), '{}.tar.gz'.format(normal_name))) as bundle:
 -        assert os.path.join(normal_name, 'source', normal_name, 'llamas.txt') in bundle.getnames()
 -        assert os.path.join(normal_name, 'build.sh') in bundle.getnames()

tests/frontend/source_checkout.py

  import os
  import pytest
 +import tarfile
 +from pathlib import Path
  from tests.testutils import cli
@@ -39,6 +41,39 @@ def test_source_checkout(datafiles, cli):
      assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
 +@pytest.mark.datafiles(DATA_DIR)
 +@pytest.mark.parametrize('force_flag', ['--force', '-f'])
 +def test_source_checkout_force(datafiles, cli, force_flag):
 +    project = os.path.join(datafiles.dirname, datafiles.basename)
 +    checkout = os.path.join(cli.directory, 'source-checkout')
 +    target = 'checkout-deps.bst'
++
 +    os.makedirs(os.path.join(checkout, 'some-thing'))
 +    # Path(os.path.join(checkout, 'some-file')).touch()
++
 +    result = cli.run(project=project, args=['source-checkout', force_flag, target, '--deps', 'none', checkout])
 +    result.assert_success()
++
 +    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_source_checkout_tar(datafiles, cli):
 +    project = os.path.join(datafiles.dirname, datafiles.basename)
 +    checkout = os.path.join(cli.directory, 'source-checkout.tar')
 +    target = 'checkout-deps.bst'
++
 +    result = cli.run(project=project, args=['source-checkout', '--tar', target, '--deps', 'none', checkout])
 +    result.assert_success()
++
 +    assert os.path.exists(checkout)
 +    with tarfile.open(checkout) as tf:
 +        expected_content = os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config')
 +        tar_members = [f.name for f in tf]
 +        for member in tar_members:
 +            assert member in expected_content
++
++
  @pytest.mark.datafiles(DATA_DIR)
  @pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
  def test_source_checkout_deps(datafiles, cli, deps):
@@ -119,3 +154,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
          assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
      else:
          result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
 +    project_path = os.path.join(datafiles.dirname, datafiles.basename)
 +    element_name = 'source-bundle/source-bundle-hello.bst'
 +    normal_name = 'source-bundle-source-bundle-hello'
 +    checkout = os.path.join(str(tmpdir), 'source-checkout')
++
 +    args = ['source-checkout', '--include-build-scripts', element_name, checkout]
 +    result = cli.run(project=project_path, args=args)
 +    result.assert_success()
++
 +    # There sould be a script for each element (just one in this case) and a top level build script
 +    expected_scripts = ['build.sh', 'build-' + normal_name]
 +    for script in expected_scripts:
 +        assert script in os.listdir(checkout)
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
 +    project_path = os.path.join(datafiles.dirname, datafiles.basename)
 +    element_name = 'source-bundle/source-bundle-hello.bst'
 +    normal_name = 'source-bundle-source-bundle-hello'
 +    tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
++
 +    args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
 +    result = cli.run(project=project_path, args=args)
 +    result.assert_success()
++
 +    expected_scripts = ['build.sh', 'build-' + normal_name]
++
 +    with tarfile.open(tar_file, 'r') as tf:
 +        for script in expected_scripts:
 +            assert script in tf.getnames()

[Notes] [Git][BuildStream/buildstream][phil/source-checkout-options] 6 commits: Add --tar option to source-checkout command

Phil Dawson pushed to branch phil/source-checkout-options at BuildStream / buildstream

Commits:

8 changed files:

Changes: