[Notes] [Git][BuildStream/buildstream][bschubert/remove-pytest-runner] 7

Jonathan Maw pushed to branch bschubert/remove-pytest-runner at BuildStream / buildstream

Commits:

2c6cb230
by Valentin David at 2018-10-26T14:21:18Z
```
Fix some ruamel warnings during tests
```

a8250ca4

by Valentin David at 2018-10-26T15:04:25Z

Merge branch 'valentindavid/ruamel_warnings' into 'master'

Fix some ruamel warnings during tests

See merge request BuildStream/buildstream!889

1b308fe3

by Daniel Silverstone at 2018-10-26T15:50:42Z

_stream.py: Preserve stdout FD across checkout-to-stdout

In order to support things which might need to use the stdout FD
after checkout has completed writing a tarball to it, preserve it
by means of `os.dup()` and `os.dup2()`

Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>

65d90c31

by Daniel Silverstone at 2018-10-26T15:50:54Z

tests/testutils/runcli.py: Support binary-mode capture of stdout

In order to test things which write tarballs to stdout correctly,
we need to capture the binary output cleanly.  This ensures we're not
potentially mismatching encodings in and out.

Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>

ed733f3e

by Daniel Silverstone at 2018-10-26T15:51:40Z

tests/frontend/buildcheckout.py: Enable test_build_checkout_tarball_stdout

With the new support for saving the stdout FDs cleanly, and binary mode
capture in runcli, enable the tarball-to-stdout test.

Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>

e97d2a75

by Benjamin Schubert at 2018-10-29T10:17:07Z

Merge branch 'bschubert/fix-binary-output-capture' into 'master'

Don't close stdout when running 'checkout' to stdout

Closes #486

See merge request BuildStream/buildstream!905

cb7f1538

by Benjamin Schubert at 2018-10-29T10:30:06Z

Remove dependency on pytest-runner

This includes a new command mimicking pytest-runner so that we
can drop this dependency

Changes:

buildstream/_stream.py

@@ -423,9 +423,16 @@ class Stream():
                  else:
                      if location == '-':
                          with target.timed_activity("Creating tarball"):
 -                            with os.fdopen(sys.stdout.fileno(), 'wb') as fo:
 -                                with tarfile.open(fileobj=fo, mode="w|") as tf:
 -                                    sandbox_vroot.export_to_tar(tf, '.')
 +                            # Save the stdout FD to restore later
 +                            saved_fd = os.dup(sys.stdout.fileno())
 +                            try:
 +                                with os.fdopen(sys.stdout.fileno(), 'wb') as fo:
 +                                    with tarfile.open(fileobj=fo, mode="w|") as tf:
 +                                        sandbox_vroot.export_to_tar(tf, '.')
 +                            finally:
 +                                # No matter what, restore stdout for further use
 +                                os.dup2(saved_fd, sys.stdout.fileno())
 +                                os.close(saved_fd)
                      else:
                          with target.timed_activity("Creating tarball '{}'"
                                                     .format(location)):

setup.py

@@ -39,6 +39,7 @@ if sys.version_info[0] != REQUIRED_PYTHON_MAJOR or sys.version_info[1] < REQUIRE
  try:
      from setuptools import setup, find_packages, Command
      from setuptools.command.easy_install import ScriptWriter
 +    from setuptools.command.test import test as TestCommand
  except ImportError:
      print("BuildStream requires setuptools in order to build. Install it using"
            " your package manager (usually python3-setuptools) or via pip (pip3"
@@ -219,9 +220,48 @@ class BuildGRPC(Command):
                          f.write(code)
 +#####################################################
 +#                   Pytest command                  #
 +#####################################################
 +class PyTest(TestCommand):
 +    """Defines a pytest command class to run tests from setup.py"""
++
 +    user_options = TestCommand.user_options + [
 +        ("addopts=", None, "Arguments to pass to pytest"),
 +        ('index-url=''build_grpc': BuildGRPC,
 +        'pytest': PyTest,
+     }
      cmdclass.update(versioneer.get_cmdclass())
      return cmdclass
@@ -305,6 +345,5 @@ setup(name='BuildStream',
            'grpcio >= 1.10',
        ],
        entry_points=bst_install_entry_points,
 -      setup_requires=['pytest-runner'],
        tests_require=dev_requires,
        zip_safe=False)

tests/frontend/buildcheckout.py

@@ -128,7 +128,6 @@ def test_build_checkout_tarball(datafiles, cli):
      assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames()
 -@pytest.mark.skip(reason="Capturing the binary output is causing a stacktrace")
  @pytest.mark.datafiles(DATA_DIR)
  def test_build_checkout_tarball_stdout(datafiles, cli):
      project = os.path.join(datafiles.dirname, datafiles.basename)
@@ -143,7 +142,7 @@ def test_build_checkout_tarball_stdout(datafiles, cli):
      checkout_args = ['checkout', '--tar', 'target.bst', '-']
 -    result = cli.run(project=project, args=checkout_args)
 +    result = cli.run(project=project, args=checkout_args, binary_capture=True)
      result.assert_success()
      with open(tarball, 'wb') as f:

tests/frontend/yamlcache.py

@@ -103,7 +103,7 @@ def test_yamlcache_used(cli, tmpdir, ref_storage, with_junction, move_project):
          yc.put_from_key(prj, element_path, key, contents)
      # Show that a variable has been added
 -    result = cli.run(project=project, args=['show', '--format', '%{vars}', 'test.bst'])
 +    result = cli.run(project=project, args=['show', '--deps', 'none', '--format', '%{vars}', 'test.bst'])
      result.assert_success()
      data = yaml.safe_load(result.output)
      assert 'modified' in data
@@ -135,7 +135,7 @@ def test_yamlcache_changed_file(cli, tmpdir, ref_storage, with_junction):
          _yaml.load(element_path, copy_tree=False, project=prj, yaml_cache=yc)
      # Show that a variable has been added
 -    result = cli.run(project=project, args=['show', '--format', '%{vars}', 'test.bst'])
 +    result = cli.run(project=project, args=['show', '--deps', 'none', '--format', '%{vars}', 'test.bst'])
      result.assert_success()
      data = yaml.safe_load(result.output)
      assert 'modified' in data

tests/testutils/runcli.py

@@ -17,7 +17,7 @@ import pytest
  # CliRunner convenience API (click.testing module) does not support
  # separation of stdout/stderr.
+ #
 -from _pytest.capture import MultiCapture, FDCapture
 +from _pytest.capture import MultiCapture, FDCapture, FDCaptureBinary
  # Import the main cli entrypoint
  from buildstream._frontend import cli as bst_cli
@@ -234,9 +234,10 @@ class Cli():
      #    silent (bool): Whether to pass --no-verbose
      #    env (dict): Environment variables to temporarily set during the test
      #    args (list): A list of arguments to pass buildstream
 +    #    binary_capture (bool): Whether to capture the stdout/stderr as binary
+     #
      def run(self, configure=True, project=None, silent=False, env=None,
 -            cwd=None, options=None, args=None):
 +            cwd=None, options=None, args=None, binary_capture=False):
          if args is None:
              args = []
          if options is None:
@@ -278,7 +279,7 @@ class Cli():
              except ValueError:
                  sys.__stdout__ = open('/dev/stdout', 'w')
 -            result = self.invoke(bst_cli, bst_args)
 +            result = self.invoke(bst_cli, bst_args, binary_capture=binary_capture)
          # Some informative stdout we can observe when anything fails
          if self.verbose:
@@ -295,7 +296,7 @@ class Cli():
          return result
 -    def invoke(self, cli, args=None, color=False, **extra):
 +    def invoke(self, cli, args=None, color=False, binary_capture=False, **extra):
          exc_info = None
          exception = None
          exit_code = 0
@@ -305,8 +306,8 @@ class Cli():
          old_stdin = sys.stdin
          with open(os.devnull) as devnull:
              sys.stdin = devnull
+-
 -            capture = MultiCapture(out=True, err=True, in_=False, Capture=FDCapture)
 +            capture_kind = FDCaptureBinary if binary_capture else FDCapture
 +            capture = MultiCapture(out=True, err=True, in_=False, Capture=capture_kind)
              capture.start_capturing()
              try:

[Notes] [Git][BuildStream/buildstream][bschubert/remove-pytest-runner] 7 commits: Fix some ruamel warnings during tests

Jonathan Maw pushed to branch bschubert/remove-pytest-runner at BuildStream / buildstream

Commits:

5 changed files:

Changes: