[Notes] [Git][BuildStream/buildstream][bschubert/tests-no-chroot-linux] 7 commits: _yaml.py: Implement `get()` for `ChainMap`

[Benjamin Schubert <gitlab mg gitlab com>]

Title: GitLab

Benjamin Schubert pushed to branch bschubert/tests-no-chroot-linux at BuildStream / buildstream

Commits:

• d868b409
  by Daniel Silverstone at 2018-11-01T13:40:24Z

  _yaml.py: Implement `get()` for `ChainMap`
  
  Since the core Python `ChainMap.get()` implements with:
  
      self[key] if key in self else default
  
  The double-chain-lookup is expensive.  This simple change solves that
  for our ChainMap derived structure.  As such it improves matters
  for #466 somewhat.
  
  Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>
  

• 7f79b9ce
  by Tristan Van Berkom at 2018-11-01T14:25:57Z

  Merge branch 'danielsilverstone-ct/improve-chainmap-get' into 'master'
  
  _yaml.py: Implement `get()` for `ChainMap`
  
  See merge request BuildStream/buildstream!919

• de59ebdb
  by ctolentino8 at 2018-11-02T16:41:54Z

  plugins/sources/pip.py: Accomodate characters '-','.','_' for packages
  

• 8d7cf806
  by ctolentino8 at 2018-11-02T16:41:54Z

  tests/integration/project/files: Remove pypi-repo
  

• 9c2f9bf7
  by Chandan Singh at 2018-11-02T17:09:46Z

  Merge branch 'chiaratolentino/fix-pip-source-regex' into 'master'
  
  plugins/sources/pip.py: Accomodate characters '-','.','_' for packages
  
  See merge request BuildStream/buildstream!914

• 48a78679
  by Benjamin Schubert at 2018-11-05T08:30:41Z

  Ensure platform is not maintained between tests
  
  This removes the `_instance` on the platform object that we use
  for caching and not recreating the object everytime at the start
  of every test.
  
  This is to ensure our tests share the least amount of state.
  
  The performance penalty is from 2 to 5% accross the whole test suite.
  The readings were done 5 times for each before and after the change
  and on the same computer.
  

• 49bb1ff8
  by Benjamin Schubert at 2018-11-05T08:30:59Z

  Ensure helpful messages are raised when missing dependencies
  

13 changed files:

• buildstream/_yaml.py
• buildstream/plugins/sources/pip.py
• buildstream/sandbox/_sandboxdummy.py
• conftest.py
• + tests/integration/missing_dependencies.py
• tests/integration/pip_source.py
• − tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz
• − tests/integration/project/files/pypi-repo/app2/index.html
• − tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz
• − tests/integration/project/files/pypi-repo/hellolib/index.html
• tests/sources/pip.py
• tests/testutils/__init__.py
• + tests/testutils/python_repo.py

Changes:

buildstream/_yaml.py

---

...	...	@@ -1049,6 +1049,12 @@ class ChainMap(collections.ChainMap):
1049	1049	for key in clearable:
1050	1050	del self[key]
1051	1051
	1052	+ def get(self, key, default=None):
	1053	+ try:
	1054	+ return self[key]
	1055	+ except KeyError:
	1056	+ return default
	1057	+
1052	1058
1053	1059	def node_chain_copy(source):
1054	1060	copy = ChainMap({}, source)

buildstream/plugins/sources/pip.py

---

...	...	@@ -96,7 +96,7 @@ _PYTHON_VERSIONS = [
96	96	# Names of source distribution archives must be of the form
97	97	# '%{package-name}-%{version}.%{extension}'.
98	98	_SDIST_RE = re.compile(
99		- r'^([a-zA-Z0-9]+?)-(.+).(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
	99	+ r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
100	100	re.IGNORECASE)
101	101
102	102
...	...	@@ -225,12 +225,27 @@ class PipSource(Source):
225	225	def _parse_sdist_names(self, basedir):
226	226	reqs = []
227	227	for f in os.listdir(basedir):
228		- pkg_match = _SDIST_RE.match(f)
229		- if pkg_match:
230		- reqs.append(pkg_match.groups())
	228	+ pkg = _match_package_name(f)
	229	+ if pkg is not None:
	230	+ reqs.append(pkg)
231	231
232	232	return sorted(reqs)
233	233
234	234
	235	+# Extract the package name and version of a source distribution
	236	+#
	237	+# Args:
	238	+# filename (str): Filename of the source distribution
	239	+#
	240	+# Returns:
	241	+# (tuple): A tuple of (package_name, version)
	242	+#
	243	+def _match_package_name(filename):
	244	+ pkg_match = _SDIST_RE.match(filename)
	245	+ if pkg_match is None:
	246	+ return None
	247	+ return pkg_match.groups()
	248	+
	249	+
235	250	def setup():
236	251	return PipSource

buildstream/sandbox/_sandboxdummy.py

---

...	...	@@ -42,4 +42,5 @@ class SandboxDummy(Sandbox):
42	42	"'{}'".format(command[0]),
43	43	reason='missing-command')
44	44
45		- raise SandboxError("This platform does not support local builds: {}".format(self._reason))
	45	+ raise SandboxError("This platform does not support local builds: {}".format(self._reason),
	46	+ reason="unavailable-local-sandbox")

conftest.py

---

...	...	@@ -23,6 +23,8 @@ import shutil
23	23
24	24	import pytest
25	25
	26	+from buildstream._platform.platform import Platform
	27	+
26	28
27	29	def pytest_addoption(parser):
28	30	parser.addoption('--integration', action='store_true', default=False,
...	...	@@ -52,3 +54,8 @@ def integration_cache(request):
52	54	shutil.rmtree(os.path.join(cache_dir, 'artifacts'))
53	55	except FileNotFoundError:
54	56	pass
	57	+
	58	+
	59	+@pytest.fixture(autouse=True)
	60	+def clean_platform_cache():
	61	+ Platform._instance = None

tests/integration/missing_dependencies.py

---

	1	+import os
	2	+import pytest
	3	+from tests.testutils import cli
	4	+from tests.testutils.site import IS_LINUX
	5	+
	6	+from buildstream import _yaml
	7	+from buildstream._exceptions import ErrorDomain
	8	+
	9	+
	10	+# Project directory
	11	+DATA_DIR = os.path.join(
	12	+ os.path.dirname(os.path.realpath(__file__)),
	13	+ 'project',
	14	+)
	15	+
	16	+
	17	+@pytest.mark.integration
	18	+@pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux')
	19	+@pytest.mark.datafiles(DATA_DIR)
	20	+def test_missing_brwap_has_nice_error_message(cli, datafiles):
	21	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	22	+ element_path = os.path.join(project, 'elements', 'element.bst')
	23	+
	24	+ # Write out our test target
	25	+ element = {
	26	+ 'kind': 'script',
	27	+ 'depends': [
	28	+ {
	29	+ 'filename': 'base.bst',
	30	+ 'type': 'build',
	31	+ },
	32	+ ],
	33	+ 'config': {
	34	+ 'commands': [
	35	+ 'false',
	36	+ ],
	37	+ },
	38	+ }
	39	+ _yaml.dump(element, element_path)
	40	+
	41	+ # Build without access to host tools, this should fail with a nice error
	42	+ result = cli.run(
	43	+ project=project, args=['build', 'element.bst'], env={'PATH': ''})
	44	+ result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
	45	+ assert "not found" in result.stderr
	46	+
	47	+
	48	+@pytest.mark.integration
	49	+@pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux')
	50	+@pytest.mark.datafiles(DATA_DIR)
	51	+def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path):
	52	+ import buildstream._platform
	53	+ buildstream._platform.Platform._instance = None
	54	+
	55	+ bwrap = tmp_path.joinpath('bin/bwrap')
	56	+ bwrap.parent.mkdir()
	57	+ with bwrap.open('w') as fp:
	58	+ fp.write('''
	59	+ #!/bin/bash
	60	+ echo bubblewrap 0.0.1
	61	+ '''.strip())
	62	+
	63	+ bwrap.chmod(0o755)
	64	+
	65	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	66	+ element_path = os.path.join(project, 'elements', 'element3.bst')
	67	+
	68	+ # Write out our test target
	69	+ element = {
	70	+ 'kind': 'script',
	71	+ 'depends': [
	72	+ {
	73	+ 'filename': 'base.bst',
	74	+ 'type': 'build',
	75	+ },
	76	+ ],
	77	+ 'config': {
	78	+ 'commands': [
	79	+ 'true && true && false',
	80	+ ],
	81	+ },
	82	+ }
	83	+ _yaml.dump(element, element_path)
	84	+
	85	+ # Build without access to host tools, this should fail with a nice error
	86	+ result = cli.run(
	87	+ project=project,
	88	+ args=['--debug', '--verbose', 'build', 'element3.bst'],
	89	+ env={'PATH': str(tmp_path.joinpath('bin'))})
	90	+ result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
	91	+ assert "too old" in result.stderr

tests/integration/pip_source.py

---

...	...	@@ -4,6 +4,7 @@ import pytest
4	4	from buildstream import _yaml
5	5
6	6	from tests.testutils import cli_integration as cli
	7	+from tests.testutils.python_repo import setup_pypi_repo
7	8	from tests.testutils.integration import assert_contains
8	9
9	10
...	...	@@ -17,12 +18,21 @@ DATA_DIR = os.path.join(
17	18
18	19
19	20	@pytest.mark.datafiles(DATA_DIR)
20		-def test_pip_source_import(cli, tmpdir, datafiles):
	21	+def test_pip_source_import(cli, tmpdir, datafiles, setup_pypi_repo):
21	22	project = os.path.join(datafiles.dirname, datafiles.basename)
22	23	checkout = os.path.join(cli.directory, 'checkout')
23	24	element_path = os.path.join(project, 'elements')
24	25	element_name = 'pip/hello.bst'
25	26
	27	+ # check that exotically named packages are imported correctly
	28	+ myreqs_packages = ['hellolib']
	29	+ packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
	30	+
	31	+ # create mock pypi repository
	32	+ pypi_repo = os.path.join(project, 'files', 'pypi-repo')
	33	+ os.makedirs(pypi_repo, exist_ok=True)
	34	+ setup_pypi_repo(myreqs_packages + packages, pypi_repo)
	35	+
26	36	element = {
27	37	'kind': 'import',
28	38	'sources': [
...	...	@@ -32,9 +42,9 @@ def test_pip_source_import(cli, tmpdir, datafiles):
32	42	},
33	43	{
34	44	'kind': 'pip',
35		- 'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
	45	+ 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
36	46	'requirements-files': ['myreqs.txt'],
37		- 'packages': ['app2']
	47	+ 'packages': packages
38	48	}
39	49	]
40	50	}
...	...	@@ -51,16 +61,31 @@ def test_pip_source_import(cli, tmpdir, datafiles):
51	61	assert result.exit_code == 0
52	62
53	63	assert_contains(checkout, ['/.bst_pip_downloads',
54		- '/.bst_pip_downloads/HelloLib-0.1.tar.gz',
55		- '/.bst_pip_downloads/App2-0.1.tar.gz'])
	64	+ '/.bst_pip_downloads/hellolib-0.1.tar.gz',
	65	+ '/.bst_pip_downloads/app2-0.1.tar.gz',
	66	+ '/.bst_pip_downloads/app.3-0.1.tar.gz',
	67	+ '/.bst_pip_downloads/app-4-0.1.tar.gz',
	68	+ '/.bst_pip_downloads/app_5-0.1.tar.gz',
	69	+ '/.bst_pip_downloads/app.no.6-0.1.tar.gz',
	70	+ '/.bst_pip_downloads/app-no-7-0.1.tar.gz',
	71	+ '/.bst_pip_downloads/app_no_8-0.1.tar.gz'])
56	72
57	73
58	74	@pytest.mark.datafiles(DATA_DIR)
59		-def test_pip_source_build(cli, tmpdir, datafiles):
	75	+def test_pip_source_build(cli, tmpdir, datafiles, setup_pypi_repo):
60	76	project = os.path.join(datafiles.dirname, datafiles.basename)
61	77	element_path = os.path.join(project, 'elements')
62	78	element_name = 'pip/hello.bst'
63	79
	80	+ # check that exotically named packages are imported correctly
	81	+ myreqs_packages = ['hellolib']
	82	+ packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
	83	+
	84	+ # create mock pypi repository
	85	+ pypi_repo = os.path.join(project, 'files', 'pypi-repo')
	86	+ os.makedirs(pypi_repo, exist_ok=True)
	87	+ setup_pypi_repo(myreqs_packages + packages, pypi_repo)
	88	+
64	89	element = {
65	90	'kind': 'manual',
66	91	'depends': ['base.bst'],
...	...	@@ -71,16 +96,15 @@ def test_pip_source_build(cli, tmpdir, datafiles):
71	96	},
72	97	{
73	98	'kind': 'pip',
74		- 'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
	99	+ 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
75	100	'requirements-files': ['myreqs.txt'],
76		- 'packages': ['app2']
	101	+ 'packages': packages
77	102	}
78	103	],
79	104	'config': {
80	105	'install-commands': [
81	106	'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz',
82		- 'chmod +x app1.py',
83		- 'install app1.py %{install-root}/usr/bin/'
	107	+ 'install app1.py %{install-root}/usr/bin/'
84	108	]
85	109	}
86	110	}
...	...	@@ -95,5 +119,4 @@ def test_pip_source_build(cli, tmpdir, datafiles):
95	119
96	120	result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py'])
97	121	assert result.exit_code == 0
98		- assert result.output == """Hello App1!
99		-"""
	122	+ assert result.output == "Hello App1! This is hellolib\n"

tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz deleted

---

No preview for this file type

tests/integration/project/files/pypi-repo/app2/index.html deleted

---

1		-<html>
2		- <head>
3		- <title>Links for app1</title>
4		- </head>
5		- <body>
6		- <a href="">'App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
7		- </body>
8		-</html>

tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz deleted

---

No preview for this file type

tests/integration/project/files/pypi-repo/hellolib/index.html deleted

---

1		-<html>
2		- <head>
3		- <title>Links for app1</title>
4		- </head>
5		- <body>
6		- <a href="">'HelloLib-0.1.tar.gz'>HelloLib-0.1.tar.gz</a><br />
7		- </body>
8		-</html>

tests/sources/pip.py

---

...	...	@@ -3,6 +3,7 @@ import pytest
3	3
4	4	from buildstream._exceptions import ErrorDomain
5	5	from buildstream import _yaml
	6	+from buildstream.plugins.sources.pip import _match_package_name
6	7	from tests.testutils import cli
7	8
8	9	DATA_DIR = os.path.join(
...	...	@@ -45,3 +46,22 @@ def test_no_packages(cli, tmpdir, datafiles):
45	46	'show', 'target.bst'
46	47	])
47	48	result.assert_main_error(ErrorDomain.SOURCE, None)
	49	+
	50	+
	51	+# Test that pip source parses tar ball names correctly for the ref
	52	+@pytest.mark.parametrize(
	53	+ 'tarball, expected_name, expected_version',
	54	+ [
	55	+ ('dotted.package-0.9.8.tar.gz', 'dotted.package', '0.9.8'),
	56	+ ('hyphenated-package-2.6.0.tar.gz', 'hyphenated-package', '2.6.0'),
	57	+ ('underscore_pkg-3.1.0.tar.gz', 'underscore_pkg', '3.1.0'),
	58	+ ('numbers2and5-1.0.1.tar.gz', 'numbers2and5', '1.0.1'),
	59	+ ('multiple.dots.package-5.6.7.tar.gz', 'multiple.dots.package', '5.6.7'),
	60	+ ('multiple-hyphens-package-1.2.3.tar.gz', 'multiple-hyphens-package', '1.2.3'),
	61	+ ('multiple_underscore_pkg-3.4.5.tar.gz', 'multiple_underscore_pkg', '3.4.5'),
	62	+ ('shortversion-1.0.tar.gz', 'shortversion', '1.0'),
	63	+ ('longversion-1.2.3.4.tar.gz', 'longversion', '1.2.3.4')
	64	+ ])
	65	+def test_match_package_name(tarball, expected_name, expected_version):
	66	+ name, version = _match_package_name(tarball)
	67	+ assert (expected_name, expected_version) == (name, version)

tests/testutils/__init__.py

---

...	...	@@ -29,3 +29,4 @@ from .artifactshare import create_artifact_share
29	29	from .element_generators import create_element_size, update_element_size
30	30	from .junction import generate_junction
31	31	from .runner_integration import wait_for_cache_granularity
	32	+from .python_repo import setup_pypi_repo

tests/testutils/python_repo.py

---

	1	+from setuptools.sandbox import run_setup
	2	+import os
	3	+import pytest
	4	+import re
	5	+import shutil
	6	+
	7	+
	8	+SETUP_TEMPLATE = '''\
	9	+from setuptools import setup
	10	+
	11	+setup(
	12	+ name='{name}',
	13	+ version='{version}',
	14	+ description='{name}',
	15	+ packages=['{pkgdirname}'],
	16	+ entry_points={{
	17	+ 'console_scripts': [
	18	+ '{pkgdirname}={pkgdirname}:main'
	19	+ ]
	20	+ }}
	21	+)
	22	+'''
	23	+
	24	+# All packages generated via generate_pip_package will have the functions below
	25	+INIT_TEMPLATE = '''\
	26	+def main():
	27	+ print('This is {name}')
	28	+
	29	+def hello(actor='world'):
	30	+ print('Hello {{}}! This is {name}'.format(actor))
	31	+'''
	32	+
	33	+HTML_TEMPLATE = '''\
	34	+<html>
	35	+ <head>
	36	+ <title>Links for {name}</title>
	37	+ </head>
	38	+ <body>
	39	+ <a href=''>{name}-{version}.tar.gz</a><br />
	40	+ </body>
	41	+</html>
	42	+'''
	43	+
	44	+
	45	+# Creates a simple python source distribution and copies this into a specified
	46	+# directory which is to serve as a mock python repository
	47	+#
	48	+# Args:
	49	+# tmpdir (str): Directory in which the source files will be created
	50	+# pypi (str): Directory serving as a mock python repository
	51	+# name (str): The name of the package to be created
	52	+# version (str): The version of the package to be created
	53	+#
	54	+# Returns:
	55	+# None
	56	+#
	57	+def generate_pip_package(tmpdir, pypi, name, version='0.1'):
	58	+ # check if package already exists in pypi
	59	+ pypi_package = os.path.join(pypi, re.sub('[^0-9a-zA-Z]+', '-', name))
	60	+ if os.path.exists(pypi_package):
	61	+ return
	62	+
	63	+ # create the package source files in tmpdir resulting in a directory
	64	+ # tree resembling the following structure:
	65	+ #
	66	+ # tmpdir
	67	+ # |-- setup.py
	68	+ # `-- package
	69	+ # `-- __init__.py
	70	+ #
	71	+ setup_file = os.path.join(tmpdir, 'setup.py')
	72	+ pkgdirname = re.sub('[^0-9a-zA-Z]+', '', name)
	73	+ with open(setup_file, 'w') as f:
	74	+ f.write(
	75	+ SETUP_TEMPLATE.format(
	76	+ name=name,
	77	+ version=version,
	78	+ pkgdirname=pkgdirname
	79	+ )
	80	+ )
	81	+ os.chmod(setup_file, 0o755)
	82	+
	83	+ package = os.path.join(tmpdir, pkgdirname)
	84	+ os.makedirs(package)
	85	+
	86	+ main_file = os.path.join(package, '__init__.py')
	87	+ with open(main_file, 'w') as f:
	88	+ f.write(INIT_TEMPLATE.format(name=name))
	89	+ os.chmod(main_file, 0o644)
	90	+
	91	+ run_setup(setup_file, ['sdist'])
	92	+
	93	+ # create directory for this package in pypi resulting in a directory
	94	+ # tree resembling the following structure:
	95	+ #
	96	+ # pypi
	97	+ # `-- pypi_package
	98	+ # |-- index.html
	99	+ # `-- foo-0.1.tar.gz
	100	+ #
	101	+ os.makedirs(pypi_package)
	102	+
	103	+ # add an index html page
	104	+ index_html = os.path.join(pypi_package, 'index.html')
	105	+ with open(index_html, 'w') as f:
	106	+ f.write(HTML_TEMPLATE.format(name=name, version=version))
	107	+
	108	+ # copy generated tarfile to pypi package
	109	+ dist_dir = os.path.join(tmpdir, 'dist')
	110	+ for tar in os.listdir(dist_dir):
	111	+ tarpath = os.path.join(dist_dir, tar)
	112	+ shutil.copy(tarpath, pypi_package)
	113	+
	114	+
	115	+@pytest.fixture
	116	+def setup_pypi_repo(tmpdir):
	117	+ def create_pkgdir(package):
	118	+ pkgdirname = re.sub('[^0-9a-zA-Z]+', '', package)
	119	+ pkgdir = os.path.join(str(tmpdir), pkgdirname)
	120	+ os.makedirs(pkgdir)
	121	+ return pkgdir
	122	+
	123	+ def add_packages(packages, pypi_repo):
	124	+ for package in packages:
	125	+ pkgdir = create_pkgdir(package)
	126	+ generate_pip_package(pkgdir, pypi_repo, package)
	127	+
	128	+ return add_packages

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.