[Notes] [Git][BuildStream/buildstream][jonathan/workspace-fragment-create] 13 commits: plugins/sources/pip.py: Accomodate characters '-', '.', '_' for packages

[Jonathan Maw <gitlab mg gitlab com>]

Title: GitLab

Jonathan Maw pushed to branch jonathan/workspace-fragment-create at BuildStream / buildstream

Commits:

• de59ebdb
  by ctolentino8 at 2018-11-02T16:41:54Z

  plugins/sources/pip.py: Accomodate characters '-','.','_' for packages
  

• 8d7cf806
  by ctolentino8 at 2018-11-02T16:41:54Z

  tests/integration/project/files: Remove pypi-repo
  

• 9c2f9bf7
  by Chandan Singh at 2018-11-02T17:09:46Z

  Merge branch 'chiaratolentino/fix-pip-source-regex' into 'master'
  
  plugins/sources/pip.py: Accomodate characters '-','.','_' for packages
  
  See merge request BuildStream/buildstream!914

• f75f6706
  by Jonathan Maw at 2018-11-02T18:01:39Z

  _workspaces.py: Add a WorkspaceLocal object
  
  WorkspaceLocal contains all the information needed to generate data
  that's stored within the workspace. Currently this is just a list of
  project directories.
  

• 233cb7bb
  by Jonathan Maw at 2018-11-02T18:01:39Z

  _stream.py: Opened workspaces store the project that opened them
  

• d7c8dfb1
  by Jonathan Maw at 2018-11-02T18:01:39Z

  _stream.py: workspace_close deletes the stored project reference
  

• 3776af56
  by Jonathan Maw at 2018-11-02T18:01:39Z

  tests: Workspace tests check existence of .bstproject.yaml
  

• a3fbf644
  by Jonathan Maw at 2018-11-02T18:01:39Z

  _project.py: Rename _ensure_project_dir to _find_project_dir
  

• 58d9ed76
  by Jonathan Maw at 2018-11-02T18:01:39Z

  utils.py: Add a helper for searching upwards for files
  
  i.e. with a given directory and filename, check parent directories until
  either a directory with the filename is found, or you reach the root of
  the filesystem.
  

• 282995a4
  by Jonathan Maw at 2018-11-02T18:01:39Z

  cli.py: Use utils' search upwards helper when searching for project.conf
  

• ed93d1e3
  by Jonathan Maw at 2018-11-02T18:01:39Z

  _project.py: Use utils' search upwards helper when searching for project.conf
  

• 7d4c5bb3
  by Jonathan Maw at 2018-11-02T18:01:39Z

  tests: Try building from a workspace
  

• 500df50b
  by Jonathan Maw at 2018-11-02T18:01:40Z

  _project.py: Find project from workspace if outside of a project
  

15 changed files:

• buildstream/_frontend/cli.py
• buildstream/_project.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/plugins/sources/pip.py
• buildstream/utils.py
• tests/frontend/workspace.py
• tests/integration/pip_source.py
• − tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz
• − tests/integration/project/files/pypi-repo/app2/index.html
• − tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz
• − tests/integration/project/files/pypi-repo/hellolib/index.html
• tests/sources/pip.py
• tests/testutils/__init__.py
• + tests/testutils/python_repo.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -59,18 +59,9 @@ def complete_target(args, incomplete):
59	59	:return: all the possible user-specified completions for the param
60	60	"""
61	61
	62	+ from .. import utils
62	63	project_conf = 'project.conf'
63	64
64		- def ensure_project_dir(directory):
65		- directory = os.path.abspath(directory)
66		- while not os.path.isfile(os.path.join(directory, project_conf)):
67		- parent_dir = os.path.dirname(directory)
68		- if directory == parent_dir:
69		- break
70		- directory = parent_dir
71		-
72		- return directory
73		-
74	65	# First resolve the directory, in case there is an
75	66	# active --directory/-C option
76	67	#
...	...	@@ -89,7 +80,7 @@ def complete_target(args, incomplete):
89	80	else:
90	81	# Check if this directory or any of its parent directories
91	82	# contain a project config file
92		- base_directory = ensure_project_dir(base_directory)
	83	+ base_directory = utils._search_upward_for_file(base_directory, project_conf)
93	84
94	85	# Now parse the project.conf just to find the element path,
95	86	# this is unfortunately a bit heavy.

buildstream/_project.py

---

...	...	@@ -40,6 +40,7 @@ from .element import Element
40	40	from ._message import Message, MessageType
41	41	from ._includes import Includes
42	42	from ._platform import Platform
	43	+from ._workspaces import WorkspaceLocal
43	44
44	45
45	46	# Project Configuration file
...	...	@@ -95,7 +96,7 @@ class Project():
95	96	self.name = None
96	97
97	98	# The project directory
98		- self.directory = self._ensure_project_dir(directory)
	99	+ self.directory = self._find_project_dir(directory)
99	100
100	101	# Absolute path to where elements are loaded from within the project
101	102	self.element_path = None
...	...	@@ -647,7 +648,7 @@ class Project():
647	648	# Source url aliases
648	649	output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
649	650
650		- # _ensure_project_dir()
	651	+ # _find_project_dir()
651	652	#
652	653	# Returns path of the project directory, if a configuration file is found
653	654	# in given directory or any of its parent directories.
...	...	@@ -658,18 +659,19 @@ class Project():
658	659	# Raises:
659	660	# LoadError if project.conf is not found
660	661	#
661		- def _ensure_project_dir(self, directory):
662		- directory = os.path.abspath(directory)
663		- while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
664		- parent_dir = os.path.dirname(directory)
665		- if directory == parent_dir:
	662	+ def _find_project_dir(self, directory):
	663	+ project_directory = utils._search_upward_for_file(directory, _PROJECT_CONF_FILE)
	664	+ if not project_directory:
	665	+ workspace_local = WorkspaceLocal.load(directory)
	666	+ if workspace_local:
	667	+ project_directory = workspace_local.get_default_path()
	668	+ else:
666	669	raise LoadError(
667	670	LoadErrorReason.MISSING_PROJECT_CONF,
668	671	'{} not found in current directory or any of its parent directories'
669	672	.format(_PROJECT_CONF_FILE))
670		- directory = parent_dir
671	673
672		- return directory
	674	+ return project_directory
673	675
674	676	def _load_plugin_factories(self, config, output):
675	677	plugin_source_origins = [] # Origins of custom sources

buildstream/_stream.py

---

...	...	@@ -32,6 +32,7 @@ from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
32	32	from ._message import Message, MessageType
33	33	from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
34	34	from ._pipeline import Pipeline, PipelineSelection
	35	+from ._workspaces import WorkspaceLocal
35	36	from . import utils, _yaml, _site
36	37	from . import Scope, Consistency
37	38
...	...	@@ -516,6 +517,10 @@ class Stream():
516	517	with target.timed_activity("Staging sources to {}".format(directory)):
517	518	target._open_workspace()
518	519
	520	+ project = self._context.get_toplevel_project()
	521	+ workspace_local = WorkspaceLocal.create(directory, project.directory, target._get_full_name())
	522	+ workspace_local.write()
	523	+
519	524	workspaces.save_config()
520	525	self._message(MessageType.INFO, "Saved workspace configuration")
521	526
...	...	@@ -540,6 +545,11 @@ class Stream():
540	545	except OSError as e:
541	546	raise StreamError("Could not remove '{}': {}"
542	547	.format(workspace.get_absolute_path(), e)) from e
	548	+ else:
	549	+ # TODO: At some point, closing a workspace only deletes the file if no projects are using it.
	550	+ workspace_local = WorkspaceLocal.load(workspace.get_absolute_path())
	551	+ if workspace_local:
	552	+ workspace_local.delete()
543	553
544	554	# Delete the workspace and save the configuration
545	555	workspaces.delete_workspace(element_name)

buildstream/_workspaces.py

---

...	...	@@ -25,6 +25,141 @@ from ._exceptions import LoadError, LoadErrorReason
25	25
26	26
27	27	BST_WORKSPACE_FORMAT_VERSION = 3
	28	+BST_WORKSPACE_LOCAL_FORMAT_VERSION = 1
	29	+WORKSPACE_LOCAL_FILE = ".bstproject.yaml"
	30	+
	31	+
	32	+# WorkspaceLocal()
	33	+#
	34	+# An object to contain various helper functions and data required for
	35	+# referring from a workspace back to buildstream.
	36	+#
	37	+# Args:
	38	+# directory (str): The directory that the workspace exists in
	39	+# project_path (str): The project path used to refer back
	40	+# to buildstream projects.
	41	+# element_name (str): The name of the element used to create this workspace.
	42	+class WorkspaceLocal():
	43	+ def __init__(self, directory, project_path="", element_name=""):
	44	+ self._projects = []
	45	+ self._directory = directory
	46	+
	47	+ assert (project_path and element_name) or (not project_path and not element_name)
	48	+ if project_path:
	49	+ self._add_project(project_path, element_name)
	50	+
	51	+ # get_default_path()
	52	+ #
	53	+ # Retrieves the default path to a project.
	54	+ #
	55	+ # Returns:
	56	+ # (str): The path to a project
	57	+ def get_default_path(self):
	58	+ return self._projects[0]['project-path']
	59	+
	60	+ # to_dict()
	61	+ #
	62	+ # Turn the members data into a dict for serialization purposes
	63	+ #
	64	+ # Returns:
	65	+ # (dict): A dict representation of the WorkspaceLocal
	66	+ #
	67	+ def to_dict(self):
	68	+ ret = {
	69	+ 'projects': self._projects,
	70	+ 'format-version': BST_WORKSPACE_LOCAL_FORMAT_VERSION,
	71	+ }
	72	+ return ret
	73	+
	74	+ # from_dict()
	75	+ #
	76	+ # Loads a new WorkspaceLocal from a simple dictionary
	77	+ #
	78	+ # Args:
	79	+ # directory (str): The directory that the workspace exists in
	80	+ # dictionary (dict): The dict to generate a WorkspaceLocal from
	81	+ #
	82	+ # Returns:
	83	+ # (WorkspaceLocal): A newly instantiated WorkspaceLocal
	84	+ @classmethod
	85	+ def from_dict(cls, directory, dictionary):
	86	+ # Only know how to handle one format-version at the moment.
	87	+ format_version = int(dictionary['format-version'])
	88	+ assert format_version == BST_WORKSPACE_LOCAL_FORMAT_VERSION, \
	89	+ "Format version {} not found in {}".format(BST_WORKSPACE_LOCAL_FORMAT_VERSION, dictionary)
	90	+
	91	+ workspace_local = cls(directory)
	92	+ for item in dictionary['projects']:
	93	+ workspace_local._add_project(item['project-path'], item['element-name'])
	94	+
	95	+ return workspace_local
	96	+
	97	+ # create()
	98	+ #
	99	+ # Creates a new WorkspaceLocal
	100	+ #
	101	+ # Args:
	102	+ # directory (str): The directory that the workspace exists in
	103	+ # project_path (str): The path to the project to store
	104	+ # element_name (str): The name of the element within the project
	105	+ #
	106	+ # Returns:
	107	+ # (WorkspaceLocal): The created WorkspaceLocal
	108	+ @classmethod
	109	+ def create(cls, directory, project_path, element_name):
	110	+ # TODO: Load WorkspaceLocal if it exists, and maybe add project_path to it
	111	+ return cls(directory, project_path, element_name)
	112	+
	113	+ # load()
	114	+ #
	115	+ # Loads the WorkspaceLocal for a given directory. This directory may be a
	116	+ # subdirectory of the workspace's directory.
	117	+ #
	118	+ # Args:
	119	+ # directory (str): The directory
	120	+ # Returns:
	121	+ # (WorkspaceLocal): The created WorkspaceLocal, if in a workspace, or
	122	+ # (NoneType): None, if the directory is not inside a workspace.
	123	+ @classmethod
	124	+ def load(cls, directory):
	125	+ local_dir = cls.search_for_dir(directory)
	126	+ if local_dir:
	127	+ workspace_file = os.path.join(local_dir, WORKSPACE_LOCAL_FILE)
	128	+ data_dict = _yaml.load(workspace_file)
	129	+ return cls.from_dict(local_dir, data_dict)
	130	+ else:
	131	+ return None
	132	+
	133	+ # write()
	134	+ #
	135	+ # Writes the WorkspaceLocal to disk
	136	+ def write(self):
	137	+ os.makedirs(self._directory, exist_ok=True)
	138	+ _yaml.dump(self.to_dict(), self._get_filename())
	139	+
	140	+ # delete()
	141	+ #
	142	+ # Deletes the WorkspaceLocal from disk, if it exists.
	143	+ def delete(self):
	144	+ try:
	145	+ os.unlink(self._get_filename())
	146	+ except FileNotFoundError:
	147	+ pass
	148	+
	149	+ # search_for_dir()
	150	+ #
	151	+ # Returns the directory that contains the workspace local file,
	152	+ # searching upwards from search_dir.
	153	+ @staticmethod
	154	+ def search_for_dir(search_dir):
	155	+ return utils._search_upward_for_file(search_dir, WORKSPACE_LOCAL_FILE)
	156	+
	157	+ def _get_filename(self):
	158	+ return os.path.join(self._directory, WORKSPACE_LOCAL_FILE)
	159	+
	160	+ def _add_project(self, project_path, element_name):
	161	+ self._projects.append({'project-path': project_path, 'element-name': element_name})
	162	+
28	163
29	164
30	165	# Workspace()

buildstream/plugins/sources/pip.py

---

...	...	@@ -96,7 +96,7 @@ _PYTHON_VERSIONS = [
96	96	# Names of source distribution archives must be of the form
97	97	# '%{package-name}-%{version}.%{extension}'.
98	98	_SDIST_RE = re.compile(
99		- r'^([a-zA-Z0-9]+?)-(.+).(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
	99	+ r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
100	100	re.IGNORECASE)
101	101
102	102
...	...	@@ -225,12 +225,27 @@ class PipSource(Source):
225	225	def _parse_sdist_names(self, basedir):
226	226	reqs = []
227	227	for f in os.listdir(basedir):
228		- pkg_match = _SDIST_RE.match(f)
229		- if pkg_match:
230		- reqs.append(pkg_match.groups())
	228	+ pkg = _match_package_name(f)
	229	+ if pkg is not None:
	230	+ reqs.append(pkg)
231	231
232	232	return sorted(reqs)
233	233
234	234
	235	+# Extract the package name and version of a source distribution
	236	+#
	237	+# Args:
	238	+# filename (str): Filename of the source distribution
	239	+#
	240	+# Returns:
	241	+# (tuple): A tuple of (package_name, version)
	242	+#
	243	+def _match_package_name(filename):
	244	+ pkg_match = _SDIST_RE.match(filename)
	245	+ if pkg_match is None:
	246	+ return None
	247	+ return pkg_match.groups()
	248	+
	249	+
235	250	def setup():
236	251	return PipSource

buildstream/utils.py

---

...	...	@@ -1198,3 +1198,17 @@ def _deduplicate(iterable, key=None):
1198	1198	def _get_link_mtime(path):
1199	1199	path_stat = os.lstat(path)
1200	1200	return path_stat.st_mtime
	1201	+
	1202	+
	1203	+# Returns the first directory to contain filename, or an empty string if
	1204	+# none found
	1205	+#
	1206	+def _search_upward_for_file(directory, filename):
	1207	+ directory = os.path.abspath(directory)
	1208	+ while not os.path.isfile(os.path.join(directory, filename)):
	1209	+ parent_dir = os.path.dirname(directory)
	1210	+ if directory == parent_dir:
	1211	+ return ""
	1212	+ directory = parent_dir
	1213	+
	1214	+ return directory

tests/frontend/workspace.py

---

...	...	@@ -93,6 +93,13 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir
93	93
94	94	result.assert_success()
95	95
	96	+ # Assert that a .bstproject.yaml file has been created
	97	+ # and contains the path to the project
	98	+ bstproject_path = os.path.join(workspace_dir, '.bstproject.yaml')
	99	+ assert os.path.exists(bstproject_path)
	100	+ with open(bstproject_path) as f:
	101	+ assert project_path in f.read()
	102	+
96	103	# Assert that we are now buildable because the source is
97	104	# now cached.
98	105	assert cli.get_element_state(project_path, element_name) == 'buildable'
...	...	@@ -148,6 +155,10 @@ def test_open_force(cli, tmpdir, datafiles, kind):
148	155	# Assert the workspace dir still exists
149	156	assert os.path.exists(workspace)
150	157
	158	+ # Assert the bstproject doesn't exist
	159	+ bstproject_path = os.path.join(workspace, '.bstproject.yaml')
	160	+ assert not os.path.exists(bstproject_path)
	161	+
151	162	# Now open the workspace again with --force, this should happily succeed
152	163	result = cli.run(project=project, args=[
153	164	'workspace', 'open', '--force', element_name, workspace
...	...	@@ -436,7 +447,9 @@ def test_list(cli, tmpdir, datafiles):
436	447	@pytest.mark.datafiles(DATA_DIR)
437	448	@pytest.mark.parametrize("kind", repo_kinds)
438	449	@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
439		-def test_build(cli, tmpdir, datafiles, kind, strict):
	450	+@pytest.mark.parametrize("call_from", [("project"), ("workspace")])
	451	+def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
	452	+ tmpdir = tmpdir_factory.mktemp('')
440	453	element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
441	454	checkout = os.path.join(str(tmpdir), 'checkout')
442	455
...	...	@@ -461,7 +474,8 @@ def test_build(cli, tmpdir, datafiles, kind, strict):
461	474	# Build modified workspace
462	475	assert cli.get_element_state(project, element_name) == 'buildable'
463	476	assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
464		- result = cli.run(project=project, args=['build', element_name])
	477	+ result = cli.run(project=project if call_from == "project" else workspace,
	478	+ args=['build', element_name])
465	479	result.assert_success()
466	480	assert cli.get_element_state(project, element_name) == 'cached'
467	481	assert cli.get_element_key(project, element_name) != "{:?<64}".format('')

tests/integration/pip_source.py

---

...	...	@@ -4,6 +4,7 @@ import pytest
4	4	from buildstream import _yaml
5	5
6	6	from tests.testutils import cli_integration as cli
	7	+from tests.testutils.python_repo import setup_pypi_repo
7	8	from tests.testutils.integration import assert_contains
8	9
9	10
...	...	@@ -17,12 +18,21 @@ DATA_DIR = os.path.join(
17	18
18	19
19	20	@pytest.mark.datafiles(DATA_DIR)
20		-def test_pip_source_import(cli, tmpdir, datafiles):
	21	+def test_pip_source_import(cli, tmpdir, datafiles, setup_pypi_repo):
21	22	project = os.path.join(datafiles.dirname, datafiles.basename)
22	23	checkout = os.path.join(cli.directory, 'checkout')
23	24	element_path = os.path.join(project, 'elements')
24	25	element_name = 'pip/hello.bst'
25	26
	27	+ # check that exotically named packages are imported correctly
	28	+ myreqs_packages = ['hellolib']
	29	+ packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
	30	+
	31	+ # create mock pypi repository
	32	+ pypi_repo = os.path.join(project, 'files', 'pypi-repo')
	33	+ os.makedirs(pypi_repo, exist_ok=True)
	34	+ setup_pypi_repo(myreqs_packages + packages, pypi_repo)
	35	+
26	36	element = {
27	37	'kind': 'import',
28	38	'sources': [
...	...	@@ -32,9 +42,9 @@ def test_pip_source_import(cli, tmpdir, datafiles):
32	42	},
33	43	{
34	44	'kind': 'pip',
35		- 'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
	45	+ 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
36	46	'requirements-files': ['myreqs.txt'],
37		- 'packages': ['app2']
	47	+ 'packages': packages
38	48	}
39	49	]
40	50	}
...	...	@@ -51,16 +61,31 @@ def test_pip_source_import(cli, tmpdir, datafiles):
51	61	assert result.exit_code == 0
52	62
53	63	assert_contains(checkout, ['/.bst_pip_downloads',
54		- '/.bst_pip_downloads/HelloLib-0.1.tar.gz',
55		- '/.bst_pip_downloads/App2-0.1.tar.gz'])
	64	+ '/.bst_pip_downloads/hellolib-0.1.tar.gz',
	65	+ '/.bst_pip_downloads/app2-0.1.tar.gz',
	66	+ '/.bst_pip_downloads/app.3-0.1.tar.gz',
	67	+ '/.bst_pip_downloads/app-4-0.1.tar.gz',
	68	+ '/.bst_pip_downloads/app_5-0.1.tar.gz',
	69	+ '/.bst_pip_downloads/app.no.6-0.1.tar.gz',
	70	+ '/.bst_pip_downloads/app-no-7-0.1.tar.gz',
	71	+ '/.bst_pip_downloads/app_no_8-0.1.tar.gz'])
56	72
57	73
58	74	@pytest.mark.datafiles(DATA_DIR)
59		-def test_pip_source_build(cli, tmpdir, datafiles):
	75	+def test_pip_source_build(cli, tmpdir, datafiles, setup_pypi_repo):
60	76	project = os.path.join(datafiles.dirname, datafiles.basename)
61	77	element_path = os.path.join(project, 'elements')
62	78	element_name = 'pip/hello.bst'
63	79
	80	+ # check that exotically named packages are imported correctly
	81	+ myreqs_packages = ['hellolib']
	82	+ packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
	83	+
	84	+ # create mock pypi repository
	85	+ pypi_repo = os.path.join(project, 'files', 'pypi-repo')
	86	+ os.makedirs(pypi_repo, exist_ok=True)
	87	+ setup_pypi_repo(myreqs_packages + packages, pypi_repo)
	88	+
64	89	element = {
65	90	'kind': 'manual',
66	91	'depends': ['base.bst'],
...	...	@@ -71,16 +96,15 @@ def test_pip_source_build(cli, tmpdir, datafiles):
71	96	},
72	97	{
73	98	'kind': 'pip',
74		- 'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
	99	+ 'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
75	100	'requirements-files': ['myreqs.txt'],
76		- 'packages': ['app2']
	101	+ 'packages': packages
77	102	}
78	103	],
79	104	'config': {
80	105	'install-commands': [
81	106	'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz',
82		- 'chmod +x app1.py',
83		- 'install app1.py %{install-root}/usr/bin/'
	107	+ 'install app1.py %{install-root}/usr/bin/'
84	108	]
85	109	}
86	110	}
...	...	@@ -95,5 +119,4 @@ def test_pip_source_build(cli, tmpdir, datafiles):
95	119
96	120	result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py'])
97	121	assert result.exit_code == 0
98		- assert result.output == """Hello App1!
99		-"""
	122	+ assert result.output == "Hello App1! This is hellolib\n"

tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz deleted

---

No preview for this file type

tests/integration/project/files/pypi-repo/app2/index.html deleted

---

1		-<html>
2		- <head>
3		- <title>Links for app1</title>
4		- </head>
5		- <body>
6		- <a href="">'App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
7		- </body>
8		-</html>

tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz deleted

---

No preview for this file type

tests/integration/project/files/pypi-repo/hellolib/index.html deleted

---

1		-<html>
2		- <head>
3		- <title>Links for app1</title>
4		- </head>
5		- <body>
6		- <a href="">'HelloLib-0.1.tar.gz'>HelloLib-0.1.tar.gz</a><br />
7		- </body>
8		-</html>

tests/sources/pip.py

---

...	...	@@ -3,6 +3,7 @@ import pytest
3	3
4	4	from buildstream._exceptions import ErrorDomain
5	5	from buildstream import _yaml
	6	+from buildstream.plugins.sources.pip import _match_package_name
6	7	from tests.testutils import cli
7	8
8	9	DATA_DIR = os.path.join(
...	...	@@ -45,3 +46,22 @@ def test_no_packages(cli, tmpdir, datafiles):
45	46	'show', 'target.bst'
46	47	])
47	48	result.assert_main_error(ErrorDomain.SOURCE, None)
	49	+
	50	+
	51	+# Test that pip source parses tar ball names correctly for the ref
	52	+@pytest.mark.parametrize(
	53	+ 'tarball, expected_name, expected_version',
	54	+ [
	55	+ ('dotted.package-0.9.8.tar.gz', 'dotted.package', '0.9.8'),
	56	+ ('hyphenated-package-2.6.0.tar.gz', 'hyphenated-package', '2.6.0'),
	57	+ ('underscore_pkg-3.1.0.tar.gz', 'underscore_pkg', '3.1.0'),
	58	+ ('numbers2and5-1.0.1.tar.gz', 'numbers2and5', '1.0.1'),
	59	+ ('multiple.dots.package-5.6.7.tar.gz', 'multiple.dots.package', '5.6.7'),
	60	+ ('multiple-hyphens-package-1.2.3.tar.gz', 'multiple-hyphens-package', '1.2.3'),
	61	+ ('multiple_underscore_pkg-3.4.5.tar.gz', 'multiple_underscore_pkg', '3.4.5'),
	62	+ ('shortversion-1.0.tar.gz', 'shortversion', '1.0'),
	63	+ ('longversion-1.2.3.4.tar.gz', 'longversion', '1.2.3.4')
	64	+ ])
	65	+def test_match_package_name(tarball, expected_name, expected_version):
	66	+ name, version = _match_package_name(tarball)
	67	+ assert (expected_name, expected_version) == (name, version)

tests/testutils/__init__.py

---

...	...	@@ -29,3 +29,4 @@ from .artifactshare import create_artifact_share
29	29	from .element_generators import create_element_size, update_element_size
30	30	from .junction import generate_junction
31	31	from .runner_integration import wait_for_cache_granularity
	32	+from .python_repo import setup_pypi_repo

tests/testutils/python_repo.py

---

	1	+from setuptools.sandbox import run_setup
	2	+import os
	3	+import pytest
	4	+import re
	5	+import shutil
	6	+
	7	+
	8	+SETUP_TEMPLATE = '''\
	9	+from setuptools import setup
	10	+
	11	+setup(
	12	+ name='{name}',
	13	+ version='{version}',
	14	+ description='{name}',
	15	+ packages=['{pkgdirname}'],
	16	+ entry_points={{
	17	+ 'console_scripts': [
	18	+ '{pkgdirname}={pkgdirname}:main'
	19	+ ]
	20	+ }}
	21	+)
	22	+'''
	23	+
	24	+# All packages generated via generate_pip_package will have the functions below
	25	+INIT_TEMPLATE = '''\
	26	+def main():
	27	+ print('This is {name}')
	28	+
	29	+def hello(actor='world'):
	30	+ print('Hello {{}}! This is {name}'.format(actor))
	31	+'''
	32	+
	33	+HTML_TEMPLATE = '''\
	34	+<html>
	35	+ <head>
	36	+ <title>Links for {name}</title>
	37	+ </head>
	38	+ <body>
	39	+ <a href=''>{name}-{version}.tar.gz</a><br />
	40	+ </body>
	41	+</html>
	42	+'''
	43	+
	44	+
	45	+# Creates a simple python source distribution and copies this into a specified
	46	+# directory which is to serve as a mock python repository
	47	+#
	48	+# Args:
	49	+# tmpdir (str): Directory in which the source files will be created
	50	+# pypi (str): Directory serving as a mock python repository
	51	+# name (str): The name of the package to be created
	52	+# version (str): The version of the package to be created
	53	+#
	54	+# Returns:
	55	+# None
	56	+#
	57	+def generate_pip_package(tmpdir, pypi, name, version='0.1'):
	58	+ # check if package already exists in pypi
	59	+ pypi_package = os.path.join(pypi, re.sub('[^0-9a-zA-Z]+', '-', name))
	60	+ if os.path.exists(pypi_package):
	61	+ return
	62	+
	63	+ # create the package source files in tmpdir resulting in a directory
	64	+ # tree resembling the following structure:
	65	+ #
	66	+ # tmpdir
	67	+ # |-- setup.py
	68	+ # `-- package
	69	+ # `-- __init__.py
	70	+ #
	71	+ setup_file = os.path.join(tmpdir, 'setup.py')
	72	+ pkgdirname = re.sub('[^0-9a-zA-Z]+', '', name)
	73	+ with open(setup_file, 'w') as f:
	74	+ f.write(
	75	+ SETUP_TEMPLATE.format(
	76	+ name=name,
	77	+ version=version,
	78	+ pkgdirname=pkgdirname
	79	+ )
	80	+ )
	81	+ os.chmod(setup_file, 0o755)
	82	+
	83	+ package = os.path.join(tmpdir, pkgdirname)
	84	+ os.makedirs(package)
	85	+
	86	+ main_file = os.path.join(package, '__init__.py')
	87	+ with open(main_file, 'w') as f:
	88	+ f.write(INIT_TEMPLATE.format(name=name))
	89	+ os.chmod(main_file, 0o644)
	90	+
	91	+ run_setup(setup_file, ['sdist'])
	92	+
	93	+ # create directory for this package in pypi resulting in a directory
	94	+ # tree resembling the following structure:
	95	+ #
	96	+ # pypi
	97	+ # `-- pypi_package
	98	+ # |-- index.html
	99	+ # `-- foo-0.1.tar.gz
	100	+ #
	101	+ os.makedirs(pypi_package)
	102	+
	103	+ # add an index html page
	104	+ index_html = os.path.join(pypi_package, 'index.html')
	105	+ with open(index_html, 'w') as f:
	106	+ f.write(HTML_TEMPLATE.format(name=name, version=version))
	107	+
	108	+ # copy generated tarfile to pypi package
	109	+ dist_dir = os.path.join(tmpdir, 'dist')
	110	+ for tar in os.listdir(dist_dir):
	111	+ tarpath = os.path.join(dist_dir, tar)
	112	+ shutil.copy(tarpath, pypi_package)
	113	+
	114	+
	115	+@pytest.fixture
	116	+def setup_pypi_repo(tmpdir):
	117	+ def create_pkgdir(package):
	118	+ pkgdirname = re.sub('[^0-9a-zA-Z]+', '', package)
	119	+ pkgdir = os.path.join(str(tmpdir), pkgdirname)
	120	+ os.makedirs(pkgdir)
	121	+ return pkgdir
	122	+
	123	+ def add_packages(packages, pypi_repo):
	124	+ for package in packages:
	125	+ pkgdir = create_pkgdir(package)
	126	+ generate_pip_package(pkgdir, pypi_repo, package)
	127	+
	128	+ return add_packages

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.