[Notes] [Git][BuildStream/buildstream][chandan/source-checkout] 4 commits: element.py: Fix cache check in non-strict mode

[Chandan Singh <gitlab mg gitlab com>]

Title: GitLab

Chandan Singh pushed to branch chandan/source-checkout at BuildStream / buildstream

Commits:

• da7e038b
  by Jürg Billeter at 2018-09-24T13:58:55Z

  element.py: Fix cache check in non-strict mode
  
  The behavior of __assert_cached() was changed when the keystrength
  parameter was introduced. This restores the previous behavior.
  
  Fixes #607.
  
  Fixes: 3d308894 ("element.py: Cache the result of checking whether...")
  

• 86ea1173
  by Jürg Billeter at 2018-09-24T13:59:59Z

  tests/frontend/rebuild.py: Add strict/non-strict rebuild test
  
  Test rebuild that requires staging of weakly cached dependency.
  
  This is a regression test for #607.
  

• a76339de
  by Jürg Billeter at 2018-09-24T16:02:38Z

  Merge branch 'juerg/rebuild' into 'master'
  
  element.py: Fix cache check in non-strict mode
  
  Closes #607
  
  See merge request BuildStream/buildstream!822

• c2690049
  by Chandan Singh at 2018-09-25T00:43:57Z

  Add `bst source-checkout` command
  
  As discussed in
  https://mail.gnome.org/archives/buildstream-list/2018-September/msg00064.html,
  add `bst source-checkout` command. This will allow users to checkout
  sources for a given target.
  

9 changed files:

• buildstream/_frontend/cli.py
• buildstream/_stream.py
• buildstream/element.py
• tests/completions/completions.py
• + tests/frontend/project/elements/checkout-deps.bst
• + tests/frontend/project/elements/rebuild-target.bst
• + tests/frontend/project/files/etc-files/etc/buildstream/config
• + tests/frontend/rebuild.py
• + tests/frontend/source_checkout.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -662,6 +662,30 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
662	662	tar=tar)
663	663
664	664
	665	+##################################################################
	666	+# Source Checkout Command #
	667	+##################################################################
	668	+@cli.command(name="source-checkout", short_help="Checkout sources for an element")
	669	+@click.option('--except', 'except_', multiple=True,
	670	+ type=click.Path(readable=False),
	671	+ help="Except certain dependencies")
	672	+@click.option('--deps', '-d', default='none',
	673	+ type=click.Choice(['build', 'none', 'run', 'all']),
	674	+ help='The dependencies whose sources to checkout (default: none)')
	675	+@click.argument('element',
	676	+ type=click.Path(readable=False))
	677	+@click.argument('location', type=click.Path())
	678	+@click.pass_obj
	679	+def source_checkout(app, element, location, deps, except_):
	680	+ """Checkout sources of an element to the specified location
	681	+ """
	682	+ with app.initialized():
	683	+ app.stream.source_checkout(element,
	684	+ location=location,
	685	+ deps=deps,
	686	+ except_targets=except_)
	687	+
	688	+
665	689	##################################################################
666	690	# Workspace Command #
667	691	##################################################################

buildstream/_stream.py

---

...	...	@@ -381,27 +381,7 @@ class Stream():
381	381	elements, _ = self._load((target,), (), fetch_subprojects=True)
382	382	target = elements[0]
383	383
384		- if not tar:
385		- try:
386		- os.makedirs(location, exist_ok=True)
387		- except OSError as e:
388		- raise StreamError("Failed to create checkout directory: '{}'"
389		- .format(e)) from e
390		-
391		- if not tar:
392		- if not os.access(location, os.W_OK):
393		- raise StreamError("Checkout directory '{}' not writable"
394		- .format(location))
395		- if not force and os.listdir(location):
396		- raise StreamError("Checkout directory '{}' not empty"
397		- .format(location))
398		- elif os.path.exists(location) and location != '-':
399		- if not os.access(location, os.W_OK):
400		- raise StreamError("Output file '{}' not writable"
401		- .format(location))
402		- if not force and os.path.exists(location):
403		- raise StreamError("Output file '{}' already exists"
404		- .format(location))
	384	+ self.__check_location_writable(location, force=force, tar=tar)
405	385
406	386	# Stage deps into a temporary sandbox first
407	387	try:
...	...	@@ -438,6 +418,35 @@ class Stream():
438	418	raise StreamError("Error while staging dependencies into a sandbox"
439	419	": '{}'".format(e), detail=e.detail, reason=e.reason) from e
440	420
	421	+ # source_checkout()
	422	+ #
	423	+ # Checkout sources of the target element to the specified location
	424	+ #
	425	+ # Args:
	426	+ # target (str): The target element whose sources to checkout
	427	+ # location (str): Location to checkout the sources to
	428	+ # deps (str): The dependencies to checkout
	429	+ # except_targets (list): List of targets to except from staging
	430	+ #
	431	+ def source_checkout(self, target, *,
	432	+ location=None,
	433	+ deps='none',
	434	+ except_targets=()):
	435	+
	436	+ self.__check_location_writable(location)
	437	+
	438	+ elements, _ = self._load((target,), (),
	439	+ selection=deps,
	440	+ except_targets=except_targets,
	441	+ fetch_subprojects=True)
	442	+
	443	+ # Stage all sources determined by scope
	444	+ try:
	445	+ self._write_element_sources(location, elements)
	446	+ except BstError as e:
	447	+ raise StreamError("Error while writing sources"
	448	+ ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
	449	+
441	450	# workspace_open
442	451	#
443	452	# Open a project workspace
...	...	@@ -721,7 +730,7 @@ class Stream():
721	730	if self._write_element_script(source_directory, element)
722	731	]
723	732
724		- self._write_element_sources(tempdir, elements)
	733	+ self._write_element_sources(os.path.join(tempdir, "source"), elements)
725	734	self._write_build_script(tempdir, elements)
726	735	self._collect_sources(tempdir, tar_location,
727	736	target.normal_name, compression)
...	...	@@ -1084,11 +1093,10 @@ class Stream():
1084	1093	# Write all source elements to the given directory
1085	1094	def _write_element_sources(self, directory, elements):
1086	1095	for element in elements:
1087		- source_dir = os.path.join(directory, "source")
1088		- element_source_dir = os.path.join(source_dir, element.normal_name)
1089		- os.makedirs(element_source_dir)
1090		-
1091		- element._stage_sources_at(element_source_dir)
	1096	+ element_source_dir = os.path.join(directory, element.normal_name)
	1097	+ if list(element.sources()):
	1098	+ os.makedirs(element_source_dir)
	1099	+ element._stage_sources_at(element_source_dir)
1092	1100
1093	1101	# Write a master build script to the sandbox
1094	1102	def _write_build_script(self, directory, elements):
...	...	@@ -1117,3 +1125,29 @@ class Stream():
1117	1125
1118	1126	with tarfile.open(tar_name, permissions) as tar:
1119	1127	tar.add(directory, arcname=element_name)
	1128	+
	1129	+ #############################################################
	1130	+ # Private Methods #
	1131	+ #############################################################
	1132	+
	1133	+ # Check if given location is writable
	1134	+ def __check_location_writable(self, location, force=False, tar=False):
	1135	+ if not tar:
	1136	+ try:
	1137	+ os.makedirs(location, exist_ok=True)
	1138	+ except OSError as e:
	1139	+ raise StreamError("Failed to create checkout directory: '{}'"
	1140	+ .format(e)) from e
	1141	+ if not os.access(location, os.W_OK):
	1142	+ raise StreamError("Checkout directory '{}' not writable"
	1143	+ .format(location))
	1144	+ if not force and os.listdir(location):
	1145	+ raise StreamError("Checkout directory '{}' not empty"
	1146	+ .format(location))
	1147	+ elif os.path.exists(location) and location != '-':
	1148	+ if not os.access(location, os.W_OK):
	1149	+ raise StreamError("Output file '{}' not writable"
	1150	+ .format(location))
	1151	+ if not force and os.path.exists(location):
	1152	+ raise StreamError("Output file '{}' already exists"
	1153	+ .format(location))

buildstream/element.py

---

...	...	@@ -1366,6 +1366,14 @@ class Element(Plugin):
1366	1366	def _stage_sources_at(self, vdirectory, mount_workspaces=True):
1367	1367	with self.timed_activity("Staging sources", silent_nested=True):
1368	1368
	1369	+ if self._get_consistency() != Consistency.CACHED:
	1370	+ detail = "Sources for this element have not been catched yet.\n" + \
	1371	+ "Try fetching the sources first with `bst fetch`. \n" + \
	1372	+ "Alternatively, run this command with `--fetch` option."
	1373	+ raise ElementError("Sources are not cached",
	1374	+ detail=detail,
	1375	+ reason="uncached-source-checkout-attempt")
	1376	+
1369	1377	if not isinstance(vdirectory, Directory):
1370	1378	vdirectory = FileBasedDirectory(vdirectory)
1371	1379	if not vdirectory.is_empty():
...	...	@@ -2084,7 +2092,7 @@ class Element(Plugin):
2084	2092	#
2085	2093	# Raises an error if the artifact is not cached.
2086	2094	#
2087		- def __assert_cached(self, keystrength=_KeyStrength.STRONG):
	2095	+ def __assert_cached(self, keystrength=None):
2088	2096	assert self.__is_cached(keystrength=keystrength), "{}: Missing artifact {}".format(
2089	2097	self, self._get_brief_display_key())
2090	2098

tests/completions/completions.py

---

...	...	@@ -15,6 +15,7 @@ MAIN_COMMANDS = [
15	15	'push ',
16	16	'shell ',
17	17	'show ',
	18	+ 'source-checkout ',
18	19	'source-bundle ',
19	20	'track ',
20	21	'workspace '

tests/frontend/project/elements/checkout-deps.bst

---

	1	+kind: import
	2	+description: It is important for this element to have both build and runtime dependencies
	3	+sources:
	4	+- kind: local
	5	+ path: files/etc-files
	6	+depends:
	7	+- filename: import-dev.bst
	8	+ type: build
	9	+- filename: import-bin.bst
	10	+ type: runtime

tests/frontend/project/elements/rebuild-target.bst

---

	1	+kind: compose
	2	+
	3	+build-depends:
	4	+- target.bst

tests/frontend/project/files/etc-files/etc/buildstream/config

---

1

+config

tests/frontend/rebuild.py

---

	1	+import os
	2	+import pytest
	3	+from tests.testutils import cli
	4	+
	5	+# Project directory
	6	+DATA_DIR = os.path.join(
	7	+ os.path.dirname(os.path.realpath(__file__)),
	8	+ "project",
	9	+)
	10	+
	11	+
	12	+def strict_args(args, strict):
	13	+ if strict != "strict":
	14	+ return ['--no-strict'] + args
	15	+ return args
	16	+
	17	+
	18	+@pytest.mark.datafiles(DATA_DIR)
	19	+@pytest.mark.parametrize("strict", ["strict", "non-strict"])
	20	+def test_rebuild(datafiles, cli, strict):
	21	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	22	+ checkout = os.path.join(cli.directory, 'checkout')
	23	+
	24	+ # First build intermediate target.bst
	25	+ result = cli.run(project=project, args=strict_args(['build', 'target.bst'], strict))
	26	+ result.assert_success()
	27	+
	28	+ # Modify base import
	29	+ with open(os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'new.h'), "w") as f:
	30	+ f.write("#define NEW")
	31	+
	32	+ # Rebuild base import and build top-level rebuild-target.bst
	33	+ # In non-strict mode, this does not rebuild intermediate target.bst,
	34	+ # which means that a weakly cached target.bst will be staged as dependency.
	35	+ result = cli.run(project=project, args=strict_args(['build', 'rebuild-target.bst'], strict))
	36	+ result.assert_success()

tests/frontend/source_checkout.py

---

	1	+import os
	2	+import pytest
	3	+
	4	+from tests.testutils import cli
	5	+
	6	+# Project directory
	7	+DATA_DIR = os.path.join(
	8	+ os.path.dirname(os.path.realpath(__file__)),
	9	+ "project",
	10	+)
	11	+
	12	+
	13	+@pytest.mark.datafiles(DATA_DIR)
	14	+def test_source_checkout(datafiles, cli):
	15	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	16	+ checkout = os.path.join(cli.directory, 'source-checkout')
	17	+ target = 'checkout-deps.bst'
	18	+
	19	+ result = cli.run(project=project, args=['source-checkout', target, '--deps', 'none', checkout])
	20	+ result.assert_success()
	21	+
	22	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	23	+
	24	+
	25	+@pytest.mark.datafiles(DATA_DIR)
	26	+@pytest.mark.parametrize('deps', [("build"), ("none"), ("run"), ("all")])
	27	+def test_source_checkout_deps(datafiles, cli, deps):
	28	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	29	+ checkout = os.path.join(cli.directory, 'source-checkout')
	30	+ target = 'checkout-deps.bst'
	31	+
	32	+ result = cli.run(project=project, args=['source-checkout', target, '--deps', deps, checkout])
	33	+ result.assert_success()
	34	+
	35	+ # Sources of the target
	36	+ if deps == 'build':
	37	+ assert not os.path.exists(os.path.join(checkout, 'checkout-deps'))
	38	+ else:
	39	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	40	+
	41	+ # Sources of the target's build dependencies
	42	+ if deps in ('build', 'all'):
	43	+ assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
	44	+ else:
	45	+ assert not os.path.exists(os.path.join(checkout, 'import-dev'))
	46	+
	47	+ # Sources of the target's runtime dependencies
	48	+ if deps in ('run', 'all'):
	49	+ assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello'))
	50	+ else:
	51	+ assert not os.path.exists(os.path.join(checkout, 'import-bin'))
	52	+
	53	+
	54	+@pytest.mark.datafiles(DATA_DIR)
	55	+def test_source_checkout_except(datafiles, cli):
	56	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	57	+ checkout = os.path.join(cli.directory, 'source-checkout')
	58	+ target = 'checkout-deps.bst'
	59	+
	60	+ result = cli.run(project=project, args=['source-checkout', target,
	61	+ '--deps', 'all',
	62	+ '--except', 'import-bin.bst',
	63	+ checkout])
	64	+ result.assert_success()
	65	+
	66	+ # Sources for the target should be present
	67	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	68	+
	69	+ # Sources for import-bin.bst should not be present
	70	+ assert not os.path.exists(os.path.join(checkout, 'import-bin'))
	71	+
	72	+ # Sources for other dependencies should be present
	73	+ assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.