[Notes] [Git][BuildStream/buildstream][master] 2 commits: remote.py: Add support for marking downloaded files executable

[Tristan Van Berkom <gitlab mg gitlab com>]

Title: GitLab

Tristan Van Berkom pushed to branch master at BuildStream / buildstream

Commits:

• 3d26ff6e
  by Ed Baunton at 2018-08-07T05:04:50Z

  remote.py: Add support for marking downloaded files executable
  
  Add an optional flag to make files executable after having downloaded them.
  
  Instead of leaving the permissioning of downloaded file in remote.py up
  to the user's umask; expressly set permissions to 0644 or 0755 if
  executable.
  
  Bump format version to 13.
  

• 60a552a5
  by Tristan Van Berkom at 2018-08-07T06:13:53Z

  Merge branch 'edbaunton/executable-remote-source' into 'master'
  
  remote.py: Add support for marking downloaded files executable
  
  See merge request BuildStream/buildstream!581

4 changed files:

• buildstream/_versions.py
• buildstream/plugins/sources/remote.py
• tests/sources/remote.py
• + tests/sources/remote/unique-keys/target-custom-executable.bst

Changes:

buildstream/_versions.py

---

...	...	@@ -23,7 +23,7 @@
23	23	# This version is bumped whenever enhancements are made
24	24	# to the `project.conf` format or the core element format.
25	25	#
26		-BST_FORMAT_VERSION = 12
	26	+BST_FORMAT_VERSION = 13
27	27
28	28
29	29	# The base BuildStream artifact version

buildstream/plugins/sources/remote.py

---

...	...	@@ -35,6 +35,10 @@ remote - stage files from remote urls
35	35	# If not specified, the basename of the url will be used.
36	36	# filename: customfilename
37	37
	38	+ # Optionally specify whether the downloaded file should be
	39	+ # marked executable.
	40	+ # executable: true
	41	+
38	42	# Specify the url. Using an alias defined in your project
39	43	# configuration is encouraged. 'bst track' will update the
40	44	# sha256sum in 'ref' to the downloaded file's sha256sum.
...	...	@@ -43,6 +47,8 @@ remote - stage files from remote urls
43	47	# Specify the ref. It's a sha256sum of the file you download.
44	48	ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
45	49
	50	+
	51	+
46	52	.. note::
47	53
48	54	The ``remote`` plugin is available since :ref:`format version 10 <project_format_version>`
...	...	@@ -60,22 +66,31 @@ class RemoteSource(DownloadableFileSource):
60	66	super().configure(node)
61	67
62	68	self.filename = self.node_get_member(node, str, 'filename', os.path.basename(self.url))
	69	+ self.executable = self.node_get_member(node, bool, 'executable', False)
63	70
64	71	if os.sep in self.filename:
65	72	raise SourceError('{}: filename parameter cannot contain directories'.format(self),
66	73	reason="filename-contains-directory")
67		- self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename'])
	74	+ self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename', 'executable'])
68	75
69	76	def get_unique_key(self):
70		- return super().get_unique_key() + [self.filename]
	77	+ return super().get_unique_key() + [self.filename, self.executable]
71	78
72	79	def stage(self, directory):
73	80	# Same as in local plugin, don't use hardlinks to stage sources, they
74	81	# are not write protected in the sandbox.
75	82	dest = os.path.join(directory, self.filename)
76	83	with self.timed_activity("Staging remote file to {}".format(dest)):
	84	+
77	85	utils.safe_copy(self._get_mirror_file(), dest)
78	86
	87	+ # To prevent user's umask introducing variability here, explicitly set
	88	+ # file modes.
	89	+ if self.executable:
	90	+ os.chmod(dest, 0o755)
	91	+ else:
	92	+ os.chmod(dest, 0o644)
	93	+
79	94
80	95	def setup():
81	96	return RemoteSource

tests/sources/remote.py

---

1	1	import os
	2	+import stat
2	3	import pytest
3	4
4	5	from buildstream._exceptions import ErrorDomain
...	...	@@ -82,7 +83,14 @@ def test_simple_file_build(cli, tmpdir, datafiles):
82	83	result.assert_success()
83	84	# Note that the url of the file in target.bst is actually /dir/file
84	85	# but this tests confirms we take the basename
85		- assert(os.path.exists(os.path.join(checkoutdir, 'file')))
	86	+ checkout_file = os.path.join(checkoutdir, 'file')
	87	+ assert(os.path.exists(checkout_file))
	88	+
	89	+ mode = os.stat(checkout_file).st_mode
	90	+ # Assert not executable by anyone
	91	+ assert(not (mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)))
	92	+ # Assert not writeable by anyone other than me
	93	+ assert(not (mode & (stat.S_IWGRP | stat.S_IWOTH)))
86	94
87	95
88	96	@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file-custom-name'))
...	...	@@ -119,6 +127,7 @@ def test_unique_key(cli, tmpdir, datafiles):
119	127	generate_project(project, tmpdir)
120	128	assert cli.get_element_state(project, 'target.bst') == "fetch needed"
121	129	assert cli.get_element_state(project, 'target-custom.bst') == "fetch needed"
	130	+ assert cli.get_element_state(project, 'target-custom-executable.bst') == "fetch needed"
122	131	# Try to fetch it
123	132	result = cli.run(project=project, args=[
124	133	'fetch', 'target.bst'
...	...	@@ -127,7 +136,31 @@ def test_unique_key(cli, tmpdir, datafiles):
127	136	# We should download the file only once
128	137	assert cli.get_element_state(project, 'target.bst') == 'buildable'
129	138	assert cli.get_element_state(project, 'target-custom.bst') == 'buildable'
	139	+ assert cli.get_element_state(project, 'target-custom-executable.bst') == 'buildable'
130	140
131	141	# But the cache key is different because the 'filename' is different.
132	142	assert cli.get_element_key(project, 'target.bst') != \
133		- cli.get_element_key(project, 'target-custom.bst')
	143	+ cli.get_element_key(project, 'target-custom.bst') != \
	144	+ cli.get_element_key(project, 'target-custom-executable.bst')
	145	+
	146	+
	147	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'unique-keys'))
	148	+def test_executable(cli, tmpdir, datafiles):
	149	+ '''This test confirms that the 'ecxecutable' parameter is honoured.
	150	+ '''
	151	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	152	+ generate_project(project, tmpdir)
	153	+ checkoutdir = os.path.join(str(tmpdir), "checkout")
	154	+ assert cli.get_element_state(project, 'target-custom-executable.bst') == "fetch needed"
	155	+ # Try to fetch it
	156	+ result = cli.run(project=project, args=[
	157	+ 'build', 'target-custom-executable.bst'
	158	+ ])
	159	+
	160	+ result = cli.run(project=project, args=[
	161	+ 'checkout', 'target-custom-executable.bst', checkoutdir
	162	+ ])
	163	+ mode = os.stat(os.path.join(checkoutdir, 'some-custom-file')).st_mode
	164	+ assert (mode & stat.S_IEXEC)
	165	+ # Assert executable by anyone
	166	+ assert(mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH))

tests/sources/remote/unique-keys/target-custom-executable.bst

---

	1	+kind: import
	2	+description: test
	3	+sources:
	4	+- kind: remote
	5	+ url: tmpdir:/dir/file
	6	+ ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
	7	+ filename: some-custom-file
	8	+ executable: true

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.