[Notes] [Git][BuildStream/buildstream][willsalmon/CacheExpiryTest] Trying to mitigate a mtime granularity braking the cache tests

[Will Salmon <gitlab mg gitlab com>]

Title: GitLab

Will Salmon pushed to branch willsalmon/CacheExpiryTest at BuildStream / buildstream

Commits:

• 177c4264
  by William Salmon at 2018-08-16T09:52:38Z

  Trying to mitigate a mtime granularity braking the cache tests
  
  This patch mitigates the granularity of the mtimes used on the default
  gitlab runners, Allowing the test suite to pass on these runners.
  

5 changed files:

• tests/artifactcache/expiry.py
• tests/frontend/push.py
• tests/frontend/workspace.py
• tests/testutils/__init__.py
• + tests/testutils/runner_integration.py

Changes:

tests/artifactcache/expiry.py

---

	1	+#
	2	+# Copyright (C) 2018 Codethink Limited
	3	+#
	4	+# This program is free software; you can redistribute it and/or
	5	+# modify it under the terms of the GNU Lesser General Public
	6	+# License as published by the Free Software Foundation; either
	7	+# version 2 of the License, or (at your option) any later version.
	8	+#
	9	+# This library is distributed in the hope that it will be useful,
	10	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	12	+# Lesser General Public License for more details.
	13	+#
	14	+# You should have received a copy of the GNU Lesser General Public
	15	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	16	+#
	17	+# Authors: Tristan Maat <tristan maat codethink co uk>
	18	+#
	19	+
1	20	import os
2	21
3	22	import pytest
...	...	@@ -5,7 +24,7 @@ import pytest
5	24	from buildstream import _yaml
6	25	from buildstream._exceptions import ErrorDomain, LoadErrorReason
7	26
8		-from tests.testutils import cli, create_element_size
	27	+from tests.testutils import cli, create_element_size, wait_for_cache_granularity
9	28
10	29
11	30	DATA_DIR = os.path.join(
...	...	@@ -108,6 +127,8 @@ def test_expiry_order(cli, datafiles, tmpdir):
108	127	res = cli.run(project=project, args=['build', 'target2.bst'])
109	128	res.assert_success()
110	129
	130	+ wait_for_cache_granularity()
	131	+
111	132	# Now extract dep.bst
112	133	res = cli.run(project=project, args=['checkout', 'dep.bst', checkout])
113	134	res.assert_success()

tests/frontend/push.py

---

	1	+#
	2	+# Copyright (C) 2018 Codethink Limited
	3	+# Copyright (C) 2018 Bloomberg Finance LP
	4	+#
	5	+# This program is free software; you can redistribute it and/or
	6	+# modify it under the terms of the GNU Lesser General Public
	7	+# License as published by the Free Software Foundation; either
	8	+# version 2 of the License, or (at your option) any later version.
	9	+#
	10	+# This library is distributed in the hope that it will be useful,
	11	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	+# Lesser General Public License for more details.
	14	+#
	15	+# You should have received a copy of the GNU Lesser General Public
	16	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	17	+#
	18	+# Authors: Tristan Van Berkom <tristan vanberkom codethink co uk>
	19	+# Sam Thursfield <sam thursfield codethink co uk>
	20	+# Jürg Billeter <juerg billeter codethink co uk>
	21	+#
	22	+
1	23	import os
2	24	import pytest
3	25
4	26	from buildstream._exceptions import ErrorDomain
5	27	from tests.testutils import cli, create_artifact_share, create_element_size
6		-from tests.testutils import generate_junction
	28	+from tests.testutils import generate_junction, wait_for_cache_granularity
7	29	from . import configure_project
8	30
9	31
...	...	@@ -327,6 +349,8 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
327	349	# Ensure element1 is cached locally
328	350	assert cli.get_element_state(project, 'element1.bst') == 'cached'
329	351
	352	+ wait_for_cache_granularity()
	353	+
330	354	# Create and build the element3 (of 5 MB)
331	355	create_element_size('element3.bst', project, element_path, [], int(5e6))
332	356	result = cli.run(project=project, args=['build', 'element3.bst'])

tests/frontend/workspace.py

---

	1	+#
	2	+# Copyright (C) 2018 Codethink Limited
	3	+# Copyright (C) 2018 Bloomberg Finance LP
	4	+#
	5	+# This program is free software; you can redistribute it and/or
	6	+# modify it under the terms of the GNU Lesser General Public
	7	+# License as published by the Free Software Foundation; either
	8	+# version 2 of the License, or (at your option) any later version.
	9	+#
	10	+# This library is distributed in the hope that it will be useful,
	11	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	+# Lesser General Public License for more details.
	14	+#
	15	+# You should have received a copy of the GNU Lesser General Public
	16	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	17	+#
	18	+# Authors: Tristan Van Berkom <tristan vanberkom codethink co uk>
	19	+# Tristan Maat <tristan maat codethink co uk>
	20	+# Chandan Singh <csingh43 bloomberg net>
	21	+# Phillip Smyth <phillip smyth codethink co uk>
	22	+# Jonathan Maw <jonathan maw codethink co uk>
	23	+# Richard Maw <richard maw codethink co uk>
	24	+#
	25	+
1	26	import os
2	27	import pytest
3	28	import shutil
4	29	import subprocess
5	30	from ruamel.yaml.comments import CommentedSet
6		-from tests.testutils import cli, create_repo, ALL_REPO_KINDS
	31	+from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
7	32
8	33	from buildstream import _yaml
9	34	from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
...	...	@@ -507,6 +532,8 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
507	532	assert cli.get_element_state(project, element_name) == 'cached'
508	533	assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
509	534
	535	+ wait_for_cache_granularity()
	536	+
510	537	# Modify the workspace in various different ways, ensuring we
511	538	# properly detect the changes.
512	539	#

tests/testutils/__init__.py

---

	1	+#
	2	+# Copyright (C) 2018 Codethink Limited
	3	+# Copyright (C) 2018 Bloomberg Finance LP
	4	+#
	5	+# This program is free software; you can redistribute it and/or
	6	+# modify it under the terms of the GNU Lesser General Public
	7	+# License as published by the Free Software Foundation; either
	8	+# version 2 of the License, or (at your option) any later version.
	9	+#
	10	+# This library is distributed in the hope that it will be useful,
	11	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	+# Lesser General Public License for more details.
	14	+#
	15	+# You should have received a copy of the GNU Lesser General Public
	16	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	17	+#
	18	+# Authors: Tristan Van Berkom <tristan vanberkom codethink co uk>
	19	+# Tristan Maat <tristan maat codethink co uk>
	20	+# Sam Thursfield <sam thursfield codethink co uk>
	21	+# James Ennis <james ennis codethink co uk>
	22	+# Valentin David <valentin david codethink co uk>
	23	+# William Salmon <will salmon codethink co uk>
	24	+#
	25	+
1	26	from .runcli import cli, cli_integration
2	27	from .repo import create_repo, ALL_REPO_KINDS
3	28	from .artifactshare import create_artifact_share
4	29	from .element_generators import create_element_size
5	30	from .junction import generate_junction
	31	+from .runner_integration import wait_for_cache_granularity

tests/testutils/runner_integration.py

---

	1	+#
	2	+# Copyright (C) 2018 Bloomberg Finance LP
	3	+#
	4	+# This program is free software; you can redistribute it and/or
	5	+# modify it under the terms of the GNU Lesser General Public
	6	+# License as published by the Free Software Foundation; either
	7	+# version 2 of the License, or (at your option) any later version.
	8	+#
	9	+# This library is distributed in the hope that it will be useful,
	10	+# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	12	+# Lesser General Public License for more details.
	13	+#
	14	+# You should have received a copy of the GNU Lesser General Public
	15	+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	16	+#
	17	+# Authors:
	18	+# Will Salmon <will salmon codethink co uk>
	19	+
	20	+import time
	21	+
	22	+
	23	+def wait_for_cache_granularity():
	24	+ # This isn't called very often so has minimal impact on test runtime.
	25	+ # If this changes it may be worth while adding a more sophisticated approach.
	26	+ """
	27	+ Mitigate the coarse granularity of the gitlab runners mtime
	28	+
	29	+ This function waits for the mtime to increment so that the cache can sort by mtime and
	30	+ get the most recent results.
	31	+ """
	32	+ time.sleep(1.1)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.