[Notes] [Git][BuildStream/buildstream][tiagogomes/issue-577] 5 commits:

Tiago Gomes pushed to branch tiagogomes/issue-577 at BuildStream / buildstream

Commits:

177c4264

by William Salmon at 2018-08-16T09:52:38Z

Trying to mitigate a mtime granularity braking the cache tests

This patch mitigates the granularity of the mtimes used on the default
gitlab runners, Allowing the test suite to pass on these runners.

01c4ac57

by Tom Pollard at 2018-08-16T11:01:45Z

Merge branch 'willsalmon/CacheExpiryTest' into 'master'

Trying to mitigate a file system issue

See merge request BuildStream/buildstream!595

0066d701

by Tiago Gomes at 2018-08-16T12:33:01Z

Use http instead of https for the gnu ftp mirror

https seems broken on the Debian image:

START   autotools/hello/42930621-fetch.499.log
START   Fetching https://ftpmirror.gnu.org/gnu/automake/automake-1.16.tar.gz
FAILURE Fetching https://ftpmirror.gnu.org/gnu/automake/automake-1.16.tar.gz
FAILURE tar source at hello.bst [line 16 column 2]: Error mirroring https://ftpmirror.gnu.org/gnu/automake/automake-1.16.tar.gz: <urlopen error [SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:720)>

898aa8a0

by Tiago Gomes at 2018-08-16T13:43:06Z

Merge branch 'tiagogomes/disable-https-gnu-repo' into 'master'

Use http instead of https for the gnu ftp mirror

See merge request BuildStream/buildstream!676

b62c361c

by Tiago Gomes at 2018-08-16T13:54:54Z

cascache: use errno module

os.errno does no longer work with Python 3.7

Closes #577.

Changes:

buildstream/_artifactcache/cascache.py

@@ -25,6 +25,7 @@ import signal
  import stat
  import tempfile
  import uuid
 +import errno
  from urllib.parse import urlparse
  import grpc
@@ -82,7 +83,8 @@ class CASCache(ArtifactCache):
          tree = self.resolve_ref(ref, update_mtime=True)
 -        dest = os.path.join(self.extractdir, element._get_project().name, element.normal_name, tree.hash)
 +        dest = os.path.join(self.extractdir, element._get_project().name,
 +                            element.normal_name, tree.hash)
          if os.path.isdir(dest):
              # artifact has already been extracted
              return dest
@@ -100,7 +102,7 @@ class CASCache(ArtifactCache):
+                 #
                  # If rename fails with these errors, another process beat
                  # us to it so just ignore.
 -                if e.errno not in [os.errno.ENOTEMPTY, os.errno.EEXIST]:
 +                if e.errno not in [errno.ENOTEMPTY, errno.EEXIST]:
                      raise ArtifactError("Failed to extract artifact for ref '{}': {}"
                                          .format(ref, e)) from e

doc/examples/autotools/project.conf

@@ -10,4 +10,4 @@ element-path: elements
  # Define some aliases for the tarballs we download
  aliases:
    alpine: https://gnome7.codethink.co.uk/tarballs/
 -  gnu: https://ftpmirror.gnu.org/gnu/automake/
 +  gnu: http://ftpmirror.gnu.org/gnu/automake/

tests/artifactcache/expiry.py

 +#
 +#  Copyright (C) 2018 Codethink Limited
 +#
 +#  This program is free software; you can redistribute it and/or
 +#  modify it under the terms of the GNU Lesser General Public
 +#  License as published by the Free Software Foundation; either
 +#  version 2 of the License, or (at your option) any later version.
 +#
 +#  This library is distributed in the hope that it will be useful,
 +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
 +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 +#  Lesser General Public License for more details.
 +#
 +#  You should have received a copy of the GNU Lesser General Public
 +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 +#
 +#  Authors: Tristan Maat <tristan maat codethink co uk>
 +#
++
  import os
  import pytest
@@ -5,7 +24,7 @@ import pytest
  from buildstream import _yaml
  from buildstream._exceptions import ErrorDomain, LoadErrorReason
 -from tests.testutils import cli, create_element_size
 +from tests.testutils import cli, create_element_size, wait_for_cache_granularity
  DATA_DIR = os.path.join(
@@ -108,6 +127,8 @@ def test_expiry_order(cli, datafiles, tmpdir):
      res = cli.run(project=project, args=['build', 'target2.bst'])
      res.assert_success()
 +    wait_for_cache_granularity()
++
      # Now extract dep.bst
      res = cli.run(project=project, args=['checkout', 'dep.bst', checkout])
      res.assert_success()

tests/frontend/push.py

 +#
 +#  Copyright (C) 2018 Codethink Limited
 +#  Copyright (C) 2018 Bloomberg Finance LP
 +#
 +#  This program is free software; you can redistribute it and/or
 +#  modify it under the terms of the GNU Lesser General Public
 +#  License as published by the Free Software Foundation; either
 +#  version 2 of the License, or (at your option) any later version.
 +#
 +#  This library is distributed in the hope that it will be useful,
 +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
 +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 +#  Lesser General Public License for more details.
 +#
 +#  You should have received a copy of the GNU Lesser General Public
 +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 +#
 +#  Authors: Tristan Van Berkom <tristan vanberkom codethink co uk>
 +#           Sam Thursfield <sam thursfield codethink co uk>
 +#           Jürg Billeter <juerg billeter codethink co uk>
 +#
++
  import os
  import pytest
  from buildstream._exceptions import ErrorDomain
  from tests.testutils import cli, create_artifact_share, create_element_size
 -from tests.testutils import generate_junction
 +from tests.testutils import generate_junction, wait_for_cache_granularity
  from . import configure_project
@@ -327,6 +349,8 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
          # Ensure element1 is cached locally
          assert cli.get_element_state(project, 'element1.bst') == 'cached'
 +        wait_for_cache_granularity()
++
          # Create and build the element3 (of 5 MB)
          create_element_size('element3.bst', project, element_path, [], int(5e6))
          result = cli.run(project=project, args=['build', 'element3.bst'])

tests/frontend/workspace.py

 +#
 +#  Copyright (C) 2018 Codethink Limited
 +#  Copyright (C) 2018 Bloomberg Finance LP
 +#
 +#  This program is free software; you can redistribute it and/or
 +#  modify it under the terms of the GNU Lesser General Public
 +#  License as published by the Free Software Foundation; either
 +#  version 2 of the License, or (at your option) any later version.
 +#
 +#  This library is distributed in the hope that it will be useful,
 +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
 +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 +#  Lesser General Public License for more details.
 +#
 +#  You should have received a copy of the GNU Lesser General Public
 +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 +#
 +#  Authors: Tristan Van Berkom <tristan vanberkom codethink co uk>
 +#           Tristan Maat <tristan maat codethink co uk>
 +#           Chandan Singh <csingh43 bloomberg net>
 +#           Phillip Smyth <phillip smyth codethink co uk>
 +#           Jonathan Maw <jonathan maw codethink co uk>
 +#           Richard Maw <richard maw codethink co uk>
 +#
++
  import os
  import pytest
  import shutil
  import subprocess
  from ruamel.yaml.comments import CommentedSet
 -from tests.testutils import cli, create_repo, ALL_REPO_KINDS
 +from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
  from buildstream import _yaml
  from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
@@ -507,6 +532,8 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
      assert cli.get_element_state(project, element_name) == 'cached'
      assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
 +    wait_for_cache_granularity()
++
      # Modify the workspace in various different ways, ensuring we
      # properly detect the changes.
+     #

tests/testutils/__init__.py

 +#
 +#  Copyright (C) 2018 Codethink Limited
 +#  Copyright (C) 2018 Bloomberg Finance LP
 +#
 +#  This program is free software; you can redistribute it and/or
 +#  modify it under the terms of the GNU Lesser General Public
 +#  License as published by the Free Software Foundation; either
 +#  version 2 of the License, or (at your option) any later version.
 +#
 +#  This library is distributed in the hope that it will be useful,
 +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
 +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 +#  Lesser General Public License for more details.
 +#
 +#  You should have received a copy of the GNU Lesser General Public
 +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 +#
 +#  Authors: Tristan Van Berkom <tristan vanberkom codethink co uk>
 +#           Tristan Maat <tristan maat codethink co uk>
 +#           Sam Thursfield <sam thursfield codethink co uk>
 +#           James Ennis <james ennis codethink co uk>
 +#           Valentin David <valentin david codethink co uk>
 +#           William Salmon <will salmon codethink co uk>
 +#
++
  from .runcli import cli, cli_integration
  from .repo import create_repo, ALL_REPO_KINDS
  from .artifactshare import create_artifact_share
  from .element_generators import create_element_size
  from .junction import generate_junction
 +from .runner_integration import wait_for_cache_granularity

tests/testutils/runner_integration.py

 +#
 +#  Copyright (C) 2018 Bloomberg Finance LP
 +#
 +#  This program is free software; you can redistribute it and/or
 +#  modify it under the terms of the GNU Lesser General Public
 +#  License as published by the Free Software Foundation; either
 +#  version 2 of the License, or (at your option) any later version.
 +#
 +#  This library is distributed in the hope that it will be useful,
 +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
 +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 +#  Lesser General Public License for more details.
 +#
 +#  You should have received a copy of the GNU Lesser General Public
 +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 +#
 +#  Authors:
 +#         Will Salmon <will salmon codethink co uk>
++
 +import time
++
++
 +def wait_for_cache_granularity():
 +    # This isn't called very often so has minimal impact on test runtime.
 +    # If this changes it may be worth while adding a more sophisticated approach.
 +    """
 +    Mitigate the coarse granularity of the gitlab runners mtime
++
 +    This function waits for the mtime to increment so that the cache can sort by mtime and
 +    get the most recent results.
 +    """
 +    time.sleep(1.1)

[Notes] [Git][BuildStream/buildstream][tiagogomes/issue-577] 5 commits: Trying to mitigate a mtime granularity braking the cache tests

Tiago Gomes pushed to branch tiagogomes/issue-577 at BuildStream / buildstream

Commits:

7 changed files:

Changes: