[Notes] [Git][BuildStream/buildstream][juerg/cas-mtime] 8 commits:

Jürg Billeter pushed to branch juerg/cas-mtime at BuildStream / buildstream

Commits:

b8f69946

by Daniel Silverstone at 2018-10-18T09:32:09Z

_loader/loader.py: Be more explanatory in CircDep exception

When a circular dependency is detected, the reported error hilights
only one part of the loop in question.  This change ensures that
the entire loop is listed so that the user has a greater chance of
easily determining the correct fix.

Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>

892866e1

by richardmaw-codethink at 2018-10-18T10:04:32Z

Merge branch 'danielsilverstone-ct/report-loop' into 'master'

_loader/loader.py: Be more explanatory in CircDep exception

See merge request BuildStream/buildstream!878

f1aa0974

by Javier Jardón at 2018-10-18T14:13:35Z

Fix python warnings: Use collections.abc instead collections

Since python 3.3, collections has been moved to collections.abc module.
For backwards compatibility, they continue to be visible in this module through Python 3.7.
Subsequently, they will be removed entirely.

See https://docs.python.org/3/library/collections.html

f1550c8e

by Valentin David at 2018-10-18T16:02:45Z

Merge branch 'jjardon/warnings_collections' into 'master'

Fix python warnings: Use collections.abc instead collections

See merge request BuildStream/buildstream!883

1c24979a

by Angelos Evripiotis at 2018-10-19T21:17:01Z

Spelling fixes: 'affect', not 'effect' as verb

Fix most instances of 'effect' being used as a verb instead of a noun.
Skipped generated instance in doc/source/conf.py.

43a2eee3

by Angelos Evripiotis at 2018-10-19T21:48:59Z

Merge branch 'aevri/affect' into 'master'

Spelling fixes: 'affect', not 'effect' as verb

See merge request BuildStream/buildstream!885

12719f0d

by Jürg Billeter at 2018-10-22T17:05:41Z

_artifactcache: Rename update_atime() to update_mtime()

os.utime() updates both, however, list_artifacts() sorts refs by mtime,
i.e., atime is irrelevant.

a7a28d14

by Jürg Billeter at 2018-10-22T17:05:41Z

_artifactcache: Fix ref in update_mtime()

get_artifact_fullname() is required to construct the ref. The cache key
alone does not suffice.

24 changed files:

buildstream/_artifactcache/artifactcache.py
buildstream/_artifactcache/cascache.py
buildstream/_context.py
buildstream/_includes.py
buildstream/_loader/loadelement.py
buildstream/_loader/loader.py
buildstream/_options/optionpool.py
buildstream/_project.py
buildstream/_scheduler/queues/queue.py
buildstream/_yaml.py
buildstream/buildelement.py
buildstream/element.py
buildstream/plugin.py
buildstream/plugins/elements/autotools.yaml
buildstream/plugins/elements/cmake.yaml
buildstream/plugins/elements/junction.py
buildstream/plugins/elements/make.yaml
buildstream/plugins/elements/manual.yaml
buildstream/plugins/elements/meson.yaml
buildstream/plugins/elements/qmake.yaml
buildstream/plugins/sources/git.py
buildstream/source.py
tests/loader/dependencies.py
tests/yaml/yaml.py

Changes:

buildstream/_artifactcache/artifactcache.py

@@ -19,7 +19,8 @@
  import os
  import string
 -from collections import Mapping, namedtuple
 +from collections import namedtuple
 +from collections.abc import Mapping
  from ..types import _KeyStrength
  from .._exceptions import ArtifactError, ImplError, LoadError, LoadErrorReason
@@ -227,7 +228,7 @@ class ArtifactCache():
          self._required_elements.update(elements)
          # For the cache keys which were resolved so far, we bump
 -        # the atime of them.
 +        # the mtime of them.
+         #
          # This is just in case we have concurrent instances of
          # BuildStream running with the same artifact cache, it will
@@ -239,7 +240,7 @@ class ArtifactCache():
              for key in (strong_key, weak_key):
                  if key:
                      try:
 -                        self.update_atime(key)
 +                        self.update_mtime(element, key)
                      except ArtifactError:
                          pass
@@ -390,15 +391,16 @@ class ArtifactCache():
      def preflight(self):
          pass
 -    # update_atime()
 +    # update_mtime()
+     #
 -    # Update the atime of an artifact.
 +    # Update the mtime of an artifact.
+     #
      # Args:
 +    #     element (Element): The Element to update
      #     key (str): The key of the artifact.
+     #
 -    def update_atime(self, key):
 -        raise ImplError("Cache '{kind}' does not implement contains()"
 +    def update_mtime(self, element, key):
 +        raise ImplError("Cache '{kind}' does not implement update_mtime()"
                          .format(kind=type(self).__name__))
      # initialize_remotes():

buildstream/_artifactcache/cascache.py

@@ -538,8 +538,9 @@ class CASCache(ArtifactCache):
          except FileNotFoundError as e:
              raise ArtifactError("Attempt to access unavailable artifact: {}".format(e)) from e
 -    def update_atime(self, ref):
 +    def update_mtime(self, element, key):
          try:
 +            ref = self.get_artifact_fullname(element, key)
              os.utime(self._refpath(ref))
          except FileNotFoundError as e:
              raise ArtifactError("Attempt to access unavailable artifact: {}".format(e)) from e

buildstream/_context.py

@@ -19,7 +19,8 @@
  import os
  import datetime
 -from collections import deque, Mapping
 +from collections import deque
 +from collections.abc import Mapping
  from contextlib import contextmanager
  from . import utils
  from . import _cachekey

buildstream/_includes.py

  import os
 -from collections import Mapping
 +from collections.abc import Mapping
  from . import _yaml
  from ._exceptions import LoadError, LoadErrorReason

buildstream/_loader/loadelement.py

@@ -18,7 +18,7 @@
  #        Tristan Van Berkom <tristan vanberkom codethink co uk>
  # System imports
 -from collections import Mapping
 +from collections.abc import Mapping
  # BuildStream toplevel imports
  from .._exceptions import LoadError, LoadErrorReason

buildstream/_loader/loader.py

@@ -19,7 +19,8 @@
  import os
  from functools import cmp_to_key
 -from collections import Mapping, namedtuple
 +from collections import namedtuple
 +from collections.abc import Mapping
  import tempfile
  import shutil
@@ -296,12 +297,14 @@ class Loader():
      # Raises:
      #    (LoadError): In case there was a circular dependency error
+     #
 -    def _check_circular_deps(self, element_name, check_elements=None, validated=None):
 +    def _check_circular_deps(self, element_name, check_elements=None, validated=None, sequence=None):
          if check_elements is None:
              check_elements = {}
          if validated is None:
              validated = {}
 +        if sequence is None:
 +            sequence = []
          element = self._elements[element_name]
@@ -314,16 +317,24 @@ class Loader():
              return
          if check_elements.get(element_name) is not None:
 +            # Create `chain`, the loop of element dependencies from this
 +            # element back to itself, by trimming everything before this
 +            # element from the sequence under consideration.
 +            chain = sequence[sequence.index(element_name):]
 +            chain.append(element_name)
              raise LoadError(LoadErrorReason.CIRCULAR_DEPENDENCY,
 -                            "Circular dependency detected for element: {}"
 -                            .format(element.name))
 +                            ("Circular dependency detected at element: {}\n" +
 +                             "Dependency chain: {}")
 +                            .format(element.name, " -> ".join(chain)))
          # Push / Check each dependency / Pop
          check_elements[element_name] = True
 +        sequence.append(element_name)
          for dep in element.deps:
              loader = self._get_loader_for_dep(dep)
 -            loader._check_circular_deps(dep.name, check_elements, validated)
 +            loader._check_circular_deps(dep.name, check_elements, validated, sequence)
          del check_elements[element_name]
 +        sequence.pop()
          # Eliminate duplicate paths
          validated[element_name] = True

buildstream/_options/optionpool.py

@@ -18,7 +18,7 @@
  #        Tristan Van Berkom <tristan vanberkom codethink co uk>
+ #
 -from collections import Mapping
 +from collections.abc import Mapping
  import jinja2
  from .. import _yaml

buildstream/_project.py

@@ -19,7 +19,8 @@
  #        Tiago Gomes <tiago gomes codethink co uk>
  import os
 -from collections import Mapping, OrderedDict
 +from collections import OrderedDict
 +from collections.abc import Mapping
  from pluginbase import PluginBase
  from . import utils
  from . import _cachekey

buildstream/_scheduler/queues/queue.py


     # This will have different results for elements depending
     # on the Queue.status() implementation.
     #
-    #   o Elements which are QueueStatus.WAIT will not be effected
+    #   o Elements which are QueueStatus.WAIT will not be affected
     #
     #   o Elements which are QueueStatus.SKIP will move directly
     #     to the dequeue pool

buildstream/_yaml.py


 #
 # The purpose of this is to create a virtual copy-on-write
 # copy of a dictionary, so that mutating it in any way does
-# not effect the underlying dictionaries.
+# not affect the underlying dictionaries.
 #
 # collections.ChainMap covers this already mostly, but fails
 # to record internal state so as to hide keys which have been

buildstream/buildelement.py

@@ -176,7 +176,7 @@ class BuildElement(Element):
          # Specifying notparallel for a given element effects the
          # cache key, while having the side effect of setting max-jobs to 1,
 -        # which is normally automatically resolved and does not effect
 +        # which is normally automatically resolved and does not affect
          # the cache key.
          if self.get_variable('notparallel'):
              dictionary['notparallel'] = True

buildstream/element.py

@@ -76,7 +76,8 @@ import os
  import re
  import stat
  import copy
 -from collections import Mapping, OrderedDict
 +from collections import OrderedDict
 +from collections.abc import Mapping
  from contextlib import contextmanager
  import tempfile
  import shutil

buildstream/plugin.py

@@ -266,7 +266,7 @@ class Plugin():
          such as an sha256 sum of a tarball content.
          Elements and Sources should implement this by collecting any configurations
 -        which could possibly effect the output and return a dictionary of these settings.
 +        which could possibly affect the output and return a dictionary of these settings.
          For Sources, this is guaranteed to only be called if
          :func:`Source.get_consistency() <buildstream.source.Source.get_consistency>`

buildstream/plugins/elements/autotools.yaml

@@ -123,7 +123,7 @@ environment:
    V: 1
  # And dont consider MAKEFLAGS or V as something which may
 -# effect build output.
 +# affect build output.
  environment-nocache:
  - MAKEFLAGS
  - V

buildstream/plugins/elements/cmake.yaml

@@ -66,7 +66,7 @@ environment:
    V: 1
  # And dont consider JOBS or V as something which may
 -# effect build output.
 +# affect build output.
  environment-nocache:
  - JOBS
  - V

buildstream/plugins/elements/junction.py

@@ -124,7 +124,7 @@ the user to resolve possibly conflicting nested junctions by creating a junction
  with the same name in the top-level project, which then takes precedence.
  """
 -from collections import Mapping
 +from collections.abc import Mapping
  from buildstream import Element
  from buildstream._pipeline import PipelineError

buildstream/plugins/elements/make.yaml

@@ -36,7 +36,7 @@ environment:
    V: 1
  # And dont consider MAKEFLAGS or V as something which may
 -# effect build output.
 +# affect build output.
  environment-nocache:
  - MAKEFLAGS
  - V

buildstream/plugins/elements/manual.yaml

@@ -35,7 +35,7 @@ environment:
    V: 1
  # And dont consider MAKEFLAGS or V as something which may
 -# effect build output.
 +# affect build output.
  environment-nocache:
  - MAKEFLAGS
  - V

buildstream/plugins/elements/meson.yaml

@@ -74,6 +74,6 @@ environment:
      %{max-jobs}
  # And dont consider NINJAJOBS as something which may
 -# effect build output.
 +# affect build output.
  environment-nocache:
  - NINJAJOBS

buildstream/plugins/elements/qmake.yaml

@@ -44,7 +44,7 @@ environment:
    V: 1
  # And dont consider MAKEFLAGS or V as something which may
 -# effect build output.
 +# affect build output.
  environment-nocache:
  - MAKEFLAGS
  - V

buildstream/plugins/sources/git.py

@@ -89,7 +89,7 @@ import os
  import errno
  import re
  import shutil
 -from collections import Mapping
 +from collections.abc import Mapping
  from io import StringIO
  from configparser import RawConfigParser
@@ -415,7 +415,7 @@ class GitSource(Source):
      def get_unique_key(self):
          # Here we want to encode the local name of the repository and
          # the ref, if the user changes the alias to fetch the same sources
 -        # from another location, it should not effect the cache key.
 +        # from another location, it should not affect the cache key.
          key = [self.original_url, self.mirror.ref]
          # Only modify the cache key with checkout_submodules if it's something

buildstream/source.py

@@ -155,7 +155,7 @@ Class Reference
  """
  import os
 -from collections import Mapping
 +from collections.abc import Mapping
  from contextlib import contextmanager
  from . import Plugin, Consistency

tests/loader/dependencies.py

@@ -110,6 +110,7 @@ def test_circular_dependency(datafiles):
          element = loader.load(['elements/circulartarget.bst'])[0]
      assert (exc.value.reason == LoadErrorReason.CIRCULAR_DEPENDENCY)
 +    assert ("seconddep" in exc.value.args[0])
  @pytest.mark.datafiles(DATA_DIR)

tests/yaml/yaml.py

  import os
  import pytest
  import tempfile
 -from collections import Mapping
 +from collections.abc import Mapping
  from buildstream import _yaml
  from buildstream._exceptions import LoadError, LoadErrorReason

...	...	@@ -208,7 +208,7 @@ class Queue():
208	208	# This will have different results for elements depending
209	209	# on the Queue.status() implementation.
210	210	#
211		- # o Elements which are QueueStatus.WAIT will not be effected
	211	+ # o Elements which are QueueStatus.WAIT will not be affected
212	212	#
213	213	# o Elements which are QueueStatus.SKIP will move directly
214	214	# to the dequeue pool

...	...	@@ -972,7 +972,7 @@ def node_validate(node, valid_keys):
972	972	#
973	973	# The purpose of this is to create a virtual copy-on-write
974	974	# copy of a dictionary, so that mutating it in any way does
975		-# not effect the underlying dictionaries.
	975	+# not affect the underlying dictionaries.
976	976	#
977	977	# collections.ChainMap covers this already mostly, but fails
978	978	# to record internal state so as to hide keys which have been

[Notes] [Git][BuildStream/buildstream][juerg/cas-mtime] 8 commits: _loader/loader.py: Be more explanatory in CircDep exception

Jürg Billeter pushed to branch juerg/cas-mtime at BuildStream / buildstream

Commits:

24 changed files:

Changes: