[Notes] [Git][BuildStream/buildstream][Qinusty/597-non-alias-url-fix] 4

Qinusty pushed to branch Qinusty/597-non-alias-url-fix at BuildStream / buildstream

Commits:

f3fbdac0

by Jonathan Maw at 2018-08-21T11:33:34Z

Pipeline: Skip except_elements logic if no elements to except

3a4c147a

by Jonathan Maw at 2018-08-21T12:43:41Z

Merge branch 'jonathan/faster-except-elements' into 'master'

Pipeline: Skip except_elements logic if no elements to except

See merge request BuildStream/buildstream!688

2a959972

by Josh Smith at 2018-08-22T10:22:27Z

source.py: Take first part of url split for alias

This fixes #597 and prevents ValueError being raised when a url is
specified without an alias or URI scheme.

6c7e8c91

by Josh Smith at 2018-08-22T10:22:50Z

Add UNKNOWN_ALIAS and URL_WITHOUT_ALIAS

Added configurable warnings for the warnings produced in
Source.__validate_raw_url()

Changes:

buildstream/_pipeline.py

@@ -235,6 +235,9 @@ class Pipeline():
      #                       exceptions removed
+     #
      def except_elements(self, targets, elements, except_targets):
 +        if not except_targets:
 +            return elements
++
          targeted = list(self.dependencies(targets, Scope.ALL))
          visited = []

buildstream/plugin.py

@@ -757,6 +757,16 @@ class CoreWarnings():
      which is found to be invalid based on the configured track
      """
 +    URL_WITHOUT_ALIAS = "url-without-alias"
 +    """
 +    This warning will be produced when a source is configured with a url which does not contain an alias.
 +    """
++
 +    UNKNOWN_ALIAS = "unknown-alias"
 +    """
 +    This warning will be produced when a source is configured with a url which does not contain an alias.
 +    """
++
  __CORE_WARNINGS = [
      value

buildstream/source.py

@@ -137,6 +137,7 @@ from . import Plugin
  from . import _yaml, utils
  from ._exceptions import BstError, ImplError, ErrorDomain
  from ._projectrefs import ProjectRefStorage
 +from .plugin import CoreWarnings
  class Consistency():
@@ -219,9 +220,7 @@ class SourceFetcher():
          Args:
             url (str): The url used to download.
          """
 -        # Not guaranteed to be a valid alias yet.
 -        # Ensuring it's a valid alias currently happens in Project.get_alias_uris
 -        alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
 +        alias = url.split(utils._ALIAS_SEPARATOR, 1)[0]
          self.__alias = alias
      #############################################################
@@ -459,8 +458,8 @@ class Source(Plugin):
          *Since: 1.2*
          """
 -        alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
 -        self.__expected_alias = alias
 +        url_alias = url.split(utils._ALIAS_SEPARATOR, 1)[0]
 +        self.__expected_alias = url_alias
      def get_source_fetchers(self):
          """Get the objects that are used for fetching
@@ -505,6 +504,7 @@ class Source(Plugin):
          Returns:
             str: The fully qualified url, with aliases resolved
          """
 +        self.__validate_raw_url(url)
          # Alias overriding can happen explicitly (by command-line) or
          # implicitly (the Source being constructed with an __alias_override).
          if alias_override or self.__alias_override:
@@ -525,8 +525,7 @@ class Source(Plugin):
          else:
              # Sneakily store the alias if it hasn't already been stored
              if not self.__expected_alias and url and utils._ALIAS_SEPARATOR in url:
 -                url_alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
 -                self.__expected_alias = url_alias
 +                self.mark_download_url(url)
              project = self._get_project()
              return project.translate_url(url, first_pass=self.__first_pass)
@@ -988,3 +987,23 @@ class Source(Plugin):
              if src.get_consistency() == Consistency.RESOLVED:
                  src._fetch(previous_sources[0:index])
++
 +    # __validate_raw_url():
 +    #
 +    # Validates a url prior to url substitution.
 +    # This checks the url and produces warnings.
 +    #
 +    def __validate_raw_url(self, url):
 +        url_parts = url.split(utils._ALIAS_SEPARATOR, 1)
 +        alias = url_parts[0]
++
 +        # Check and warn when detecting no valid alias.
 +        project = self._get_project()
 +        if alias.lower() in utils._URI_SCHEMES:
 +            self.warn("{}: The use of an alias in urls is strongly advised.".format(self._get_provenance()),
 +                      detail="See https://buildstream.gitlab.io/buildstream/format_project.html#source-aliases",
 +                      warning_token=CoreWarnings.URL_WITHOUT_ALIAS)
++
 +        elif not project.get_alias_uri(alias, first_pass=self.__first_pass) and len(url_parts) > 1:
 +            self.warn("{}: Unknown alias '{}' in url".format(self._get_provenance(), alias),
 +                      warning_token=CoreWarnings.UNKNOWN_ALIAS)

buildstream/utils.py

@@ -47,6 +47,7 @@ _magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11])
  # The separator we use for user specified aliases
  _ALIAS_SEPARATOR = ':'
 +_URI_SCHEMES = ["http", "https", "ftp", "file", "git", "sftp", "ssh"]
  class UtilError(BstError):

[Notes] [Git][BuildStream/buildstream][Qinusty/597-non-alias-url-fix] 4 commits: Pipeline: Skip except_elements logic if no elements to except

Qinusty pushed to branch Qinusty/597-non-alias-url-fix at BuildStream / buildstream

Commits:

4 changed files:

Changes: