GNOME.org
 

[Notes] [Git][BuildStream/buildstream][chandan/sourcetransform] 6 commits: install_linux_distro.rst: Change install instructions for python-arpy package on Arch

Title: GitLab

Chandan Singh pushed to branch chandan/sourcetransform at BuildStream / buildstream

Commits:

28 changed files:

Changes:

  • buildstream/_loader/loader.py
    ... ... @@ -522,14 +522,15 @@ class Loader():
    522 522 
             element = Element._new_from_meta(meta_element, platform.artifactcache)
    523 523 
             element._preflight()
    524 524
    525 
    -        for source in element.sources():
    525 
    +        sources = list(element.sources())
    526 
    +        for idx, source in enumerate(sources):
    526 527 
                 # Handle the case where a subproject needs to be fetched
    527 528 
                 #
    528 529 
                 if source.get_consistency() == Consistency.RESOLVED:
    529 530 
                     if fetch_subprojects:
    530 531 
                         if ticker:
    531 532 
                             ticker(filename, 'Fetching subproject from {} source'.format(source.get_kind()))
    532 
    -                    source._fetch()
    533 
    +                    source._fetch(sources[0:idx])
    533 534 
                     else:
    534 535 
                         detail = "Try fetching the project with `bst fetch {}`".format(filename)
    535 536 
                         raise LoadError(LoadErrorReason.SUBPROJECT_FETCH_NEEDED,

  • buildstream/_pipeline.py
    ... ... @@ -358,10 +358,24 @@ class Pipeline():
    358 358 
                         inconsistent.append(element)
    359 359
    360 360 
             if inconsistent:
    361 
    -            detail = "Exact versions are missing for the following elements\n" + \
    362 
    -                     "Try tracking these elements first with `bst track`\n\n"
    361 
    +            detail = "Exact versions are missing for the following elements:\n\n"
    362 
    +
    363 
    +            missingTrack = 0
    363 364 
                 for element in inconsistent:
    364 
    -                detail += "  " + element._get_full_name() + "\n"
    365 
    +                detail += "  " + element._get_full_name()
    366 
    +                for source in element.sources():
    367 
    +                    if not source._get_consistency() and not source.get_ref():
    368 
    +                        if hasattr(source, 'tracking') and source.tracking is None:
    369 
    +                            detail += ": Source {} is missing ref and track. ".format(source._get_full_name()) + \
    370 
    +                                      "Please specify a ref or branch/tag to track."
    371 
    +                            missingTrack = 1
    372 
    +
    373 
    +                detail += "\n"
    374 
    +
    375 
    +            if missingTrack:
    376 
    +                detail += "\nThen track these elements with `bst track`\n"
    377 
    +            else:
    378 
    +                detail += "\nTry tracking these elements first with `bst track`\n"
    365 379 
                 raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline")
    366 380
    367 381 
         #############################################################

  • buildstream/_scheduler/queues/fetchqueue.py
    ... ... @@ -40,8 +40,10 @@ class FetchQueue(Queue):
    40 40 
             self._skip_cached = skip_cached
    41 41
    42 42 
         def process(self, element):
    43 
    +        previous_sources = []
    43 44 
             for source in element.sources():
    44 
    -            source._fetch()
    45 
    +            source._fetch(previous_sources)
    46 
    +            previous_sources.append(source)
    45 47
    46 48 
         def status(self, element):
    47 49 
             # state of dependencies may have changed, recalculate element state

  • buildstream/element.py
    ... ... @@ -1270,6 +1270,12 @@ class Element(Plugin):
    1270 1270 
                 # Prepend provenance to the error
    1271 1271 
                 raise ElementError("{}: {}".format(self, e), reason=e.reason) from e
    1272 1272
    1273 
    +        # Ensure that the first source does not need access to previous soruces
    1274 
    +        if self.__sources and self.__sources[0]._requires_previous_sources():
    1275 
    +            raise ElementError("{}: {} cannot be the first source of an element "
    1276 
    +                               "as it requires access to previous sources"
    1277 
    +                               .format(self, self.__sources[0]))
    1278 
    +
    1273 1279 
             # Preflight the sources
    1274 1280 
             for source in self.sources():
    1275 1281 
                 source._preflight()
    ... ... @@ -1313,9 +1319,9 @@ class Element(Plugin):
    1313 1319 
         #
    1314 1320 
         def _track(self):
    1315 1321 
             refs = []
    1316 
    -        for source in self.__sources:
    1322 
    +        for index, source in enumerate(self.__sources):
    1317 1323 
                 old_ref = source.get_ref()
    1318 
    -            new_ref = source._track()
    1324 
    +            new_ref = source._track(self.__sources[0:index])
    1319 1325 
                 refs.append((source._get_unique_id(), new_ref))
    1320 1326
    1321 1327 
                 # Complimentary warning that the new ref will be unused.

  • buildstream/plugins/sources/git.py
    ... ... @@ -363,6 +363,12 @@ class GitSource(Source):
    363 363
    364 364 
             # If self.tracking is not specified it's not an error, just silently return
    365 365 
             if not self.tracking:
    366 
    +            # Is there a better way to check if a ref is given.
    367 
    +            if self.mirror.ref is None:
    368 
    +                detail = 'Without a tracking branch ref can not be updated. Please ' + \
    369 
    +                         'provide a ref or a track.'
    370 
    +                raise SourceError("{}: No track or ref".format(self),
    371 
    +                                  detail=detail, reason="track-attempt-no-track")
    366 372 
                 return None
    367 373
    368 374 
             with self.timed_activity("Tracking {} from {}"

  • buildstream/plugins/sources/pip.py
    1 
    +#
    2 
    +#  Copyright 2018 Bloomberg Finance LP
    3 
    +#
    4 
    +#  This program is free software; you can redistribute it and/or
    5 
    +#  modify it under the terms of the GNU Lesser General Public
    6 
    +#  License as published by the Free Software Foundation; either
    7 
    +#  version 2 of the License, or (at your option) any later version.
    8 
    +#
    9 
    +#  This library is distributed in the hope that it will be useful,
    10 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    11 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    12 
    +#  Lesser General Public License for more details.
    13 
    +#
    14 
    +#  You should have received a copy of the GNU Lesser General Public
    15 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    16 
    +#
    17 
    +#  Authors:
    18 
    +#        Chandan Singh <csingh43 bloomberg net>
    19 
    +
    20 
    +"""
    21 
    +pip - stage python packages using pip
    22 
    +=====================================
    23 
    +
    24 
    +**Host depndencies:**
    25 
    +
    26 
    +  * ``virtualenv`` CLI tool or ``venv`` python module
    27 
    +
    28 
    +**Usage:**
    29 
    +
    30 
    +.. code:: yaml
    31 
    +
    32 
    +   # Specify the pip source kind
    33 
    +   kind: pip
    34 
    +
    35 
    +   # Optionally specify the python executable, defaults to system "python"
    36 
    +   # Note that either the venv module or the virtualenv CLI tool must be
    37 
    +   # available
    38 
    +   python-exe: python3.6
    39 
    +
    40 
    +   # Optionally specify index url, defaults to PyPi
    41 
    +   index-url: https://mypypi.example.com/simple
    42 
    +
    43 
    +   # Optionally specify the path to requirements files
    44 
    +   # Note that either 'requirements-files' or 'packages' must be defined
    45 
    +   requirements-files:
    46 
    +   - requirements.txt
    47 
    +
    48 
    +   # Optionally specify a list of additional packages
    49 
    +   # Note that either 'requirements-files' or 'packages' must be defined
    50 
    +   packages:
    51 
    +   - flake8
    52 
    +
    53 
    +   # Optionally specify a relative staging directory
    54 
    +   directory: path/to/stage
    55 
    +
    56 
    +   # Specify the ref. It is a list of strings of format
    57 
    +   # `<package-name>==<version>`, one per line.
    58 
    +   # Usually this will be contents of a requirements.txt file where all
    59 
    +   # package versions have been frozen.
    60 
    +   ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0"
    61 
    +
    62 
    +.. note::
    63 
    +
    64 
    +   The ``pip`` plugin is available since :ref:`format version 12 <project_format_version>`
    65 
    +
    66 
    +"""
    67 
    +
    68 
    +import hashlib
    69 
    +import os
    70 
    +
    71 
    +from buildstream import Consistency, Source, SourceError, utils
    72 
    +
    73 
    +_PYPI_INDEX_URL = 'https://pypi.org/simple/'
    74 
    +
    75 
    +
    76 
    +class PipSource(Source):
    77 
    +    # pylint: disable=attribute-defined-outside-init
    78 
    +
    79 
    +    # We need access to previous sources at track time to use requirements.txt
    80 
    +    # but not at fetch time as self.ref should contain sufficient information
    81 
    +    # for this plugin
    82 
    +    requires_previous_sources_track = True
    83 
    +
    84 
    +    def configure(self, node):
    85 
    +        self.node_validate(node, ['index-url', 'packages', 'python-exe', 'ref', 'requirements-files'] +
    86 
    +                           Source.COMMON_CONFIG_KEYS)
    87 
    +        self.ref = self.node_get_member(node, str, 'ref', None)
    88 
    +        self.python_exe = self.node_get_member(node, str, 'python-exe', 'python')
    89 
    +        self.index_url = self.node_get_member(node, str, 'index-url', _PYPI_INDEX_URL)
    90 
    +        self.packages = self.node_get_member(node, list, 'packages', [])
    91 
    +        self.requirements_files = self.node_get_member(node, list, 'requirements-files', [])
    92 
    +
    93 
    +        if not (self.packages or self.requirements_files):
    94 
    +            raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self))
    95 
    +
    96 
    +    def preflight(self):
    97 
    +        # Try to find a way to open virtual environments on the host
    98 
    +        try:
    99 
    +            # Look for the virtualenv CLI first
    100 
    +            venv = utils.get_host_tool('virtualenv')
    101 
    +            self.venv_cmd = [venv, '--python', self.python_exe]
    102 
    +        except utils.ProgramNotFoundError:
    103 
    +            # Fall back to venv module if it is installed
    104 
    +            python_exe = utils.get_host_tool(self.python_exe)
    105 
    +            rc = self.call([python_exe, '-m', 'venv', '--help'])
    106 
    +            if rc == 0:
    107 
    +                self.venv_cmd = [python_exe, '-m', 'venv']
    108 
    +            else:
    109 
    +                raise SourceError("{}: venv module not found using python: {}"
    110 
    +                                  .format(self, python_exe))
    111 
    +
    112 
    +    def get_unique_key(self):
    113 
    +        return [self.venv_cmd, self.index_url, self.ref]
    114 
    +
    115 
    +    def get_consistency(self):
    116 
    +        if not self.ref:
    117 
    +            return Consistency.INCONSISTENT
    118 
    +        # FIXME add a stronger consistency check
    119 
    +        # Currently we take the presence of "something" as an indication that
    120 
    +        # we have the right things in cache
    121 
    +        if os.path.exists(self.mirror) and os.listdir(self.mirror):
    122 
    +            return Consistency.CACHED
    123 
    +        return Consistency.RESOLVED
    124 
    +
    125 
    +    def get_ref(self):
    126 
    +        return self.ref
    127 
    +
    128 
    +    def load_ref(self, node):
    129 
    +        self.ref = self.node_get_member(node, str, 'ref', None)
    130 
    +
    131 
    +    def set_ref(self, ref, node):
    132 
    +        node['ref'] = self.ref = ref
    133 
    +
    134 
    +    def track(self, previous_sources_dir):
    135 
    +        # XXX pip does not offer any public API other than the CLI tool so it
    136 
    +        # is not feasible to correctly parse the requirements file or to check
    137 
    +        # which package versions pip is going to install.
    138 
    +        # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
    139 
    +        # for details.
    140 
    +        # As a result, we have to wastefully install the packages during track.
    141 
    +        with self.tempdir() as tmpdir:
    142 
    +            pip = self._venv_pip(tmpdir)
    143 
    +
    144 
    +            install_args = [pip, 'install', '--index-url', self.index_url]
    145 
    +            for requirement_file in self.requirements_files:
    146 
    +                fpath = os.path.join(previous_sources_dir, requirement_file)
    147 
    +                install_args += ['-r', fpath]
    148 
    +            install_args += self.packages
    149 
    +
    150 
    +            self.call(install_args, fail="Failed to install python packages")
    151 
    +            _, reqs = self.check_output([pip, 'freeze'])
    152 
    +
    153 
    +        return reqs.strip()
    154 
    +
    155 
    +    def fetch(self):
    156 
    +        with self.tempdir() as tmpdir:
    157 
    +            pip = self._venv_pip(tmpdir)
    158 
    +            packages = self.ref.strip().split('\n')
    159 
    +            self.call([pip, 'install',
    160 
    +                       '--index-url', self.index_url,
    161 
    +                       '--prefix', self.mirror] +
    162 
    +                      packages,
    163 
    +                      fail="Failed to install python packages: {}".format(packages))
    164 
    +
    165 
    +    def stage(self, directory):
    166 
    +        with self.timed_activity("Staging Python packages", silent_nested=True):
    167 
    +            utils.copy_files(self.mirror, directory)
    168 
    +
    169 
    +    @property
    170 
    +    def mirror(self):
    171 
    +        """Directory where this source should stage its files
    172 
    +
    173 
    +        """
    174 
    +        path = os.path.join(self.get_mirror_directory(),
    175 
    +                            self.index_url,
    176 
    +                            hashlib.sha256(self.ref.encode()).hexdigest())
    177 
    +        os.makedirs(path, exist_ok=True)
    178 
    +        return path
    179 
    +
    180 
    +    def _venv_pip(self, directory):
    181 
    +        """Open a virtual environment in given directory and return pip path
    182 
    +
    183 
    +        """
    184 
    +        self.call(self.venv_cmd + [directory], fail="Failed to initialize virtual environment")
    185 
    +        pip_exe = os.path.join(directory, 'bin', 'pip')
    186 
    +        if not os.path.isfile(pip_exe):
    187 
    +            raise SourceError("Failed to initialize virtual environment")
    188 
    +        return pip_exe
    189 
    +
    190 
    +
    191 
    +def setup():
    192 
    +    return PipSource

  • buildstream/source.py
    ... ... @@ -76,6 +76,39 @@ these methods are mandatory to implement.
    76 76 
       :ref:`SourceFetcher <core_source_fetcher>`.
    77 77
    78 78
    79 
    +Accessing previous sources
    80 
    +--------------------------
    81 
    +*Since: 1.4*
    82 
    +
    83 
    +In the general case, all sources are fetched and tracked independently of one
    84 
    +another. In situations where a source needs to access previous source(s) in
    85 
    +order to perform its own track and/or fetch, following attributes can be set to
    86 
    +request access to previous sources:
    87 
    +
    88 
    +* :attr:`~buildstream.source.Source.requires_previous_sources_track`
    89 
    +
    90 
    +  Indicate that access to previous sources is required during track
    91 
    +
    92 
    +* :attr:`~buildstream.source.Source.requires_previous_sources_fetch`
    93 
    +
    94 
    +  Indicate that access to previous sources is required during fetch
    95 
    +
    96 
    +The intended use of such plugins is to fetch external dependencies of other
    97 
    +sources, typically using some kind of package manager, such that all the
    98 
    +dependencies of the original source(s) are available at build time.
    99 
    +
    100 
    +When implementing such a plugin, implementors should adhere to the following
    101 
    +guidelines:
    102 
    +
    103 
    +* Implementations must be able to store the obtained artifacts in a
    104 
    +  subdirectory.
    105 
    +
    106 
    +* Implementations must be able to deterministically generate a unique ref, such
    107 
    +  that two refs are different if and only if they produce different outputs.
    108 
    +
    109 
    +* Implementations must not introduce host contamination.
    110 
    +
    111 
    +
    79 112 
     .. _core_source_fetcher:
    80 113
    81 114 
     SourceFetcher - Object for fetching individual URLs
    ... ... @@ -92,6 +125,8 @@ mentioned, these methods are mandatory to implement.
    92 125 
       Fetches the URL associated with this SourceFetcher, optionally taking an
    93 126 
       alias override.
    94 127
    128 
    +Class Reference
    129 
    +---------------
    95 130 
     """
    96 131
    97 132 
     import os
    ... ... @@ -156,7 +191,7 @@ class SourceFetcher():
    156 191 
         #############################################################
    157 192 
         #                      Abstract Methods                     #
    158 193 
         #############################################################
    159 
    -    def fetch(self, alias_override=None):
    194 
    +    def fetch(self, alias_override=None, **kwargs):
    160 195 
             """Fetch remote sources and mirror them locally, ensuring at least
    161 196 
             that the specific reference is cached locally.
    162 197
    ... ... @@ -209,6 +244,32 @@ class Source(Plugin):
    209 244 
         __defaults = {}          # The defaults from the project
    210 245 
         __defaults_set = False   # Flag, in case there are not defaults at all
    211 246
    247 
    +    requires_previous_sources_track = False
    248 
    +    """Whether access to previous sources is required during track
    249 
    +
    250 
    +    When set to True:
    251 
    +      * all sources listed before this source in the given element will be
    252 
    +        fetched before this source is tracked
    253 
    +      * Source.track() will be called with an additional keyword argument
    254 
    +        `previous_sources_dir` where previous sources will be staged
    255 
    +      * this source can not be the first source for an element
    256 
    +
    257 
    +    *Since: 1.4*
    258 
    +    """
    259 
    +
    260 
    +    requires_previous_sources_fetch = False
    261 
    +    """Whether access to previous sources is required during fetch
    262 
    +
    263 
    +    When set to True:
    264 
    +      * all sources listed before this source in the given element will be
    265 
    +        fetched before this source is fetched
    266 
    +      * Source.fetch() will be called with an additional keyword argument
    267 
    +        `previous_sources_dir` where previous sources will be staged
    268 
    +      * this source can not be the first source for an element
    269 
    +
    270 
    +    *Since: 1.4*
    271 
    +    """
    272 
    +
    212 273 
         def __init__(self, context, project, meta, *, alias_override=None):
    213 274 
             provenance = _yaml.node_get_provenance(meta.config)
    214 275 
             super().__init__("{}-{}".format(meta.element_name, meta.element_index),
    ... ... @@ -305,9 +366,15 @@ class Source(Plugin):
    305 366 
             """
    306 367 
             raise ImplError("Source plugin '{}' does not implement set_ref()".format(self.get_kind()))
    307 368
    308 
    -    def track(self):
    369 
    +    def track(self, **kwargs):
    309 370 
             """Resolve a new ref from the plugin's track option
    310 371
    372 
    +        Args:
    373 
    +           previous_sources_dir (str): directory where previous sources are staged.
    374 
    +                                       Note that this keyword argument is available only when
    375 
    +                                       :attr:`~buildstream.source.Source.requires_previous_sources_track`
    376 
    +                                       is set to True.
    377 
    +
    311 378 
             Returns:
    312 379 
                (simple object): A new internal source reference, or None
    313 380
    ... ... @@ -326,10 +393,16 @@ class Source(Plugin):
    326 393 
             # Allow a non implementation
    327 394 
             return None
    328 395
    329 
    -    def fetch(self):
    396 
    +    def fetch(self, **kwargs):
    330 397 
             """Fetch remote sources and mirror them locally, ensuring at least
    331 398 
             that the specific reference is cached locally.
    332 399
    400 
    +        Args:
    401 
    +           previous_sources_dir (str): directory where previous sources are staged.
    402 
    +                                       Note that this keyword argument is available only when
    403 
    +                                       :attr:`~buildstream.source.Source.requires_previous_sources_fetch`
    404 
    +                                       is set to True.
    405 
    +
    333 406 
             Raises:
    334 407 
                :class:`.SourceError`
    335 408
    ... ... @@ -519,50 +592,19 @@ class Source(Plugin):
    519 592
    520 593 
         # Wrapper function around plugin provided fetch method
    521 594 
         #
    522 
    -    def _fetch(self):
    523 
    -        project = self._get_project()
    524 
    -        source_fetchers = self.get_source_fetchers()
    525 
    -        if source_fetchers:
    526 
    -            for fetcher in source_fetchers:
    527 
    -                alias = fetcher._get_alias()
    528 
    -                success = False
    529 
    -                for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
    530 
    -                    try:
    531 
    -                        fetcher.fetch(uri)
    532 
    -                    # FIXME: Need to consider temporary vs. permanent failures,
    533 
    -                    #        and how this works with retries.
    534 
    -                    except BstError as e:
    535 
    -                        last_error = e
    536 
    -                        continue
    537 
    -                    success = True
    538 
    -                    break
    539 
    -                if not success:
    540 
    -                    raise last_error
    595 
    +    # Args:
    596 
    +    #   previous_sources (list): List of Sources listed prior to this source
    597 
    +    #
    598 
    +    def _fetch(self, previous_sources):
    599 
    +
    600 
    +        if self.requires_previous_sources_fetch:
    601 
    +            self.__ensure_previous_sources(previous_sources)
    602 
    +            with self.tempdir() as staging_directory:
    603 
    +                for src in previous_sources:
    604 
    +                    src._stage(staging_directory)
    605 
    +                self.__do_fetch(previous_sources_dir=staging_directory)
    541 606 
             else:
    542 
    -            alias = self._get_alias()
    543 
    -            if self.__first_pass:
    544 
    -                mirrors = project.first_pass_config.mirrors
    545 
    -            else:
    546 
    -                mirrors = project.config.mirrors
    547 
    -            if not mirrors or not alias:
    548 
    -                self.fetch()
    549 
    -                return
    550 
    -
    551 
    -            context = self._get_context()
    552 
    -            source_kind = type(self)
    553 
    -            for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
    554 
    -                new_source = source_kind(context, project, self.__meta,
    555 
    -                                         alias_override=(alias, uri))
    556 
    -                new_source._preflight()
    557 
    -                try:
    558 
    -                    new_source.fetch()
    559 
    -                # FIXME: Need to consider temporary vs. permanent failures,
    560 
    -                #        and how this works with retries.
    561 
    -                except BstError as e:
    562 
    -                    last_error = e
    563 
    -                    continue
    564 
    -                return
    565 
    -            raise last_error
    607 
    +            self.__do_fetch()
    566 608
    567 609 
         # Wrapper for stage() api which gives the source
    568 610 
         # plugin a fully constructed path considering the
    ... ... @@ -773,8 +815,19 @@ class Source(Plugin):
    773 815
    774 816 
         # Wrapper for track()
    775 817 
         #
    776 
    -    def _track(self):
    777 
    -        new_ref = self.__do_track()
    818 
    +    # Args:
    819 
    +    #   previous_sources (list): List of Sources listed prior to this source
    820 
    +    #
    821 
    +    def _track(self, previous_sources):
    822 
    +        if self.requires_previous_sources_track:
    823 
    +            self.__ensure_previous_sources(previous_sources)
    824 
    +            with self.tempdir() as staging_directory:
    825 
    +                for src in previous_sources:
    826 
    +                    src._stage(staging_directory)
    827 
    +                new_ref = self.__do_track(previous_sources_dir=staging_directory)
    828 
    +        else:
    829 
    +            new_ref = self.__do_track()
    830 
    +
    778 831 
             current_ref = self.get_ref()
    779 832
    780 833 
             if new_ref is None:
    ... ... @@ -786,6 +839,17 @@ class Source(Plugin):
    786 839
    787 840 
             return new_ref
    788 841
    842 
    +    # _requires_previous_sources()
    843 
    +    #
    844 
    +    # If a plugin requires access to previous sources at track or fetch time,
    845 
    +    # then it cannot be the first source of an elemenet.
    846 
    +    #
    847 
    +    # Returns:
    848 
    +    #   (bool): Whether this source requires access to previous sources
    849 
    +    #
    850 
    +    def _requires_previous_sources(self):
    851 
    +        return self.requires_previous_sources_track or self.requires_previous_sources_fetch
    852 
    +
    789 853 
         # Returns the alias if it's defined in the project
    790 854 
         def _get_alias(self):
    791 855 
             alias = self.__expected_alias
    ... ... @@ -801,8 +865,54 @@ class Source(Plugin):
    801 865 
         #                   Local Private Methods                   #
    802 866 
         #############################################################
    803 867
    868 
    +    # Tries to call fetch for every mirror, stopping once it succeeds
    869 
    +    def __do_fetch(self, **kwargs):
    870 
    +        project = self._get_project()
    871 
    +        source_fetchers = self.get_source_fetchers()
    872 
    +        if source_fetchers:
    873 
    +            for fetcher in source_fetchers:
    874 
    +                alias = fetcher._get_alias()
    875 
    +                success = False
    876 
    +                for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
    877 
    +                    try:
    878 
    +                        fetcher.fetch(uri)
    879 
    +                    # FIXME: Need to consider temporary vs. permanent failures,
    880 
    +                    #        and how this works with retries.
    881 
    +                    except BstError as e:
    882 
    +                        last_error = e
    883 
    +                        continue
    884 
    +                    success = True
    885 
    +                    break
    886 
    +                if not success:
    887 
    +                    raise last_error
    888 
    +        else:
    889 
    +            alias = self._get_alias()
    890 
    +            if self.__first_pass:
    891 
    +                mirrors = project.first_pass_config.mirrors
    892 
    +            else:
    893 
    +                mirrors = project.config.mirrors
    894 
    +            if not mirrors or not alias:
    895 
    +                self.fetch(**kwargs)
    896 
    +                return
    897 
    +
    898 
    +            context = self._get_context()
    899 
    +            source_kind = type(self)
    900 
    +            for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
    901 
    +                new_source = source_kind(context, project, self.__meta,
    902 
    +                                         alias_override=(alias, uri))
    903 
    +                new_source._preflight()
    904 
    +                try:
    905 
    +                    new_source.fetch(**kwargs)
    906 
    +                # FIXME: Need to consider temporary vs. permanent failures,
    907 
    +                #        and how this works with retries.
    908 
    +                except BstError as e:
    909 
    +                    last_error = e
    910 
    +                    continue
    911 
    +                return
    912 
    +            raise last_error
    913 
    +
    804 914 
         # Tries to call track for every mirror, stopping once it succeeds
    805 
    -    def __do_track(self):
    915 
    +    def __do_track(self, **kwargs):
    806 916 
             project = self._get_project()
    807 917 
             # If there are no mirrors, or no aliases to replace, there's nothing to do here.
    808 918 
             alias = self._get_alias()
    ... ... @@ -811,7 +921,7 @@ class Source(Plugin):
    811 921 
             else:
    812 922 
                 mirrors = project.config.mirrors
    813 923 
             if not mirrors or not alias:
    814 
    -            return self.track()
    924 
    +            return self.track(**kwargs)
    815 925
    816 926 
             context = self._get_context()
    817 927 
             source_kind = type(self)
    ... ... @@ -823,7 +933,7 @@ class Source(Plugin):
    823 933 
                                          alias_override=(alias, uri))
    824 934 
                 new_source._preflight()
    825 935 
                 try:
    826 
    -                ref = new_source.track()
    936 
    +                ref = new_source.track(**kwargs)
    827 937 
                 # FIXME: Need to consider temporary vs. permanent failures,
    828 938 
                 #        and how this works with retries.
    829 939 
                 except BstError as e:
    ... ... @@ -867,3 +977,14 @@ class Source(Plugin):
    867 977 
             _yaml.node_final_assertions(config)
    868 978
    869 979 
             return config
    980 
    +
    981 
    +    # Ensures that previous sources have been tracked and fetched.
    982 
    +    #
    983 
    +    def __ensure_previous_sources(self, previous_sources):
    984 
    +        for index, src in enumerate(previous_sources):
    985 
    +            if src.get_consistency() == Consistency.RESOLVED:
    986 
    +                src._fetch(previous_sources[0:index])
    987 
    +            elif src.get_consistency() == Consistency.INCONSISTENT:
    988 
    +                new_ref = src._track(previous_sources[0:index])
    989 
    +                src._save_ref(new_ref)
    990 
    +                src._fetch(previous_sources[0:index])

  • doc/source/core_plugins.rst
    ... ... @@ -58,6 +58,7 @@ Sources
    58 58 
        sources/ostree
    59 59 
        sources/patch
    60 60 
        sources/deb
    61 
    +   sources/pip
    61 62
    62 63
    63 64 
     External plugins

  • doc/source/install_linux_distro.rst
    ... ... @@ -57,9 +57,20 @@ Install the dependencies with::
    57 57 
     For the default plugins::
    58 58
    59 59 
       sudo pacman -S \
    60 
    -      bzr git lzip ostree patch python-arpy python-gobject
    60 
    +      bzr git lzip ostree patch python-gobject
    61 61
    62 62
    63 
    +The package *python-arpy* is required by the deb source plugin. This is not
    64 
    +obtainable via `pacman`, you must get *python-arpy* from AUR:
    65 
    +https://aur.archlinux.org/packages/python-arpy/
    66 
    +
    67 
    +To install::
    68 
    +
    69 
    +  wget https://aur.archlinux.org/cgit/aur.git/snapshot/python-arpy.tar.gz
    70 
    +  tar -xvf python-arpy.tar.gz
    71 
    +  cd python-arpy
    72 
    +  makepkg -si
    73 
    +
    63 74 
     Debian
    64 75 
     ++++++
    65 76 
     Install the dependencies with::
    ... ... @@ -227,7 +238,7 @@ Fedora
    227 238 
     ~~~~~~
    228 239
    229 240 
     BuildStream is not yet in the official Fedora repositories, but you can
    230 
    -install it from a Copr:
    241 
    +install it from a Copr::
    231 242
    232 243 
       sudo dnf copr enable bochecha/buildstream
    233 244 
       sudo dnf install buildstream

  • tests/integration/pip.py → tests/integration/pip_element.py

  • tests/integration/pip_source.py
    1 
    +import os
    2 
    +import pytest
    3 
    +
    4 
    +from buildstream import _yaml
    5 
    +
    6 
    +from tests.testutils import cli_integration as cli
    7 
    +from tests.testutils.integration import assert_contains
    8 
    +
    9 
    +
    10 
    +pytestmark = pytest.mark.integration
    11 
    +
    12 
    +
    13 
    +DATA_DIR = os.path.join(
    14 
    +    os.path.dirname(os.path.realpath(__file__)),
    15 
    +    "project"
    16 
    +)
    17 
    +
    18 
    +
    19 
    +@pytest.mark.datafiles(DATA_DIR)
    20 
    +def test_pip_source(cli, tmpdir, datafiles):
    21 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    22 
    +    checkout = os.path.join(cli.directory, 'checkout')
    23 
    +    element_path = os.path.join(project, 'elements')
    24 
    +    element_name = 'pip/hello.bst'
    25 
    +
    26 
    +    element = {
    27 
    +        'kind': 'import',
    28 
    +        'sources': [
    29 
    +            {
    30 
    +                'kind': 'local',
    31 
    +                'path': 'files/pip-source'
    32 
    +            },
    33 
    +            {
    34 
    +                'kind': 'pip',
    35 
    +                'python-exe': 'python3',
    36 
    +                'index-url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
    37 
    +                'requirements-files': ['myreqs.txt'],
    38 
    +                'packages': ['app2']
    39 
    +            }
    40 
    +        ]
    41 
    +    }
    42 
    +    os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
    43 
    +    _yaml.dump(element, os.path.join(element_path, element_name))
    44 
    +
    45 
    +    result = cli.run(project=project, args=['track', element_name])
    46 
    +    assert result.exit_code == 0
    47 
    +
    48 
    +    result = cli.run(project=project, args=['build', element_name])
    49 
    +    assert result.exit_code == 0
    50 
    +
    51 
    +    result = cli.run(project=project, args=['checkout', element_name, checkout])
    52 
    +    assert result.exit_code == 0
    53 
    +
    54 
    +    assert_contains(checkout, ['/bin', '/bin/app1', '/bin/app2'])

  • tests/integration/project/files/pip-source/myreqs.txt
    1 
    +app1

  • tests/integration/project/files/pypi-repo/app1/App1-0.1.tar.gz
    No preview for this file type
  • tests/integration/project/files/pypi-repo/app1/index.html
    1 
    +<html>
    2 
    +  <head>
    3 
    +    <title>Links for app1</title>
    4 
    +  </head>
    5 
    +  <body>
    6 
    +    <a href="">'App1-0.1.tar.gz'>App1-0.1.tar.gz</a><br />
    7 
    +  </body>
    8 
    +</html>

  • tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz
    No preview for this file type
  • tests/integration/project/files/pypi-repo/app2/index.html
    1 
    +<html>
    2 
    +  <head>
    3 
    +    <title>Links for app1</title>
    4 
    +  </head>
    5 
    +  <body>
    6 
    +    <a href="">'App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
    7 
    +  </body>
    8 
    +</html>

  • tests/sources/git.py
    ... ... @@ -359,3 +359,45 @@ def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
    359 359
    360 360 
         # Assert that we are just fine without it, and emit a warning to the user.
    361 361 
         assert "Ignoring inconsistent submodule" in result.stderr
    362 
    +
    363 
    +
    364 
    +@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
    365 
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
    366 
    +def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
    367 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    368 
    +
    369 
    +    # Create the repo from 'repofiles' subdir
    370 
    +    repo = create_repo('git', str(tmpdir))
    371 
    +    ref = repo.create(os.path.join(project, 'repofiles'))
    372 
    +
    373 
    +    # Write out our test target
    374 
    +    gitsource = repo.source_config(ref=None)
    375 
    +    gitsource.pop('track')
    376 
    +    element = {
    377 
    +        'kind': 'import',
    378 
    +        'sources': [
    379 
    +            gitsource
    380 
    +        ]
    381 
    +    }
    382 
    +
    383 
    +    _yaml.dump(element, os.path.join(project, 'target.bst'))
    384 
    +
    385 
    +    # Track will encounter an inconsistent submodule without any ref
    386 
    +    result = cli.run(project=project, args=['track', 'target.bst'])
    387 
    +    result.assert_main_error(ErrorDomain.STREAM, None)
    388 
    +    result.assert_task_error(ErrorDomain.SOURCE, 'track-attempt-no-track')
    389 
    +
    390 
    +    # Assert that we are just fine without it, and emit a warning to the user.
    391 
    +    assert "FAILURE git source at" in result.stderr
    392 
    +    assert "Without a tracking branch ref can not be updated. Please " + \
    393 
    +        "provide a ref or a track." in result.stderr
    394 
    +
    395 
    +    # Track will encounter an inconsistent submodule without any ref
    396 
    +    result = cli.run(project=project, args=['build', 'target.bst'])
    397 
    +    result.assert_main_error(ErrorDomain.PIPELINE, 'inconsistent-pipeline')
    398 
    +    result.assert_task_error(None, None)
    399 
    +
    400 
    +    # Assert that we are just fine without it, and emit a warning to the user.
    401 
    +    assert "Exact versions are missing for the following elements" in result.stderr
    402 
    +    assert "is missing ref and track." in result.stderr
    403 
    +    assert "Then track these elements with `bst track`" in result.stderr

  • tests/sources/pip.py
    1 
    +import os
    2 
    +import pytest
    3 
    +
    4 
    +from buildstream._exceptions import ErrorDomain
    5 
    +from buildstream import _yaml
    6 
    +from tests.testutils import cli
    7 
    +
    8 
    +DATA_DIR = os.path.join(
    9 
    +    os.path.dirname(os.path.realpath(__file__)),
    10 
    +    'pip',
    11 
    +)
    12 
    +
    13 
    +
    14 
    +def generate_project(project_dir, tmpdir):
    15 
    +    project_file = os.path.join(project_dir, "project.conf")
    16 
    +    _yaml.dump({'name': 'foo'}, project_file)
    17 
    +
    18 
    +
    19 
    +# Test that without ref, consistency is set appropriately.
    20 
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
    21 
    +def test_no_ref(cli, tmpdir, datafiles):
    22 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    23 
    +    generate_project(project, tmpdir)
    24 
    +    assert cli.get_element_state(project, 'target.bst') == 'no reference'
    25 
    +
    26 
    +
    27 
    +# Test that pip is not allowed to be the first source
    28 
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'first-source-pip'))
    29 
    +def test_first_source(cli, tmpdir, datafiles):
    30 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    31 
    +    generate_project(project, tmpdir)
    32 
    +    result = cli.run(project=project, args=[
    33 
    +        'show', 'target.bst'
    34 
    +    ])
    35 
    +    result.assert_main_error(ErrorDomain.ELEMENT, None)
    36 
    +
    37 
    +
    38 
    +# Test that error is raised when neither packges nor requirements files
    39 
    +# have been specified
    40 
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-packages'))
    41 
    +def test_no_packages(cli, tmpdir, datafiles):
    42 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    43 
    +    generate_project(project, tmpdir)
    44 
    +    result = cli.run(project=project, args=[
    45 
    +        'show', 'target.bst'
    46 
    +    ])
    47 
    +    result.assert_main_error(ErrorDomain.SOURCE, None)

  • tests/sources/pip/first-source-pip/target.bst
    1 
    +kind: import
    2 
    +description: pip should not be allowed to be the first source
    3 
    +sources:
    4 
    +- kind: pip
    5 
    +  python-exe: python3
    6 
    +  packages:
    7 
    +  - flake8

  • tests/sources/pip/no-packages/file
    1 
    +Hello World!

  • tests/sources/pip/no-packages/target.bst
    1 
    +kind: import
    2 
    +description: The kind of this element is irrelevant.
    3 
    +sources:
    4 
    +- kind: local
    5 
    +  path: file
    6 
    +- kind: pip
    7 
    +  python-exe: python3

  • tests/sources/pip/no-ref/file
    1 
    +Hello World!

  • tests/sources/pip/no-ref/target.bst
    1 
    +kind: import
    2 
    +description: The kind of this element is irrelevant.
    3 
    +sources:
    4 
    +- kind: local
    5 
    +  path: file
    6 
    +- kind: pip
    7 
    +  python-exe: python3
    8 
    +  packages:
    9 
    +  - flake8

  • tests/sources/previous_source_access.py
    1 
    +import os
    2 
    +import pytest
    3 
    +
    4 
    +from tests.testutils import cli
    5 
    +
    6 
    +DATA_DIR = os.path.join(
    7 
    +    os.path.dirname(os.path.realpath(__file__)),
    8 
    +    'previous_source_access'
    9 
    +)
    10 
    +
    11 
    +
    12 
    +##################################################################
    13 
    +#                              Tests                             #
    14 
    +##################################################################
    15 
    +# Test that plugins can access data from previous sources
    16 
    +@pytest.mark.datafiles(DATA_DIR)
    17 
    +def test_custom_transform_source(cli, tmpdir, datafiles):
    18 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    19 
    +
    20 
    +    # Ensure we can track
    21 
    +    result = cli.run(project=project, args=[
    22 
    +        'track', 'target.bst'
    23 
    +    ])
    24 
    +    result.assert_success()
    25 
    +
    26 
    +    # Ensure we can fetch
    27 
    +    result = cli.run(project=project, args=[
    28 
    +        'fetch', 'target.bst'
    29 
    +    ])
    30 
    +    result.assert_success()
    31 
    +
    32 
    +    # Ensure we get correct output from foo_transform
    33 
    +    result = cli.run(project=project, args=[
    34 
    +        'build', 'target.bst'
    35 
    +    ])
    36 
    +    destpath = os.path.join(cli.directory, 'checkout')
    37 
    +    result = cli.run(project=project, args=[
    38 
    +        'checkout', 'target.bst', destpath
    39 
    +    ])
    40 
    +    result.assert_success()
    41 
    +    # Assert that files from both sources exist, and that they have
    42 
    +    # the same content
    43 
    +    assert os.path.exists(os.path.join(destpath, 'file'))
    44 
    +    assert os.path.exists(os.path.join(destpath, 'filetransform'))
    45 
    +    with open(os.path.join(destpath, 'file')) as file1:
    46 
    +        with open(os.path.join(destpath, 'filetransform')) as file2:
    47 
    +            assert file1.read() == file2.read()

  • tests/sources/previous_source_access/elements/target.bst
    1 
    +kind: import
    2 
    +
    3 
    +sources:
    4 
    +- kind: local
    5 
    +  path: files/file
    6 
    +- kind: foo_transform

  • tests/sources/previous_source_access/files/file
    1 
    +Hello World!

  • tests/sources/previous_source_access/plugins/sources/foo_transform.py
    1 
    +"""
    2 
    +foo_transform - transform "file" from previous sources into "filetransform"
    3 
    +===========================================================================
    4 
    +
    5 
    +This is a test source plugin that looks for a file named "file" staged by
    6 
    +previous sources, and copies its contents to a file called "filetransform".
    7 
    +
    8 
    +"""
    9 
    +
    10 
    +import os
    11 
    +import hashlib
    12 
    +
    13 
    +from buildstream import Consistency, Source, SourceError, utils
    14 
    +
    15 
    +
    16 
    +class FooTransformSource(Source):
    17 
    +
    18 
    +    # We need access to previous both at track time and fetch time
    19 
    +    requires_previous_sources_track = True
    20 
    +    requires_previous_sources_fetch = True
    21 
    +
    22 
    +    @property
    23 
    +    def mirror(self):
    24 
    +        """Directory where this source should stage its files
    25 
    +
    26 
    +        """
    27 
    +        path = os.path.join(self.get_mirror_directory(), self.name,
    28 
    +                            self.ref.strip())
    29 
    +        os.makedirs(path, exist_ok=True)
    30 
    +        return path
    31 
    +
    32 
    +    def configure(self, node):
    33 
    +        self.node_validate(node, ['ref'] + Source.COMMON_CONFIG_KEYS)
    34 
    +        self.ref = self.node_get_member(node, str, 'ref', None)
    35 
    +
    36 
    +    def preflight(self):
    37 
    +        pass
    38 
    +
    39 
    +    def get_unique_key(self):
    40 
    +        return (self.ref,)
    41 
    +
    42 
    +    def get_consistency(self):
    43 
    +        if self.ref is None:
    44 
    +            return Consistency.INCONSISTENT
    45 
    +        # If we have a file called "filetransform", verify that its checksum
    46 
    +        # matches our ref. Otherwise, it resolved but not cached.
    47 
    +        fpath = os.path.join(self.mirror, 'filetransform')
    48 
    +        try:
    49 
    +            with open(fpath, 'rb') as f:
    50 
    +                if hashlib.sha256(f.read()).hexdigest() == self.ref.strip():
    51 
    +                    return Consistency.CACHED
    52 
    +        except Exception:
    53 
    +            pass
    54 
    +        return Consistency.RESOLVED
    55 
    +
    56 
    +    def get_ref(self):
    57 
    +        return self.ref
    58 
    +
    59 
    +    def set_ref(self, ref, node):
    60 
    +        self.ref = node['ref'] = ref
    61 
    +
    62 
    +    def track(self, previous_sources_dir):
    63 
    +        # Store the checksum of the file from previous source as our ref
    64 
    +        fpath = os.path.join(previous_sources_dir, 'file')
    65 
    +        with open(fpath, 'rb') as f:
    66 
    +            return hashlib.sha256(f.read()).hexdigest()
    67 
    +
    68 
    +    def fetch(self, previous_sources_dir):
    69 
    +        fpath = os.path.join(previous_sources_dir, 'file')
    70 
    +        # Verify that the checksum of the file from previous source matches
    71 
    +        # our ref
    72 
    +        with open(fpath, 'rb') as f:
    73 
    +            if hashlib.sha256(f.read()).hexdigest() != self.ref.strip():
    74 
    +                raise SourceError("Element references do not match")
    75 
    +
    76 
    +        # Copy "file" as "filetransform"
    77 
    +        newfpath = os.path.join(self.mirror, 'filetransform')
    78 
    +        utils.safe_copy(fpath, newfpath)
    79 
    +
    80 
    +    def stage(self, directory):
    81 
    +        # Simply stage the "filetransform" file
    82 
    +        utils.safe_copy(os.path.join(self.mirror, 'filetransform'),
    83 
    +                        os.path.join(directory, 'filetransform'))
    84 
    +
    85 
    +
    86 
    +def setup():
    87 
    +    return FooTransformSource

  • tests/sources/previous_source_access/project.conf
    1 
    +# Project with local source plugins
    2 
    +name: foo
    3 
    +
    4 
    +element-path: elements
    5 
    +
    6 
    +plugins:
    7 
    +- origin: local
    8 
    +  path: plugins/sources
    9 
    +  sources:
    10 
    +    foo_transform: 0

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]