[Notes] [Git][BuildStream/buildstream][chandan/sourcetransform] Add pip source plugin



Title: GitLab

Chandan Singh pushed to branch chandan/sourcetransform at BuildStream / buildstream

Commits:

21 changed files:

Changes:

  • buildstream/_versions.py
    ... ... @@ -23,7 +23,7 @@
    23 23
     # This version is bumped whenever enhancements are made
    
    24 24
     # to the `project.conf` format or the core element format.
    
    25 25
     #
    
    26
    -BST_FORMAT_VERSION = 13
    
    26
    +BST_FORMAT_VERSION = 14
    
    27 27
     
    
    28 28
     
    
    29 29
     # The base BuildStream artifact version
    

  • buildstream/plugins/sources/pip.py
    1
    +#
    
    2
    +#  Copyright 2018 Bloomberg Finance LP
    
    3
    +#
    
    4
    +#  This program is free software; you can redistribute it and/or
    
    5
    +#  modify it under the terms of the GNU Lesser General Public
    
    6
    +#  License as published by the Free Software Foundation; either
    
    7
    +#  version 2 of the License, or (at your option) any later version.
    
    8
    +#
    
    9
    +#  This library is distributed in the hope that it will be useful,
    
    10
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    
    11
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    
    12
    +#  Lesser General Public License for more details.
    
    13
    +#
    
    14
    +#  You should have received a copy of the GNU Lesser General Public
    
    15
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    
    16
    +#
    
    17
    +#  Authors:
    
    18
    +#        Chandan Singh <csingh43 bloomberg net>
    
    19
    +
    
    20
    +"""
    
    21
    +pip - stage python packages using pip
    
    22
    +=====================================
    
    23
    +
    
    24
    +**Host depndencies:**
    
    25
    +
    
    26
    +  * ``pip`` python module
    
    27
    +
    
    28
    +This plugin will download source distributions for specified packages using
    
    29
    +``pip`` but will not install them. It is expected that the elements using this
    
    30
    +source will install the downloaded packages.
    
    31
    +
    
    32
    +**Usage:**
    
    33
    +
    
    34
    +.. code:: yaml
    
    35
    +
    
    36
    +   # Specify the pip source kind
    
    37
    +   kind: pip
    
    38
    +
    
    39
    +   # Optionally specify index url, defaults to PyPi
    
    40
    +   # This url is used to discover new versions of packages and download them
    
    41
    +   # Projects intending to mirror their sources to a permanent location should
    
    42
    +   # use an aliased url, and declare the alias in the project configuration
    
    43
    +   url: https://mypypi.example.com/simple
    
    44
    +
    
    45
    +   # Optionally specify the path to requirements files
    
    46
    +   # Note that either 'requirements-files' or 'packages' must be defined
    
    47
    +   requirements-files:
    
    48
    +   - requirements.txt
    
    49
    +
    
    50
    +   # Optionally specify a list of additional packages
    
    51
    +   # Note that either 'requirements-files' or 'packages' must be defined
    
    52
    +   packages:
    
    53
    +   - flake8
    
    54
    +
    
    55
    +   # Optionally specify a relative staging directory
    
    56
    +   directory: path/to/stage
    
    57
    +
    
    58
    +   # Specify the ref. It is a list of strings of format
    
    59
    +   # "<package-name>==<version>", separated by "\\n".
    
    60
    +   # Usually this will be contents of a requirements.txt file where all
    
    61
    +   # package versions have been frozen.
    
    62
    +   ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0"
    
    63
    +
    
    64
    +.. note::
    
    65
    +
    
    66
    +   The ``pip`` plugin is available since :ref:`format version 14 <project_format_version>`
    
    67
    +
    
    68
    +"""
    
    69
    +
    
    70
    +import errno
    
    71
    +import hashlib
    
    72
    +import os
    
    73
    +import re
    
    74
    +
    
    75
    +from buildstream import Consistency, Source, SourceError, utils
    
    76
    +
    
    77
    +_PYPI_INDEX_URL = 'https://pypi.org/simple/'
    
    78
    +
    
    79
    +# Used only for finding pip command
    
    80
    +_PYTHON_VERSIONS = [
    
    81
    +    'python2.7',
    
    82
    +    'python3.0',
    
    83
    +    'python3.1',
    
    84
    +    'python3.2',
    
    85
    +    'python3.3',
    
    86
    +    'python3.4',
    
    87
    +    'python3.5',
    
    88
    +    'python3.6',
    
    89
    +    'python3.7',
    
    90
    +]
    
    91
    +
    
    92
    +# List of allowed extensions taken from
    
    93
    +# https://docs.python.org/3/distutils/sourcedist.html.
    
    94
    +# Names of source distribution archives must be of the form
    
    95
    +# '%{package-name}-%{version}.%{extension}'.
    
    96
    +_SDIST_RE = re.compile(
    
    97
    +    r'^([a-zA-Z0-9]+?)-(.+).(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
    
    98
    +    re.IGNORECASE)
    
    99
    +
    
    100
    +
    
    101
    +class PipSource(Source):
    
    102
    +    # pylint: disable=attribute-defined-outside-init
    
    103
    +
    
    104
    +    # We need access to previous sources at track time to use requirements.txt
    
    105
    +    # but not at fetch time as self.ref should contain sufficient information
    
    106
    +    # for this plugin
    
    107
    +    BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True
    
    108
    +
    
    109
    +    def configure(self, node):
    
    110
    +        self.node_validate(node, ['url', 'packages', 'ref', 'requirements-files'] +
    
    111
    +                           Source.COMMON_CONFIG_KEYS)
    
    112
    +        self.ref = self.node_get_member(node, str, 'ref', None)
    
    113
    +        self.original_url = self.node_get_member(node, str, 'url', _PYPI_INDEX_URL)
    
    114
    +        self.index_url = self.translate_url(self.original_url)
    
    115
    +        self.packages = self.node_get_member(node, list, 'packages', [])
    
    116
    +        self.requirements_files = self.node_get_member(node, list, 'requirements-files', [])
    
    117
    +
    
    118
    +        if not (self.packages or self.requirements_files):
    
    119
    +            raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self))
    
    120
    +
    
    121
    +    def preflight(self):
    
    122
    +        # Try to find a pip version that supports download command
    
    123
    +        self.host_pip = None
    
    124
    +        for python in reversed(_PYTHON_VERSIONS):
    
    125
    +            try:
    
    126
    +                host_python = utils.get_host_tool(python)
    
    127
    +                rc = self.call([host_python, '-m', 'pip', 'download', '--help'])
    
    128
    +                if rc == 0:
    
    129
    +                    self.host_pip = [host_python, '-m', 'pip']
    
    130
    +                    break
    
    131
    +            except utils.ProgramNotFoundError:
    
    132
    +                pass
    
    133
    +
    
    134
    +        if self.host_pip is None:
    
    135
    +            raise SourceError("{}: Unable to find a suitable pip command".format(self))
    
    136
    +
    
    137
    +    def get_unique_key(self):
    
    138
    +        return [self.original_url, self.ref]
    
    139
    +
    
    140
    +    def get_consistency(self):
    
    141
    +        if not self.ref:
    
    142
    +            return Consistency.INCONSISTENT
    
    143
    +        if os.path.exists(self._mirror) and os.listdir(self._mirror):
    
    144
    +            return Consistency.CACHED
    
    145
    +        return Consistency.RESOLVED
    
    146
    +
    
    147
    +    def get_ref(self):
    
    148
    +        return self.ref
    
    149
    +
    
    150
    +    def load_ref(self, node):
    
    151
    +        self.ref = self.node_get_member(node, str, 'ref', None)
    
    152
    +
    
    153
    +    def set_ref(self, ref, node):
    
    154
    +        node['ref'] = self.ref = ref
    
    155
    +
    
    156
    +    def track(self, previous_sources_dir):
    
    157
    +        # XXX pip does not offer any public API other than the CLI tool so it
    
    158
    +        # is not feasible to correctly parse the requirements file or to check
    
    159
    +        # which package versions pip is going to install.
    
    160
    +        # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
    
    161
    +        # for details.
    
    162
    +        # As a result, we have to wastefully install the packages during track.
    
    163
    +        with self.tempdir() as tmpdir:
    
    164
    +            install_args = self.host_pip + ['download',
    
    165
    +                                            '--no-binary', ':all:',
    
    166
    +                                            '--index-url', self.index_url,
    
    167
    +                                            '--dest', tmpdir]
    
    168
    +            for requirement_file in self.requirements_files:
    
    169
    +                fpath = os.path.join(previous_sources_dir, requirement_file)
    
    170
    +                install_args += ['-r', fpath]
    
    171
    +            install_args += self.packages
    
    172
    +
    
    173
    +            self.call(install_args, fail="Failed to install python packages")
    
    174
    +            reqs = self._parse_sdist_names(tmpdir)
    
    175
    +
    
    176
    +        return '\n'.join(["{}=={}".format(pkg, ver) for pkg, ver in reqs])
    
    177
    +
    
    178
    +    def fetch(self):
    
    179
    +        with self.tempdir() as tmpdir:
    
    180
    +            packages = self.ref.strip().split('\n')
    
    181
    +            package_dir = os.path.join(tmpdir, 'packages')
    
    182
    +            os.makedirs(package_dir)
    
    183
    +            self.call(self.host_pip + ['download',
    
    184
    +                                       '--no-binary', ':all:',
    
    185
    +                                       '--index-url', self.index_url,
    
    186
    +                                       '--dest', package_dir] + packages,
    
    187
    +                      fail="Failed to install python packages: {}".format(packages))
    
    188
    +
    
    189
    +            # If the mirror directory already exists, assume that some other
    
    190
    +            # process has fetched the sources before us and ensure that we do
    
    191
    +            # not raise an error in that case.
    
    192
    +            try:
    
    193
    +                os.makedirs(self._mirror)
    
    194
    +                os.rename(package_dir, self._mirror)
    
    195
    +            except FileExistsError:
    
    196
    +                return
    
    197
    +            except OSError as e:
    
    198
    +                if e.errno != errno.ENOTEMPTY:
    
    199
    +                    raise
    
    200
    +
    
    201
    +    def stage(self, directory):
    
    202
    +        with self.timed_activity("Staging Python packages", silent_nested=True):
    
    203
    +            utils.copy_files(self._mirror, directory)
    
    204
    +
    
    205
    +    # Directory where this source should stage its files
    
    206
    +    #
    
    207
    +    @property
    
    208
    +    def _mirror(self):
    
    209
    +        if not self.ref:
    
    210
    +            return None
    
    211
    +        return os.path.join(self.get_mirror_directory(),
    
    212
    +                            self.original_url,
    
    213
    +                            hashlib.sha256(self.ref.encode()).hexdigest())
    
    214
    +
    
    215
    +    # Parse names of downloaded source distributions
    
    216
    +    #
    
    217
    +    # Args:
    
    218
    +    #    basedir (str): Directory containing source distribution archives
    
    219
    +    #
    
    220
    +    # Returns:
    
    221
    +    #    (list): List of (package_name, version) tuples in sorted order
    
    222
    +    #
    
    223
    +    def _parse_sdist_names(self, basedir):
    
    224
    +        reqs = []
    
    225
    +        for f in os.listdir(basedir):
    
    226
    +            pkg_match = _SDIST_RE.match(f)
    
    227
    +            if pkg_match:
    
    228
    +                reqs.append(pkg_match.groups())
    
    229
    +
    
    230
    +        return sorted(reqs)
    
    231
    +
    
    232
    +
    
    233
    +def setup():
    
    234
    +    return PipSource

  • doc/source/core_plugins.rst
    ... ... @@ -58,6 +58,7 @@ Sources
    58 58
        sources/ostree
    
    59 59
        sources/patch
    
    60 60
        sources/deb
    
    61
    +   sources/pip
    
    61 62
     
    
    62 63
     
    
    63 64
     External plugins
    

  • tests/cachekey/project/sources/pip1.bst
    1
    +kind: import
    
    2
    +
    
    3
    +sources:
    
    4
    +- kind: git
    
    5
    +  url: https://example.com/foo/foobar.git
    
    6
    +  ref: b99955530263172ed1beae52aed7a33885ef781f
    
    7
    +- kind: pip
    
    8
    +  url: https://pypi.example.com/simple
    
    9
    +  packages:
    
    10
    +  - horses
    
    11
    +  - ponies
    
    12
    +  ref: 'horses==0.0.1\nponies==0.0.2'

  • tests/cachekey/project/sources/pip1.expected
    1
    +494b2f80acdc21eebd007c0a7255b4c80ef9c4e8d79da5c7291c148944fd272b
    \ No newline at end of file

  • tests/cachekey/project/target.bst
    ... ... @@ -13,6 +13,7 @@ depends:
    13 13
     - sources/patch1.bst
    
    14 14
     - sources/patch2.bst
    
    15 15
     - sources/patch3.bst
    
    16
    +- sources/pip1.bst
    
    16 17
     - sources/remote1.bst
    
    17 18
     - sources/remote2.bst
    
    18 19
     - sources/tar1.bst
    

  • tests/cachekey/project/target.expected
    1
    -b11fe7a9883cee2846a6148703b53785dd031482b359ca56d1ff22b08a1b5cd2
    \ No newline at end of file
    1
    +e324201f54b2bff05f4d69d9f3aece9db62cf320524024c1351b199af589e631
    \ No newline at end of file

  • tests/integration/pip.pytests/integration/pip_element.py

  • tests/integration/pip_source.py
    1
    +import os
    
    2
    +import pytest
    
    3
    +
    
    4
    +from buildstream import _yaml
    
    5
    +
    
    6
    +from tests.testutils import cli_integration as cli
    
    7
    +from tests.testutils.integration import assert_contains
    
    8
    +
    
    9
    +
    
    10
    +pytestmark = pytest.mark.integration
    
    11
    +
    
    12
    +
    
    13
    +DATA_DIR = os.path.join(
    
    14
    +    os.path.dirname(os.path.realpath(__file__)),
    
    15
    +    "project"
    
    16
    +)
    
    17
    +
    
    18
    +
    
    19
    +@pytest.mark.datafiles(DATA_DIR)
    
    20
    +def test_pip_source_import(cli, tmpdir, datafiles):
    
    21
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    22
    +    checkout = os.path.join(cli.directory, 'checkout')
    
    23
    +    element_path = os.path.join(project, 'elements')
    
    24
    +    element_name = 'pip/hello.bst'
    
    25
    +
    
    26
    +    element = {
    
    27
    +        'kind': 'import',
    
    28
    +        'sources': [
    
    29
    +            {
    
    30
    +                'kind': 'local',
    
    31
    +                'path': 'files/pip-source'
    
    32
    +            },
    
    33
    +            {
    
    34
    +                'kind': 'pip',
    
    35
    +                'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
    
    36
    +                'requirements-files': ['myreqs.txt'],
    
    37
    +                'packages': ['app2']
    
    38
    +            }
    
    39
    +        ]
    
    40
    +    }
    
    41
    +    os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
    
    42
    +    _yaml.dump(element, os.path.join(element_path, element_name))
    
    43
    +
    
    44
    +    result = cli.run(project=project, args=['track', element_name])
    
    45
    +    assert result.exit_code == 0
    
    46
    +
    
    47
    +    result = cli.run(project=project, args=['build', element_name])
    
    48
    +    assert result.exit_code == 0
    
    49
    +
    
    50
    +    result = cli.run(project=project, args=['checkout', element_name, checkout])
    
    51
    +    assert result.exit_code == 0
    
    52
    +
    
    53
    +    assert_contains(checkout, ['/HelloLib-0.1.tar.gz', '/App2-0.1.tar.gz'])
    
    54
    +
    
    55
    +
    
    56
    +@pytest.mark.datafiles(DATA_DIR)
    
    57
    +def test_pip_source_build(cli, tmpdir, datafiles):
    
    58
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    59
    +    element_path = os.path.join(project, 'elements')
    
    60
    +    element_name = 'pip/hello.bst'
    
    61
    +
    
    62
    +    element = {
    
    63
    +        'kind': 'manual',
    
    64
    +        'depends': ['base.bst'],
    
    65
    +        'sources': [
    
    66
    +            {
    
    67
    +                'kind': 'local',
    
    68
    +                'path': 'files/pip-source'
    
    69
    +            },
    
    70
    +            {
    
    71
    +                'kind': 'pip',
    
    72
    +                'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
    
    73
    +                'requirements-files': ['myreqs.txt'],
    
    74
    +                'packages': ['app2'],
    
    75
    +                'directory': 'bst-pip-source-vendored'
    
    76
    +            }
    
    77
    +        ],
    
    78
    +        'config': {
    
    79
    +            'install-commands': [
    
    80
    +                'pip3 install --no-index --prefix %{install-root}/usr bst-pip-source-vendored/*.tar.gz',
    
    81
    +                'install app1.py  %{install-root}/usr/bin/'
    
    82
    +            ]
    
    83
    +        }
    
    84
    +    }
    
    85
    +    os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
    
    86
    +    _yaml.dump(element, os.path.join(element_path, element_name))
    
    87
    +
    
    88
    +    result = cli.run(project=project, args=['track', element_name])
    
    89
    +    assert result.exit_code == 0
    
    90
    +
    
    91
    +    result = cli.run(project=project, args=['build', element_name])
    
    92
    +    assert result.exit_code == 0
    
    93
    +
    
    94
    +    result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py'])
    
    95
    +    assert result.exit_code == 0
    
    96
    +    assert result.output == """Hello App1!
    
    97
    +"""

  • tests/integration/project/files/pip-source/app1.py
    1
    +from hellolib import hello
    
    2
    +
    
    3
    +
    
    4
    +def main():
    
    5
    +    hello('App1')
    
    6
    +
    
    7
    +
    
    8
    +if __name__ == '__main__':
    
    9
    +    main()

  • tests/integration/project/files/pip-source/myreqs.txt
    1
    +hellolib

  • tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz
    No preview for this file type
  • tests/integration/project/files/pypi-repo/app2/index.html
    1
    +<html>
    
    2
    +  <head>
    
    3
    +    <title>Links for app1</title>
    
    4
    +  </head>
    
    5
    +  <body>
    
    6
    +    <a href="">'App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
    
    7
    +  </body>
    
    8
    +</html>

  • tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz
    No preview for this file type
  • tests/integration/project/files/pypi-repo/hellolib/index.html
    1
    +<html>
    
    2
    +  <head>
    
    3
    +    <title>Links for app1</title>
    
    4
    +  </head>
    
    5
    +  <body>
    
    6
    +    <a href="">'HelloLib-0.1.tar.gz'>HelloLib-0.1.tar.gz</a><br />
    
    7
    +  </body>
    
    8
    +</html>

  • tests/sources/pip.py
    1
    +import os
    
    2
    +import pytest
    
    3
    +
    
    4
    +from buildstream._exceptions import ErrorDomain
    
    5
    +from buildstream import _yaml
    
    6
    +from tests.testutils import cli
    
    7
    +
    
    8
    +DATA_DIR = os.path.join(
    
    9
    +    os.path.dirname(os.path.realpath(__file__)),
    
    10
    +    'pip',
    
    11
    +)
    
    12
    +
    
    13
    +
    
    14
    +def generate_project(project_dir, tmpdir):
    
    15
    +    project_file = os.path.join(project_dir, "project.conf")
    
    16
    +    _yaml.dump({'name': 'foo'}, project_file)
    
    17
    +
    
    18
    +
    
    19
    +# Test that without ref, consistency is set appropriately.
    
    20
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
    
    21
    +def test_no_ref(cli, tmpdir, datafiles):
    
    22
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    23
    +    generate_project(project, tmpdir)
    
    24
    +    assert cli.get_element_state(project, 'target.bst') == 'no reference'
    
    25
    +
    
    26
    +
    
    27
    +# Test that pip is not allowed to be the first source
    
    28
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'first-source-pip'))
    
    29
    +def test_first_source(cli, tmpdir, datafiles):
    
    30
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    31
    +    generate_project(project, tmpdir)
    
    32
    +    result = cli.run(project=project, args=[
    
    33
    +        'show', 'target.bst'
    
    34
    +    ])
    
    35
    +    result.assert_main_error(ErrorDomain.ELEMENT, None)
    
    36
    +
    
    37
    +
    
    38
    +# Test that error is raised when neither packges nor requirements files
    
    39
    +# have been specified
    
    40
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-packages'))
    
    41
    +def test_no_packages(cli, tmpdir, datafiles):
    
    42
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    43
    +    generate_project(project, tmpdir)
    
    44
    +    result = cli.run(project=project, args=[
    
    45
    +        'show', 'target.bst'
    
    46
    +    ])
    
    47
    +    result.assert_main_error(ErrorDomain.SOURCE, None)

  • tests/sources/pip/first-source-pip/target.bst
    1
    +kind: import
    
    2
    +description: pip should not be allowed to be the first source
    
    3
    +sources:
    
    4
    +- kind: pip
    
    5
    +  packages:
    
    6
    +  - flake8

  • tests/sources/pip/no-packages/file
    1
    +Hello World!

  • tests/sources/pip/no-packages/target.bst
    1
    +kind: import
    
    2
    +description: The kind of this element is irrelevant.
    
    3
    +sources:
    
    4
    +- kind: local
    
    5
    +  path: file
    
    6
    +- kind: pip

  • tests/sources/pip/no-ref/file
    1
    +Hello World!

  • tests/sources/pip/no-ref/target.bst
    1
    +kind: import
    
    2
    +description: The kind of this element is irrelevant.
    
    3
    +sources:
    
    4
    +- kind: local
    
    5
    +  path: file
    
    6
    +- kind: pip
    
    7
    +  packages:
    
    8
    +  - flake8



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]