GNOME.org
 

[Notes] [Git][BuildStream/buildstream][chandan/pip-source] 2 commits: Add pip source plugin

Title: GitLab

Chandan Singh pushed to branch chandan/pip-source at BuildStream / buildstream

Commits:

10 changed files:

Changes:

  • buildstream/_versions.py
    ... ... @@ -23,7 +23,7 @@
    23 23 
     # This version is bumped whenever enhancements are made
    24 24 
     # to the `project.conf` format or the core element format.
    25 25 
     #
    26 
    -BST_FORMAT_VERSION = 11
    26 
    +BST_FORMAT_VERSION = 12
    27 27
    28 28
    29 29 
     # The base BuildStream artifact version

  • buildstream/plugins/sources/pip.py
    1 
    +#
    2 
    +#  Copyright Bloomberg Finance LP
    3 
    +#
    4 
    +#  This program is free software; you can redistribute it and/or
    5 
    +#  modify it under the terms of the GNU Lesser General Public
    6 
    +#  License as published by the Free Software Foundation; either
    7 
    +#  version 2 of the License, or (at your option) any later version.
    8 
    +#
    9 
    +#  This library is distributed in the hope that it will be useful,
    10 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    11 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    12 
    +#  Lesser General Public License for more details.
    13 
    +#
    14 
    +#  You should have received a copy of the GNU Lesser General Public
    15 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    16 
    +#
    17 
    +#  Authors:
    18 
    +#        Chandan Singh <csingh43 bloomberg net>
    19 
    +
    20 
    +"""
    21 
    +pip - stage python packages using pip
    22 
    +=====================================
    23 
    +
    24 
    +**Usage:**
    25 
    +
    26 
    +.. code:: yaml
    27 
    +
    28 
    +   # Specify the pip source kind
    29 
    +   kind: pip
    30 
    +
    31 
    +   # Optionally specify the python executable, defaults to system "python"
    32 
    +   # Note that either the venv module or the virtualenv CLI tool must be
    33 
    +   # available
    34 
    +   python-exe: python3.6
    35 
    +
    36 
    +   # Optionally specify index url, defaults to PyPi
    37 
    +   index-url: https://mypypi.example.com/simple
    38 
    +
    39 
    +   # Optionally specify the path to requirements files
    40 
    +   # Note that either 'requirements-files' or 'packages' must be defined
    41 
    +   requirements-files:
    42 
    +   - requirements.txt
    43 
    +
    44 
    +   # Optionally specify a list of additional packages
    45 
    +   # Note that either 'requirements-files' or 'packages' must be defined
    46 
    +   packages:
    47 
    +   - flake8
    48 
    +
    49 
    +   # Optionally specify a relative staging directory
    50 
    +   directory: path/to/stage
    51 
    +
    52 
    +   # Specify the ref. It is a list of strings of format
    53 
    +   # ``<package-name>==<version>`` separated by ``\n``.
    54 
    +   # Usually this will be contents of a requirements.txt file where all
    55 
    +   # package versions have been frozen.
    56 
    +   ref: "flake8==3.5.0\nmccabe==0.6.1\npkg-resources==0.0.0\npycodestyle==2.3.1\npyflakes==1.6.0"
    57 
    +
    58 
    +.. note::
    59 
    +
    60 
    +   The ``pip`` plugin is available since :ref:`format version 12 <project_format_version>`
    61 
    +
    62 
    +"""
    63 
    +
    64 
    +import hashlib
    65 
    +import os
    66 
    +
    67 
    +from buildstream import Consistency, Source, SourceError, utils
    68 
    +
    69 
    +_PYPI_INDEX_URL = 'https://pypi.org/simple/'
    70 
    +
    71 
    +
    72 
    +class PipSource(Source):
    73 
    +    # pylint: disable=attribute-defined-outside-init
    74 
    +
    75 
    +    # We need access to previous sources at track time to use requirements.txt
    76 
    +    # but not at fetch time as self.ref should contain sufficient information
    77 
    +    # for this plugin
    78 
    +    requires_previous_sources_track = True
    79 
    +
    80 
    +    def configure(self, node):
    81 
    +        self.node_validate(node, ['index-url', 'packages', 'python-exe', 'ref', 'requirements-files'] +
    82 
    +                           Source.COMMON_CONFIG_KEYS)
    83 
    +        self.ref = self.node_get_member(node, str, 'ref', None)
    84 
    +        self.python_exe = self.node_get_member(node, str, 'python-exe', 'python')
    85 
    +        self.index_url = self.node_get_member(node, str, 'index-url', _PYPI_INDEX_URL)
    86 
    +        self.packages = self.node_get_member(node, list, 'packages', [])
    87 
    +        self.requirements_files = self.node_get_member(node, list, 'requirements-files', [])
    88 
    +
    89 
    +    def preflight(self):
    90 
    +        # Try to find a way to open virtual environments on the host
    91 
    +        try:
    92 
    +            # Look for the virtualenv CLI first
    93 
    +            venv = utils.get_host_tool('virtualenv')
    94 
    +            self.venv_cmd = [venv, '--python', self.python_exe]
    95 
    +        except utils.ProgramNotFoundError:
    96 
    +            # Fall back to venv module if it is installed
    97 
    +            python_exe = utils.get_host_tool(self.python_exe)
    98 
    +            rc = self.call([python_exe, '-m', 'venv', '--help'])
    99 
    +            if rc == 0:
    100 
    +                self.venv_cmd = [python_exe, '-m', 'venv']
    101 
    +            else:
    102 
    +                raise SourceError("{}: venv module not found using python: {}"
    103 
    +                                  .format(self, python_exe))
    104 
    +
    105 
    +    def get_unique_key(self):
    106 
    +        return [self.venv_cmd, self.index_url, self.ref]
    107 
    +
    108 
    +    def get_consistency(self):
    109 
    +        if not self.ref:
    110 
    +            return Consistency.INCONSISTENT
    111 
    +        # FIXME add a stronger consistency check
    112 
    +        # Currently we take the presence of "something" as an indication that
    113 
    +        # we have the right things in cache
    114 
    +        if os.path.exists(self.mirror) and os.listdir(self.mirror):
    115 
    +            return Consistency.CACHED
    116 
    +        return Consistency.RESOLVED
    117 
    +
    118 
    +    def get_ref(self):
    119 
    +        return self.ref
    120 
    +
    121 
    +    def load_ref(self, node):
    122 
    +        self.ref = self.node_get_member(node, str, 'ref', None)
    123 
    +
    124 
    +    def set_ref(self, ref, node):
    125 
    +        node['ref'] = self.ref = ref
    126 
    +
    127 
    +    def track(self, previous_sources_dir):
    128 
    +        # XXX pip does not offer any public API other than the CLI tool so it
    129 
    +        # is not feasible to correctly parse the requirements file or to check
    130 
    +        # which package versions pip is going to install.
    131 
    +        # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
    132 
    +        # for details.
    133 
    +        # As a result, we have to wastefully install the packages during track.
    134 
    +        with self.tempdir() as tmpdir:
    135 
    +            pip = self._venv_pip(tmpdir)
    136 
    +
    137 
    +            install_args = [pip, 'install', '--index-url', self.index_url]
    138 
    +            for requirement_file in self.requirements_files:
    139 
    +                fpath = os.path.join(previous_sources_dir, requirement_file)
    140 
    +                install_args += ['-r', fpath]
    141 
    +            install_args += self.packages
    142 
    +
    143 
    +            self.call(install_args, fail="Failed to install python packages")
    144 
    +            _, reqs = self.check_output([pip, 'freeze'])
    145 
    +
    146 
    +        return reqs.strip()
    147 
    +
    148 
    +    def fetch(self):
    149 
    +        with self.tempdir() as tmpdir:
    150 
    +            pip = self._venv_pip(tmpdir)
    151 
    +            packages = self.ref.strip().split('\n')
    152 
    +            self.call([pip, 'install',
    153 
    +                       '--index-url', self.index_url,
    154 
    +                       '--prefix', self.mirror] +
    155 
    +                      packages,
    156 
    +                      fail="Failed to install python packages: {}".format(packages))
    157 
    +
    158 
    +    def stage(self, directory):
    159 
    +        with self.timed_activity("Staging Python packages", silent_nested=True):
    160 
    +            utils.copy_files(self.mirror, directory)
    161 
    +
    162 
    +    @property
    163 
    +    def mirror(self):
    164 
    +        """Directory where this source should stage its files
    165 
    +
    166 
    +        """
    167 
    +        path = os.path.join(self.get_mirror_directory(),
    168 
    +                            self.index_url,
    169 
    +                            hashlib.sha256(self.ref.encode()).hexdigest())
    170 
    +        os.makedirs(path, exist_ok=True)
    171 
    +        return path
    172 
    +
    173 
    +    def _venv_pip(self, directory):
    174 
    +        """Open a virtual environment in given directory and return pip path
    175 
    +
    176 
    +        """
    177 
    +        self.call(self.venv_cmd + [directory], fail="Failed to initialize virtual environment")
    178 
    +        pip_exe = os.path.join(directory, 'bin', 'pip')
    179 
    +        if not os.path.isfile(pip_exe):
    180 
    +            raise SourceError("Failed to initialize virtual environment")
    181 
    +        return pip_exe
    182 
    +
    183 
    +
    184 
    +def setup():
    185 
    +    return PipSource

  • tests/integration/pip.py → tests/integration/pip_element.py

  • tests/integration/pip_source.py
    1 
    +import os
    2 
    +import pytest
    3 
    +
    4 
    +from buildstream import _yaml
    5 
    +
    6 
    +from tests.testutils import cli_integration as cli
    7 
    +from tests.testutils.integration import assert_contains
    8 
    +
    9 
    +
    10 
    +pytestmark = pytest.mark.integration
    11 
    +
    12 
    +
    13 
    +DATA_DIR = os.path.join(
    14 
    +    os.path.dirname(os.path.realpath(__file__)),
    15 
    +    "project"
    16 
    +)
    17 
    +
    18 
    +
    19 
    +@pytest.mark.datafiles(DATA_DIR)
    20 
    +def test_pip_source(cli, tmpdir, datafiles):
    21 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    22 
    +    checkout = os.path.join(cli.directory, 'checkout')
    23 
    +    element_name = 'pip_source/pip-packages.bst'
    24 
    +
    25 
    +    result = cli.run(project=project, args=['track', element_name])
    26 
    +    assert result.exit_code == 0
    27 
    +
    28 
    +    result = cli.run(project=project, args=['build', element_name])
    29 
    +    assert result.exit_code == 0
    30 
    +
    31 
    +    result = cli.run(project=project, args=['checkout', element_name, checkout])
    32 
    +    assert result.exit_code == 0
    33 
    +
    34 
    +    assert_contains(checkout, ['/lib/python3.6/site-packages/mccabe.py',
    35 
    +                               '/lib/python3.6/site-packages/urllib3'])

  • tests/integration/project/elements/pip_source/pip-packages.bst
    1 
    +kind: import
    2 
    +description: Specify both `requirements-files` and `packages`, dependencies should get merged
    3 
    +
    4 
    +sources:
    5 
    +- kind: local
    6 
    +  path: files/pip-source
    7 
    +- kind: pip
    8 
    +  python-exe: python3.6
    9 
    +  requirements-files:
    10 
    +  - myreqs.txt
    11 
    +  packages:
    12 
    +  - urllib3

  • tests/integration/project/files/pip-source/myreqs.txt
    1 
    +mccabe==0.6.1

  • tests/sources/pip.py
    1 
    +import os
    2 
    +import pytest
    3 
    +
    4 
    +from buildstream._exceptions import ErrorDomain
    5 
    +from buildstream import _yaml
    6 
    +from tests.testutils import cli
    7 
    +
    8 
    +DATA_DIR = os.path.join(
    9 
    +    os.path.dirname(os.path.realpath(__file__)),
    10 
    +    'pip',
    11 
    +)
    12 
    +
    13 
    +
    14 
    +def generate_project(project_dir, tmpdir):
    15 
    +    project_file = os.path.join(project_dir, "project.conf")
    16 
    +    _yaml.dump({'name': 'foo'}, project_file)
    17 
    +
    18 
    +
    19 
    +# Test that without ref, consistency is set appropriately.
    20 
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
    21 
    +def test_no_ref(cli, tmpdir, datafiles):
    22 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    23 
    +    generate_project(project, tmpdir)
    24 
    +    assert cli.get_element_state(project, 'target.bst') == 'no reference'
    25 
    +
    26 
    +
    27 
    +# Test that pip is not allowed to be the first source
    28 
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'first-source-pip'))
    29 
    +def test_first_source(cli, tmpdir, datafiles):
    30 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    31 
    +    generate_project(project, tmpdir)
    32 
    +    result = cli.run(project=project, args=[
    33 
    +        'show', 'target.bst'
    34 
    +    ])
    35 
    +    result.assert_main_error(ErrorDomain.ELEMENT, None)

  • tests/sources/pip/first-source-pip/target.bst
    1 
    +kind: import
    2 
    +description: pip should not be allowed to be the first source
    3 
    +sources:
    4 
    +- kind: pip

  • tests/sources/pip/no-ref/file
    1 
    +Hello World!

  • tests/sources/pip/no-ref/target.bst
    1 
    +kind: import
    2 
    +description: The kind of this element is irrelevant.
    3 
    +sources:
    4 
    +- kind: local
    5 
    +  path: file
    6 
    +- kind: pip

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]