GNOME.org
 

[Notes] [Git][BuildStream/buildstream][chiaratolentino/fix-pip-source-regex] 2 commits: plugins/sources/pip.py: Accomodate characters '-', '.', '_' for packages

Title: GitLab

Chiara Tolentino pushed to branch chiaratolentino/fix-pip-source-regex at BuildStream / buildstream

Commits:

9 changed files:

Changes:

  • buildstream/plugins/sources/pip.py
    ... ... @@ -96,7 +96,7 @@ _PYTHON_VERSIONS = [
    96 96 
     # Names of source distribution archives must be of the form
    97 97 
     # '%{package-name}-%{version}.%{extension}'.
    98 98 
     _SDIST_RE = re.compile(
    99 
    -    r'^([a-zA-Z0-9]+?)-(.+).(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
    99 
    +    r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
    100 100 
         re.IGNORECASE)
    101 101
    102 102
    ... ... @@ -225,12 +225,27 @@ class PipSource(Source):
    225 225 
         def _parse_sdist_names(self, basedir):
    226 226 
             reqs = []
    227 227 
             for f in os.listdir(basedir):
    228 
    -            pkg_match = _SDIST_RE.match(f)
    229 
    -            if pkg_match:
    230 
    -                reqs.append(pkg_match.groups())
    228 
    +            pkg = _match_package_name(f)
    229 
    +            if pkg is not None:
    230 
    +                reqs.append(pkg)
    231 231
    232 232 
             return sorted(reqs)
    233 233
    234 234
    235 
    +# Extract the package name and version of a source distribution
    236 
    +#
    237 
    +# Args:
    238 
    +#    filename (str): Filename of the source distribution
    239 
    +#
    240 
    +# Returns:
    241 
    +#    (tuple): A tuple of (package_name, version)
    242 
    +#
    243 
    +def _match_package_name(filename):
    244 
    +    pkg_match = _SDIST_RE.match(filename)
    245 
    +    if pkg_match is None:
    246 
    +        return None
    247 
    +    return pkg_match.groups()
    248 
    +
    249 
    +
    235 250 
     def setup():
    236 251 
         return PipSource

  • tests/integration/pip_source.py
    ... ... @@ -4,6 +4,7 @@ import pytest
    4 4 
     from buildstream import _yaml
    5 5
    6 6 
     from tests.testutils import cli_integration as cli
    7 
    +from tests.testutils.python_repo import setup_pypi_repo
    7 8 
     from tests.testutils.integration import assert_contains
    8 9
    9 10
    ... ... @@ -17,12 +18,21 @@ DATA_DIR = os.path.join(
    17 18
    18 19
    19 20 
     @pytest.mark.datafiles(DATA_DIR)
    20 
    -def test_pip_source_import(cli, tmpdir, datafiles):
    21 
    +def test_pip_source_import(cli, tmpdir, datafiles, setup_pypi_repo):
    21 22 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    22 23 
         checkout = os.path.join(cli.directory, 'checkout')
    23 24 
         element_path = os.path.join(project, 'elements')
    24 25 
         element_name = 'pip/hello.bst'
    25 26
    27 
    +    # check that exotically named packages are imported correctly
    28 
    +    myreqs_packages = ['hellolib']
    29 
    +    packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
    30 
    +
    31 
    +    # create mock pypi repository
    32 
    +    pypi_repo = os.path.join(project, 'files', 'pypi-repo')
    33 
    +    os.makedirs(pypi_repo, exist_ok=True)
    34 
    +    setup_pypi_repo(myreqs_packages + packages, pypi_repo)
    35 
    +
    26 36 
         element = {
    27 37 
             'kind': 'import',
    28 38 
             'sources': [
    ... ... @@ -32,9 +42,9 @@ def test_pip_source_import(cli, tmpdir, datafiles):
    32 42 
                 },
    33 43 
                 {
    34 44 
                     'kind': 'pip',
    35 
    -                'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
    45 
    +                'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
    36 46 
                     'requirements-files': ['myreqs.txt'],
    37 
    -                'packages': ['app2']
    47 
    +                'packages': packages
    38 48 
                 }
    39 49 
             ]
    40 50 
         }
    ... ... @@ -51,16 +61,31 @@ def test_pip_source_import(cli, tmpdir, datafiles):
    51 61 
         assert result.exit_code == 0
    52 62
    53 63 
         assert_contains(checkout, ['/.bst_pip_downloads',
    54 
    -                               '/.bst_pip_downloads/HelloLib-0.1.tar.gz',
    55 
    -                               '/.bst_pip_downloads/App2-0.1.tar.gz'])
    64 
    +                               '/.bst_pip_downloads/hellolib-0.1.tar.gz',
    65 
    +                               '/.bst_pip_downloads/app2-0.1.tar.gz',
    66 
    +                               '/.bst_pip_downloads/app.3-0.1.tar.gz',
    67 
    +                               '/.bst_pip_downloads/app-4-0.1.tar.gz',
    68 
    +                               '/.bst_pip_downloads/app_5-0.1.tar.gz',
    69 
    +                               '/.bst_pip_downloads/app.no.6-0.1.tar.gz',
    70 
    +                               '/.bst_pip_downloads/app-no-7-0.1.tar.gz',
    71 
    +                               '/.bst_pip_downloads/app_no_8-0.1.tar.gz'])
    56 72
    57 73
    58 74 
     @pytest.mark.datafiles(DATA_DIR)
    59 
    -def test_pip_source_build(cli, tmpdir, datafiles):
    75 
    +def test_pip_source_build(cli, tmpdir, datafiles, setup_pypi_repo):
    60 76 
         project = os.path.join(datafiles.dirname, datafiles.basename)
    61 77 
         element_path = os.path.join(project, 'elements')
    62 78 
         element_name = 'pip/hello.bst'
    63 79
    80 
    +    # check that exotically named packages are imported correctly
    81 
    +    myreqs_packages = ['hellolib']
    82 
    +    packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
    83 
    +
    84 
    +    # create mock pypi repository
    85 
    +    pypi_repo = os.path.join(project, 'files', 'pypi-repo')
    86 
    +    os.makedirs(pypi_repo, exist_ok=True)
    87 
    +    setup_pypi_repo(myreqs_packages + packages, pypi_repo)
    88 
    +
    64 89 
         element = {
    65 90 
             'kind': 'manual',
    66 91 
             'depends': ['base.bst'],
    ... ... @@ -71,16 +96,15 @@ def test_pip_source_build(cli, tmpdir, datafiles):
    71 96 
                 },
    72 97 
                 {
    73 98 
                     'kind': 'pip',
    74 
    -                'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
    99 
    +                'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
    75 100 
                     'requirements-files': ['myreqs.txt'],
    76 
    -                'packages': ['app2']
    101 
    +                'packages': packages
    77 102 
                 }
    78 103 
             ],
    79 104 
             'config': {
    80 105 
                 'install-commands': [
    81 106 
                     'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz',
    82 
    -                'chmod +x app1.py',
    83 
    -                'install app1.py  %{install-root}/usr/bin/'
    107 
    +                'install app1.py %{install-root}/usr/bin/'
    84 108 
                 ]
    85 109 
             }
    86 110 
         }
    ... ... @@ -95,5 +119,4 @@ def test_pip_source_build(cli, tmpdir, datafiles):
    95 119
    96 120 
         result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py'])
    97 121 
         assert result.exit_code == 0
    98 
    -    assert result.output == """Hello App1!
    99 
    -"""
    122 
    +    assert result.output == "Hello App1! This is hellolib\n"

  • tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz deleted
    No preview for this file type
  • tests/integration/project/files/pypi-repo/app2/index.html deleted
    1 
    -<html>
    2 
    -  <head>
    3 
    -    <title>Links for app1</title>
    4 
    -  </head>
    5 
    -  <body>
    6 
    -    <a href="">'App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
    7 
    -  </body>
    8 
    -</html>

  • tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz deleted
    No preview for this file type
  • tests/integration/project/files/pypi-repo/hellolib/index.html deleted
    1 
    -<html>
    2 
    -  <head>
    3 
    -    <title>Links for app1</title>
    4 
    -  </head>
    5 
    -  <body>
    6 
    -    <a href="">'HelloLib-0.1.tar.gz'>HelloLib-0.1.tar.gz</a><br />
    7 
    -  </body>
    8 
    -</html>

  • tests/sources/pip.py
    ... ... @@ -3,6 +3,7 @@ import pytest
    3 3
    4 4 
     from buildstream._exceptions import ErrorDomain
    5 5 
     from buildstream import _yaml
    6 
    +from buildstream.plugins.sources.pip import _match_package_name
    6 7 
     from tests.testutils import cli
    7 8
    8 9 
     DATA_DIR = os.path.join(
    ... ... @@ -45,3 +46,22 @@ def test_no_packages(cli, tmpdir, datafiles):
    45 46 
             'show', 'target.bst'
    46 47 
         ])
    47 48 
         result.assert_main_error(ErrorDomain.SOURCE, None)
    49 
    +
    50 
    +
    51 
    +# Test that pip source parses tar ball names correctly for the ref
    52 
    +@pytest.mark.parametrize(
    53 
    +    'tarball, expected_name, expected_version',
    54 
    +    [
    55 
    +        ('dotted.package-0.9.8.tar.gz', 'dotted.package', '0.9.8'),
    56 
    +        ('hyphenated-package-2.6.0.tar.gz', 'hyphenated-package', '2.6.0'),
    57 
    +        ('underscore_pkg-3.1.0.tar.gz', 'underscore_pkg', '3.1.0'),
    58 
    +        ('numbers2and5-1.0.1.tar.gz', 'numbers2and5', '1.0.1'),
    59 
    +        ('multiple.dots.package-5.6.7.tar.gz', 'multiple.dots.package', '5.6.7'),
    60 
    +        ('multiple-hyphens-package-1.2.3.tar.gz', 'multiple-hyphens-package', '1.2.3'),
    61 
    +        ('multiple_underscore_pkg-3.4.5.tar.gz', 'multiple_underscore_pkg', '3.4.5'),
    62 
    +        ('shortversion-1.0.tar.gz', 'shortversion', '1.0'),
    63 
    +        ('longversion-1.2.3.4.tar.gz', 'longversion', '1.2.3.4')
    64 
    +    ])
    65 
    +def test_match_package_name(tarball, expected_name, expected_version):
    66 
    +    name, version = _match_package_name(tarball)
    67 
    +    assert (expected_name, expected_version) == (name, version)

  • tests/testutils/__init__.py
    ... ... @@ -29,3 +29,4 @@ from .artifactshare import create_artifact_share
    29 29 
     from .element_generators import create_element_size, update_element_size
    30 30 
     from .junction import generate_junction
    31 31 
     from .runner_integration import wait_for_cache_granularity
    32 
    +from .python_repo import setup_pypi_repo

  • tests/testutils/python_repo.py
    1 
    +from setuptools.sandbox import run_setup
    2 
    +import os
    3 
    +import pytest
    4 
    +import re
    5 
    +import shutil
    6 
    +
    7 
    +
    8 
    +SETUP_TEMPLATE = '''\
    9 
    +from setuptools import setup
    10 
    +
    11 
    +setup(
    12 
    +    name='{name}',
    13 
    +    version='{version}',
    14 
    +    description='{name}',
    15 
    +    packages=['{pkgdirname}'],
    16 
    +    entry_points={{
    17 
    +        'console_scripts': [
    18 
    +            '{pkgdirname}={pkgdirname}:main'
    19 
    +        ]
    20 
    +    }}
    21 
    +)
    22 
    +'''
    23 
    +
    24 
    +# All packages generated via generate_pip_package will have the functions below
    25 
    +INIT_TEMPLATE = '''\
    26 
    +def main():
    27 
    +    print('This is {name}')
    28 
    +
    29 
    +def hello(actor='world'):
    30 
    +    print('Hello {{}}! This is {name}'.format(actor))
    31 
    +'''
    32 
    +
    33 
    +HTML_TEMPLATE = '''\
    34 
    +<html>
    35 
    +  <head>
    36 
    +    <title>Links for {name}</title>
    37 
    +  </head>
    38 
    +  <body>
    39 
    +    <a href=''>{name}-{version}.tar.gz</a><br />
    40 
    +  </body>
    41 
    +</html>
    42 
    +'''
    43 
    +
    44 
    +
    45 
    +# Creates a simple python source distribution and copies this into a specified
    46 
    +# directory which is to serve as a mock python repository
    47 
    +#
    48 
    +# Args:
    49 
    +#    tmpdir (str): Directory in which the source files will be created
    50 
    +#    pypi (str): Directory serving as a mock python repository
    51 
    +#    name (str): The name of the package to be created
    52 
    +#    version (str): The version of the package to be created
    53 
    +#
    54 
    +# Returns:
    55 
    +#    None
    56 
    +#
    57 
    +def generate_pip_package(tmpdir, pypi, name, version='0.1'):
    58 
    +    # check if package already exists in pypi
    59 
    +    pypi_package = os.path.join(pypi, re.sub('[^0-9a-zA-Z]+', '-', name))
    60 
    +    if os.path.exists(pypi_package):
    61 
    +        return
    62 
    +
    63 
    +    # create the package source files in tmpdir resulting in a directory
    64 
    +    # tree resembling the following structure:
    65 
    +    #
    66 
    +    # tmpdir
    67 
    +    # |-- setup.py
    68 
    +    # `-- package
    69 
    +    #     `-- __init__.py
    70 
    +    #
    71 
    +    setup_file = os.path.join(tmpdir, 'setup.py')
    72 
    +    pkgdirname = re.sub('[^0-9a-zA-Z]+', '', name)
    73 
    +    with open(setup_file, 'w') as f:
    74 
    +        f.write(
    75 
    +            SETUP_TEMPLATE.format(
    76 
    +                name=name,
    77 
    +                version=version,
    78 
    +                pkgdirname=pkgdirname
    79 
    +            )
    80 
    +        )
    81 
    +    os.chmod(setup_file, 0o755)
    82 
    +
    83 
    +    package = os.path.join(tmpdir, pkgdirname)
    84 
    +    os.makedirs(package)
    85 
    +
    86 
    +    main_file = os.path.join(package, '__init__.py')
    87 
    +    with open(main_file, 'w') as f:
    88 
    +        f.write(INIT_TEMPLATE.format(name=name))
    89 
    +    os.chmod(main_file, 0o644)
    90 
    +
    91 
    +    run_setup(setup_file, ['sdist'])
    92 
    +
    93 
    +    # create directory for this package in pypi resulting in a directory
    94 
    +    # tree resembling the following structure:
    95 
    +    #
    96 
    +    # pypi
    97 
    +    # `-- pypi_package
    98 
    +    #     |-- index.html
    99 
    +    #     `-- foo-0.1.tar.gz
    100 
    +    #
    101 
    +    os.makedirs(pypi_package)
    102 
    +
    103 
    +    # add an index html page
    104 
    +    index_html = os.path.join(pypi_package, 'index.html')
    105 
    +    with open(index_html, 'w') as f:
    106 
    +        f.write(HTML_TEMPLATE.format(name=name, version=version))
    107 
    +
    108 
    +    # copy generated tarfile to pypi package
    109 
    +    dist_dir = os.path.join(tmpdir, 'dist')
    110 
    +    for tar in os.listdir(dist_dir):
    111 
    +        tarpath = os.path.join(dist_dir, tar)
    112 
    +        shutil.copy(tarpath, pypi_package)
    113 
    +
    114 
    +
    115 
    +@pytest.fixture
    116 
    +def setup_pypi_repo(tmpdir):
    117 
    +    def create_pkgdir(package):
    118 
    +        pkgdirname = re.sub('[^0-9a-zA-Z]+', '', package)
    119 
    +        pkgdir = os.path.join(str(tmpdir), pkgdirname)
    120 
    +        os.makedirs(pkgdir)
    121 
    +        return pkgdir
    122 
    +
    123 
    +    def add_packages(packages, pypi_repo):
    124 
    +        for package in packages:
    125 
    +            pkgdir = create_pkgdir(package)
    126 
    +            generate_pip_package(pkgdir, pypi_repo, package)
    127 
    +
    128 
    +    return add_packages

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]