[Notes] [Git][BuildStream/buildstream][chiaratolentino/fix-pip-source-regex] 2 commits: plugins/sources/pip.py: Accomodate characters '-', '.', '_' for packages



Title: GitLab

Chiara Tolentino pushed to branch chiaratolentino/fix-pip-source-regex at BuildStream / buildstream

Commits:

9 changed files:

Changes:

  • buildstream/plugins/sources/pip.py
    ... ... @@ -96,7 +96,7 @@ _PYTHON_VERSIONS = [
    96 96
     # Names of source distribution archives must be of the form
    
    97 97
     # '%{package-name}-%{version}.%{extension}'.
    
    98 98
     _SDIST_RE = re.compile(
    
    99
    -    r'^([a-zA-Z0-9]+?)-(.+).(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
    
    99
    +    r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
    
    100 100
         re.IGNORECASE)
    
    101 101
     
    
    102 102
     
    
    ... ... @@ -225,12 +225,27 @@ class PipSource(Source):
    225 225
         def _parse_sdist_names(self, basedir):
    
    226 226
             reqs = []
    
    227 227
             for f in os.listdir(basedir):
    
    228
    -            pkg_match = _SDIST_RE.match(f)
    
    229
    -            if pkg_match:
    
    230
    -                reqs.append(pkg_match.groups())
    
    228
    +            pkg = _match_package_name(f)
    
    229
    +            if pkg is not None:
    
    230
    +                reqs.append(pkg)
    
    231 231
     
    
    232 232
             return sorted(reqs)
    
    233 233
     
    
    234 234
     
    
    235
    +# Extract the package name and version of a source distribution
    
    236
    +#
    
    237
    +# Args:
    
    238
    +#    filename (str): Filename of the source distribution
    
    239
    +#
    
    240
    +# Returns:
    
    241
    +#    (tuple): A tuple of (package_name, version)
    
    242
    +#
    
    243
    +def _match_package_name(filename):
    
    244
    +    pkg_match = _SDIST_RE.match(filename)
    
    245
    +    if pkg_match is None:
    
    246
    +        return None
    
    247
    +    return pkg_match.groups()
    
    248
    +
    
    249
    +
    
    235 250
     def setup():
    
    236 251
         return PipSource

  • tests/integration/pip_source.py
    ... ... @@ -4,6 +4,7 @@ import pytest
    4 4
     from buildstream import _yaml
    
    5 5
     
    
    6 6
     from tests.testutils import cli_integration as cli
    
    7
    +from tests.testutils.python_repo import setup_pypi_repo
    
    7 8
     from tests.testutils.integration import assert_contains
    
    8 9
     
    
    9 10
     
    
    ... ... @@ -17,12 +18,21 @@ DATA_DIR = os.path.join(
    17 18
     
    
    18 19
     
    
    19 20
     @pytest.mark.datafiles(DATA_DIR)
    
    20
    -def test_pip_source_import(cli, tmpdir, datafiles):
    
    21
    +def test_pip_source_import(cli, tmpdir, datafiles, setup_pypi_repo):
    
    21 22
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    22 23
         checkout = os.path.join(cli.directory, 'checkout')
    
    23 24
         element_path = os.path.join(project, 'elements')
    
    24 25
         element_name = 'pip/hello.bst'
    
    25 26
     
    
    27
    +    # create mock pypi repository
    
    28
    +    pypi_repo = os.path.join(project, 'files', 'pypi-repo')
    
    29
    +    os.makedirs(pypi_repo, exist_ok=True)
    
    30
    +
    
    31
    +    # check that exotically named packages are imported correctly
    
    32
    +    myreqs_packages = ['hellolib']
    
    33
    +    packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
    
    34
    +    setup_pypi_repo(myreqs_packages + packages, pypi_repo)
    
    35
    +
    
    26 36
         element = {
    
    27 37
             'kind': 'import',
    
    28 38
             'sources': [
    
    ... ... @@ -32,9 +42,9 @@ def test_pip_source_import(cli, tmpdir, datafiles):
    32 42
                 },
    
    33 43
                 {
    
    34 44
                     'kind': 'pip',
    
    35
    -                'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
    
    45
    +                'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
    
    36 46
                     'requirements-files': ['myreqs.txt'],
    
    37
    -                'packages': ['app2']
    
    47
    +                'packages': packages
    
    38 48
                 }
    
    39 49
             ]
    
    40 50
         }
    
    ... ... @@ -51,16 +61,31 @@ def test_pip_source_import(cli, tmpdir, datafiles):
    51 61
         assert result.exit_code == 0
    
    52 62
     
    
    53 63
         assert_contains(checkout, ['/.bst_pip_downloads',
    
    54
    -                               '/.bst_pip_downloads/HelloLib-0.1.tar.gz',
    
    55
    -                               '/.bst_pip_downloads/App2-0.1.tar.gz'])
    
    64
    +                               '/.bst_pip_downloads/hellolib-0.1.tar.gz',
    
    65
    +                               '/.bst_pip_downloads/app2-0.1.tar.gz',
    
    66
    +                               '/.bst_pip_downloads/app.3-0.1.tar.gz',
    
    67
    +                               '/.bst_pip_downloads/app-4-0.1.tar.gz',
    
    68
    +                               '/.bst_pip_downloads/app_5-0.1.tar.gz',
    
    69
    +                               '/.bst_pip_downloads/app.no.6-0.1.tar.gz',
    
    70
    +                               '/.bst_pip_downloads/app-no-7-0.1.tar.gz',
    
    71
    +                               '/.bst_pip_downloads/app_no_8-0.1.tar.gz'])
    
    56 72
     
    
    57 73
     
    
    58 74
     @pytest.mark.datafiles(DATA_DIR)
    
    59
    -def test_pip_source_build(cli, tmpdir, datafiles):
    
    75
    +def test_pip_source_build(cli, tmpdir, datafiles, setup_pypi_repo):
    
    60 76
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    61 77
         element_path = os.path.join(project, 'elements')
    
    62 78
         element_name = 'pip/hello.bst'
    
    63 79
     
    
    80
    +    # create mock pypi repository
    
    81
    +    pypi_repo = os.path.join(project, 'files', 'pypi-repo')
    
    82
    +    os.makedirs(pypi_repo, exist_ok=True)
    
    83
    +
    
    84
    +    # check that exotically named packages are imported correctly
    
    85
    +    myreqs_packages = ['hellolib']
    
    86
    +    packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
    
    87
    +    setup_pypi_repo(myreqs_packages + packages, pypi_repo)
    
    88
    +
    
    64 89
         element = {
    
    65 90
             'kind': 'manual',
    
    66 91
             'depends': ['base.bst'],
    
    ... ... @@ -71,16 +96,15 @@ def test_pip_source_build(cli, tmpdir, datafiles):
    71 96
                 },
    
    72 97
                 {
    
    73 98
                     'kind': 'pip',
    
    74
    -                'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
    
    99
    +                'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
    
    75 100
                     'requirements-files': ['myreqs.txt'],
    
    76
    -                'packages': ['app2']
    
    101
    +                'packages': packages
    
    77 102
                 }
    
    78 103
             ],
    
    79 104
             'config': {
    
    80 105
                 'install-commands': [
    
    81 106
                     'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz',
    
    82
    -                'chmod +x app1.py',
    
    83
    -                'install app1.py  %{install-root}/usr/bin/'
    
    107
    +                'install app1.py %{install-root}/usr/bin/'
    
    84 108
                 ]
    
    85 109
             }
    
    86 110
         }
    
    ... ... @@ -95,5 +119,4 @@ def test_pip_source_build(cli, tmpdir, datafiles):
    95 119
     
    
    96 120
         result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py'])
    
    97 121
         assert result.exit_code == 0
    
    98
    -    assert result.output == """Hello App1!
    
    99
    -"""
    122
    +    assert result.output == "Hello App1! This is hellolib\n"

  • tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz deleted
    No preview for this file type
  • tests/integration/project/files/pypi-repo/app2/index.html deleted
    1
    -<html>
    
    2
    -  <head>
    
    3
    -    <title>Links for app1</title>
    
    4
    -  </head>
    
    5
    -  <body>
    
    6
    -    <a href="">'App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
    
    7
    -  </body>
    
    8
    -</html>

  • tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz deleted
    No preview for this file type
  • tests/integration/project/files/pypi-repo/hellolib/index.html deleted
    1
    -<html>
    
    2
    -  <head>
    
    3
    -    <title>Links for app1</title>
    
    4
    -  </head>
    
    5
    -  <body>
    
    6
    -    <a href="">'HelloLib-0.1.tar.gz'>HelloLib-0.1.tar.gz</a><br />
    
    7
    -  </body>
    
    8
    -</html>

  • tests/sources/pip.py
    ... ... @@ -3,6 +3,7 @@ import pytest
    3 3
     
    
    4 4
     from buildstream._exceptions import ErrorDomain
    
    5 5
     from buildstream import _yaml
    
    6
    +from buildstream.plugins.sources.pip import _match_package_name
    
    6 7
     from tests.testutils import cli
    
    7 8
     
    
    8 9
     DATA_DIR = os.path.join(
    
    ... ... @@ -45,3 +46,22 @@ def test_no_packages(cli, tmpdir, datafiles):
    45 46
             'show', 'target.bst'
    
    46 47
         ])
    
    47 48
         result.assert_main_error(ErrorDomain.SOURCE, None)
    
    49
    +
    
    50
    +
    
    51
    +# Test that pip source parses tar ball names correctly for the ref
    
    52
    +@pytest.mark.parametrize(
    
    53
    +    'tarball, expected_name, expected_version',
    
    54
    +    [
    
    55
    +        ('dotted.package-0.9.8.tar.gz', 'dotted.package', '0.9.8'),
    
    56
    +        ('hyphenated-package-2.6.0.tar.gz', 'hyphenated-package', '2.6.0'),
    
    57
    +        ('underscore_pkg-3.1.0.tar.gz', 'underscore_pkg', '3.1.0'),
    
    58
    +        ('numbers2and5-1.0.1.tar.gz', 'numbers2and5', '1.0.1'),
    
    59
    +        ('multiple.dots.package-5.6.7.tar.gz', 'multiple.dots.package', '5.6.7'),
    
    60
    +        ('multiple-hyphens-package-1.2.3.tar.gz', 'multiple-hyphens-package', '1.2.3'),
    
    61
    +        ('multiple_underscore_pkg-3.4.5.tar.gz', 'multiple_underscore_pkg', '3.4.5'),
    
    62
    +        ('shortversion-1.0.tar.gz', 'shortversion', '1.0'),
    
    63
    +        ('longversion-1.2.3.4.tar.gz', 'longversion', '1.2.3.4')
    
    64
    +    ])
    
    65
    +def test_match_package_name(tarball, expected_name, expected_version):
    
    66
    +    name, version = _match_package_name(tarball)
    
    67
    +    assert (expected_name, expected_version) == (name, version)

  • tests/testutils/__init__.py
    ... ... @@ -29,3 +29,4 @@ from .artifactshare import create_artifact_share
    29 29
     from .element_generators import create_element_size, update_element_size
    
    30 30
     from .junction import generate_junction
    
    31 31
     from .runner_integration import wait_for_cache_granularity
    
    32
    +from .python_repo import setup_pypi_repo

  • tests/testutils/python_repo.py
    1
    +from setuptools.sandbox import run_setup
    
    2
    +import os
    
    3
    +import pytest
    
    4
    +import re
    
    5
    +import shutil
    
    6
    +
    
    7
    +
    
    8
    +SETUP_TEMPLATE = '''\
    
    9
    +from setuptools import setup
    
    10
    +
    
    11
    +setup(
    
    12
    +    name='{0}',
    
    13
    +    version='{1}',
    
    14
    +    description='{0}',
    
    15
    +    packages=['{2}'],
    
    16
    +    entry_points={{
    
    17
    +        'console_scripts': [
    
    18
    +            '{2}={2}:main'
    
    19
    +        ]
    
    20
    +    }}
    
    21
    +)
    
    22
    +'''
    
    23
    +
    
    24
    +# All packages generated via generate_pip_package will have the functions below
    
    25
    +INIT_TEMPLATE = '''\
    
    26
    +def main():
    
    27
    +    print('This is {0}')
    
    28
    +
    
    29
    +def hello(actor='world'):
    
    30
    +    print('Hello {{}}! This is {0}'.format(actor))
    
    31
    +'''
    
    32
    +
    
    33
    +HTML_TEMPLATE = '''\
    
    34
    +<html>
    
    35
    +  <head>
    
    36
    +    <title>Links for {0}</title>
    
    37
    +  </head>
    
    38
    +  <body>
    
    39
    +    <a href=''>{0}-{1}.tar.gz</a><br />
    
    40
    +  </body>
    
    41
    +</html>
    
    42
    +'''
    
    43
    +
    
    44
    +
    
    45
    +# Creates a simple python source distribution and copies this into a specified
    
    46
    +# directory which is to serve as a mock python repository
    
    47
    +#
    
    48
    +# Args:
    
    49
    +#    tmpdir (Path): Directory in which the source files will be created
    
    50
    +#    pypi (Path): Directory serving as a mock python repository
    
    51
    +#    name (str): The name of the package to be created
    
    52
    +#    version (str): The version of the package to be created
    
    53
    +#
    
    54
    +# Returns:
    
    55
    +#    None
    
    56
    +#
    
    57
    +def generate_pip_package(tmpdir, pypi, name, version='0.1'):
    
    58
    +    # check if package already exists in pypi
    
    59
    +    pypi_package = os.path.join(pypi, re.sub('[^0-9a-zA-Z]+', '-', name))
    
    60
    +    if os.path.exists(pypi_package):
    
    61
    +        return
    
    62
    +
    
    63
    +    # create the package source files in tmpdir resulting in a directory
    
    64
    +    # tree resembling the following structure:
    
    65
    +    #
    
    66
    +    # tmpdir
    
    67
    +    # |-- setup.py
    
    68
    +    # `-- package
    
    69
    +    #     `-- __init__.py
    
    70
    +    #
    
    71
    +    setup_file = os.path.join(tmpdir, 'setup.py')
    
    72
    +    pkgdirname = re.sub('[^0-9a-zA-Z]+', '', name)
    
    73
    +    with open(setup_file, 'w') as f:
    
    74
    +        f.write(SETUP_TEMPLATE.format(name, version, pkgdirname))
    
    75
    +    os.chmod(setup_file, 0o755)
    
    76
    +
    
    77
    +    package = os.path.join(tmpdir, pkgdirname)
    
    78
    +    os.makedirs(package)
    
    79
    +
    
    80
    +    main_file = os.path.join(package, '__init__.py')
    
    81
    +    with open(main_file, 'w') as f:
    
    82
    +        f.write(INIT_TEMPLATE.format(name))
    
    83
    +    os.chmod(main_file, 0o644)
    
    84
    +
    
    85
    +    run_setup(setup_file, ['sdist'])
    
    86
    +
    
    87
    +    # create directory for this package in pypi resulting in a directory
    
    88
    +    # tree resembling the following structure:
    
    89
    +    #
    
    90
    +    # pypi
    
    91
    +    # `-- pypi_package
    
    92
    +    #     |-- index.html
    
    93
    +    #     `-- foo-0.1.tar.gz
    
    94
    +    #
    
    95
    +    os.makedirs(pypi_package)
    
    96
    +
    
    97
    +    # add an index html page
    
    98
    +    index_html = os.path.join(pypi_package, 'index.html')
    
    99
    +    with open(index_html, 'w') as f:
    
    100
    +        f.write(HTML_TEMPLATE.format(name, version))
    
    101
    +
    
    102
    +    # copy generated tarfile to pypi package
    
    103
    +    dist_dir = os.path.join(tmpdir, 'dist')
    
    104
    +    for tar in os.listdir(dist_dir):
    
    105
    +        tarpath = os.path.join(dist_dir, tar)
    
    106
    +        shutil.copy(tarpath, pypi_package)
    
    107
    +
    
    108
    +
    
    109
    +@pytest.fixture
    
    110
    +def setup_pypi_repo(tmpdir):
    
    111
    +    def create_pkgdir(package):
    
    112
    +        pkgdirname = re.sub('[^0-9a-zA-Z]+', '', package)
    
    113
    +        pkgdir = os.path.join(str(tmpdir), pkgdirname)
    
    114
    +        os.makedirs(pkgdir)
    
    115
    +        return pkgdir
    
    116
    +
    
    117
    +    def add_packages(packages, pypi_repo):
    
    118
    +        for package in packages:
    
    119
    +            pkgdir = create_pkgdir(package)
    
    120
    +            generate_pip_package(pkgdir, pypi_repo, package)
    
    121
    +
    
    122
    +    return add_packages



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]