[Notes] [Git][BuildStream/buildstream][chandan/toxify] 7 commits: Add requirements files for install, test and plugin dependencies



Title: GitLab

Chandan Singh pushed to branch chandan/toxify at BuildStream / buildstream

Commits:

12 changed files:

Changes:

  • .gitignore
    ... ... @@ -17,6 +17,7 @@ tmp
    17 17
     .cache
    
    18 18
     .pytest_cache/
    
    19 19
     *.bst/
    
    20
    +.tox
    
    20 21
     
    
    21 22
     # Pycache, in case buildstream is ran directly from within the source
    
    22 23
     # tree
    

  • .gitlab-ci.yml
    ... ... @@ -13,7 +13,7 @@ stages:
    13 13
     variables:
    
    14 14
       PYTEST_ADDOPTS: "--color=yes"
    
    15 15
       INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
    
    16
    -  TEST_COMMAND: 'python3 setup.py test --index-url invalid://uri --addopts --integration'
    
    16
    +  TEST_COMMAND: 'tox -- --color=yes --integration'
    
    17 17
     
    
    18 18
     #####################################################
    
    19 19
     #                  Prepare stage                    #
    
    ... ... @@ -67,10 +67,10 @@ source_dist:
    67 67
       # Diagnostics
    
    68 68
       - mount
    
    69 69
       - df -h
    
    70
    -
    
    71
    -  # Unpack
    
    72
    -  - cd dist && ./unpack.sh
    
    73
    -  - cd buildstream
    
    70
    +  # FIXME: this deps should ideally be provided by the testsuite images
    
    71
    +  - pip3 install tox
    
    72
    +  - |
    
    73
    +    bash -c '( apt-get update && apt-get install -y python3-dev gcc libcairo2-dev pkg-config libgirepository1.0-dev ) || ( dnf install -y gcc python3-devel gobject-introspection-devel glib2-devel cairo-gobject-devel )'
    
    74 74
     
    
    75 75
       script:
    
    76 76
       - useradd -Um buildstream
    
    ... ... @@ -83,7 +83,7 @@ source_dist:
    83 83
       after_script:
    
    84 84
       # Collect our reports
    
    85 85
       - mkdir -p ${COVERAGE_DIR}
    
    86
    -  - cp dist/buildstream/.coverage ${COVERAGE_DIR}/coverage."${CI_JOB_NAME}"
    
    86
    +  - cp .coverage ${COVERAGE_DIR}/coverage."${CI_JOB_NAME}"
    
    87 87
       except:
    
    88 88
       - schedules
    
    89 89
       artifacts:
    
    ... ... @@ -158,20 +158,18 @@ tests-fedora-missing-deps:
    158 158
     
    
    159 159
     # Automatically build documentation for every commit, we want to know
    
    160 160
     # if building documentation fails even if we're not deploying it.
    
    161
    -# Note: We still do not enforce a consistent installation of python3-sphinx,
    
    162
    -#       as it will significantly grow the backing image.
    
    163 161
     docs:
    
    164 162
       stage: test
    
    163
    +  variables:
    
    164
    +    BST_FORCE_SESSION_REBUILD: 1
    
    165
    +  before_script:
    
    166
    +  # FIXME: this deps should ideally be provided by the testsuite images
    
    167
    +  - pip3 install tox
    
    168
    +  - |
    
    169
    +    bash -c '( apt-get update && apt-get install -y python3-dev gcc libcairo2-dev pkg-config libgirepository1.0-dev ) || ( dnf install -y gcc python3-devel gobject-introspection-devel glib2-devel cairo-gobject-devel )'
    
    165 170
       script:
    
    166
    -  - export BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources"
    
    167
    -  # Currently sphinx_rtd_theme does not support Sphinx >1.8, this breaks search functionality
    
    168
    -  - pip3 install sphinx==1.7.9
    
    169
    -  - pip3 install sphinx-click
    
    170
    -  - pip3 install sphinx_rtd_theme
    
    171
    -  - cd dist && ./unpack.sh && cd buildstream
    
    172
    -  - make BST_FORCE_SESSION_REBUILD=1 -C doc
    
    173
    -  - cd ../..
    
    174
    -  - mv dist/buildstream/doc/build/html public
    
    171
    +  - env BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources" tox -e docs
    
    172
    +  - mv doc/build/html public
    
    175 173
       except:
    
    176 174
       - schedules
    
    177 175
       artifacts:
    

  • CONTRIBUTING.rst
    ... ... @@ -1222,27 +1222,13 @@ For further information about using the reStructuredText with sphinx, please see
    1222 1222
     
    
    1223 1223
     Building Docs
    
    1224 1224
     ~~~~~~~~~~~~~
    
    1225
    -The documentation build is not integrated into the ``setup.py`` and is
    
    1226
    -difficult (or impossible) to do so, so there is a little bit of setup
    
    1227
    -you need to take care of first.
    
    1228
    -
    
    1229
    -Before you can build the BuildStream documentation yourself, you need
    
    1230
    -to first install ``sphinx`` along with some additional plugins and dependencies,
    
    1231
    -using pip or some other mechanism::
    
    1232
    -
    
    1233
    -  # Install sphinx
    
    1234
    -  pip3 install --user sphinx
    
    1235
    -
    
    1236
    -  # Install some sphinx extensions
    
    1237
    -  pip3 install --user sphinx-click
    
    1238
    -  pip3 install --user sphinx_rtd_theme
    
    1239
    -
    
    1240
    -  # Additional optional dependencies required
    
    1241
    -  pip3 install --user arpy
    
    1225
    +Before you can build the docs, you will end to ensure that you have installed
    
    1226
    +the required :ref:`buid dependencies <contributing_build_deps>` as mentioned
    
    1227
    +in the testing section above.
    
    1242 1228
     
    
    1243 1229
     To build the documentation, just run the following::
    
    1244 1230
     
    
    1245
    -  make -C doc
    
    1231
    +  tox -e docs
    
    1246 1232
     
    
    1247 1233
     This will give you a ``doc/build/html`` directory with the html docs which
    
    1248 1234
     you can view in your browser locally to test.
    
    ... ... @@ -1260,9 +1246,10 @@ will make the docs build reuse already downloaded sources::
    1260 1246
     
    
    1261 1247
       export BST_SOURCE_CACHE=~/.cache/buildstream/sources
    
    1262 1248
     
    
    1263
    -To force rebuild session html while building the doc, simply build the docs like this::
    
    1249
    +To force rebuild session html while building the doc, simply run `tox` with the
    
    1250
    +``BST_FORCE_SESSION_REBUILD`` environment variable set, like so::
    
    1264 1251
     
    
    1265
    -  make BST_FORCE_SESSION_REBUILD=1 -C doc
    
    1252
    +  env BST_FORCE_SESSION_REBUILD=1 tox -e docs
    
    1266 1253
     
    
    1267 1254
     
    
    1268 1255
     Man pages
    
    ... ... @@ -1468,58 +1455,115 @@ regenerate them locally in order to build the docs.
    1468 1455
     
    
    1469 1456
     Testing
    
    1470 1457
     -------
    
    1471
    -BuildStream uses pytest for regression tests and testing out
    
    1472
    -the behavior of newly added components.
    
    1458
    +BuildStream uses `tox <https://tox.readthedocs.org/>`_ as a frontend to run the
    
    1459
    +tests which are implemented using `pytest <https://pytest.org/>`_. We use
    
    1460
    +pytest for regression tests and testing out the behavior of newly added
    
    1461
    +components.
    
    1473 1462
     
    
    1474 1463
     The elaborate documentation for pytest can be found here: http://doc.pytest.org/en/latest/contents.html
    
    1475 1464
     
    
    1476 1465
     Don't get lost in the docs if you don't need to, follow existing examples instead.
    
    1477 1466
     
    
    1478 1467
     
    
    1468
    +.. _contributing_build_deps:
    
    1469
    +
    
    1470
    +Installing build dependencies
    
    1471
    +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    1472
    +Some of BuildStream's dependencies have non-python build dependencies. When
    
    1473
    +running tests with ``tox``, you will first need to install these dependencies.
    
    1474
    +Exact steps to install these will depend on your oprtation systemm. Commands
    
    1475
    +for installing them for some common distributions are lised below.
    
    1476
    +
    
    1477
    +For Fedora-based systems::
    
    1478
    +
    
    1479
    +  dnf install gcc pkg-config python3-devel cairo-gobject-devel glib2-devel gobject-introspection-devel
    
    1480
    +
    
    1481
    +
    
    1482
    +For Debian-based systems::
    
    1483
    +
    
    1484
    +  apt install gcc pkg-config python3-dev libcairo2-dev libgirepository1.0-dev
    
    1485
    +
    
    1486
    +
    
    1479 1487
     Running tests
    
    1480 1488
     ~~~~~~~~~~~~~
    
    1481
    -To run the tests, just type::
    
    1489
    +To run the tests, simply navigate to the toplevel directory of your BuildStream
    
    1490
    +checkout and run::
    
    1482 1491
     
    
    1483
    -  ./setup.py test
    
    1492
    +  tox
    
    1484 1493
     
    
    1485
    -At the toplevel.
    
    1494
    +By default, the test suite will be run against every supported python version
    
    1495
    +found on your host. If you have multiple python versions installed, you may
    
    1496
    +want to run tests against only one version and you can do that using the ``-e``
    
    1497
    +option when running tox::
    
    1486 1498
     
    
    1487
    -When debugging a test, it can be desirable to see the stdout
    
    1488
    -and stderr generated by a test, to do this use the ``--addopts``
    
    1489
    -function to feed arguments to pytest as such::
    
    1499
    +  tox -e py37
    
    1490 1500
     
    
    1491
    -  ./setup.py test --addopts -s
    
    1501
    +The output of all failing tests will always be printed in the summary, but
    
    1502
    +if you want to observe the stdout and stderr generated by a passing test,
    
    1503
    +you can pass the ``-s`` option to pytest as such::
    
    1504
    +
    
    1505
    +  tox -- -s
    
    1506
    +
    
    1507
    +.. tip::
    
    1508
    +
    
    1509
    +   The ``-s`` option is `a pytest option <https://docs.pytest.org/latest/usage.html>`_.
    
    1510
    +
    
    1511
    +   Any options specified before the ``--`` separator are consumed by ``tox``,
    
    1512
    +   and any options after the ``--`` separator will be passed along to pytest.
    
    1492 1513
     
    
    1493 1514
     You can always abort on the first failure by running::
    
    1494 1515
     
    
    1495
    -  ./setup.py test --addopts -x
    
    1516
    +  tox -- -x
    
    1496 1517
     
    
    1497 1518
     If you want to run a specific test or a group of tests, you
    
    1498 1519
     can specify a prefix to match. E.g. if you want to run all of
    
    1499 1520
     the frontend tests you can do::
    
    1500 1521
     
    
    1501
    -  ./setup.py test --addopts 'tests/frontend/'
    
    1522
    +  tox -- tests/frontend/
    
    1502 1523
     
    
    1503 1524
     Specific tests can be chosen by using the :: delimeter after the test module.
    
    1504 1525
     If you wanted to run the test_build_track test within frontend/buildtrack.py you could do::
    
    1505 1526
     
    
    1506
    -  ./setup.py test --addopts 'tests/frontend/buildtrack.py::test_build_track'
    
    1527
    +  tox -- tests/frontend/buildtrack.py::test_build_track
    
    1507 1528
     
    
    1508 1529
     We also have a set of slow integration tests that are disabled by
    
    1509 1530
     default - you will notice most of them marked with SKIP in the pytest
    
    1510 1531
     output. To run them, you can use::
    
    1511 1532
     
    
    1512
    -  ./setup.py test --addopts '--integration'
    
    1533
    +  tox -- --integration
    
    1513 1534
     
    
    1514 1535
     By default, buildstream also runs pylint on all files. Should you want
    
    1515 1536
     to run just pylint (these checks are a lot faster), you can do so
    
    1516 1537
     with::
    
    1517 1538
     
    
    1518
    -  ./setup.py test --addopts '-m pylint'
    
    1539
    +  tox -- -m pylint
    
    1519 1540
     
    
    1520 1541
     Alternatively, any IDE plugin that uses pytest should automatically
    
    1521 1542
     detect the ``.pylintrc`` in the project's root directory.
    
    1522 1543
     
    
    1544
    +.. note::
    
    1545
    +
    
    1546
    +   By default, we do not allow use of site packages in our ``tox``
    
    1547
    +   confguration to enable running the tests in an isolated environment.
    
    1548
    +   If you need to enable use of site packages for whatever reason, you can
    
    1549
    +   do so by passing the ``--sitepackages`` option to ``tox``. Also, you will
    
    1550
    +   not need to install any of the build dependencies mentioned above if you
    
    1551
    +   use this approach.
    
    1552
    +
    
    1553
    +.. note::
    
    1554
    +
    
    1555
    +   While using ``tox`` is practical for developers running tests in
    
    1556
    +   more predictable execution environments, it is still possible to
    
    1557
    +   execute the test suite against a specific installation environment
    
    1558
    +   using pytest directly::
    
    1559
    +
    
    1560
    +     ./setup.py test
    
    1561
    +
    
    1562
    +   Specific options can be passed to ``pytest`` using the ``--addopts``
    
    1563
    +   option::
    
    1564
    +
    
    1565
    +     ./setup.py test --addopts 'tests/frontend/buildtrack.py::test_build_track'
    
    1566
    +
    
    1523 1567
     
    
    1524 1568
     Adding tests
    
    1525 1569
     ~~~~~~~~~~~~
    

  • MANIFEST.in
    ... ... @@ -24,6 +24,7 @@ recursive-include doc/sessions *.run
    24 24
     # Tests
    
    25 25
     recursive-include tests *
    
    26 26
     include conftest.py
    
    27
    +include tox.ini
    
    27 28
     include .coveragerc
    
    28 29
     include .pylintrc
    
    29 30
     
    
    ... ... @@ -31,7 +32,12 @@ include .pylintrc
    31 32
     recursive-include buildstream/_protos *.proto
    
    32 33
     
    
    33 34
     # Requirements files
    
    34
    -include dev-requirements.txt
    
    35
    +include tools/requirements.in
    
    36
    +include tools/requirements.txt
    
    37
    +include tools/dev-requirements.in
    
    38
    +include tools/dev-requirements.txt
    
    39
    +include tools/plugin-requirements.in
    
    40
    +include tools/plugin-requirements.txt
    
    35 41
     
    
    36 42
     # Versioneer
    
    37 43
     include versioneer.py

  • setup.py
    ... ... @@ -270,9 +270,12 @@ def get_cmdclass():
    270 270
     #####################################################
    
    271 271
     #               Gather requirements                 #
    
    272 272
     #####################################################
    
    273
    -with open('dev-requirements.txt') as dev_reqs:
    
    273
    +with open('tools/dev-requirements.in') as dev_reqs:
    
    274 274
         dev_requires = dev_reqs.read().splitlines()
    
    275 275
     
    
    276
    +with open('tools/requirements.in') as install_reqs:
    
    277
    +    install_requires = install_reqs.read().splitlines()
    
    278
    +
    
    276 279
     #####################################################
    
    277 280
     #     Prepare package description from README       #
    
    278 281
     #####################################################
    
    ... ... @@ -334,23 +337,7 @@ setup(name='BuildStream',
    334 337
                   os.path.join('buildstream', 'data', 'bst')
    
    335 338
               ])
    
    336 339
           ],
    
    337
    -      install_requires=[
    
    338
    -          'setuptools',
    
    339
    -          'psutil',
    
    340
    -          # According to ruamel.yaml's PyPI page, we are suppose to use
    
    341
    -          # "<=0.15" in production until 0.15 becomes API stable.
    
    342
    -          # However we need ruamel.yaml 0.15.41 or greater for Python 3.7.
    
    343
    -          # We know that ruamel.yaml 0.15.52 breaks API in a way that
    
    344
    -          # is incompatible with BuildStream.
    
    345
    -          #
    
    346
    -          # See issues #571 and #790.
    
    347
    -          'ruamel.yaml >= 0.15.41, < 0.15.52',
    
    348
    -          'pluginbase',
    
    349
    -          'Click >= 7.0',
    
    350
    -          'jinja2 >= 2.10',
    
    351
    -          'protobuf >= 3.5',
    
    352
    -          'grpcio >= 1.10',
    
    353
    -      ],
    
    340
    +      install_requires=install_requires,
    
    354 341
           entry_points=bst_install_entry_points,
    
    355 342
           tests_require=dev_requires,
    
    356 343
           zip_safe=False)

  • dev-requirements.txttools/dev-requirements.in

  • tools/dev-requirements.txt
    1
    +coverage==4.4
    
    2
    +pylint==2.2.2
    
    3
    +pytest==4.0.2
    
    4
    +pytest-codestyle==1.4.0
    
    5
    +pytest-cov==2.6.0
    
    6
    +pytest-datafiles==2.0
    
    7
    +pytest-env==0.6.2
    
    8
    +pytest-pylint==0.13.0
    
    9
    +pytest-xdist==1.25.0
    
    10
    +pytest-timeout==1.3.3
    
    11
    +pyftpdlib==1.5.4
    
    12
    +## The following requirements were added by pip freeze:
    
    13
    +apipkg==1.5
    
    14
    +astroid==2.1.0
    
    15
    +atomicwrites==1.2.1
    
    16
    +attrs==18.2.0
    
    17
    +execnet==1.5.0
    
    18
    +isort==4.3.4
    
    19
    +lazy-object-proxy==1.3.1
    
    20
    +mccabe==0.6.1
    
    21
    +more-itertools==5.0.0
    
    22
    +pluggy==0.8.0
    
    23
    +py==1.7.0
    
    24
    +pycodestyle==2.4.0
    
    25
    +pytest-forked==0.2
    
    26
    +six==1.12.0
    
    27
    +typed-ast==1.1.1
    
    28
    +wrapt==1.10.11

  • tools/plugin-requirements.in
    1
    +arpy
    
    2
    +PyGObject

  • tools/plugin-requirements.txt
    1
    +arpy==1.1.1
    
    2
    +PyGObject==3.30.4
    
    3
    +## The following requirements were added by pip freeze:
    
    4
    +pycairo==1.18.0

  • tools/requirements.in
    1
    +Click
    
    2
    +grpcio >= 1.10
    
    3
    +jinja2 >= 2.10
    
    4
    +pluginbase
    
    5
    +protobuf >= 3.5
    
    6
    +psutil
    
    7
    +# According to ruamel.yaml's PyPI page, we are suppose to use
    
    8
    +# "<=0.15" in production until 0.15 becomes API stable.
    
    9
    +# However we need ruamel.yaml 0.15.41 or greater for Python 3.7.
    
    10
    +# We know that ruamel.yaml 0.15.52 breaks API in a way that
    
    11
    +# is incompatible with BuildStream.
    
    12
    +#
    
    13
    +# See issues #571 and #790.
    
    14
    +ruamel.yaml >= 0.15.41, < 0.15.52
    
    15
    +setuptools

  • tools/requirements.txt
    1
    +Click==7.0
    
    2
    +grpcio==1.17.1
    
    3
    +pluginbase==0.7
    
    4
    +protobuf==3.6.1
    
    5
    +psutil==5.4.8
    
    6
    +# According to ruamel.yaml's PyPI page, we are suppose to use
    
    7
    +# "<=0.15" in production until 0.15 becomes API stable.
    
    8
    +# However we need ruamel.yaml 0.15.41 or greater for Python 3.7.
    
    9
    +# We know that ruamel.yaml 0.15.52 breaks API in a way that
    
    10
    +# is incompatible with BuildStream.
    
    11
    +#
    
    12
    +# See issues #571 and #790.
    
    13
    +ruamel.yaml==0.15.51
    
    14
    +setuptools==39.0.1
    
    15
    +## The following requirements were added by pip freeze:
    
    16
    +Jinja2==2.10
    
    17
    +MarkupSafe==1.1.0
    
    18
    +six==1.12.0

  • tox.ini
    1
    +[tox]
    
    2
    +envlist = py35,py36,py37
    
    3
    +skip_missing_interpreters = true
    
    4
    +
    
    5
    +[testenv]
    
    6
    +commands = pytest {posargs}
    
    7
    +deps =
    
    8
    +    -rtools/requirements.txt
    
    9
    +    -rtools/dev-requirements.txt
    
    10
    +    -rtools/plugin-requirements.txt
    
    11
    +passenv =
    
    12
    +    BST_FORCE_BACKEND
    
    13
    +    GI_TYPELIB_PATH
    
    14
    +    INTEGRATION_CACHE
    
    15
    +
    
    16
    +[testenv:docs]
    
    17
    +commands =
    
    18
    +    make -C doc
    
    19
    +# Currently sphinx_rtd_theme does not support Sphinx >1.8, this breaks search functionality
    
    20
    +deps =
    
    21
    +    sphinx==1.7.9
    
    22
    +    sphinx-click
    
    23
    +    sphinx_rtd_theme
    
    24
    +    -rtools/requirements.txt
    
    25
    +    -rtools/plugin-requirements.txt
    
    26
    +passenv =
    
    27
    +    BST_FORCE_SESSION_REBUILD
    
    28
    +    BST_SOURCE_CACHE
    
    29
    +    HOME
    
    30
    +    LANG
    
    31
    +    LC_ALL
    
    32
    +whitelist_externals =
    
    33
    +    make



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]