[Notes] [Git][BuildStream/buildstream][chandan/automate-pypi-release] 2

Chandan Singh pushed to branch chandan/automate-pypi-release at BuildStream / buildstream

Commits:

db537ab8

by Chandan Singh at 2018-11-28T16:46:00Z

.gitlab-ci.yml: Publish to PyPI when new tags are pushed

Add a new CI job to publish souce distribution archive to PyPi any time
new tags are pushed to the repository. Releases will be made on
test.pypi.org first.

Part of https://gitlab.com/BuildStream/buildstream/issues/587.

0a42d35b
by Chandan Singh at 2018-11-28T16:47:14Z
```
TEST: verify shell script
```

1 changed file:

.gitlab-ci.yml

Changes:

.gitlab-ci.yml

@@ -6,14 +6,16 @@ cache:
      - cache/
  stages:
 -  - prepare
 -  - test
    - post
  variables:
    PYTEST_ADDOPTS: "--color=yes"
    INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
    TEST_COMMAND: 'python3 setup.py test --index-url invalid://uri --addopts --integration'
 +  TEST_PYPI_UPLOAD_URL: "https://test.pypi.org/legacy/"
 +  TEST_PYPI_INDEX_URL: "https://test.pypi.org/simple/"
 +  PYPI_UPLOAD_URL: "https://upload.pypi.org/legacy/"
 +  CI_COMMIT_TAG: "1.2.3"
  #####################################################
  #                  Prepare stage                    #
@@ -51,275 +53,30 @@ source_dist:
      - dist/
 -#####################################################
 -#                    Test stage                     #
 -#####################################################
+-
 -# Run premerge commits
 -#
 -.tests-template: &tests
 -  stage: test
+-
 -  variables:
 -    COVERAGE_DIR: coverage-linux
+-
 -  before_script:
 -  # Diagnostics
 -  - mount
 -  - df -h
+-
 -  # Unpack
 -  - cd dist && ./unpack.sh
 -  - cd buildstream
+-
 -  script:
 -  - useradd -Um buildstream
 -  - chown -R buildstream:buildstream .
+-
 -  # Run the tests from the source distribution, We run as a simple
 -  # user to test for permission issues
 -  - su buildstream -c "${TEST_COMMAND}"
+-
 -  after_script:
 -  # Collect our reports
 -  - mkdir -p ${COVERAGE_DIR}
 -  - cp dist/buildstream/.coverage ${COVERAGE_DIR}/coverage."${CI_JOB_NAME}"
 -  except:
 -  - schedules
 -  artifacts:
 -    paths:
 -    - ${COVERAGE_DIR}
+-
 -tests-debian-9:
 -  image: buildstream/testsuite-debian:9-master-123-7ce6581b
 -  <<: *tests
+-
 -tests-fedora-27:
 -  image: buildstream/testsuite-fedora:27-master-123-7ce6581b
 -  <<: *tests
+-
 -tests-fedora-28:
 -  image: buildstream/testsuite-fedora:28-master-123-7ce6581b
 -  <<: *tests
+-
 -tests-ubuntu-18.04:
 -  image: buildstream/testsuite-ubuntu:18.04-master-123-7ce6581b
 -  <<: *tests
+-
 -overnight-fedora-28-aarch64:
 -  image: buildstream/testsuite-fedora:aarch64-28-master-123-7ce6581b
 -  tags:
 -    - aarch64
 -  <<: *tests
 -  # We need to override the exclusion from the template
 -  # in order to run on schedules
 -  except: []
 -  only:
 -  - schedules
+-
 -tests-unix:
 -  # Use fedora here, to a) run a test on fedora and b) ensure that we
 -  # can get rid of ostree - this is not possible with debian-8
 -  image: buildstream/testsuite-fedora:27-master-123-7ce6581b
 -  <<: *tests
 -  variables:
 -    BST_FORCE_BACKEND: "unix"
 -    COVERAGE_DIR: coverage-unix
+-
 -  script:
+-
 -    # We remove the Bubblewrap and OSTree packages here so that we catch any
 -    # codepaths that try to use them. Removing OSTree causes fuse-libs to
 -    # disappear unless we mark it as user-installed.
 -    - dnf mark install fuse-libs
 -    - dnf erase -y bubblewrap ostree
+-
 -    # Since the unix platform is required to run as root, no user change required
 -    - ${TEST_COMMAND}
+-
+-
 -tests-fedora-missing-deps:
 -  # Ensure that tests behave nicely while missing bwrap and ostree
 -  image: buildstream/testsuite-fedora:28-master-119-552f5fc6
 -  <<: *tests
+-
 -  script:
 -    # We remove the Bubblewrap and OSTree packages here so that we catch any
 -    # codepaths that try to use them. Removing OSTree causes fuse-libs to
 -    # disappear unless we mark it as user-installed.
 -    - dnf mark install fuse-libs
 -    - dnf erase -y bubblewrap ostree
+-
 -    - useradd -Um buildstream
 -    - chown -R buildstream:buildstream .
+-
 -    - ${TEST_COMMAND}
+-
+-
 -# Automatically build documentation for every commit, we want to know
 -# if building documentation fails even if we're not deploying it.
 -# Note: We still do not enforce a consistent installation of python3-sphinx,
 -#       as it will significantly grow the backing image.
 -docs:
 -  stage: test
 -  script:
 -  - export BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources"
 -  # Currently sphinx_rtd_theme does not support Sphinx >1.8, this breaks search functionality
 -  - pip3 install sphinx==1.7.9
 -  - pip3 install sphinx-click
 -  - pip3 install sphinx_rtd_theme
 -  - cd dist && ./unpack.sh && cd buildstream
 -  - make BST_FORCE_SESSION_REBUILD=1 -C doc
 -  - cd ../..
 -  - mv dist/buildstream/doc/build/html public
 -  except:
 -  - schedules
 -  artifacts:
 -    paths:
 -    - public/
+-
 -.overnight-tests: &overnight-tests-template
 -  stage: test
 -  variables:
 -    BST_EXT_URL: git+https://gitlab.com/BuildStream/bst-external.git
 -    BST_EXT_REF: 573843768f4d297f85dc3067465b3c7519a8dcc3 # 0.7.0
 -    FD_SDK_REF: 612f66e218445eee2b1a9d7dd27c9caba571612e # freedesktop-sdk-18.08.19-54-g612f66e2
 -  before_script:
 -  - |
 -    mkdir -p "${HOME}/.config"
 -    cat <<EOF >"${HOME}/.config/buildstream.conf"
 -    scheduler:
 -      fetchers: 2
 -    EOF
 -  - (cd dist && ./unpack.sh && cd buildstream && pip3 install .)
 -  - pip3 install --user -e ${BST_EXT_URL}@${BST_EXT_REF}#egg=bst_ext
 -  - git clone https://gitlab.com/freedesktop-sdk/freedesktop-sdk.git
 -  - git -C freedesktop-sdk checkout ${FD_SDK_REF}
 -  only:
 -  - schedules
+-
 -overnight-tests:
 -  <<: *overnight-tests-template
 -  script:
 -  - make -C freedesktop-sdk
 -  tags:
 -  - overnight-tests
+-
 -overnight-tests-no-cache:
 -  <<: *overnight-tests-template
 -  script:
 -  - sed -i '/artifacts:/,+1 d' freedesktop-sdk/bootstrap/project.conf
 -  - sed -i '/artifacts:/,+1 d' freedesktop-sdk/project.conf
 -  - make -C freedesktop-sdk
 -  tags:
 -  - overnight-tests
+-
 -# Check code quality with gitlab's built-in feature.
 +# Release to PyPI, only for tags.
+ #
 -code_quality:
 -  image: docker:stable
 -  stage: test
 -  variables:
 -    DOCKER_DRIVER: overlay2
 -  allow_failure: true
 -  services:
 -    - docker:stable-dind
 -  script:
 -    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
 -    - docker run
 -        --env SOURCE_CODE="$PWD"
 -        --volume "$PWD":/code
 -        --volume /var/run/docker.sock:/var/run/docker.sock
 -        "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
 -  except:
 -  - schedules
 -  artifacts:
 -    paths: [gl-code-quality-report.json]
+-
 -#####################################################
 -#                    Post stage                     #
 -#####################################################
+-
 -analysis:
 -  stage: post
 -  script:
 -  - |
 -    pip3 install radon
 -    mkdir analysis
+-
 -  - |
 -    echo "Calculating Maintainability Index"
 -    radon mi -s -j buildstream > analysis/mi.json
 -    radon mi -s buildstream
+-
 -  - |
 -    echo "Calculating Cyclomatic Complexity"
 -    radon cc -a -s -j buildstream > analysis/cc.json
 -    radon cc -a -s buildstream
+-
 -  - |
 -    echo "Calculating Raw Metrics"
 -    radon raw -s -j buildstream > analysis/raw.json
 -    radon raw -s buildstream
+-
 -  except:
 -  - schedules
 -  artifacts:
 -    paths:
 -    - analysis/
+-
 -# Collate coverage reports
 -#
 -coverage:
 -  stage: post
 -  coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
 -  script:
 -    - cd dist && ./unpack.sh && cd buildstream
 -    - pip3 install --no-index .
 -    - mkdir report
 -    - cd report
 -    - cp ../../../coverage-unix/coverage.* .
 -    - cp ../../../coverage-linux/coverage.* .
 -    - ls coverage.*
 -    - coverage combine --rcfile=../.coveragerc -a coverage.*
 -    - coverage report --rcfile=../.coveragerc -m
 -  dependencies:
 -  - tests-debian-9
 -  - tests-fedora-27
 -  - tests-fedora-28
 -  - tests-unix
 -  - source_dist
 -  except:
 -  - schedules
+-
 -# Deploy, only for merges which land on master branch.
 +# TODO: To begin with, we only upload to test.pypi. Later we should also push
 +# to real PyPI after verifying that it can be installed correctly.
+ #
 -pages:
 +pypi_release:
    stage: post
    dependencies:
    - source_dist
 -  - docs
 -  variables:
 -    ACME_DIR: public/.well-known/acme-challenge
    script:
 -  - mkdir -p ${ACME_DIR}
 -    # Required to finish the creation of the Let's Encrypt certificate,
 -    # which allows using https://docs.buildstream.build/ for accessing
 -    # the documentation.
 -  - echo ${ACME_CHALLENGE} > ${ACME_DIR}/$(echo ${ACME_CHALLENGE} | cut -c1-43)
 -  artifacts:
 -    paths:
 -    - public/
 -  only:
 -  #
 -  # FIXME:
 -  #
 -  # Ideally we want to publish to a different subdir of
 -  # pages depending on which stable branch we are building here,
 -  # not currently automatically supported but can be worked around.
 -  #
 -  # See https://gitlab.com/gitlab-org/gitlab-ce/issues/35141
 -  #
 -  - master
 -  except:
 -  - schedules
 +  - |
 +    if [[ ! "$CI_COMMIT_TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
 +        echo "Not processing non-numeric tag: $CI_COMMIT_TAG for PyPI"
 +    fi
 +    minor_version="$(echo "$CI_COMMIT_TAG" | cut -d. -f2)"
 +    if [[ "$(( minor_version %2 ))" -ne 0 ]]; then
 +        echo "Not uploading development release: $CI_COMMIT_TAG to PyPI"
 +    fi
++
 +    # Credentials for PyPI are defined in secret CI variables "TWINE_USERNAME"
 +    # and "TWINE_PASSWORD".
 +    echo pip3 install twine
 +    echo twine upload --repository-url "$TEST_PYPI_UPLOAD_URL" dist/*.tar.gz
 +    echo pip3 install --extra-index-url "$TEST_PYPI_INDEX_URL" "BuildStream==$CI_COMMIT_TAG"
 +    echo bst --version
 +  # only:
 +  # - tags

[Notes] [Git][BuildStream/buildstream][chandan/automate-pypi-release] 2 commits: .gitlab-ci.yml: Publish to PyPI when new tags are pushed

Chandan Singh pushed to branch chandan/automate-pypi-release at BuildStream / buildstream

Commits:

1 changed file:

Changes: