GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][mablanch/160-docker-compose-feed-bin] 4 commits: .gitlab-ci.yml: Fix and refactor the whole pipeline

Title: GitLab

Martin Blanchard pushed to branch mablanch/160-docker-compose-feed-bin at BuildGrid / buildgrid

Commits:

9 changed files:

Changes:

  • .coveragerc
    1 1 
     [run]
    2 2 
     concurrency = multiprocessing
    3 3 
     data_file = .coverage
    4 
    -include =
    5 
    -  */buildgrid/*
    6 
    -
    4 
    +source =
    5 
    +  buildgrid/*
    7 6 
     omit =
    8 
    -  # Omit profiling helper module
    9 
    -  # Omit generated code
    10 
    -  */buildgrid/google/*
    7 
    +  buildgrid/_app/*
    8 
    +  buildgrid/_protos/*
    11 9 
       *_pb2.py
    12 10 
       *_pb2_grpc.py
    13 
    -  */.eggs/*
    14 
    -  */_app/*
    15 11
    16 12 
     [report]
    17 13 
     show_missing = True
    18 14 
     precision = 2
    19 
    -
    20 
    -[paths]
    21 
    -source =
    22 
    -    buildgrid/
    23 
    -    */site-packages/buildgrid/
    24 
    -    */buildgrid/buildgrid/

  • .gitlab-ci.yml
    1 
    -# Use a debian image with python 3.5 (and pip3) pre-installed
    1 
    +# Debian Stretch base image with python 3.5:
    2 
    +#
    2 3 
     image: python:3.5-stretch
    3 4
    4 
    -variables:
    5 
    -  BGD: bgd
    6 
    -
    7 5 
     stages:
    8 6 
       - test
    9 7 
       - post
    10 8 
       - deploy
    11 9
    12 10
    13 
    -before_script:
    14 
    -  - python3 -m pip install --upgrade setuptools pip
    15 
    -  - export PATH=~/.local/bin:${PATH}
    16 
    -  - python3 -m pip install --user --editable ".[tests]"
    11 
    +# Templates and global variables.
    12 
    +#
    13 
    +variables:
    14 
    +  BGD: "${CI_PROJECT_DIR}/env/bin/bgd"
    15 
    +  PYTHON: "${CI_PROJECT_DIR}/env/bin/python"
    16 
    +  TOOLS: "${CI_PROJECT_DIR}/env/bin"
    17 
    +
    18 
    +.build-template:
    19 
    +  before_script: &install-base
    20 
    +    - apt-get update && apt-get install -y python3 python3-venv python3-pip graphviz
    21 
    +    - python3 -m venv env  # Creates a virtual environment
    22 
    +    - ${PYTHON} -m pip --cache-dir=.pip install --upgrade setuptools pip wheel
    23 
    +    - ${PYTHON} -m pip --cache-dir=.pip install --editable ".[auth,docs,tests]"
    24 
    +  cache:
    25 
    +    paths:
    26 
    +      - .pip
    27 
    +
    17 28
    18 
    -.tests-template: &linux-tests
    29 
    +# Test stage, build and test the code.
    30 
    +#
    31 
    +unit-tests:
    19 32 
       stage: test
    20 
    -  variables:
    21 
    -    PYTEST_ADDOPTS: "--color=yes"
    33 
    +  extends: .build-template
    22 34 
       script:
    23 
    -    - python3 -m pip install --user --editable ".[auth]"
    24 
    -    - python3 setup.py test
    35 
    +    - ${PYTHON} setup.py test
    36 
    +  after_script:
    25 37 
         - mkdir -p coverage/
    26 38 
         - cp .coverage coverage/coverage."${CI_JOB_NAME}"
    39 
    +  variables:
    40 
    +    PYTEST_ADDOPTS: "--color=yes"
    27 41 
       artifacts:
    28 42 
         paths:
    29 
    -    - coverage/
    43 
    +      - coverage/
    30 44
    31 
    -.run-dummy-job-template: &dummy-job
    45 
    +dummy-tests:
    32 46 
       stage: test
    47 
    +  extends: .build-template
    33 48 
       script:
    34 49 
         - ${BGD} server start data/config/default.conf &
    35 
    -    - sleep 1 # Allow server to boot
    50 
    +    - sleep 1  # Allows server to boot
    36 51 
         - ${BGD} bot dummy &
    37 52 
         - ${BGD} cas upload-dummy
    38 53 
         - ${BGD} execute request-dummy --wait-for-completion
    39 54
    40 
    -
    41 
    -# Test stage, build and test the code.
    42 
    -#
    43 
    -tests-debian-stretch:
    44 
    -  <<: *linux-tests
    45 
    -
    46 
    -run-dummy-job-debian:
    47 
    -  <<: *dummy-job
    48 
    -
    49 
    -buildbox-e2e:
    50 
    -  image: registry.gitlab.com/buildgrid/buildbox/buildbox-e2e:latest
    55 
    +e2e-tests:
    51 56 
       stage: test
    52 
    -  before_script: []
    57 
    +  image: registry.gitlab.com/buildgrid/buildbox/buildbox-e2e:latest
    53 58 
       script:
    54 59 
         - BUILDGRID_SOURCE_ROOT=`pwd` end-to-end-test.sh
    55 60
    61 
    +
    56 62 
     # Post-build stage, documentation, coverage report...
    57 63 
     #
    58 64 
     documentation:
    59 65 
       stage: post
    66 
    +  extends: .build-template
    60 67 
       script:
    61 
    -    - apt-get update && apt-get install -y graphviz
    62 
    -    - python3 -m pip install --user --editable ".[docs]"
    63 
    -    - make -C docs html
    68 
    +    - PATH="${PATH}:${TOOLS}" make -C docs html
    69 
    +  after_script:
    64 70 
         - mkdir -p documentation/
    65 71 
         - cp -a docs/build/html/. documentation/
    66 72 
       artifacts:
    67 73 
         paths:
    68 
    -    - documentation/
    74 
    +      - documentation/
    69 75
    70 76 
     coverage:
    71 77 
       stage: post
    78 
    +  extends: .build-template
    72 79 
       dependencies:
    73 
    -  - tests-debian-stretch
    80 
    +    - unit-tests
    74 81 
       coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
    75 82 
       script:
    76 
    -    - python3 -m pip install --user --editable ".[tests]"
    77 
    -    - cd coverage/
    78 
    -    - ls -l .
    79 
    -    - python3 -m coverage combine --rcfile=../.coveragerc --append coverage.*
    80 
    -    - python3 -m coverage html --rcfile=../.coveragerc --directory .
    81 
    -    - python3 -m coverage report --rcfile=../.coveragerc --show-missing
    82 
    -    - python3 -m coverage erase --rcfile=../.coveragerc
    83 
    +    - cd coverage/ && ls -l .
    84 
    +    - ${PYTHON} -m coverage combine --rcfile=../.coveragerc --append coverage.*
    85 
    +    - ${PYTHON} -m coverage html --rcfile=../.coveragerc --directory .
    86 
    +    - ${PYTHON} -m coverage report --rcfile=../.coveragerc --show-missing
    83 87 
       artifacts:
    84 88 
         paths:
    85 
    -    - coverage/
    89 
    +      - coverage/
    86 90
    87 91 
     # Deployement stage, only for merges which land on master branch.
    88 92 
     #
    89 93 
     pages:
    90 94 
       stage: deploy
    91 95 
       dependencies:
    92 
    -  - coverage
    93 
    -  - documentation
    96 
    +    - coverage
    97 
    +    - documentation
    94 98 
       script:
    95 
    -  - mkdir -p public/coverage/
    96 
    -  - cp -a coverage/* public/coverage/
    97 
    -  - ls -la public/coverage/
    98 
    -  - cp -a documentation/* public/
    99 
    -  - ls -la public/
    99 
    +    - mkdir -p public/coverage/
    100 
    +    - cp -a coverage/* public/coverage/
    101 
    +    - ls -la public/coverage/
    102 
    +    - cp -a documentation/* public/
    103 
    +    - ls -la public/
    100 104 
       artifacts:
    101 105 
         paths:
    102 
    -    - public/
    106 
    +      - public/
    103 107 
       only:
    104 
    -  - master
    108 
    +    - master
    105 109
    106 
    -trigger-buildbox-e2e-rebuild:
    110 
    +triggers:
    107 111 
       stage: deploy
    108 
    -  variables:
    109 
    -    GIT_STRATEGY: none
    110 
    -  script:
    112 
    +  before_script:
    111 113 
         - apt-get update && apt-get install -y curl
    114 
    +  script:
    112 115 
         - curl --request POST --form "token=$CI_JOB_TOKEN" --form ref=master https://gitlab.com/api/v4/projects/buildgrid%2Fbuildbox%2Fbuildbox-e2e/trigger/pipeline
    116 
    +  variables:
    117 
    +    GIT_STRATEGY: none
    113 118 
       only:
    114 119 
         - master

  • Dockerfile
    ... ... @@ -6,7 +6,15 @@
    6 6 
     # Builds an image from local sources.
    7 7 
     #
    8 8
    9 
    -FROM python:3.5-stretch
    9 
    +FROM debian:buster
    10 
    +
    11 
    +RUN  [ \
    12 
    +"apt-get", "update" ]
    13 
    +RUN  [ \
    14 
    +"apt-get", "install", "-y", \
    15 
    +"python3", "python3-venv", "python3-pip", \
    16 
    +"bubblewrap", "fuse3" \
    17 
    +]
    10 18
    11 19 
     # Use /app as working directory:
    12 20 
     WORKDIR /app
    ... ... @@ -42,6 +50,9 @@ RUN [ \
    42 50 
     # Copy the repo. contents:
    43 51 
     COPY . /app
    44 52
    53 
    +# Add tools directory to the PATH:
    54 
    +ENV PATH=$PATH:/app/tools
    55 
    +
    45 56 
     # Install BuildGrid:
    46 57 
     RUN [ \
    47 58 
     "/app/env/bin/python", "-m", "pip", \

  • data/bin/.gitignore
    1 
    +buildbox
    2 
    +buildbox-fuse
    3 
    +buildbox-run-bubblewrap
    4 
    +buildbox-run-hosttools
    5 
    +buildbox-run-userchroot
    6 
    +buildbox-worker

  • data/bin/README
    1 
    +    ¡FOR LOCAL DEVELOPMENT ONLY!
    2 
    +
    3 
    +This directory is used by BuildGrid's main Docker Compose manifest: it gets
    4 
    +bind mounted in worker bot containers and is included in PATH. Place here
    5 
    +static binaries of the external tools needed by the bots. Typical use case is
    6 
    +to copy here the `buildbox` binary used as a backend by the `buildbox` bot.
    7 
    +
    8 
    +    ¡FOR LOCAL DEVELOPMENT ONLY!

  • data/config/controller.conf
    ... ... @@ -6,7 +6,7 @@ server:
    6 6 
     description: >
    7 7 
       Docker Compose controller configuration:
    8 8 
         - Unauthenticated plain HTTP at :50051
    9 
    -    - Single instance: local
    9 
    +    - Single instance: [unnamed]
    10 10 
         - Expects a remote CAS at :50052
    11 11 
         - Hosted services:
    12 12 
            - ActionCache
    ... ... @@ -19,14 +19,14 @@ monitoring:
    19 19 
       enabled: false
    20 20
    21 21 
     instances:
    22 
    -  - name: local
    22 
    +  - name: ''
    23 23 
         description: |
    24 
    -      The unique 'local' instance.
    24 
    +      The unique unnamed instance.
    25 25
    26 26 
         storages:
    27 27 
           - !remote-storage &data-store
    28 28 
             url: http://storage:50052
    29 
    -        instance-name: local
    29 
    +        instance-name: ''
    30 30
    31 31 
         services:
    32 32 
           - !action-cache &build-cache

  • data/config/storage.conf
    ... ... @@ -6,7 +6,7 @@ server:
    6 6 
     description: >
    7 7 
       Docker Compose storage configuration:
    8 8 
         - Unauthenticated plain HTTP at :50052
    9 
    -    - Single instance: local
    9 
    +    - Single instance: [unnamed]
    10 10 
         - On-disk data stored in /var
    11 11 
         - Hosted services:
    12 12 
            - ContentAddressableStorage
    ... ... @@ -19,9 +19,9 @@ monitoring:
    19 19 
       enabled: false
    20 20
    21 21 
     instances:
    22 
    -  - name: local
    22 
    +  - name: ''
    23 23 
         description: |
    24 
    -      The unique 'local' instance.
    24 
    +      The unique unnamed instance.
    25 25
    26 26 
         storages:
    27 27 
           - !disk-storage &data-store

  • docker-compose.yml
    ... ... @@ -58,10 +58,24 @@ services:
    58 58 
       bots:  # To be scaled horizontaly
    59 59 
         image: buildgrid:local
    60 60 
         command: [
    61 
    -      "bot", "--parent=local",
    61 
    +      "bot", "--parent=",
    62 62 
           "--remote=http://controller:50051",
    63 63 
           "--remote-cas=http://storage:50052",
    64 64 
           "host-tools"]
    65 
    +    #   "bot", "--parent=",
    66 
    +    #   "--remote=http://controller:50051",
    67 
    +    #   "--remote-cas=http://storage:50052",
    68 
    +    #   "buildbox",
    69 
    +    #   "--local-cas", "/var/lib/buildgrid/cache",
    70 
    +    #   "--fuse-dir", "/mnt"]
    71 
    +    privileged: true
    72 
    +    volumes:
    73 
    +      - type: volume
    74 
    +        source: cache
    75 
    +        target: /var/lib/buildgrid/cache
    76 
    +      - type: bind
    77 
    +        source: ./data/bin
    78 
    +        target: /app/tools
    65 79 
         depends_on:
    66 80 
           - controller
    67 81 
         networks:
    ... ... @@ -72,4 +86,5 @@ networks:
    72 86 
       host:
    73 87
    74 88 
     volumes:
    89 
    +  cache:
    75 90 
       data:

  • docs/source/installation.rst
    ... ... @@ -168,7 +168,7 @@ Once completed, you can start a minimal grid by running:
    168 168 
        - An execution and action-cache service available at
    169 169 
          ``http://localhost:50051``.
    170 170 
        - An CAS service available at ``http://localhost:50052``.
    171 
    -   - A single ``local`` instance with one host-tools based worker bot attached.
    171 
    +   - A single unnamed instance with one host-tools based worker bot attached.
    172 172
    173 173 
     .. hint::
    174 174

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]