[Notes] [Git][BuildGrid/buildgrid][mablanch/160-docker-compose-feed-bin] 4 commits: .gitlab-ci.yml: Fix and refactor the whole pipeline



Title: GitLab

Martin Blanchard pushed to branch mablanch/160-docker-compose-feed-bin at BuildGrid / buildgrid

Commits:

9 changed files:

Changes:

  • .coveragerc
    1 1
     [run]
    
    2 2
     concurrency = multiprocessing
    
    3 3
     data_file = .coverage
    
    4
    -include =
    
    5
    -  */buildgrid/*
    
    6
    -
    
    4
    +source =
    
    5
    +  buildgrid/*
    
    7 6
     omit =
    
    8
    -  # Omit profiling helper module
    
    9
    -  # Omit generated code
    
    10
    -  */buildgrid/google/*
    
    7
    +  buildgrid/_app/*
    
    8
    +  buildgrid/_protos/*
    
    11 9
       *_pb2.py
    
    12 10
       *_pb2_grpc.py
    
    13
    -  */.eggs/*
    
    14
    -  */_app/*
    
    15 11
     
    
    16 12
     [report]
    
    17 13
     show_missing = True
    
    18 14
     precision = 2
    19
    -
    
    20
    -[paths]
    
    21
    -source =
    
    22
    -    buildgrid/
    
    23
    -    */site-packages/buildgrid/
    
    24
    -    */buildgrid/buildgrid/

  • .gitlab-ci.yml
    1
    -# Use a debian image with python 3.5 (and pip3) pre-installed
    
    1
    +# Debian Stretch base image with python 3.5:
    
    2
    +#
    
    2 3
     image: python:3.5-stretch
    
    3 4
     
    
    4
    -variables:
    
    5
    -  BGD: bgd
    
    6
    -
    
    7 5
     stages:
    
    8 6
       - test
    
    9 7
       - post
    
    10 8
       - deploy
    
    11 9
     
    
    12 10
     
    
    13
    -before_script:
    
    14
    -  - python3 -m pip install --upgrade setuptools pip
    
    15
    -  - export PATH=~/.local/bin:${PATH}
    
    16
    -  - python3 -m pip install --user --editable ".[tests]"
    
    11
    +# Templates and global variables.
    
    12
    +#
    
    13
    +variables:
    
    14
    +  BGD: "${CI_PROJECT_DIR}/env/bin/bgd"
    
    15
    +  PYTHON: "${CI_PROJECT_DIR}/env/bin/python"
    
    16
    +  TOOLS: "${CI_PROJECT_DIR}/env/bin"
    
    17
    +
    
    18
    +.build-template:
    
    19
    +  before_script: &install-base
    
    20
    +    - apt-get update && apt-get install -y python3 python3-venv python3-pip graphviz
    
    21
    +    - python3 -m venv env  # Creates a virtual environment
    
    22
    +    - ${PYTHON} -m pip --cache-dir=.pip install --upgrade setuptools pip wheel
    
    23
    +    - ${PYTHON} -m pip --cache-dir=.pip install --editable ".[auth,docs,tests]"
    
    24
    +  cache:
    
    25
    +    paths:
    
    26
    +      - .pip
    
    27
    +
    
    17 28
     
    
    18
    -.tests-template: &linux-tests
    
    29
    +# Test stage, build and test the code.
    
    30
    +#
    
    31
    +unit-tests:
    
    19 32
       stage: test
    
    20
    -  variables:
    
    21
    -    PYTEST_ADDOPTS: "--color=yes"
    
    33
    +  extends: .build-template
    
    22 34
       script:
    
    23
    -    - python3 -m pip install --user --editable ".[auth]"
    
    24
    -    - python3 setup.py test
    
    35
    +    - ${PYTHON} setup.py test
    
    36
    +  after_script:
    
    25 37
         - mkdir -p coverage/
    
    26 38
         - cp .coverage coverage/coverage."${CI_JOB_NAME}"
    
    39
    +  variables:
    
    40
    +    PYTEST_ADDOPTS: "--color=yes"
    
    27 41
       artifacts:
    
    28 42
         paths:
    
    29
    -    - coverage/
    
    43
    +      - coverage/
    
    30 44
     
    
    31
    -.run-dummy-job-template: &dummy-job
    
    45
    +dummy-tests:
    
    32 46
       stage: test
    
    47
    +  extends: .build-template
    
    33 48
       script:
    
    34 49
         - ${BGD} server start data/config/default.conf &
    
    35
    -    - sleep 1 # Allow server to boot
    
    50
    +    - sleep 1  # Allows server to boot
    
    36 51
         - ${BGD} bot dummy &
    
    37 52
         - ${BGD} cas upload-dummy
    
    38 53
         - ${BGD} execute request-dummy --wait-for-completion
    
    39 54
     
    
    40
    -
    
    41
    -# Test stage, build and test the code.
    
    42
    -#
    
    43
    -tests-debian-stretch:
    
    44
    -  <<: *linux-tests
    
    45
    -
    
    46
    -run-dummy-job-debian:
    
    47
    -  <<: *dummy-job
    
    48
    -
    
    49
    -buildbox-e2e:
    
    50
    -  image: registry.gitlab.com/buildgrid/buildbox/buildbox-e2e:latest
    
    55
    +e2e-tests:
    
    51 56
       stage: test
    
    52
    -  before_script: []
    
    57
    +  image: registry.gitlab.com/buildgrid/buildbox/buildbox-e2e:latest
    
    53 58
       script:
    
    54 59
         - BUILDGRID_SOURCE_ROOT=`pwd` end-to-end-test.sh
    
    55 60
     
    
    61
    +
    
    56 62
     # Post-build stage, documentation, coverage report...
    
    57 63
     #
    
    58 64
     documentation:
    
    59 65
       stage: post
    
    66
    +  extends: .build-template
    
    60 67
       script:
    
    61
    -    - apt-get update && apt-get install -y graphviz
    
    62
    -    - python3 -m pip install --user --editable ".[docs]"
    
    63
    -    - make -C docs html
    
    68
    +    - PATH="${PATH}:${TOOLS}" make -C docs html
    
    69
    +  after_script:
    
    64 70
         - mkdir -p documentation/
    
    65 71
         - cp -a docs/build/html/. documentation/
    
    66 72
       artifacts:
    
    67 73
         paths:
    
    68
    -    - documentation/
    
    74
    +      - documentation/
    
    69 75
     
    
    70 76
     coverage:
    
    71 77
       stage: post
    
    78
    +  extends: .build-template
    
    72 79
       dependencies:
    
    73
    -  - tests-debian-stretch
    
    80
    +    - unit-tests
    
    74 81
       coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
    
    75 82
       script:
    
    76
    -    - python3 -m pip install --user --editable ".[tests]"
    
    77
    -    - cd coverage/
    
    78
    -    - ls -l .
    
    79
    -    - python3 -m coverage combine --rcfile=../.coveragerc --append coverage.*
    
    80
    -    - python3 -m coverage html --rcfile=../.coveragerc --directory .
    
    81
    -    - python3 -m coverage report --rcfile=../.coveragerc --show-missing
    
    82
    -    - python3 -m coverage erase --rcfile=../.coveragerc
    
    83
    +    - cd coverage/ && ls -l .
    
    84
    +    - ${PYTHON} -m coverage combine --rcfile=../.coveragerc --append coverage.*
    
    85
    +    - ${PYTHON} -m coverage html --rcfile=../.coveragerc --directory .
    
    86
    +    - ${PYTHON} -m coverage report --rcfile=../.coveragerc --show-missing
    
    83 87
       artifacts:
    
    84 88
         paths:
    
    85
    -    - coverage/
    
    89
    +      - coverage/
    
    86 90
     
    
    87 91
     # Deployement stage, only for merges which land on master branch.
    
    88 92
     #
    
    89 93
     pages:
    
    90 94
       stage: deploy
    
    91 95
       dependencies:
    
    92
    -  - coverage
    
    93
    -  - documentation
    
    96
    +    - coverage
    
    97
    +    - documentation
    
    94 98
       script:
    
    95
    -  - mkdir -p public/coverage/
    
    96
    -  - cp -a coverage/* public/coverage/
    
    97
    -  - ls -la public/coverage/
    
    98
    -  - cp -a documentation/* public/
    
    99
    -  - ls -la public/
    
    99
    +    - mkdir -p public/coverage/
    
    100
    +    - cp -a coverage/* public/coverage/
    
    101
    +    - ls -la public/coverage/
    
    102
    +    - cp -a documentation/* public/
    
    103
    +    - ls -la public/
    
    100 104
       artifacts:
    
    101 105
         paths:
    
    102
    -    - public/
    
    106
    +      - public/
    
    103 107
       only:
    
    104
    -  - master
    
    108
    +    - master
    
    105 109
     
    
    106
    -trigger-buildbox-e2e-rebuild:
    
    110
    +triggers:
    
    107 111
       stage: deploy
    
    108
    -  variables:
    
    109
    -    GIT_STRATEGY: none
    
    110
    -  script:
    
    112
    +  before_script:
    
    111 113
         - apt-get update && apt-get install -y curl
    
    114
    +  script:
    
    112 115
         - curl --request POST --form "token=$CI_JOB_TOKEN" --form ref=master https://gitlab.com/api/v4/projects/buildgrid%2Fbuildbox%2Fbuildbox-e2e/trigger/pipeline
    
    116
    +  variables:
    
    117
    +    GIT_STRATEGY: none
    
    113 118
       only:
    
    114 119
         - master

  • Dockerfile
    ... ... @@ -6,7 +6,15 @@
    6 6
     # Builds an image from local sources.
    
    7 7
     #
    
    8 8
     
    
    9
    -FROM python:3.5-stretch
    
    9
    +FROM debian:buster
    
    10
    +
    
    11
    +RUN  [ \
    
    12
    +"apt-get", "update" ]
    
    13
    +RUN  [ \
    
    14
    +"apt-get", "install", "-y", \
    
    15
    +"python3", "python3-venv", "python3-pip", \
    
    16
    +"bubblewrap", "fuse3" \
    
    17
    +]
    
    10 18
     
    
    11 19
     # Use /app as working directory:
    
    12 20
     WORKDIR /app
    
    ... ... @@ -42,6 +50,9 @@ RUN [ \
    42 50
     # Copy the repo. contents:
    
    43 51
     COPY . /app
    
    44 52
     
    
    53
    +# Add tools directory to the PATH:
    
    54
    +ENV PATH=$PATH:/app/tools
    
    55
    +
    
    45 56
     # Install BuildGrid:
    
    46 57
     RUN [ \
    
    47 58
     "/app/env/bin/python", "-m", "pip", \
    

  • data/bin/.gitignore
    1
    +buildbox
    
    2
    +buildbox-fuse
    
    3
    +buildbox-run-bubblewrap
    
    4
    +buildbox-run-hosttools
    
    5
    +buildbox-run-userchroot
    
    6
    +buildbox-worker

  • data/bin/README
    1
    +    ¡FOR LOCAL DEVELOPMENT ONLY!
    
    2
    +
    
    3
    +This directory is used by BuildGrid's main Docker Compose manifest: it gets
    
    4
    +bind mounted in worker bot containers and is included in PATH. Place here
    
    5
    +static binaries of the external tools needed by the bots. Typical use case is
    
    6
    +to copy here the `buildbox` binary used as a backend by the `buildbox` bot.
    
    7
    +
    
    8
    +    ¡FOR LOCAL DEVELOPMENT ONLY!

  • data/config/controller.conf
    ... ... @@ -6,7 +6,7 @@ server:
    6 6
     description: >
    
    7 7
       Docker Compose controller configuration:
    
    8 8
         - Unauthenticated plain HTTP at :50051
    
    9
    -    - Single instance: local
    
    9
    +    - Single instance: [unnamed]
    
    10 10
         - Expects a remote CAS at :50052
    
    11 11
         - Hosted services:
    
    12 12
            - ActionCache
    
    ... ... @@ -19,14 +19,14 @@ monitoring:
    19 19
       enabled: false
    
    20 20
     
    
    21 21
     instances:
    
    22
    -  - name: local
    
    22
    +  - name: ''
    
    23 23
         description: |
    
    24
    -      The unique 'local' instance.
    
    24
    +      The unique unnamed instance.
    
    25 25
     
    
    26 26
         storages:
    
    27 27
           - !remote-storage &data-store
    
    28 28
             url: http://storage:50052
    
    29
    -        instance-name: local
    
    29
    +        instance-name: ''
    
    30 30
     
    
    31 31
         services:
    
    32 32
           - !action-cache &build-cache
    

  • data/config/storage.conf
    ... ... @@ -6,7 +6,7 @@ server:
    6 6
     description: >
    
    7 7
       Docker Compose storage configuration:
    
    8 8
         - Unauthenticated plain HTTP at :50052
    
    9
    -    - Single instance: local
    
    9
    +    - Single instance: [unnamed]
    
    10 10
         - On-disk data stored in /var
    
    11 11
         - Hosted services:
    
    12 12
            - ContentAddressableStorage
    
    ... ... @@ -19,9 +19,9 @@ monitoring:
    19 19
       enabled: false
    
    20 20
     
    
    21 21
     instances:
    
    22
    -  - name: local
    
    22
    +  - name: ''
    
    23 23
         description: |
    
    24
    -      The unique 'local' instance.
    
    24
    +      The unique unnamed instance.
    
    25 25
     
    
    26 26
         storages:
    
    27 27
           - !disk-storage &data-store
    

  • docker-compose.yml
    ... ... @@ -58,10 +58,24 @@ services:
    58 58
       bots:  # To be scaled horizontaly
    
    59 59
         image: buildgrid:local
    
    60 60
         command: [
    
    61
    -      "bot", "--parent=local",
    
    61
    +      "bot", "--parent=",
    
    62 62
           "--remote=http://controller:50051",
    
    63 63
           "--remote-cas=http://storage:50052",
    
    64 64
           "host-tools"]
    
    65
    +    #   "bot", "--parent=",
    
    66
    +    #   "--remote=http://controller:50051",
    
    67
    +    #   "--remote-cas=http://storage:50052",
    
    68
    +    #   "buildbox",
    
    69
    +    #   "--local-cas", "/var/lib/buildgrid/cache",
    
    70
    +    #   "--fuse-dir", "/mnt"]
    
    71
    +    privileged: true
    
    72
    +    volumes:
    
    73
    +      - type: volume
    
    74
    +        source: cache
    
    75
    +        target: /var/lib/buildgrid/cache
    
    76
    +      - type: bind
    
    77
    +        source: ./data/bin
    
    78
    +        target: /app/tools
    
    65 79
         depends_on:
    
    66 80
           - controller
    
    67 81
         networks:
    
    ... ... @@ -72,4 +86,5 @@ networks:
    72 86
       host:
    
    73 87
     
    
    74 88
     volumes:
    
    89
    +  cache:
    
    75 90
       data:

  • docs/source/installation.rst
    ... ... @@ -168,7 +168,7 @@ Once completed, you can start a minimal grid by running:
    168 168
        - An execution and action-cache service available at
    
    169 169
          ``http://localhost:50051``.
    
    170 170
        - An CAS service available at ``http://localhost:50052``.
    
    171
    -   - A single ``local`` instance with one host-tools based worker bot attached.
    
    171
    +   - A single unnamed instance with one host-tools based worker bot attached.
    
    172 172
     
    
    173 173
     .. hint::
    
    174 174
     
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]