GNOME.org
 

[gnome-build-meta/abderrahim/child-pipelines] .gitlab-ci.yml: use child pipelines for deploying to flat-manager


commit cd552cac38000a36618909af6e2ab24dca8ab2b1
Author: Abderrahim Kitouni <akitouni gnome org>
Date:   Wed Aug 5 11:41:35 2020 +0100

    .gitlab-ci.yml: use child pipelines for deploying to flat-manager

 .gitlab-ci.yml              | 281 ++++++++++++++++----------------------------
 .gitlab-ci/flat-manager.yml |  70 +++++++++++
 2 files changed, 173 insertions(+), 178 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5b8ffaf4..8484fad9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,9 @@ variables:
 stages:
 - track
 - build
+- deploy-config
 - image
+- deploy
 - prepare_flatpak
 - flatpak
 - finish_flatpak
@@ -48,35 +50,6 @@ default:
     sourcedir: /cache/buildstream/sources
     EOF
 
-  - |
-    case "$FLATPAK_BRANCH" in
-      master)
-        if [ -n "$NIGHTLY_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$NIGHTLY_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://nightly.gnome.org/
-          export FLAT_MANAGER_REPO=nightly
-        fi
-      ;;
-      *beta)
-        if [ -n "$FLATHUB_BETA_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$FLATHUB_BETA_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://hub.flathub.org/
-          export FLAT_MANAGER_REPO=beta
-        fi
-      ;;
-      *)
-        if [ -n "$FLATHUB_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$FLATHUB_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://hub.flathub.org/
-          export FLAT_MANAGER_REPO=stable
-        fi
-      ;;
-    esac
-
-  - |
-    export OSTREE_FLAT_MANAGER_SERVER=https://flatmgr-ostree.openshift.gnome.org/
-    export OSTREE_FLAT_MANAGER_REPO=gnomeos
-
 #
 # Templates shared by multiple jobs
 #
@@ -127,62 +100,12 @@ default:
     - image
     expire_in: '2 days'
 
-.flatpak-template:
-  stage: flatpak
-  dependencies: [track, flatpak-prepare]
-  script:
-  - ostree init --repo repo/ --mode archive
-
-  - ${BST} -o arch "${ARCH}" pull flatpak-runtimes.bst
-  - ${BST} -o arch "${ARCH}" checkout --hardlinks flatpak-runtimes.bst checkout-repo/
-  - flatpak build-commit-from --src-repo=checkout-repo/ repo/
-  - rm -rf checkout-repo/
-
-  - |
-    if test $FLAT_MANAGER_REPO == nightly; then
-        ${BST} -o arch "${ARCH}" pull flatpak-platform-extensions.bst
-        ${BST} -o arch "${ARCH}" checkout --hardlinks flatpak-platform-extensions.bst checkout-repo/
-        flatpak build-commit-from --src-repo=checkout-repo/ repo/
-        rm -rf checkout-repo/
-    fi
-
-  - |
-    if test $FLAT_MANAGER_REPO == nightly -a $ARCH == x86_64; then
-        ${BST} pull openh264-extension.bst:flatpak-repo.bst
-        ${BST} checkout --hardlinks openh264-extension.bst:flatpak-repo.bst checkout-repo/
-        flatpak build-commit-from --src-repo=checkout-repo/ repo/
-        rm -rf checkout-repo/
-    fi
-
-  - flatpak build-update-repo --generate-static-deltas repo/
-
-  - flat-manager-client push $(cat build.txt) repo/
-  rules: &flatpak-publish-rules
+.deploy-rules:
+  rules:
   - if: $EMERGENCY || $BST_TRACK_TAGS != "False"
     when: never
   - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^gnome-\d-\d\d$/
 
-.ostree-template:
-  stage: ostree
-  dependencies: [track, ostree-prepare]
-  script:
-  - export REPO_TOKEN=$OSTREE_REPO_TOKEN
-
-  - ostree init --repo repo/ --mode archive
-
-  - ${BST_NO_PUSH} -o arch "${ARCH}" build vm/repo.bst vm/repo-devel.bst
-
-  - ${BST_NO_PUSH} -o arch "${ARCH}" checkout --hardlinks vm/repo.bst checkout-repo/
-  - ostree pull-local --repo=repo/ checkout-repo/
-  - rm -rf checkout-repo/
-
-  - ${BST_NO_PUSH} -o arch "${ARCH}" checkout --hardlinks vm/repo-devel.bst checkout-devel-repo/
-  - ostree pull-local --repo=repo/ checkout-devel-repo/
-  - rm -rf checkout-devel-repo/
-
-  - flat-manager-client push $(cat build.txt) repo/ $(ostree --repo=repo/ refs)
-  rules: *flatpak-publish-rules
-
 
 #
 # "Real" CI jobs
@@ -195,6 +118,7 @@ track:
   script:
   - 'sed -i "s/track-tags: False/track-tags: ${BST_TRACK_TAGS}/" project.conf'
   - ${BST} track --deps all core.bst flatpak-runtimes.bst vm/image.bst boards/pinebook-pro/image.bst 
boards/rock64/image.bst boards/raspberrypi-4/image.bst vm/repo-devel.bst
+  - echo $CI_JOB_ID > track-job.id
   retry: 2
   # only run on branches targeting master
   rules:
@@ -202,6 +126,7 @@ track:
   artifacts:
     paths:
     - project.refs
+    - track-job.id
 
 
 build-gnome-core-x86_64:
@@ -230,6 +155,103 @@ build-gnome-core-ppc64le:
   - .ppc64le
 
 
+generate-deploy-config:
+  extends:
+  - .x86_64
+  - .deploy-rules
+  dependencies: [track]
+  stage: deploy-config
+  script:
+  - TRACK_JOB_ID=$(cat track-job.id)
+  - |
+    case "$FLATPAK_BRANCH" in
+      master)
+        FLAT_MANAGER_SERVER=https://nightly.gnome.org/
+        FLAT_MANAGER_REPO=nightly
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm ppc64le"
+      ;;
+      *beta)
+        FLAT_MANAGER_SERVER=https://hub.flathub.org/
+        FLAT_MANAGER_REPO=beta
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm"
+      ;;
+      *)
+        FLAT_MANAGER_SERVER=https://hub.flathub.org/
+        FLAT_MANAGER_REPO=stable
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm"
+      ;;
+    esac
+
+  - |
+    cat > flatpak-variables.yml << END
+    variables:
+      BST: '$BST'
+      TRACK_JOB_ID: '$TRACK_JOB_ID'
+      FLAT_MANAGER_SERVER: '$FLAT_MANAGER_SERVER'
+      FLAT_MANAGER_REPO: '$FLAT_MANAGER_REPO'
+      TARGETS: flatpak-runtimes.bst
+      TARGETS_nightly: flatpak-platform-extensions.bst
+      TARGETS_nightly_x86_64: openh264-extension.bst:flatpak-repo.bst
+      ENVIRONMENT_NAME: flatpak/$FLATPAK_BRANCH
+
+    END
+  - cat flatpak-variables.yml .gitlab-ci/flat-manager.yml > flatpak-deploy.yml
+
+  - |
+    for ARCH in $SUPPORTED_ARCHES; do
+    cat >> flatpak-deploy.yml << END
+
+    flatmgr-$ARCH:
+      extends:
+      - .flatmgr-template
+      - .$ARCH
+    END
+    done
+
+  - |
+    cat > ostree-variables.yml << END
+    variables:
+      BST: '$BST'
+      TRACK_JOB_ID: '$TRACK_JOB_ID'
+      FLAT_MANAGER_SERVER: 'https://flatmgr-ostree.openshift.gnome.org/'
+      FLAT_MANAGER_REPO: 'gnomeos'
+      BUILD_TARGETS: vm/repo.bst vm/repo-devel.bst
+      TARGETS: \$BUILD_TARGETS
+      ENVIRONMENT_NAME: gnomeos/master
+
+    END
+  - cat ostree-variables.yml .gitlab-ci/flat-manager.yml > ostree-deploy.yml
+  - |
+    cat >> ostree-deploy.yml << END
+
+    flatmgr-x86_64:
+      extends:
+      - .flatmgr-template
+      - .x86_64
+    END
+  artifacts:
+    paths:
+    - flatpak-deploy.yml
+    - ostree-deploy.yml
+
+deploy-flatpak:
+  extends: .deploy-rules
+  stage: deploy
+  trigger:
+    strategy: depend
+    include:
+    - artifact: flatpak-deploy.yml
+      job: generate-deploy-config
+
+deploy-ostree:
+  extends: .deploy-rules
+  stage: deploy
+  trigger:
+    strategy: depend
+    include:
+    - artifact: ostree-deploy.yml
+      job: generate-deploy-config
+
 vm-image-x86_64:
   extends:
   - .vm-image-template
@@ -298,103 +320,6 @@ raspberrypi-4-image:
     artifacts: false
 
 
-flatpak-prepare:
-  extends: .x86_64
-  stage: prepare_flatpak
-  dependencies: []
-  script:
-  - flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO > build.txt
-  artifacts:
-    paths:
-    - build.txt
-  rules: *flatpak-publish-rules
-
-ostree-prepare:
-  extends: .x86_64
-  stage: prepare_ostree
-  dependencies: []
-  script:
-  - export REPO_TOKEN=$OSTREE_REPO_TOKEN
-  - flat-manager-client create $OSTREE_FLAT_MANAGER_SERVER $OSTREE_FLAT_MANAGER_REPO > build.txt
-  artifacts:
-    paths:
-    - build.txt
-  rules: *flatpak-publish-rules
-
-flatpak-x86_64:
-  extends:
-  - .flatpak-template
-  - .x86_64
-
-flatpak-i386:
-  extends:
-  - .flatpak-template
-  - .i686
-
-flatpak-aarch64:
-  extends:
-  - .flatpak-template
-  - .aarch64
-
-flatpak-arm:
-  extends:
-  - .flatpak-template
-  - .arm
-
-flatpak-ppc64le:
-  extends:
-  - .flatpak-template
-  - .ppc64le
-
-ostree-x86_64:
-  extends:
-  - .ostree-template
-  - .x86_64
-
-flatpak-finish:
-  extends: .x86_64
-  stage: finish_flatpak
-  dependencies: [flatpak-prepare]
-  script:
-  - flat-manager-client commit --wait $(cat build.txt)
-  - flat-manager-client publish --wait $(cat build.txt)
-  - flat-manager-client purge $(cat build.txt)
-  rules: *flatpak-publish-rules
-
-flatpak-finish-failed:
-  extends: .x86_64
-  stage: finish_flatpak
-  dependencies: [flatpak-prepare]
-  script:
-  - flat-manager-client purge $(cat build.txt)
-  rules: &flatpak-failed-rules
-  - if: $EMERGENCY || $BST_TRACK_TAGS != "False"
-    when: never
-  - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^gnome-\d-\d\d$/
-    when: on_failure
-
-ostree-finish:
-  extends: .x86_64
-  stage: finish_ostree
-  dependencies: [ostree-prepare]
-  script:
-  - export REPO_TOKEN=$OSTREE_REPO_TOKEN
-  - flat-manager-client commit --wait $(cat build.txt)
-  - flat-manager-client publish --wait $(cat build.txt)
-  - flat-manager-client purge $(cat build.txt)
-  rules: *flatpak-publish-rules
-
-ostree-finish-failed:
-  extends: .x86_64
-  stage: finish_ostree
-  dependencies: [ostree-prepare]
-  script:
-  - export REPO_TOKEN=$OSTREE_REPO_TOKEN
-  - flat-manager-client purge $(cat build.txt)
-  rules: *flatpak-failed-rules
-  - if: $CI_COMMIT_REF_NAME == "master"
-    when: on_failure
-
 cve_report:
   extends: .x86_64
   stage: reports
diff --git a/.gitlab-ci/flat-manager.yml b/.gitlab-ci/flat-manager.yml
new file mode 100644
index 00000000..eb3dafc1
--- /dev/null
+++ b/.gitlab-ci/flat-manager.yml
@@ -0,0 +1,70 @@
+
+include: .gitlab-ci/arch.yml
+
+stages:
+- prepare
+- deploy
+- finish
+
+.flatmgr-template:
+  stage: deploy
+  dependencies: [flatmgr-prepare]
+  script:
+  - test -n "$BUILD_TARGETS" && $BST -o arch $ARCH build $BUILD_TARGETS
+
+  - TARGETS_REPO=TARGETS_${FLAT_MANAGER_REPO}
+  - TARGETS_REPO_ARCH=TARGETS_{$FLAT_MANAGER_REPO}_${ARCH}
+  - TARGETS="$TARGETS ${!TARGETS_REPO} ${!TARGETS_REPO_ARCH}"
+
+  - $BST -o arch $ARCH pull $TARGETS
+
+  - ostree init --repo repo/ --mode archive
+
+  - |
+    for target in $TARGETS; do
+      $BST -o arch $ARCH checkout --hardlinks $target checkout-repo/
+      ostree pull-local --repo repo/ checkout-repo/
+      rm -rf checkout-repo/
+    done
+
+  - flatpak build-update-repo --generate-static-deltas repo/
+  - flat-manager-client push $(cat build.txt) repo/
+  environment:
+    name: $ENVIRONMENT_NAME
+    action: prepare
+
+
+flatmgr-prepare:
+  extends: .x86_64
+  stage: prepare
+  script:
+  - curl -O $CI_PROJECT_URL/-/jobs/$TRACK_JOB_ID/artifacts/raw/project.refs
+  - flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO > build.txt
+  artifacts:
+    paths:
+    - build.txt
+    - project.refs
+  environment:
+    name: $ENVIRONMENT_NAME
+    action: prepare
+
+flatmgr-finish:
+  extends: .x86_64
+  stage: finish
+  dependencies: [flatmgr-prepare]
+  script:
+  - flat-manager-client commit --publish --wait $(cat build.txt)
+  - flat-manager-client publish --wait $(cat build.txt)
+  after_script:
+  - flat-manager-client purge $(cat build.txt)
+  environment:
+    name: $ENVIRONMENT_NAME
+
+flatmgr-finish-failed:
+  extends: .x86_64
+  stage: finish
+  dependencies: [flatmgr-prepare]
+  script:
+  - flat-manager-client purge $(cat build.txt)
+  when: on_failure
+
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]