[gnome-build-meta/abderrahim/child-pipelines: 7/7] .gitlab-ci.yml: use a child pipeline for deploying flatpak runtimes

[Abderrahim Kitouni <akitouni src gnome org>]

commit eab3456b21a91373e267de28aa61e980355a4907
Author: Abderrahim Kitouni <akitouni gnome org>
Date:   Wed Aug 5 11:41:35 2020 +0100

    .gitlab-ci.yml: use a child pipeline for deploying flatpak runtimes
    
    This allows other jobs to proceed without waiting for the different
    stages of publishing flatpaks
    
    This also makes ppc64le only published for nightly runtimes and not
    beta and stable, and moves publishing to ostree to the new deploy
    stage

 .gitlab-ci.yml              | 193 +++++++++++++++++---------------------------
 .gitlab-ci/flat-manager.yml |  70 ++++++++++++++++
 2 files changed, 144 insertions(+), 119 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eeeab8ed..30766250 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,11 +14,12 @@ variables:
 stages:
 - track
 - build
+- deploy-config
 - image
+- deploy
 - prepare_flatpak
 - flatpak
 - finish_flatpak
-- ostree
 - reports
 
 default:
@@ -46,31 +47,6 @@ default:
     sourcedir: /cache/buildstream/sources
     EOF
 
-  - |
-    case "$FLATPAK_BRANCH" in
-      master)
-        if [ -n "$NIGHTLY_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$NIGHTLY_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://nightly.gnome.org/
-          export FLAT_MANAGER_REPO=nightly
-        fi
-      ;;
-      *beta)
-        if [ -n "$FLATHUB_BETA_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$FLATHUB_BETA_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://hub.flathub.org/
-          export FLAT_MANAGER_REPO=beta
-        fi
-      ;;
-      *)
-        if [ -n "$FLATHUB_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$FLATHUB_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://hub.flathub.org/
-          export FLAT_MANAGER_REPO=stable
-        fi
-      ;;
-    esac
-
 #
 # Templates shared by multiple jobs
 #
@@ -121,43 +97,15 @@ default:
     - image
     expire_in: '2 days'
 
-.flatpak-template:
-  stage: flatpak
-  dependencies: [track, flatpak-prepare]
-  script:
-  - ostree init --repo repo/ --mode archive
-
-  - ${BST} -o arch "${ARCH}" pull flatpak-runtimes.bst
-  - ${BST} -o arch "${ARCH}" checkout --hardlinks flatpak-runtimes.bst checkout-repo/
-  - flatpak build-commit-from --src-repo=checkout-repo/ repo/
-  - rm -rf checkout-repo/
-
-  - |
-    if test $FLAT_MANAGER_REPO == nightly; then
-        ${BST} -o arch "${ARCH}" pull flatpak-platform-extensions.bst
-        ${BST} -o arch "${ARCH}" checkout --hardlinks flatpak-platform-extensions.bst checkout-repo/
-        flatpak build-commit-from --src-repo=checkout-repo/ repo/
-        rm -rf checkout-repo/
-    fi
-
-  - |
-    if test $FLAT_MANAGER_REPO == nightly -a $ARCH == x86_64; then
-        ${BST} pull openh264-extension.bst:flatpak-repo.bst
-        ${BST} checkout --hardlinks openh264-extension.bst:flatpak-repo.bst checkout-repo/
-        flatpak build-commit-from --src-repo=checkout-repo/ repo/
-        rm -rf checkout-repo/
-    fi
-
-  - flatpak build-update-repo --generate-static-deltas repo/
-
-  - flat-manager-client push $(cat build.txt) repo/
-  rules: &flatpak-publish-rules
+.deploy-rules:
+  rules:
   - if: $EMERGENCY || $BST_TRACK_TAGS != "False"
     when: never
   - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^gnome-\d-\d\d$/
 
 .ostree-template:
-  stage: ostree
+  extends: .deploy-rules
+  stage: deploy
   dependencies: [track]
   script:
   - |
@@ -179,8 +127,6 @@ default:
   - BUILD_ID=$(flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO)
   - flat-manager-client push --commit --publish --wait $BUILD_ID repo/ $(ostree --repo=repo/ refs) || 
result=$?
   - flat-manager-client purge $BUILD_ID
-  rules: *flatpak-publish-rules
-
 
 #
 # "Real" CI jobs
@@ -193,6 +139,7 @@ track:
   script:
   - 'sed -i "s/track-tags: False/track-tags: ${BST_TRACK_TAGS}/" project.conf'
   - ${BST} track --deps all core.bst flatpak-runtimes.bst vm/image.bst boards/pinebook-pro/image.bst 
boards/rock64/image.bst boards/raspberrypi-4/image.bst vm/repo-devel.bst
+  - echo $CI_JOB_ID > track-job.id
   retry: 2
   # only run on branches targeting master
   rules:
@@ -200,6 +147,7 @@ track:
   artifacts:
     paths:
     - project.refs
+    - track-job.id
 
 
 build-x86_64:
@@ -228,6 +176,72 @@ build-ppc64le:
   - .ppc64le
 
 
+generate-deploy-config:
+  extends:
+  - .x86_64
+  - .deploy-rules
+  dependencies: [track]
+  stage: deploy-config
+  script:
+  - TRACK_JOB_ID=$(cat track-job.id)
+  - |
+    case "$FLATPAK_BRANCH" in
+      master)
+        FLAT_MANAGER_SERVER=https://nightly.gnome.org/
+        FLAT_MANAGER_REPO=nightly
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm ppc64le"
+      ;;
+      *beta)
+        FLAT_MANAGER_SERVER=https://hub.flathub.org/
+        FLAT_MANAGER_REPO=beta
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm"
+      ;;
+      *)
+        FLAT_MANAGER_SERVER=https://hub.flathub.org/
+        FLAT_MANAGER_REPO=stable
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm"
+      ;;
+    esac
+
+  - |
+    cat > flatpak-variables.yml << END
+    variables:
+      BST: '$BST'
+      TRACK_JOB_ID: '$TRACK_JOB_ID'
+      FLAT_MANAGER_SERVER: '$FLAT_MANAGER_SERVER'
+      FLAT_MANAGER_REPO: '$FLAT_MANAGER_REPO'
+      TARGETS: flatpak-runtimes.bst
+      TARGETS_nightly: flatpak-platform-extensions.bst
+      TARGETS_nightly_x86_64: openh264-extension.bst:flatpak-repo.bst
+      ENVIRONMENT_NAME: flatpak/$FLATPAK_BRANCH
+
+    END
+  - cat flatpak-variables.yml .gitlab-ci/flat-manager.yml > flatpak-deploy.yml
+
+  - |
+    for ARCH in $SUPPORTED_ARCHES; do
+    cat >> flatpak-deploy.yml << END
+
+    flatmgr-$ARCH:
+      extends:
+      - .flatmgr-template
+      - .$ARCH
+    END
+    done
+
+  artifacts:
+    paths:
+    - flatpak-deploy.yml
+
+deploy-flatpak:
+  extends: .deploy-rules
+  stage: deploy
+  trigger:
+    strategy: depend
+    include:
+    - artifact: flatpak-deploy.yml
+      job: generate-deploy-config
+
 vm-image-x86_64:
   extends:
   - .vm-image-template
@@ -295,65 +309,6 @@ raspberrypi-4-image:
   - job: build-aarch64
     artifacts: false
 
-
-flatpak-prepare:
-  extends: .x86_64
-  stage: prepare_flatpak
-  dependencies: []
-  script:
-  - flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO > build.txt
-  artifacts:
-    paths:
-    - build.txt
-  rules: *flatpak-publish-rules
-
-flatpak-x86_64:
-  extends:
-  - .flatpak-template
-  - .x86_64
-
-flatpak-i386:
-  extends:
-  - .flatpak-template
-  - .i686
-
-flatpak-aarch64:
-  extends:
-  - .flatpak-template
-  - .aarch64
-
-flatpak-arm:
-  extends:
-  - .flatpak-template
-  - .arm
-
-flatpak-ppc64le:
-  extends:
-  - .flatpak-template
-  - .ppc64le
-
-flatpak-finish:
-  extends: .x86_64
-  stage: finish_flatpak
-  dependencies: [flatpak-prepare]
-  script:
-  - flat-manager-client commit --wait $(cat build.txt)
-  - flat-manager-client publish --wait $(cat build.txt)
-  - flat-manager-client purge $(cat build.txt)
-  rules: *flatpak-publish-rules
-
-flatpak-finish-failed:
-  extends: .x86_64
-  stage: finish_flatpak
-  dependencies: [flatpak-prepare]
-  script:
-  - flat-manager-client purge $(cat build.txt)
-  rules: &flatpak-failed-rules
-  - if: $EMERGENCY || $BST_TRACK_TAGS != "False"
-    when: never
-  - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^gnome-\d-\d\d$/
-    when: on_failure
-
 ostree-x86_64:
   extends:
   - .ostree-template
diff --git a/.gitlab-ci/flat-manager.yml b/.gitlab-ci/flat-manager.yml
new file mode 100644
index 00000000..eb3dafc1
--- /dev/null
+++ b/.gitlab-ci/flat-manager.yml
@@ -0,0 +1,70 @@
+
+include: .gitlab-ci/arch.yml
+
+stages:
+- prepare
+- deploy
+- finish
+
+.flatmgr-template:
+  stage: deploy
+  dependencies: [flatmgr-prepare]
+  script:
+  - test -n "$BUILD_TARGETS" && $BST -o arch $ARCH build $BUILD_TARGETS
+
+  - TARGETS_REPO=TARGETS_${FLAT_MANAGER_REPO}
+  - TARGETS_REPO_ARCH=TARGETS_{$FLAT_MANAGER_REPO}_${ARCH}
+  - TARGETS="$TARGETS ${!TARGETS_REPO} ${!TARGETS_REPO_ARCH}"
+
+  - $BST -o arch $ARCH pull $TARGETS
+
+  - ostree init --repo repo/ --mode archive
+
+  - |
+    for target in $TARGETS; do
+      $BST -o arch $ARCH checkout --hardlinks $target checkout-repo/
+      ostree pull-local --repo repo/ checkout-repo/
+      rm -rf checkout-repo/
+    done
+
+  - flatpak build-update-repo --generate-static-deltas repo/
+  - flat-manager-client push $(cat build.txt) repo/
+  environment:
+    name: $ENVIRONMENT_NAME
+    action: prepare
+
+
+flatmgr-prepare:
+  extends: .x86_64
+  stage: prepare
+  script:
+  - curl -O $CI_PROJECT_URL/-/jobs/$TRACK_JOB_ID/artifacts/raw/project.refs
+  - flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO > build.txt
+  artifacts:
+    paths:
+    - build.txt
+    - project.refs
+  environment:
+    name: $ENVIRONMENT_NAME
+    action: prepare
+
+flatmgr-finish:
+  extends: .x86_64
+  stage: finish
+  dependencies: [flatmgr-prepare]
+  script:
+  - flat-manager-client commit --publish --wait $(cat build.txt)
+  - flat-manager-client publish --wait $(cat build.txt)
+  after_script:
+  - flat-manager-client purge $(cat build.txt)
+  environment:
+    name: $ENVIRONMENT_NAME
+
+flatmgr-finish-failed:
+  extends: .x86_64
+  stage: finish
+  dependencies: [flatmgr-prepare]
+  script:
+  - flat-manager-client purge $(cat build.txt)
+  when: on_failure
+