[gnome-build-meta/abderrahim/child-pipelines: 11/11] .gitlab-ci.yml: use a child pipeline for deploying flatpak runtimes

[Abderrahim Kitouni <akitouni src gnome org>]

commit 73f3c5d1ff03a0d512a1cf63ff054b62367b8e5b
Author: Abderrahim Kitouni <akitouni gnome org>
Date:   Wed Aug 5 11:41:35 2020 +0100

    .gitlab-ci.yml: use a child pipeline for deploying flatpak runtimes
    
    This allows other jobs to proceed without waiting for the different
    stages of publishing flatpaks
    
    This also makes ppc64le only published for nightly runtimes and not
    beta and stable

 .gitlab-ci.yml              | 194 +++++++++++++++++---------------------------
 .gitlab-ci/flat-manager.yml |  70 ++++++++++++++++
 2 files changed, 144 insertions(+), 120 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6a6d1410..a78a2280 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,11 +14,12 @@ variables:
 stages:
 - track
 - build
+- deploy-config
 - image
+- deploy
 - prepare_flatpak
 - flatpak
 - finish_flatpak
-- ostree
 - reports
 
 default:
@@ -46,31 +47,6 @@ default:
     sourcedir: /cache/buildstream/sources
     EOF
 
-  - |
-    case "$FLATPAK_BRANCH" in
-      master)
-        if [ -n "$NIGHTLY_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$NIGHTLY_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://nightly.gnome.org/
-          export FLAT_MANAGER_REPO=nightly
-        fi
-      ;;
-      *beta)
-        if [ -n "$FLATHUB_BETA_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$FLATHUB_BETA_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://hub.flathub.org/
-          export FLAT_MANAGER_REPO=beta
-        fi
-      ;;
-      *)
-        if [ -n "$FLATHUB_REPO_TOKEN" ]; then
-          export REPO_TOKEN=$FLATHUB_REPO_TOKEN
-          export FLAT_MANAGER_SERVER=https://hub.flathub.org/
-          export FLAT_MANAGER_REPO=stable
-        fi
-      ;;
-    esac
-
 #
 # Templates shared by multiple jobs
 #
@@ -121,43 +97,15 @@ default:
     - image
     expire_in: '2 days'
 
-.flatpak-template:
-  stage: flatpak
-  dependencies: [track, flatpak-prepare]
-  script:
-  - ostree init --repo repo/ --mode archive
-
-  - ${BST} -o arch "${ARCH}" pull flatpak-runtimes.bst
-  - ${BST} -o arch "${ARCH}" checkout --hardlinks flatpak-runtimes.bst checkout-repo/
-  - flatpak build-commit-from --src-repo=checkout-repo/ repo/
-  - rm -rf checkout-repo/
-
-  - |
-    if test $FLAT_MANAGER_REPO == nightly; then
-        ${BST} -o arch "${ARCH}" pull flatpak-platform-extensions.bst
-        ${BST} -o arch "${ARCH}" checkout --hardlinks flatpak-platform-extensions.bst checkout-repo/
-        flatpak build-commit-from --src-repo=checkout-repo/ repo/
-        rm -rf checkout-repo/
-    fi
-
-  - |
-    if test $FLAT_MANAGER_REPO == nightly -a $ARCH == x86_64; then
-        ${BST} pull openh264-extension.bst:flatpak-repo.bst
-        ${BST} checkout --hardlinks openh264-extension.bst:flatpak-repo.bst checkout-repo/
-        flatpak build-commit-from --src-repo=checkout-repo/ repo/
-        rm -rf checkout-repo/
-    fi
-
-  - flatpak build-update-repo --generate-static-deltas repo/
-
-  - flat-manager-client push $(cat build.txt) repo/
-  rules: &flatpak-publish-rules
+.deploy-rules:
+  rules:
   - if: $EMERGENCY || $BST_TRACK_TAGS != "False"
     when: never
   - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^gnome-\d-\d\d$/
 
 .ostree-template:
-  stage: ostree
+  extends: .deploy-rules
+  stage: deploy
   dependencies: [track]
   script:
   - |
@@ -181,9 +129,6 @@ default:
   - flat-manager-client push --commit --publish --wait $BUILD_ID repo/ $(ostree --repo=repo/ refs) || 
result=$?
   - flat-manager-client purge $BUILD_ID
 
-  rules: *flatpak-publish-rules
-
-
 #
 # "Real" CI jobs
 #
@@ -195,6 +140,7 @@ track:
   script:
   - 'sed -i "s/track-tags: False/track-tags: ${BST_TRACK_TAGS}/" project.conf'
   - ${BST} track --deps all core.bst flatpak-runtimes.bst vm/image.bst boards/pinebook-pro/image.bst 
boards/rock64/image.bst boards/raspberrypi-4/image.bst vm/repo-devel.bst
+  - echo $CI_JOB_ID > track-job.id
   retry: 2
   # only run on branches targeting master
   rules:
@@ -202,6 +148,7 @@ track:
   artifacts:
     paths:
     - project.refs
+    - track-job.id
 
 
 build-gnome-core-x86_64:
@@ -230,6 +177,72 @@ build-gnome-core-ppc64le:
   - .ppc64le
 
 
+generate-deploy-config:
+  extends:
+  - .x86_64
+  - .deploy-rules
+  dependencies: [track]
+  stage: deploy-config
+  script:
+  - TRACK_JOB_ID=$(cat track-job.id)
+  - |
+    case "$FLATPAK_BRANCH" in
+      master)
+        FLAT_MANAGER_SERVER=https://nightly.gnome.org/
+        FLAT_MANAGER_REPO=nightly
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm ppc64le"
+      ;;
+      *beta)
+        FLAT_MANAGER_SERVER=https://hub.flathub.org/
+        FLAT_MANAGER_REPO=beta
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm"
+      ;;
+      *)
+        FLAT_MANAGER_SERVER=https://hub.flathub.org/
+        FLAT_MANAGER_REPO=stable
+        SUPPORTED_ARCHES="x86_64 i686 aarch64 arm"
+      ;;
+    esac
+
+  - |
+    cat > flatpak-variables.yml << END
+    variables:
+      BST: '$BST'
+      TRACK_JOB_ID: '$TRACK_JOB_ID'
+      FLAT_MANAGER_SERVER: '$FLAT_MANAGER_SERVER'
+      FLAT_MANAGER_REPO: '$FLAT_MANAGER_REPO'
+      TARGETS: flatpak-runtimes.bst
+      TARGETS_nightly: flatpak-platform-extensions.bst
+      TARGETS_nightly_x86_64: openh264-extension.bst:flatpak-repo.bst
+      ENVIRONMENT_NAME: flatpak/$FLATPAK_BRANCH
+
+    END
+  - cat flatpak-variables.yml .gitlab-ci/flat-manager.yml > flatpak-deploy.yml
+
+  - |
+    for ARCH in $SUPPORTED_ARCHES; do
+    cat >> flatpak-deploy.yml << END
+
+    flatmgr-$ARCH:
+      extends:
+      - .flatmgr-template
+      - .$ARCH
+    END
+    done
+
+  artifacts:
+    paths:
+    - flatpak-deploy.yml
+
+deploy-flatpak:
+  extends: .deploy-rules
+  stage: deploy
+  trigger:
+    strategy: depend
+    include:
+    - artifact: flatpak-deploy.yml
+      job: generate-deploy-config
+
 vm-image-x86_64:
   extends:
   - .vm-image-template
@@ -297,65 +310,6 @@ raspberrypi-4-image:
   - job: build-gnome-core-aarch64
     artifacts: false
 
-
-flatpak-prepare:
-  extends: .x86_64
-  stage: prepare_flatpak
-  dependencies: []
-  script:
-  - flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO > build.txt
-  artifacts:
-    paths:
-    - build.txt
-  rules: *flatpak-publish-rules
-
-flatpak-x86_64:
-  extends:
-  - .flatpak-template
-  - .x86_64
-
-flatpak-i386:
-  extends:
-  - .flatpak-template
-  - .i686
-
-flatpak-aarch64:
-  extends:
-  - .flatpak-template
-  - .aarch64
-
-flatpak-arm:
-  extends:
-  - .flatpak-template
-  - .arm
-
-flatpak-ppc64le:
-  extends:
-  - .flatpak-template
-  - .ppc64le
-
-flatpak-finish:
-  extends: .x86_64
-  stage: finish_flatpak
-  dependencies: [flatpak-prepare]
-  script:
-  - flat-manager-client commit --wait $(cat build.txt)
-  - flat-manager-client publish --wait $(cat build.txt)
-  - flat-manager-client purge $(cat build.txt)
-  rules: *flatpak-publish-rules
-
-flatpak-finish-failed:
-  extends: .x86_64
-  stage: finish_flatpak
-  dependencies: [flatpak-prepare]
-  script:
-  - flat-manager-client purge $(cat build.txt)
-  rules: &flatpak-failed-rules
-  - if: $EMERGENCY || $BST_TRACK_TAGS != "False"
-    when: never
-  - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^gnome-\d-\d\d$/
-    when: on_failure
-
 ostree-x86_64:
   extends:
   - .ostree-template
diff --git a/.gitlab-ci/flat-manager.yml b/.gitlab-ci/flat-manager.yml
new file mode 100644
index 00000000..eb3dafc1
--- /dev/null
+++ b/.gitlab-ci/flat-manager.yml
@@ -0,0 +1,70 @@
+
+include: .gitlab-ci/arch.yml
+
+stages:
+- prepare
+- deploy
+- finish
+
+.flatmgr-template:
+  stage: deploy
+  dependencies: [flatmgr-prepare]
+  script:
+  - test -n "$BUILD_TARGETS" && $BST -o arch $ARCH build $BUILD_TARGETS
+
+  - TARGETS_REPO=TARGETS_${FLAT_MANAGER_REPO}
+  - TARGETS_REPO_ARCH=TARGETS_{$FLAT_MANAGER_REPO}_${ARCH}
+  - TARGETS="$TARGETS ${!TARGETS_REPO} ${!TARGETS_REPO_ARCH}"
+
+  - $BST -o arch $ARCH pull $TARGETS
+
+  - ostree init --repo repo/ --mode archive
+
+  - |
+    for target in $TARGETS; do
+      $BST -o arch $ARCH checkout --hardlinks $target checkout-repo/
+      ostree pull-local --repo repo/ checkout-repo/
+      rm -rf checkout-repo/
+    done
+
+  - flatpak build-update-repo --generate-static-deltas repo/
+  - flat-manager-client push $(cat build.txt) repo/
+  environment:
+    name: $ENVIRONMENT_NAME
+    action: prepare
+
+
+flatmgr-prepare:
+  extends: .x86_64
+  stage: prepare
+  script:
+  - curl -O $CI_PROJECT_URL/-/jobs/$TRACK_JOB_ID/artifacts/raw/project.refs
+  - flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO > build.txt
+  artifacts:
+    paths:
+    - build.txt
+    - project.refs
+  environment:
+    name: $ENVIRONMENT_NAME
+    action: prepare
+
+flatmgr-finish:
+  extends: .x86_64
+  stage: finish
+  dependencies: [flatmgr-prepare]
+  script:
+  - flat-manager-client commit --publish --wait $(cat build.txt)
+  - flat-manager-client publish --wait $(cat build.txt)
+  after_script:
+  - flat-manager-client purge $(cat build.txt)
+  environment:
+    name: $ENVIRONMENT_NAME
+
+flatmgr-finish-failed:
+  extends: .x86_64
+  stage: finish
+  dependencies: [flatmgr-prepare]
+  script:
+  - flat-manager-client purge $(cat build.txt)
+  when: on_failure
+