[gnome-build-meta/abderrahim/child-pipelines: 4/4] .gitlab-ci.yml: use child pipelines for deploying to flat-manager

[Abderrahim Kitouni <akitouni src gnome org>]

commit f88286b590be0d575f193b35852047de99041929
Author: Abderrahim Kitouni <akitouni gnome org>
Date:   Wed Aug 5 11:41:35 2020 +0100

    .gitlab-ci.yml: use child pipelines for deploying to flat-manager

 .gitlab-ci.yml              | 210 +++++++++++++-------------------------------
 .gitlab-ci/flat-manager.yml |  87 ++++++++++++++++++
 2 files changed, 147 insertions(+), 150 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 78df71b9..4286b5c2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,9 @@ variables:
 stages:
 - track
 - build
+- deploy-config
 - image
+- deploy
 - prepare_flatpak
 - flatpak
 - finish_flatpak
@@ -127,63 +129,13 @@ default:
     - image
     expire_in: '2 days'
 
-.flatpak-template:
-  stage: flatpak
-  dependencies: [track, flatpak-prepare]
-  script:
-  - ostree init --repo repo/ --mode archive
-
-  - ${BST} -o arch "${ARCH}" pull flatpak-runtimes.bst
-  - ${BST} -o arch "${ARCH}" checkout --hardlinks flatpak-runtimes.bst checkout-repo/
-  - flatpak build-commit-from --src-repo=checkout-repo/ repo/
-  - rm -rf checkout-repo/
-
-  - |
-    if test $FLAT_MANAGER_REPO == nightly; then
-        ${BST} -o arch "${ARCH}" pull flatpak-platform-extensions.bst
-        ${BST} -o arch "${ARCH}" checkout --hardlinks flatpak-platform-extensions.bst checkout-repo/
-        flatpak build-commit-from --src-repo=checkout-repo/ repo/
-        rm -rf checkout-repo/
-    fi
-
-  - |
-    if test $FLAT_MANAGER_REPO == nightly -a $ARCH == x86_64; then
-        ${BST} pull openh264-extension.bst:flatpak-repo.bst
-        ${BST} checkout --hardlinks openh264-extension.bst:flatpak-repo.bst checkout-repo/
-        flatpak build-commit-from --src-repo=checkout-repo/ repo/
-        rm -rf checkout-repo/
-    fi
-
-  - flatpak build-update-repo --generate-static-deltas repo/
-
-  - flat-manager-client push $(cat build.txt) repo/
-  rules: &flatpak-publish-rules
+.deploy-rules:
+  stage: deploy
+  rules:
   - if: $EMERGENCY || $BST_TRACK_TAGS != "False"
     when: never
   - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^gnome-\d-\d\d$/
 
-.ostree-template:
-  stage: ostree
-  dependencies: [track, ostree-prepare]
-  script:
-  - export REPO_TOKEN=$OSTREE_REPO_TOKEN
-
-  - ostree init --repo repo/ --mode archive
-
-  - ${BST_NO_PUSH} -o arch "${ARCH}" build vm/repo.bst vm/repo-devel.bst
-
-  - ${BST_NO_PUSH} -o arch "${ARCH}" checkout --hardlinks vm/repo.bst checkout-repo/
-  - ostree pull-local --repo=repo/ checkout-repo/
-  - rm -rf checkout-repo/
-
-  - ${BST_NO_PUSH} -o arch "${ARCH}" checkout --hardlinks vm/repo-devel.bst checkout-devel-repo/
-  - ostree pull-local --repo=repo/ checkout-devel-repo/
-  - rm -rf checkout-devel-repo/
-
-  - flat-manager-client push $(cat build.txt) repo/ $(ostree --repo=repo/ refs)
-  rules: &ostree-publish-rules
-  - if: $CI_COMMIT_REF_NAME == "master"
-
 
 #
 # "Real" CI jobs
@@ -196,6 +148,7 @@ track:
   script:
   - 'sed -i "s/track-tags: False/track-tags: ${BST_TRACK_TAGS}/" project.conf'
   - ${BST} track --deps all core.bst flatpak-runtimes.bst vm/image.bst boards/pinebook-pro/image.bst 
boards/rock64/image.bst boards/raspberrypi-4/image.bst vm/repo-devel.bst
+  - echo $CI_JOB_ID > track-job.id
   retry: 2
   # only run on branches targeting master
   rules:
@@ -203,6 +156,7 @@ track:
   artifacts:
     paths:
     - project.refs
+    - track-job.id
 
 
 build-gnome-core-x86_64:
@@ -231,6 +185,59 @@ build-gnome-core-ppc64le:
   - .ppc64le
 
 
+generate-deploy-config:
+  extends: .x86_64
+  dependencies: [track]
+  stage: deploy-config
+  script:
+  - TRACK_JOB_ID=$(cat track-job.id)
+
+  - |
+    cat > flatpak-variables.yml << END
+    variables:
+      BST: '$BST'
+      FLATPAK_REPO: '$FLATPAK_REPO'
+      TRACK_JOB_ID: '$TRACK_JOB_ID'
+      TARGETS: flatpak-runtimes.bst
+      TARGETS_nightly: flatpak-platform-extensions.bst
+      TARGETS_nightly_x86_64: openh264-extension.bst:flatpak-repo.bst
+
+    END
+  - cat flatpak-variables.yml .gitlab-ci/flat-manager.yml > flatpak-deploy.yml
+
+  - |
+    cat > ostree-variables.yml << END
+    variables:
+      BST: '$BST'
+      FLATPAK_REPO: '$FLATPAK_REPO'
+      TRACK_JOB_ID: '$TRACK_JOB_ID'
+      BUILD_TARGETS: vm/repo.bst vm/repo-devel.bst
+      TARGETS: \$BUILD_TARGETS
+
+    END
+  - cat ostree-variables.yml .gitlab-ci/flat-manager.yml > ostree-deploy.yml
+
+  artifacts:
+    paths:
+    - flatpak-deploy.yml
+    - ostree-deploy.yml
+
+deploy-flatpak:
+  extends: .deploy-rules
+  trigger:
+    strategy: depend
+    include:
+    - artifact: flatpak-deploy.yml
+      job: generate-deploy-config
+
+deploy-ostree:
+  extends: .deploy-rules
+  trigger:
+    strategy: depend
+    include:
+    - artifact: ostree-deploy.yml
+      job: generate-deploy-config
+
 vm-image-x86_64:
   extends:
   - .vm-image-template
@@ -299,103 +306,6 @@ raspberrypi-4-image:
     artifacts: false
 
 
-flatpak-prepare:
-  extends: .x86_64
-  stage: prepare_flatpak
-  dependencies: []
-  script:
-  - flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO > build.txt
-  artifacts:
-    paths:
-    - build.txt
-  rules: *flatpak-publish-rules
-
-ostree-prepare:
-  extends: .x86_64
-  stage: prepare_ostree
-  dependencies: []
-  script:
-  - export REPO_TOKEN=$OSTREE_REPO_TOKEN
-  - flat-manager-client create $OSTREE_FLAT_MANAGER_SERVER $OSTREE_FLAT_MANAGER_REPO > build.txt
-  artifacts:
-    paths:
-    - build.txt
-  rules: *ostree-publish-rules
-
-flatpak-x86_64:
-  extends:
-  - .flatpak-template
-  - .x86_64
-
-flatpak-i386:
-  extends:
-  - .flatpak-template
-  - .i686
-
-flatpak-aarch64:
-  extends:
-  - .flatpak-template
-  - .aarch64
-
-flatpak-arm:
-  extends:
-  - .flatpak-template
-  - .arm
-
-flatpak-ppc64le:
-  extends:
-  - .flatpak-template
-  - .ppc64le
-
-ostree-x86_64:
-  extends:
-  - .ostree-template
-  - .x86_64
-
-flatpak-finish:
-  extends: .x86_64
-  stage: finish_flatpak
-  dependencies: [flatpak-prepare]
-  script:
-  - flat-manager-client commit --wait $(cat build.txt)
-  - flat-manager-client publish --wait $(cat build.txt)
-  - flat-manager-client purge $(cat build.txt)
-  rules: *flatpak-publish-rules
-
-flatpak-finish-failed:
-  extends: .x86_64
-  stage: finish_flatpak
-  dependencies: [flatpak-prepare]
-  script:
-  - flat-manager-client purge $(cat build.txt)
-  rules:
-  - if: $EMERGENCY || $BST_TRACK_TAGS != "False"
-    when: never
-  - if: $CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^gnome-\d-\d\d$/
-    when: on_failure
-
-ostree-finish:
-  extends: .x86_64
-  stage: finish_ostree
-  dependencies: [ostree-prepare]
-  script:
-  - export REPO_TOKEN=$OSTREE_REPO_TOKEN
-  - flat-manager-client commit --wait $(cat build.txt)
-  - flat-manager-client publish --wait $(cat build.txt)
-  - flat-manager-client purge $(cat build.txt)
-  rules: *ostree-publish-rules
-
-ostree-finish-failed:
-  extends: .x86_64
-  stage: finish_ostree
-  dependencies: [ostree-prepare]
-  script:
-  - export REPO_TOKEN=$OSTREE_REPO_TOKEN
-  - flat-manager-client purge $(cat build.txt)
-  rules:
-  - if: $CI_COMMIT_REF_NAME == "master"
-    when: on_failure
-
 cve_report:
   extends: .x86_64
   stage: reports
diff --git a/.gitlab-ci/flat-manager.yml b/.gitlab-ci/flat-manager.yml
new file mode 100644
index 00000000..1bdf626c
--- /dev/null
+++ b/.gitlab-ci/flat-manager.yml
@@ -0,0 +1,87 @@
+include: .gitlab-ci/arch.yml
+
+stages:
+- prepare
+- deploy
+- finish
+
+.flatmgr-template:
+  stage: deploy
+  dependencies: [flatmgr-prepare]
+  script:
+  - test -n "$BUILD_TARGETS" && $BST -o arch $ARCH build $BUILD_TARGETS
+
+  - TARGETS_REPO=TARGETS_${FLAT_MANAGER_REPO}
+  - TARGETS_REPO_ARCH=TARGETS_{$FLAT_MANAGER_REPO}_${ARCH}
+  - TARGETS="$TARGETS ${!TARGETS_REPO} ${!TARGETS_REPO_ARCH}"
+
+  - $BST -o arch $ARCH pull $TARGETS
+  
+  - ostree init --repo repo/ --mode archive
+
+  - |
+    for target in $TARGETS; do
+      $BST -o arch $ARCH checkout --hardlinks $target checkout-repo/
+      ostree pull-local --repo repo/ checkout-repo/
+      rm -rf checkout-repo/
+    done
+
+  - flatpak build-update-repo --generate-static-deltas repo/
+  - flat-manager-client push $(cat build.txt) repo/
+
+
+flatmgr-prepare:
+  extends: .x86_64
+  stage: prepare
+  script:
+  - curl -O $CI_PROJECT_URL/-/jobs/$TRACK_JOB_ID/artifacts/raw/project.refs
+  - flat-manager-client create $FLAT_MANAGER_SERVER $FLAT_MANAGER_REPO > build.txt
+  artifacts:
+    paths:
+    - build.txt
+    - project.refs
+
+
+flatmgr-x86_64:
+  extends:
+  - .flatmgr-template
+  - .x86_64
+
+flatmgr-i386:
+  extends:
+  - .flatmgr-template
+  - .i686
+
+flatmgr-aarch64:
+  extends:
+  - .flatmgr-template
+  - .aarch64
+
+flatmgr-arm:
+  extends:
+  - .flatmgr-template
+  - .arm
+
+flatmgr-ppc64le:
+  extends:
+  - .flatmgr-template
+  - .ppc64le
+
+
+flatmgr-finish:
+  extends: .x86_64
+  stage: finish
+  dependencies: [flatmgr-prepare]
+  script:
+  - flat-manager-client commit --publish --wait $(cat build.txt)
+  - flat-manager-client publish --wait $(cat build.txt)
+  after_script:
+  - flat-manager-client purge $(cat build.txt)
+
+flatmgr-finish-failed:
+  extends: .x86_64
+  stage: finish
+  dependencies: [flatmgr-prepare]
+  script:
+  - flat-manager-client purge $(cat build.txt)
+  when: on_failure