GNOME.org
 

[gnome-build-meta/valentindavid/gnome-os-ostree] Re-use already built OSTree repository in image

commit e2f3101e80ab7c0815cffa67b39698982257dd72
Author: Valentin David <valentin david codethink co uk>
Date:   Wed Jan 8 11:48:44 2020 +0100

    Re-use already built OSTree repository in image

 .gitlab-ci.yml                       |  52 +--------------
 elements/vm/deps.bst                 |   1 +
 elements/vm/dev-repo-script.bst      |  19 ++++++
 elements/vm/image.bst                |  44 +++++++------
 elements/vm/ostree-config.bst        |   6 +-
 elements/vm/repo.bst                 |   5 +-
 files/vm/enable-developer-repository |  70 ++++++++++++++++++++
 plugins/ostree_mirror.py             | 120 -----------------------------------
 project.conf                         |  17 +----
 utils/update-local-repo.sh           |   9 +--
 utils/update-repo.sh                 |  20 ++++--
 11 files changed, 139 insertions(+), 224 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ac52a7dd..bebabb2e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,8 +22,6 @@ stages:
 - flatpak
 - finish_flatpak
 - reports
-- publish_ostree
-- build_bootable_image
 
 default:
   before_script:
@@ -87,7 +85,7 @@ default:
     if [ "${ARCH}" == "x86_64" ]; then
         ${BST} build openh264-extension.bst:flatpak-repo.bst
 
-        ${BST} --max-jobs $(( $(nproc) / 4 )) -o arch "${ARCH}" build vm/repo.bst
+        ${BST} --max-jobs $(( $(nproc) / 4 )) -o arch "${ARCH}" build vm/image.bst
     fi
 
   # Store all the downloaded git repos and tarballs in the cache.
@@ -188,7 +186,7 @@ track:
   stage: track
   script:
   - 'sed -i "s/track-tags: False/track-tags: ${BST_TRACK_TAGS}/" project.conf'
-  - ${BST} track --deps all core.bst flatpak-runtimes.bst vm/repo.bst vm/qemu-tools.bst
+  - ${BST} track --deps all core.bst flatpak-runtimes.bst vm/image.bst
   retry: 2
   # only run on branches targeting master
   only:
@@ -310,49 +308,3 @@ cve_report:
   only:
   - master
   - /^gnome-\d-\d\d$/
-
-publish_ostree_x86_64:
-  stage: publish_ostree
-  dependencies: [track]
-  script:
-  - |
-    if [ -n "${GBM_SSH_KEY}" ]; then
-      mkdir -p ~/.ssh
-      rm -f ~/.ssh/id_rsa
-      touch ~/.ssh/id_rsa
-      chmod 600 ~/.ssh/id_rsa
-      echo "${GBM_SSH_KEY}" | base64 --decode >>~/.ssh/id_rsa
-      cat >~/.ssh/known_hosts <<EOF
-    |1|aok+cG86yan2cP42RSwCy2E7X+8=|KBvi7XIpU/CgWzQvRLOVQ+c7+50= ecdsa-sha2-nistp256 
AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAGixpYwAfvRyHOlqpKnrpo3iG5zPjQNL0hcay90SlNMfZ4W6+QUoFTZ/rQXDQFbBrOgTXzFDEQxKxxo350n6dw=
-    EOF
-    fi
-
-  - |
-    OSTREE_BRANCH="$(bst show --format "%{vars}" --deps none vm/repo.bst | sed '/ostree-branch: 
/{;s///;q;};d')"
-  - BUILD_BRANCH="${OSTREE_BRANCH}-${CI_COMMIT_SHA}"
-  - ${BST} build vm/repo.bst
-  - ${BST} checkout --hardlinks vm/repo.bst import-repo
-  - ostree --repo=import-repo/ostree/repo refs -A --create="${BUILD_BRANCH}" "${OSTREE_BRANCH}"
-  - ostree-push --repo=import-repo/ostree/repo "${GBM_PUSH_ADDRESS}:incoming-repo" "${BUILD_BRANCH}"
-  - ssh "${GBM_PUSH_ADDRESS}" commit "${BUILD_BRANCH}" "${OSTREE_BRANCH}"
-  variables:
-    ARCH: x86_64
-  only:
-  - master
-  # For testing only:
-  - valentindavid/gnome-os-ostree
-
-build_bootable_image_x86_64:
-  stage: build_bootable_image
-  dependencies: [track]
-  script:
-  - ${BST} -o arch "${ARCH}" track vm/image.bst
-  - ${BST} --max-jobs $(( $(nproc) / 4 )) -o arch "${ARCH}" build vm/image.bst
-  - ${BST} -o arch "${ARCH}" checkout vm/image.bst image
-  variables:
-    ARCH: x86_64
-  only:
-  - master
-  # For testing only:
-  - valentindavid/gnome-os-ostree
-  artifacts: *logging-artifacts
diff --git a/elements/vm/deps.bst b/elements/vm/deps.bst
index 427a11f1..ee2e667b 100644
--- a/elements/vm/deps.bst
+++ b/elements/vm/deps.bst
@@ -16,6 +16,7 @@ runtime-depends:
 
 - vm/plymouth-gnome-theme.bst
 - vm/kbd.bst
+- vm/dev-repo-script.bst
 
 - freedesktop-sdk.bst:vm/config/pulseaudio.bst
 - freedesktop-sdk.bst:vm/config/sudo.bst
diff --git a/elements/vm/dev-repo-script.bst b/elements/vm/dev-repo-script.bst
new file mode 100644
index 00000000..9728f0bc
--- /dev/null
+++ b/elements/vm/dev-repo-script.bst
@@ -0,0 +1,19 @@
+kind: manual
+
+depends:
+- freedesktop-sdk.bst:bootstrap-import.bst
+
+runtime-depends:
+- core-deps/ostree.bst
+- freedesktop-sdk.bst:components/curl.bst
+
+config:
+  build-commands:
+    - sed -i "s/@@ARCH@@/%{arch}/g" enable-developer-repository
+
+  install-commands:
+  - install -Dm755 -t "%{install-root}%{bindir}" enable-developer-repository
+
+sources:
+- kind: local
+  path: files/vm/enable-developer-repository
diff --git a/elements/vm/image.bst b/elements/vm/image.bst
index 6a203d72..a1a4cccc 100644
--- a/elements/vm/image.bst
+++ b/elements/vm/image.bst
@@ -1,10 +1,11 @@
-kind: manual
+kind: script
 
 build-depends:
 - core-deps/ostree.bst
 - vm/qemu-tools.bst
 - freedesktop-sdk.bst:integration/mtab.bst
 - freedesktop-sdk.bst:components/genimage.bst
+- vm/repo.bst
 
 variables:
   (?):
@@ -17,18 +18,34 @@ variables:
   - arch == "aarch64":
       linux-root: B921B045-1DF0-41C3-AF44-4C6F280D3FAE
 
-  sysroot: "%{build-root}/sysroot"
+  sysroot: "/sysroot"
 
 environment:
   OSTREE_REPO: "%{sysroot}/ostree/repo"
 
 config:
-  build-commands:
+  layout:
+  - element: ''
+    destination: '/sysroot'
+  - element: ''
+    destination: '/genimage'
+  - element: vm/repo.bst
+    destination: '/source-repo'
+  - element: core-deps/ostree.bst
+    destination: '/'
+  - element: vm/qemu-tools.bst
+    destination: '/'
+  - element: integration/mtab.bst
+    destination: '/'
+  - element: components/genimage.bst
+    destination: '/'
+
+  commands:
   - mkdir -p "${OSTREE_REPO}"
   - ostree init --repo="${OSTREE_REPO}" --mode=bare
 
   - ostree config --group sysroot set bootloader none
-  - ostree pull-local "%{build-root}/source-repo" %{ostree-branch}
+  - ostree pull-local "/source-repo" %{ostree-branch}
 
   - mkdir -p "%{sysroot}/boot"
 
@@ -53,10 +70,7 @@ config:
     cp -r "%{sysroot}"/ostree/boot.1/gnome-os/*/*/boot/EFI/ "%{sysroot}/boot/"
 
   - |
-    mkdir -p genimage
-
-  - |
-    cat >genimage/genimage.cfg <<EOF
+    cat >/genimage/genimage.cfg <<EOF
     image efi.img {
         vfat {
             extraargs = "-F32 -n EFI"
@@ -94,20 +108,8 @@ config:
     EOF
 
   - |
-    cd genimage
+    cd /genimage
     genimage --rootpath "%{sysroot}"
 
   - |
     install -Dm644 -t "%{install-root}" genimage/images/disk.qcow2
-
-sources:
-- kind: ostree_mirror
-  (?):
-  - repo_mode == 'local':
-      path: ostree-repo
-      gpg: local.gpg
-  - repo_mode == 'nightly':
-      url: https://nightly.gnome.org/
-      gpg: files/vm/gnome.gpg
-  match: 'gnome-os/*'
-  directory: source-repo
diff --git a/elements/vm/ostree-config.bst b/elements/vm/ostree-config.bst
index 3cb885c1..8bc5232e 100644
--- a/elements/vm/ostree-config.bst
+++ b/elements/vm/ostree-config.bst
@@ -43,8 +43,4 @@ sources:
 - kind: local
   path: files/vm/ostree-config
 - kind: local
-  (?):
-  - repo_mode == 'local':
-      path: local.gpg
-  - repo_mode == 'nightly':
-      path: files/vm/gnome.gpg
+  path: files/vm/gnome.gpg
diff --git a/elements/vm/repo.bst b/elements/vm/repo.bst
index 90e8bea2..b2f10f3a 100644
--- a/elements/vm/repo.bst
+++ b/elements/vm/repo.bst
@@ -10,7 +10,7 @@ variables:
   uuidnamespace: aea54278-2587-4075-ae67-8688ace4ce3d
 
 environment:
-  OSTREE_REPO: "%{install-root}/ostree/repo"
+  OSTREE_REPO: "%{install-root}"
 
 config:
   layout:
@@ -36,4 +36,5 @@ config:
   - mkdir -p "${OSTREE_REPO}"
   - ostree init --repo="${OSTREE_REPO}" --mode=archive
   - mv /sysroot/etc /sysroot/usr/etc
-  - ostree commit --branch=%{ostree-branch} /sysroot
+  - ostree commit --branch=%{ostree-branch} --timestamp="$(date --date="@${SOURCE_DATE_EPOCH}" 
--rfc-3339=seconds)" /sysroot
+
diff --git a/files/vm/enable-developer-repository b/files/vm/enable-developer-repository
new file mode 100644
index 00000000..4980d428
--- /dev/null
+++ b/files/vm/enable-developer-repository
@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -eu
+
+DEFAULT_REF="gnome-os/@@ARCH@@/devel"
+DEFAULT_NAME="local"
+DEFAULT_URL="http://10.0.2.2:8000/";
+DEFAULT_GPG="http://10.0.2.2:8000/key.gpg";
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --name=*)
+            NAME="${1#--name=}"
+            ;;
+        --name)
+            shift
+            NAME="$1"
+            ;;
+        --ref=*)
+            REF="${1#--ref=}"
+            ;;
+        --ref)
+            shift
+            REF="${1}"
+            ;;
+        --url=*)
+            URL="${1#--url=}"
+            ;;
+        --url)
+            shift
+            URL="${1}"
+            ;;
+        --gpg=*)
+            GPG="${1#--gpg=}"
+            ;;
+        --gpg)
+            shift
+            GPG="${1}"
+            ;;
+        --help) ;&
+        -h)
+            cat <<EOF
+$0 [OPTIONS]
+
+Options:
+  --name NAME  OSTree remote name to add (default: ${DEFAULT_NAME})
+  --ref REF    OSTree ref to use (default: ${DEFAULT_REF})
+  --url URL    URL for remote OSTree repository (default: ${DEFAULT_URL})
+  --gpg URL    URL to the public key (default: ${DEFAULT_GPG}))
+  --help       Display this help message and exit
+EOF
+            exit 0
+            ;;
+    esac
+done
+
+: ${REF:="${DEFAULT_REF}"}
+: ${NAME:="${DEFAULT_NAME}"}
+: ${URL:="${DEFAULT_URL}"}
+: ${GPG:="${DEFAULT_GPG}"}
+
+ostree remote add --gpg-import=<(curl "${GPG}") "${NAME}" "${URL}"
+
+ostree pull "${NAME}" "${REF}"
+
+ostree admin deploy "${NAME}:${REF}"
+
+cat <<EOF
+You can now reboot.
+EOF
diff --git a/project.conf b/project.conf
index c20d1695..6d8de00c 100644
--- a/project.conf
+++ b/project.conf
@@ -32,14 +32,6 @@ options:
     - i686
     - x86_64
 
-  repo_mode:
-    type: enum
-    description: Which remote repository should the bootable image use
-    default: nightly
-    values:
-    - nightly
-    - local
-
 # Source aliases.
 #
 # These are used in the individual element.bst files in
@@ -213,6 +205,7 @@ shell:
 variables:
   branch: master
   ostree-branch: "gnome-os/%{arch}/%{branch}"
+  ostree-remote-url: "https://nightly.gnome.org/gnomeos/repo";
   gcc_arch: "%{arch}"
   abi: "gnu"
   gcc_triplet: "%{gcc_arch}-linux-%{abi}"
@@ -236,12 +229,6 @@ variables:
     - arch == "arm":
         abi: "gnueabihf"
 
-    - repo_mode == 'local':
-        # This is the default address to the host using QEMU user netdev
-        ostree-remote-url: "http://10.0.2.2:8000/";
-    - repo_mode == 'nightly':
-        ostree-remote-url: "https://nightly.gnome.org/gnomeos/repo";
-
   # This should eventually done with buildstream includes
   strip-binaries: |
     touch source-files
@@ -302,5 +289,3 @@ plugins:
   path: plugins
   elements:
     collect_initial_scripts: 0
-  sources:
-    ostree_mirror: 0
diff --git a/utils/update-local-repo.sh b/utils/update-local-repo.sh
index ce534f08..64bc24cd 100755
--- a/utils/update-local-repo.sh
+++ b/utils/update-local-repo.sh
@@ -2,12 +2,11 @@
 
 set -eu
 
-export BST="${BST:-bst} -o repo_mode local"
+: ${BST:=bst}
+export BST
 
 ref="$(${BST} show --format "%{vars}" --deps none vm/repo.bst | sed '/ostree-branch: /{;s///;q;};d')"
 
-
-
 if ! [ -d ostree-gpg ]; then
     rm -rf ostree-gpg.tmp
     mkdir ostree-gpg.tmp
@@ -25,7 +24,6 @@ Expire-Date: 0
 EOF
     gpg --batch --homedir=ostree-gpg.tmp --generate-key ostree-gpg.tmp/key-config
     gpg --homedir=ostree-gpg.tmp -k --with-colons | sed '/^fpr:/q;d' | cut -d: -f10 

ostree-gpg.tmp/default-id

-    gpg --homedir=ostree-gpg.tmp --export --armor >local.gpg
     mv ostree-gpg.tmp ostree-gpg
 fi
 
@@ -33,5 +31,8 @@ utils/update-repo.sh \
     --gpg-homedir=ostree-gpg \
     --gpg-sign="$(cat ostree-gpg/default-id)" \
     --collection-id=org.gnome.GnomeOS \
+    --target-ref="${ref%/*}/devel" \
     ostree-repo vm/repo.bst \
     "${ref}"
+
+gpg --homedir=ostree-gpg --export --armor >ostree-repo/key.gpg
diff --git a/utils/update-repo.sh b/utils/update-repo.sh
index 2e5b2b1a..c2e8f0d1 100755
--- a/utils/update-repo.sh
+++ b/utils/update-repo.sh
@@ -13,6 +13,13 @@ EOF
 
 while [ $# -gt 0 ]; do
     case "$1" in
+       --target-ref=*)
+           target_ref="${1#--target-ref=}"
+           ;;
+       --target-ref)
+           target_ref="$2"
+           shift
+           ;;
        --gpg-*=*)
            gpg_opts+=("$1")
            ;;
@@ -69,6 +76,7 @@ OSTREE_REPO="${main_opts[0]}"
 export OSTREE_REPO
 element="${main_opts[1]}"
 ref="${main_opts[2]}"
+: ${target_ref:="${ref}"}
 
 checkout="$(mktemp --suffix="-update-repo" -d -p "$(dirname ${OSTREE_REPO})")"
 
@@ -84,21 +92,21 @@ if ! [ -d ${OSTREE_REPO} ]; then
     ostree init --repo=${OSTREE_REPO} --mode=archive
 fi
 
-commit="$(ostree --repo="${checkout}/ostree/repo" rev-parse "${ref}")"
-ostree pull-local "${checkout}/ostree/repo" "${commit}"
+commit="$(ostree --repo="${checkout}" rev-parse "${ref}")"
+ostree pull-local "${checkout}" "${commit}"
 
-prev_commit="$(ostree rev-parse "${ref}" 2>/dev/null || true)"
+prev_commit="$(ostree rev-parse "${target_ref}" 2>/dev/null || true)"
 
 ostree commit ${gpg_opts[*]} \
-       --branch="${ref}" --tree=ref="${commit}" --skip-if-unchanged
+       --branch="${target_ref}" --tree=ref="${commit}" --skip-if-unchanged
 
-new_commit="$(ostree rev-parse "${ref}")"
+new_commit="$(ostree rev-parse "${target_ref}")"
 
 if [ "${new_commit}" != "${prev_commit}" ]; then
     ostree prune --refs-only --keep-younger-than="6 months ago"
 
     if [ -n "${prev_commit}" ]; then
-        ostree static-delta generate "${ref}"
+        ostree static-delta generate "${target_ref}"
     fi
 
     ostree summary \
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]