[gnome-build-meta/abderrahim/ostree-speedup] vm/repo*: add plugin to commit to ostree

[Abderrahim Kitouni <akitouni src gnome org>]

commit 3c1f24c4f27b8c0d9c3ae777013cc29f57d86c82
Author: Abderrahim Kitouni <akitouni gnome org>
Date:   Sun Aug 9 18:22:45 2020 +0100

    vm/repo*: add plugin to commit to ostree
    
    This reduces the time it takes for a commit from ~20min to ~5min on my machine
    
    Fixes #296

 elements/vm/repo-devel.bst | 31 +++++----------
 elements/vm/repo.bst       | 32 +++++----------
 plugins/ostree.py          | 99 ++++++++++++++++++++++++++++++++++++++++++++++
 plugins/ostree.yaml        |  9 +++++
 project.conf               |  1 +
 5 files changed, 127 insertions(+), 45 deletions(-)
---
diff --git a/elements/vm/repo-devel.bst b/elements/vm/repo-devel.bst
index 4ce7d054..517bd38b 100644
--- a/elements/vm/repo-devel.bst
+++ b/elements/vm/repo-devel.bst
@@ -1,4 +1,4 @@
-kind: script
+kind: ostree
 
 build-depends:
 - core-deps/libostree.bst
@@ -10,32 +10,19 @@ variables:
   uuidnamespace: aea54278-2587-4075-ae67-8688ace4ce3d
   ostree-layer: devel
 
-environment:
-  OSTREE_REPO: "%{install-root}"
-
 config:
-  layout:
-  - element: ''
-    destination: /tmp
-  - element: core-deps/libostree.bst
-    destination: /
-  - element: vm/filesystem-devel.bst
-    destination: /sysroot
-  - element: vm/initial-scripts-devel.bst
-    destination: /
-  - element: vm/prepare-image.bst
-    destination: /
-
-  commands:
+  environment:
+  - core-deps/libostree.bst
+  - vm/prepare-image.bst
+  - vm/initial-scripts-devel.bst
+  ostree-branch: '%{ostree-branch}'
+  initial-commands:
   - |
     prepare-image.sh \
-       --sysroot /sysroot \
+       --sysroot %{sysroot} \
        --seed "%{uuidnamespace}" \
        --rootsource /dev/gpt-auto-root \
        --efisource LABEL=EFI \
        --efipath /boot >/dev/null
 
-  - mkdir -p "${OSTREE_REPO}"
-  - ostree init --repo="${OSTREE_REPO}" --mode=archive
-  - mv /sysroot/etc /sysroot/usr/etc
-  - ostree commit --fsync=false --branch=%{ostree-branch} --timestamp="$(date --date="@${SOURCE_DATE_EPOCH}" 
--rfc-3339=seconds)" /sysroot
+  - mv %{sysroot}/etc %{sysroot}/usr/etc
diff --git a/elements/vm/repo.bst b/elements/vm/repo.bst
index 0ad012ab..f9ecc7ca 100644
--- a/elements/vm/repo.bst
+++ b/elements/vm/repo.bst
@@ -1,4 +1,4 @@
-kind: script
+kind: ostree
 
 build-depends:
 - core-deps/libostree.bst
@@ -9,33 +9,19 @@ build-depends:
 variables:
   uuidnamespace: aea54278-2587-4075-ae67-8688ace4ce3d
 
-environment:
-  OSTREE_REPO: "%{install-root}"
-
 config:
-  layout:
-  - element: ''
-    destination: /tmp
-  - element: core-deps/libostree.bst
-    destination: /
-  - element: vm/filesystem.bst
-    destination: /sysroot
-  - element: vm/initial-scripts.bst
-    destination: /
-  - element: vm/prepare-image.bst
-    destination: /
-
-  commands:
+  environment:
+  - core-deps/libostree.bst
+  - vm/prepare-image.bst
+  - vm/initial-scripts.bst
+  ostree-branch: '%{ostree-branch}'
+  initial-commands:
   - |
     prepare-image.sh \
-       --sysroot /sysroot \
+       --sysroot %{sysroot} \
        --seed "%{uuidnamespace}" \
        --rootsource /dev/gpt-auto-root \
        --efisource LABEL=EFI \
        --efipath /boot >/dev/null
 
-  - mkdir -p "${OSTREE_REPO}"
-  - ostree init --repo="${OSTREE_REPO}" --mode=archive
-  - mv /sysroot/etc /sysroot/usr/etc
-  - ostree commit --fsync=false --branch=%{ostree-branch} --timestamp="$(date --date="@${SOURCE_DATE_EPOCH}" 
--rfc-3339=seconds)" /sysroot
-
+  - mv %{sysroot}/etc %{sysroot}/usr/etc
diff --git a/plugins/ostree.py b/plugins/ostree.py
new file mode 100644
index 00000000..d5de139d
--- /dev/null
+++ b/plugins/ostree.py
@@ -0,0 +1,99 @@
+from buildstream import Element, Scope, SandboxFlags, ElementError
+import os
+
+
+class OstreeElement(Element):
+    BST_FORBID_RDEPENDS = True
+    BST_FORBID_SOURCES = True
+    BST_STRICT_REBUILD = True
+
+    def preflight(self):
+        pass
+
+    def configure(self, node):
+        self.node_validate(node, ["environment", "ostree-branch", "initial-commands"])
+
+        self.env = self.node_subst_list(node, "environment")
+        self.branch = self.node_subst_member(node, "ostree-branch")
+        self.initial_commands = self.node_subst_list(node, "initial-commands")
+
+    def get_unique_key(self):
+        return {"branch": self.branch, "initial-commands": self.initial_commands}
+
+    def configure_sandbox(self, sandbox):
+        sandbox.mark_directory(self.get_variable("build-root"), artifact=True)
+        sandbox.mark_directory(self.get_variable("install-root"))
+
+        sandbox.set_environment(self.get_environment())
+
+    def stage(self, sandbox):
+        env = []
+        source_deps = []
+        for dep in self.dependencies(Scope.BUILD, recurse=False):
+            if dep.name in self.env:
+                self.status("{} in environment".format(dep.name))
+                env.append(dep)
+            else:
+                self.status("{} in sysroot".format(dep.name))
+                source_deps.append(dep)
+
+        with self.timed_activity("Staging environment", silent_nested=True):
+            for build_dep in env:
+                build_dep.stage_dependency_artifacts(sandbox, Scope.RUN)
+
+        with self.timed_activity("Integrating sandbox", silent_nested=True):
+            for build_dep in env:
+                for dep in build_dep.dependencies(Scope.RUN):
+                    dep.integrate(sandbox)
+
+        for build_dep in source_deps:
+            build_dep.stage_dependency_artifacts(
+                sandbox, Scope.RUN, path=self.get_variable("sysroot")
+            )
+
+    def assemble(self, sandbox):
+        def run_command(*command):
+            exitcode = sandbox.run(command, SandboxFlags.ROOT_READ_ONLY)
+            if exitcode != 0:
+                raise ElementError(
+                    "Command '{}' failed with exitcode {}".format(
+                        " ".join(command), exitcode
+                    )
+                )
+
+        sysroot = self.get_variable("sysroot")
+        barerepopath = os.path.join(self.get_variable("build-root"), "barerepo")
+        repopath = self.get_variable("install-root")
+
+        with self.timed_activity("Running initial commands"):
+            for command in self.initial_commands:
+                run_command("sh", "-c", command)
+
+        with self.timed_activity("Initial commit"):
+            # ostree doesn't like the fuse filesystem buildstream uses to prevent artifact corruption
+            # so disable it. This should be safe as ostree shouldn't modify the files contents now
+            sandbox.mark_directory(self.get_variable("build-root"), artifact=False)
+
+            run_command("ostree", "init", "--repo", barerepopath)
+            run_command(
+                "ostree",
+                "commit",
+                "--repo",
+                barerepopath,
+                "--consume",
+                sysroot,
+                "--branch",
+                self.branch,
+                "--timestamp",
+                "2011-11-11 11:11:11+00:00",
+            )
+
+        with self.timed_activity("Pull"):
+            run_command("ostree", "init", "--repo", repopath, "--mode", "archive")
+            run_command("ostree", "pull-local", "--repo", repopath, barerepopath)
+
+        return repopath
+
+
+def setup():
+    return OstreeElement
diff --git a/plugins/ostree.yaml b/plugins/ostree.yaml
new file mode 100644
index 00000000..c7a87d4f
--- /dev/null
+++ b/plugins/ostree.yaml
@@ -0,0 +1,9 @@
+variables:
+  sysroot: '%{build-root}/sysroot'
+
+config:
+  # environment: []
+  # ostree-branch: ''
+
+  # Commands to run on the sysroot before comitting
+  initial-commands: []
diff --git a/project.conf b/project.conf
index 028fa287..b2efac37 100644
--- a/project.conf
+++ b/project.conf
@@ -257,3 +257,4 @@ plugins:
   path: plugins
   elements:
     collect_initial_scripts: 0
+    ostree: 0
\ No newline at end of file