[gnome-build-meta/abderrahim/ostree-speedup] vm/repo*: add plugin to commit to ostree

[Abderrahim Kitouni <akitouni src gnome org>]

commit f78badb21d6a905841f69ed9e9efb3153d1c5f97
Author: Abderrahim Kitouni <akitouni gnome org>
Date:   Sun Aug 9 18:22:45 2020 +0100

    vm/repo*: add plugin to commit to ostree
    
    This reduces the time it takes for a commit from ~20min to ~5min on my machine
    
    Fixes #296

 elements/vm/repo-devel.bst | 31 +++++------------
 elements/vm/repo.bst       | 32 +++++-------------
 plugins/ostree.py          | 84 ++++++++++++++++++++++++++++++++++++++++++++++
 plugins/ostree.yaml        |  9 +++++
 project.conf               |  1 +
 5 files changed, 112 insertions(+), 45 deletions(-)
---
diff --git a/elements/vm/repo-devel.bst b/elements/vm/repo-devel.bst
index 4ce7d054..517bd38b 100644
--- a/elements/vm/repo-devel.bst
+++ b/elements/vm/repo-devel.bst
@@ -1,4 +1,4 @@
-kind: script
+kind: ostree
 
 build-depends:
 - core-deps/libostree.bst
@@ -10,32 +10,19 @@ variables:
   uuidnamespace: aea54278-2587-4075-ae67-8688ace4ce3d
   ostree-layer: devel
 
-environment:
-  OSTREE_REPO: "%{install-root}"
-
 config:
-  layout:
-  - element: ''
-    destination: /tmp
-  - element: core-deps/libostree.bst
-    destination: /
-  - element: vm/filesystem-devel.bst
-    destination: /sysroot
-  - element: vm/initial-scripts-devel.bst
-    destination: /
-  - element: vm/prepare-image.bst
-    destination: /
-
-  commands:
+  environment:
+  - core-deps/libostree.bst
+  - vm/prepare-image.bst
+  - vm/initial-scripts-devel.bst
+  ostree-branch: '%{ostree-branch}'
+  initial-commands:
   - |
     prepare-image.sh \
-       --sysroot /sysroot \
+       --sysroot %{sysroot} \
        --seed "%{uuidnamespace}" \
        --rootsource /dev/gpt-auto-root \
        --efisource LABEL=EFI \
        --efipath /boot >/dev/null
 
-  - mkdir -p "${OSTREE_REPO}"
-  - ostree init --repo="${OSTREE_REPO}" --mode=archive
-  - mv /sysroot/etc /sysroot/usr/etc
-  - ostree commit --fsync=false --branch=%{ostree-branch} --timestamp="$(date --date="@${SOURCE_DATE_EPOCH}" 
--rfc-3339=seconds)" /sysroot
+  - mv %{sysroot}/etc %{sysroot}/usr/etc
diff --git a/elements/vm/repo.bst b/elements/vm/repo.bst
index 0ad012ab..f9ecc7ca 100644
--- a/elements/vm/repo.bst
+++ b/elements/vm/repo.bst
@@ -1,4 +1,4 @@
-kind: script
+kind: ostree
 
 build-depends:
 - core-deps/libostree.bst
@@ -9,33 +9,19 @@ build-depends:
 variables:
   uuidnamespace: aea54278-2587-4075-ae67-8688ace4ce3d
 
-environment:
-  OSTREE_REPO: "%{install-root}"
-
 config:
-  layout:
-  - element: ''
-    destination: /tmp
-  - element: core-deps/libostree.bst
-    destination: /
-  - element: vm/filesystem.bst
-    destination: /sysroot
-  - element: vm/initial-scripts.bst
-    destination: /
-  - element: vm/prepare-image.bst
-    destination: /
-
-  commands:
+  environment:
+  - core-deps/libostree.bst
+  - vm/prepare-image.bst
+  - vm/initial-scripts.bst
+  ostree-branch: '%{ostree-branch}'
+  initial-commands:
   - |
     prepare-image.sh \
-       --sysroot /sysroot \
+       --sysroot %{sysroot} \
        --seed "%{uuidnamespace}" \
        --rootsource /dev/gpt-auto-root \
        --efisource LABEL=EFI \
        --efipath /boot >/dev/null
 
-  - mkdir -p "${OSTREE_REPO}"
-  - ostree init --repo="${OSTREE_REPO}" --mode=archive
-  - mv /sysroot/etc /sysroot/usr/etc
-  - ostree commit --fsync=false --branch=%{ostree-branch} --timestamp="$(date --date="@${SOURCE_DATE_EPOCH}" 
--rfc-3339=seconds)" /sysroot
-
+  - mv %{sysroot}/etc %{sysroot}/usr/etc
diff --git a/plugins/ostree.py b/plugins/ostree.py
new file mode 100644
index 00000000..89f79ff4
--- /dev/null
+++ b/plugins/ostree.py
@@ -0,0 +1,84 @@
+from buildstream import Element, Scope, SandboxFlags, ElementError
+import os, shutil
+
+class OstreeElement(Element):
+    BST_FORBID_RDEPENDS = True
+    BST_FORBID_SOURCES = True
+    BST_STRICT_REBUILD = True
+
+    def preflight(self):
+        pass
+
+    def configure(self, node):
+        self.node_validate(node, ['environment', 'ostree-branch', 'initial-commands'])
+
+        self.env = self.node_subst_list(node, 'environment')
+        self.branch = self.node_subst_member(node, 'ostree-branch')
+        self.initial_commands = self.node_subst_list(node, 'initial-commands')
+
+    def get_unique_key(self):
+        return {
+            'branch': self.branch,
+            'initial-commands': self.initial_commands
+        }
+
+    def configure_sandbox(self, sandbox):
+        sandbox.mark_directory(self.get_variable('build-root'), artifact=True)
+        sandbox.mark_directory(self.get_variable('install-root'))
+
+        sandbox.set_environment(self.get_environment())
+
+    def stage(self, sandbox):
+        env = []
+        source_deps = []
+        for dep in self.dependencies(Scope.BUILD, recurse=False):
+            if dep.name in self.env:
+                self.status("{} in environment".format(dep.name))
+                env.append(dep)
+            else:
+                self.status("{} in sysroot".format(dep.name))
+                source_deps.append(dep)
+
+        with self.timed_activity("Staging environment", silent_nested=True):
+            for build_dep in env:
+                build_dep.stage_dependency_artifacts(sandbox, Scope.RUN)
+
+        with self.timed_activity("Integrating sandbox", silent_nested=True):
+            for build_dep in env:
+                for dep in build_dep.dependencies(Scope.RUN):
+                    dep.integrate(sandbox)
+
+        for build_dep in source_deps:
+            build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path=self.get_variable('sysroot'))
+
+    def assemble(self, sandbox):
+        def run_command(*command):
+            exitcode = sandbox.run(command, SandboxFlags.ROOT_READ_ONLY)
+            if exitcode != 0:
+                raise ElementError("Command '{}' failed with exitcode {}".format(" ".join(command), 
exitcode))
+
+        sysroot = self.get_variable('sysroot')
+        barerepopath = os.path.join(self.get_variable('build-root'), 'barerepo')
+        repopath = self.get_variable('install-root')
+
+        with self.timed_activity('Running initial commands'):
+            for command in self.initial_commands:
+                run_command('sh', '-c', command)
+
+        with self.timed_activity('Initial commit'):
+            # ostree doesn't like the fuse filesystem buildstream uses to prevent artifact corruption
+            # so disable it. This should be safe as ostree shouldn't modify the files contents now
+            sandbox.mark_directory(self.get_variable('build-root'), artifact=False)
+
+            run_command('ostree', 'init', '--repo', barerepopath)
+            run_command('ostree', 'commit', '--repo', barerepopath, '--consume', sysroot,
+                        '--branch', self.branch, '--timestamp', '2011-11-11 11:11:11+00:00')
+
+        with self.timed_activity('Pull'):
+            run_command('ostree', 'init', '--repo', repopath, '--mode', 'archive')
+            run_command('ostree', 'pull-local', '--repo', repopath, barerepopath)
+
+        return repopath
+
+def setup():
+    return OstreeElement
diff --git a/plugins/ostree.yaml b/plugins/ostree.yaml
new file mode 100644
index 00000000..c7a87d4f
--- /dev/null
+++ b/plugins/ostree.yaml
@@ -0,0 +1,9 @@
+variables:
+  sysroot: '%{build-root}/sysroot'
+
+config:
+  # environment: []
+  # ostree-branch: ''
+
+  # Commands to run on the sysroot before comitting
+  initial-commands: []
diff --git a/project.conf b/project.conf
index 028fa287..b2efac37 100644
--- a/project.conf
+++ b/project.conf
@@ -257,3 +257,4 @@ plugins:
   path: plugins
   elements:
     collect_initial_scripts: 0
+    ostree: 0
\ No newline at end of file