[gnome-build-meta/valentindavid/partial-tracking] Copy project.refs from last scheduled strict build and track only modified files

[Valentin David <valentindavid src gnome org>]

commit 558fea9624202cce6b609706a605335212c14788
Author: Valentin David <valentin david codethink co uk>
Date:   Sat Apr 18 14:54:08 2020 +0200

    Copy project.refs from last scheduled strict build and track only modified files

 .gitlab-ci.yml         |  8 +++--
 utils/partial-track.sh | 12 +++++++
 utils/partial_track.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+), 2 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ef5ef1b6..4efb974f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -200,7 +200,12 @@ track:
   interruptible: true
   script:
   - 'sed -i "s/track-tags: False/track-tags: ${BST_TRACK_TAGS}/" project.conf'
-  - ${BST} track --deps all core.bst flatpak-runtimes.bst vm/image.bst boards/pinebook-pro/image.bst
+  - |
+    if [ "${BST_STRICT}" = "--strict" ]; then
+      ${BST} track --deps all core.bst flatpak-runtimes.bst vm/image.bst boards/pinebook-pro/image.bst
+    else
+      ${BST} track $(bash utils/partial-track.sh core.bst flatpak-runtimes.bst vm/image.bst 
boards/pinebook-pro/image.bst)
+    fi
   retry: 2
   # only run on branches targeting master
   rules:
@@ -210,7 +215,6 @@ track:
     paths:
     - project.refs
 
-
 build-gnome-core-x86_64:
   extends: .build-template
   <<: *x86_64
diff --git a/utils/partial-track.sh b/utils/partial-track.sh
new file mode 100644
index 00000000..2c83d6db
--- /dev/null
+++ b/utils/partial-track.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+rm -f project.refs.commit
+python3 ./utils/partial_track.py
+
+if [ -f project.refs.commit ]; then
+    echo --deps none
+    git diff --name-only $(cat project.refs.commit) HEAD | sed '\,elements/,{;s///;p;};d'
+else
+    echo --deps all
+    echo "$@"
+fi
diff --git a/utils/partial_track.py b/utils/partial_track.py
new file mode 100644
index 00000000..f0453086
--- /dev/null
+++ b/utils/partial_track.py
@@ -0,0 +1,96 @@
+import json
+import urllib.request
+import os
+from contextlib import contextmanager
+import datetime
+import zipfile
+import tempfile
+
+class token_handler(urllib.request.BaseHandler):
+    def __init__(self, tokens):
+        self._tokens = tokens
+
+    def http_request(self, req):
+        if req.host in self._tokens:
+            type, value = self._tokens[req.host]
+            req.add_unredirected_header(f'{type}-TOKEN', value)
+        return req
+
+    https_request = http_request
+
+tokens = {}
+job_token = os.environ.get('CI_JOB_TOKEN')
+if job_token:
+    tokens['gitlab.gnome.org'] = ('JOB', job_token)
+else:
+    private_token = os.environ.get('PRIVATE_TOKEN')
+    tokens['gitlab.gnome.org'] = ('PRIVATE', private_token)
+
+opener = urllib.request.build_opener(token_handler(tokens))
+urllib.request.install_opener(opener)
+
+project_id = os.environ.get('CI_PROJECT_ID', '456')
+base_url = f'https://gitlab.gnome.org/api/v4/projects/{project_id}'
+
+@contextmanager
+def download(path, **kwargs):
+    with urllib.request.urlopen(f'{base_url}/{path}') as resp:
+        yield resp
+
+def call(path, **kwargs):
+    query = urllib.parse.urlencode(kwargs)
+    with download(path, **kwargs) as resp:
+        return json.load(resp)
+
+found = []
+for schedule in call('pipeline_schedules'):
+    id = schedule.get('id')
+    if id:
+        schedule = call(f'pipeline_schedules/{id}')
+        if schedule.get('ref') != 'master':
+            continue
+        if not schedule.get('active'):
+            continue
+        for var in schedule.get('variables', []):
+            if var.get('variable_type') == 'env_var' and \
+               var.get('key') == 'BST_STRICT' and \
+               var.get('value') == '--strict':
+                found.append(schedule)
+
+def get_last_build(schedule):
+    up = schedule.get('updated_at')
+    if up:
+        return datetime.datetime.strptime(up, "%Y-%m-%dT%H:%M:%S.%fZ")
+    else:
+        return datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
+
+found = sorted(found, key=get_last_build)
+last_pipeline = found[0].get('last_pipeline', {}).get('id')
+
+found_job = None
+for job in call(f'pipelines/{last_pipeline}/jobs'):
+    if job.get('name') == 'track':
+        found_job = job.get('id')
+        break
+
+job = call(f'jobs/{found_job}')
+
+with tempfile.TemporaryFile() as temp:
+    with download(f'jobs/{found_job}/artifacts') as resp:
+        while True:
+            data = resp.read(4096)
+            if not data:
+                break
+            temp.write(data)
+    temp.seek(0, 0)
+    with zipfile.ZipFile(temp) as zip:
+        with zip.open('project.refs') as refs:
+            with open('project.refs', 'wb') as local_refs:
+                while True:
+                    data = refs.read(4096)
+                    if not data:
+                        break
+                    local_refs.write(data)
+            with open('project.refs.commit', 'w') as commit:
+                commit.write(job.get('commit', {}).get('id'))
+                commit.write('\n')