[gnome-continuous-yocto/gnomeostree-3.28-rocko: 1840/8267] Allow for simultaneous do_rootfs tasks with rpm

[Emmanuele Bassi <ebassi src gnome org>]

commit 6b66e9317f4ec3a69f98f29836aafa35b52f3fc7
Author: Stephano Cetola <stephano cetola linux intel com>
Date:   Wed Aug 10 13:03:16 2016 -0700

    Allow for simultaneous do_rootfs tasks with rpm
    
    Give each rootfs its own RPM channel to use.  This puts the RPM metadata
    in a private subdirectory of $WORKDIR, rather than living in DEPLOY_DIR
    where other tasks may race with it.
    
    This allows us to reduce the time that the rpm.lock is held to only the
    time needed to hardlink the RPMs, allowing the majority of the rootfs
    operation to run in parallel.
    
    Also, this fixes the smart tests by generating an index for all packages
    at the time of the test, rather than using the one provided by the
    rootfs process.
    
    Original credit for the enhancement should go to Steven Walter
    stevenrwalter gmail com.
    
    (From OE-Core rev: a92c196449c516fe51786d429078bbb1213bb029)
    
    Signed-off-by: Stephano Cetola <stephano cetola linux intel com>
    Signed-off-by: Richard Purdie <richard purdie linuxfoundation org>

 meta/classes/rootfs_rpm.bbclass |    5 ----
 meta/lib/oe/package_manager.py  |   17 +++++++++++++--
 meta/lib/oeqa/runtime/smart.py  |   41 ++++++++++++++++++++++++++++++++++++++-
 3 files changed, 54 insertions(+), 9 deletions(-)
---
diff --git a/meta/classes/rootfs_rpm.bbclass b/meta/classes/rootfs_rpm.bbclass
index 38b3c99..37730a7 100644
--- a/meta/classes/rootfs_rpm.bbclass
+++ b/meta/classes/rootfs_rpm.bbclass
@@ -24,11 +24,6 @@ do_populate_sdk[depends] += "${RPMROOTFSDEPENDS}"
 do_rootfs[recrdeptask] += "do_package_write_rpm"
 do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
 
-# RPM doesn't work with multiple rootfs generation at once due to collisions in the use of files 
-# in ${DEPLOY_DIR_RPM}. This can be removed if package_update_index_rpm can be called concurrently
-do_rootfs[lockfiles] += "${DEPLOY_DIR_RPM}/rpm.lock"
-do_populate_sdk[lockfiles] += "${DEPLOY_DIR_RPM}/rpm.lock"
-
 python () {
     if d.getVar('BUILD_IMAGES_FROM_FEEDS', True):
         flags = d.getVarFlag('do_rootfs', 'recrdeptask', True)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 47f6831..2802254 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -9,6 +9,7 @@ import collections
 import bb
 import tempfile
 import oe.utils
+import oe.path
 import string
 from oe.gpg_sign import get_signer
 
@@ -175,7 +176,7 @@ class RpmIndexer(Indexer):
             dbpath = os.path.join(self.d.getVar('WORKDIR', True), 'rpmdb', arch)
             if os.path.exists(dbpath):
                 bb.utils.remove(dbpath, True)
-            arch_dir = os.path.join(self.deploy_dir, arch)
+            arch_dir = os.path.join(self.d.getVar('WORKDIR', True), 'rpms', arch)
             if not os.path.isdir(arch_dir):
                 continue
 
@@ -1010,8 +1011,18 @@ class RpmPM(PackageManager):
         ch_already_added = []
         for canonical_arch in platform_extra:
             arch = canonical_arch.split('-')[0]
-            arch_channel = os.path.join(self.deploy_dir, arch)
-            if os.path.exists(arch_channel) and not arch in ch_already_added:
+            arch_channel = os.path.join(self.d.getVar('WORKDIR', True), 'rpms', arch)
+            oe.path.remove(arch_channel)
+            deploy_arch_dir = os.path.join(self.deploy_dir, arch)
+            if not os.path.exists(deploy_arch_dir):
+                    continue
+
+            lockfilename = self.d.getVar('DEPLOY_DIR_RPM', True) + "/rpm.lock"
+            lf = bb.utils.lockfile(lockfilename, False)
+            oe.path.copyhardlinktree(deploy_arch_dir, arch_channel)
+            bb.utils.unlockfile(lf)
+
+            if not arch in ch_already_added:
                 bb.note('Adding Smart channel %s (%s)' %
                         (arch, channel_priority))
                 self._invoke_smart('channel --add %s type=rpm-md baseurl=%s -y'
diff --git a/meta/lib/oeqa/runtime/smart.py b/meta/lib/oeqa/runtime/smart.py
index c7a5753..c8ba433 100644
--- a/meta/lib/oeqa/runtime/smart.py
+++ b/meta/lib/oeqa/runtime/smart.py
@@ -1,5 +1,7 @@
 import unittest
 import re
+import oe
+import subprocess
 from oeqa.oetest import oeRuntimeTest, skipModule
 from oeqa.utils.decorators import *
 from oeqa.utils.httpserver import HTTPService
@@ -53,9 +55,46 @@ class SmartBasicTest(SmartTest):
 class SmartRepoTest(SmartTest):
 
     @classmethod
+    def create_index(self, arg):
+        index_cmd = arg
+        try:
+            bb.note("Executing '%s' ..." % index_cmd)
+            result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
+        except subprocess.CalledProcessError as e:
+            return("Index creation command '%s' failed with return code %d:\n%s" %
+                    (e.cmd, e.returncode, e.output.decode("utf-8")))
+        if result:
+            bb.note(result)
+        return None
+
+    @classmethod
     def setUpClass(self):
         self.repolist = []
-        self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), 
oeRuntimeTest.tc.target.server_ip)
+
+        # Index RPMs
+        rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo")
+        index_cmds = []
+        rpm_dirs_found = False
+        archs = (oeRuntimeTest.tc.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', 
'_').split()
+        for arch in archs:
+            rpm_dir = os.path.join(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR_RPM', True), arch)
+            idx_path = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 'rpm', arch)
+            db_path = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 'rpmdb', arch)
+            if not os.path.isdir(rpm_dir):
+                continue
+            if os.path.exists(db_path):
+                bb.utils.remove(dbpath, True)
+            lockfilename = oeRuntimeTest.tc.d.getVar('DEPLOY_DIR_RPM', True) + "/rpm.lock"
+            lf = bb.utils.lockfile(lockfilename, False)
+            oe.path.copyhardlinktree(rpm_dir, idx_path)
+            bb.utils.unlockfile(lf)
+            index_cmds.append("%s --dbpath %s --update -q %s" % (rpm_createrepo, db_path, idx_path))
+            rpm_dirs_found = True
+         # Create repodata¬
+        result = oe.utils.multiprocess_exec(index_cmds, self.create_index)
+        if result:
+            bb.fatal('%s' % ('\n'.join(result)))
+        self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 
oeRuntimeTest.tc.target.server_ip)
         self.repo_server.start()
 
     @classmethod