[gnome-continuous-yocto/gnomeostree-3.28-rocko: 7603/8267] package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages
- From: Emmanuele Bassi <ebassi src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-continuous-yocto/gnomeostree-3.28-rocko: 7603/8267] package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages
- Date: Sun, 17 Dec 2017 06:28:48 +0000 (UTC)
commit f6f6b3e5d5a77010d5716b2cb980b271ee5dc8e8
Author: Leonardo Sandoval <leonardo sandoval gonzalez linux intel com>
Date: Mon Sep 4 14:35:48 2017 -0700
package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages
Current implementation does not handle possible exceptions coming from child
processes, the latter responsible for creating packages. With the aim to have more
control, use pipes to communicate exceptions and stop package creation in case
of failure.
Helps to debug [YOCTO #12012].
(From OE-Core rev: 11350a67ba137f560d04aa643ff500a7ff112c73)
Signed-off-by: Leonardo Sandoval <leonardo sandoval gonzalez linux intel com>
Signed-off-by: Ross Burton <ross burton intel com>
Signed-off-by: Richard Purdie <richard purdie linuxfoundation org>
meta/classes/package_deb.bbclass | 36 +++++++++++++++++++++++++++++++++---
meta/classes/package_ipk.bbclass | 36 +++++++++++++++++++++++++++++++++---
2 files changed, 66 insertions(+), 6 deletions(-)
---
diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass
index 3060534..5d29793 100644
--- a/meta/classes/package_deb.bbclass
+++ b/meta/classes/package_deb.bbclass
@@ -41,7 +41,29 @@ def debian_arch_map(arch, tune):
return arch
python do_package_deb () {
- from multiprocessing import Process
+
+ import multiprocessing
+ import traceback
+
+ class DebianWritePkgProcess(multiprocessing.Process):
+ def __init__(self, *args, **kwargs):
+ multiprocessing.Process.__init__(self, *args, **kwargs)
+ self._pconn, self._cconn = multiprocessing.Pipe()
+ self._exception = None
+
+ def run(self):
+ try:
+ multiprocessing.Process.run(self)
+ self._cconn.send(None)
+ except Exception as e:
+ tb = traceback.format_exc()
+ self._cconn.send((e, tb))
+
+ @property
+ def exception(self):
+ if self._pconn.poll():
+ self._exception = self._pconn.recv()
+ return self._exception
oldcwd = os.getcwd()
@@ -56,20 +78,28 @@ python do_package_deb () {
max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
launched = []
+ error = None
pkgs = packages.split()
- while pkgs:
+ while not error and pkgs:
if len(launched) < max_process:
- p = Process(target=deb_write_pkg, args=(pkgs.pop(), d))
+ p = DebianWritePkgProcess(target=deb_write_pkg, args=(pkgs.pop(), d))
p.start()
launched.append(p)
for q in launched:
# The finished processes are joined when calling is_alive()
if not q.is_alive():
launched.remove(q)
+ if q.exception:
+ error, traceback = q.exception
+ break
+
for p in launched:
p.join()
os.chdir(oldcwd)
+
+ if error:
+ raise error
}
do_package_deb[vardeps] += "deb_write_pkg"
do_package_deb[vardepsexclude] = "BB_NUMBER_THREADS"
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index ec90996..8439cda 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -17,7 +17,29 @@ OPKG_ARGS += "${@['', '--add-exclude ' + ' --add-exclude '.join((d.getVar('PACKA
OPKGLIBDIR = "${localstatedir}/lib"
python do_package_ipk () {
- from multiprocessing import Process
+ import multiprocessing
+ import traceback
+
+ class IPKWritePkgProcess(multiprocessing.Process):
+ def __init__(self, *args, **kwargs):
+ multiprocessing.Process.__init__(self, *args, **kwargs)
+ self._pconn, self._cconn = multiprocessing.Pipe()
+ self._exception = None
+
+ def run(self):
+ try:
+ multiprocessing.Process.run(self)
+ self._cconn.send(None)
+ except Exception as e:
+ tb = traceback.format_exc()
+ self._cconn.send((e, tb))
+
+ @property
+ def exception(self):
+ if self._pconn.poll():
+ self._exception = self._pconn.recv()
+ return self._exception
+
oldcwd = os.getcwd()
@@ -41,20 +63,28 @@ python do_package_ipk () {
max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
launched = []
+ error = None
pkgs = packages.split()
- while pkgs:
+ while not error and pkgs:
if len(launched) < max_process:
- p = Process(target=ipk_write_pkg, args=(pkgs.pop(), d))
+ p = IPKWritePkgProcess(target=ipk_write_pkg, args=(pkgs.pop(), d))
p.start()
launched.append(p)
for q in launched:
# The finished processes are joined when calling is_alive()
if not q.is_alive():
launched.remove(q)
+ if q.exception:
+ error, traceback = q.exception
+ break
+
for p in launched:
p.join()
os.chdir(oldcwd)
+
+ if error:
+ raise error
}
do_package_ipk[vardeps] += "ipk_write_pkg"
do_package_ipk[vardepsexclude] = "BB_NUMBER_THREADS"
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]