[damned-lies] Add its gettext extraction support

[Claude Paroz <claudep src gnome org>]

commit bf56f5c84690cd336bcb4f5321d0e82f841ac73a
Author: Claude Paroz <claude 2xlibre net>
Date:   Wed Sep 7 11:29:36 2016 +0200

    Add its gettext extraction support
    
    Fixes #755466. This needs gettext >= 0.19.8 to work properly.

 damnedlies/settings.py |   10 ++++++++++
 stats/models.py        |    5 ++++-
 stats/utils.py         |   43 ++++++++++++++++++++++++++++++++++++-------
 3 files changed, 50 insertions(+), 8 deletions(-)
---
diff --git a/damnedlies/settings.py b/damnedlies/settings.py
index 84d8dde..a3a65c5 100644
--- a/damnedlies/settings.py
+++ b/damnedlies/settings.py
@@ -159,6 +159,16 @@ LOGIN_REDIRECT_URL = '/'
 # Members of this group can edit all team's details and change team coordinatorship
 ADMIN_GROUP = ''
 
+# https://www.gnu.org/software/gettext/manual/html_node/Preparing-ITS-Rules.html#Preparing-ITS-Rules
+GETTEXT_ITS_DATA = {
+    'gtk+': [os.path.join('gtk', 'gtkbuilder.its'), os.path.join('gtk', 'gtkbuilder.loc')],
+    'glib': [os.path.join('gio', 'gschema.its'), os.path.join('gio', 'gschema.loc')],
+    'gnome-control-center': [
+        os.path.join('panels', 'keyboard', 'gnome-keybindings.its'),
+        os.path.join('panels', 'keyboard', 'gnome-keybindings.loc'),
+    ]
+}
+
 try:
     from .local_settings import *
 except ImportError:
diff --git a/stats/models.py b/stats/models.py
index 9f0e3e0..14ce287 100644
--- a/stats/models.py
+++ b/stats/models.py
@@ -817,6 +817,7 @@ class Domain(models.Model):
 
         vcs_path = os.path.join(current_branch.co_path(), self.directory)
         pot_command = self.pot_method
+        env = None
         podir = vcs_path
         if not self.pot_method:  # default is intltool
             pot_command = ['intltool-update', '-g', self.potbase(), '-p']
@@ -828,6 +829,9 @@ class Domain(models.Model):
                            '--add-comments',
                            '--output', '%s.pot' % self.potbase(),
                           ]
+            if not os.path.exists(utils.ITS_DATA_DIR):
+                utils.collect_its_data()
+            env = {'GETTEXTDATADIRS': os.path.dirname(utils.ITS_DATA_DIR)}
             # Parse and use content from: "XGETTEXT_OPTIONS = --keyword=_ --keyword=N_"
             kwds_vars = utils.read_makefile_variable([vcs_path], 'XGETTEXT_OPTIONS', 
makefile_name='Makevars')
             if kwds_vars:
@@ -850,7 +854,6 @@ class Domain(models.Model):
             podir = "."
             vcs_path = "./po"
 
-        env = None
         if 'intltool-update' in pot_command:
             env = {"XGETTEXT_ARGS": "\"--msgid-bugs-address=%s\"" % self.module.get_bugs_enter_url()}
 
diff --git a/stats/utils.py b/stats/utils.py
index 9454575..ddd38a3 100644
--- a/stats/utils.py
+++ b/stats/utils.py
@@ -18,12 +18,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
-import sys, os, re, time
+import errno
 import hashlib
 import logging
+import os
+import re
+import shutil
+import sys
+import time
 from itertools import islice
 from subprocess import Popen, PIPE
-import errno
 
 try:
     from translate.tools import pogrep, pocount
@@ -50,6 +54,7 @@ CHANGED_WITH_ADDITIONS  = 2
 CHANGED_NO_ADDITIONS    = 3
 
 ITSTOOL_PATH = getattr(settings, 'ITSTOOL_PATH', '')
+ITS_DATA_DIR = os.path.join(settings.SCRATCHDIR, 'data', 'its')
 
 class DocFormat(object):
     itstool_regex = re.compile("^msgid \"external ref=\'(?P<path>[^\']*)\' md5=\'(?P<hash>[^\']*)\'\"")
@@ -122,21 +127,20 @@ def ellipsize(val, length):
         val = "%s..." % val[:length]
     return val
 
-def run_shell_command(cmd, input_data=None, raise_on_error=False, **popen_kwargs):
+def run_shell_command(cmd, input_data=None, raise_on_error=False, env=None, **popen_kwargs):
     logging.debug(cmd)
 
     stdin = None
     if input_data:
         stdin = PIPE
-    if popen_kwargs.get('env'):
-        os.environ.update(popen_kwargs['env'])
-        popen_kwargs['env'] = os.environ
+    if env is not None:
+        env = dict(os.environ, **env)
     shell = not isinstance(cmd, list)
     if isinstance(cmd, six.text_type):
         cmd = cmd.encode('utf-8')
     elif isinstance(cmd, list):
         cmd = [c.encode('utf-8') for c in cmd]
-    pipe = Popen(cmd, shell=shell, stdin=stdin, stdout=PIPE, stderr=PIPE, **popen_kwargs)
+    pipe = Popen(cmd, shell=shell, stdin=stdin, stdout=PIPE, stderr=PIPE, env=env, **popen_kwargs)
     if input_data:
         try:
             pipe.stdin.write(force_bytes(input_data))
@@ -458,6 +462,31 @@ def get_doc_linguas(module_path, po_path):
     return {'langs': linguas.split(),
             'error': ugettext_noop("DOC_LINGUAS list doesn't include this language.") }
 
+
+def collect_its_data():
+    """
+    Fill a data directory with *.loc/*its files needed by gettext to extract
+    XML strings for GNOME modules .
+    """
+    from .models import Branch, ModuleLock
+
+    if not os.path.exists(ITS_DATA_DIR):
+        os.makedirs(ITS_DATA_DIR)
+    data_to_collect = getattr(settings, 'GETTEXT_ITS_DATA', {})
+    for module, files in data_to_collect.items():
+        try:
+            branch = Branch.objects.get(module__name=module, name='master')
+        except Branch.DoesNotExist:
+            logging.error("Unable to find the master branch of module '%s'" % module)
+            continue
+        with ModuleLock(branch.module):
+            branch.checkout()
+            for file_path in files:
+                src = os.path.join(branch.co_path(), file_path)
+                dest = os.path.join(ITS_DATA_DIR, os.path.basename(file_path))
+                shutil.copyfile(src, dest)
+
+
 def get_fig_stats(pofile, doc_format):
     """ Extract image strings from pofile and return a list of figures dict:
         [{'path':, 'hash':, 'fuzzy':, 'translated':}, ...] """