[damned-lies: 6/9] Consider possible reduced pot file in vertimus view (diff/merge)

[Claude Paroz <claudep src gnome org>]

commit 993685bf12b648ebe3f53221af129fe0c20293a0
Author: Claude Paroz <claude 2xlibre net>
Date:   Sat Mar 5 21:00:12 2011 +0100

    Consider possible reduced pot file in vertimus view (diff/merge)

 stats/models.py    |   97 +++++++++++++++++++++++++++++----------------------
 stats/utils.py     |   20 +++++++++++
 vertimus/models.py |    7 +++-
 vertimus/views.py  |    4 ++-
 4 files changed, 83 insertions(+), 45 deletions(-)
---
diff --git a/stats/models.py b/stats/models.py
index 14dd692..2000c74 100644
--- a/stats/models.py
+++ b/stats/models.py
@@ -394,11 +394,11 @@ class Branch(models.Model):
 
                 # Prepare statistics object
                 try:
-                    stat = Statistics.objects.get(language=None, branch=self, domain=dom)
-                    Information.objects.filter(statistics=stat).delete() # Reset errors
+                    pot_stat = Statistics.objects.get(language=None, branch=self, domain=dom)
+                    Information.objects.filter(statistics=pot_stat).delete() # Reset errors
                 except Statistics.DoesNotExist:
-                    stat = Statistics(language=None, branch=self, domain=dom)
-                    stat.save()
+                    pot_stat = Statistics(language=None, branch=self, domain=dom)
+                    pot_stat.save()
 
                 # 4. Compare with old pot files, various checks
                 # *****************************
@@ -411,9 +411,21 @@ class Branch(models.Model):
                         errors.append(("error", ugettext_noop("Can't generate POT file, using old one.")))
                     else:
                         errors.append(("error", ugettext_noop("Can't generate POT file, statistics aborted.")))
-                        stat.set_errors(errors)
+                        pot_stat.set_errors(errors)
                         continue
 
+                # 5. Generate pot stats and update DB
+                # ***********************************
+                pot_stats = utils.po_file_stats(potfile, False)
+                errors.extend(pot_stats['errors'])
+                if potfile != previous_pot and not utils.copy_file(potfile, previous_pot):
+                    errors.append(('error', ugettext_noop("Can't copy new POT file to public location.")))
+
+                pot_stat.set_translation_stats(previous_pot, untranslated=int(pot_stats['untranslated']), num_figures=int(pot_stats['num_figures']))
+                pot_stat.set_errors(errors)
+
+                # 6. Check if pot changed
+                # ***********************************
                 changed_status = utils.CHANGED_WITH_ADDITIONS
 
                 if os.access(previous_pot, os.R_OK):
@@ -425,17 +437,7 @@ class Branch(models.Model):
                     if changed_status != utils.NOT_CHANGED:
                         signals.pot_has_changed.send(sender=self, potfile=potfile, branch=self, domain=dom)
 
-                # 5. Generate pot stats and update DB
-                # ***********************************
-                pot_stats = utils.po_file_stats(potfile, False)
-                errors.extend(pot_stats['errors'])
-                if potfile != previous_pot and not utils.copy_file(potfile, previous_pot):
-                    errors.append(('error', ugettext_noop("Can't copy new POT file to public location.")))
-
-                stat.set_translation_stats(previous_pot, untranslated=int(pot_stats['untranslated']), num_figures=int(pot_stats['num_figures']))
-                stat.set_errors(errors)
-
-                # 6. Update language po files and update DB
+                # 7. Update language po files and update DB
                 # *****************************************
                 command = "msgmerge --previous -o %(outpo)s %(pofile)s %(potfile)s"
                 stats_with_ext_errors = Statistics.objects.filter(branch=self, domain=dom, information__type__endswith='-ext')
@@ -1251,7 +1253,7 @@ class Statistics(models.Model):
         if not self.is_pot_stats() and not potfile:
             return "%s.%s.%s.%spo" % (self.domain.potbase(), self.branch.name, self.language.locale, reduced and "reduced." or "")
         else:
-            return "%s.%s.pot" % (self.domain.potbase(), self.branch.name)
+            return "%s.%s.%spot" % (self.domain.potbase(), self.branch.name, reduced and "reduced." or "")
 
     def pot_text(self):
         pot_size = self.full_po.pot_size()
@@ -1309,12 +1311,12 @@ class Statistics(models.Model):
         """ Return the Web interface path of file on remote vcs """
         return utils.url_join(self.branch.get_vcs_web_url(), self.domain.directory)
 
-    def po_path(self, potfile=False):
+    def po_path(self, potfile=False, reduced=False):
         """ Return path of po file on local filesystem """
         subdir = ""
         if self.domain.dtype == "doc":
             subdir = "docs"
-        return os.path.join(settings.POTDIR, self.module_name()+'.'+self.branch.name, subdir, self.filename(potfile))
+        return os.path.join(settings.POTDIR, self.module_name()+'.'+self.branch.name, subdir, self.filename(potfile, reduced))
 
     def po_url(self, potfile=False, reduced=False):
         """ Return URL of po file, e.g. for downloading the file """
@@ -1337,31 +1339,42 @@ class Statistics(models.Model):
         self.full_po.num_figures = num_figures
         self.full_po.updated = datetime.now()
         self.full_po.save()
-        # Try to compute a reduced po file
-        if self.domain.dtype == "ui" and (self.full_po.fuzzy + self.full_po.untranslated) > 0:
-            # Generate partial_po and store partial stats
-            part_po_path = self.full_po.path[:-3] + ".reduced.po"
-            cmd = "pogrep --invert-match --header --search=locations \"gschema.xml.in\" %(full_po)s %(part_po)s" % {
-                'full_po': self.full_po.path,
-                'part_po': part_po_path,
-            }
-            utils.run_shell_command(cmd)
-            part_stats = utils.po_file_stats(part_po_path, msgfmt_checks=False, count_images=False)
-            if part_stats['translated'] + part_stats['fuzzy'] + part_stats['untranslated'] == translated + fuzzy + untranslated:
-                # No possible gain, set part_po = full_po so it is possible to compute complete stats at database level
+        if self.domain.dtype == "ui":
+            def part_po_equals_full_po():
+                if self.part_po and self.part_po != self.full_po:
+                    self.part_po.delete()
                 self.part_po = self.full_po
                 self.save()
-                os.remove(part_po_path)
-                return
-            if not self.part_po:
-                self.part_po = PoFile.objects.create(path=part_po_path)
-                self.save()
-            self.part_po.path = part_po_path
-            self.part_po.translated = part_stats['translated']
-            self.part_po.fuzzy = part_stats['fuzzy']
-            self.part_po.untranslated = part_stats['untranslated']
-            self.part_po.updated = datetime.now()
-            self.part_po.save()
+            # Try to compute a reduced po file
+            if (self.full_po.fuzzy + self.full_po.untranslated) > 0:
+                # Generate partial_po and store partial stats
+                if self.full_po.path.endswith('.pot'):
+                    part_po_path = self.full_po.path[:-3] + "reduced.pot"
+                else:
+                    part_po_path = self.full_po.path[:-3] + ".reduced.po"
+                cmd = "pogrep --invert-match --header --search=locations \"gschema.xml.in\" %(full_po)s %(part_po)s" % {
+                    'full_po': self.full_po.path,
+                    'part_po': part_po_path,
+                }
+                utils.run_shell_command(cmd)
+                part_stats = utils.po_file_stats(part_po_path, msgfmt_checks=False, count_images=False)
+                if part_stats['translated'] + part_stats['fuzzy'] + part_stats['untranslated'] == translated + fuzzy + untranslated:
+                    # No possible gain, set part_po = full_po so it is possible to compute complete stats at database level
+                    part_po_equals_full_po()
+                    os.remove(part_po_path)
+                    return
+                utils.add_custom_header(part_po_path, "X-DamnedLies-Scope", "partial")
+                if not self.part_po:
+                    self.part_po = PoFile.objects.create(path=part_po_path)
+                    self.save()
+                self.part_po.path = part_po_path
+                self.part_po.translated = part_stats['translated']
+                self.part_po.fuzzy = part_stats['fuzzy']
+                self.part_po.untranslated = part_stats['untranslated']
+                self.part_po.updated = datetime.now()
+                self.part_po.save()
+            else:
+                part_po_equals_full_po()
 
     def set_errors(self, errors):
         for err in errors:
diff --git a/stats/utils.py b/stats/utils.py
index 072c3ac..5ac82ac 100644
--- a/stats/utils.py
+++ b/stats/utils.py
@@ -367,6 +367,26 @@ def get_fig_stats(pofile):
         figures.append(fig)
     return figures
 
+def add_custom_header(po_path, header, value):
+    """ Add a custom po file header """
+    grep_cmd = """grep "%s" %s""" % (header, po_path)
+    status = 1
+    last_headers = ["Content-Transfer-Encoding", "Plural-Forms"]
+    while status != 0 and last_headers != []:
+        (status, output, errs) = run_shell_command(grep_cmd)
+        if status != 0:
+            # Try to add header
+            cmd = '''sed -i '/^\"%s/ a\\"%s: %s\\\\n"' %s''' % (last_headers.pop(), header, value, po_path)
+            (stat, out, err) = run_shell_command(cmd)
+    if status == 0 and not "%s: %s" % (header, value) in output:
+        # Set header value
+        cmd = '''sed -i '/^\"%s/ c\\"%s: %s\\\\n"' %s''' % (header, header, value, po_path)
+        (stat, out, err) = run_shell_command(cmd)
+
+def is_po_reduced(file_path):
+    status, output, errs = run_shell_command("""grep "X-DamnedLies-Scope: partial" %s""" % file_path)
+    return (status == 0)
+
 def copy_file(file1, file2):
     try:
         fin = open(file1, "rb")
diff --git a/vertimus/models.py b/vertimus/models.py
index b5db15c..e807d52 100644
--- a/vertimus/models.py
+++ b/vertimus/models.py
@@ -31,7 +31,7 @@ from django.utils.translation import get_language, activate, ugettext, ugettext_
 
 from stats.models import Branch, Domain, Statistics, PoFile
 from stats.signals import pot_has_changed
-from stats.utils import run_shell_command
+from stats.utils import run_shell_command, is_po_reduced
 from languages.models import Language
 from people.models import Person
 
@@ -372,6 +372,9 @@ class ActionDb(models.Model):
     def merge_file_with_pot(self, pot_file):
         """Merge the uploaded translated file with current pot."""
         if self.file:
+            pot_file_reduced = pot_file[:-3] + "reduced.pot"
+            if is_po_reduced(self.file) and os.path.exists(pot_file_reduced):
+                pot_file = pot_file_reduced
             command = "msgmerge --previous -o %(out_po)s %(po_file)s %(pot_file)s" % {
                 'out_po': self.file.path[:-3] + ".merged.po",
                 'po_file': self.file.path,
@@ -828,7 +831,7 @@ def merge_uploaded_file(sender, instance, **kwargs):
             stat = Statistics.objects.get(branch=instance.state_db.branch, domain=instance.state_db.domain, language=None)
         except Statistics.DoesNotExist:
             return
-        potfile = stat.po_path()
+        potfile = stat.po_path(reduced=is_po_reduced(instance.file.path))
         instance.merge_file_with_pot(potfile)
 post_save.connect(merge_uploaded_file, sender=ActionDb)
 
diff --git a/vertimus/views.py b/vertimus/views.py
index 247df60..c618b57 100644
--- a/vertimus/views.py
+++ b/vertimus/views.py
@@ -26,6 +26,7 @@ from django.template import RequestContext
 from django.core import urlresolvers
 
 from stats.models import Statistics, Module, Branch, Domain, Language
+from stats.utils import is_po_reduced
 from vertimus.models import StateDb, ActionDb, ActionDbArchived, ActionAbstract
 from vertimus.forms import ActionForm
 
@@ -152,6 +153,7 @@ def vertimus_diff(request, action_id_1, action_id_2, level):
     state = action_1.state
 
     file_path_1 = action_1.merged_file()['path'] or action_1.file.path
+    reduced = is_po_reduced(file_path_1)
 
     try:
         content_1 = [l.decode('utf-8') for l in open(file_path_1, 'U').readlines()]
@@ -184,7 +186,7 @@ def vertimus_diff(request, action_id_1, action_id_2, level):
             except Statistics.DoesNotExist:
                 stats = get_object_or_404(Statistics, branch=state.branch, domain=state.domain, language=None)
                 descr_2 = _("Latest POT file")
-            file_path_2 = stats.po_path()
+            file_path_2 = stats.po_path(reduced=reduced)
     try:
         content_2 = [l.decode('utf-8') for l in open(file_path_2, 'U').readlines()]
     except UnicodeDecodeError: