damned-lies r1294 - in trunk: . stats stats/tests

[claudep svn gnome org]

Author: claudep
Date: Fri Jan  2 22:55:17 2009
New Revision: 1294
URL: http://svn.gnome.org/viewvc/damned-lies?rev=1294&view=rev

Log:
2009-01-02  Claude Paroz  <claude 2xlibre net>

	* stats/models.py: Add identical doc figures detection in update_stats.
	* stats/utils.py: Moved get_fig_stats from models to utils.
	* stats/tests/__init__.py: Add test for identical doc figures detection.
	Fixes bug #566227.

Modified:
   trunk/ChangeLog
   trunk/stats/models.py
   trunk/stats/tests/__init__.py
   trunk/stats/utils.py

Modified: trunk/stats/models.py
==============================================================================
--- trunk/stats/models.py	(original)
+++ trunk/stats/models.py	Fri Jan  2 22:55:17 2009
@@ -19,11 +19,10 @@
 # along with Damned Lies; if not, write to the Free Software Foundation, Inc.,
 # 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-import os, sys, re
+import os, sys, re, hashlib
 import threading
 from datetime import datetime
 from time import tzname
-from itertools import islice
 from django.db import models, connection
 from django.utils.translation import ungettext, ugettext as _, ugettext_noop
 from stats.conf import settings
@@ -344,6 +343,14 @@
                 elif dom.dtype == "doc":
                     if lang not in doclinguas:
                         langstats['errors'].append(("warn", ugettext_noop("DOC_LINGUAS list doesn't include this language.")))
+                    fig_stats = utils.get_fig_stats(outpo)
+                    for fig in fig_stats:
+                        trans_path = os.path.join(domain_path, lang, fig['path'])
+                        if os.access(trans_path, os.R_OK):
+                            fig_file = open(trans_path, 'rb').read()
+                            trans_hash = hashlib.md5(fig_file).hexdigest()
+                            if fig['hash'] == trans_hash:
+                                langstats['errors'].append(("warn", "Figures should not be copied when identical to original (%s)." % trans_path))
 
                 if settings.DEBUG: print >>sys.stderr, lang + ":\n" + str(langstats)
                 # Save in DB
@@ -931,33 +938,13 @@
     
     def get_figures(self):
         if self.figures is None and self.domain.dtype == 'doc':
-            # Extract image strings: beforeline/msgid/msgstr/grep auto output a fourth line 
-            command = "msgcat --no-wrap %(pofile)s| grep -A 1 -B 1 '^msgid \"@@image:'" % { 'pofile': self.po_path() }
-            (status, output) = utils.run_shell_command(command)
-            if status != utils.STATUS_OK:
-                # FIXME: something should be logged here
-                return []
-            lines = output.split('\n')
-            while lines[0][0] != "#":
-                lines = lines[1:] # skip warning messages at the top of the output
-            re_path = re.compile('^msgid \"@@image: \'([^\']*)\'')
-            self.figures = []
-            
-            for i, line in islice(enumerate(lines), 0, None, 4):
-                fig = {}
-                fig['fuzzy'] = line=='#, fuzzy'
-                path_match = re_path.match(lines[i+1])
-                if path_match and len(path_match.groups()):
-                    fig['path'] = path_match.group(1)
-                else:
-                    fig['path'] = '' # This should not happen
-                fig['translated'] = len(lines[i+2])>9 and not fig['fuzzy']
+            self.figures = utils.get_fig_stats(self.po_path())
+            for fig in self.figures:
                 fig['translated_file'] = False
                 if self.language:
                     # Check if a translated figure really exists or if the English one is used
                     if os.path.exists(os.path.join(self.branch.co_path(), self.domain.directory, self.language.locale, fig['path'])):
                         fig['translated_file'] = True
-                self.figures.append(fig)
         return self.figures
     
     def fig_count(self):

Modified: trunk/stats/tests/__init__.py
==============================================================================
--- trunk/stats/tests/__init__.py	(original)
+++ trunk/stats/tests/__init__.py	Fri Jan  2 22:55:17 2009
@@ -18,10 +18,10 @@
 # along with Damned Lies; if not, write to the Free Software Foundation, Inc.,
 # 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-import os, unittest
+import os, shutil, unittest
 import threading
 from django.core import mail
-from stats.models import Module, Domain, Branch, Category, Release, Statistics
+from stats.models import Module, Domain, Branch, Category, Release, Statistics, Information
 
 class ModuleTestCase(unittest.TestCase):
     def setUp(self):
@@ -87,6 +87,14 @@
         self.assertEquals(len(mail.outbox), 1);
         self.assertEquals(mail.outbox[0].subject, "String additions to '%s'")
         self.assertTrue(new_string in mail.outbox[0].message)
+        
+        # Detect warning if translated figure is identical to original figure
+        orig_figure = os.path.join(branch.co_path(), "help", "C", "figures", "gnome-hello.png")
+        shutil.copy(orig_figure, os.path.join(branch.co_path(), "help", "fr", "figures", "gnome-hello.png"))
+        branch.update_stats(force=True)
+        stat = Statistics.objects.get(branch=branch, domain__name='help', language__locale='fr')
+        warn_infos = Information.objects.filter(statistics=stat, type='warn')
+        self.assertEquals(len(warn_infos), 1);
 
         # Delete the branch (removing the repo checkout in the file system)
         checkout_path = branch.co_path()

Modified: trunk/stats/utils.py
==============================================================================
--- trunk/stats/utils.py	(original)
+++ trunk/stats/utils.py	Fri Jan  2 22:55:17 2009
@@ -20,6 +20,7 @@
 # 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 import sys, os, re, time
+from itertools import islice
 from subprocess import Popen, PIPE, STDOUT
 
 from django.utils.translation import ugettext as _, ugettext_noop
@@ -290,6 +291,34 @@
     errors.append(("warn", ugettext_noop("Don't know where to look if this language is actually used, ask the module maintainer.")))
     return errors
 
+def get_fig_stats(pofile):
+    """ Extract image strings from pofile and return a list of figures dict {'path':, 'fuzzy':, 'translated':} """
+    # Extract image strings: beforeline/msgid/msgstr/grep auto output a fourth line 
+    command = "msgcat --no-wrap %(pofile)s| grep -A 1 -B 1 '^msgid \"@@image:'" % locals()
+    (status, output) = run_shell_command(command)
+    if status != STATUS_OK:
+        # FIXME: something should be logged here
+        return []
+    lines = output.split('\n')
+    while lines[0][0] != "#":
+        lines = lines[1:] # skip warning messages at the top of the output
+    re_path = re.compile('^msgid \"@@image: \'([^\']*)\'')
+    re_hash = re.compile('.*md5=(.*)\"')
+    figures = []
+    
+    for i, line in islice(enumerate(lines), 0, None, 4):
+        fig = {'path': '', 'hash': ''}
+        fig['fuzzy'] = line=='#, fuzzy'
+        path_match = re_path.match(lines[i+1])
+        if path_match and len(path_match.groups()):
+            fig['path'] = path_match.group(1)
+        hash_match = re_hash.match(lines[i+1])
+        if hash_match and len(hash_match.groups()):
+            fig['hash'] = hash_match.group(1)
+        fig['translated'] = len(lines[i+2])>9 and not fig['fuzzy']
+        figures.append(fig)
+    return figures
+
 def copy_file(file1, file2):
     try:
         fin = open(file1, "rb")