[damned-lies] Moved run_shell_command to common.utils

[Claude Paroz <claudep src gnome org>]

commit 5d0424c06dfa6da9e84cafb06d92dd73817644fe
Author: Claude Paroz <claude 2xlibre net>
Date:   Tue Apr 24 14:23:30 2018 +0200

    Moved run_shell_command to common.utils

 common/tests.py                             |    4 +-
 common/utils.py                             |   34 +++++++++++++++++++++++++++
 stats/management/commands/migrate-to-git.py |    5 ++-
 stats/models.py                             |   34 +++++++++++++-------------
 stats/tests/tests.py                        |    3 +-
 stats/tests/utils.py                        |    4 +-
 stats/utils.py                              |   28 +---------------------
 stats/views.py                              |    9 ++++---
 vertimus/models.py                          |    4 +-
 9 files changed, 68 insertions(+), 57 deletions(-)
---
diff --git a/common/tests.py b/common/tests.py
index d72d11a..ce14382 100644
--- a/common/tests.py
+++ b/common/tests.py
@@ -3,6 +3,7 @@ from datetime import datetime, timedelta
 from unittest import skipUnless
 from unittest.mock import MagicMock, patch
 
+from django.core.management import call_command
 from django.test import TestCase
 from django.urls import reverse
 from django.utils import translation
@@ -46,8 +47,7 @@ class CommonTest(TestCase):
         response = self.client.get('/register/activate/a_activation_key')
         self.assertContains(response,  'Your account has been activated.')
 
-        from django.core import management
-        management.call_command('run-maintenance')
+        call_command('run-maintenance')
 
         # Testing if the non-activated accounts were deleted
         jn = Person.objects.filter(first_name='John', last_name='Note')
diff --git a/common/utils.py b/common/utils.py
index a1f2ec9..4757fdf 100644
--- a/common/utils.py
+++ b/common/utils.py
@@ -1,3 +1,7 @@
+import logging
+import os
+from subprocess import Popen, PIPE
+
 from django.conf import settings
 from django.core.mail import EmailMessage
 from django.utils.translation import ugettext as _, get_language
@@ -12,6 +16,36 @@ MIME_TYPES = {
     'json': 'application/json',
     'xml':  'text/xml'
 }
+STATUS_OK = 0
+
+
+def run_shell_command(cmd, input_data=None, raise_on_error=False, env=None,
+                      cwd=None, **popen_kwargs):
+    logging.debug(cmd)
+
+    stdin = None
+    if input_data:
+        stdin = PIPE
+    if env is not None:
+        env = dict(os.environ, **env)
+    if cwd is not None:
+        cwd = str(cwd)  # Popen cwd doesn't support Path on Python 3.5
+    shell = not isinstance(cmd, list)
+    pipe = Popen(cmd, shell=shell, stdin=stdin, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd, **popen_kwargs)
+    if input_data:
+        try:
+            pipe.stdin.write(input_data)
+        except IOError as e:
+            if e.errno != errno.EPIPE:
+                raise
+    output, errout = pipe.communicate()
+    status = pipe.returncode
+    logging.debug(output + errout)
+    if raise_on_error and status != STATUS_OK:
+        raise OSError(status, errout if errout else output)
+
+    return (status, output.decode('utf-8'), errout.decode('utf-8'))
+
 
 def lc_sorted(*args, **kwargs):
     """
diff --git a/stats/management/commands/migrate-to-git.py b/stats/management/commands/migrate-to-git.py
index 65bb40d..926501e 100644
--- a/stats/management/commands/migrate-to-git.py
+++ b/stats/management/commands/migrate-to-git.py
@@ -1,7 +1,8 @@
 import shutil
 from django.core.management.base import BaseCommand
 from stats.models import Module, Branch
-from stats import utils
+from common.utils import run_shell_command
+
 
 class Command(BaseCommand):
 
@@ -34,7 +35,7 @@ class Command(BaseCommand):
                 # Checkout branch (other than master)
                 cmd = ['git', 'checkout', '--track', '-b', branch.name, 'origin/%s' % branch.name]
                 try:
-                    utils.run_shell_command(cmd, raise_on_error=True, cwd=branch.co_path)
+                    run_shell_command(cmd, raise_on_error=True, cwd=branch.co_path)
                 except Exception, e:
                     self.stderr.write("Unable to checkout branch '%s' of module '%s': %s" % (branch.name, 
module.name, e))
                     continue
diff --git a/stats/models.py b/stats/models.py
index 042bee3..fcfa1f3 100644
--- a/stats/models.py
+++ b/stats/models.py
@@ -23,7 +23,7 @@ from django.utils import dateformat
 from django.db import models
 
 from common.fields import DictionaryField, JSONField
-from common.utils import is_site_admin
+from common.utils import is_site_admin, run_shell_command
 from stats import utils, signals
 from stats.doap import update_doap_infos
 from people.models import Person
@@ -234,8 +234,8 @@ class Branch(models.Model):
                     shutil.rmtree(str(wdir))
                 else:
                     with ModuleLock(self.module):
-                        utils.run_shell_command(['git', 'checkout', 'master'], cwd=wdir)
-                        utils.run_shell_command(['git', 'branch', '-D', self.name], cwd=wdir)
+                        run_shell_command(['git', 'checkout', 'master'], cwd=wdir)
+                        run_shell_command(['git', 'branch', '-D', self.name], cwd=wdir)
         #To be implemented for hg/bzr
 
         # Remove the pot/po generated files
@@ -513,7 +513,7 @@ class Branch(models.Model):
                         continue
 
                     # msgmerge po with recent pot
-                    utils.run_shell_command([
+                    run_shell_command([
                         'msgmerge', '--previous', '-o', str(outpo), str(pofile), str(potfile)
                     ])
 
@@ -555,7 +555,7 @@ class Branch(models.Model):
             if not self.co_path.exists():
                 return False
             command = "git branch | grep %s" % self.name
-            status, output, errs = utils.run_shell_command(command, cwd=self.co_path)
+            status, output, errs = run_shell_command(command, cwd=self.co_path)
             return output != ""
         elif self.module.vcs_type == 'hg':
             return self.id != None and os.access(str(self.co_path), os.X_OK | os.W_OK)
@@ -610,7 +610,7 @@ class Branch(models.Model):
         # Run command(s)
         logging.debug("Checking '%s.%s' out to '%s'..." % (module_name, self.name, modulepath))
         for working_dir, command in command_list:
-            utils.run_shell_command(command, raise_on_error=True, cwd=working_dir)
+            run_shell_command(command, raise_on_error=True, cwd=working_dir)
 
     def update_repo(self, execute=True):
         """
@@ -639,7 +639,7 @@ class Branch(models.Model):
             command_list.append((modulepath, ['bzr', 'up']))
         if execute:
             for working_dir, command in command_list:
-                utils.run_shell_command(command, raise_on_error=True, cwd=working_dir)
+                run_shell_command(command, raise_on_error=True, cwd=working_dir)
         else:
             return command_list
 
@@ -668,14 +668,14 @@ class Branch(models.Model):
                 shutil.copyfile(str(po_file), str(dest_path))
 
                 # git add file.po
-                utils.run_shell_command(
+                run_shell_command(
                     ['git', 'add', dest_filename], raise_on_error=True, cwd=commit_dir)
                 if not already_exist:
                     # Add locale to LINGUAS
                     linguas_file = commit_dir / "LINGUAS"
                     if linguas_file.exists():
                         utils.insert_locale_in_linguas(linguas_file, locale)
-                        utils.run_shell_command(
+                        run_shell_command(
                             ['git', 'add', 'LINGUAS'], raise_on_error=True, cwd=commit_dir)
                     msg = "Add %s translation" % language.name
                 else:
@@ -684,18 +684,18 @@ class Branch(models.Model):
                 commit_cmd = ['git', 'commit', '-m', msg]
                 if author:
                     commit_cmd.extend(['--author', author])
-                utils.run_shell_command(commit_cmd, raise_on_error=True, cwd=commit_dir)
+                run_shell_command(commit_cmd, raise_on_error=True, cwd=commit_dir)
                 # git push
                 try:
-                    utils.run_shell_command(
+                    run_shell_command(
                         ['git', 'push', 'origin', self.name], raise_on_error=True, cwd=commit_dir)
                 except OSError:
                     # Revert the commit
-                    utils.run_shell_command(
+                    run_shell_command(
                         ['git', 'reset', '--hard', 'origin/%s' % self.name], cwd=commit_dir)
                     raise
                 else:
-                    _, out, _ = utils.run_shell_command(
+                    _, out, _ = run_shell_command(
                         ['git', 'log', '-n1', '--format=oneline'], cwd=commit_dir)
                     commit_hash = out.split()[0] if out else ''
 
@@ -721,15 +721,15 @@ class Branch(models.Model):
         with ModuleLock(self.module):
             self.update_repo()
             commit_dir = self.co_path / domain.directory
-            result = utils.run_shell_command(
+            result = run_shell_command(
                 ['git', 'cherry-pick', '-x', commit_hash], cwd=commit_dir)
             if result[0] == utils.STATUS_OK:
-                utils.run_shell_command(
+                run_shell_command(
                     ['git', 'push', 'origin', self.name], raise_on_error=True, cwd=commit_dir)
                 return True
             else:
                 # Revert
-                utils.run_shell_command(['git', 'cherry-pick', '--abort'], cwd=commit_dir)
+                run_shell_command(['git', 'cherry-pick', '--abort'], cwd=commit_dir)
                 return False
 
 
@@ -856,7 +856,7 @@ class Domain(models.Model):
                 env = {"XGETTEXT_ARGS": "\"--msgid-bugs-address=%s\"" % bugs_url}
 
         if pot_command:
-            status, output, errs = utils.run_shell_command(pot_command, env=env, cwd=podir)
+            status, output, errs = run_shell_command(pot_command, env=env, cwd=podir)
 
         potfile = vcs_path / (self.potbase() + ".pot")
         if not potfile.exists():
diff --git a/stats/tests/tests.py b/stats/tests/tests.py
index 5013346..ea0ab46 100644
--- a/stats/tests/tests.py
+++ b/stats/tests/tests.py
@@ -14,6 +14,7 @@ from django.urls import reverse
 from django.test import TestCase
 from django.test.utils import override_settings
 
+from common.utils import run_shell_command
 from stats.models import (
     Module, Domain, Branch, Release, CategoryName, Statistics, FakeLangStatistics,
     Information,
@@ -369,7 +370,7 @@ class ModuleTestCase(TestCase):
         """
         domain = self.mod.domain_set.get(name='po')
         commit_dir = self.branch.co_path
-        utils.run_shell_command(
+        run_shell_command(
             ['git', 'checkout', '-b', 'gnome-3-18', 'origin/master'], cwd=commit_dir, raise_on_error=True
         )
         branch = Branch.objects.create(module=self.mod, name='gnome-3-18')
diff --git a/stats/tests/utils.py b/stats/tests/utils.py
index 6822e03..48160cc 100644
--- a/stats/tests/utils.py
+++ b/stats/tests/utils.py
@@ -6,12 +6,12 @@ from functools import wraps
 
 from django.conf import settings
 
-from stats import utils
+from common import utils
 
 
 class patch_shell_command:
     """
-    Mock utils.run_shell_commands and gather all passed commands.
+    Mock common.utils.run_shell_commands and gather all passed commands.
     `only` is an optional list of commands to limit mocking to (empty -> all).
     """
     def __init__(self, only=None):
diff --git a/stats/utils.py b/stats/utils.py
index 80baa69..0ee7239 100644
--- a/stats/utils.py
+++ b/stats/utils.py
@@ -7,7 +7,6 @@ import shutil
 import time
 from itertools import islice
 from pathlib import Path
-from subprocess import Popen, PIPE
 from unittest.mock import MagicMock
 
 from translate.tools import pogrep, pocount
@@ -19,7 +18,7 @@ from django.utils.encoding import force_text
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_noop
 
-from common.utils import send_mail
+from common.utils import run_shell_command, send_mail
 from . import potdiff
 
 STATUS_OK = 0
@@ -334,31 +333,6 @@ def ellipsize(val, length):
         val = "%s..." % val[:length]
     return val
 
-def run_shell_command(cmd, input_data=None, raise_on_error=False, env=None, cwd=None, **popen_kwargs):
-    logging.debug(cmd)
-
-    stdin = None
-    if input_data:
-        stdin = PIPE
-    if env is not None:
-        env = dict(os.environ, **env)
-    if cwd is not None:
-        cwd = str(cwd)  # Popen cwd doesn't support Path yet
-    shell = not isinstance(cmd, list)
-    pipe = Popen(cmd, shell=shell, stdin=stdin, stdout=PIPE, stderr=PIPE, env=env, cwd=cwd, **popen_kwargs)
-    if input_data:
-        try:
-            pipe.stdin.write(input_data)
-        except IOError as e:
-            if e.errno != errno.EPIPE:
-                raise
-    output, errout = pipe.communicate()
-    status = pipe.returncode
-    logging.debug(output + errout)
-    if raise_on_error and status != STATUS_OK:
-        raise OSError(status, errout if errout else output)
-
-    return (status, output.decode('utf-8'), errout.decode('utf-8'))
 
 def check_program_presence(prog_name):
     """ Test if prog_name is an available command on the system """
diff --git a/stats/views.py b/stats/views.py
index 1bd7aed..e7a2b86 100644
--- a/stats/views.py
+++ b/stats/views.py
@@ -10,13 +10,14 @@ from django.shortcuts import render, get_object_or_404
 from django.urls import reverse
 from django.utils.translation import ugettext as _
 
-from common.utils import MIME_TYPES, get_user_locale
+from common.utils import MIME_TYPES, get_user_locale, run_shell_command
 from stats.models import Statistics, FakeLangStatistics, Module, ModuleLock, Branch, Category, Release
 from stats.forms import ModuleBranchForm
-from stats import utils
+from stats.utils import sort_object_list
 from languages.models import Language
 from people.models import Person
 
+
 def modules(request, format='html'):
     all_modules = Module.objects.all()
     if format in ('json', 'xml'):
@@ -24,7 +25,7 @@ def modules(request, format='html'):
         return HttpResponse(data, content_type=MIME_TYPES[format])
     context = {
         'pageSection':  "module",
-        'modules': utils.sort_object_list(all_modules, 'get_description'),
+        'modules': sort_object_list(all_modules, 'get_description'),
     }
     return render(request, 'module_list.html', context)
 
@@ -224,7 +225,7 @@ def dynamic_po(request, module_name, domain, branch_name, filename):
         dyn_content += "# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.\n#\n"
 
     command = "msginit --locale=%s --no-translator --input=%s --output-file=-" % (locale, file_path)
-    status, output, err = utils.run_shell_command(command, raise_on_error=True)
+    status, output, err = run_shell_command(command, raise_on_error=True)
     lines = output.split("\n")
     skip_next_line = False
     for i, line in enumerate(lines):
diff --git a/vertimus/models.py b/vertimus/models.py
index 178d893..1692d39 100644
--- a/vertimus/models.py
+++ b/vertimus/models.py
@@ -10,10 +10,10 @@ from django.dispatch import receiver
 from django.urls import reverse
 from django.utils.translation import override, ugettext, ugettext_noop, ugettext_lazy as _
 
-from common.utils import send_mail
+from common.utils import run_shell_command, send_mail
 from stats.models import Branch, Domain, Statistics, PoFile
 from stats.signals import pot_has_changed
-from stats.utils import STATUS_OK, run_shell_command, is_po_reduced, po_grep
+from stats.utils import STATUS_OK, is_po_reduced, po_grep
 from languages.models import Language
 from people.models import Person