[damned-lies] Replace legacy structured fields by Django's JSONField

[Claude Paroz <claudep src gnome org>]

commit 5f4a5dcd57cfce0a6a94d52be95f20463d0aa0ad
Author: Claude Paroz <claude 2xlibre net>
Date:   Sat Apr 17 12:49:50 2021 +0200

    Replace legacy structured fields by Django's JSONField

 common/fields.py                                   |  3 +++
 stats/fixtures/sample_data.json                    |  6 +++--
 stats/migrations/0001_initial.py                   |  2 +-
 stats/migrations/0018_new_jsonfields.py            | 31 ++++++++++++++++++++++
 stats/migrations/0019_migrate_old_custom_files.py  | 23 ++++++++++++++++
 stats/migrations/0020_remove_pofile_figures_old.py | 19 +++++++++++++
 stats/models.py                                    |  7 ++---
 7 files changed, 85 insertions(+), 6 deletions(-)
---
diff --git a/common/fields.py b/common/fields.py
index 9023deb3..5158502d 100644
--- a/common/fields.py
+++ b/common/fields.py
@@ -1,3 +1,6 @@
+# Unused since April 2021
+# Keep this file until migrations are squashed and don't refer any more to these fields
+
 # From: https://djangosnippets.org/snippets/1979/
 import json
 
diff --git a/stats/fixtures/sample_data.json b/stats/fixtures/sample_data.json
index 1082cfe6..e273a2b2 100644
--- a/stats/fixtures/sample_data.json
+++ b/stats/fixtures/sample_data.json
@@ -629,7 +629,9 @@
   "fuzzy_words": 0,
   "untranslated": 20,
   "translated_words": 0,
-  "figures": "[{\"path\": \"figures/gnome-hello-new.png\", \"hash\": \"8a1fcc6f46a22a1f500cfef9ca51b481\"}, 
{\"path\": \"figures/gnome-hello-logo.png\", \"hash\": \"1ae47b7a7c4fbeb1f6bb72c0cf18d389\"}]",
+  "figures": [
+        {"path": "figures/gnome-hello-new.png", "hash": "8a1fcc6f46a22a1f500cfef9ca51b481"},
+        {"path": "figures/gnome-hello-logo.png", "hash": "1ae47b7a7c4fbeb1f6bb72c0cf18d389"}],
   "fuzzy": 0,
   "translated": 0,
   "path": ""
@@ -644,7 +646,7 @@
   "fuzzy_words": 0,
   "untranslated": 0,
   "translated_words": 0,
-  "figures": "[{\"translated\": false, \"path\": \"figures/gnome-hello-new.png\", \"hash\": 
\"8a1fcc6f46a22a1f500cfef9ca51b481\", \"fuzzy\": true}, {\"translated\": false, \"path\": 
\"figures/gnome-hello-logo.png\", \"hash\": \"1ae47b7a7c4fbeb1f6bb72c0cf18d389\", \"fuzzy\": false}]",
+  "figures": [{"translated": false, "path": "figures/gnome-hello-new.png", "hash": 
"8a1fcc6f46a22a1f500cfef9ca51b481", "fuzzy": true}, {"translated": false, "path": 
"figures/gnome-hello-logo.png", "hash": "1ae47b7a7c4fbeb1f6bb72c0cf18d389", "fuzzy": false}],
   "fuzzy": 0,
   "translated": 20,
   "path": ""
diff --git a/stats/migrations/0001_initial.py b/stats/migrations/0001_initial.py
index 611069aa..6afaecfa 100644
--- a/stats/migrations/0001_initial.py
+++ b/stats/migrations/0001_initial.py
@@ -88,7 +88,7 @@ class Migration(migrations.Migration):
                 ('bugs_base', models.CharField(max_length=50, null=True, blank=True)),
                 ('bugs_product', models.CharField(max_length=50, null=True, blank=True)),
                 ('bugs_component', models.CharField(max_length=50, null=True, blank=True)),
-                ('vcs_type', models.CharField(max_length=5, choices=[('cvs', 'CVS'), ('svn', 'Subversion'), 
('git', 'Git'), ('hg', 'Mercurial'), ('bzr', 'Bazaar')])),
+                ('vcs_type', models.CharField(max_length=5, choices=[('cvs', 'CVS'), ('svn', 'Subversion'), 
('git', 'Git'), ('hg', 'Mercurial')])),
                 ('vcs_root', models.CharField(max_length=200)),
                 ('vcs_web', models.URLField()),
                 ('ext_platform', models.URLField(help_text='URL to external translation platform, if any', 
null=True, blank=True)),
diff --git a/stats/migrations/0018_new_jsonfields.py b/stats/migrations/0018_new_jsonfields.py
new file mode 100644
index 00000000..a263addc
--- /dev/null
+++ b/stats/migrations/0018_new_jsonfields.py
@@ -0,0 +1,31 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stats', '0017_pofile_path_relative'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='pofile',
+            old_name='figures',
+            new_name='figures_old',
+        ),
+        migrations.RenameField(
+            model_name='branch',
+            old_name='file_hashes',
+            new_name='file_hashes_old',
+        ),
+        migrations.AddField(
+            model_name='pofile',
+            name='figures',
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='branch',
+            name='file_hashes',
+            field=models.JSONField(blank=True, null=True, editable=False),
+        ),
+    ]
diff --git a/stats/migrations/0019_migrate_old_custom_files.py 
b/stats/migrations/0019_migrate_old_custom_files.py
new file mode 100644
index 00000000..064ab872
--- /dev/null
+++ b/stats/migrations/0019_migrate_old_custom_files.py
@@ -0,0 +1,23 @@
+from django.db import migrations
+
+
+def migrate_figures(apps, schema_editor):
+    PoFile = apps.get_model("stats", "PoFile")
+    Branch = apps.get_model("stats", "Branch")
+    for file_ in PoFile.objects.exclude(figures_old__isnull=False):
+        file_.figures = file_.figures_old
+        file_.save()
+    for branch in Branch.objects.exclude(file_hashes_old=''):
+        branch.file_hashes = branch.file_hashes_old
+        branch.save(update_statistics=False)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stats', '0018_new_jsonfields'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_figures),
+    ]
diff --git a/stats/migrations/0020_remove_pofile_figures_old.py 
b/stats/migrations/0020_remove_pofile_figures_old.py
new file mode 100644
index 00000000..d9edf09f
--- /dev/null
+++ b/stats/migrations/0020_remove_pofile_figures_old.py
@@ -0,0 +1,19 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stats', '0019_migrate_old_custom_files'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='pofile',
+            name='figures_old',
+        ),
+        migrations.RemoveField(
+            model_name='branch',
+            name='file_hashes_old',
+        ),
+    ]
diff --git a/stats/models.py b/stats/models.py
index 848bc9b9..eea291d9 100644
--- a/stats/models.py
+++ b/stats/models.py
@@ -24,7 +24,6 @@ from django.utils import dateformat
 from django.db import models
 from django.db.models.functions import Coalesce
 
-from common.fields import DictionaryField, JSONField
 from common.utils import is_site_admin, run_shell_command
 from stats import repos, signals, utils
 from stats.doap import update_doap_infos
@@ -221,7 +220,7 @@ class Branch(models.Model):
     vcs_subpath = models.CharField(max_length=50, blank=True)
     module      = models.ForeignKey(Module, on_delete=models.CASCADE)
     weight      = models.IntegerField(default=0, help_text="Smaller weight is displayed first")
-    file_hashes = DictionaryField(default='', blank=True, editable=False)
+    file_hashes = models.JSONField(blank=True, null=True, editable=False)
     # 'releases' is the backward relation name from Release model
 
     # May be set to False by test suite
@@ -320,6 +319,8 @@ class Branch(models.Model):
         if self.file_hashes and self.file_hashes.get(rel_path, None) == new_hash:
             return False
         else:
+            if not self.file_hashes:
+                self.file_hashes = {}
             self.file_hashes[rel_path] = new_hash
             self.save(update_statistics=False)
             return True
@@ -1224,7 +1225,7 @@ class PoFile(models.Model):
     fuzzy        = models.IntegerField(default=0)
     untranslated = models.IntegerField(default=0)
     # List of figure dict
-    figures      = JSONField(blank=True, null=True)
+    figures = models.JSONField(blank=True, null=True)
     # words statistics
     translated_words   = models.IntegerField(default=0)
     fuzzy_words        = models.IntegerField(default=0)