[damned-lies] feat: use jinja to template configurations

From: Guillaume Bernard <gbernard src gnome org>
To: commits-list gnome org
Cc:
Subject: [damned-lies] feat: use jinja to template configurations
Date: Tue, 10 May 2022 08:30:02 +0000 (UTC)
commit c0ebfb22b4552d0137525f841587301ac1562880
Author: Guillaume Bernard <associations guillaume-bernard fr>
Date:   Mon May 9 15:06:22 2022 +0200

    feat: use jinja to template configurations

 containers/production/.containerignore             |  2 +
 containers/production/.gitignore                   |  3 ++
 containers/production/build_buildah_production.sh  | 12 +++--
 .../{entrypoint.sh => entrypoint.sh.jinja2}        | 26 ++++++----
 containers/production/httpd.conf.jinja2            | 57 ++++++++++++++++++++++
 containers/production/l10n.gnome.org.conf          | 55 ---------------------
 ...{local_settings.py => local_settings.py.jinja2} | 29 ++++++++---
 containers/production/production.json              | 28 +++++++++++
 .../production/render_configuration_templates.sh   | 18 +++++++
 containers/production/staging.json                 | 32 ++++++++++++
 containers/production/test.json                    | 28 +++++++++++
 11 files changed, 214 insertions(+), 76 deletions(-)
---
diff --git a/containers/production/.containerignore b/containers/production/.containerignore
index c3cf174c..eb59ec9f 100644
--- a/containers/production/.containerignore
+++ b/containers/production/.containerignore
@@ -11,4 +11,6 @@ containers
 *.md
 setup.*
 
+# Databases
+*.db
 
diff --git a/containers/production/.gitignore b/containers/production/.gitignore
new file mode 100644
index 00000000..21b384b0
--- /dev/null
+++ b/containers/production/.gitignore
@@ -0,0 +1,3 @@
+production
+staging
+test
diff --git a/containers/production/build_buildah_production.sh 
b/containers/production/build_buildah_production.sh
index e1ccb2fc..90fd1b3a 100755
--- a/containers/production/build_buildah_production.sh
+++ b/containers/production/build_buildah_production.sh
@@ -30,6 +30,10 @@ if [[ -z ${RUNTIME_IMAGE_NAME+x}  ]]; then
     declare -r RUNTIME_IMAGE_NAME="damned-lies-runtime:${CURRENT_VCS_REF_NAME}-latest"
 fi
 
+# KIND_OF_DEPLOYMENT: kind of image to build (the configuration will depend on this parameter).
+# Possible values are test, staging or production (the default), the names of the configuration files in JSON
+declare KIND_OF_DEPLOYMENT="${1:-"production"}"
+
 
#######################################################################################################################
 
 # Stop the shell script if at least one command fails
@@ -61,8 +65,8 @@ buildah add \
 buildah config --workingdir "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/damnedlies "${container}"
 
 # Copy project settings files
-buildah add --chown "${PROCESS_USER}:${PROCESS_USER}" --chmod 660 "${container}" 
"${THIS_SCRIPT_DIRECTORY}/local_settings.py" 
"${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/damnedlies/damnedlies/local_settings.py
-buildah add "${container}" "${THIS_SCRIPT_DIRECTORY}/l10n*.gnome.org.conf" 
/etc/httpd/conf.d/l10n.gnome.org.conf
+buildah add --chown "${PROCESS_USER}:${PROCESS_USER}" --chmod 660 "${container}" 
"${THIS_SCRIPT_DIRECTORY}/${KIND_OF_DEPLOYMENT}/local_settings.py" 
"${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/damnedlies/damnedlies/local_settings.py
+buildah add --chmod 644 "${container}" "${THIS_SCRIPT_DIRECTORY}/${KIND_OF_DEPLOYMENT}/httpd.conf" 
/etc/httpd/conf.d/l10n.gnome.org.conf
 
 # Install dependencies in the virtual environment
 buildah run "${container}" -- python3 -m venv --system-site-packages 
"${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/venv
@@ -70,7 +74,7 @@ buildah run "${container}" -- "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/venv/bin
 buildah run "${container}" -- "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/venv/bin/pip install -r 
requirements.txt
 
 # Set the image entrypoint
-buildah add --chown "${PROCESS_USER}:${PROCESS_USER}" --chmod 770 "${container}" 
"${THIS_SCRIPT_DIRECTORY}/entrypoint.sh" "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}/entrypoint.sh"
+buildah add --chown "${PROCESS_USER}:${PROCESS_USER}" --chmod 770 "${container}" 
"${THIS_SCRIPT_DIRECTORY}/${KIND_OF_DEPLOYMENT}/entrypoint.sh" 
"${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}/entrypoint.sh"
 buildah config --entrypoint "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}/entrypoint.sh" "${container}"
 
 # Change ownership of HTTPd server and project to the user in the container
@@ -80,5 +84,5 @@ buildah run "${container}" -- chown -R "${PROCESS_USER}":"${PROCESS_USER}" "${DA
 buildah config --user "${PROCESS_USER}" "${container}"
 
 # Commit container to image
-buildah commit "${container}" "${DAMNED_LIES_IMAGE_NAME}:${CURRENT_VCS_REF_NAME}"
+buildah commit "${container}" "${DAMNED_LIES_IMAGE_NAME}:${CURRENT_VCS_REF_NAME}-${KIND_OF_DEPLOYMENT}"
 
diff --git a/containers/production/entrypoint.sh b/containers/production/entrypoint.sh.jinja2
similarity index 56%
rename from containers/production/entrypoint.sh
rename to containers/production/entrypoint.sh.jinja2
index 010182b1..d9967a78 100755
--- a/containers/production/entrypoint.sh
+++ b/containers/production/entrypoint.sh.jinja2
@@ -1,9 +1,9 @@
 #!/bin/bash
 
-declare -r DAMNED_LIES_PROJECT_DIRECTORY="/${USER}"
-declare -r LOCAL_SETTINGS_FILE="${DAMNED_LIES_PROJECT_DIRECTORY}/damnedlies/damnedlies/local_settings.py"
+declare -r LOCAL_SETTINGS_FILE="{{ damned_lies_project_directory }}/damnedlies/local_settings.py"
 declare -r SSH_CONFIG_URL="https://gitlab.gnome.org/Infrastructure/damned-lies/-/raw/oscp/httpd/ssh_config";
 
+{% if kind == "production" %}
 sed -i "s/\$DB_NAME/${DB_NAME}/" "${LOCAL_SETTINGS_FILE}"
 sed -i "s/\$DB_USER/${DB_USER}/" "${LOCAL_SETTINGS_FILE}"
 sed -i "s/\$DB_PASSWORD/${DB_PASSWORD}/" "${LOCAL_SETTINGS_FILE}"
@@ -13,20 +13,26 @@ sed -i "s/\$SECRET_KEY/${SECRET_KEY}/" "${LOCAL_SETTINGS_FILE}"
 sed -i "s/\$SENTRY_DSN/${SENTRY_DSN}/" "${LOCAL_SETTINGS_FILE}"
 sed -i "s/\$GITLAB_TOKEN/${GITLAB_TOKEN}/" "${LOCAL_SETTINGS_FILE}"
 
-echo "Populating .ssh for the ${USER} user"
+echo "Populating .ssh for the {{ process.user }} user"
 mkdir -p "${HOME}/.ssh"
 ln -s "${HOME}/.ssh_secret/ssh-privatekey" "${HOME}/.ssh/ssh-privatekey"
 ln -s "${HOME}/.ssh_secret/.gitconfig" "${HOME}/.gitconfig"
-wget "${SSH_CONFIG_URL}" -O "${HOME}.ssh/config"
+wget "${SSH_CONFIG_URL}" -O "${HOME}/.ssh/config"
+{% endif %}
 
-echo 'Applying migrations'
-"${DAMNED_LIES_PROJECT_DIRECTORY}/venv/bin/python" manage.py migrate
+echo "Applying migrations"
+{{ process.virtual_env }}/bin/python manage.py migrate
 
-echo 'Generating static files'
-"${DAMNED_LIES_PROJECT_DIRECTORY}/venv/bin/python" manage.py collectstatic
+echo "Generating static files"
+{{ process.virtual_env }}/bin/python manage.py collectstatic
 
-echo 'Compiling translations'
-"${DAMNED_LIES_PROJECT_DIRECTORY}/venv/bin/python" manage.py compile-trans
+echo "Compiling translations"
+{{ process.virtual_env }}/bin/python manage.py compile-trans
+
+{% if kind == "test" %}
+echo "Load sample data"
+{{ process.virtual_env }}/bin/python manage.py loaddata sample_data
+{% endif %}
 
 if [ $# -eq 0 ]; then
   exec httpd -DFOREGROUND
diff --git a/containers/production/httpd.conf.jinja2 b/containers/production/httpd.conf.jinja2
new file mode 100644
index 00000000..e041c85c
--- /dev/null
+++ b/containers/production/httpd.conf.jinja2
@@ -0,0 +1,57 @@
+<VirtualHost *:8080>
+    ServerName {{ server.name }}
+    ServerAlias localhost
+    ServerAlias 127.0.0.1
+
+    Alias /HTML {{ damned_lies_data_directory }}/scratchdir/HTML
+    Alias /POT {{ damned_lies_data_directory }}/scratchdir/POT
+    Alias /static {{ damned_lies_project_directory }}/static
+    Alias /media {{ damned_lies_data_directory }}/media
+    Alias /robots.txt {{ damned_lies_project_directory }}/static/robots.txt
+
+    # mod_wsgi
+    WSGIDaemonProcess {{ process.user.name }} display-name=%{GROUP} processes=4 home={{ process.user.home }} 
python-home={{ process.virtual_env }} python-path={{ damned_lies_project_directory }} user={{ 
process.user.name }} group={{ process.user.group }}
+    WSGIProcessGroup {{ process.user.group }}
+    WSGIScriptAlias / {{ damned_lies_project_directory }}/damnedlies/wsgi.py
+
+    # Map .po and .pot files accordingly to the correct Content-Type,
+    # and also make sure they are rendered as utf-8.
+    AddCharset UTF-8 .po
+    AddCharset UTF-8 .pot
+    AddType text/plain .po
+    AddType text/plain .pot
+
+    <Directory "{{ damned_lies_project_directory }}">
+        # Avoid sending request to Django when host is not correct.
+        Require expr %{HTTP_HOST} == "l10n.gnome.org"
+    </Directory>
+
+    <DirectoryMatch "{{ damned_lies_project_directory }}/(static|damnedlies)">
+        Options Indexes
+        Require all granted
+    </DirectoryMatch>
+
+    <Directory "{{ damned_lies_data_directory }}/media">
+        Options Indexes
+        Require all granted
+    </Directory>
+
+    <DirectoryMatch "^{{ damned_lies_data_directory }}/scratchdir/(POT|HTML)">
+        Options Indexes
+        <RequireAll>
+          Require all granted
+          Require not ip 95.108.249.30
+          Require not ip 218.30.103.149
+          Require not ip 65.55.24.217
+          Require not ip 180.76.5.0/24
+          Require not ip 218.30.103.0/24
+          Require not ip 1.202.219.0/24
+        </RequireAll>
+    </DirectoryMatch>
+</VirtualHost>
+
+<LocationMatch "^/vertimus">
+  <If "%{HTTP_USER_AGENT} =~ /Sogou web spider/">
+    Require all denied
+  </If>
+</LocationMatch>
diff --git a/containers/production/local_settings.py b/containers/production/local_settings.py.jinja2
similarity index 61%
rename from containers/production/local_settings.py
rename to containers/production/local_settings.py.jinja2
index 8dae1a43..36770e5b 100644
--- a/containers/production/local_settings.py
+++ b/containers/production/local_settings.py.jinja2
@@ -4,10 +4,15 @@ DEBUG = False
 STATIC_SERVE = False
 USE_DJANGO_OPENID = False
 
-ADMINS = (("Guillaume Bernard", "associations guillaume-bernard fr"),)
+ADMINS = (
+{% for admin_name, admin_email in admins %}
+    ("{{ admin_name }}", "{{ admin_email }}"),
+{% endfor %}
+)
 
 MANAGERS = ADMINS
 
+{% if kind == "production" %}
 DATABASES = {
     "default": {
         "ENGINE": "django.db.backends.mysql",
@@ -22,20 +27,30 @@ DATABASES = {
         },
     }
 }
+{% endif %}
+
+{% if kind == "test" %}
+DEBUG = True
+{% else %}
+DEBUG = False
+{% endif %}
+
 SECRET_KEY = """$SECRET_KEY"""
-ALLOWED_HOSTS = ["l10n.gnome.org", "8.43.85.13", "8.43.85.14", "8.43.85.29"]
+
+SITE_DOMAIN = "{{ server.name }}"
+ALLOWED_HOSTS = ["{{ server.name }}", "8.43.85.13", "8.43.85.14", "8.43.85.29", "localhost", "127.0.0.1"]
 SESSION_COOKIE_SECURE = True
 CSRF_COOKIE_SECURE = True
 SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
 
-EMAIL_HOST = "smtp-int.gnome.org"
-EMAIL_SUBJECT_PREFIX = "[DL]"
-DEFAULT_FROM_EMAIL = "noreply gnome org"
-SERVER_EMAIL = "gnomeweb gnome org"
+EMAIL_HOST = "{{ email.smtp_server }}"
+EMAIL_SUBJECT_PREFIX = "{{ email.subject_prefix }}"
+DEFAULT_FROM_EMAIL = "{{ email.from_address }}"
+SERVER_EMAIL = "{{ email.server }}"
 
 TIME_ZONE = "UTC"
 
-DATADIR = Path("/var/www/djamnedlies/data/")
+DATADIR = Path("{{ damned_lies_data_directory }}")
 MEDIA_ROOT = DATADIR / "media"
 
 # Local directory path for VCS checkout
diff --git a/containers/production/production.json b/containers/production/production.json
new file mode 100644
index 00000000..d889d6d5
--- /dev/null
+++ b/containers/production/production.json
@@ -0,0 +1,28 @@
+{
+    "server": {
+        "name": "l10n.gnome.org"
+    },
+    "kind": "production",
+    "damned_lies_project_directory": "/l10n/damnedlies",
+    "damned_lies_data_directory": "/var/www/djamnedlies/data",
+    "process": {
+        "user": {
+            "name": "l10n",
+            "group": "l10n",
+            "home": "/home/l10n"
+        },
+        "virtual_env": "/l10n/venv"
+    },
+    "admins": [
+        [
+            "Guillaume Bernard",
+            "associations guillaume-bernard fr"
+        ]
+    ],
+    "email": {
+        "smtp_server": "smtp-int.gnome.org",
+        "subject_prefix": "[DL]",
+        "from_address": "noreply gnome org",
+        "server": "gnomeweb gnome org"
+    }
+}
diff --git a/containers/production/render_configuration_templates.sh 
b/containers/production/render_configuration_templates.sh
new file mode 100755
index 00000000..c604173f
--- /dev/null
+++ b/containers/production/render_configuration_templates.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Render a specific configuration. All the files will be rendered in the KIND_OF_DEPLOYMENT directory
+
+set +C
+
+# KIND_OF_DEPLOYMENT: kind of deployment to render (the configuration will depend on this parameter).
+# Possible values are test, staging or production (the default), the names of the configuration files in JSON
+declare -r KIND_OF_DEPLOYMENT="${1:-production}"
+
+# THIS_SCRIPT_DIRECTORY: the current script directory name
+THIS_SCRIPT_DIRECTORY="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+declare -r THIS_SCRIPT_DIRECTORY
+
+mkdir -p "${THIS_SCRIPT_DIRECTORY}/${KIND_OF_DEPLOYMENT}"
+for template in "${THIS_SCRIPT_DIRECTORY}"/*.jinja2; do
+    jinja2 --strict "${template}" "${THIS_SCRIPT_DIRECTORY}/${KIND_OF_DEPLOYMENT}.json" > 
"${THIS_SCRIPT_DIRECTORY}/${KIND_OF_DEPLOYMENT}/$(basename "${template}" .jinja2)"
+done
diff --git a/containers/production/staging.json b/containers/production/staging.json
new file mode 100644
index 00000000..3f5c511d
--- /dev/null
+++ b/containers/production/staging.json
@@ -0,0 +1,32 @@
+{
+    "server": {
+        "name": "l10n-staging.openshift.gnome.org"
+    },
+    "kind": "production",
+    "damned_lies_project_directory": "/l10n/damnedlies",
+    "damned_lies_data_directory": "/var/www/djamnedlies/data",
+    "process": {
+        "user": {
+            "name": "l10n",
+            "group": "l10n",
+            "home": "/home/l10n"
+        },
+        "virtual_env": "/l10n/venv"
+    },
+    "admins": [
+        [
+            "Guillaume Bernard",
+            "associations guillaume-bernard fr"
+        ],
+        [
+            "Andrea Veri",
+            "averi gnome org"
+        ]
+    ],
+    "email": {
+        "smtp_server": "smtp-int.gnome.org",
+        "subject_prefix": "[Damned Lies − Test Staging]",
+        "from_address": "noreply gnome org",
+        "server": "gnomeweb gnome org"
+    }
+}
diff --git a/containers/production/test.json b/containers/production/test.json
new file mode 100644
index 00000000..a7e71786
--- /dev/null
+++ b/containers/production/test.json
@@ -0,0 +1,28 @@
+{
+    "server": {
+        "name": "l10n-testing.openshift.gnome.org"
+    },
+    "kind": "test",
+    "damned_lies_project_directory": "/l10n/damnedlies",
+    "damned_lies_data_directory": "/var/www/djamnedlies/data",
+    "process": {
+        "user": {
+            "name": "l10n",
+            "group": "l10n",
+            "home": "/home/l10n"
+        },
+        "virtual_env": "/l10n/venv"
+    },
+    "admins": [
+        [
+            "Guillaume Bernard",
+            "associations guillaume-bernard fr"
+        ]
+    ],
+    "email": {
+        "smtp_server": "smtp-int.gnome.org",
+        "subject_prefix": "[Damned Lies − Test Instance]",
+        "from_address": "noreply gnome org",
+        "server": "gnomeweb gnome org"
+    }
+}
[Date Prev][Date Next] [Thread Prev][Thread Next] [Thread Index] [Date Index] [Author Index]