Init

app/db/admin.py

@@ -0,0 +1,18 @@
+from django.contrib import admin
+
+from .models import DB, DBBackup, DBCredential
+
+
+@admin.register(DB)
+class DBAdmin(admin.ModelAdmin):
+    pass
+
+
+@admin.register(DBCredential)
+class DBCredentialAdmin(admin.ModelAdmin):
+    pass
+
+
+@admin.register(DBBackup)
+class BackupDBAdmin(admin.ModelAdmin):
+    readonly_fields = ("task_id", "status", "rel_path",)

app/db/apps.py

@@ -0,0 +1,13 @@
+from django.apps import AppConfig
+from django.db.models import signals as db_signals
+
+
+class DbConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'db'
+
+    def ready(self):
+        from . import signals, models
+
+        db_signals.post_save.connect(signals.on_backupdb_save, sender=models.DBBackup)
+        db_signals.post_delete.connect(signals.on_backupdb_delete, sender=models.DBBackup)

app/db/backup_engine.py

@@ -0,0 +1,98 @@
+import time
+from abc import ABC, abstractmethod
+import gzip
+import subprocess
+import uuid
+from typing import Any
+
+from .models import DB, DBBackup
+# https://medium.com/poka-techblog/5-different-ways-to-backup-your-postgresql-database-using-python-3f06cea4f51
+
+
+class BackupEngineBase(ABC):
+    def __init__(self, backupdb: DBBackup):
+        self.backup_instance = backupdb
+        self.db_instance = self.backup_instance.db
+
+        # self.db_instance = db_instance
+        # self.backup_instance = BackupDB.objects.create(
+        #     db=self.db_instance,
+        #     rel_path="{db_id}-{timestamp}-{rand_gen}.{db_type}.{ext}".format(
+        #         db_id=self.db_instance.id,
+        #         timestamp=int(time.time()),
+        #         rand_gen=uuid.uuid4().hex[:5],
+        #         db_type=self.db_instance.db_type,
+        #         ext='sql.gz' if self.gzip else 'sql'
+        #     ),
+        # )
+        self.credentials = {"database": self.db_instance.db_name, **self.db_instance.credential.credentials}
+
+    @property
+    @abstractmethod
+    def cmd_kwargs(self) -> dict[str, Any]:
+        ...
+
+    @property
+    def format_cmd_kwargs(self):
+        return " ".join([
+            key if isinstance(value, bool) else f"{key} {value}"
+            for key, value in self.cmd_kwargs.items()
+            if value
+        ])
+
+    @property
+    @abstractmethod
+    def cmd(self):
+        ...
+
+    @property
+    def cmd_format(self):
+        return self.cmd.format(cmd_kwargs=self.format_cmd_kwargs, **self.credentials)
+
+    def popen(self):
+        return subprocess.Popen(self.cmd_format,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.STDOUT,
+                                shell=True,
+                                universal_newlines=True)
+
+    def file_obj(self, path):
+        return gzip.open(path, "wb")
+
+    def run(self, task_id=None):
+        self.backup_instance.update_task_id(task_id)
+        try:
+            with self.file_obj(self.backup_instance.abs_path) as f:
+                with self.popen() as proc:
+                    self.backup_instance.update_status("running")
+                    for line in iter(proc.stdout.readline, ""):
+                        f.write(line.encode())
+        except Exception as e:
+            self.backup_instance.update_status("finished")
+            raise e
+
+
+class MySQLBackupEngine(BackupEngineBase):
+    @property
+    def cmd_kwargs(self) -> dict[str, Any]:
+        return {
+            "--no-tablespaces": True
+        }
+
+    @property
+    def cmd(self):
+        return "mysqldump {cmd_kwargs} -h {host} -P {port} -u {user} -p{password} {database}"
+
+
+class PostgreSQLBackupEngine(BackupEngineBase):
+    @property
+    def cmd_kwargs(self) -> dict[str, Any]:
+        return {
+            "--no-owner": True,
+            "--no-privileges": True,
+            "--clean": False
+        }
+
+    @property
+    def cmd(self):
+        return "pg_dump {cmd_kwargs} --dbname=postgresql://{user}:{password}@{host}:{port}/{database}"

app/db/migrations/0001_initial.py

@@ -0,0 +1,55 @@
+# Generated by Django 5.0.6 on 2024-05-28 09:06
+
+import django.db.models.deletion
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DB',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=100)),
+                ('date_created', models.DateTimeField(auto_now_add=True)),
+                ('date_modified', models.DateTimeField(auto_now=True)),
+                ('db_type', models.CharField(max_length=100)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='DBBackup',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('date_created', models.DateTimeField(auto_now_add=True)),
+                ('date_modified', models.DateTimeField(auto_now=True)),
+                ('status', models.CharField(choices=[('waiting', 'waiting'), ('running', 'running'), ('ok', 'Ok'), ('error', 'Error'), ('expired', 'Expired')], default='waiting', max_length=20)),
+                ('task_id', models.CharField(max_length=100, null=True)),
+                ('rel_path', models.CharField(max_length=100)),
+                ('db', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='db.db')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='DBCredential',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('credentials', models.JSONField(default=dict)),
+                ('create_db_perm', models.BooleanField(default=False)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='db',
+            name='credential',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='db.dbcredential'),
+        ),
+    ]

app/db/migrations/0002_db_db_name_alter_db_db_type_alter_dbbackup_db.py

@@ -0,0 +1,30 @@
+# Generated by Django 5.0.6 on 2024-06-04 10:27
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('db', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='db',
+            name='db_name',
+            field=models.CharField(default='test', max_length=255),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='db',
+            name='db_type',
+            field=models.CharField(choices=[('mysql', 'MySQL/MariaDB'), ('postgres', 'PostgreSQL')], max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='dbbackup',
+            name='db',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='backups', to='db.db'),
+        ),
+    ]

app/db/models.py

@@ -0,0 +1,79 @@
+from django.db import models
+from django.core.files.storage import FileSystemStorage
+from django.conf import settings
+
+import pathlib
+import uuid
+import time
+
+credentials_storage = FileSystemStorage(pathlib.Path(__file__).parent / 'credentials')
+
+
+class DBCredential(models.Model):
+    # credentials = models.FileField(storage=credentials_storage)
+    credentials = models.JSONField(default=dict)
+    create_db_perm = models.BooleanField(default=False)
+    user = models.ForeignKey("user.User", on_delete=models.CASCADE)
+
+
+class DB(models.Model):
+    name = models.CharField(max_length=100)
+    db_name = models.CharField(max_length=255)
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+    credential = models.ForeignKey("DBCredential", on_delete=models.CASCADE)
+    user = models.ForeignKey("user.User", on_delete=models.CASCADE)
+    db_types = (
+        ("mysql", "MySQL/MariaDB"),
+        ("postgres", "PostgreSQL"),
+    )
+    db_type = models.CharField(max_length=100, choices=db_types)
+
+
+class DBBackup(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_modified = models.DateTimeField(auto_now=True)
+    db = models.ForeignKey("DB", on_delete=models.CASCADE, related_name="backups")
+    status_choices = [
+        ("waiting", "waiting"),
+        ("running", "running"),
+        ("ok", "Ok"),
+        ("error", "Error"),
+        ("expired", "Expired")
+    ]
+    status = models.CharField(max_length=20, default="waiting", choices=status_choices)
+    task_id = models.CharField(max_length=100, null=True)
+    rel_path = models.CharField(max_length=100)
+
+    def save(self, *args, **kwargs):
+        if not self.rel_path:
+            self.rel_path = "{db_id}-{timestamp}-{rand_gen}.{db_type}.{ext}".format(
+                db_id=self.db.id,
+                timestamp=int(time.time()),
+                rand_gen=uuid.uuid4().hex[:5],
+                db_type=self.db.db_type,
+                ext='sql.gz'
+            )
+        return super().save(*args, **kwargs)
+
+    def update_status(self, status):
+        self.status = status
+        self.save()
+
+    def update_task_id(self, task_id):
+        if task_id != self.task_id:
+            self.task_id = task_id
+            self.save()
+
+    def set_expired(self):
+        self.status = "expired"
+        self.abs_path.unlink(True)
+        self.save()
+
+    @property
+    def abs_path(self):
+        return settings.BACKUPS_PATH / self.rel_path
+
+
+

app/db/signals.py

@@ -0,0 +1,12 @@
+from .models import DBBackup
+
+from .tasks import start_backup
+
+
+def on_backupdb_save(instance: DBBackup, created, **kwargs):
+    if created:
+        start_backup.delay(instance.id)
+
+
+def on_backupdb_delete(instance: DBBackup, **kwargs):
+    instance.abs_path.unlink(missing_ok=True)

app/db/tasks.py

@@ -0,0 +1,53 @@
+from django.db.models import Prefetch, Q
+from django.utils import timezone
+
+import celery
+from datetime import timedelta
+
+from .models import DBBackup, DB
+from .backup_engine import MySQLBackupEngine, PostgreSQLBackupEngine
+
+
+@celery.shared_task
+def start_backup(backup_id):
+    backupdb = DBBackup.objects.get(id=backup_id)
+    # task_id can be none if it's not a celery task (for ex : if running the function without apply it)
+    task_id = start_backup.request.id
+
+    klass = None
+    match backupdb.db.db_type:
+        case "mysql":
+            klass = MySQLBackupEngine
+
+        case "postgres":
+            klass = PostgreSQLBackupEngine
+
+        case _:
+            pass
+
+    if klass:
+        be = klass(backupdb=backupdb)
+        be.run(task_id)
+
+    # if start_backup.request.id:
+    #     # running from celery task
+    #     pass
+    # else:
+    #     # running from standard function
+    #     pass
+
+
+@celery.shared_task
+def job_check_daily_backup():
+    now = timezone.now()
+
+    for db in DB.objects.all():
+        # check if need backup
+        last_backup: DBBackup = (db.backups.order_by("-date_created")
+                                 .filter(Q(status="ok") | Q(status="running") | Q(status="waiting"))
+                                 .first())
+        if not last_backup or last_backup.date_created < (now - timedelta(hours=24)):
+            DBBackup.objects.create(db=db)
+
+        for backup in db.backups.filter(date_created__lt=now - timedelta(days=30)):
+            backup.set_expired()

app/db/templates/db/index.html

@@ -0,0 +1,6 @@
+{% extends "db/layout.html" %}
+{% load django_vite %}
+
+{% block js %}
+    {% vite_asset "entrypoint/db.js" %}
+{% endblock %}

app/db/templates/db/layout.html

@@ -0,0 +1,5 @@
+{% extends "base.html" %}
+
+{% block js %}
+
+{% endblock %}

app/db/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

app/db/urls.py

@@ -0,0 +1,8 @@
+from django.urls import path
+
+from .views import IndexView
+
+app_name = 'db'
+urlpatterns = [
+    path("", IndexView.as_view(), name='index'),
+]

app/db/views.py

@@ -0,0 +1,5 @@
+from django.views.generic import TemplateView
+
+
+class IndexView(TemplateView):
+    template_name = "db/index.html"