Open source video conferencing app powered by LiveKit. Built with Django and React.
✨(backend) add command to clean pending and deleted files
Florent Chehab5c1514078315
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 32771f32..787a817d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,11 @@ and this project adheres to## [Unreleased]+### Added++- ✨(backend) add command to clean pending and deleted files+- 🧱(helm) run clean files command as cronjob+## [1.21.0] - 2026-06-15### Addeddiff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.pyindex 242d8255..e731cf42 100644--- a/src/backend/core/api/viewsets.py+++ b/src/backend/core/api/viewsets.py@@ -1187,7 +1187,10 @@ class FileViewSet(serializer.save(creator=self.request.user)def perform_destroy(self, instance):- """Override to implement a soft delete instead of dumping the record in database."""+ """Override to implement a soft delete instead of dumping the record in database.++ Files are actually purged by commands that should run periodically.+ """instance.soft_delete()@decorators.action(detail=True, methods=["post"], url_path="upload-ended")diff --git a/src/backend/core/management/commands/clean_pending_files.py b/src/backend/core/management/commands/clean_pending_files.pynew file mode 100644index 00000000..9a8b8041--- /dev/null+++ b/src/backend/core/management/commands/clean_pending_files.py@@ -0,0 +1,47 @@+"""Clean stale pending files that were never fully uploaded."""++from datetime import timedelta++from django.core.management.base import BaseCommand, CommandError+from django.utils import timezone++from core.models import File, FileUploadStateChoices+from core.tasks.file import process_file_deletion+++class Command(BaseCommand):+ """Remove pending files older than a given threshold."""++ help = "Delete pending files that have been stuck for too long"++ def add_arguments(self, parser):+ parser.add_argument(+ "--hours",+ type=int,+ default=24,+ help="Age threshold in hours (default: 24)",+ )++ def handle(self, *args, **options):+ hours = options["hours"]+ if hours < 0:+ raise CommandError("Hours must be greater than 0")++ threshold = timezone.now() - timedelta(hours=hours)++ files = File.objects.filter(+ upload_state=FileUploadStateChoices.PENDING,+ created_at__lt=threshold,+ hard_deleted_at__isnull=True,+ )++ count = 0+ for file in files.iterator():+ # This check shouldn't happen, but just in case we do it to avoid an error+ if not file.deleted_at:+ file.soft_delete()+ file.hard_delete()+ process_file_deletion(file.id)+ count += 1++ self.stdout.write(f"Cleaned {count} stale pending file(s).")diff --git a/src/backend/core/management/commands/purge_deleted_files.py b/src/backend/core/management/commands/purge_deleted_files.pynew file mode 100644index 00000000..10d71516--- /dev/null+++ b/src/backend/core/management/commands/purge_deleted_files.py@@ -0,0 +1,40 @@+"""Purge deleted files."""++from datetime import timedelta++from django.conf import settings+from django.core.management.base import BaseCommand+from django.db.models import Q+from django.utils import timezone++from core.models import File+from core.tasks.file import process_file_deletion+++class Command(BaseCommand):+ """+ Purge deleted files (object storage and database object):+ - files marked as hard deleted in database+ - files marked as soft deleted and for which the trashbin retention period has expired+ """++ help = "Purge deleted files"++ def handle(self, *args, **options):+ """Browse purgeable files and queue them through the file deletion task."""++ is_hard_deleted = Q(hard_deleted_at__isnull=False)+ is_purgeable = Q(+ deleted_at__lte=timezone.now()+ - timedelta(days=settings.FILE_PURGE_GRACE_DAYS)+ )++ count = 0+ for file in File.objects.filter(is_hard_deleted | is_purgeable).iterator():+ if file.hard_deleted_at is None:+ file.hard_delete()++ process_file_deletion.delay(file.id)+ count += 1++ self.stdout.write(f"Purged {count} deleted file(s).")diff --git a/src/backend/core/tests/commands/__init__.py b/src/backend/core/tests/commands/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/src/backend/core/tests/commands/test_clean_pending_files.py b/src/backend/core/tests/commands/test_clean_pending_files.pynew file mode 100644index 00000000..264f4dbc--- /dev/null+++ b/src/backend/core/tests/commands/test_clean_pending_files.py@@ -0,0 +1,90 @@+"""Tests for the clean_pending_files management command."""++from datetime import timedelta++from django.core.files.storage import default_storage+from django.core.management import call_command+from django.utils import timezone++import pytest++from core import factories, models++pytestmark = pytest.mark.django_db+++def test_clean_pending_files_no_stale_files():+ """Nothing happens when there are no stale pending files."""+ call_command("clean_pending_files")+++def test_clean_pending_files_recent_pending_not_deleted():+ """Recent pending files (within threshold) should not be deleted."""+ file = factories.FileFactory(+ type=models.FileTypeChoices.BACKGROUND_IMAGE,+ update_upload_state=models.FileUploadStateChoices.PENDING,+ upload_bytes=b"hello",+ )++ call_command("clean_pending_files")++ file.refresh_from_db()+ assert file.deleted_at is None+ assert default_storage.exists(file.file_key)+++def test_clean_pending_files_old_pending_deleted():+ """Pending files older than the threshold should be deleted."""+ old_date = timezone.now() - timedelta(hours=49)+ file = factories.FileFactory(+ type=models.FileTypeChoices.BACKGROUND_IMAGE,+ update_upload_state=models.FileUploadStateChoices.PENDING,+ upload_bytes=b"hello",+ )+ assert default_storage.exists(file.file_key)+ models.File.objects.filter(pk=file.pk).update(created_at=old_date)++ call_command("clean_pending_files")++ assert not models.File.objects.filter(pk=file.pk).exists()+ assert not default_storage.exists(file.file_key)+++def test_clean_pending_files_old_non_pending_not_deleted():+ """Old files that are not pending should not be deleted."""+ old_date = timezone.now() - timedelta(hours=49)+ file = factories.FileFactory(+ type=models.FileTypeChoices.BACKGROUND_IMAGE,+ update_upload_state=models.FileUploadStateChoices.READY,+ )+ models.File.objects.filter(pk=file.pk).update(created_at=old_date)++ call_command("clean_pending_files")++ file.refresh_from_db()+ assert file.deleted_at is None+ assert file.hard_deleted_at is None+++def test_clean_pending_files_custom_hours():+ """The --hours argument controls the age threshold."""+ old_date = timezone.now() - timedelta(hours=10)+ file = factories.FileFactory(+ type=models.FileTypeChoices.BACKGROUND_IMAGE,+ update_upload_state=models.FileUploadStateChoices.PENDING,+ upload_bytes=b"hello",+ )+ models.File.objects.filter(pk=file.pk).update(created_at=old_date)++ # Default 24h threshold -> file not deleted+ call_command("clean_pending_files")++ file.refresh_from_db()+ assert file.deleted_at is None+ assert default_storage.exists(file.file_key)++ # 8h threshold -> file deleted+ call_command("clean_pending_files", "--hours=8")++ assert not models.File.objects.filter(pk=file.pk).exists()+ assert not default_storage.exists(file.file_key)diff --git a/src/backend/core/tests/commands/test_purge_deleted_files.py b/src/backend/core/tests/commands/test_purge_deleted_files.pynew file mode 100644index 00000000..28e51605--- /dev/null+++ b/src/backend/core/tests/commands/test_purge_deleted_files.py@@ -0,0 +1,85 @@+"""Tests for the purge_deleted_files management command."""++from datetime import timedelta+from io import StringIO+from random import randint+from unittest.mock import patch++from django.core.files.storage import default_storage+from django.core.management import call_command+from django.utils import timezone++import pytest++from core import factories, models+from core.tasks.file import process_file_deletion++pytestmark = pytest.mark.django_db+++def test_purge_deleted_files_no_deleted_files(django_assert_num_queries):+ """Nothing happens when there are no purgeable files."""+ with django_assert_num_queries(1):+ call_command("purge_deleted_files")++[email protected]_db(transaction=True)+def test_purge_deleted_files_success(settings):+ """+ Queue deletion for:+ - hard-deleted files+ - soft-deleted files past retention period + grace period.+ """+ out = StringIO()++ settings.FILE_PURGE_GRACE_DAYS = grace = randint(1, 20)++ now = timezone.now()+ purge_now = now - timedelta(days=grace)++ not_deleted_file = factories.FileFactory(+ type=models.FileTypeChoices.BACKGROUND_IMAGE,+ upload_bytes=b"hello",+ )++ with patch("django.utils.timezone.now", return_value=now):+ not_purgeable_file = factories.FileFactory(+ type=models.FileTypeChoices.BACKGROUND_IMAGE,+ upload_bytes=b"hello",+ )+ not_purgeable_file.soft_delete()++ with patch("django.utils.timezone.now", return_value=purge_now):+ purgeable_file = factories.FileFactory(+ type=models.FileTypeChoices.BACKGROUND_IMAGE,+ upload_bytes=b"hello",+ )+ purgeable_file.soft_delete()++ hard_deleted_file = factories.FileFactory(+ type=models.FileTypeChoices.BACKGROUND_IMAGE,+ upload_bytes=b"hello",+ )+ hard_deleted_file.soft_delete()+ hard_deleted_file.hard_delete()++ with patch(+ "core.management.commands.purge_deleted_files.process_file_deletion.delay",+ side_effect=process_file_deletion,+ ) as mock_delay:+ call_command("purge_deleted_files", stdout=out)++ assert "Purged 2 deleted file(s)." in out.getvalue()+ assert mock_delay.call_count == 2+ called_ids = {call.args[0] for call in mock_delay.call_args_list}+ assert called_ids == {purgeable_file.id, hard_deleted_file.id}++ assert models.File.objects.filter(id=not_deleted_file.id).exists()+ assert models.File.objects.filter(id=not_purgeable_file.id).exists()+ assert not models.File.objects.filter(id=purgeable_file.id).exists()+ assert not models.File.objects.filter(id=hard_deleted_file.id).exists()++ assert default_storage.exists(not_deleted_file.file_key)+ assert default_storage.exists(not_purgeable_file.file_key)+ assert not default_storage.exists(purgeable_file.file_key)+ assert not default_storage.exists(hard_deleted_file.file_key)diff --git a/src/backend/meet/settings.py b/src/backend/meet/settings.pyindex 05ce3edb..72d6fbd5 100755--- a/src/backend/meet/settings.py+++ b/src/backend/meet/settings.py@@ -182,7 +182,9 @@ class Base(Configuration):environ_name="FILE_UPLOAD_ENABLED",environ_prefix=None,)-+ FILE_PURGE_GRACE_DAYS = values.PositiveIntegerValue(+ default=7, environ_name="FILE_PURGE_GRACE_DAYS", environ_prefix=None+ )FILE_UPLOAD_PATH = values.Value("files", environ_name="FILE_UPLOAD_PATH", environ_prefix=None)