Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

✨(backend) add command to clean pending and deleted files

Florent Chehab5c1514078315
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 32771f32..787a817d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,11 @@ and this project adheres to
## [Unreleased]
+### Added
+
+- ✨(backend) add command to clean pending and deleted files
+- 🧱(helm) run clean files command as cronjob
+
## [1.21.0] - 2026-06-15
### Added
diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py
index 242d8255..e731cf42 100644
--- a/src/backend/core/api/viewsets.py
+++ b/src/backend/core/api/viewsets.py
@@ -1187,7 +1187,10 @@ class FileViewSet(
serializer.save(creator=self.request.user)
def perform_destroy(self, instance):
- """Override to implement a soft delete instead of dumping the record in database."""
+ """Override to implement a soft delete instead of dumping the record in database.
+
+ Files are actually purged by commands that should run periodically.
+ """
instance.soft_delete()
@decorators.action(detail=True, methods=["post"], url_path="upload-ended")
diff --git a/src/backend/core/management/commands/clean_pending_files.py b/src/backend/core/management/commands/clean_pending_files.py
new file mode 100644
index 00000000..9a8b8041
--- /dev/null
+++ b/src/backend/core/management/commands/clean_pending_files.py
@@ -0,0 +1,47 @@
+"""Clean stale pending files that were never fully uploaded."""
+
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+
+from core.models import File, FileUploadStateChoices
+from core.tasks.file import process_file_deletion
+
+
+class Command(BaseCommand):
+ """Remove pending files older than a given threshold."""
+
+ help = "Delete pending files that have been stuck for too long"
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ "--hours",
+ type=int,
+ default=24,
+ help="Age threshold in hours (default: 24)",
+ )
+
+ def handle(self, *args, **options):
+ hours = options["hours"]
+ if hours < 0:
+ raise CommandError("Hours must be greater than 0")
+
+ threshold = timezone.now() - timedelta(hours=hours)
+
+ files = File.objects.filter(
+ upload_state=FileUploadStateChoices.PENDING,
+ created_at__lt=threshold,
+ hard_deleted_at__isnull=True,
+ )
+
+ count = 0
+ for file in files.iterator():
+ # This check shouldn't happen, but just in case we do it to avoid an error
+ if not file.deleted_at:
+ file.soft_delete()
+ file.hard_delete()
+ process_file_deletion(file.id)
+ count += 1
+
+ self.stdout.write(f"Cleaned {count} stale pending file(s).")
diff --git a/src/backend/core/management/commands/purge_deleted_files.py b/src/backend/core/management/commands/purge_deleted_files.py
new file mode 100644
index 00000000..10d71516
--- /dev/null
+++ b/src/backend/core/management/commands/purge_deleted_files.py
@@ -0,0 +1,40 @@
+"""Purge deleted files."""
+
+from datetime import timedelta
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+from django.utils import timezone
+
+from core.models import File
+from core.tasks.file import process_file_deletion
+
+
+class Command(BaseCommand):
+ """
+ Purge deleted files (object storage and database object):
+ - files marked as hard deleted in database
+ - files marked as soft deleted and for which the trashbin retention period has expired
+ """
+
+ help = "Purge deleted files"
+
+ def handle(self, *args, **options):
+ """Browse purgeable files and queue them through the file deletion task."""
+
+ is_hard_deleted = Q(hard_deleted_at__isnull=False)
+ is_purgeable = Q(
+ deleted_at__lte=timezone.now()
+ - timedelta(days=settings.FILE_PURGE_GRACE_DAYS)
+ )
+
+ count = 0
+ for file in File.objects.filter(is_hard_deleted | is_purgeable).iterator():
+ if file.hard_deleted_at is None:
+ file.hard_delete()
+
+ process_file_deletion.delay(file.id)
+ count += 1
+
+ self.stdout.write(f"Purged {count} deleted file(s).")
diff --git a/src/backend/core/tests/commands/__init__.py b/src/backend/core/tests/commands/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/backend/core/tests/commands/test_clean_pending_files.py b/src/backend/core/tests/commands/test_clean_pending_files.py
new file mode 100644
index 00000000..264f4dbc
--- /dev/null
+++ b/src/backend/core/tests/commands/test_clean_pending_files.py
@@ -0,0 +1,90 @@
+"""Tests for the clean_pending_files management command."""
+
+from datetime import timedelta
+
+from django.core.files.storage import default_storage
+from django.core.management import call_command
+from django.utils import timezone
+
+import pytest
+
+from core import factories, models
+
+pytestmark = pytest.mark.django_db
+
+
+def test_clean_pending_files_no_stale_files():
+ """Nothing happens when there are no stale pending files."""
+ call_command("clean_pending_files")
+
+
+def test_clean_pending_files_recent_pending_not_deleted():
+ """Recent pending files (within threshold) should not be deleted."""
+ file = factories.FileFactory(
+ type=models.FileTypeChoices.BACKGROUND_IMAGE,
+ update_upload_state=models.FileUploadStateChoices.PENDING,
+ upload_bytes=b"hello",
+ )
+
+ call_command("clean_pending_files")
+
+ file.refresh_from_db()
+ assert file.deleted_at is None
+ assert default_storage.exists(file.file_key)
+
+
+def test_clean_pending_files_old_pending_deleted():
+ """Pending files older than the threshold should be deleted."""
+ old_date = timezone.now() - timedelta(hours=49)
+ file = factories.FileFactory(
+ type=models.FileTypeChoices.BACKGROUND_IMAGE,
+ update_upload_state=models.FileUploadStateChoices.PENDING,
+ upload_bytes=b"hello",
+ )
+ assert default_storage.exists(file.file_key)
+ models.File.objects.filter(pk=file.pk).update(created_at=old_date)
+
+ call_command("clean_pending_files")
+
+ assert not models.File.objects.filter(pk=file.pk).exists()
+ assert not default_storage.exists(file.file_key)
+
+
+def test_clean_pending_files_old_non_pending_not_deleted():
+ """Old files that are not pending should not be deleted."""
+ old_date = timezone.now() - timedelta(hours=49)
+ file = factories.FileFactory(
+ type=models.FileTypeChoices.BACKGROUND_IMAGE,
+ update_upload_state=models.FileUploadStateChoices.READY,
+ )
+ models.File.objects.filter(pk=file.pk).update(created_at=old_date)
+
+ call_command("clean_pending_files")
+
+ file.refresh_from_db()
+ assert file.deleted_at is None
+ assert file.hard_deleted_at is None
+
+
+def test_clean_pending_files_custom_hours():
+ """The --hours argument controls the age threshold."""
+ old_date = timezone.now() - timedelta(hours=10)
+ file = factories.FileFactory(
+ type=models.FileTypeChoices.BACKGROUND_IMAGE,
+ update_upload_state=models.FileUploadStateChoices.PENDING,
+ upload_bytes=b"hello",
+ )
+ models.File.objects.filter(pk=file.pk).update(created_at=old_date)
+
+ # Default 24h threshold -> file not deleted
+ call_command("clean_pending_files")
+
+ file.refresh_from_db()
+ assert file.deleted_at is None
+ assert default_storage.exists(file.file_key)
+
+ # 8h threshold -> file deleted
+ call_command("clean_pending_files", "--hours=8")
+
+ assert not models.File.objects.filter(pk=file.pk).exists()
+ assert not default_storage.exists(file.file_key)
diff --git a/src/backend/core/tests/commands/test_purge_deleted_files.py b/src/backend/core/tests/commands/test_purge_deleted_files.py
new file mode 100644
index 00000000..28e51605
--- /dev/null
+++ b/src/backend/core/tests/commands/test_purge_deleted_files.py
@@ -0,0 +1,85 @@
+"""Tests for the purge_deleted_files management command."""
+
+from datetime import timedelta
+from io import StringIO
+from random import randint
+from unittest.mock import patch
+
+from django.core.files.storage import default_storage
+from django.core.management import call_command
+from django.utils import timezone
+
+import pytest
+
+from core import factories, models
+from core.tasks.file import process_file_deletion
+
+pytestmark = pytest.mark.django_db
+
+
+def test_purge_deleted_files_no_deleted_files(django_assert_num_queries):
+ """Nothing happens when there are no purgeable files."""
+ with django_assert_num_queries(1):
+ call_command("purge_deleted_files")
+
+
[email protected]_db(transaction=True)
+def test_purge_deleted_files_success(settings):
+ """
+ Queue deletion for:
+ - hard-deleted files
+ - soft-deleted files past retention period + grace period.
+ """
+ out = StringIO()
+
+ settings.FILE_PURGE_GRACE_DAYS = grace = randint(1, 20)
+
+ now = timezone.now()
+ purge_now = now - timedelta(days=grace)
+
+ not_deleted_file = factories.FileFactory(
+ type=models.FileTypeChoices.BACKGROUND_IMAGE,
+ upload_bytes=b"hello",
+ )
+
+ with patch("django.utils.timezone.now", return_value=now):
+ not_purgeable_file = factories.FileFactory(
+ type=models.FileTypeChoices.BACKGROUND_IMAGE,
+ upload_bytes=b"hello",
+ )
+ not_purgeable_file.soft_delete()
+
+ with patch("django.utils.timezone.now", return_value=purge_now):
+ purgeable_file = factories.FileFactory(
+ type=models.FileTypeChoices.BACKGROUND_IMAGE,
+ upload_bytes=b"hello",
+ )
+ purgeable_file.soft_delete()
+
+ hard_deleted_file = factories.FileFactory(
+ type=models.FileTypeChoices.BACKGROUND_IMAGE,
+ upload_bytes=b"hello",
+ )
+ hard_deleted_file.soft_delete()
+ hard_deleted_file.hard_delete()
+
+ with patch(
+ "core.management.commands.purge_deleted_files.process_file_deletion.delay",
+ side_effect=process_file_deletion,
+ ) as mock_delay:
+ call_command("purge_deleted_files", stdout=out)
+
+ assert "Purged 2 deleted file(s)." in out.getvalue()
+ assert mock_delay.call_count == 2
+ called_ids = {call.args[0] for call in mock_delay.call_args_list}
+ assert called_ids == {purgeable_file.id, hard_deleted_file.id}
+
+ assert models.File.objects.filter(id=not_deleted_file.id).exists()
+ assert models.File.objects.filter(id=not_purgeable_file.id).exists()
+ assert not models.File.objects.filter(id=purgeable_file.id).exists()
+ assert not models.File.objects.filter(id=hard_deleted_file.id).exists()
+
+ assert default_storage.exists(not_deleted_file.file_key)
+ assert default_storage.exists(not_purgeable_file.file_key)
+ assert not default_storage.exists(purgeable_file.file_key)
+ assert not default_storage.exists(hard_deleted_file.file_key)
diff --git a/src/backend/meet/settings.py b/src/backend/meet/settings.py
index 05ce3edb..72d6fbd5 100755
--- a/src/backend/meet/settings.py
+++ b/src/backend/meet/settings.py
@@ -182,7 +182,9 @@ class Base(Configuration):
environ_name="FILE_UPLOAD_ENABLED",
environ_prefix=None,
)
-
+ FILE_PURGE_GRACE_DAYS = values.PositiveIntegerValue(
+ default=7, environ_name="FILE_PURGE_GRACE_DAYS", environ_prefix=None
+ )
FILE_UPLOAD_PATH = values.Value(
"files", environ_name="FILE_UPLOAD_PATH", environ_prefix=None
)