Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(dify): retry failed archive deletion instead of reporting success (#44061)

CRYPTONIKAV8857292ad7b6
diff --git a/api/tasks/workflow_run_archive_deletion_tasks.py b/api/tasks/workflow_run_archive_deletion_tasks.py
index 41899e3e96c..940103e6aac 100644
--- a/api/tasks/workflow_run_archive_deletion_tasks.py
+++ b/api/tasks/workflow_run_archive_deletion_tasks.py
@@ -6,9 +6,10 @@ from celery import shared_task
from core.db.session_factory import get_session_maker
from extensions.ext_redis import redis_client
-from libs.archive_storage import get_archive_storage, get_export_storage
+from libs.archive_storage import ArchiveStorageError, get_archive_storage, get_export_storage
from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository
from services.retention.workflow_run.archive_deletion import WorkflowRunArchiveDeleter
+from services.retention.workflow_run.archive_deletion_task import WorkflowRunArchiveDeletionStatus
from services.retention.workflow_run.archive_month_lock import (
RedisWorkflowRunArchiveMonthLock,
)
@@ -29,9 +30,15 @@ logger = logging.getLogger(__name__)
def delete_workflow_run_archive_task(tenant_id: str, year: int, month: int) -> None:
"""Redelivery is safe because only DB rows with durable deletion requests are selected."""
logger.info("Deleting workflow run archive month: tenant=%s year=%s month=%s", tenant_id, year, month)
- WorkflowRunArchiveDeleter(
+ result = WorkflowRunArchiveDeleter(
bundles=WorkflowRunArchiveBundleQueryRepository(session_factory=get_session_maker()),
archive_storage_provider=get_archive_storage,
download_storage_provider=get_export_storage,
month_lock=RedisWorkflowRunArchiveMonthLock(redis=redis_client),
).delete(tenant_id=tenant_id, year=year, month=month)
+ if result.status is WorkflowRunArchiveDeletionStatus.FAILED:
+ # The deleter persists FAILED durably but swallows the fault: without
+ # raising, Celery reports SUCCESS and the configured autoretry/backoff
+ # never engages (#44059). Exhausting the retry budget then surfaces as
+ # a task failure instead of silent success.
+ raise ArchiveStorageError(result.error or "Failed to delete archive files. Please retry.")
diff --git a/api/tests/unit_tests/services/retention/workflow_run/test_archive_deletion.py b/api/tests/unit_tests/services/retention/workflow_run/test_archive_deletion.py
index 6cf64d6015c..1341618602c 100644
--- a/api/tests/unit_tests/services/retention/workflow_run/test_archive_deletion.py
+++ b/api/tests/unit_tests/services/retention/workflow_run/test_archive_deletion.py
@@ -200,6 +200,76 @@ def test_archive_deletion_removes_only_requested_files_indexes_and_downloads(
]
+def test_failed_cleanup_reaches_celery_retry_instead_of_success(
+ sqlite_session_factory: sessionmaker[Session],
+ month_lease: WorkflowRunArchiveMonthLease,
+) -> None:
+ """The entry point must surface FAILED so autoretry engages (#44059)."""
+ from tasks import workflow_run_archive_deletion_tasks as deletion_tasks
+
+ archive, downloads = MemoryStorage(), MemoryStorage()
+ target = _bundle()
+ prefix = _put_bundle(archive, target)
+ download_key = build_archive_download_month_prefix(tenant_id=TENANT_ID, year=2025, month=3) + "download.zip"
+ downloads.objects[download_key] = b"download"
+ archive.fail_key = prefix + "workflow_runs.parquet"
+ with sqlite_session_factory.begin() as session:
+ session.add(target)
+ repository = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory)
+ repository.request_deletion(TENANT_ID, year=2025, month=3)
+
+ task = deletion_tasks.delete_workflow_run_archive_task
+ # apply() executes eagerly without changing process-global Celery settings.
+ with (
+ patch.object(deletion_tasks, "get_session_maker", return_value=sqlite_session_factory),
+ patch.object(deletion_tasks, "get_archive_storage", return_value=archive),
+ patch.object(deletion_tasks, "get_export_storage", return_value=downloads),
+ patch.object(
+ deletion_tasks,
+ "RedisWorkflowRunArchiveMonthLock",
+ lambda **_kwargs: lambda **_kw: nullcontext(month_lease),
+ ),
+ ):
+ result = task.apply(args=(TENANT_ID, 2025, 3))
+
+ assert result.failed()
+ records = repository.list_for_tenant_month(TENANT_ID, year=2025, month=3)
+ assert len(records) == 1
+ assert records[0].deletion_status == WorkflowRunArchiveDeletionStatus.FAILED
+
+
+def test_completed_cleanup_returns_success(
+ sqlite_session_factory: sessionmaker[Session],
+ month_lease: WorkflowRunArchiveMonthLease,
+) -> None:
+ """A clean run still reports success with no retry."""
+ from tasks import workflow_run_archive_deletion_tasks as deletion_tasks
+
+ archive, downloads = MemoryStorage(), MemoryStorage()
+ target = _bundle()
+ _put_bundle(archive, target)
+ with sqlite_session_factory.begin() as session:
+ session.add(target)
+ repository = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory)
+ repository.request_deletion(TENANT_ID, year=2025, month=3)
+
+ task = deletion_tasks.delete_workflow_run_archive_task
+ with (
+ patch.object(deletion_tasks, "get_session_maker", return_value=sqlite_session_factory),
+ patch.object(deletion_tasks, "get_archive_storage", return_value=archive),
+ patch.object(deletion_tasks, "get_export_storage", return_value=downloads),
+ patch.object(
+ deletion_tasks,
+ "RedisWorkflowRunArchiveMonthLock",
+ lambda **_kwargs: lambda **_kw: nullcontext(month_lease),
+ ),
+ ):
+ result = task.apply(args=(TENANT_ID, 2025, 3))
+
+ assert result.successful()
+ assert not repository.list_for_tenant_month(TENANT_ID, year=2025, month=3)
+
+
@pytest.mark.parametrize("failure_stage", ["archive", "download"])
def test_failed_storage_delete_keeps_indexes_and_can_retry_partial_deletion(
sqlite_session_factory: sessionmaker[Session],