Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

feat: support permanent deletion of monthly workflow archives (#44052)

非法操作c11b36c70acf
diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py
index 6236e026701..52a6703f909 100644
--- a/api/controllers/console/__init__.py
+++ b/api/controllers/console/__init__.py
@@ -9,6 +9,12 @@ from flask_restx import Namespace
from controllers.common.errors import register_permission_error_handler
from libs.external_api import ExternalApi
from machinery.errors import ActiveWorkspaceRequiredError
+from services.retention.workflow_run.archive_log_service import (
+ WorkflowRunArchiveDownloadNotReadyError,
+ WorkflowRunArchiveDownloadTaskNotFoundError,
+ WorkflowRunArchiveInvalidMonthError,
+ WorkflowRunArchiveNotFoundError,
+)
bp = Blueprint("console", __name__, url_prefix="/console/api")
@@ -33,9 +39,44 @@ def _handle_active_workspace_required_error(error: ActiveWorkspaceRequiredError)
}, status.value
[email protected](WorkflowRunArchiveInvalidMonthError)
+def _handle_workflow_run_archive_invalid_month_error(
+ error: WorkflowRunArchiveInvalidMonthError,
+) -> tuple[dict[str, str | int], int]:
+ got_request_exception.send(current_app, exception=error)
+ status = HTTPStatus.BAD_REQUEST
+ return {"code": "bad_request", "message": str(error), "status": status.value}, status.value
+
+
[email protected](WorkflowRunArchiveNotFoundError)
[email protected](WorkflowRunArchiveDownloadTaskNotFoundError)
+def _handle_workflow_run_archive_not_found_error(
+ error: WorkflowRunArchiveNotFoundError | WorkflowRunArchiveDownloadTaskNotFoundError,
+) -> tuple[dict[str, str | int], int]:
+ got_request_exception.send(current_app, exception=error)
+ status = HTTPStatus.NOT_FOUND
+ return {"code": "not_found", "message": str(error), "status": status.value}, status.value
+
+
[email protected](WorkflowRunArchiveDownloadNotReadyError)
+def _handle_workflow_run_archive_download_not_ready_error(
+ error: WorkflowRunArchiveDownloadNotReadyError,
+) -> tuple[dict[str, str | int], int]:
+ got_request_exception.send(current_app, exception=error)
+ status = HTTPStatus.CONFLICT
+ return {"code": "conflict", "message": str(error), "status": status.value}, status.value
+
+
# Flask-RESTX dispatches in registration order; the shared Exception handler
-# must not shadow this Console-specific error mapping.
-cast(OrderedDict[type[Exception], object], api.error_handlers).move_to_end(ActiveWorkspaceRequiredError, last=False)
+# must not shadow these Console-specific error mappings.
+for error_type in (
+ ActiveWorkspaceRequiredError,
+ WorkflowRunArchiveInvalidMonthError,
+ WorkflowRunArchiveNotFoundError,
+ WorkflowRunArchiveDownloadTaskNotFoundError,
+ WorkflowRunArchiveDownloadNotReadyError,
+):
+ cast(OrderedDict[type[Exception], object], api.error_handlers).move_to_end(error_type, last=False)
console_ns = Namespace("console", description="Console management API operations", path="/")
diff --git a/api/controllers/console/workflow_run_archive.py b/api/controllers/console/workflow_run_archive.py
index 88b85fd1091..1edd5fe21d8 100644
--- a/api/controllers/console/workflow_run_archive.py
+++ b/api/controllers/console/workflow_run_archive.py
@@ -1,10 +1,10 @@
import datetime
from http import HTTPStatus
+from typing import Literal
from flask import redirect
from flask_restx import Resource
-from pydantic import BaseModel, Field
-from werkzeug.exceptions import Conflict, NotFound
+from pydantic import BaseModel, Field, ValidationError
from controllers.common.fields import RedirectResponse
from controllers.common.schema import register_response_schema_models, register_schema_models
@@ -20,13 +20,12 @@ from fields.base import ResponseModel
from libs.helper import dump_response
from machinery.context import RequestContext
from models.account import TenantAccountRole
+from services.retention.workflow_run.archive_deletion_task import WorkflowRunArchiveDeletionStatus
from services.retention.workflow_run.archive_download_task import (
WorkflowRunArchiveDownloadStatus,
)
from services.retention.workflow_run.archive_log_service import (
- WorkflowRunArchiveDownloadNotReadyError,
- WorkflowRunArchiveDownloadTaskNotFoundError,
- WorkflowRunArchiveNotFoundError,
+ WorkflowRunArchiveInvalidMonthError,
)
_WORKFLOW_RUN_ARCHIVE_ALLOWED_ROLES = frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN})
@@ -35,7 +34,7 @@ _WORKFLOW_RUN_ARCHIVE_ALLOWED_ROLES = frozenset({TenantAccountRole.OWNER, Tenant
class WorkflowRunArchiveDownloadPayload(BaseModel):
"""Request body for preparing one monthly workflow-run archive download."""
- year: int = Field(ge=1)
+ year: int = Field(ge=1, le=9999)
month: int = Field(ge=1, le=12)
@@ -72,6 +71,21 @@ class WorkflowRunArchiveMonthResponse(ResponseModel):
archive_bytes: int
latest_archived_at: datetime.datetime
download_task: WorkflowRunArchiveDownloadTaskResponse | None = None
+ deletion_status: (
+ Literal[
+ WorkflowRunArchiveDeletionStatus.PENDING,
+ WorkflowRunArchiveDeletionStatus.PROCESSING,
+ WorkflowRunArchiveDeletionStatus.FAILED,
+ ]
+ | None
+ ) = None
+
+
+class WorkflowRunArchiveDeletionTaskResponse(ResponseModel):
+ year: int
+ month: int
+ status: WorkflowRunArchiveDeletionStatus
+ error: str | None = None
class WorkflowRunArchiveListResponse(ResponseModel):
@@ -86,6 +100,7 @@ register_response_schema_models(
WorkflowRunArchiveMonthResponse,
WorkflowRunArchiveListResponse,
WorkflowRunArchiveDownloadTaskResponse,
+ WorkflowRunArchiveDeletionTaskResponse,
RedirectResponse,
)
@@ -105,6 +120,38 @@ class WorkflowRunArchivesApi(Resource):
return dump_response(WorkflowRunArchiveListResponse, archives)
+@console_ns.route("/workflow-run-archives/<int:year>/<int:month>")
+class WorkflowRunArchiveMonthApi(Resource):
+ @console_ns.doc("delete_workflow_run_archive")
+ @console_ns.doc(description="Permanently delete this workspace's monthly archive files and archive index")
+ @console_ns.doc(
+ params={
+ "year": {"in": "path", "type": "integer", "minimum": 1, "maximum": 9999},
+ "month": {"in": "path", "type": "integer", "minimum": 1, "maximum": 12},
+ }
+ )
+ @console_ns.response(
+ HTTPStatus.ACCEPTED,
+ "Archive deletion accepted",
+ console_ns.models[WorkflowRunArchiveDeletionTaskResponse.__name__],
+ )
+ @console_account_admission(
+ editions=frozenset({DeploymentEdition.CLOUD}),
+ # shortcut: allowed_roles only applies with RBAC disabled; add RBAC checks here when enabling it.
+ allowed_roles=frozenset({TenantAccountRole.OWNER}),
+ )
+ @cloud_edition_billing_paid_plan_required
+ def delete(self, request_context: RequestContext, year: int, month: int):
+ try:
+ archive_month = WorkflowRunArchiveDownloadPayload(year=year, month=month)
+ except ValidationError as exc:
+ raise WorkflowRunArchiveInvalidMonthError(str(exc)) from exc
+ task = application_services().workflow_run_archives.delete_archive(
+ request_context, year=archive_month.year, month=archive_month.month
+ )
+ return dump_response(WorkflowRunArchiveDeletionTaskResponse, task), HTTPStatus.ACCEPTED
+
+
@console_ns.route("/workflow-run-archives/downloads")
class WorkflowRunArchiveDownloadsApi(Resource):
@console_ns.doc("create_workflow_run_archive_download")
@@ -122,14 +169,11 @@ class WorkflowRunArchiveDownloadsApi(Resource):
@cloud_edition_billing_paid_plan_required
@model_validate(WorkflowRunArchiveDownloadPayload)
def post(self, req_data: WorkflowRunArchiveDownloadPayload, request_context: RequestContext):
- try:
- task = application_services().workflow_run_archives.create_download(
- request_context,
- year=req_data.year,
- month=req_data.month,
- )
- except WorkflowRunArchiveNotFoundError as exc:
- raise NotFound(str(exc)) from exc
+ task = application_services().workflow_run_archives.create_download(
+ request_context,
+ year=req_data.year,
+ month=req_data.month,
+ )
return dump_response(WorkflowRunArchiveDownloadTaskResponse, task), HTTPStatus.ACCEPTED
@@ -144,13 +188,10 @@ class WorkflowRunArchiveDownloadApi(Resource):
)
@cloud_edition_billing_paid_plan_required
def get(self, request_context: RequestContext, download_id: str):
- try:
- task = application_services().workflow_run_archives.get_download(
- request_context,
- download_id=download_id,
- )
- except WorkflowRunArchiveDownloadTaskNotFoundError as exc:
- raise NotFound(str(exc)) from exc
+ task = application_services().workflow_run_archives.get_download(
+ request_context,
+ download_id=download_id,
+ )
return dump_response(WorkflowRunArchiveDownloadTaskResponse, task)
@@ -170,13 +211,8 @@ class WorkflowRunArchiveDownloadFileApi(Resource):
)
@cloud_edition_billing_paid_plan_required
def get(self, request_context: RequestContext, download_id: str):
- try:
- presigned_url = application_services().workflow_run_archives.get_download_url(
- request_context,
- download_id=download_id,
- )
- except WorkflowRunArchiveDownloadTaskNotFoundError as exc:
- raise NotFound(str(exc)) from exc
- except WorkflowRunArchiveDownloadNotReadyError as exc:
- raise Conflict(str(exc)) from exc
+ presigned_url = application_services().workflow_run_archives.get_download_url(
+ request_context,
+ download_id=download_id,
+ )
return redirect(presigned_url, code=HTTPStatus.FOUND)
diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.py
index ab8e2e9dc8f..1108149efd6 100644
--- a/api/extensions/ext_application_services.py
+++ b/api/extensions/ext_application_services.py
@@ -167,6 +167,7 @@ from services.recommended_app_query_service import RecommendedAppQueryService
from services.remote_file_service import RemoteFileService
from services.resource_access_token_service import ResourceAccessTokenService
from services.retention.workflow_run.archive_download_adapters import (
+ dispatch_workflow_run_archive_deletion_task,
dispatch_workflow_run_archive_download_task,
sign_workflow_run_archive_download_url,
)
@@ -678,6 +679,7 @@ def build_application_services(
tasks=WorkflowRunArchiveDownloadTaskCache(redis=redis),
dispatcher=dispatch_workflow_run_archive_download_task,
sign_download_url=sign_workflow_run_archive_download_url,
+ deletion_dispatcher=dispatch_workflow_run_archive_deletion_task,
),
workflow_runs=WorkflowRunService(
workflow_runs=workflow_run_repository,
diff --git a/api/extensions/ext_celery.py b/api/extensions/ext_celery.py
index 7befa412af2..b3ad8391e18 100644
--- a/api/extensions/ext_celery.py
+++ b/api/extensions/ext_celery.py
@@ -182,6 +182,7 @@ def init_app(app: DifyApp) -> Celery:
"tasks.install_default_plugins_task", # tenant default plugin installation
"tasks.refresh_billing_vector_space_task", # billing vector-space cache refresh
"tasks.workflow_run_archive_download_tasks", # workflow-run archive download preparation
+ "tasks.workflow_run_archive_deletion_tasks", # permanent monthly archive deletion
]
day = dify_config.CELERY_BEAT_SCHEDULER_TIME
diff --git a/api/migrations/versions/2026_10_12_1200-f1d3c5b7a902_add_archive_bundle_deletion_status.py b/api/migrations/versions/2026_10_12_1200-f1d3c5b7a902_add_archive_bundle_deletion_status.py
new file mode 100644
index 00000000000..0e2fb1dca5f
--- /dev/null
+++ b/api/migrations/versions/2026_10_12_1200-f1d3c5b7a902_add_archive_bundle_deletion_status.py
@@ -0,0 +1,23 @@
+"""add archive bundle deletion status
+
+Revision ID: f1d3c5b7a902
+Revises: a8c9e2f1b704
+Create Date: 2026-10-12 12:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "f1d3c5b7a902"
+down_revision = "a8c9e2f1b704"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.add_column("workflow_run_archive_bundles", sa.Column("deletion_status", sa.String(16), nullable=True))
+
+
+def downgrade() -> None:
+ op.drop_column("workflow_run_archive_bundles", "deletion_status")
diff --git a/api/models/workflow.py b/api/models/workflow.py
index 5f0323eba23..42d17032762 100644
--- a/api/models/workflow.py
+++ b/api/models/workflow.py
@@ -1468,6 +1468,8 @@ class WorkflowRunArchiveBundle(DefaultFieldsDCMixin, TypeBase):
row_count: Mapped[int] = mapped_column(sa.BigInteger, nullable=False)
archive_bytes: Mapped[int] = mapped_column(sa.BigInteger, nullable=False)
archived_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
+ # A non-null value is a durable deletion request, including failed requests that may be retried.
+ deletion_status: Mapped[str | None] = mapped_column(String(16), nullable=True, default=None)
class ConversationVariable(TypeBase):
diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md
index 2c9fd6c4604..43bc6cb946e 100644
--- a/api/openapi/markdown/console-openapi.md
+++ b/api/openapi/markdown/console-openapi.md
@@ -9865,6 +9865,22 @@ Redirect to a prepared workflow-run archive ZIP file
| 302 | Redirect to pre-signed archive storage URL | **application/json**: [RedirectResponse](#redirectresponse)<br> |
| 409 | Download task is not ready | |
+### [DELETE] /workflow-run-archives/{year}/{month}
+Permanently delete this workspace's monthly archive files and archive index
+
+#### Parameters
+
+| Name | Located in | Description | Required | Schema |
+| ---- | ---------- | ----------- | -------- | ------ |
+| month | path | | Yes | integer |
+| year | path | | Yes | integer |
+
+#### Responses
+
+| Code | Description | Schema |
+| ---- | ----------- | ------ |
+| 202 | Archive deletion accepted | **application/json**: [WorkflowRunArchiveDeletionTaskResponse](#workflowrunarchivedeletiontaskresponse)<br> |
+
### [GET] /workflow/{workflow_run_id}/events
**Get workflow execution events stream after resume**
@@ -25985,6 +26001,21 @@ Enabled routes require at least two exits; drafts may omit conditions.
| result | string | | Yes |
| updated_at | integer | | Yes |
+#### WorkflowRunArchiveDeletionStatus
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| WorkflowRunArchiveDeletionStatus | string | | |
+
+#### WorkflowRunArchiveDeletionTaskResponse
+
+| Name | Type | Description | Required |
+| ---- | ---- | ----------- | -------- |
+| error | string | | No |
+| month | integer | | Yes |
+| status | [WorkflowRunArchiveDeletionStatus](#workflowrunarchivedeletionstatus) | | Yes |
+| year | integer | | Yes |
+
#### WorkflowRunArchiveDownloadPayload
Request body for preparing one monthly workflow-run archive download.
@@ -26034,6 +26065,7 @@ Lifecycle state for an asynchronous archive download request.
| ---- | ---- | ----------- | -------- |
| archive_bytes | integer | | Yes |
| bundle_count | integer | | Yes |
+| deletion_status | string, <br>**Available values:** "failed", "pending", "processing" | | No |
| download_task | [WorkflowRunArchiveDownloadTaskResponse](#workflowrunarchivedownloadtaskresponse) | | No |
| latest_archived_at | dateTime | | Yes |
| month | integer | | Yes |
diff --git a/api/repositories/workflow_run_archive_repository.py b/api/repositories/workflow_run_archive_repository.py
index c8296bc59ed..595da1cfecf 100644
--- a/api/repositories/workflow_run_archive_repository.py
+++ b/api/repositories/workflow_run_archive_repository.py
@@ -1,18 +1,20 @@
-"""Database repository for workflow-run archive bundle queries."""
+"""Tenant-scoped queries and deletion requests for workflow-run archive bundles."""
+from collections.abc import Sequence
from typing import override
-from sqlalchemy import select
+from sqlalchemy import delete, select, update
from sqlalchemy.orm import Session, sessionmaker
from models.workflow import WorkflowRunArchiveBundle
+from services.retention.workflow_run.archive_deletion_task import WorkflowRunArchiveDeletionStatus
from services.retention.workflow_run.archive_log_service import (
- WorkflowRunArchiveBundleQuery,
WorkflowRunArchiveBundleRecord,
+ WorkflowRunArchiveBundleStore,
)
-class WorkflowRunArchiveBundleQueryRepository(WorkflowRunArchiveBundleQuery):
+class WorkflowRunArchiveBundleQueryRepository(WorkflowRunArchiveBundleStore):
def __init__(self, *, session_factory: sessionmaker[Session]) -> None:
self._session_factory = session_factory
@@ -28,6 +30,8 @@ class WorkflowRunArchiveBundleQueryRepository(WorkflowRunArchiveBundleQuery):
WorkflowRunArchiveBundle.row_count,
WorkflowRunArchiveBundle.archive_bytes,
WorkflowRunArchiveBundle.archived_at,
+ WorkflowRunArchiveBundle.deletion_status,
+ WorkflowRunArchiveBundle.id,
)
.where(WorkflowRunArchiveBundle.tenant_id == tenant_id)
.order_by(
@@ -50,6 +54,8 @@ class WorkflowRunArchiveBundleQueryRepository(WorkflowRunArchiveBundleQuery):
row_count=row.row_count,
archive_bytes=row.archive_bytes,
archived_at=row.archived_at,
+ deletion_status=WorkflowRunArchiveDeletionStatus(row.deletion_status) if row.deletion_status else None,
+ catalog_id=row.id,
)
for row in rows
)
@@ -72,6 +78,8 @@ class WorkflowRunArchiveBundleQueryRepository(WorkflowRunArchiveBundleQuery):
WorkflowRunArchiveBundle.row_count,
WorkflowRunArchiveBundle.archive_bytes,
WorkflowRunArchiveBundle.archived_at,
+ WorkflowRunArchiveBundle.deletion_status,
+ WorkflowRunArchiveBundle.id,
)
.where(
WorkflowRunArchiveBundle.tenant_id == tenant_id,
@@ -93,6 +101,62 @@ class WorkflowRunArchiveBundleQueryRepository(WorkflowRunArchiveBundleQuery):
row_count=row.row_count,
archive_bytes=row.archive_bytes,
archived_at=row.archived_at,
+ deletion_status=WorkflowRunArchiveDeletionStatus(row.deletion_status) if row.deletion_status else None,
+ catalog_id=row.id,
)
for row in rows
)
+
+ @override
+ def request_deletion(self, tenant_id: str, *, year: int, month: int) -> WorkflowRunArchiveDeletionStatus | None:
+ """Commit the safety marker under the same row locks used by source-row maintenance."""
+ with self._session_factory.begin() as session:
+ bundles = session.scalars(
+ select(WorkflowRunArchiveBundle)
+ .where(
+ WorkflowRunArchiveBundle.tenant_id == tenant_id,
+ WorkflowRunArchiveBundle.year == year,
+ WorkflowRunArchiveBundle.month == month,
+ )
+ .order_by(WorkflowRunArchiveBundle.id)
+ .with_for_update()
+ ).all()
+ for bundle in bundles:
+ if bundle.deletion_status in (None, WorkflowRunArchiveDeletionStatus.FAILED):
+ bundle.deletion_status = WorkflowRunArchiveDeletionStatus.PENDING
+ if not bundles:
+ return None
+ if any(bundle.deletion_status == WorkflowRunArchiveDeletionStatus.PROCESSING for bundle in bundles):
+ return WorkflowRunArchiveDeletionStatus.PROCESSING
+ return WorkflowRunArchiveDeletionStatus.PENDING
+
+ @override
+ def set_deletion_status(
+ self, tenant_id: str, *, year: int, month: int, status: WorkflowRunArchiveDeletionStatus
+ ) -> None:
+ with self._session_factory.begin() as session:
+ session.execute(
+ update(WorkflowRunArchiveBundle)
+ .where(
+ WorkflowRunArchiveBundle.tenant_id == tenant_id,
+ WorkflowRunArchiveBundle.year == year,
+ WorkflowRunArchiveBundle.month == month,
+ WorkflowRunArchiveBundle.deletion_status.is_not(None),
+ )
+ .values(deletion_status=status)
+ )
+
+ @override
+ def remove_deleted_bundles(self, tenant_id: str, *, catalog_ids: Sequence[str]) -> None:
+ """Remove only requested bundle indexes after their object-store deletion succeeds."""
+ if not catalog_ids:
+ return
+ with self._session_factory.begin() as session:
+ for offset in range(0, len(catalog_ids), 1000):
+ session.execute(
+ delete(WorkflowRunArchiveBundle).where(
+ WorkflowRunArchiveBundle.tenant_id == tenant_id,
+ WorkflowRunArchiveBundle.id.in_(catalog_ids[offset : offset + 1000]),
+ WorkflowRunArchiveBundle.deletion_status.is_not(None),
+ )
+ )
diff --git a/api/services/retention/workflow_run/archive_deletion.py b/api/services/retention/workflow_run/archive_deletion.py
new file mode 100644
index 00000000000..912f01eef09
--- /dev/null
+++ b/api/services/retention/workflow_run/archive_deletion.py
@@ -0,0 +1,180 @@
+"""Permanently delete requested archive files without changing live workflow-run data."""
+
+import logging
+import re
+from collections.abc import Callable
+from uuid import UUID
+
+from libs.archive_storage import ArchiveStorage, ArchiveStorageError
+from services.retention.workflow_run.archive_bundle_index import (
+ ARCHIVE_BUNDLE_ROOT_PREFIX,
+ decode_archive_bundle_manifest,
+)
+from services.retention.workflow_run.archive_deletion_task import (
+ WorkflowRunArchiveDeletionStatus,
+ WorkflowRunArchiveDeletionTask,
+)
+from services.retention.workflow_run.archive_download_preparation import build_archive_download_month_prefix
+from services.retention.workflow_run.archive_log_service import (
+ WorkflowRunArchiveBundleRecord,
+ WorkflowRunArchiveBundleStore,
+)
+from services.retention.workflow_run.archive_month_lock import (
+ WorkflowRunArchiveMonthBusyError,
+ WorkflowRunArchiveMonthLease,
+ WorkflowRunArchiveMonthLock,
+)
+from services.retention.workflow_run.constants import ARCHIVE_BUNDLE_MANIFEST_NAME
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowRunArchiveDeleter:
+ """Delete only durably requested DB bundle identities; redelivery and partial storage failures are retryable."""
+
+ def __init__(
+ self,
+ *,
+ bundles: WorkflowRunArchiveBundleStore,
+ archive_storage_provider: Callable[[], ArchiveStorage],
+ download_storage_provider: Callable[[], ArchiveStorage],
+ month_lock: WorkflowRunArchiveMonthLock,
+ ) -> None:
+ self._bundles = bundles
+ self._archive_storage_provider = archive_storage_provider
+ self._download_storage_provider = download_storage_provider
+ self._month_lock = month_lock
+
+ def delete(self, *, tenant_id: str, year: int, month: int) -> WorkflowRunArchiveDeletionTask:
+ """Keep all requested indexes until archive objects and temporary downloads are successfully removed."""
+ try:
+ # Tenant and immutable bundle paths are reconstructed from scoped DB rows, never task-supplied keys.
+ UUID(tenant_id)
+ if not 1 <= year <= 9999 or not 1 <= month <= 12:
+ raise ValueError("Invalid workflow run archive month")
+ with self._month_lock(tenant_id=tenant_id, year=year, month=month) as lease:
+ return self._delete_locked(tenant_id=tenant_id, year=year, month=month, lease=lease)
+ except Exception as exc:
+ logger.exception(
+ "audit: workflow_run_archive.deletion_failed tenant_id=%s year=%s month=%s",
+ tenant_id,
+ year,
+ month,
+ extra={
+ "attributes": {
+ "audit": True,
+ "event": "workflow_run_archive.deletion_failed",
+ "tenant_id": tenant_id,
+ "year": year,
+ "month": month,
+ }
+ },
+ )
+ self._bundles.set_deletion_status(
+ tenant_id, year=year, month=month, status=WorkflowRunArchiveDeletionStatus.FAILED
+ )
+ if isinstance(exc, WorkflowRunArchiveMonthBusyError):
+ raise
+ return WorkflowRunArchiveDeletionTask(
+ year, month, WorkflowRunArchiveDeletionStatus.FAILED, "Failed to delete archive files. Please retry."
+ )
+
+ def _delete_locked(
+ self, *, tenant_id: str, year: int, month: int, lease: WorkflowRunArchiveMonthLease
+ ) -> WorkflowRunArchiveDeletionTask:
+ lease.assert_owned()
+ records = self._bundles.list_for_tenant_month(tenant_id, year=year, month=month)
+ requested = [record for record in records if record.deletion_status is not None]
+ if not requested:
+ return WorkflowRunArchiveDeletionTask(year, month, WorkflowRunArchiveDeletionStatus.COMPLETED)
+ self._bundles.set_deletion_status(
+ tenant_id, year=year, month=month, status=WorkflowRunArchiveDeletionStatus.PROCESSING
+ )
+ archive_storage = self._archive_storage_provider()
+ for record in requested:
+ lease.assert_owned()
+ prefix = _bundle_prefix(tenant_id=tenant_id, record=record)
+ logger.info("Permanently deleting workflow run archive bundle: tenant=%s prefix=%s", tenant_id, prefix)
+ self._delete_bundle(archive_storage, tenant_id=tenant_id, record=record, prefix=prefix)
+
+ # The month lease serializes ZIP uploads with this cleanup, including interrupted upload workers.
+ # Remove ZIPs before the last durable marker disappears.
+ download_storage = self._download_storage_provider()
+ download_prefix = build_archive_download_month_prefix(tenant_id=tenant_id, year=year, month=month)
+ for key in _list_scoped_objects(download_storage, download_prefix):
+ lease.assert_owned()
+ download_storage.delete_object(key)
+ if _list_scoped_objects(download_storage, download_prefix):
+ raise ArchiveStorageError("Temporary workflow run archive downloads could not be deleted")
+
+ # shortcut: rare catalog-writer races are tolerated; share this lease if deletion becomes frequent.
+ lease.assert_owned()
+ self._bundles.remove_deleted_bundles(tenant_id, catalog_ids=[record.catalog_id for record in requested])
+ logger.info(
+ "audit: workflow_run_archive.deletion_completed tenant_id=%s year=%s month=%s bundle_count=%s",
+ tenant_id,
+ year,
+ month,
+ len(requested),
+ extra={
+ "attributes": {
+ "audit": True,
+ "event": "workflow_run_archive.deletion_completed",
+ "tenant_id": tenant_id,
+ "year": year,
+ "month": month,
+ "bundle_count": len(requested),
+ }
+ },
+ )
+ return WorkflowRunArchiveDeletionTask(year, month, WorkflowRunArchiveDeletionStatus.COMPLETED)
+
+ @staticmethod
+ def _delete_bundle(
+ storage: ArchiveStorage, *, tenant_id: str, record: WorkflowRunArchiveBundleRecord, prefix: str
+ ) -> None:
+ keys = _list_scoped_objects(storage, prefix)
+ manifest_key = f"{prefix}{ARCHIVE_BUNDLE_MANIFEST_NAME}"
+ if manifest_key in keys:
+ try:
+ manifest = decode_archive_bundle_manifest(storage.get_object(manifest_key))
+ except FileNotFoundError:
+ # Another delivery may have completed this same immutable bundle.
+ manifest = None
+ if manifest is not None:
+ expected = (tenant_id, record.year, record.month, record.shard, record.bundle_id, prefix.rstrip("/"))
+ actual = (
+ manifest["tenant_id"],
+ manifest["year"],
+ manifest["month"],
+ manifest["shard"],
+ manifest["bundle_id"],
+ manifest["object_prefix"],
+ )
+ if actual != expected:
+ raise ValueError("Archive manifest identity does not match the requested bundle")
+ for key in keys:
+ if key != manifest_key:
+ storage.delete_object(key)
+ if any(key != manifest_key for key in _list_scoped_objects(storage, prefix)):
+ raise ArchiveStorageError("Workflow run archive objects could not be deleted")
+ # Preserve the manifest until its payloads have been removed, including on a partially failed attempt.
+ storage.delete_object(manifest_key)
+ if _list_scoped_objects(storage, prefix):
+ raise ArchiveStorageError("Workflow run archive manifest could not be deleted")
+
+
+def _bundle_prefix(*, tenant_id: str, record: WorkflowRunArchiveBundleRecord) -> str:
+ if not re.fullmatch(r"\d{2}-of-\d{2}", record.shard) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", record.bundle_id):
+ raise ValueError("Invalid workflow run archive bundle identity")
+ return (
+ f"{ARCHIVE_BUNDLE_ROOT_PREFIX}tenant_prefix={tenant_id[0].lower()}/tenant_id={tenant_id}/"
+ f"year={record.year:04d}/month={record.month:02d}/shard={record.shard}/bundle={record.bundle_id}/"
+ )
+
+
+def _list_scoped_objects(storage: ArchiveStorage, prefix: str) -> list[str]:
+ keys = storage.list_objects(prefix)
+ if any(not key.startswith(prefix) for key in keys):
+ raise ValueError("Archive storage returned an object outside the requested bundle")
+ return keys
diff --git a/api/services/retention/workflow_run/archive_deletion_task.py b/api/services/retention/workflow_run/archive_deletion_task.py
new file mode 100644
index 00000000000..0d2340b831d
--- /dev/null
+++ b/api/services/retention/workflow_run/archive_deletion_task.py
@@ -0,0 +1,24 @@
+"""States returned for permanent deletion of a monthly workflow-run archive."""
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Protocol
+
+
+class WorkflowRunArchiveDeletionStatus(StrEnum):
+ PENDING = "pending"
+ PROCESSING = "processing"
+ COMPLETED = "completed"
+ FAILED = "failed"
+
+
+@dataclass(frozen=True)
+class WorkflowRunArchiveDeletionTask:
+ year: int
+ month: int
+ status: WorkflowRunArchiveDeletionStatus
+ error: str | None = None
+
+
+class WorkflowRunArchiveDeletionDispatcher(Protocol):
+ def __call__(self, *, tenant_id: str, year: int, month: int) -> None: ...
diff --git a/api/services/retention/workflow_run/archive_download_adapters.py b/api/services/retention/workflow_run/archive_download_adapters.py
index 2586bb6d4cc..abaa5e13586 100644
--- a/api/services/retention/workflow_run/archive_download_adapters.py
+++ b/api/services/retention/workflow_run/archive_download_adapters.py
@@ -6,6 +6,13 @@ from services.retention.workflow_run.archive_download_task import WorkflowRunArc
_ARCHIVE_DOWNLOAD_MIME_TYPE = "application/zip"
+def dispatch_workflow_run_archive_deletion_task(*, tenant_id: str, year: int, month: int) -> None:
+ """Enqueue an archive-only deletion whose durable request is already committed."""
+ from tasks.workflow_run_archive_deletion_tasks import delete_workflow_run_archive_task
+
+ delete_workflow_run_archive_task.apply_async(args=(tenant_id, year, month))
+
+
def dispatch_workflow_run_archive_download_task(
task: WorkflowRunArchiveDownloadTask,
) -> None:
diff --git a/api/services/retention/workflow_run/archive_download_preparation.py b/api/services/retention/workflow_run/archive_download_preparation.py
index 69aaf002351..302236a018e 100644
--- a/api/services/retention/workflow_run/archive_download_preparation.py
+++ b/api/services/retention/workflow_run/archive_download_preparation.py
@@ -13,7 +13,10 @@ import logging
import re
import zipfile
from collections.abc import Callable, Sequence
+from functools import partial
+import gevent
+import gevent.monkey
import pyarrow as pa
import pyarrow.compute as pc
import pyarrow.csv as pa_csv
@@ -37,6 +40,7 @@ from services.retention.workflow_run.archive_log_service import (
WorkflowRunArchiveBundleRecord,
WorkflowRunArchiveDownloadTaskStore,
)
+from services.retention.workflow_run.archive_month_lock import WorkflowRunArchiveMonthLock
from services.retention.workflow_run.constants import (
ARCHIVE_BUNDLE_FORMAT,
ARCHIVE_BUNDLE_MANIFEST_NAME,
@@ -71,11 +75,13 @@ class WorkflowRunArchiveDownloadPreparer:
download_storage_provider: Callable[[], ArchiveStorage],
bundles: WorkflowRunArchiveBundleQuery,
cache: WorkflowRunArchiveDownloadTaskStore,
+ month_lock: WorkflowRunArchiveMonthLock,
) -> None:
self._archive_storage_provider = archive_storage_provider
self._download_storage_provider = download_storage_provider
self._bundles = bundles
self.cache = cache
+ self._month_lock = month_lock
def prepare(self, *, tenant_id: str, download_id: str) -> WorkflowRunArchiveDownloadTask | None:
"""Prepare a ZIP for an existing Redis task and persist terminal task state."""
@@ -89,13 +95,24 @@ class WorkflowRunArchiveDownloadPreparer:
return task
processing_task = self._mark_processing(task)
try:
- archive_storage = self._archive_storage_provider()
- download_storage = self._download_storage_provider()
- bundles = self._get_task_bundles(processing_task)
- payload = self._build_zip_payload(archive_storage, processing_task, bundles)
- storage_key = build_archive_download_storage_key(processing_task)
- download_storage.put_object(storage_key, payload)
- return self._mark_ready(processing_task, storage_key=storage_key, file_size_bytes=len(payload))
+ with self._month_lock(tenant_id=tenant_id, year=processing_task.year, month=processing_task.month) as lease:
+ lease.assert_owned()
+ bundles = self._get_task_bundles(processing_task)
+ archive_storage = self._archive_storage_provider()
+ download_storage = self._download_storage_provider()
+ payload = self._build_zip_payload(archive_storage, processing_task, bundles)
+ storage_key = build_archive_download_storage_key(processing_task)
+ lease.assert_owned()
+ download_storage.put_object(storage_key, payload)
+ try:
+ # A deletion request may be committed during ZIP preparation. The deleter waits on this month
+ # lease and cleans the ZIP even if compensation fails or this worker crashes after uploading.
+ self._get_task_bundles(processing_task)
+ lease.assert_owned()
+ except Exception:
+ download_storage.delete_object(storage_key)
+ raise
+ return self._mark_ready(processing_task, storage_key=storage_key, file_size_bytes=len(payload))
except Exception as exc:
logger.exception("Failed to prepare workflow run archive download %s", download_id)
return self._mark_failed(processing_task, error=str(exc))
@@ -126,20 +143,19 @@ class WorkflowRunArchiveDownloadPreparer:
object_key = entry["object_key"]
table_payload = storage.get_object(object_key)
_validate_table_payload(object_key=object_key, entry=entry, payload=table_payload)
- csv_payload = _parquet_payload_to_csv(
- table_payload,
- include_header=table_name not in csv_headers_written,
+ csv_payload = _run_archive_cpu_work(
+ partial(
+ _parquet_payload_to_csv,
+ table_payload,
+ include_header=table_name not in csv_headers_written,
+ )
)
if not csv_payload:
continue
csv_buffers.setdefault(table_name, io.BytesIO()).write(csv_payload)
csv_headers_written.add(table_name)
- buffer = io.BytesIO()
- with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as archive:
- for table_name, csv_buffer in sorted(csv_buffers.items()):
- archive.writestr(f"{zip_root}/{table_name}.csv", csv_buffer.getvalue())
- return buffer.getvalue()
+ return _run_archive_cpu_work(lambda: _zip_csv_buffers(csv_buffers, zip_root=zip_root))
def _mark_processing(self, task: WorkflowRunArchiveDownloadTask) -> WorkflowRunArchiveDownloadTask:
now = datetime.datetime.now(datetime.UTC)
@@ -214,9 +230,14 @@ def build_archive_download_file_name(task: WorkflowRunArchiveDownloadTask) -> st
def build_archive_download_storage_key(task: WorkflowRunArchiveDownloadTask) -> str:
"""Return the deterministic object-store key for a prepared download ZIP."""
+ prefix = build_archive_download_month_prefix(tenant_id=task.tenant_id, year=task.year, month=task.month)
+ return f"{prefix}{task.download_id}.zip"
+
+
+def build_archive_download_month_prefix(*, tenant_id: str, year: int, month: int) -> str:
return (
- f"{ARCHIVE_DOWNLOAD_ROOT_PREFIX}tenant_prefix={task.tenant_id[0].lower()}/tenant_id={task.tenant_id}/"
- f"year={task.year:04d}/month={task.month:02d}/{task.download_id}.zip"
+ f"{ARCHIVE_DOWNLOAD_ROOT_PREFIX}tenant_prefix={tenant_id[0].lower()}/tenant_id={tenant_id}/"
+ f"year={year:04d}/month={month:02d}/"
)
@@ -224,6 +245,8 @@ def _select_task_bundles(
indexed_bundles: Sequence[WorkflowRunArchiveBundleRecord],
task: WorkflowRunArchiveDownloadTask,
) -> list[WorkflowRunArchiveBundleRecord]:
+ if any(bundle.deletion_status is not None for bundle in indexed_bundles):
+ raise ValueError("This workflow run archive is being permanently deleted.")
if task.bundle_refs:
requested_refs = [(ref.shard, ref.bundle_id) for ref in task.bundle_refs]
else:
@@ -346,3 +369,18 @@ def _parquet_payload_to_csv(payload: bytes, *, include_header: bool) -> bytes:
write_options=pa_csv.WriteOptions(include_header=include_header),
)
return buffer.getvalue()
+
+
+def _zip_csv_buffers(csv_buffers: dict[str, io.BytesIO], *, zip_root: str) -> bytes:
+ buffer = io.BytesIO()
+ with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as archive:
+ for table_name, csv_buffer in sorted(csv_buffers.items()):
+ archive.writestr(f"{zip_root}/{table_name}.csv", csv_buffer.getvalue())
+ return buffer.getvalue()
+
+
+def _run_archive_cpu_work[T](work: Callable[[], T]) -> T:
+ """Keep lease renewal responsive in gevent workers; storage I/O stays on the calling greenlet."""
+ if gevent.monkey.is_module_patched("threading"):
+ return gevent.get_hub().threadpool.apply(work)
+ return work()
diff --git a/api/services/retention/workflow_run/archive_log_service.py b/api/services/retention/workflow_run/archive_log_service.py
index 875e9bd499b..c8106000bd9 100644
--- a/api/services/retention/workflow_run/archive_log_service.py
+++ b/api/services/retention/workflow_run/archive_log_service.py
@@ -9,6 +9,11 @@ from dataclasses import dataclass
from typing import Any, NamedTuple, Protocol
from machinery.context import RequestContext
+from services.retention.workflow_run.archive_deletion_task import (
+ WorkflowRunArchiveDeletionDispatcher,
+ WorkflowRunArchiveDeletionStatus,
+ WorkflowRunArchiveDeletionTask,
+)
from services.retention.workflow_run.archive_download_task import (
WorkflowRunArchiveDownloadStatus,
WorkflowRunArchiveDownloadTask,
@@ -30,6 +35,8 @@ class WorkflowRunArchiveBundleRecord(NamedTuple):
row_count: int
archive_bytes: int
archived_at: datetime.datetime
+ deletion_status: WorkflowRunArchiveDeletionStatus | None = None
+ catalog_id: str = ""
class WorkflowRunArchiveBundleQuery(Protocol):
@@ -44,6 +51,16 @@ class WorkflowRunArchiveBundleQuery(Protocol):
) -> Sequence[WorkflowRunArchiveBundleRecord]: ...
+class WorkflowRunArchiveBundleStore(WorkflowRunArchiveBundleQuery, Protocol):
+ def request_deletion(self, tenant_id: str, *, year: int, month: int) -> WorkflowRunArchiveDeletionStatus | None: ...
+
+ def set_deletion_status(
+ self, tenant_id: str, *, year: int, month: int, status: WorkflowRunArchiveDeletionStatus
+ ) -> None: ...
+
+ def remove_deleted_bundles(self, tenant_id: str, *, catalog_ids: Sequence[str]) -> None: ...
+
+
class WorkflowRunArchiveDownloadTaskStore(Protocol):
def get(self, *, tenant_id: str, download_id: str) -> WorkflowRunArchiveDownloadTask | None: ...
@@ -78,6 +95,7 @@ class WorkflowRunArchiveMonth:
archive_bytes: int
latest_archived_at: datetime.datetime
download_task: WorkflowRunArchiveDownloadTask | None
+ deletion_status: WorkflowRunArchiveDeletionStatus | None = None
@dataclass(frozen=True)
@@ -102,6 +120,10 @@ class WorkflowRunArchiveNotFoundError(Exception):
"""Raised when no archive bundles exist for a requested tenant/month."""
+class WorkflowRunArchiveInvalidMonthError(Exception):
+ """Raised when a requested archive year or month is outside the supported range."""
+
+
class WorkflowRunArchiveDownloadTaskNotFoundError(Exception):
"""Raised when the temporary download task has expired or never existed."""
@@ -114,15 +136,17 @@ class WorkflowRunArchiveService:
def __init__(
self,
*,
- bundles: WorkflowRunArchiveBundleQuery,
+ bundles: WorkflowRunArchiveBundleStore,
tasks: WorkflowRunArchiveDownloadTaskStore,
dispatcher: WorkflowRunArchiveDownloadTaskDispatcher,
sign_download_url: WorkflowRunArchiveDownloadUrlSigner,
+ deletion_dispatcher: WorkflowRunArchiveDeletionDispatcher,
) -> None:
self._bundles = bundles
self._tasks = tasks
self._dispatcher = dispatcher
self._sign_download_url = sign_download_url
+ self._deletion_dispatcher = deletion_dispatcher
def list_archives(self, context: RequestContext) -> WorkflowRunArchiveList:
"""Return monthly archive metadata for the active workspace."""
@@ -135,6 +159,7 @@ class WorkflowRunArchiveService:
for year, month in sorted(records_by_month, reverse=True):
records = records_by_month[(year, month)]
bundle_refs = [(record.shard, record.bundle_id) for record in records]
+ deletion_status = _month_deletion_status(records)
months.append(
WorkflowRunArchiveMonth(
year=year,
@@ -144,12 +169,15 @@ class WorkflowRunArchiveService:
row_count=sum(record.row_count for record in records),
archive_bytes=sum(record.archive_bytes for record in records),
latest_archived_at=max(record.archived_at for record in records),
- download_task=self._get_cached_month_download_task(
+ download_task=None
+ if deletion_status
+ else self._get_cached_month_download_task(
tenant_id=tenant_id,
year=year,
month=month,
bundle_refs=bundle_refs,
),
+ deletion_status=deletion_status,
)
)
@@ -176,6 +204,8 @@ class WorkflowRunArchiveService:
bundles = self._bundles.list_for_tenant_month(tenant_id, year=year, month=month)
if not bundles:
raise WorkflowRunArchiveNotFoundError(f"Workflow run archive not found: {year:04d}-{month:02d}")
+ if _month_deletion_status(bundles):
+ raise WorkflowRunArchiveDownloadNotReadyError("This workflow run archive is being permanently deleted.")
bundle_refs = [(bundle.shard, bundle.bundle_id) for bundle in bundles]
download_id = build_archive_download_id(
@@ -221,8 +251,88 @@ class WorkflowRunArchiveService:
task = self._tasks.get(tenant_id=tenant_id, download_id=download_id)
if task is None:
raise WorkflowRunArchiveDownloadTaskNotFoundError(f"Workflow run archive download not found: {download_id}")
+ bundles = self._bundles.list_for_tenant_month(tenant_id, year=task.year, month=task.month)
+ available_refs = {(bundle.shard, bundle.bundle_id) for bundle in bundles}
+ requested_refs = {(ref.shard, ref.bundle_id) for ref in task.bundle_refs}
+ available_ids = {bundle.bundle_id for bundle in bundles}
+ if (
+ task.tenant_id != tenant_id
+ or _month_deletion_status(bundles)
+ or not bundles
+ or (requested_refs and not requested_refs.issubset(available_refs))
+ or not set(task.bundle_ids).issubset(available_ids)
+ ):
+ raise WorkflowRunArchiveDownloadTaskNotFoundError("This workflow run archive is no longer available.")
return task
+ def delete_archive(self, context: RequestContext, *, year: int, month: int) -> WorkflowRunArchiveDeletionTask:
+ """Durably disable downloads before enqueueing an idempotent archive-only deletion."""
+ tenant_id = context.active_workspace_id
+ status = self._bundles.request_deletion(tenant_id, year=year, month=month)
+ event = (
+ "workflow_run_archive.deletion_requested" if status is not None else "workflow_run_archive.deletion_noop"
+ )
+ audit_attributes = {
+ "audit": True,
+ "tenant_id": tenant_id,
+ "account_id": context.account_id,
+ "year": year,
+ "month": month,
+ "request_id": context.request_id,
+ }
+ # The repository has committed the deletion marker; a request is not proof of completed deletion.
+ logger.info(
+ "audit: %s tenant_id=%s account_id=%s year=%s month=%s request_id=%s status=%s",
+ event,
+ tenant_id,
+ context.account_id,
+ year,
+ month,
+ context.request_id,
+ status,
+ extra={"attributes": {**audit_attributes, "event": event, "deletion_status": status}},
+ )
+ if status is None:
+ return WorkflowRunArchiveDeletionTask(year, month, WorkflowRunArchiveDeletionStatus.COMPLETED)
+ try:
+ # Re-dispatch pending/processing requests too: a lost or interrupted worker must be recoverable.
+ self._deletion_dispatcher(tenant_id=tenant_id, year=year, month=month)
+ except Exception:
+ logger.exception(
+ "audit: workflow_run_archive.deletion_enqueue_failed "
+ "tenant_id=%s account_id=%s year=%s month=%s request_id=%s",
+ tenant_id,
+ context.account_id,
+ year,
+ month,
+ context.request_id,
+ extra={"attributes": {**audit_attributes, "event": "workflow_run_archive.deletion_enqueue_failed"}},
+ )
+ self._bundles.set_deletion_status(
+ tenant_id, year=year, month=month, status=WorkflowRunArchiveDeletionStatus.FAILED
+ )
+ return WorkflowRunArchiveDeletionTask(
+ year, month, WorkflowRunArchiveDeletionStatus.FAILED, "Failed to enqueue archive deletion."
+ )
+ logger.info(
+ "audit: workflow_run_archive.deletion_enqueued "
+ "tenant_id=%s account_id=%s year=%s month=%s request_id=%s status=%s",
+ tenant_id,
+ context.account_id,
+ year,
+ month,
+ context.request_id,
+ status,
+ extra={
+ "attributes": {
+ **audit_attributes,
+ "event": "workflow_run_archive.deletion_enqueued",
+ "deletion_status": status,
+ }
+ },
+ )
+ return WorkflowRunArchiveDeletionTask(year, month, status)
+
def get_download_url(self, context: RequestContext, *, download_id: str) -> str:
"""Return a short-lived URL for a ready archive download."""
task = self.get_download(context, download_id=download_id)
@@ -284,3 +394,18 @@ class WorkflowRunArchiveService:
expires_at_utc = expires_at if expires_at.tzinfo else expires_at.replace(tzinfo=datetime.UTC)
remaining_seconds = int((expires_at_utc - datetime.datetime.now(datetime.UTC)).total_seconds())
return max(1, min(3600, remaining_seconds))
+
+
+def _month_deletion_status(
+ records: Sequence[WorkflowRunArchiveBundleRecord],
+) -> WorkflowRunArchiveDeletionStatus | None:
+ """Any durable deletion request disables the entire month's downloads."""
+ statuses = {record.deletion_status for record in records}
+ for status in (
+ WorkflowRunArchiveDeletionStatus.FAILED,
+ WorkflowRunArchiveDeletionStatus.PROCESSING,
+ WorkflowRunArchiveDeletionStatus.PENDING,
+ ):
+ if status in statuses:
+ return status
+ return None
diff --git a/api/services/retention/workflow_run/archive_month_lock.py b/api/services/retention/workflow_run/archive_month_lock.py
new file mode 100644
index 00000000000..97a4e1010ba
--- /dev/null
+++ b/api/services/retention/workflow_run/archive_month_lock.py
@@ -0,0 +1,76 @@
+"""Serialize monthly archive storage work while renewing the lease during blocking S3 operations."""
+
+import logging
+import threading
+from collections.abc import Generator
+from contextlib import AbstractContextManager, contextmanager
+from typing import Protocol
+
+from redis.lock import Lock
+
+from extensions.ext_redis import RedisClientWrapper
+
+logger = logging.getLogger(__name__)
+
+_LOCK_TTL_SECONDS = 120
+_RENEW_INTERVAL_SECONDS = 30
+
+
+class WorkflowRunArchiveMonthBusyError(RuntimeError):
+ """Another upload or deletion still owns the monthly storage lease."""
+
+
+class WorkflowRunArchiveMonthLease(Protocol):
+ def assert_owned(self) -> None: ...
+
+
+class WorkflowRunArchiveMonthLock(Protocol):
+ def __call__(
+ self, *, tenant_id: str, year: int, month: int
+ ) -> AbstractContextManager[WorkflowRunArchiveMonthLease]: ...
+
+
+class _MonthLease:
+ def __init__(self, lock: Lock, lost: threading.Event) -> None:
+ self._lock = lock
+ self._lost = lost
+
+ def assert_owned(self) -> None:
+ if self._lost.is_set() or not self._lock.owned():
+ raise RuntimeError("Workflow run archive month lock was lost")
+
+
+class RedisWorkflowRunArchiveMonthLock:
+ """A shared month lease prevents ZIP uploads from racing permanent deletion or surviving worker crashes."""
+
+ def __init__(self, *, redis: RedisClientWrapper) -> None:
+ self._redis = redis
+
+ @contextmanager
+ def __call__(self, *, tenant_id: str, year: int, month: int) -> Generator[WorkflowRunArchiveMonthLease]:
+ name = f"workflow_run_archive_month:{tenant_id}:{year:04d}-{month:02d}"
+ lock = self._redis.lock(name, timeout=_LOCK_TTL_SECONDS, blocking_timeout=_LOCK_TTL_SECONDS, thread_local=False)
+ if not lock.acquire():
+ raise WorkflowRunArchiveMonthBusyError("Workflow run archive month is busy; please retry")
+ stopped, lost = threading.Event(), threading.Event()
+
+ def renew() -> None:
+ while not stopped.wait(_RENEW_INTERVAL_SECONDS):
+ try:
+ lock.reacquire()
+ except Exception:
+ lost.set()
+ logger.exception("Lost workflow run archive month lease: %s", name)
+ return
+
+ heartbeat = threading.Thread(target=renew, daemon=True, name=f"ArchiveMonthLease({name})")
+ heartbeat.start()
+ try:
+ yield _MonthLease(lock, lost)
+ finally:
+ stopped.set()
+ heartbeat.join(timeout=5)
+ try:
+ lock.release()
+ except Exception:
+ logger.warning("Failed to release workflow run archive month lease: %s", name, exc_info=True)
diff --git a/api/services/retention/workflow_run/bundle_archive_maintenance.py b/api/services/retention/workflow_run/bundle_archive_maintenance.py
index a2fa0a6db94..c08bab98c1d 100644
--- a/api/services/retention/workflow_run/bundle_archive_maintenance.py
+++ b/api/services/retention/workflow_run/bundle_archive_maintenance.py
@@ -318,6 +318,7 @@ class WorkflowRunBundleArchiveMaintenance:
conditions = [
WorkflowRunArchiveBundle.year == target_year,
WorkflowRunArchiveBundle.month == target_month,
+ WorkflowRunArchiveBundle.deletion_status.is_(None),
]
if tenant_ids is not None:
conditions.append(WorkflowRunArchiveBundle.tenant_id.in_(tenant_ids))
@@ -419,6 +420,7 @@ class WorkflowRunBundleArchiveMaintenance:
conditions = [
WorkflowRunArchiveBundle.year == target_year,
WorkflowRunArchiveBundle.month == target_month,
+ WorkflowRunArchiveBundle.deletion_status.is_(None),
]
if tenant_ids:
conditions.append(WorkflowRunArchiveBundle.tenant_id.in_(tenant_ids))
@@ -465,9 +467,10 @@ class WorkflowRunBundleArchiveMaintenance:
target_month: int,
shard: str | None,
) -> None:
- """Reject a keyset cursor that cannot safely represent the requested catalog scope."""
+ """Validate the cursor scope when its catalog row still exists."""
if cursor_bundle is None:
- raise ValueError("after_catalog_id does not exist in the workflow run archive bundle catalog")
+ # Permanent archive deletion can remove a saved cursor; its immutable ID remains an exclusive boundary.
+ return
if cursor_bundle.year != target_year or cursor_bundle.month != target_month:
raise ValueError("after_catalog_id is outside the requested archive month")
if tenant_ids is not None and cursor_bundle.tenant_id not in tenant_ids:
@@ -697,16 +700,18 @@ class WorkflowRunBundleArchiveMaintenance:
@staticmethod
def _lock_catalog_entry(session: Session, catalog_entry: ArchiveBundleCatalogEntry) -> None:
- locked_catalog_id = session.scalar(
- select(WorkflowRunArchiveBundle.id)
+ locked_catalog = session.scalar(
+ select(WorkflowRunArchiveBundle)
.where(
WorkflowRunArchiveBundle.id == catalog_entry.catalog_id,
WorkflowRunArchiveBundle.tenant_id == catalog_entry.tenant_id,
)
.with_for_update()
)
- if locked_catalog_id is None:
+ if locked_catalog is None:
raise ValueError("archive catalog row disappeared before bundle maintenance")
+ if locked_catalog.deletion_status is not None:
+ raise ValueError("archive files are pending permanent deletion; source rows cannot be deleted or restored")
def _validate_archive_object(
self,
diff --git a/api/tasks/workflow_run_archive_deletion_tasks.py b/api/tasks/workflow_run_archive_deletion_tasks.py
new file mode 100644
index 00000000000..41899e3e96c
--- /dev/null
+++ b/api/tasks/workflow_run_archive_deletion_tasks.py
@@ -0,0 +1,37 @@
+"""Archive-only deletion jobs on the existing workflow archive queue."""
+
+import logging
+
+from celery import shared_task
+
+from core.db.session_factory import get_session_maker
+from extensions.ext_redis import redis_client
+from libs.archive_storage import get_archive_storage, get_export_storage
+from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository
+from services.retention.workflow_run.archive_deletion import WorkflowRunArchiveDeleter
+from services.retention.workflow_run.archive_month_lock import (
+ RedisWorkflowRunArchiveMonthLock,
+)
+from tasks.workflow_run_archive_download_tasks import WORKFLOW_RUN_ARCHIVE_DOWNLOAD_QUEUE
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task(
+ queue=WORKFLOW_RUN_ARCHIVE_DOWNLOAD_QUEUE,
+ acks_late=True,
+ reject_on_worker_lost=True,
+ autoretry_for=(Exception,),
+ retry_backoff=30,
+ retry_backoff_max=120,
+ retry_kwargs={"max_retries": 5},
+)
+def delete_workflow_run_archive_task(tenant_id: str, year: int, month: int) -> None:
+ """Redelivery is safe because only DB rows with durable deletion requests are selected."""
+ logger.info("Deleting workflow run archive month: tenant=%s year=%s month=%s", tenant_id, year, month)
+ WorkflowRunArchiveDeleter(
+ bundles=WorkflowRunArchiveBundleQueryRepository(session_factory=get_session_maker()),
+ archive_storage_provider=get_archive_storage,
+ download_storage_provider=get_export_storage,
+ month_lock=RedisWorkflowRunArchiveMonthLock(redis=redis_client),
+ ).delete(tenant_id=tenant_id, year=year, month=month)
diff --git a/api/tasks/workflow_run_archive_download_tasks.py b/api/tasks/workflow_run_archive_download_tasks.py
index c230b427982..7d630e64a6a 100644
--- a/api/tasks/workflow_run_archive_download_tasks.py
+++ b/api/tasks/workflow_run_archive_download_tasks.py
@@ -10,6 +10,7 @@ from libs.archive_storage import get_archive_storage, get_export_storage
from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository
from services.retention.workflow_run.archive_download_preparation import WorkflowRunArchiveDownloadPreparer
from services.retention.workflow_run.archive_download_task_cache import WorkflowRunArchiveDownloadTaskCache
+from services.retention.workflow_run.archive_month_lock import RedisWorkflowRunArchiveMonthLock
logger = logging.getLogger(__name__)
@@ -25,4 +26,5 @@ def prepare_workflow_run_archive_download_task(tenant_id: str, download_id: str)
bundles=WorkflowRunArchiveBundleQueryRepository(session_factory=get_session_maker()),
cache=WorkflowRunArchiveDownloadTaskCache(redis=redis_client),
download_storage_provider=get_export_storage,
+ month_lock=RedisWorkflowRunArchiveMonthLock(redis=redis_client),
).prepare(tenant_id=tenant_id, download_id=download_id)
diff --git a/api/tests/unit_tests/controllers/console/test_workflow_run_archive.py b/api/tests/unit_tests/controllers/console/test_workflow_run_archive.py
index 852c2ca9e6c..1f26fa80609 100644
--- a/api/tests/unit_tests/controllers/console/test_workflow_run_archive.py
+++ b/api/tests/unit_tests/controllers/console/test_workflow_run_archive.py
@@ -4,29 +4,40 @@ import inspect
from collections.abc import Callable
from types import CodeType, SimpleNamespace
from typing import Protocol, cast
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
import pytest
from flask import Flask
-from werkzeug.exceptions import Conflict, Forbidden, NotFound
+from flask.testing import FlaskClient
+from flask_restx import Api, Resource
+from werkzeug.exceptions import Forbidden, NotFound
from werkzeug.wrappers import Response
+from controllers.console import api as console_api
from controllers.console import flask_admission, workflow_run_archive
from controllers.console.workflow_run_archive import (
WorkflowRunArchiveDownloadApi,
WorkflowRunArchiveDownloadFileApi,
WorkflowRunArchiveDownloadPayload,
WorkflowRunArchiveDownloadsApi,
+ WorkflowRunArchiveMonthApi,
WorkflowRunArchivesApi,
)
from enums import CloudPlan, DeploymentEdition
+from extensions.ext_application_services import ApplicationServices
from libs.login import AccountWithTenant
from machinery.context import RequestContext
from models import Account, TenantAccountRole
+from services.retention.workflow_run.archive_deletion_task import (
+ WorkflowRunArchiveDeletionStatus,
+ WorkflowRunArchiveDeletionTask,
+)
from services.retention.workflow_run.archive_log_service import (
WorkflowRunArchiveDownloadNotReadyError,
WorkflowRunArchiveDownloadTaskNotFoundError,
+ WorkflowRunArchiveInvalidMonthError,
WorkflowRunArchiveNotFoundError,
+ WorkflowRunArchiveService,
)
from tests.unit_tests.config_override import apply_config_overrides
from tests.unit_tests.model_factories import make_account
@@ -42,6 +53,7 @@ _ENDPOINTS = [
WorkflowRunArchiveDownloadsApi.post,
WorkflowRunArchiveDownloadApi.get,
WorkflowRunArchiveDownloadFileApi.get,
+ WorkflowRunArchiveMonthApi.delete,
]
@@ -67,6 +79,15 @@ def _admission_injector(method: Callable[..., object]) -> Callable[..., object]:
return cast(Callable[..., object], current)
+def archive_service(
+ account_application_services: ApplicationServices,
+ monkeypatch: pytest.MonkeyPatch,
+) -> WorkflowRunArchiveService:
+ monkeypatch.setattr(workflow_run_archive, "application_services", lambda: account_application_services)
+ return account_application_services.workflow_run_archives
+
+
@pytest.mark.parametrize("method", _ENDPOINTS)
def test_workflow_run_archive_endpoints_declare_cloud_paid_plan_admission(method: Callable[..., object]) -> None:
decorator_names = set()
@@ -81,7 +102,12 @@ def test_workflow_run_archive_endpoints_declare_cloud_paid_plan_admission(method
assert "console_account_admission" in decorator_names
assert "cloud_edition_billing_paid_plan_required" in decorator_names
assert "only_edition_cloud" not in decorator_names
- assert allowed_roles == frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN})
+ expected_roles = (
+ {TenantAccountRole.OWNER}
+ if method == WorkflowRunArchiveMonthApi.delete
+ else {TenantAccountRole.OWNER, TenantAccountRole.ADMIN}
+ )
+ assert allowed_roles == frozenset(expected_roles)
@pytest.mark.parametrize("method", _ENDPOINTS)
@@ -109,6 +135,7 @@ def test_workflow_run_archive_endpoints_reject_non_manager_when_rbac_is_disabled
(WorkflowRunArchiveDownloadsApi.post, ()),
(WorkflowRunArchiveDownloadApi.get, ("download-1",)),
(WorkflowRunArchiveDownloadFileApi.get, ("download-1",)),
+ (WorkflowRunArchiveMonthApi.delete, (2025, 3)),
],
)
def test_workflow_run_archive_endpoints_reject_unsupported_edition_after_initialization(
@@ -164,6 +191,52 @@ def test_workflow_run_archive_endpoint_allows_admitted_role_when_rbac_is_enabled
assert context.active_workspace_id == "tenant-1"
[email protected]("role", [TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.NORMAL])
+def test_workflow_run_archive_role_permissions_when_rbac_is_disabled(
+ monkeypatch: pytest.MonkeyPatch,
+ archive_service: WorkflowRunArchiveService,
+ role: TenantAccountRole,
+) -> None:
+ apply_config_overrides(monkeypatch, RBAC_ENABLED=False)
+ identity = AccountWithTenant(account=_account(role), tenant_id="tenant-1")
+ monkeypatch.setattr(flask_admission, "current_account_with_tenant", lambda: identity)
+ monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", lambda: identity)
+ monkeypatch.setattr(
+ "controllers.console.wraps.BillingService.get_info",
+ lambda _tenant_id, **_kwargs: {"subscription": {"plan": CloudPlan.TEAM}},
+ )
+ task = object()
+ archives = object()
+ monkeypatch.setattr(workflow_run_archive, "dump_response", lambda _model, value: value)
+ app = Flask(__name__)
+
+ with (
+ patch.object(archive_service, "delete_archive", return_value=task) as delete_archive,
+ patch.object(archive_service, "create_download", return_value=task) as create_download,
+ patch.object(archive_service, "list_archives", return_value=archives) as list_archives,
+ app.test_request_context(method="POST", json={"year": 2025, "month": 3}),
+ ):
+ if role == TenantAccountRole.OWNER:
+ assert _admission_injector(WorkflowRunArchiveMonthApi.delete)(None, 2025, 3) == (task, 202)
+ context = delete_archive.call_args.args[0]
+ assert context.account_id == "account-1"
+ assert context.active_workspace_id == "tenant-1"
+ delete_archive.assert_called_once_with(context, year=2025, month=3)
+ else:
+ with pytest.raises(Forbidden):
+ _admission_injector(WorkflowRunArchiveMonthApi.delete)(None, 2025, 3)
+ delete_archive.assert_not_called()
+
+ if role == TenantAccountRole.ADMIN:
+ assert _admission_injector(WorkflowRunArchivesApi.get)(None) is archives
+ assert _admission_injector(WorkflowRunArchiveDownloadsApi.post)(None) == (task, 202)
+ context = list_archives.call_args.args[0]
+ assert context.account_id == "account-1"
+ assert context.active_workspace_id == "tenant-1"
+ list_archives.assert_called_once_with(context)
+ create_download.assert_called_once_with(context, year=2025, month=3)
+
+
def test_workflow_run_archive_endpoints_delegate_to_application_service(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -173,6 +246,7 @@ def test_workflow_run_archive_endpoints_delegate_to_application_service(
service.list_archives.return_value = archives
service.create_download.return_value = task
service.get_download.return_value = task
+ service.delete_archive.return_value = task
monkeypatch.setattr(
workflow_run_archive,
"application_services",
@@ -188,51 +262,54 @@ def test_workflow_run_archive_endpoints_delegate_to_application_service(
_CONTEXT,
) == (task, 202)
assert _original(WorkflowRunArchiveDownloadApi.get)(None, _CONTEXT, "download-1") is task
+ assert _original(WorkflowRunArchiveMonthApi.delete)(None, _CONTEXT, 2025, 3) == (task, 202)
service.list_archives.assert_called_once_with(_CONTEXT)
service.create_download.assert_called_once_with(_CONTEXT, year=2025, month=3)
service.get_download.assert_called_once_with(_CONTEXT, download_id="download-1")
+ service.delete_archive.assert_called_once_with(_CONTEXT, year=2025, month=3)
@pytest.mark.parametrize(
- ("method", "service_method", "service_error", "http_error", "args"),
+ ("method", "service_method", "service_error", "args"),
[
(
WorkflowRunArchiveDownloadsApi.post,
"create_download",
WorkflowRunArchiveNotFoundError("archive missing"),
- NotFound,
+ (WorkflowRunArchiveDownloadPayload(year=2025, month=3), _CONTEXT),
+ ),
+ (
+ WorkflowRunArchiveDownloadsApi.post,
+ "create_download",
+ WorkflowRunArchiveDownloadNotReadyError("archive deleting"),
(WorkflowRunArchiveDownloadPayload(year=2025, month=3), _CONTEXT),
),
(
WorkflowRunArchiveDownloadApi.get,
"get_download",
WorkflowRunArchiveDownloadTaskNotFoundError("task missing"),
- NotFound,
(_CONTEXT, "download-1"),
),
(
WorkflowRunArchiveDownloadFileApi.get,
"get_download_url",
WorkflowRunArchiveDownloadTaskNotFoundError("task missing"),
- NotFound,
(_CONTEXT, "download-1"),
),
(
WorkflowRunArchiveDownloadFileApi.get,
"get_download_url",
WorkflowRunArchiveDownloadNotReadyError("task pending"),
- Conflict,
(_CONTEXT, "download-1"),
),
],
)
-def test_workflow_run_archive_endpoints_translate_service_errors(
+def test_workflow_run_archive_endpoints_propagate_domain_errors(
monkeypatch: pytest.MonkeyPatch,
method: Callable[..., object],
service_method: str,
service_error: Exception,
- http_error: type[Exception],
args: tuple[object, ...],
) -> None:
service = MagicMock()
@@ -243,8 +320,9 @@ def test_workflow_run_archive_endpoints_translate_service_errors(
lambda: SimpleNamespace(workflow_run_archives=service),
)
- with pytest.raises(http_error):
+ with pytest.raises(type(service_error)) as raised:
_original(method)(None, *args)
+ assert raised.value is service_error
def test_workflow_run_archive_file_redirects_to_signed_url(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -263,3 +341,154 @@ def test_workflow_run_archive_file_redirects_to_signed_url(monkeypatch: pytest.M
assert isinstance(response, Response)
assert response.status_code == 302
assert response.location == "https://storage.example.com/archive.zip"
+
+
[email protected](("year", "month"), [(0, 3), (-1, 3), (10000, 3), (2025, 0), (2025, 13), (2025, -1)])
+def test_delete_archive_validates_path_month_before_dispatch(
+ archive_service: WorkflowRunArchiveService,
+ year: int,
+ month: int,
+) -> None:
+ with patch.object(archive_service, "delete_archive") as delete_archive:
+ with pytest.raises(WorkflowRunArchiveInvalidMonthError):
+ _original(WorkflowRunArchiveMonthApi.delete)(None, _CONTEXT, year, month)
+
+ delete_archive.assert_not_called()
+
+
+def archive_http_client() -> FlaskClient:
+ app = Flask(__name__)
+ app.config["RESTX_ERROR_404_HELP"] = False
+ api = Api(app, doc=False)
+ # Exercise the production handler registrations and their dispatch order.
+ api.error_handlers = console_api.error_handlers.copy()
+
+ class ArchiveMonth(Resource):
+ def delete(self, year: int, month: int) -> object:
+ return _original(WorkflowRunArchiveMonthApi.delete)(self, _CONTEXT, year, month)
+
+ class ArchiveDownloads(Resource):
+ def post(self) -> object:
+ return _original(WorkflowRunArchiveDownloadsApi.post)(
+ self, WorkflowRunArchiveDownloadPayload(year=2025, month=3), _CONTEXT
+ )
+
+ class ArchiveDownload(Resource):
+ def get(self, download_id: str) -> object:
+ return _original(WorkflowRunArchiveDownloadApi.get)(self, _CONTEXT, download_id)
+
+ class ArchiveDownloadFile(Resource):
+ def get(self, download_id: str) -> object:
+ return _original(WorkflowRunArchiveDownloadFileApi.get)(self, _CONTEXT, download_id)
+
+ api.add_resource(ArchiveMonth, "/workflow-run-archives/<int:year>/<int:month>")
+ api.add_resource(ArchiveDownloads, "/workflow-run-archives/downloads")
+ api.add_resource(ArchiveDownload, "/workflow-run-archives/downloads/<string:download_id>")
+ api.add_resource(ArchiveDownloadFile, "/workflow-run-archives/downloads/<string:download_id>/file")
+ return app.test_client()
+
+
+ ("method", "path", "service_method", "error", "status", "code"),
+ [
+ (
+ "delete",
+ "/workflow-run-archives/2025/3",
+ "delete_archive",
+ WorkflowRunArchiveInvalidMonthError("Invalid archive month"),
+ 400,
+ "bad_request",
+ ),
+ (
+ "post",
+ "/workflow-run-archives/downloads",
+ "create_download",
+ WorkflowRunArchiveNotFoundError("archive missing"),
+ 404,
+ "not_found",
+ ),
+ (
+ "post",
+ "/workflow-run-archives/downloads",
+ "create_download",
+ WorkflowRunArchiveDownloadNotReadyError("archive deleting"),
+ 409,
+ "conflict",
+ ),
+ (
+ "get",
+ "/workflow-run-archives/downloads/download-1",
+ "get_download",
+ WorkflowRunArchiveDownloadTaskNotFoundError("task missing"),
+ 404,
+ "not_found",
+ ),
+ (
+ "get",
+ "/workflow-run-archives/downloads/download-1/file",
+ "get_download_url",
+ WorkflowRunArchiveDownloadTaskNotFoundError("task missing"),
+ 404,
+ "not_found",
+ ),
+ (
+ "get",
+ "/workflow-run-archives/downloads/download-1/file",
+ "get_download_url",
+ WorkflowRunArchiveDownloadNotReadyError("task pending"),
+ 409,
+ "conflict",
+ ),
+ ],
+)
+def test_workflow_run_archive_domain_errors_keep_http_responses(
+ archive_service: WorkflowRunArchiveService,
+ archive_http_client: FlaskClient,
+ method: str,
+ path: str,
+ service_method: str,
+ error: Exception,
+ status: int,
+ code: str,
+) -> None:
+ with patch.object(archive_service, service_method, side_effect=error):
+ response = archive_http_client.open(path, method=method.upper())
+
+ assert response.status_code == status
+ assert response.json == {"code": code, "message": str(error), "status": status}
+
+
+def test_invalid_archive_month_keeps_bad_request_response(
+ archive_service: WorkflowRunArchiveService,
+ archive_http_client: FlaskClient,
+) -> None:
+ with patch.object(archive_service, "delete_archive") as delete_archive:
+ response = archive_http_client.delete("/workflow-run-archives/2025/13")
+
+ delete_archive.assert_not_called()
+
+ assert response.status_code == 400
+ payload = response.json
+ assert isinstance(payload, dict)
+ assert payload["code"] == "bad_request"
+ assert payload["status"] == 400
+ assert "month" in payload["message"]
+
+
+def test_archive_domain_error_handlers_preserve_accepted_deletion_and_download_redirect(
+ archive_service: WorkflowRunArchiveService,
+ archive_http_client: FlaskClient,
+) -> None:
+ task = WorkflowRunArchiveDeletionTask(year=2025, month=3, status=WorkflowRunArchiveDeletionStatus.PENDING)
+ with (
+ patch.object(archive_service, "delete_archive", return_value=task),
+ patch.object(archive_service, "get_download_url", return_value="https://storage.example.com/archive.zip"),
+ ):
+ deletion = archive_http_client.delete("/workflow-run-archives/2025/3")
+ download = archive_http_client.get("/workflow-run-archives/downloads/download-1/file")
+
+ assert deletion.status_code == 202
+ assert deletion.json == {"year": 2025, "month": 3, "status": "pending", "error": None}
+ assert download.status_code == 302
+ assert download.location == "https://storage.example.com/archive.zip"
diff --git a/api/tests/unit_tests/extensions/test_celery_ssl.py b/api/tests/unit_tests/extensions/test_celery_ssl.py
index d6495785d71..7fb3ba6c8e3 100644
--- a/api/tests/unit_tests/extensions/test_celery_ssl.py
+++ b/api/tests/unit_tests/extensions/test_celery_ssl.py
@@ -1,6 +1,7 @@
"""Tests for Celery SSL configuration."""
import ssl
+from importlib import import_module
from unittest.mock import MagicMock, patch
from enums import DeploymentEdition
@@ -248,6 +249,15 @@ class TestCelerySSLConfiguration:
assert celery_app.conf["result_backend_transport_options"]["global_keyprefix"] == "enterprise-a:"
assert "tasks.collect_agent_resources_task" in celery_app.conf["imports"]
assert "tasks.delete_conversation_task" in celery_app.conf["imports"]
+ assert "tasks.workflow_run_archive_deletion_tasks" in celery_app.conf["imports"]
+ import_module("tasks.workflow_run_archive_deletion_tasks")
+ task_name = "tasks.workflow_run_archive_deletion_tasks.delete_workflow_run_archive_task"
+ archive_deletion = celery_app.tasks[task_name]
+ assert archive_deletion.queue == "workflow_archive"
+ assert archive_deletion.acks_late is True
+ assert archive_deletion.reject_on_worker_lost is True
+ assert archive_deletion.autoretry_for == (Exception,)
+ assert archive_deletion.retry_kwargs["max_retries"] == 5
assert celery_app.conf["beat_schedule"]["conversation_cleanup_sweeper"]["task"] == (
"tasks.delete_conversation_task.sweep_deleted_conversations"
)
diff --git a/api/tests/unit_tests/repositories/test_workflow_run_archive_repository.py b/api/tests/unit_tests/repositories/test_workflow_run_archive_repository.py
index c6c763a5410..6ac3346257e 100644
--- a/api/tests/unit_tests/repositories/test_workflow_run_archive_repository.py
+++ b/api/tests/unit_tests/repositories/test_workflow_run_archive_repository.py
@@ -4,6 +4,7 @@ from sqlalchemy.orm import Session, sessionmaker
from models.workflow import WorkflowRunArchiveBundle
from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository
+from services.retention.workflow_run.archive_deletion_task import WorkflowRunArchiveDeletionStatus
from services.retention.workflow_run.archive_log_service import WorkflowRunArchiveBundleRecord
@@ -83,7 +84,7 @@ def test_list_for_tenant_returns_ordered_pure_records(
"tenant-1"
)
- assert records == (
+ assert tuple(record._replace(catalog_id="") for record in records) == (
WorkflowRunArchiveBundleRecord(2025, 3, "00-of-02", "bundle-a", 40, 360, 1024, previous),
WorkflowRunArchiveBundleRecord(2025, 3, "01-of-02", "bundle-b", 60, 540, 3072, latest),
WorkflowRunArchiveBundleRecord(2025, 2, "00-of-01", "bundle-c", 1, 9, 1024, previous),
@@ -132,7 +133,32 @@ def test_list_for_tenant_month_filters_and_orders_bundles(
"tenant-1", year=2025, month=3
)
- assert records == (
+ assert tuple(record._replace(catalog_id="") for record in records) == (
WorkflowRunArchiveBundleRecord(2025, 3, "00-of-01", "bundle-a", 1, 9, 1024, archived_at),
WorkflowRunArchiveBundleRecord(2025, 3, "00-of-01", "bundle-b", 1, 9, 1024, archived_at),
)
+
+
+def test_request_deletion_is_scoped_and_preserves_processing_requests(
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ active = _bundle(tenant_id="tenant-1", year=2025, month=3, shard="00-of-01", bundle_id="active")
+ processing = _bundle(tenant_id="tenant-1", year=2025, month=3, shard="00-of-01", bundle_id="processing")
+ other_tenant = _bundle(tenant_id="tenant-2", year=2025, month=3, shard="00-of-01", bundle_id="other")
+ other_month = _bundle(tenant_id="tenant-1", year=2025, month=2, shard="00-of-01", bundle_id="other-month")
+ processing.deletion_status = "processing"
+ with sqlite_session_factory.begin() as session:
+ session.add_all([active, processing, other_tenant, other_month])
+ repository = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory)
+
+ assert repository.request_deletion("tenant-1", year=2025, month=3) == WorkflowRunArchiveDeletionStatus.PROCESSING
+ with sqlite_session_factory() as session:
+ for bundle, expected_status in (
+ (active, "pending"),
+ (processing, "processing"),
+ (other_tenant, None),
+ (other_month, None),
+ ):
+ reloaded = session.get(WorkflowRunArchiveBundle, bundle.id)
+ assert reloaded is not None
+ assert reloaded.deletion_status == expected_status
diff --git a/api/tests/unit_tests/services/retention/workflow_run/conftest.py b/api/tests/unit_tests/services/retention/workflow_run/conftest.py
new file mode 100644
index 00000000000..012e3956484
--- /dev/null
+++ b/api/tests/unit_tests/services/retention/workflow_run/conftest.py
@@ -0,0 +1,16 @@
+import threading
+from collections.abc import Generator
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from extensions.ext_redis import RedisClientWrapper
+from services.retention.workflow_run.archive_month_lock import _MonthLease
+
+
+def month_lease(redis_transport: tuple[RedisClientWrapper, MagicMock]) -> Generator[_MonthLease]:
+ redis, _commands = redis_transport
+ lock = redis.lock("archive-test", timeout=120, thread_local=False)
+ with patch.object(lock, "owned", return_value=True):
+ yield _MonthLease(lock, threading.Event())
diff --git a/api/tests/unit_tests/services/retention/workflow_run/test_archive_deletion.py b/api/tests/unit_tests/services/retention/workflow_run/test_archive_deletion.py
new file mode 100644
index 00000000000..6cf64d6015c
--- /dev/null
+++ b/api/tests/unit_tests/services/retention/workflow_run/test_archive_deletion.py
@@ -0,0 +1,428 @@
+"""Archive deletion uses real tenant-scoped catalog rows and leaves live workflow data intact."""
+
+import datetime
+import json
+import logging
+import threading
+from collections.abc import Generator
+from contextlib import contextmanager, nullcontext
+from typing import override
+from unittest.mock import MagicMock, patch
+
+import pytest
+from sqlalchemy import event, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from extensions.ext_redis import RedisClientWrapper
+from graphon.enums import WorkflowExecutionStatus
+from libs.archive_storage import ArchiveStorage, ArchiveStorageError
+from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom
+from models.workflow import WorkflowRun, WorkflowRunArchiveBundle, WorkflowType
+from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository
+from services.retention.workflow_run.archive_deletion import WorkflowRunArchiveDeleter
+from services.retention.workflow_run.archive_deletion_task import WorkflowRunArchiveDeletionStatus
+from services.retention.workflow_run.archive_download_preparation import (
+ WorkflowRunArchiveDownloadPreparer,
+ build_archive_download_month_prefix,
+)
+from services.retention.workflow_run.archive_download_task import (
+ WorkflowRunArchiveDownloadStatus,
+ build_archive_download_id,
+ build_pending_archive_download_task,
+)
+from services.retention.workflow_run.archive_download_task_cache import WorkflowRunArchiveDownloadTaskCache
+from services.retention.workflow_run.archive_month_lock import WorkflowRunArchiveMonthLease
+
+TENANT_ID = "1251fe32-c0c7-4fe2-a7bd-a8105267faf5"
+OTHER_TENANT_ID = "2251fe32-c0c7-4fe2-a7bd-a8105267faf5"
+
+
+class MemoryStorage(ArchiveStorage):
+ def __init__(self) -> None:
+ self.objects: dict[str, bytes] = {}
+ self.deleted: list[str] = []
+ self.fail_key: str | None = None
+
+ @override
+ def get_object(self, key: str) -> bytes:
+ try:
+ return self.objects[key]
+ except KeyError as exc:
+ raise FileNotFoundError(key) from exc
+
+ @override
+ def list_objects(self, prefix: str) -> list[str]:
+ return sorted(key for key in self.objects if key.startswith(prefix))
+
+ @override
+ def delete_object(self, key: str) -> None:
+ if key == self.fail_key:
+ raise ArchiveStorageError("storage unavailable")
+ self.deleted.append(key)
+ self.objects.pop(key, None)
+
+ @override
+ def put_object(self, key: str, data: bytes) -> str:
+ self.objects[key] = data
+ return "checksum"
+
+
+def _bundle(*, tenant_id: str = TENANT_ID, month: int = 3, bundle_id: str = "bundle-a") -> WorkflowRunArchiveBundle:
+ return WorkflowRunArchiveBundle(
+ tenant_id=tenant_id,
+ year=2025,
+ month=month,
+ shard="00-of-01",
+ bundle_id=bundle_id,
+ workflow_run_count=1,
+ row_count=7,
+ archive_bytes=512,
+ archived_at=datetime.datetime(2026, 6, 25),
+ )
+
+
+def _put_bundle(storage: MemoryStorage, bundle: WorkflowRunArchiveBundle) -> str:
+ prefix = (
+ f"workflow-runs/v2/tenant_prefix={bundle.tenant_id[0]}/tenant_id={bundle.tenant_id}/"
+ f"year={bundle.year:04d}/month={bundle.month:02d}/shard={bundle.shard}/bundle={bundle.bundle_id}/"
+ )
+ storage.objects[f"{prefix}manifest.json"] = json.dumps(
+ {
+ "tenant_id": bundle.tenant_id,
+ "year": bundle.year,
+ "month": bundle.month,
+ "shard": bundle.shard,
+ "bundle_id": bundle.bundle_id,
+ "object_prefix": prefix.rstrip("/"),
+ }
+ ).encode()
+ storage.objects[f"{prefix}workflow_runs.parquet"] = b"runs"
+ storage.objects[f"{prefix}workflow_node_executions.parquet"] = b"nodes"
+ storage.objects[f"{prefix}.deleted"] = b"marker"
+ return prefix
+
+
+def _deleter(
+ repository: WorkflowRunArchiveBundleQueryRepository,
+ archive: MemoryStorage,
+ downloads: MemoryStorage,
+ lease: WorkflowRunArchiveMonthLease,
+) -> WorkflowRunArchiveDeleter:
+ return WorkflowRunArchiveDeleter(
+ bundles=repository,
+ archive_storage_provider=lambda: archive,
+ download_storage_provider=lambda: downloads,
+ month_lock=lambda **_kwargs: nullcontext(lease),
+ )
+
+
+def test_archive_deletion_removes_only_requested_files_indexes_and_downloads(
+ sqlite_session_factory: sessionmaker[Session],
+ caplog: pytest.LogCaptureFixture,
+ month_lease: WorkflowRunArchiveMonthLease,
+) -> None:
+ caplog.set_level(logging.INFO, logger="services.retention.workflow_run.archive_deletion")
+ archive, downloads = MemoryStorage(), MemoryStorage()
+ target, other_tenant, other_month = _bundle(), _bundle(tenant_id=OTHER_TENANT_ID), _bundle(month=2)
+ target_prefix = _put_bundle(archive, target)
+ _put_bundle(archive, other_tenant)
+ _put_bundle(archive, other_month)
+ index_key = target_prefix.rsplit("bundle=", 1)[0] + "index.json"
+ archive.objects[index_key] = b'{"run_ids":["run-1"]}'
+ target_download = build_archive_download_month_prefix(tenant_id=TENANT_ID, year=2025, month=3) + "download.zip"
+ other_download = build_archive_download_month_prefix(tenant_id=OTHER_TENANT_ID, year=2025, month=3) + "download.zip"
+ downloads.objects.update({target_download: b"target", other_download: b"other"})
+ run = WorkflowRun(
+ id="run-1",
+ tenant_id=TENANT_ID,
+ app_id="app-1",
+ workflow_id="workflow-1",
+ type=WorkflowType.WORKFLOW,
+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
+ version="1",
+ graph="{}",
+ inputs="{}",
+ status=WorkflowExecutionStatus.SUCCEEDED,
+ outputs="{}",
+ error=None,
+ elapsed_time=0,
+ total_tokens=0,
+ total_steps=0,
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by="account-1",
+ created_at=datetime.datetime(2025, 3, 15),
+ finished_at=datetime.datetime(2025, 3, 15),
+ )
+ with sqlite_session_factory.begin() as session:
+ session.add_all([target, other_tenant, other_month, run])
+ repository = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory)
+ writes: list[str] = []
+
+ def record_writes(_conn, _cursor, statement: str, _parameters, _context, _executemany) -> None:
+ if statement.lower().startswith(("delete ", "update ", "insert ")):
+ writes.append(statement.lower())
+
+ engine = sqlite_session_factory.kw["bind"]
+ event.listen(engine, "before_cursor_execute", record_writes)
+ try:
+ repository.request_deletion(TENANT_ID, year=2025, month=3)
+ result = _deleter(repository, archive, downloads, month_lease).delete(tenant_id=TENANT_ID, year=2025, month=3)
+ finally:
+ event.remove(engine, "before_cursor_execute", record_writes)
+
+ assert result.status == WorkflowRunArchiveDeletionStatus.COMPLETED
+ assert not archive.list_objects(target_prefix)
+ assert archive.deleted[-1] == target_prefix + "manifest.json"
+ assert archive.objects[index_key] == b'{"run_ids":["run-1"]}'
+ assert downloads.objects == {other_download: b"other"}
+ assert writes
+ assert all("workflow_run_archive_bundles" in statement for statement in writes)
+ with sqlite_session_factory() as session:
+ assert session.get(WorkflowRun, "run-1") is not None
+ assert {bundle.id for bundle in session.scalars(select(WorkflowRunArchiveBundle))} == {
+ other_tenant.id,
+ other_month.id,
+ }
+ completed = [
+ record.__dict__.get("attributes", {})
+ for record in caplog.records
+ if record.__dict__.get("attributes", {}).get("event") == "workflow_run_archive.deletion_completed"
+ ]
+ assert completed == [
+ {
+ "audit": True,
+ "event": "workflow_run_archive.deletion_completed",
+ "tenant_id": TENANT_ID,
+ "year": 2025,
+ "month": 3,
+ "bundle_count": 1,
+ }
+ ]
+
+
[email protected]("failure_stage", ["archive", "download"])
+def test_failed_storage_delete_keeps_indexes_and_can_retry_partial_deletion(
+ sqlite_session_factory: sessionmaker[Session],
+ failure_stage: str,
+ caplog: pytest.LogCaptureFixture,
+ month_lease: WorkflowRunArchiveMonthLease,
+) -> None:
+ caplog.set_level(logging.INFO, logger="services.retention.workflow_run.archive_deletion")
+ archive, downloads = MemoryStorage(), MemoryStorage()
+ target = _bundle()
+ prefix = _put_bundle(archive, target)
+ download_key = build_archive_download_month_prefix(tenant_id=TENANT_ID, year=2025, month=3) + "download.zip"
+ downloads.objects[download_key] = b"download"
+ if failure_stage == "archive":
+ archive.fail_key = prefix + "workflow_runs.parquet"
+ else:
+ downloads.fail_key = download_key
+ with sqlite_session_factory.begin() as session:
+ session.add(target)
+ repository = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory)
+ repository.request_deletion(TENANT_ID, year=2025, month=3)
+ deleter = _deleter(repository, archive, downloads, month_lease)
+
+ assert deleter.delete(tenant_id=TENANT_ID, year=2025, month=3).status == WorkflowRunArchiveDeletionStatus.FAILED
+ records = repository.list_for_tenant_month(TENANT_ID, year=2025, month=3)
+ assert len(records) == 1
+ assert records[0].deletion_status == WorkflowRunArchiveDeletionStatus.FAILED
+ if failure_stage == "archive":
+ assert prefix + "manifest.json" in archive.objects
+ assert download_key in downloads.objects
+ assert not any(
+ record.__dict__.get("attributes", {}).get("event") == "workflow_run_archive.deletion_completed"
+ for record in caplog.records
+ )
+ failed = next(
+ record.__dict__.get("attributes", {})
+ for record in caplog.records
+ if record.__dict__.get("attributes", {}).get("event") == "workflow_run_archive.deletion_failed"
+ )
+ assert (failed["tenant_id"], failed["year"], failed["month"]) == (TENANT_ID, 2025, 3)
+
+ archive.fail_key = downloads.fail_key = None
+ repository.request_deletion(TENANT_ID, year=2025, month=3)
+ assert deleter.delete(tenant_id=TENANT_ID, year=2025, month=3).status == WorkflowRunArchiveDeletionStatus.COMPLETED
+ assert not repository.list_for_tenant_month(TENANT_ID, year=2025, month=3)
+ assert not archive.list_objects(prefix)
+ assert not downloads.objects
+ assert deleter.delete(tenant_id=TENANT_ID, year=2025, month=3).status == WorkflowRunArchiveDeletionStatus.COMPLETED
+ assert (
+ sum(
+ record.__dict__.get("attributes", {}).get("event") == "workflow_run_archive.deletion_completed"
+ for record in caplog.records
+ )
+ == 1
+ )
+
+
+def test_deletion_requires_durable_request_and_never_deletes_new_active_bundles(
+ sqlite_session_factory: sessionmaker[Session],
+ month_lease: WorkflowRunArchiveMonthLease,
+) -> None:
+ archive, downloads = MemoryStorage(), MemoryStorage()
+ target, new_bundle = _bundle(), _bundle(bundle_id="new-bundle")
+ prefix = _put_bundle(archive, target)
+ new_prefix = _put_bundle(archive, new_bundle)
+ with sqlite_session_factory.begin() as session:
+ session.add(target)
+ repository = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory)
+ deleter = _deleter(repository, archive, downloads, month_lease)
+ deleter.delete(tenant_id=TENANT_ID, year=2025, month=3)
+ assert not archive.deleted
+ repository.request_deletion(TENANT_ID, year=2025, month=3)
+ with sqlite_session_factory.begin() as session:
+ session.add(new_bundle)
+
+ deleter.delete(tenant_id=TENANT_ID, year=2025, month=3)
+
+ assert not archive.list_objects(prefix)
+ assert archive.list_objects(new_prefix)
+ assert [record.bundle_id for record in repository.list_for_tenant_month(TENANT_ID, year=2025, month=3)] == [
+ "new-bundle"
+ ]
+
+
+def test_manifest_mismatch_fails_without_deleting_objects(
+ sqlite_session_factory: sessionmaker[Session], month_lease: WorkflowRunArchiveMonthLease
+) -> None:
+ archive, downloads = MemoryStorage(), MemoryStorage()
+ target = _bundle()
+ prefix = _put_bundle(archive, target)
+ manifest = json.loads(archive.objects[prefix + "manifest.json"])
+ manifest["tenant_id"] = OTHER_TENANT_ID
+ archive.objects[prefix + "manifest.json"] = json.dumps(manifest).encode()
+ with sqlite_session_factory.begin() as session:
+ session.add(target)
+ repository = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory)
+ repository.request_deletion(TENANT_ID, year=2025, month=3)
+
+ result = _deleter(repository, archive, downloads, month_lease).delete(tenant_id=TENANT_ID, year=2025, month=3)
+
+ assert result.status == WorkflowRunArchiveDeletionStatus.FAILED
+ assert not archive.deleted
+ assert repository.list_for_tenant_month(TENANT_ID, year=2025, month=3)
+
+
+def test_lost_month_lease_keeps_durable_index_for_retry(
+ sqlite_session_factory: sessionmaker[Session], month_lease: WorkflowRunArchiveMonthLease
+) -> None:
+ archive, downloads = MemoryStorage(), MemoryStorage()
+ target = _bundle()
+ prefix = _put_bundle(archive, target)
+ with sqlite_session_factory.begin() as session:
+ session.add(target)
+ repository = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory)
+ repository.request_deletion(TENANT_ID, year=2025, month=3)
+ deleter = WorkflowRunArchiveDeleter(
+ bundles=repository,
+ archive_storage_provider=lambda: archive,
+ download_storage_provider=lambda: downloads,
+ month_lock=lambda **_kwargs: nullcontext(month_lease),
+ )
+
+ with patch.object(month_lease, "assert_owned", side_effect=[None, None, RuntimeError("lease lost")]):
+ result = deleter.delete(tenant_id=TENANT_ID, year=2025, month=3)
+
+ assert result.status == WorkflowRunArchiveDeletionStatus.FAILED
+ assert not archive.list_objects(prefix)
+ records = repository.list_for_tenant_month(TENANT_ID, year=2025, month=3)
+ assert len(records) == 1
+ assert records[0].deletion_status == WorkflowRunArchiveDeletionStatus.FAILED
+
+
+def test_month_lease_lets_deletion_clean_zip_after_failed_upload_compensation(
+ sqlite_session_factory: sessionmaker[Session],
+ month_lease: WorkflowRunArchiveMonthLease,
+ redis_transport: tuple[RedisClientWrapper, MagicMock],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ archive, downloads = MemoryStorage(), MemoryStorage()
+ target = _bundle()
+ _put_bundle(archive, target)
+ with sqlite_session_factory.begin() as session:
+ session.add(target)
+ repository = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory)
+ task = build_pending_archive_download_task(
+ tenant_id=TENANT_ID,
+ requested_by="account-1",
+ year=2025,
+ month=3,
+ bundle_ids=[target.bundle_id],
+ bundle_refs=[(target.shard, target.bundle_id)],
+ archive_bytes=512,
+ download_id=build_archive_download_id(
+ tenant_id=TENANT_ID,
+ year=2025,
+ month=3,
+ bundle_refs=[(target.shard, target.bundle_id)],
+ ),
+ )
+ state = [task]
+ redis, _commands = redis_transport
+ cache = WorkflowRunArchiveDownloadTaskCache(redis=redis)
+ monkeypatch.setattr(cache, "get", lambda **_kwargs: state[0])
+ monkeypatch.setattr(cache, "save", lambda saved: state.__setitem__(0, saved))
+ monkeypatch.setattr(cache, "lock", lambda **_kwargs: nullcontext())
+ mutex = threading.Lock()
+ deletion_waiting = threading.Event()
+
+ @contextmanager
+ def month_lock(*, tenant_id: str, year: int, month: int) -> Generator[WorkflowRunArchiveMonthLease]:
+ assert (tenant_id, year, month) == (TENANT_ID, 2025, 3)
+ if threading.current_thread() is not threading.main_thread():
+ deletion_waiting.set()
+ with mutex:
+ yield month_lease
+
+ deleter = WorkflowRunArchiveDeleter(
+ bundles=repository,
+ archive_storage_provider=lambda: archive,
+ download_storage_provider=lambda: downloads,
+ month_lock=month_lock,
+ )
+ deletion_results = []
+ deletion_worker = threading.Thread(
+ target=lambda: deletion_results.append(deleter.delete(tenant_id=TENANT_ID, year=2025, month=3))
+ )
+ original_put, original_delete = downloads.put_object, downloads.delete_object
+ compensation_failed = False
+
+ def upload(key: str, data: bytes) -> str:
+ result = original_put(key, data)
+ repository.request_deletion(TENANT_ID, year=2025, month=3)
+ deletion_worker.start()
+ assert deletion_waiting.wait(timeout=1)
+ assert deletion_worker.is_alive()
+ assert key in downloads.objects
+ return result
+
+ def fail_compensation_once(key: str) -> None:
+ nonlocal compensation_failed
+ if not compensation_failed:
+ compensation_failed = True
+ raise ArchiveStorageError("compensation failed")
+ original_delete(key)
+
+ downloads.put_object = upload
+ downloads.delete_object = fail_compensation_once
+ preparer = WorkflowRunArchiveDownloadPreparer(
+ archive_storage_provider=lambda: archive,
+ download_storage_provider=lambda: downloads,
+ bundles=repository,
+ cache=cache,
+ month_lock=month_lock,
+ )
+ monkeypatch.setattr(preparer, "_build_zip_payload", lambda *_args: b"prepared ZIP")
+
+ result = preparer.prepare(tenant_id=TENANT_ID, download_id=task.download_id)
+ deletion_worker.join(timeout=5)
+
+ assert result is not None
+ assert result.status == WorkflowRunArchiveDownloadStatus.FAILED
+ assert compensation_failed
+ assert not deletion_worker.is_alive()
+ assert deletion_results[0].status == WorkflowRunArchiveDeletionStatus.COMPLETED
+ assert not downloads.objects
+ assert not repository.list_for_tenant_month(TENANT_ID, year=2025, month=3)
diff --git a/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_preparation.py b/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_preparation.py
index 9e6689e2756..f4a5ee08b32 100644
--- a/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_preparation.py
+++ b/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_preparation.py
@@ -2,16 +2,21 @@ import datetime
import hashlib
import io
import json
+import subprocess
+import sys
import zipfile
from contextlib import nullcontext
+from pathlib import Path
from typing import cast
from unittest.mock import MagicMock
import pyarrow as pa
import pyarrow.parquet as pq
+import pytest
from libs.archive_storage import ArchiveStorage
from services.retention.workflow_run.archive_bundle_index import ARCHIVE_BUNDLE_ROOT_PREFIX, ArchiveBundleManifest
+from services.retention.workflow_run.archive_deletion_task import WorkflowRunArchiveDeletionStatus
from services.retention.workflow_run.archive_download_preparation import (
WorkflowRunArchiveDownloadPreparer,
build_archive_download_storage_key,
@@ -23,6 +28,7 @@ from services.retention.workflow_run.archive_download_task import (
build_pending_archive_download_task,
)
from services.retention.workflow_run.archive_log_service import WorkflowRunArchiveBundleRecord
+from services.retention.workflow_run.archive_month_lock import WorkflowRunArchiveMonthLease
from services.retention.workflow_run.constants import ARCHIVE_BUNDLE_FORMAT, ARCHIVE_BUNDLE_SCHEMA_VERSION
TENANT_ID = "1251fe32-c0c7-4fe2-a7bd-a8105267faf5"
@@ -50,6 +56,10 @@ class FakeArchiveStorage:
self.put_objects[key] = data
return hashlib.md5(data).hexdigest()
+ def delete_object(self, key: str) -> None:
+ self.objects.pop(key, None)
+ self.put_objects.pop(key, None)
+
class FakeTaskCache:
task: WorkflowRunArchiveDownloadTask | None
@@ -170,6 +180,7 @@ def _preparer(
archive_storage: FakeArchiveStorage | None = None,
download_storage: FakeArchiveStorage | None = None,
cache: FakeTaskCache,
+ lease: WorkflowRunArchiveMonthLease,
bundles: list[WorkflowRunArchiveBundleRecord] | None = None,
) -> WorkflowRunArchiveDownloadPreparer:
source_storage = archive_storage or storage
@@ -181,6 +192,7 @@ def _preparer(
bundles=FakeBundleQuery(tuple([_bundle()] if bundles is None else bundles)),
cache=cache,
download_storage_provider=lambda: cast(ArchiveStorage, target_storage),
+ month_lock=lambda **_kwargs: nullcontext(lease),
)
@@ -190,7 +202,94 @@ def _parquet_bytes(records: list[dict[str, object]]) -> bytes:
return buffer.getvalue()
-def test_prepare_workflow_run_archive_download_builds_csv_zip_and_marks_ready() -> None:
[email protected]("after_upload", ["pending", "failed", "missing"])
+def test_preparation_does_not_revive_archives_deleted_during_upload(
+ after_upload: str, month_lease: WorkflowRunArchiveMonthLease
+) -> None:
+ task = _task()
+ payloads = {"workflow_runs": _parquet_bytes([{"id": "run-a"}])}
+ storage = FakeArchiveStorage(
+ {
+ MANIFEST_KEY: _manifest_bytes(payloads),
+ f"{OBJECT_PREFIX}/workflow_runs.parquet": payloads["workflow_runs"],
+ }
+ )
+ cache = FakeTaskCache(task)
+ bundles = FakeBundleQuery((_bundle(),))
+ original_upload = storage.put_object
+
+ def upload(key: str, data: bytes) -> str:
+ checksum = original_upload(key, data)
+ bundles.records = (
+ ()
+ if after_upload == "missing"
+ else (_bundle()._replace(deletion_status=WorkflowRunArchiveDeletionStatus(after_upload)),)
+ )
+ return checksum
+
+ storage.put_object = upload
+ preparer = WorkflowRunArchiveDownloadPreparer(
+ archive_storage_provider=lambda: cast(ArchiveStorage, storage),
+ download_storage_provider=lambda: cast(ArchiveStorage, storage),
+ cache=cache,
+ bundles=bundles,
+ month_lock=lambda **_kwargs: nullcontext(month_lease),
+ )
+
+ result = preparer.prepare(tenant_id=TENANT_ID, download_id=task.download_id)
+
+ assert result is not None
+ assert result.status == WorkflowRunArchiveDownloadStatus.FAILED
+ assert not storage.put_objects
+ assert cache.saved_tasks[-1].status == WorkflowRunArchiveDownloadStatus.FAILED
+
+
+def test_gevent_archive_cpu_work_keeps_month_lease_heartbeat_responsive() -> None:
+ # Celery patches threading before importing task modules. Isolate that process-wide patch from other tests.
+ script = """
+from gevent import monkey
+monkey.patch_all()
+import gevent
+import time
+from services.retention.workflow_run.archive_download_preparation import _run_archive_cpu_work
+
+native_ident = monkey.get_original('threading', 'get_ident')
+main_thread = native_ident()
+ticks = []
+finished = gevent.event.Event()
+
+def heartbeat():
+ while not finished.is_set():
+ ticks.append(time.perf_counter())
+ gevent.sleep(0.01)
+
+def cpu_work():
+ deadline = time.perf_counter() + 0.2
+ value = 0
+ while time.perf_counter() < deadline:
+ value += 1
+ return native_ident(), value
+
+worker = gevent.spawn(heartbeat)
+thread_id, count = _run_archive_cpu_work(cpu_work)
+finished.set()
+worker.join()
+assert thread_id != main_thread and count > 0
+assert len(ticks) >= 5, ticks
+"""
+ subprocess.run(
+ [sys.executable, "-c", script],
+ cwd=Path(__file__).resolve().parents[5],
+ check=True,
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+
+
+def test_prepare_workflow_run_archive_download_builds_csv_zip_and_marks_ready(
+ month_lease: WorkflowRunArchiveMonthLease,
+) -> None:
bundle_refs = [(SHARD, "bundle-a"), (SHARD, "bundle-b")]
task = _task(bundle_refs)
first_bundle_payloads = {
@@ -227,6 +326,7 @@ def test_prepare_workflow_run_archive_download_builds_csv_zip_and_marks_ready()
archive_storage=archive_storage,
download_storage=download_storage,
cache=cache,
+ lease=month_lease,
bundles=[_bundle("bundle-a"), _bundle("bundle-b")],
)
@@ -253,7 +353,9 @@ def test_prepare_workflow_run_archive_download_builds_csv_zip_and_marks_ready()
assert '"run-b","failed","safe"' in workflow_runs_csv
-def test_prepare_workflow_run_archive_download_marks_failed_on_checksum_mismatch() -> None:
+def test_prepare_workflow_run_archive_download_marks_failed_on_checksum_mismatch(
+ month_lease: WorkflowRunArchiveMonthLease,
+) -> None:
task = _task()
table_payloads = {"workflow_runs": _parquet_bytes([{"id": "run-a", "status": "succeeded"}])}
manifest_data = json.loads(_manifest_bytes(table_payloads).decode("utf-8"))
@@ -265,7 +367,7 @@ def test_prepare_workflow_run_archive_download_marks_failed_on_checksum_mismatch
}
)
cache = FakeTaskCache(task)
- preparer = _preparer(storage=storage, cache=cache)
+ preparer = _preparer(storage=storage, cache=cache, lease=month_lease)
result = preparer.prepare(tenant_id=TENANT_ID, download_id=task.download_id)
@@ -275,16 +377,27 @@ def test_prepare_workflow_run_archive_download_marks_failed_on_checksum_mismatch
assert storage.put_objects == {}
-def test_prepare_workflow_run_archive_download_marks_failed_when_storage_is_unavailable() -> None:
+def test_prepare_workflow_run_archive_download_marks_failed_when_storage_is_unavailable(
+ month_lease: WorkflowRunArchiveMonthLease,
+) -> None:
task = _task()
cache = FakeTaskCache(task)
- archive_storage_provider = MagicMock(side_effect=RuntimeError("archive storage unavailable"))
- download_storage_provider = MagicMock()
+
+ def archive_storage_provider() -> ArchiveStorage:
+ raise RuntimeError("archive storage unavailable")
+
+ download_storage_calls: list[None] = []
+
+ def download_storage_provider() -> ArchiveStorage:
+ download_storage_calls.append(None)
+ raise AssertionError("download storage must not be accessed when archive storage is unavailable")
+
preparer = WorkflowRunArchiveDownloadPreparer(
archive_storage_provider=archive_storage_provider,
bundles=FakeBundleQuery((_bundle(),)),
cache=cache,
download_storage_provider=download_storage_provider,
+ month_lock=lambda **_kwargs: nullcontext(month_lease),
)
result = preparer.prepare(tenant_id=TENANT_ID, download_id=task.download_id)
@@ -292,16 +405,19 @@ def test_prepare_workflow_run_archive_download_marks_failed_when_storage_is_unav
assert result is not None
assert result.status == WorkflowRunArchiveDownloadStatus.FAILED
assert result.error == "archive storage unavailable"
- download_storage_provider.assert_not_called()
+ assert not download_storage_calls
-def test_prepare_workflow_run_archive_download_skips_duplicate_worker() -> None:
+def test_prepare_workflow_run_archive_download_skips_duplicate_worker(
+ month_lease: WorkflowRunArchiveMonthLease,
+) -> None:
task = _task().model_copy(update={"celery_task_id": "celery-task-1"})
storage = FakeArchiveStorage({})
cache = FakeTaskCache(task)
preparer = _preparer(
storage=storage,
cache=cache,
+ lease=month_lease,
bundles=[],
)
nested_results: list[WorkflowRunArchiveDownloadTask | None] = []
@@ -322,13 +438,13 @@ def test_prepare_workflow_run_archive_download_skips_duplicate_worker() -> None:
preparer._build_zip_payload.assert_called_once()
-def test_failed_worker_cannot_overwrite_ready_task() -> None:
+def test_failed_worker_cannot_overwrite_ready_task(month_lease: WorkflowRunArchiveMonthLease) -> None:
processing_task = _task().model_copy(
update={"status": WorkflowRunArchiveDownloadStatus.PROCESSING, "celery_task_id": "celery-task-1"}
)
ready_task = processing_task.model_copy(update={"status": WorkflowRunArchiveDownloadStatus.READY})
cache = FakeTaskCache(ready_task)
- preparer = _preparer(storage=FakeArchiveStorage({}), cache=cache)
+ preparer = _preparer(storage=FakeArchiveStorage({}), cache=cache, lease=month_lease)
result = preparer._mark_failed(processing_task, error="late failure")
diff --git a/api/tests/unit_tests/services/retention/workflow_run/test_archive_log_service.py b/api/tests/unit_tests/services/retention/workflow_run/test_archive_log_service.py
index 2d0946e7232..fdb75337978 100644
--- a/api/tests/unit_tests/services/retention/workflow_run/test_archive_log_service.py
+++ b/api/tests/unit_tests/services/retention/workflow_run/test_archive_log_service.py
@@ -1,9 +1,17 @@
import datetime
+import json
+import logging
+from collections.abc import Sequence
from contextlib import nullcontext
import pytest
+from core.logging.structured_formatter import StructuredJSONFormatter
from machinery.context import RequestContext
+from services.retention.workflow_run.archive_deletion_task import (
+ WorkflowRunArchiveDeletionDispatcher,
+ WorkflowRunArchiveDeletionStatus,
+)
from services.retention.workflow_run.archive_download_task import (
WorkflowRunArchiveDownloadStatus,
WorkflowRunArchiveDownloadTask,
@@ -45,6 +53,32 @@ class FakeBundleQuery:
assert tenant_id == "tenant-1"
return tuple(record for record in self.records if record.year == year and record.month == month)
+ def request_deletion(self, tenant_id: str, *, year: int, month: int) -> WorkflowRunArchiveDeletionStatus | None:
+ records = self.list_for_tenant_month(tenant_id, year=year, month=month)
+ if not records:
+ return None
+ self.records = tuple(
+ record._replace(deletion_status=WorkflowRunArchiveDeletionStatus.PENDING) if record in records else record
+ for record in self.records
+ )
+ return WorkflowRunArchiveDeletionStatus.PENDING
+
+ def set_deletion_status(
+ self, tenant_id: str, *, year: int, month: int, status: WorkflowRunArchiveDeletionStatus
+ ) -> None:
+ records = self.list_for_tenant_month(tenant_id, year=year, month=month)
+ self.records = tuple(
+ record._replace(deletion_status=status) if record in records else record for record in self.records
+ )
+
+ def remove_deleted_bundles(self, tenant_id: str, *, catalog_ids: Sequence[str]) -> None:
+ assert tenant_id == "tenant-1"
+ self.records = tuple(record for record in self.records if record.catalog_id not in catalog_ids)
+
+
+def _noop_deletion_dispatcher(*, tenant_id: str, year: int, month: int) -> None:
+ pass
+
class FakeTaskStore:
def __init__(
@@ -102,6 +136,8 @@ def _service(
dispatched_tasks: list[WorkflowRunArchiveDownloadTask] | None = None,
signed_urls: list[tuple[str, int, str]] | None = None,
dispatcher: WorkflowRunArchiveDownloadTaskDispatcher | None = None,
+ deletion_dispatcher: WorkflowRunArchiveDeletionDispatcher | None = None,
+ bundles: FakeBundleQuery | None = None,
) -> WorkflowRunArchiveService:
task_store = tasks or FakeTaskStore()
@@ -115,10 +151,11 @@ def _service(
return "https://storage.example.com/archive.zip"
return WorkflowRunArchiveService(
- bundles=FakeBundleQuery(records),
+ bundles=bundles or FakeBundleQuery(records),
tasks=task_store,
dispatcher=dispatcher or dispatch,
sign_download_url=sign,
+ deletion_dispatcher=deletion_dispatcher or _noop_deletion_dispatcher,
)
@@ -366,6 +403,7 @@ def test_create_download_claims_dispatch_once() -> None:
tasks=task_store,
dispatcher=dispatch,
sign_download_url=sign_download_url,
+ deletion_dispatcher=_noop_deletion_dispatcher,
)
result = service.create_download(_CONTEXT, year=2025, month=3)
@@ -396,7 +434,10 @@ def test_get_download_url_requires_ready_file() -> None:
)
with pytest.raises(WorkflowRunArchiveDownloadNotReadyError):
- _service(tasks=FakeTaskStore({pending.download_id: pending})).get_download_url(
+ _service(
+ records=(_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024),),
+ tasks=FakeTaskStore({pending.download_id: pending}),
+ ).get_download_url(
_CONTEXT,
download_id=pending.download_id,
)
@@ -422,9 +463,112 @@ def test_get_download_url_signs_ready_file_for_at_most_one_hour() -> None:
signed_urls: list[tuple[str, int, str]] = []
url = _service(
+ records=(_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024),),
tasks=FakeTaskStore({ready.download_id: ready}),
signed_urls=signed_urls,
).get_download_url(_CONTEXT, download_id=ready.download_id)
assert url == "https://storage.example.com/archive.zip"
assert signed_urls == [("downloads/download-1.zip", 3600, "workflow-run-logs-2025-03.zip")]
+
+
[email protected]("status", [WorkflowRunArchiveDeletionStatus.PENDING, WorkflowRunArchiveDeletionStatus.FAILED])
+def test_downloads_are_disabled_for_durable_deletion_requests(status: WorkflowRunArchiveDeletionStatus) -> None:
+ bundle = _bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024)._replace(deletion_status=status)
+ task = build_pending_archive_download_task(
+ tenant_id="tenant-1",
+ requested_by="account-1",
+ year=2025,
+ month=3,
+ bundle_ids=["bundle-a"],
+ archive_bytes=1024,
+ download_id="download-1",
+ )
+ service = _service(records=(bundle,), tasks=FakeTaskStore({task.download_id: task}))
+
+ assert service.list_archives(_CONTEXT).months[0].deletion_status == status
+ assert service.list_archives(_CONTEXT).months[0].download_task is None
+ with pytest.raises(WorkflowRunArchiveDownloadNotReadyError):
+ service.create_download(_CONTEXT, year=2025, month=3)
+ with pytest.raises(WorkflowRunArchiveDownloadTaskNotFoundError):
+ service.get_download(_CONTEXT, download_id=task.download_id)
+
+
+def test_old_download_is_unavailable_after_its_bundles_are_deleted() -> None:
+ task = build_pending_archive_download_task(
+ tenant_id="tenant-1",
+ requested_by="account-1",
+ year=2025,
+ month=3,
+ bundle_ids=["bundle-a"],
+ archive_bytes=1024,
+ download_id="download-1",
+ )
+ with pytest.raises(WorkflowRunArchiveDownloadTaskNotFoundError):
+ _service(tasks=FakeTaskStore({task.download_id: task})).get_download(_CONTEXT, download_id=task.download_id)
+
+
+def test_delete_archive_commits_request_before_dispatch_and_can_be_retried(caplog: pytest.LogCaptureFixture) -> None:
+ caplog.set_level(logging.INFO, logger="services.retention.workflow_run.archive_log_service")
+ bundles = FakeBundleQuery((_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024),))
+ dispatches: list[tuple[str, int, int]] = []
+
+ def dispatch(*, tenant_id: str, year: int, month: int) -> None:
+ assert bundles.records[0].deletion_status == WorkflowRunArchiveDeletionStatus.PENDING
+ assert caplog.records[-1].__dict__["attributes"]["event"] == "workflow_run_archive.deletion_requested"
+ dispatches.append((tenant_id, year, month))
+
+ service = _service(bundles=bundles, deletion_dispatcher=dispatch)
+ assert service.delete_archive(_CONTEXT, year=2025, month=3).status == WorkflowRunArchiveDeletionStatus.PENDING
+ assert service.delete_archive(_CONTEXT, year=2025, month=3).status == WorkflowRunArchiveDeletionStatus.PENDING
+ assert dispatches == [("tenant-1", 2025, 3), ("tenant-1", 2025, 3)]
+ events = [json.loads(StructuredJSONFormatter().format(record)) for record in caplog.records]
+ assert [event["attributes"]["event"] for event in events] == [
+ "workflow_run_archive.deletion_requested",
+ "workflow_run_archive.deletion_enqueued",
+ ] * 2
+ for event in events:
+ assert datetime.datetime.fromisoformat(event["ts"]).tzinfo == datetime.UTC
+ assert event["attributes"] == {
+ "audit": True,
+ "event": event["attributes"]["event"],
+ "tenant_id": "tenant-1",
+ "account_id": "account-1",
+ "year": 2025,
+ "month": 3,
+ "request_id": "request-1",
+ "deletion_status": "pending",
+ }
+ assert "account_id=account-1" in event["message"]
+ assert "request_id=request-1" in event["message"]
+
+
+def test_failed_deletion_dispatch_preserves_request_and_disables_downloads(caplog: pytest.LogCaptureFixture) -> None:
+ caplog.set_level(logging.INFO, logger="services.retention.workflow_run.archive_log_service")
+ bundles = FakeBundleQuery((_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024),))
+
+ def fail_dispatch(*, tenant_id: str, year: int, month: int) -> None:
+ raise RuntimeError("broker unavailable")
+
+ result = _service(bundles=bundles, deletion_dispatcher=fail_dispatch).delete_archive(_CONTEXT, year=2025, month=3)
+
+ assert result.status == WorkflowRunArchiveDeletionStatus.FAILED
+ assert bundles.records[0].deletion_status == WorkflowRunArchiveDeletionStatus.FAILED
+ assert (
+ _service(bundles=bundles).list_archives(_CONTEXT).months[0].deletion_status
+ == WorkflowRunArchiveDeletionStatus.FAILED
+ )
+ audit = [record.__dict__["attributes"] for record in caplog.records]
+ assert [event["event"] for event in audit] == [
+ "workflow_run_archive.deletion_requested",
+ "workflow_run_archive.deletion_enqueue_failed",
+ ]
+ assert audit[-1]["account_id"] == "account-1"
+ assert audit[-1]["request_id"] == "request-1"
+
+
+def test_delete_archive_without_bundles_returns_completed(caplog: pytest.LogCaptureFixture) -> None:
+ caplog.set_level(logging.INFO, logger="services.retention.workflow_run.archive_log_service")
+ assert _service().delete_archive(_CONTEXT, year=2025, month=3).status == WorkflowRunArchiveDeletionStatus.COMPLETED
+ assert len(caplog.records) == 1
+ assert caplog.records[0].__dict__["attributes"]["event"] == "workflow_run_archive.deletion_noop"
diff --git a/api/tests/unit_tests/services/retention/workflow_run/test_archive_month_lock.py b/api/tests/unit_tests/services/retention/workflow_run/test_archive_month_lock.py
new file mode 100644
index 00000000000..832aa916fde
--- /dev/null
+++ b/api/tests/unit_tests/services/retention/workflow_run/test_archive_month_lock.py
@@ -0,0 +1,81 @@
+import threading
+from unittest.mock import MagicMock, patch
+
+import pytest
+from redis.lock import Lock
+
+from extensions.ext_redis import RedisClientWrapper
+from services.retention.workflow_run import archive_month_lock
+from services.retention.workflow_run.archive_month_lock import (
+ RedisWorkflowRunArchiveMonthLock,
+ WorkflowRunArchiveMonthBusyError,
+)
+
+
+def test_month_lease_renews_during_blocking_work_and_releases_afterwards(
+ redis_transport: tuple[RedisClientWrapper, MagicMock], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ redis, commands = redis_transport
+ renewed = threading.Event()
+ monkeypatch.setattr(archive_month_lock, "_RENEW_INTERVAL_SECONDS", 0.001)
+
+ with (
+ patch.object(Lock, "acquire", autospec=True, return_value=True) as acquire,
+ patch.object(Lock, "owned", return_value=True),
+ patch.object(Lock, "reacquire", side_effect=renewed.set),
+ patch.object(Lock, "release") as release,
+ ):
+ with RedisWorkflowRunArchiveMonthLock(redis=redis)(tenant_id="tenant-1", year=2025, month=3) as lease:
+ assert renewed.wait(timeout=1)
+ lease.assert_owned()
+ release.assert_not_called()
+ acquire.assert_called_once()
+ release.assert_called_once()
+
+ lock = acquire.call_args.args[0]
+ assert isinstance(lock, Lock)
+ assert lock.name == "workflow_run_archive_month:tenant-1:2025-03"
+ assert lock.timeout == 120
+ assert lock.blocking_timeout == 120
+ assert lock.thread_local is False
+ commands.assert_not_called()
+
+
+def test_failed_renewal_prevents_committing_archive_deletion(
+ redis_transport: tuple[RedisClientWrapper, MagicMock], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ redis, commands = redis_transport
+ attempted = threading.Event()
+ renewal_threads: list[threading.Thread] = []
+
+ def fail_renewal() -> None:
+ renewal_threads.append(threading.current_thread())
+ attempted.set()
+ raise RuntimeError("redis unavailable")
+
+ monkeypatch.setattr(archive_month_lock, "_RENEW_INTERVAL_SECONDS", 0.001)
+ with (
+ patch.object(Lock, "acquire", return_value=True),
+ patch.object(Lock, "reacquire", side_effect=fail_renewal),
+ patch.object(Lock, "release") as release,
+ ):
+ with RedisWorkflowRunArchiveMonthLock(redis=redis)(tenant_id="tenant-1", year=2025, month=3) as lease:
+ assert attempted.wait(timeout=1)
+ # Join only the heartbeat: the failed renewal must stop it and record lease loss before finalization.
+ renewal_threads[0].join(timeout=1)
+ with pytest.raises(RuntimeError, match="lock was lost"):
+ lease.assert_owned()
+ release.assert_called_once()
+ commands.assert_not_called()
+
+
+def test_contended_month_lease_is_retryable(redis_transport: tuple[RedisClientWrapper, MagicMock]) -> None:
+ redis, commands = redis_transport
+
+ with patch.object(Lock, "acquire", return_value=False), patch.object(Lock, "release") as release:
+ with pytest.raises(WorkflowRunArchiveMonthBusyError):
+ with RedisWorkflowRunArchiveMonthLock(redis=redis)(tenant_id="tenant-1", year=2025, month=3):
+ pytest.fail("contended lease must not enter storage work")
+
+ release.assert_not_called()
+ commands.assert_not_called()
diff --git a/api/tests/unit_tests/services/retention/workflow_run/test_bundle_archive_maintenance.py b/api/tests/unit_tests/services/retention/workflow_run/test_bundle_archive_maintenance.py
index c0c065e07f3..1e9c17814b6 100644
--- a/api/tests/unit_tests/services/retention/workflow_run/test_bundle_archive_maintenance.py
+++ b/api/tests/unit_tests/services/retention/workflow_run/test_bundle_archive_maintenance.py
@@ -4,12 +4,14 @@ from collections.abc import Sequence
from typing import Any, cast
from unittest.mock import MagicMock, call, patch
+import boto3
import pytest
from botocore.exceptions import ClientError, ReadTimeoutError
-from sqlalchemy import event
+from botocore.stub import Stubber
+from sqlalchemy import delete, event
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
-from libs.archive_storage import ArchiveStorageError
+from libs.archive_storage import ArchiveStorage, ArchiveStorageError
from models.workflow import WorkflowRunArchiveBundle
from services.retention.workflow_run.bundle_archive_maintenance import (
ARCHIVED_TABLES,
@@ -27,6 +29,7 @@ from services.retention.workflow_run.constants import (
ARCHIVE_BUNDLE_RESTORED_MARKER_NAME,
ARCHIVE_BUNDLE_SCHEMA_VERSION,
)
+from tests.unit_tests.config_override import apply_config_overrides
TENANT_ID = "1251fe32-c0c7-4fe2-a7bd-a8105267faf5"
CATALOG_ID = "019f63b7-5ca4-7681-9ce0-800283608f39"
@@ -193,13 +196,17 @@ def _sample_archive_records() -> dict[str, list[dict[str, Any]]]:
)
[email protected]("cursor_deleted", [False, True])
def test_catalog_discovery_is_ordered_and_limited_before_storage_io(
- sqlite_session_factory: sessionmaker[Session], sqlite_session: Session
+ sqlite_session_factory: sessionmaker[Session], sqlite_session: Session, cursor_deleted: bool
) -> None:
cursor = _catalog_entry()
entry = _catalog_entry(catalog_id="019f63b7-5ca4-7681-9ce0-800283608f40", bundle_id="bundle-b")
sqlite_session.add_all([_bundle_model(cursor), _bundle_model(entry)])
sqlite_session.commit()
+ if cursor_deleted:
+ sqlite_session.execute(delete(WorkflowRunArchiveBundle).where(WorkflowRunArchiveBundle.id == cursor.catalog_id))
+ sqlite_session.commit()
storage = MagicMock()
maintenance = WorkflowRunBundleArchiveMaintenance(
storage=storage,
@@ -320,7 +327,6 @@ def test_catalog_shard_preflight_uses_requested_tenant_scope(
@pytest.mark.parametrize(
("cursor_bundle", "tenant_ids", "error_message"),
[
- (None, None, "does not exist"),
(
_cursor_bundle(year=2024, tenant_id=TENANT_ID),
None,
@@ -381,6 +387,39 @@ def test_bundle_maintenance_locks_the_existing_catalog_row(sqlite_session: Sessi
WorkflowRunBundleArchiveMaintenance._lock_catalog_entry(sqlite_session, entry)
+def test_bundle_maintenance_rejects_permanently_deleting_archive(sqlite_session: Session) -> None:
+ entry = _catalog_entry()
+ bundle = _bundle_model(entry)
+ bundle.deletion_status = "failed"
+ sqlite_session.add(bundle)
+ sqlite_session.flush()
+
+ with pytest.raises(ValueError, match="source rows cannot be deleted or restored"):
+ WorkflowRunBundleArchiveMaintenance._lock_catalog_entry(sqlite_session, entry)
+
+
+def test_source_maintenance_candidates_exclude_permanent_deletion_requests(
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ entry = _catalog_entry()
+ bundle = _bundle_model(entry)
+ bundle.deletion_status = "pending"
+ with sqlite_session_factory.begin() as session:
+ session.add(bundle)
+ maintenance = WorkflowRunBundleArchiveMaintenance(storage=MagicMock(), session_factory=sqlite_session_factory)
+
+ assert (
+ maintenance._list_catalog_entries(
+ tenant_ids=[TENANT_ID],
+ target_year=2025,
+ target_month=3,
+ after_catalog_id=None,
+ limit=10,
+ )
+ == []
+ )
+
+
def test_failure_and_dry_run_do_not_return_a_persistable_cursor(
sqlite_session_factory: sessionmaker[Session],
) -> None:
@@ -931,22 +970,32 @@ def test_delete_marker_does_not_retry_non_retryable_delete_error() -> None:
def test_delete_bundle_with_deleted_marker_clears_stale_delete_started_marker(
- sqlite_session: Session, sqlite_session_factory: sessionmaker[Session]
+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
) -> None:
entry = _catalog_entry()
_persist_catalog(sqlite_session, entry)
archive_records = _sample_archive_records()
bundle_ref = _bundle_reference(entry, table_records=archive_records)
- storage = MagicMock()
+ apply_config_overrides(
+ monkeypatch,
+ ARCHIVE_STORAGE_ENABLED=True,
+ ARCHIVE_STORAGE_ENDPOINT="https://storage.example.com",
+ ARCHIVE_STORAGE_ACCESS_KEY="test-access",
+ ARCHIVE_STORAGE_SECRET_KEY="test-secret",
+ )
+ client = boto3.client("s3", aws_access_key_id="test-access", aws_secret_access_key="test-secret")
+ with Stubber(client) as stubber, patch("libs.archive_storage.boto3.client", return_value=client):
+ stubber.add_response("head_bucket", {}, {"Bucket": "archive-test"})
+ storage = ArchiveStorage(bucket="archive-test")
existing_markers = {
f"{bundle_ref.object_prefix}/{ARCHIVE_BUNDLE_DELETED_MARKER_NAME}",
f"{bundle_ref.object_prefix}/{ARCHIVE_BUNDLE_DELETE_STARTED_MARKER_NAME}",
}
- storage.object_exists.side_effect = lambda key: key in existing_markers
- storage.delete_object.side_effect = lambda key: existing_markers.discard(key)
maintenance = WorkflowRunBundleArchiveMaintenance(session_factory=sqlite_session_factory)
with (
+ patch.object(storage, "object_exists", side_effect=lambda key: key in existing_markers),
+ patch.object(storage, "delete_object", side_effect=existing_markers.discard) as delete_object,
patch.object(
maintenance,
"_validate_archive_object",
@@ -958,7 +1007,7 @@ def test_delete_bundle_with_deleted_marker_clears_stale_delete_started_marker(
assert result.success
assert existing_markers == {f"{bundle_ref.object_prefix}/{ARCHIVE_BUNDLE_DELETED_MARKER_NAME}"}
- assert storage.delete_object.call_args_list == [
+ assert delete_object.call_args_list == [
call(f"{bundle_ref.object_prefix}/{ARCHIVE_BUNDLE_DELETE_STARTED_MARKER_NAME}"),
call(f"{bundle_ref.object_prefix}/{ARCHIVE_BUNDLE_RESTORED_MARKER_NAME}"),
]
diff --git a/packages/contracts/generated/api/console/workflow-run-archives/orpc.gen.ts b/packages/contracts/generated/api/console/workflow-run-archives/orpc.gen.ts
index 34321a36b0a..5bfa2820b7b 100644
--- a/packages/contracts/generated/api/console/workflow-run-archives/orpc.gen.ts
+++ b/packages/contracts/generated/api/console/workflow-run-archives/orpc.gen.ts
@@ -3,6 +3,8 @@
import { oc } from '@orpc/contract'
import * as z from 'zod'
import {
+ zDeleteWorkflowRunArchivesByYearByMonthPath,
+ zDeleteWorkflowRunArchivesByYearByMonthResponse,
zGetWorkflowRunArchivesDownloadsByDownloadIdPath,
zGetWorkflowRunArchivesDownloadsByDownloadIdResponse,
zGetWorkflowRunArchivesResponse,
@@ -50,6 +52,30 @@ export const downloads = {
byDownloadId,
}
+/**
+ * Permanently delete this workspace's monthly archive files and archive index
+ */
+export const delete_ = oc
+ .route({
+ description: "Permanently delete this workspace's monthly archive files and archive index",
+ inputStructure: 'detailed',
+ method: 'DELETE',
+ operationId: 'deleteWorkflowRunArchivesByYearByMonth',
+ path: '/workflow-run-archives/{year}/{month}',
+ successStatus: 202,
+ tags: ['console'],
+ })
+ .input(z.object({ params: zDeleteWorkflowRunArchivesByYearByMonthPath }))
+ .output(zDeleteWorkflowRunArchivesByYearByMonthResponse)
+
+export const byMonth = {
+ delete: delete_,
+}
+
+export const byYear = {
+ byMonth,
+}
+
/**
* List monthly workflow-run archive metadata for the current workspace
*/
@@ -67,6 +93,7 @@ export const get2 = oc
export const workflowRunArchives = {
get: get2,
downloads,
+ byYear,
}
export const contract = {
diff --git a/packages/contracts/generated/api/console/workflow-run-archives/types.gen.ts b/packages/contracts/generated/api/console/workflow-run-archives/types.gen.ts
index 9616c2d0684..e39b65c72f7 100644
--- a/packages/contracts/generated/api/console/workflow-run-archives/types.gen.ts
+++ b/packages/contracts/generated/api/console/workflow-run-archives/types.gen.ts
@@ -31,9 +31,17 @@ export type WorkflowRunArchiveDownloadTaskResponse = {
year: number
}
+export type WorkflowRunArchiveDeletionTaskResponse = {
+ error?: string | null
+ month: number
+ status: WorkflowRunArchiveDeletionStatus
+ year: number
+}
+
export type WorkflowRunArchiveMonthResponse = {
archive_bytes: number
bundle_count: number
+ deletion_status?: 'failed' | 'pending' | 'processing' | null
download_task?: WorkflowRunArchiveDownloadTaskResponse | null
latest_archived_at: string
month: number
@@ -51,6 +59,8 @@ export type WorkflowRunArchiveSummaryResponse = {
export type WorkflowRunArchiveDownloadStatus = 'failed' | 'pending' | 'processing' | 'ready'
+export type WorkflowRunArchiveDeletionStatus = 'completed' | 'failed' | 'pending' | 'processing'
+
export type GetWorkflowRunArchivesData = {
body?: never
path?: never
@@ -94,3 +104,20 @@ export type GetWorkflowRunArchivesDownloadsByDownloadIdResponses = {
export type GetWorkflowRunArchivesDownloadsByDownloadIdResponse =
GetWorkflowRunArchivesDownloadsByDownloadIdResponses[keyof GetWorkflowRunArchivesDownloadsByDownloadIdResponses]
+
+export type DeleteWorkflowRunArchivesByYearByMonthData = {
+ body?: never
+ path: {
+ month: number
+ year: number
+ }
+ query?: never
+ url: '/workflow-run-archives/{year}/{month}'
+}
+
+export type DeleteWorkflowRunArchivesByYearByMonthResponses = {
+ 202: WorkflowRunArchiveDeletionTaskResponse
+}
+
+export type DeleteWorkflowRunArchivesByYearByMonthResponse =
+ DeleteWorkflowRunArchivesByYearByMonthResponses[keyof DeleteWorkflowRunArchivesByYearByMonthResponses]
diff --git a/packages/contracts/generated/api/console/workflow-run-archives/zod.gen.ts b/packages/contracts/generated/api/console/workflow-run-archives/zod.gen.ts
index 895205a0ca3..d10693efe51 100644
--- a/packages/contracts/generated/api/console/workflow-run-archives/zod.gen.ts
+++ b/packages/contracts/generated/api/console/workflow-run-archives/zod.gen.ts
@@ -9,7 +9,7 @@ import * as z from 'zod'
*/
export const zWorkflowRunArchiveDownloadPayload = z.object({
month: z.int().gte(1).lte(12),
- year: z.int().gte(1),
+ year: z.int().gte(1).lte(9999),
})
/**
@@ -60,6 +60,7 @@ export const zWorkflowRunArchiveDownloadTaskResponse = z.object({
export const zWorkflowRunArchiveMonthResponse = z.object({
archive_bytes: z.int(),
bundle_count: z.int(),
+ deletion_status: z.enum(['failed', 'pending', 'processing']).nullish(),
download_task: zWorkflowRunArchiveDownloadTaskResponse.nullish(),
latest_archived_at: z.iso.datetime(),
month: z.int(),
@@ -76,6 +77,26 @@ export const zWorkflowRunArchiveListResponse = z.object({
summary: zWorkflowRunArchiveSummaryResponse,
})
+/**
+ * WorkflowRunArchiveDeletionStatus
+ */
+export const zWorkflowRunArchiveDeletionStatus = z.enum([
+ 'completed',
+ 'failed',
+ 'pending',
+ 'processing',
+])
+
+/**
+ * WorkflowRunArchiveDeletionTaskResponse
+ */
+export const zWorkflowRunArchiveDeletionTaskResponse = z.object({
+ error: z.string().nullish(),
+ month: z.int(),
+ status: zWorkflowRunArchiveDeletionStatus,
+ year: z.int(),
+})
+
/**
* Success
*/
@@ -97,3 +118,14 @@ export const zGetWorkflowRunArchivesDownloadsByDownloadIdPath = z.object({
*/
export const zGetWorkflowRunArchivesDownloadsByDownloadIdResponse =
zWorkflowRunArchiveDownloadTaskResponse
+
+export const zDeleteWorkflowRunArchivesByYearByMonthPath = z.object({
+ month: z.int().gte(1).lte(12),
+ year: z.int().gte(1).lte(9999),
+})
+
+/**
+ * Archive deletion accepted
+ */
+export const zDeleteWorkflowRunArchivesByYearByMonthResponse =
+ zWorkflowRunArchiveDeletionTaskResponse
diff --git a/web/app/components/header/account-setting/workflow-log-archives-page/__tests__/index.spec.tsx b/web/app/components/header/account-setting/workflow-log-archives-page/__tests__/index.spec.tsx
index 6183ab1eb93..baa26840f4f 100644
--- a/web/app/components/header/account-setting/workflow-log-archives-page/__tests__/index.spec.tsx
+++ b/web/app/components/header/account-setting/workflow-log-archives-page/__tests__/index.spec.tsx
@@ -1,5 +1,9 @@
import type { CloudPlan } from '@dify/contracts/api/console/features/types.gen'
-import type { GetWorkflowRunArchivesResponse } from '@dify/contracts/api/console/workflow-run-archives/types.gen'
+import type {
+ GetWorkflowRunArchivesResponse,
+ WorkflowRunArchiveDeletionTaskResponse,
+} from '@dify/contracts/api/console/workflow-run-archives/types.gen'
+import type { GetWorkspacesCurrentSummaryResponse } from '@dify/contracts/api/console/workspaces/types.gen'
import { act, fireEvent, screen, waitFor, within } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { NuqsTestingAdapter } from 'nuqs/adapters/testing'
@@ -12,6 +16,28 @@ import {
import WorkflowLogArchivesPage from '../index'
const onPricingUrlUpdate = vi.hoisted(() => vi.fn())
+const archiveRequests = vi.hoisted(() => ({
+ list: vi.fn(),
+ delete: vi.fn(),
+ prepareDownload: vi.fn(),
+}))
+
+vi.mock('@/service/console/browser', () => ({
+ consoleBrowserLink: {
+ call: (path: readonly string[], input: unknown) => {
+ switch (path.join('.')) {
+ case 'workflowRunArchives.get':
+ return archiveRequests.list(input)
+ case 'workflowRunArchives.byYear.byMonth.delete':
+ return archiveRequests.delete(input)
+ case 'workflowRunArchives.downloads.post':
+ return archiveRequests.prepareDownload(input)
+ default:
+ throw new Error(`Unexpected console request: ${path.join('.')}`)
+ }
+ },
+ },
+}))
vi.mock('@/config', async (importOriginal) => {
const actual = await importOriginal<typeof import('@/config')>()
@@ -37,6 +63,7 @@ const archiveMonth: GetWorkflowRunArchivesResponse['months'][number] = {
bundle_count: 2,
latest_archived_at: '2025-03-03T00:00:00Z',
download_task: null,
+ deletion_status: null,
}
const archiveData: GetWorkflowRunArchivesResponse = {
@@ -54,11 +81,13 @@ let plan: CloudPlan = 'professional'
function renderPage(
data: GetWorkflowRunArchivesResponse | null = archiveData,
queryClient = createConsoleQueryClient(),
+ role: GetWorkspacesCurrentSummaryResponse['role'] = 'owner',
) {
if (data) queryClient.setQueryData(consoleQuery.workflowRunArchives.get.queryKey(), data)
return render(<WorkflowLogArchivesPage />, {
queryClient,
+ currentWorkspace: { role },
systemFeatures: { deployment_edition: 'CLOUD' },
features: { billing: { subscription: { plan } } },
})
@@ -72,9 +101,371 @@ function render(...args: Parameters<typeof renderWithoutPricing>) {
describe('WorkflowLogArchivesPage', () => {
beforeEach(() => {
vi.clearAllMocks()
+ archiveRequests.list.mockImplementation(() => new Promise(() => {}))
+ archiveRequests.delete.mockResolvedValue({
+ year: 2025,
+ month: 3,
+ status: 'pending',
+ error: null,
+ })
+ archiveRequests.prepareDownload.mockImplementation(() => new Promise(() => {}))
plan = 'professional'
})
+ describe('Delete monthly archives', () => {
+ it('lets admins prepare downloads without offering archive deletion', async () => {
+ const user = userEvent.setup()
+ renderPage(archiveData, undefined, 'admin')
+
+ expect(screen.getByRole('row', { name: /2025-03/ })).toBeInTheDocument()
+ expect(
+ screen.queryByRole('button', { name: 'common.operation.delete 2025-03' }),
+ ).not.toBeInTheDocument()
+ await user.click(
+ screen.getByRole('button', { name: 'appLog.archives.action.prepareDownload 2025-03' }),
+ )
+
+ expect(archiveRequests.prepareDownload).toHaveBeenCalledWith({
+ body: { year: 2025, month: 3 },
+ })
+ expect(archiveRequests.delete).not.toHaveBeenCalled()
+ expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument()
+ })
+
+ it.each(['admin', 'editor', 'dataset_operator', 'normal'] as const)(
+ 'does not offer deletion retry to a workspace %s',
+ (role) => {
+ renderPage(
+ { ...archiveData, months: [{ ...archiveMonth, deletion_status: 'failed' }] },
+ undefined,
+ role,
+ )
+
+ expect(
+ screen.queryByRole('button', { name: 'appLog.archives.delete.retry 2025-03' }),
+ ).not.toBeInTheDocument()
+ expect(screen.getByRole('alert')).toHaveTextContent('appLog.archives.delete.failed')
+ expect(archiveRequests.delete).not.toHaveBeenCalled()
+ },
+ )
+
+ it('cancels confirmation without deleting any archived data', async () => {
+ const user = userEvent.setup()
+ renderPage()
+
+ await user.click(screen.getByRole('button', { name: 'common.operation.delete 2025-03' }))
+ const dialog = await screen.findByRole('alertdialog')
+ expect(dialog).toHaveAccessibleName('appLog.archives.delete.title:{"month":"2025-03"}')
+ expect(dialog).toHaveAccessibleDescription('appLog.archives.delete.description')
+ expect(archiveRequests.delete).not.toHaveBeenCalled()
+
+ await user.click(within(dialog).getByRole('button', { name: 'common.operation.cancel' }))
+ await waitFor(() => expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument())
+ expect(archiveRequests.delete).not.toHaveBeenCalled()
+ expect(screen.getByRole('row', { name: /2025-03/ })).toBeInTheDocument()
+ })
+
+ it('submits the selected month once and blocks downloads while deletion is being accepted', async () => {
+ const user = userEvent.setup()
+ let resolveDelete!: (data: WorkflowRunArchiveDeletionTaskResponse) => void
+ archiveRequests.delete.mockImplementation(
+ () =>
+ new Promise<WorkflowRunArchiveDeletionTaskResponse>((resolve) => {
+ resolveDelete = resolve
+ }),
+ )
+ renderPage({ ...archiveData, months: [{ ...archiveMonth, year: 2026, month: 8 }] })
+ const download = screen.getByRole('button', {
+ name: 'appLog.archives.action.prepareDownload 2026-08',
+ })
+
+ await user.click(screen.getByRole('button', { name: 'common.operation.delete 2026-08' }))
+ const dialog = await screen.findByRole('alertdialog')
+ await user.dblClick(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.confirm' }),
+ )
+
+ expect(archiveRequests.delete).toHaveBeenCalledTimes(1)
+ expect(archiveRequests.delete).toHaveBeenCalledWith({ params: { year: 2026, month: 8 } })
+ expect(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.deleting' }),
+ ).toHaveAttribute('aria-disabled', 'true')
+ expect(within(dialog).getByRole('button', { name: 'common.operation.cancel' })).toBeDisabled()
+ expect(download).toHaveAttribute('aria-disabled', 'true')
+
+ await act(async () => resolveDelete({ year: 2026, month: 8, status: 'pending', error: null }))
+ await waitFor(() => expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument())
+ expect(
+ screen.getByRole('button', { name: 'appLog.archives.delete.deleting 2026-08' }),
+ ).toHaveAttribute('aria-disabled', 'true')
+ expect(
+ screen.getByRole('button', { name: 'appLog.archives.action.prepareDownload 2026-08' }),
+ ).toHaveAttribute('aria-disabled', 'true')
+ expect(archiveRequests.list).toHaveBeenCalled()
+ })
+
+ it.each(['pending', 'processing'] as const)(
+ 'blocks download and duplicate deletion for a %s month',
+ async (status) => {
+ const user = userEvent.setup()
+ renderPage({ ...archiveData, months: [{ ...archiveMonth, deletion_status: status }] })
+
+ const download = screen.getByRole('button', {
+ name: 'appLog.archives.action.prepareDownload 2025-03',
+ })
+ const deletion = screen.getByRole('button', {
+ name: 'appLog.archives.delete.deleting 2025-03',
+ })
+ expect(download).toHaveAttribute('aria-disabled', 'true')
+ expect(deletion).toHaveAttribute('aria-disabled', 'true')
+ await user.click(download)
+ await user.click(deletion)
+ expect(archiveRequests.prepareDownload).not.toHaveBeenCalled()
+ expect(archiveRequests.delete).not.toHaveBeenCalled()
+ expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument()
+ },
+ )
+
+ it('keeps a rejected deletion open with an error and allows retry', async () => {
+ const user = userEvent.setup()
+ archiveRequests.delete.mockRejectedValueOnce(new Error('Storage unavailable'))
+ renderPage()
+
+ await user.click(screen.getByRole('button', { name: 'common.operation.delete 2025-03' }))
+ const dialog = await screen.findByRole('alertdialog')
+ await user.click(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.confirm' }),
+ )
+
+ expect(await within(dialog).findByRole('alert')).toHaveTextContent(
+ 'appLog.archives.delete.requestFailed',
+ )
+ expect(dialog).toBeInTheDocument()
+ await user.click(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.confirm' }),
+ )
+ await waitFor(() => expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument())
+ expect(archiveRequests.delete).toHaveBeenCalledTimes(2)
+ })
+
+ it('offers confirmed retry for an incomplete deletion while downloads remain blocked', async () => {
+ const user = userEvent.setup()
+ renderPage({ ...archiveData, months: [{ ...archiveMonth, deletion_status: 'failed' }] })
+
+ expect(screen.getByRole('alert')).toHaveTextContent('appLog.archives.delete.failed')
+ expect(
+ screen.getByRole('button', { name: 'appLog.archives.action.prepareDownload 2025-03' }),
+ ).toHaveAttribute('aria-disabled', 'true')
+ await user.click(screen.getByRole('button', { name: 'appLog.archives.delete.retry 2025-03' }))
+ const dialog = await screen.findByRole('alertdialog')
+ expect(archiveRequests.delete).not.toHaveBeenCalled()
+ await user.click(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.confirm' }),
+ )
+
+ await waitFor(() =>
+ expect(archiveRequests.delete).toHaveBeenCalledWith({ params: { year: 2025, month: 3 } }),
+ )
+ })
+
+ it('refreshes persisted failure status after a deletion request fails', async () => {
+ const user = userEvent.setup()
+ archiveRequests.delete.mockRejectedValue(new Error('Queue unavailable'))
+ archiveRequests.list.mockResolvedValue({
+ ...archiveData,
+ months: [{ ...archiveMonth, deletion_status: 'failed' }],
+ })
+ renderPage()
+
+ await user.click(screen.getByRole('button', { name: 'common.operation.delete 2025-03' }))
+ const dialog = await screen.findByRole('alertdialog')
+ await user.click(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.confirm' }),
+ )
+ expect(await within(dialog).findByRole('alert')).toHaveTextContent(
+ 'appLog.archives.delete.requestFailed',
+ )
+ await user.click(within(dialog).getByRole('button', { name: 'common.operation.cancel' }))
+
+ expect(
+ await screen.findByRole('button', { name: 'appLog.archives.delete.retry 2025-03' }),
+ ).toBeEnabled()
+ expect(
+ screen.getByRole('button', { name: 'appLog.archives.action.prepareDownload 2025-03' }),
+ ).toHaveAttribute('aria-disabled', 'true')
+ })
+
+ it('keeps confirmation open when both deletion and the following list refresh fail', async () => {
+ const user = userEvent.setup()
+ archiveRequests.delete.mockRejectedValue(new Error('Network unavailable'))
+ archiveRequests.list.mockRejectedValue(new Error('Network unavailable'))
+ renderPage()
+
+ await user.click(screen.getByRole('button', { name: 'common.operation.delete 2025-03' }))
+ const dialog = await screen.findByRole('alertdialog')
+ await user.click(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.confirm' }),
+ )
+ expect(await within(dialog).findByRole('alert')).toHaveTextContent(
+ 'appLog.archives.delete.requestFailed',
+ )
+ await waitFor(() => expect(archiveRequests.list).toHaveBeenCalled())
+ expect(dialog).toBeInTheDocument()
+ await user.click(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.confirm' }),
+ )
+ await waitFor(() => expect(archiveRequests.delete).toHaveBeenCalledTimes(2))
+ })
+
+ it('refreshes the month list and summary after an accepted deletion finishes', async () => {
+ const user = userEvent.setup()
+ archiveRequests.list.mockResolvedValue({
+ months: [],
+ summary: {
+ archived_month_count: 0,
+ workflow_run_count: 0,
+ archive_bytes: 0,
+ latest_archived_at: null,
+ },
+ })
+ renderPage()
+
+ await user.click(screen.getByRole('button', { name: 'common.operation.delete 2025-03' }))
+ await user.click(
+ within(await screen.findByRole('alertdialog')).getByRole('button', {
+ name: 'appLog.archives.delete.confirm',
+ }),
+ )
+
+ expect(await screen.findByText('appLog.archives.empty.title')).toBeInTheDocument()
+ expect(screen.queryByRole('row', { name: /2025-03/ })).not.toBeInTheDocument()
+ const runCount = screen.getByText('appLog.archives.summary.runs').parentElement
+ expect(runCount).toHaveTextContent('0')
+ const size = screen.getByText('appLog.archives.summary.size').parentElement
+ expect(size).toHaveTextContent('0 B')
+ })
+
+ it('polls deleting months until they disappear and then stops polling', async () => {
+ vi.useFakeTimers()
+ try {
+ archiveRequests.list
+ .mockResolvedValueOnce({
+ ...archiveData,
+ months: [{ ...archiveMonth, deletion_status: 'processing' }],
+ })
+ .mockResolvedValue({
+ months: [],
+ summary: {
+ archived_month_count: 0,
+ workflow_run_count: 0,
+ archive_bytes: 0,
+ latest_archived_at: null,
+ },
+ })
+ renderPage({ ...archiveData, months: [{ ...archiveMonth, deletion_status: 'pending' }] })
+
+ await act(async () => vi.advanceTimersByTimeAsync(3001))
+ expect(archiveRequests.list).toHaveBeenCalledTimes(1)
+ expect(screen.getByRole('row', { name: /2025-03/ })).toBeInTheDocument()
+
+ await act(async () => vi.advanceTimersByTimeAsync(3001))
+ expect(archiveRequests.list).toHaveBeenCalledTimes(2)
+ expect(screen.getByText('appLog.archives.empty.title')).toBeInTheDocument()
+
+ await act(async () => vi.advanceTimersByTimeAsync(3000))
+ expect(archiveRequests.list).toHaveBeenCalledTimes(2)
+ } finally {
+ vi.useRealTimers()
+ }
+ })
+
+ it('allows a confirmed retry after observing an active deletion for one minute', async () => {
+ const user = userEvent.setup()
+ vi.useFakeTimers()
+ try {
+ renderPage({ ...archiveData, months: [{ ...archiveMonth, deletion_status: 'pending' }] })
+ await act(async () => vi.advanceTimersByTimeAsync(59_999))
+ expect(
+ screen.getByRole('button', { name: 'appLog.archives.delete.deleting 2025-03' }),
+ ).toHaveAttribute('aria-disabled', 'true')
+ await act(async () => vi.advanceTimersByTimeAsync(1))
+ } finally {
+ vi.useRealTimers()
+ }
+
+ await user.click(screen.getByRole('button', { name: 'appLog.archives.delete.retry 2025-03' }))
+ const dialog = await screen.findByRole('alertdialog')
+ expect(archiveRequests.delete).not.toHaveBeenCalled()
+ expect(screen.queryByRole('alert')).not.toBeInTheDocument()
+ await user.click(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.confirm' }),
+ )
+
+ await waitFor(() => expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument())
+ expect(archiveRequests.delete).toHaveBeenCalledWith({ params: { year: 2025, month: 3 } })
+ expect(
+ screen.getByRole('button', { name: 'appLog.archives.delete.deleting 2025-03' }),
+ ).toHaveAttribute('aria-disabled', 'true')
+ expect(
+ screen.getByRole('button', { name: 'appLog.archives.action.prepareDownload 2025-03' }),
+ ).toHaveAttribute('aria-disabled', 'true')
+ })
+
+ it('shows a request error when retrying an active deletion fails', async () => {
+ const user = userEvent.setup()
+ archiveRequests.delete.mockRejectedValueOnce(new Error('Network unavailable'))
+ vi.useFakeTimers()
+ try {
+ renderPage({ ...archiveData, months: [{ ...archiveMonth, deletion_status: 'processing' }] })
+ await act(async () => vi.advanceTimersByTimeAsync(60_000))
+ } finally {
+ vi.useRealTimers()
+ }
+
+ await user.click(screen.getByRole('button', { name: 'appLog.archives.delete.retry 2025-03' }))
+ const dialog = await screen.findByRole('alertdialog')
+ await user.click(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.confirm' }),
+ )
+
+ expect(await within(dialog).findByRole('alert')).toHaveTextContent(
+ 'appLog.archives.delete.requestFailed',
+ )
+ expect(
+ within(dialog).getByRole('button', { name: 'appLog.archives.delete.deleting' }),
+ ).toHaveAttribute('aria-disabled', 'true')
+ await user.click(within(dialog).getByRole('button', { name: 'common.operation.cancel' }))
+ await waitFor(() => expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument())
+ expect(
+ screen.getByRole('button', { name: 'appLog.archives.action.prepareDownload 2025-03' }),
+ ).toHaveAttribute('aria-disabled', 'true')
+ })
+
+ it('removes a month immediately when deletion is already completed', async () => {
+ const user = userEvent.setup()
+ archiveRequests.delete.mockResolvedValue({
+ year: 2025,
+ month: 3,
+ status: 'completed',
+ error: null,
+ })
+ renderPage()
+
+ await user.click(screen.getByRole('button', { name: 'common.operation.delete 2025-03' }))
+ await user.click(
+ within(await screen.findByRole('alertdialog')).getByRole('button', {
+ name: 'appLog.archives.delete.confirm',
+ }),
+ )
+
+ expect(await screen.findByText('appLog.archives.empty.title')).toBeInTheDocument()
+ expect(screen.queryByRole('row', { name: /2025-03/ })).not.toBeInTheDocument()
+ expect(screen.getByText('appLog.archives.summary.months').parentElement).toHaveTextContent(
+ '0',
+ )
+ })
+ })
+
afterEach(() => {
vi.unstubAllGlobals()
})
@@ -220,7 +611,7 @@ describe('WorkflowLogArchivesPage', () => {
)
})
- it('opens failure details independently of the retry action', async () => {
+ it('shows failure details on the retry button without preparing a download', async () => {
const user = userEvent.setup()
renderPage({
...archiveData,
@@ -245,10 +636,16 @@ describe('WorkflowLogArchivesPage', () => {
const retry = screen.getByRole('button', { name: 'common.operation.retry 2025-03' })
expect(retry).toBeEnabled()
- await user.click(screen.getByRole('button', { name: 'common.operation.learnMore: 2025-03' }))
- expect(await screen.findByRole('dialog')).toHaveTextContent(
- 'appLog.archives.downloadHint.failedWithReason',
- )
+ await user.hover(retry)
+ expect(
+ await screen.findByText(
+ 'appLog.archives.downloadHint.failedWithReason:{"reason":"Storage unavailable"}',
+ ),
+ ).toBeVisible()
+ expect(archiveRequests.prepareDownload).not.toHaveBeenCalled()
+ expect(
+ screen.queryByRole('button', { name: 'common.operation.learnMore: 2025-03' }),
+ ).not.toBeInTheDocument()
expect(retry).toBeEnabled()
})
diff --git a/web/app/components/header/account-setting/workflow-log-archives-page/index.tsx b/web/app/components/header/account-setting/workflow-log-archives-page/index.tsx
index adce75d6828..af80479c5da 100644
--- a/web/app/components/header/account-setting/workflow-log-archives-page/index.tsx
+++ b/web/app/components/header/account-setting/workflow-log-archives-page/index.tsx
@@ -5,10 +5,21 @@ import type {
WorkflowRunArchiveDownloadTaskResponse,
WorkflowRunArchiveMonthResponse,
} from '@dify/contracts/api/console/workflow-run-archives/types.gen'
+import {
+ AlertDialog,
+ AlertDialogCancelButton,
+ AlertDialogConfirmButton,
+ AlertDialogContent,
+ AlertDialogDescription,
+ AlertDialogFooter,
+ AlertDialogTitle,
+ AlertDialogTrigger,
+} from '@langgenius/dify-ui/alert-dialog'
import { Button } from '@langgenius/dify-ui/button'
import { cn } from '@langgenius/dify-ui/cn'
-import { Infotip, InfotipContent, InfotipTrigger } from '@langgenius/dify-ui/infotip'
+import { Tooltip, TooltipContent, TooltipTrigger } from '@langgenius/dify-ui/tooltip'
import { skipToken, useMutation, useQuery, useSuspenseQuery } from '@tanstack/react-query'
+import { useAtomValue } from 'jotai'
import { useQueryState } from 'nuqs'
import { useEffect, useId, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
@@ -19,6 +30,7 @@ import {
} from '@/app/components/billing/pricing/query-params'
import { toast } from '@/app/notifications'
import { API_PREFIX } from '@/config'
+import { isCurrentWorkspaceOwnerAtom } from '@/context/workspace-state'
import { systemFeaturesQueryOptions } from '@/features/system-features/client'
import { consoleQuery } from '@/service/console'
@@ -28,6 +40,7 @@ const byteFormatter = new Intl.NumberFormat(undefined, {
})
const BYTE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB']
const DOWNLOAD_TASK_POLLING_INTERVAL = 3000
+const DELETION_RETRY_DELAY = 60_000
const ARCHIVE_MONTH_PAGE_SIZE = 20
function formatNumber(value: number) {
@@ -59,6 +72,10 @@ function isPreparingStatus(status: WorkflowRunArchiveDownloadStatus | undefined)
return status === 'pending' || status === 'processing'
}
+function isDeletingStatus(status: WorkflowRunArchiveMonthResponse['deletion_status']) {
+ return status === 'pending' || status === 'processing'
+}
+
function buildArchiveDownloadFileUrl(downloadId: string) {
return `${API_PREFIX}/workflow-run-archives/downloads/${downloadId}/file`
}
@@ -80,6 +97,10 @@ export default function WorkflowLogArchivesPage() {
const archiveListQuery = useQuery(
consoleQuery.workflowRunArchives.get.queryOptions({
enabled: deploymentEdition === 'CLOUD' && (plan === 'professional' || plan === 'team'),
+ refetchInterval: (query) =>
+ query.state.data?.months.some((archive) => isDeletingStatus(archive.deletion_status))
+ ? DOWNLOAD_TASK_POLLING_INTERVAL
+ : false,
}),
)
const archiveData = archiveListQuery.data
@@ -169,12 +190,12 @@ export default function WorkflowLogArchivesPage() {
<div className="overflow-hidden rounded-xl border-[0.5px] border-components-card-border bg-components-card-bg shadow-xs">
<div className="overflow-x-auto">
- <table className="w-full min-w-115 table-fixed">
+ <table className="w-full min-w-140 table-auto">
<colgroup>
- <col className="w-[20.5%]" />
- <col className="w-[24.2%]" />
- <col className="w-[24.2%]" />
<col />
+ <col />
+ <col />
+ <col className="w-64" />
</colgroup>
<thead>
<tr className="h-8 border-b border-divider-subtle bg-background-section-burn">
@@ -254,7 +275,6 @@ export default function WorkflowLogArchivesPage() {
</tr>
)}
{!isLoading &&
- !archiveListQuery.isError &&
visibleArchiveMonths.map((archive) => {
const archiveMonth = formatMonth(archive.year, archive.month)
@@ -307,12 +327,33 @@ function ArchivedLogsUpgradeBanner() {
function WorkflowArchiveMonthRow({ archive }: { archive: WorkflowRunArchiveMonthResponse }) {
const { t } = useTranslation(['appLog', 'common'])
+ const isCurrentWorkspaceOwner = useAtomValue(isCurrentWorkspaceOwnerAtom)
const archiveMonthLabelId = useId()
const downloadActionLabelId = useId()
+ const deleteActionLabelId = useId()
+ const confirmDeleteLabelId = useId()
+ const [deleteDialogOpen, setDeleteDialogOpen] = useState(false)
+ const [deletionRetryAvailable, setDeletionRetryAvailable] = useState(false)
const [downloadTask, setDownloadTask] = useState<WorkflowRunArchiveDownloadTaskResponse | null>(
null,
)
const archiveMonth = formatMonth(archive.year, archive.month)
+ const deleteArchiveMutation = useMutation(
+ consoleQuery.workflowRunArchives.byYear.byMonth.delete.mutationOptions({
+ context: { silent: true },
+ }),
+ )
+ const isDeletionActive = isDeletingStatus(archive.deletion_status)
+ const canRetryDeletion = isDeletionActive && deletionRetryAvailable
+ const isDeleting = deleteArchiveMutation.isPending || (isDeletionActive && !canRetryDeletion)
+ const isDownloadUnavailable = deleteArchiveMutation.isPending || archive.deletion_status != null
+
+ useEffect(() => {
+ if (!isDeletionActive || deleteArchiveMutation.isPending) return
+
+ const timer = setTimeout(() => setDeletionRetryAvailable(true), DELETION_RETRY_DELAY)
+ return () => clearTimeout(timer)
+ }, [isDeletionActive, deleteArchiveMutation.isPending, deleteArchiveMutation.submittedAt])
const cachedTask = downloadTask ?? archive.download_task ?? null
const downloadTaskId = cachedTask?.download_id
const taskQuery = useQuery(
@@ -324,9 +365,9 @@ function WorkflowArchiveMonthRow({ archive }: { archive: WorkflowRunArchiveMonth
},
}
: skipToken,
- enabled: !!downloadTaskId && isPreparingStatus(cachedTask?.status),
+ enabled: !!downloadTaskId && !isDownloadUnavailable && isPreparingStatus(cachedTask?.status),
refetchInterval: (query) =>
- isPreparingStatus(query.state.data?.status ?? cachedTask?.status)
+ !isDownloadUnavailable && isPreparingStatus(query.state.data?.status ?? cachedTask?.status)
? DOWNLOAD_TASK_POLLING_INTERVAL
: false,
}),
@@ -339,21 +380,25 @@ function WorkflowArchiveMonthRow({ archive }: { archive: WorkflowRunArchiveMonth
const isReady = currentTask?.status === 'ready'
const isFailed = currentTask?.status === 'failed'
const failureReason = isFailed ? currentTask?.error?.trim() : undefined
- const downloadHint = isReady
- ? t(($) => $['archives.downloadHint.ready'], { ns: 'appLog' })
- : isFailed
- ? failureReason
- ? t(($) => $['archives.downloadHint.failedWithReason'], {
- ns: 'appLog',
- reason: failureReason,
- })
- : t(($) => $['archives.downloadHint.failed'], { ns: 'appLog' })
- : isPreparing
- ? t(($) => $['archives.downloadHint.preparing'], { ns: 'appLog' })
- : t(($) => $['archives.downloadHint.prepare'], { ns: 'appLog' })
+ const downloadHint = isDownloadUnavailable
+ ? archive.deletion_status === 'failed'
+ ? t(($) => $['archives.delete.failed'], { ns: 'appLog' })
+ : t(($) => $['archives.delete.deleting'], { ns: 'appLog' })
+ : isReady
+ ? t(($) => $['archives.downloadHint.ready'], { ns: 'appLog' })
+ : isFailed
+ ? failureReason
+ ? t(($) => $['archives.downloadHint.failedWithReason'], {
+ ns: 'appLog',
+ reason: failureReason,
+ })
+ : t(($) => $['archives.downloadHint.failed'], { ns: 'appLog' })
+ : isPreparing
+ ? t(($) => $['archives.downloadHint.preparing'], { ns: 'appLog' })
+ : t(($) => $['archives.downloadHint.prepare'], { ns: 'appLog' })
const prepareDownload = () => {
- if (createDownloadMutation.isPending) return
+ if (createDownloadMutation.isPending || isDownloadUnavailable) return
createDownloadMutation.mutate(
{
@@ -379,7 +424,7 @@ function WorkflowArchiveMonthRow({ archive }: { archive: WorkflowRunArchiveMonth
}
const downloadArchive = () => {
- if (!currentTask || currentTask.status !== 'ready') return
+ if (isDownloadUnavailable || !currentTask || currentTask.status !== 'ready') return
globalThis.location.assign(buildArchiveDownloadFileUrl(currentTask.download_id))
}
@@ -394,6 +439,16 @@ function WorkflowArchiveMonthRow({ archive }: { archive: WorkflowRunArchiveMonth
const buttonIconClassName = isReady ? 'i-ri-download-2-line' : 'i-ri-inbox-archive-line'
const onAction = isReady ? downloadArchive : prepareDownload
+ const deleteArchive = () => {
+ if (!isCurrentWorkspaceOwner || isDeleting) return
+
+ setDeletionRetryAvailable(false)
+ deleteArchiveMutation.mutate(
+ { params: { year: archive.year, month: archive.month } },
+ { onSuccess: () => setDeleteDialogOpen(false) },
+ )
+ }
+
return (
<tr className="h-15 border-b border-divider-subtle last:border-b-0">
<td className="px-2 py-3 text-center">
@@ -408,32 +463,108 @@ function WorkflowArchiveMonthRow({ archive }: { archive: WorkflowRunArchiveMonth
{formatBytes(archive.archive_bytes)}
</td>
<td className="px-2 py-3 text-center">
- <div className="flex items-center justify-center gap-1">
- <Button
- size="small"
- variant="secondary"
- loading={isPreparing}
- className="px-2"
- aria-labelledby={`${downloadActionLabelId} ${archiveMonthLabelId}`}
- onClick={onAction}
- >
- {!isPreparing && (
- <span className={cn(buttonIconClassName, 'size-3.5')} aria-hidden="true" />
+ <div className="flex flex-col items-center gap-2">
+ <div className="flex items-center justify-center gap-1">
+ <Tooltip>
+ <TooltipTrigger
+ render={
+ <Button
+ size="small"
+ variant="secondary"
+ loading={isPreparing}
+ disabled={isDownloadUnavailable}
+ focusableWhenDisabled
+ className="shrink-0 px-2"
+ aria-labelledby={`${downloadActionLabelId} ${archiveMonthLabelId}`}
+ onClick={onAction}
+ />
+ }
+ >
+ {!isPreparing && (
+ <span
+ className={cn(buttonIconClassName, 'size-3.5 shrink-0')}
+ aria-hidden="true"
+ />
+ )}
+ <span id={downloadActionLabelId}>{buttonContent}</span>
+ </TooltipTrigger>
+ <TooltipContent>{downloadHint}</TooltipContent>
+ </Tooltip>
+ {isCurrentWorkspaceOwner && (
+ <AlertDialog
+ open={deleteDialogOpen}
+ onOpenChange={(open) => {
+ if (deleteArchiveMutation.isPending) return
+ if (open) deleteArchiveMutation.reset()
+ setDeleteDialogOpen(open)
+ }}
+ >
+ <AlertDialogTrigger
+ render={
+ <Button
+ size="small"
+ variant="tertiary"
+ tone="destructive"
+ className="shrink-0"
+ loading={isDeleting}
+ aria-labelledby={`${deleteActionLabelId} ${archiveMonthLabelId}`}
+ />
+ }
+ >
+ {!isDeleting && (
+ <span className="i-ri-delete-bin-line size-3.5" aria-hidden="true" />
+ )}
+ <span id={deleteActionLabelId}>
+ {isDeleting
+ ? t(($) => $['archives.delete.deleting'], { ns: 'appLog' })
+ : archive.deletion_status === 'failed' || canRetryDeletion
+ ? t(($) => $['archives.delete.retry'], { ns: 'appLog' })
+ : t(($) => $['operation.delete'], { ns: 'common' })}
+ </span>
+ </AlertDialogTrigger>
+ <AlertDialogContent>
+ <div className="flex flex-col gap-2 px-6 pt-6 pb-4">
+ <AlertDialogTitle className="title-2xl-semi-bold text-text-primary">
+ {t(($) => $['archives.delete.title'], { ns: 'appLog', month: archiveMonth })}
+ </AlertDialogTitle>
+ <AlertDialogDescription className="system-md-regular wrap-break-word text-text-tertiary">
+ {t(($) => $['archives.delete.description'], { ns: 'appLog' })}
+ </AlertDialogDescription>
+ {deleteArchiveMutation.isError && (
+ <div role="alert" className="system-sm-regular text-text-destructive">
+ {t(($) => $['archives.delete.requestFailed'], { ns: 'appLog' })}
+ </div>
+ )}
+ </div>
+ <AlertDialogFooter>
+ <AlertDialogCancelButton
+ variant="secondary"
+ disabled={deleteArchiveMutation.isPending}
+ >
+ {t(($) => $['operation.cancel'], { ns: 'common' })}
+ </AlertDialogCancelButton>
+ <AlertDialogConfirmButton
+ loading={isDeleting}
+ disabled={isDeletionActive && !canRetryDeletion}
+ aria-labelledby={confirmDeleteLabelId}
+ onClick={deleteArchive}
+ >
+ <span id={confirmDeleteLabelId}>
+ {isDeleting
+ ? t(($) => $['archives.delete.deleting'], { ns: 'appLog' })
+ : t(($) => $['archives.delete.confirm'], { ns: 'appLog' })}
+ </span>
+ </AlertDialogConfirmButton>
+ </AlertDialogFooter>
+ </AlertDialogContent>
+ </AlertDialog>
)}
- <span id={downloadActionLabelId}>{buttonContent}</span>
- </Button>
- <Infotip>
- <InfotipTrigger
- aria-label={`${t(($) => $['operation.learnMore'], { ns: 'common' })}: ${archiveMonth}`}
- iconVariant={isFailed ? 'warning' : 'information'}
- />
- <InfotipContent
- aria-label={`${t(($) => $['operation.learnMore'], { ns: 'common' })}: ${archiveMonth}`}
- className="whitespace-pre-wrap"
- >
- {downloadHint}
- </InfotipContent>
- </Infotip>
+ </div>
+ {archive.deletion_status === 'failed' && (
+ <div role="alert" className="max-w-56 system-xs-regular text-text-destructive">
+ {t(($) => $['archives.delete.failed'], { ns: 'appLog' })}
+ </div>
+ )}
</div>
</td>
</tr>
diff --git a/web/i18n/locales/ar-TN/app-log.json b/web/i18n/locales/ar-TN/app-log.json
index 95f8654d1d0..ab52e22fec8 100644
--- a/web/i18n/locales/ar-TN/app-log.json
+++ b/web/i18n/locales/ar-TN/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "فشل تحضير تنزيل الأرشيف.",
"archives.action.prepareStarted": "جارٍ تحضير تنزيل الأرشيف.",
"archives.action.preparing": "جارٍ التحضير",
+ "archives.delete.confirm": "حذف نهائي",
+ "archives.delete.deleting": "جارٍ الحذف",
+ "archives.delete.description": "سيؤدي هذا إلى حذف سجلات سير العمل المؤرشفة وملفات التنزيل لجميع التطبيقات لهذا الشهر نهائيًا. لن تتمكن من تنزيلها أو استعادتها بعد ذلك. نزّل نسخة احتياطية أولًا.",
+ "archives.delete.failed": "لم يكتمل الحذف. أعد المحاولة لحذف البيانات المؤرشفة المتبقية.",
+ "archives.delete.requestFailed": "تعذر بدء الحذف. يرجى إعادة المحاولة.",
+ "archives.delete.retry": "إعادة محاولة الحذف",
+ "archives.delete.title": "حذف السجلات المؤرشفة لشهر {{month}}؟",
"archives.description": "نزّل صادرات CSV شهرية لسجلات تشغيل سير العمل المؤرشفة في مساحة العمل هذه.",
"archives.downloadHint.failed": "فشل تحضير هذا الأرشيف. أعد المحاولة لبدء مهمة تنزيل جديدة.",
"archives.downloadHint.failedWithReason": "فشل تحضير هذا الأرشيف: {{reason}}",
diff --git a/web/i18n/locales/az-AZ/app-log.json b/web/i18n/locales/az-AZ/app-log.json
index 422dd3f72b3..d97c258a1a7 100644
--- a/web/i18n/locales/az-AZ/app-log.json
+++ b/web/i18n/locales/az-AZ/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Arxivin yüklənməsini hazırlamaq mümkün olmadı.",
"archives.action.prepareStarted": "Arxivin yüklənməsi hazırlanır.",
"archives.action.preparing": "Hazırlanır",
+ "archives.delete.confirm": "Həmişəlik sil",
+ "archives.delete.deleting": "Silinir",
+ "archives.delete.description": "Bu, həmin ay üzrə bütün tətbiqlərin arxivləşdirilmiş iş axını jurnallarını və endirmə fayllarını həmişəlik siləcək. Onları daha sonra endirmək və ya bərpa etmək mümkün olmayacaq. Əvvəlcə ehtiyat nüsxəni endirin.",
+ "archives.delete.failed": "Silinmə tamamlanmayıb. Qalan arxiv məlumatlarını silmək üçün yenidən cəhd edin.",
+ "archives.delete.requestFailed": "Silinməyə başlamaq mümkün olmadı. Yenidən cəhd edin.",
+ "archives.delete.retry": "Silməyə yenidən cəhd et",
+ "archives.delete.title": "{{month}} ayının arxivləşdirilmiş jurnalları silinsin?",
"archives.description": "Bu iş məkanında arxivləşdirilmiş iş axını icra jurnallarını aylıq CSV faylları şəklində yükləyin.",
"archives.downloadHint.failed": "Bu arxivi hazırlamaq mümkün olmadı. Yeni yükləmə tapşırığı başlatmaq üçün yenidən cəhd edin.",
"archives.downloadHint.failedWithReason": "Bu arxivi hazırlamaq mümkün olmadı: {{reason}}",
diff --git a/web/i18n/locales/de-DE/app-log.json b/web/i18n/locales/de-DE/app-log.json
index 543e2cbfecb..74e4e891b0c 100644
--- a/web/i18n/locales/de-DE/app-log.json
+++ b/web/i18n/locales/de-DE/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Archivdownload konnte nicht vorbereitet werden.",
"archives.action.prepareStarted": "Der Archivdownload wird vorbereitet.",
"archives.action.preparing": "Wird vorbereitet",
+ "archives.delete.confirm": "Endgültig löschen",
+ "archives.delete.deleting": "Wird gelöscht",
+ "archives.delete.description": "Dadurch werden die archivierten Workflow-Protokolle und Download-Dateien aller Apps für diesen Monat endgültig gelöscht. Sie können danach weder heruntergeladen noch wiederhergestellt werden. Laden Sie zuerst eine Sicherung herunter.",
+ "archives.delete.failed": "Die Löschung ist unvollständig. Versuchen Sie es erneut, um die verbleibenden Archivdaten zu löschen.",
+ "archives.delete.requestFailed": "Die Löschung konnte nicht gestartet werden. Bitte versuchen Sie es erneut.",
+ "archives.delete.retry": "Löschung wiederholen",
+ "archives.delete.title": "Archivierte Protokolle für {{month}} löschen?",
"archives.description": "Monatliche CSV-Exporte archivierter Workflow-Ausführungsprotokolle in diesem Workspace herunterladen.",
"archives.downloadHint.failed": "Die Vorbereitung dieses Archivs ist fehlgeschlagen. Versuchen Sie es erneut, um eine neue Downloadaufgabe zu starten.",
"archives.downloadHint.failedWithReason": "Die Vorbereitung dieses Archivs ist fehlgeschlagen: {{reason}}",
diff --git a/web/i18n/locales/en-US/app-log.json b/web/i18n/locales/en-US/app-log.json
index 0dbd2e7a21f..33f9c164c3b 100644
--- a/web/i18n/locales/en-US/app-log.json
+++ b/web/i18n/locales/en-US/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Failed to prepare archive download.",
"archives.action.prepareStarted": "Archive download is being prepared.",
"archives.action.preparing": "Preparing",
+ "archives.delete.confirm": "Permanently delete",
+ "archives.delete.deleting": "Deleting",
+ "archives.delete.description": "This will permanently delete archived workflow logs and download files for all apps in this month. You will no longer be able to download or restore them. Download a backup first.",
+ "archives.delete.failed": "Deletion is incomplete. Retry to delete the remaining archived data.",
+ "archives.delete.requestFailed": "Could not start deletion. Please try again.",
+ "archives.delete.retry": "Retry deletion",
+ "archives.delete.title": "Delete archived logs for {{month}}?",
"archives.description": "Download monthly CSV exports of archived workflow run logs across this workspace.",
"archives.downloadHint.failed": "Preparing this archive failed. Retry to start a new download task.",
"archives.downloadHint.failedWithReason": "Preparing this archive failed: {{reason}}",
diff --git a/web/i18n/locales/es-ES/app-log.json b/web/i18n/locales/es-ES/app-log.json
index 844308fdd50..f6fabe47ba1 100644
--- a/web/i18n/locales/es-ES/app-log.json
+++ b/web/i18n/locales/es-ES/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "No se pudo preparar la descarga del archivo.",
"archives.action.prepareStarted": "Se está preparando la descarga del archivo.",
"archives.action.preparing": "Preparando",
+ "archives.delete.confirm": "Eliminar permanentemente",
+ "archives.delete.deleting": "Eliminando",
+ "archives.delete.description": "Se eliminarán permanentemente los registros de flujo de trabajo archivados y los archivos de descarga de todas las aplicaciones de este mes. Ya no podrás descargarlos ni restaurarlos. Descarga primero una copia de seguridad.",
+ "archives.delete.failed": "La eliminación no se ha completado. Vuelve a intentarlo para eliminar los datos archivados restantes.",
+ "archives.delete.requestFailed": "No se pudo iniciar la eliminación. Inténtalo de nuevo.",
+ "archives.delete.retry": "Reintentar eliminación",
+ "archives.delete.title": "¿Eliminar los registros archivados de {{month}}?",
"archives.description": "Descarga exportaciones CSV mensuales de los registros archivados de ejecuciones de workflow en este espacio de trabajo.",
"archives.downloadHint.failed": "No se pudo preparar este archivo. Vuelve a intentarlo para iniciar una nueva tarea de descarga.",
"archives.downloadHint.failedWithReason": "No se pudo preparar este archivo: {{reason}}",
diff --git a/web/i18n/locales/fa-IR/app-log.json b/web/i18n/locales/fa-IR/app-log.json
index 3824f605937..2e7263e882c 100644
--- a/web/i18n/locales/fa-IR/app-log.json
+++ b/web/i18n/locales/fa-IR/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "آماده‌سازی دانلود بایگانی ناموفق بود.",
"archives.action.prepareStarted": "دانلود بایگانی در حال آماده‌سازی است.",
"archives.action.preparing": "در حال آماده‌سازی",
+ "archives.delete.confirm": "حذف دائمی",
+ "archives.delete.deleting": "در حال حذف",
+ "archives.delete.description": "این کار گزارش‌های بایگانی‌شدهٔ گردش کار و فایل‌های دانلود همهٔ برنامه‌ها در این ماه را برای همیشه حذف می‌کند. پس از حذف، امکان دانلود یا بازیابی آن‌ها وجود ندارد. ابتدا یک نسخهٔ پشتیبان دانلود کنید.",
+ "archives.delete.failed": "حذف کامل نشده است. برای حذف داده‌های بایگانی‌شدهٔ باقی‌مانده دوباره تلاش کنید.",
+ "archives.delete.requestFailed": "شروع حذف ممکن نشد. لطفاً دوباره تلاش کنید.",
+ "archives.delete.retry": "تلاش دوباره برای حذف",
+ "archives.delete.title": "گزارش‌های بایگانی‌شدهٔ {{month}} حذف شوند؟",
"archives.description": "خروجی‌های CSV ماهانه از لاگ‌های بایگانی‌شده اجرای workflow در این فضای کاری را دانلود کنید.",
"archives.downloadHint.failed": "آماده‌سازی این بایگانی ناموفق بود. برای شروع یک وظیفه دانلود جدید دوباره تلاش کنید.",
"archives.downloadHint.failedWithReason": "آماده‌سازی این بایگانی ناموفق بود: {{reason}}",
diff --git a/web/i18n/locales/fr-FR/app-log.json b/web/i18n/locales/fr-FR/app-log.json
index e88b0a0da2b..45f91f2df79 100644
--- a/web/i18n/locales/fr-FR/app-log.json
+++ b/web/i18n/locales/fr-FR/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Impossible de préparer le téléchargement de l’archive.",
"archives.action.prepareStarted": "Le téléchargement de l’archive est en cours de préparation.",
"archives.action.preparing": "Préparation",
+ "archives.delete.confirm": "Supprimer définitivement",
+ "archives.delete.deleting": "Suppression en cours",
+ "archives.delete.description": "Cela supprimera définitivement les journaux de workflow archivés et les fichiers de téléchargement de toutes les applications pour ce mois. Vous ne pourrez plus les télécharger ni les restaurer. Téléchargez d’abord une sauvegarde.",
+ "archives.delete.failed": "La suppression est incomplète. Réessayez pour supprimer les données archivées restantes.",
+ "archives.delete.requestFailed": "Impossible de lancer la suppression. Veuillez réessayer.",
+ "archives.delete.retry": "Réessayer la suppression",
+ "archives.delete.title": "Supprimer les journaux archivés de {{month}} ?",
"archives.description": "Téléchargez des exports CSV mensuels des journaux archivés d’exécutions de workflow dans cet espace de travail.",
"archives.downloadHint.failed": "La préparation de cette archive a échoué. Réessayez pour démarrer une nouvelle tâche de téléchargement.",
"archives.downloadHint.failedWithReason": "La préparation de cette archive a échoué : {{reason}}",
diff --git a/web/i18n/locales/hi-IN/app-log.json b/web/i18n/locales/hi-IN/app-log.json
index f843b6358b4..5669df57648 100644
--- a/web/i18n/locales/hi-IN/app-log.json
+++ b/web/i18n/locales/hi-IN/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "आर्काइव डाउनलोड तैयार करने में विफल रहा।",
"archives.action.prepareStarted": "आर्काइव डाउनलोड तैयार किया जा रहा है।",
"archives.action.preparing": "तैयार किया जा रहा है",
+ "archives.delete.confirm": "स्थायी रूप से हटाएँ",
+ "archives.delete.deleting": "हटाया जा रहा है",
+ "archives.delete.description": "इससे इस महीने के सभी ऐप के संग्रहित वर्कफ़्लो लॉग और डाउनलोड फ़ाइलें स्थायी रूप से हट जाएँगी। इसके बाद आप उन्हें डाउनलोड या पुनर्स्थापित नहीं कर पाएँगे। पहले बैकअप डाउनलोड कर लें।",
+ "archives.delete.failed": "हटाने की प्रक्रिया अधूरी है। शेष संग्रहित डेटा हटाने के लिए फिर से प्रयास करें।",
+ "archives.delete.requestFailed": "हटाने की प्रक्रिया शुरू नहीं हो सकी। कृपया फिर से प्रयास करें।",
+ "archives.delete.retry": "हटाने का फिर प्रयास करें",
+ "archives.delete.title": "{{month}} के संग्रहित लॉग हटाएँ?",
"archives.description": "इस वर्कस्पेस में आर्काइव किए गए workflow रन लॉग के मासिक CSV निर्यात डाउनलोड करें।",
"archives.downloadHint.failed": "इस आर्काइव को तैयार करना विफल रहा। नया डाउनलोड कार्य शुरू करने के लिए फिर से प्रयास करें।",
"archives.downloadHint.failedWithReason": "इस आर्काइव को तैयार करना विफल रहा: {{reason}}",
diff --git a/web/i18n/locales/id-ID/app-log.json b/web/i18n/locales/id-ID/app-log.json
index 63a4c77130b..9384daa9f34 100644
--- a/web/i18n/locales/id-ID/app-log.json
+++ b/web/i18n/locales/id-ID/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Gagal menyiapkan unduhan arsip.",
"archives.action.prepareStarted": "Unduhan arsip sedang disiapkan.",
"archives.action.preparing": "Menyiapkan",
+ "archives.delete.confirm": "Hapus permanen",
+ "archives.delete.deleting": "Menghapus",
+ "archives.delete.description": "Ini akan menghapus permanen log alur kerja yang diarsipkan dan file unduhan untuk semua aplikasi pada bulan ini. Anda tidak akan dapat mengunduh atau memulihkannya lagi. Unduh cadangan terlebih dahulu.",
+ "archives.delete.failed": "Penghapusan belum selesai. Coba lagi untuk menghapus data arsip yang tersisa.",
+ "archives.delete.requestFailed": "Tidak dapat memulai penghapusan. Silakan coba lagi.",
+ "archives.delete.retry": "Coba hapus lagi",
+ "archives.delete.title": "Hapus log arsip untuk {{month}}?",
"archives.description": "Unduh ekspor CSV bulanan dari log workflow run yang diarsipkan di workspace ini.",
"archives.downloadHint.failed": "Gagal menyiapkan arsip ini. Coba lagi untuk memulai tugas unduhan baru.",
"archives.downloadHint.failedWithReason": "Gagal menyiapkan arsip ini: {{reason}}",
diff --git a/web/i18n/locales/it-IT/app-log.json b/web/i18n/locales/it-IT/app-log.json
index 035ca213550..9f3564e3d65 100644
--- a/web/i18n/locales/it-IT/app-log.json
+++ b/web/i18n/locales/it-IT/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Impossibile preparare il download dell’archivio.",
"archives.action.prepareStarted": "Il download dell’archivio è in preparazione.",
"archives.action.preparing": "Preparazione",
+ "archives.delete.confirm": "Elimina definitivamente",
+ "archives.delete.deleting": "Eliminazione in corso",
+ "archives.delete.description": "Questa operazione eliminerà definitivamente i log dei flussi di lavoro archiviati e i file da scaricare di tutte le app per questo mese. Non sarà più possibile scaricarli o ripristinarli. Scarica prima una copia di backup.",
+ "archives.delete.failed": "L’eliminazione non è completa. Riprova per eliminare i dati archiviati rimanenti.",
+ "archives.delete.requestFailed": "Impossibile avviare l’eliminazione. Riprova.",
+ "archives.delete.retry": "Riprova eliminazione",
+ "archives.delete.title": "Eliminare i log archiviati di {{month}}?",
"archives.description": "Scarica esportazioni CSV mensili dei log archiviati delle esecuzioni workflow in questo workspace.",
"archives.downloadHint.failed": "La preparazione di questo archivio non è riuscita. Riprova per avviare una nuova attività di download.",
"archives.downloadHint.failedWithReason": "La preparazione di questo archivio non è riuscita: {{reason}}",
diff --git a/web/i18n/locales/ja-JP/app-log.json b/web/i18n/locales/ja-JP/app-log.json
index 4945c3dd9f5..1e5b7c8ea66 100644
--- a/web/i18n/locales/ja-JP/app-log.json
+++ b/web/i18n/locales/ja-JP/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "アーカイブのダウンロード準備に失敗しました。",
"archives.action.prepareStarted": "アーカイブのダウンロードを準備しています。",
"archives.action.preparing": "準備中",
+ "archives.delete.confirm": "完全に削除",
+ "archives.delete.deleting": "削除中",
+ "archives.delete.description": "この月のすべてのアプリのアーカイブ済みワークフローログとダウンロードファイルを完全に削除します。削除後はダウンロードや復元ができません。先にバックアップをダウンロードしてください。",
+ "archives.delete.failed": "削除が完了していません。再試行して残りのアーカイブデータを削除してください。",
+ "archives.delete.requestFailed": "削除を開始できませんでした。もう一度お試しください。",
+ "archives.delete.retry": "削除を再試行",
+ "archives.delete.title": "{{month}} のアーカイブログを削除しますか?",
"archives.description": "このワークスペース内のアーカイブされたワークフロー実行ログを月別 CSV エクスポートとしてダウンロードします。",
"archives.downloadHint.failed": "このアーカイブの準備に失敗しました。再試行すると新しいダウンロードタスクを開始します。",
"archives.downloadHint.failedWithReason": "このアーカイブの準備に失敗しました: {{reason}}",
diff --git a/web/i18n/locales/ko-KR/app-log.json b/web/i18n/locales/ko-KR/app-log.json
index 853fa3ce1c0..baeaed880ab 100644
--- a/web/i18n/locales/ko-KR/app-log.json
+++ b/web/i18n/locales/ko-KR/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "아카이브 다운로드를 준비하지 못했습니다.",
"archives.action.prepareStarted": "아카이브 다운로드를 준비하는 중입니다.",
"archives.action.preparing": "준비 중",
+ "archives.delete.confirm": "영구 삭제",
+ "archives.delete.deleting": "삭제 중",
+ "archives.delete.description": "이 달의 모든 앱에 대한 보관된 워크플로 로그와 다운로드 파일이 영구적으로 삭제됩니다. 삭제 후에는 다운로드하거나 복원할 수 없습니다. 먼저 백업을 다운로드하세요.",
+ "archives.delete.failed": "삭제가 완료되지 않았습니다. 다시 시도하여 남은 보관 데이터를 삭제하세요.",
+ "archives.delete.requestFailed": "삭제를 시작할 수 없습니다. 다시 시도하세요.",
+ "archives.delete.retry": "삭제 재시도",
+ "archives.delete.title": "{{month}}의 보관된 로그를 삭제할까요?",
"archives.description": "이 워크스페이스의 보관된 워크플로 실행 로그를 월별 CSV 내보내기로 다운로드합니다.",
"archives.downloadHint.failed": "이 아카이브를 준비하지 못했습니다. 새 다운로드 작업을 시작하려면 다시 시도하세요.",
"archives.downloadHint.failedWithReason": "이 아카이브를 준비하지 못했습니다: {{reason}}",
diff --git a/web/i18n/locales/lo-LA/app-log.json b/web/i18n/locales/lo-LA/app-log.json
index 13f41f30e47..1ff3ce7ca1f 100644
--- a/web/i18n/locales/lo-LA/app-log.json
+++ b/web/i18n/locales/lo-LA/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "ກະກຽມການດາວໂຫລດບົດຈັດເກັບບໍ່ສຳເລັດ.",
"archives.action.prepareStarted": "ກຳລັງກະກຽມການດາວໂຫລດບົດຈັດເກັບ.",
"archives.action.preparing": "ກຳລັງກະກຽມ",
+ "archives.delete.confirm": "ລຶບຖາວອນ",
+ "archives.delete.deleting": "ກຳລັງລຶບ",
+ "archives.delete.description": "ການດຳເນີນການນີ້ຈະລຶບບັນທຶກເວີກໂຟລທີ່ເກັບໄວ້ ແລະໄຟລ໌ດາວໂຫຼດຂອງທຸກແອັບໃນເດືອນນີ້ຢ່າງຖາວອນ. ຫຼັງຈາກລຶບແລ້ວຈະບໍ່ສາມາດດາວໂຫຼດ ຫຼືກູ້ຄືນໄດ້. ກະລຸນາດາວໂຫຼດສຳຮອງກ່ອນ.",
+ "archives.delete.failed": "ການລຶບຍັງບໍ່ສຳເລັດ. ລອງໃໝ່ເພື່ອລຶບຂໍ້ມູນທີ່ເກັບໄວ້ທີ່ຍັງເຫຼືອ.",
+ "archives.delete.requestFailed": "ບໍ່ສາມາດເລີ່ມລຶບໄດ້. ກະລຸນາລອງໃໝ່.",
+ "archives.delete.retry": "ລອງລຶບໃໝ່",
+ "archives.delete.title": "ລຶບບັນທຶກທີ່ເກັບໄວ້ຂອງ {{month}} ຫຼືບໍ່?",
"archives.description": "ດາວໂຫລດການສົ່ງອອກ CSV ປະຈຳເດືອນຂອງບັນທຶກການເຮັດວຽກ Workflow ທີ່ຖືກຈັດເກັບໄວ້ໃນພື້ນທີ່ເຮັດວຽກນີ້.",
"archives.downloadHint.failed": "ການກະກຽມບົດຈັດເກັບນີ້ບໍ່ສຳເລັດ. ກະລຸນາລອງໃໝ່ເພື່ອເລີ່ມຕົ້ນວຽກການດາວໂຫລດໃໝ່.",
"archives.downloadHint.failedWithReason": "ການກະກຽມບົດຈັດເກັບນີ້ບໍ່ສຳເລັດ: {{reason}}",
diff --git a/web/i18n/locales/nl-NL/app-log.json b/web/i18n/locales/nl-NL/app-log.json
index 8f24121f58e..53d8e29e05f 100644
--- a/web/i18n/locales/nl-NL/app-log.json
+++ b/web/i18n/locales/nl-NL/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Kan archiefdownload niet voorbereiden.",
"archives.action.prepareStarted": "De archiefdownload wordt voorbereid.",
"archives.action.preparing": "Voorbereiden",
+ "archives.delete.confirm": "Permanent verwijderen",
+ "archives.delete.deleting": "Bezig met verwijderen",
+ "archives.delete.description": "Hiermee worden de gearchiveerde workflowlogboeken en downloadbestanden van alle apps voor deze maand permanent verwijderd. Je kunt ze daarna niet meer downloaden of herstellen. Download eerst een back-up.",
+ "archives.delete.failed": "Het verwijderen is niet voltooid. Probeer opnieuw om de resterende gearchiveerde gegevens te verwijderen.",
+ "archives.delete.requestFailed": "Het verwijderen kon niet worden gestart. Probeer het opnieuw.",
+ "archives.delete.retry": "Verwijderen opnieuw proberen",
+ "archives.delete.title": "Gearchiveerde logboeken voor {{month}} verwijderen?",
"archives.description": "Download maandelijkse CSV-exports van gearchiveerde workflow-runlogs in deze werkruimte.",
"archives.downloadHint.failed": "Het voorbereiden van dit archief is mislukt. Probeer opnieuw om een nieuwe downloadtaak te starten.",
"archives.downloadHint.failedWithReason": "Het voorbereiden van dit archief is mislukt: {{reason}}",
diff --git a/web/i18n/locales/pl-PL/app-log.json b/web/i18n/locales/pl-PL/app-log.json
index f120892e4e1..022835d57ad 100644
--- a/web/i18n/locales/pl-PL/app-log.json
+++ b/web/i18n/locales/pl-PL/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Nie udało się przygotować pobierania archiwum.",
"archives.action.prepareStarted": "Pobieranie archiwum jest przygotowywane.",
"archives.action.preparing": "Przygotowywanie",
+ "archives.delete.confirm": "Usuń trwale",
+ "archives.delete.deleting": "Usuwanie",
+ "archives.delete.description": "Spowoduje to trwałe usunięcie zarchiwizowanych dzienników przepływów pracy i plików do pobrania dla wszystkich aplikacji z tego miesiąca. Nie będzie można ich pobrać ani przywrócić. Najpierw pobierz kopię zapasową.",
+ "archives.delete.failed": "Usuwanie nie zostało ukończone. Spróbuj ponownie, aby usunąć pozostałe zarchiwizowane dane.",
+ "archives.delete.requestFailed": "Nie udało się rozpocząć usuwania. Spróbuj ponownie.",
+ "archives.delete.retry": "Ponów usuwanie",
+ "archives.delete.title": "Usunąć zarchiwizowane dzienniki z {{month}}?",
"archives.description": "Pobierz miesięczne eksporty CSV zarchiwizowanych logów uruchomień workflow w tym obszarze roboczym.",
"archives.downloadHint.failed": "Przygotowanie tego archiwum nie powiodło się. Spróbuj ponownie, aby uruchomić nowe zadanie pobierania.",
"archives.downloadHint.failedWithReason": "Przygotowanie tego archiwum nie powiodło się: {{reason}}",
diff --git a/web/i18n/locales/pt-BR/app-log.json b/web/i18n/locales/pt-BR/app-log.json
index 69ece36c793..2b5fe031f74 100644
--- a/web/i18n/locales/pt-BR/app-log.json
+++ b/web/i18n/locales/pt-BR/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Falha ao preparar o download do arquivo.",
"archives.action.prepareStarted": "O download do arquivo está sendo preparado.",
"archives.action.preparing": "Preparando",
+ "archives.delete.confirm": "Excluir permanentemente",
+ "archives.delete.deleting": "Excluindo",
+ "archives.delete.description": "Isso excluirá permanentemente os logs de fluxo de trabalho arquivados e os arquivos de download de todos os aplicativos deste mês. Você não poderá mais baixá-los ou restaurá-los. Baixe uma cópia de segurança primeiro.",
+ "archives.delete.failed": "A exclusão está incompleta. Tente novamente para excluir os dados arquivados restantes.",
+ "archives.delete.requestFailed": "Não foi possível iniciar a exclusão. Tente novamente.",
+ "archives.delete.retry": "Tentar excluir novamente",
+ "archives.delete.title": "Excluir os logs arquivados de {{month}}?",
"archives.description": "Baixe exportações CSV mensais dos logs arquivados de execuções de workflow neste workspace.",
"archives.downloadHint.failed": "Falha ao preparar este arquivo. Tente novamente para iniciar uma nova tarefa de download.",
"archives.downloadHint.failedWithReason": "Falha ao preparar este arquivo: {{reason}}",
diff --git a/web/i18n/locales/ro-RO/app-log.json b/web/i18n/locales/ro-RO/app-log.json
index 4655bb8a5cb..71808544672 100644
--- a/web/i18n/locales/ro-RO/app-log.json
+++ b/web/i18n/locales/ro-RO/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Pregătirea descărcării arhivei a eșuat.",
"archives.action.prepareStarted": "Descărcarea arhivei este în pregătire.",
"archives.action.preparing": "Se pregătește",
+ "archives.delete.confirm": "Șterge definitiv",
+ "archives.delete.deleting": "Se șterge",
+ "archives.delete.description": "Aceasta va șterge definitiv jurnalele fluxurilor de lucru arhivate și fișierele de descărcat pentru toate aplicațiile din această lună. Nu le vei mai putea descărca sau restaura. Descarcă mai întâi o copie de rezervă.",
+ "archives.delete.failed": "Ștergerea nu este completă. Reîncearcă pentru a șterge datele arhivate rămase.",
+ "archives.delete.requestFailed": "Nu s-a putut începe ștergerea. Te rugăm să încerci din nou.",
+ "archives.delete.retry": "Reîncearcă ștergerea",
+ "archives.delete.title": "Ștergi jurnalele arhivate pentru {{month}}?",
"archives.description": "Descarcă exporturi CSV lunare ale jurnalelor arhivate de rulări workflow din acest spațiu de lucru.",
"archives.downloadHint.failed": "Pregătirea acestei arhive a eșuat. Reîncearcă pentru a porni o nouă sarcină de descărcare.",
"archives.downloadHint.failedWithReason": "Pregătirea acestei arhive a eșuat: {{reason}}",
diff --git a/web/i18n/locales/ru-RU/app-log.json b/web/i18n/locales/ru-RU/app-log.json
index 73aa447f44e..5acb678c065 100644
--- a/web/i18n/locales/ru-RU/app-log.json
+++ b/web/i18n/locales/ru-RU/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Не удалось подготовить скачивание архива.",
"archives.action.prepareStarted": "Скачивание архива подготавливается.",
"archives.action.preparing": "Подготовка",
+ "archives.delete.confirm": "Удалить навсегда",
+ "archives.delete.deleting": "Удаление",
+ "archives.delete.description": "Архивные журналы рабочих процессов и файлы для скачивания всех приложений за этот месяц будут удалены навсегда. После удаления их нельзя будет скачать или восстановить. Сначала скачайте резервную копию.",
+ "archives.delete.failed": "Удаление не завершено. Повторите попытку, чтобы удалить оставшиеся архивные данные.",
+ "archives.delete.requestFailed": "Не удалось начать удаление. Повторите попытку.",
+ "archives.delete.retry": "Повторить удаление",
+ "archives.delete.title": "Удалить архивные журналы за {{month}}?",
"archives.description": "Скачивайте ежемесячные CSV-экспорты архивных логов запусков workflow в этом рабочем пространстве.",
"archives.downloadHint.failed": "Не удалось подготовить этот архив. Повторите попытку, чтобы запустить новую задачу скачивания.",
"archives.downloadHint.failedWithReason": "Не удалось подготовить этот архив: {{reason}}",
diff --git a/web/i18n/locales/sl-SI/app-log.json b/web/i18n/locales/sl-SI/app-log.json
index c108421df26..46f281fc71a 100644
--- a/web/i18n/locales/sl-SI/app-log.json
+++ b/web/i18n/locales/sl-SI/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Priprava prenosa arhiva ni uspela.",
"archives.action.prepareStarted": "Prenos arhiva se pripravlja.",
"archives.action.preparing": "Pripravljanje",
+ "archives.delete.confirm": "Trajno izbriši",
+ "archives.delete.deleting": "Brisanje",
+ "archives.delete.description": "S tem boste trajno izbrisali arhivirane dnevnike delovnih tokov in datoteke za prenos vseh aplikacij za ta mesec. Nato jih ne bo več mogoče prenesti ali obnoviti. Najprej prenesite varnostno kopijo.",
+ "archives.delete.failed": "Brisanje ni dokončano. Poskusite znova, da izbrišete preostale arhivirane podatke.",
+ "archives.delete.requestFailed": "Brisanja ni bilo mogoče začeti. Poskusite znova.",
+ "archives.delete.retry": "Ponovi brisanje",
+ "archives.delete.title": "Izbrišem arhivirane dnevnike za {{month}}?",
"archives.description": "Prenesite mesečne izvoze CSV arhiviranih dnevnikov zagonov workflow v tem delovnem prostoru.",
"archives.downloadHint.failed": "Priprava tega arhiva ni uspela. Poskusite znova, da zaženete novo opravilo prenosa.",
"archives.downloadHint.failedWithReason": "Priprava tega arhiva ni uspela: {{reason}}",
diff --git a/web/i18n/locales/th-TH/app-log.json b/web/i18n/locales/th-TH/app-log.json
index d00bca645da..6b40dfcd479 100644
--- a/web/i18n/locales/th-TH/app-log.json
+++ b/web/i18n/locales/th-TH/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "เตรียมดาวน์โหลดไฟล์เก็บถาวรไม่สำเร็จ",
"archives.action.prepareStarted": "กำลังเตรียมดาวน์โหลดไฟล์เก็บถาวร",
"archives.action.preparing": "กำลังเตรียม",
+ "archives.delete.confirm": "ลบอย่างถาวร",
+ "archives.delete.deleting": "กำลังลบ",
+ "archives.delete.description": "การดำเนินการนี้จะลบบันทึกเวิร์กโฟลว์ที่เก็บถาวรและไฟล์ดาวน์โหลดของทุกแอปในเดือนนี้อย่างถาวร หลังจากลบแล้วจะไม่สามารถดาวน์โหลดหรือกู้คืนได้ โปรดดาวน์โหลดข้อมูลสำรองก่อน",
+ "archives.delete.failed": "การลบยังไม่เสร็จสมบูรณ์ ลองอีกครั้งเพื่อลบข้อมูลที่เก็บถาวรที่เหลือ",
+ "archives.delete.requestFailed": "ไม่สามารถเริ่มการลบได้ โปรดลองอีกครั้ง",
+ "archives.delete.retry": "ลองลบอีกครั้ง",
+ "archives.delete.title": "ลบบันทึกที่เก็บถาวรของ {{month}} หรือไม่?",
"archives.description": "ดาวน์โหลดไฟล์ส่งออก CSV รายเดือนของบันทึกการรัน workflow ที่เก็บถาวรในเวิร์กสเปซนี้",
"archives.downloadHint.failed": "เตรียมไฟล์เก็บถาวรนี้ไม่สำเร็จ ลองอีกครั้งเพื่อเริ่มงานดาวน์โหลดใหม่",
"archives.downloadHint.failedWithReason": "เตรียมไฟล์เก็บถาวรนี้ไม่สำเร็จ: {{reason}}",
diff --git a/web/i18n/locales/tr-TR/app-log.json b/web/i18n/locales/tr-TR/app-log.json
index 3571e1e4742..3ffbd440900 100644
--- a/web/i18n/locales/tr-TR/app-log.json
+++ b/web/i18n/locales/tr-TR/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Arşiv indirmesi hazırlanamadı.",
"archives.action.prepareStarted": "Arşiv indirmesi hazırlanıyor.",
"archives.action.preparing": "Hazırlanıyor",
+ "archives.delete.confirm": "Kalıcı olarak sil",
+ "archives.delete.deleting": "Siliniyor",
+ "archives.delete.description": "Bu işlem, bu ay tüm uygulamaların arşivlenmiş iş akışı günlüklerini ve indirme dosyalarını kalıcı olarak silecektir. Daha sonra bunları indiremez veya geri yükleyemezsiniz. Önce bir yedek indirin.",
+ "archives.delete.failed": "Silme tamamlanmadı. Kalan arşiv verilerini silmek için yeniden deneyin.",
+ "archives.delete.requestFailed": "Silme başlatılamadı. Lütfen yeniden deneyin.",
+ "archives.delete.retry": "Silmeyi yeniden dene",
+ "archives.delete.title": "{{month}} için arşivlenmiş günlükler silinsin mi?",
"archives.description": "Bu çalışma alanındaki arşivlenmiş workflow çalıştırma günlüklerinin aylık CSV dışa aktarımlarını indirin.",
"archives.downloadHint.failed": "Bu arşiv hazırlanamadı. Yeni bir indirme görevi başlatmak için tekrar deneyin.",
"archives.downloadHint.failedWithReason": "Bu arşiv hazırlanamadı: {{reason}}",
diff --git a/web/i18n/locales/uk-UA/app-log.json b/web/i18n/locales/uk-UA/app-log.json
index 109ca727615..6f156bf0357 100644
--- a/web/i18n/locales/uk-UA/app-log.json
+++ b/web/i18n/locales/uk-UA/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Не вдалося підготувати завантаження архіву.",
"archives.action.prepareStarted": "Завантаження архіву готується.",
"archives.action.preparing": "Підготовка",
+ "archives.delete.confirm": "Видалити назавжди",
+ "archives.delete.deleting": "Видалення",
+ "archives.delete.description": "Архівні журнали робочих процесів і файли для завантаження всіх застосунків за цей місяць буде видалено назавжди. Після видалення їх неможливо буде завантажити або відновити. Спочатку завантажте резервну копію.",
+ "archives.delete.failed": "Видалення не завершено. Спробуйте ще раз, щоб видалити решту архівних даних.",
+ "archives.delete.requestFailed": "Не вдалося розпочати видалення. Спробуйте ще раз.",
+ "archives.delete.retry": "Повторити видалення",
+ "archives.delete.title": "Видалити архівні журнали за {{month}}?",
"archives.description": "Завантажуйте щомісячні CSV-експорти архівних логів запусків workflow у цьому робочому просторі.",
"archives.downloadHint.failed": "Не вдалося підготувати цей архів. Спробуйте ще раз, щоб запустити нове завдання завантаження.",
"archives.downloadHint.failedWithReason": "Не вдалося підготувати цей архів: {{reason}}",
diff --git a/web/i18n/locales/vi-VN/app-log.json b/web/i18n/locales/vi-VN/app-log.json
index e5b89761ab6..f538f3e73c2 100644
--- a/web/i18n/locales/vi-VN/app-log.json
+++ b/web/i18n/locales/vi-VN/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "Không thể chuẩn bị bản lưu trữ để tải xuống.",
"archives.action.prepareStarted": "Đang chuẩn bị bản lưu trữ để tải xuống.",
"archives.action.preparing": "Đang chuẩn bị",
+ "archives.delete.confirm": "Xóa vĩnh viễn",
+ "archives.delete.deleting": "Đang xóa",
+ "archives.delete.description": "Thao tác này sẽ xóa vĩnh viễn nhật ký quy trình đã lưu trữ và các tệp tải xuống của tất cả ứng dụng trong tháng này. Bạn sẽ không thể tải xuống hoặc khôi phục chúng. Hãy tải bản sao lưu xuống trước.",
+ "archives.delete.failed": "Việc xóa chưa hoàn tất. Hãy thử lại để xóa dữ liệu lưu trữ còn lại.",
+ "archives.delete.requestFailed": "Không thể bắt đầu xóa. Vui lòng thử lại.",
+ "archives.delete.retry": "Thử xóa lại",
+ "archives.delete.title": "Xóa nhật ký đã lưu trữ của {{month}}?",
"archives.description": "Tải xuống các bản xuất CSV hằng tháng của log workflow run đã lưu trữ trong workspace này.",
"archives.downloadHint.failed": "Không thể chuẩn bị bản lưu trữ này. Thử lại để bắt đầu một tác vụ tải xuống mới.",
"archives.downloadHint.failedWithReason": "Không thể chuẩn bị bản lưu trữ này: {{reason}}",
diff --git a/web/i18n/locales/zh-Hans/app-log.json b/web/i18n/locales/zh-Hans/app-log.json
index 289c699259c..49ffbb2a479 100644
--- a/web/i18n/locales/zh-Hans/app-log.json
+++ b/web/i18n/locales/zh-Hans/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "归档下载准备失败。",
"archives.action.prepareStarted": "正在准备归档下载。",
"archives.action.preparing": "准备中",
+ "archives.delete.confirm": "永久删除",
+ "archives.delete.deleting": "删除中",
+ "archives.delete.description": "这将永久删除该月所有应用的已归档工作流日志及下载文件。删除后无法下载或恢复,建议先下载备份。",
+ "archives.delete.failed": "删除未完成,请重试以删除剩余归档数据。",
+ "archives.delete.requestFailed": "无法开始删除,请重试。",
+ "archives.delete.retry": "重试删除",
+ "archives.delete.title": "删除 {{month}} 的归档日志?",
"archives.description": "下载此工作空间中已归档工作流运行日志的月度 CSV 导出。",
"archives.downloadHint.failed": "此归档准备失败。重试将启动新的下载任务。",
"archives.downloadHint.failedWithReason": "此归档准备失败:{{reason}}",
diff --git a/web/i18n/locales/zh-Hant/app-log.json b/web/i18n/locales/zh-Hant/app-log.json
index 66d814778ff..5d60ca534a1 100644
--- a/web/i18n/locales/zh-Hant/app-log.json
+++ b/web/i18n/locales/zh-Hant/app-log.json
@@ -9,6 +9,13 @@
"archives.action.prepareFailed": "封存下載準備失敗。",
"archives.action.prepareStarted": "正在準備封存下載。",
"archives.action.preparing": "準備中",
+ "archives.delete.confirm": "永久刪除",
+ "archives.delete.deleting": "刪除中",
+ "archives.delete.description": "這將永久刪除該月所有應用程式的已封存工作流程日誌及下載檔案。刪除後無法下載或還原,建議先下載備份。",
+ "archives.delete.failed": "刪除尚未完成,請重試以刪除剩餘封存資料。",
+ "archives.delete.requestFailed": "無法開始刪除,請重試。",
+ "archives.delete.retry": "重試刪除",
+ "archives.delete.title": "刪除 {{month}} 的封存日誌?",
"archives.description": "下載此工作區中已封存工作流程執行記錄的每月 CSV 匯出。",
"archives.downloadHint.failed": "此封存準備失敗。重試將啟動新的下載任務。",
"archives.downloadHint.failedWithReason": "此封存準備失敗:{{reason}}",
diff --git a/web/i18n/resources.generated.d.ts b/web/i18n/resources.generated.d.ts
index 4a05e0f7540..56bf58d9086 100644
--- a/web/i18n/resources.generated.d.ts
+++ b/web/i18n/resources.generated.d.ts
@@ -1238,6 +1238,13 @@ export default interface Resources {
'archives.action.prepareFailed': 'Failed to prepare archive download.'
'archives.action.prepareStarted': 'Archive download is being prepared.'
'archives.action.preparing': 'Preparing'
+ 'archives.delete.confirm': 'Permanently delete'
+ 'archives.delete.deleting': 'Deleting'
+ 'archives.delete.description': 'This will permanently delete archived workflow logs and download files for all apps in this month. You will no longer be able to download or restore them. Download a backup first.'
+ 'archives.delete.failed': 'Deletion is incomplete. Retry to delete the remaining archived data.'
+ 'archives.delete.requestFailed': 'Could not start deletion. Please try again.'
+ 'archives.delete.retry': 'Retry deletion'
+ 'archives.delete.title': 'Delete archived logs for {{month}}?'
'archives.description': 'Download monthly CSV exports of archived workflow run logs across this workspace.'
'archives.downloadHint.failed': 'Preparing this archive failed. Retry to start a new download task.'
'archives.downloadHint.failedWithReason': 'Preparing this archive failed: {{reason}}'
diff --git a/web/service/console/query-policies.ts b/web/service/console/query-policies.ts
index dadc6b019d6..dfb33482312 100644
--- a/web/service/console/query-policies.ts
+++ b/web/service/console/query-policies.ts
@@ -5,6 +5,7 @@ import type {
import type { ApiBasedExtensionResponse } from '@dify/contracts/api/console/api-based-extension/types.gen'
import type { AppDetailWithSite } from '@dify/contracts/api/console/apps/types.gen'
import type { TagResponse as Tag } from '@dify/contracts/api/console/tags/types.gen'
+import type { GetWorkflowRunArchivesResponse } from '@dify/contracts/api/console/workflow-run-archives/types.gen'
import type { RouterUtils } from '@orpc/tanstack-query'
import type { InfiniteData, QueryClient, QueryKey } from '@tanstack/react-query'
import type { ConsoleClient } from './index'
@@ -15,6 +16,65 @@ export function createConsoleQuery(consoleClient: ConsoleClient) {
const consoleQuery: RouterUtils<ConsoleClient> = createTanstackQueryUtils(consoleClient, {
path: ['console'],
experimental_defaults: {
+ workflowRunArchives: {
+ byYear: {
+ byMonth: {
+ delete: {
+ mutationOptions: {
+ onSuccess: (task, _variables, _onMutateResult, context) => {
+ const queryKey = consoleQuery.workflowRunArchives.get.queryKey()
+ const deletionStatus = task.status
+ context.client.setQueryData<GetWorkflowRunArchivesResponse>(queryKey, (data) => {
+ if (!data) return data
+
+ if (deletionStatus === 'completed') {
+ const months = data.months.filter(
+ (archive) => archive.year !== task.year || archive.month !== task.month,
+ )
+ return {
+ months,
+ summary: {
+ archived_month_count: months.length,
+ workflow_run_count: months.reduce(
+ (count, archive) => count + archive.workflow_run_count,
+ 0,
+ ),
+ archive_bytes: months.reduce(
+ (bytes, archive) => bytes + archive.archive_bytes,
+ 0,
+ ),
+ latest_archived_at: months.reduce<string | null>(
+ (latest, archive) =>
+ archive.latest_archived_at &&
+ (!latest || archive.latest_archived_at > latest)
+ ? archive.latest_archived_at
+ : latest,
+ null,
+ ),
+ },
+ }
+ }
+
+ return {
+ ...data,
+ months: data.months.map((archive) =>
+ archive.year === task.year && archive.month === task.month
+ ? { ...archive, deletion_status: deletionStatus }
+ : archive,
+ ),
+ }
+ })
+ },
+ onSettled: (_data, _error, _variables, _onMutateResult, context) => {
+ void context.client.invalidateQueries({
+ queryKey: consoleQuery.workflowRunArchives.get.queryKey(),
+ })
+ },
+ },
+ },
+ },
+ },
+ },
rag: {
pipelines: {
datasourcePlugins: {