Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(api): extract workflow run archive application service (#41253)
非法操作a9319c86ee94
diff --git a/api/.importlinter b/api/.importlinterindex 3e0e106a1d3..f3609e5826a 100644--- a/api/.importlinter+++ b/api/.importlinter@@ -459,3 +459,20 @@ forbidden_modules =services.billing_servicesqlalchemywerkzeug++[importlinter:contract:workflow-run-archive-application-boundary]+name = Workflow run archive application service and task domain are framework and persistence neutral+type = forbidden+source_modules =+ services.retention.workflow_run.archive_download_task+ services.retention.workflow_run.archive_log_service+forbidden_modules =+ configs+ controllers+ extensions+ flask+ models+ repositories+ sqlalchemy+ tasks+ werkzeugdiff --git a/api/controllers/console/workflow_run_archive.py b/api/controllers/console/workflow_run_archive.pyindex 170d5268a6f..88b85fd1091 100644--- a/api/controllers/console/workflow_run_archive.py+++ b/api/controllers/console/workflow_run_archive.py@@ -4,38 +4,33 @@ from http import HTTPStatusfrom flask import redirectfrom flask_restx import Resourcefrom pydantic import BaseModel, Field-from werkzeug.exceptions import Conflict, Forbidden, NotFound+from werkzeug.exceptions import Conflict, NotFoundfrom controllers.common.fields import RedirectResponsefrom controllers.common.schema import register_response_schema_models, register_schema_modelsfrom controllers.console import console_ns+from controllers.console.flask_admission import console_account_admissionfrom controllers.console.wraps import (- account_initialization_required,cloud_edition_billing_paid_plan_required,model_validate,- only_edition_cloud,- setup_required,)-from extensions.ext_database import db+from enums import DeploymentEdition+from extensions.ext_application_services import application_servicesfrom fields.base import ResponseModel-from libs.archive_storage import get_export_storagefrom libs.helper import dump_response-from libs.login import current_account_with_tenant, login_required-from models import TenantAccountRole-from services.retention.workflow_run.archive_download_preparation import ARCHIVE_DOWNLOAD_MIME_TYPE-from services.retention.workflow_run.archive_download_task_cache import (+from machinery.context import RequestContext+from models.account import TenantAccountRole+from services.retention.workflow_run.archive_download_task import (WorkflowRunArchiveDownloadStatus,)from services.retention.workflow_run.archive_log_service import (WorkflowRunArchiveDownloadNotReadyError,WorkflowRunArchiveDownloadTaskNotFoundError,WorkflowRunArchiveNotFoundError,- create_workflow_run_archive_download_task,- get_ready_workflow_run_archive_download_task,- get_workflow_run_archive_download_task,- list_workflow_run_archives,)+_WORKFLOW_RUN_ARCHIVE_ALLOWED_ROLES = frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN})+class WorkflowRunArchiveDownloadPayload(BaseModel):"""Request body for preparing one monthly workflow-run archive download."""@@ -95,36 +90,19 @@ register_response_schema_models()-def _current_owner_or_admin_ids() -> tuple[str, str]:- """Return current Cloud workspace IDs for an owner or admin, independently of enterprise RBAC."""- current_user, current_tenant_id = current_account_with_tenant()- if not current_tenant_id:- raise NotFound("Current workspace not found")- if not TenantAccountRole.is_privileged_role(current_user.current_role):- raise Forbidden()- return current_tenant_id, current_user.id---def _presigned_url_expires_in(expires_at: datetime.datetime) -> int:- """Keep the storage URL no longer-lived than the Redis task and cap it for browser downloads."""- expires_at_utc = expires_at if expires_at.tzinfo else expires_at.replace(tzinfo=datetime.UTC)- remaining_seconds = int((expires_at_utc - datetime.datetime.now(datetime.UTC)).total_seconds())- return max(1, min(3600, remaining_seconds))--@console_ns.route("/workflow-run-archives")class WorkflowRunArchivesApi(Resource):@console_ns.doc("list_workflow_run_archives")@console_ns.doc(description="List monthly workflow-run archive metadata for the current workspace")@console_ns.response(200, "Success", console_ns.models[WorkflowRunArchiveListResponse.__name__])- @setup_required- @login_required- @account_initialization_required- @only_edition_cloud+ @console_account_admission(+ editions=frozenset({DeploymentEdition.CLOUD}),+ allowed_roles=_WORKFLOW_RUN_ARCHIVE_ALLOWED_ROLES,+ )@cloud_edition_billing_paid_plan_required- def get(self):- tenant_id, _ = _current_owner_or_admin_ids()- return dump_response(WorkflowRunArchiveListResponse, list_workflow_run_archives(db.session(), tenant_id))+ def get(self, request_context: RequestContext):+ archives = application_services().workflow_run_archives.list_archives(request_context)+ return dump_response(WorkflowRunArchiveListResponse, archives)@console_ns.route("/workflow-run-archives/downloads")@@ -137,19 +115,16 @@ class WorkflowRunArchiveDownloadsApi(Resource):"Download task accepted",console_ns.models[WorkflowRunArchiveDownloadTaskResponse.__name__],)- @setup_required- @login_required- @account_initialization_required- @only_edition_cloud+ @console_account_admission(+ editions=frozenset({DeploymentEdition.CLOUD}),+ allowed_roles=_WORKFLOW_RUN_ARCHIVE_ALLOWED_ROLES,+ )@cloud_edition_billing_paid_plan_required@model_validate(WorkflowRunArchiveDownloadPayload)- def post(self, req_data: WorkflowRunArchiveDownloadPayload):- tenant_id, account_id = _current_owner_or_admin_ids()+ def post(self, req_data: WorkflowRunArchiveDownloadPayload, request_context: RequestContext):try:- task = create_workflow_run_archive_download_task(- db.session(),- tenant_id=tenant_id,- requested_by=account_id,+ task = application_services().workflow_run_archives.create_download(+ request_context,year=req_data.year,month=req_data.month,)@@ -163,15 +138,17 @@ class WorkflowRunArchiveDownloadApi(Resource):@console_ns.doc("get_workflow_run_archive_download")@console_ns.doc(description="Get a temporary workflow-run archive download task")@console_ns.response(200, "Success", console_ns.models[WorkflowRunArchiveDownloadTaskResponse.__name__])- @setup_required- @login_required- @account_initialization_required- @only_edition_cloud+ @console_account_admission(+ editions=frozenset({DeploymentEdition.CLOUD}),+ allowed_roles=_WORKFLOW_RUN_ARCHIVE_ALLOWED_ROLES,+ )@cloud_edition_billing_paid_plan_required- def get(self, download_id: str):- tenant_id, _ = _current_owner_or_admin_ids()+ def get(self, request_context: RequestContext, download_id: str):try:- task = get_workflow_run_archive_download_task(tenant_id=tenant_id, download_id=download_id)+ task = application_services().workflow_run_archives.get_download(+ request_context,+ download_id=download_id,+ )except WorkflowRunArchiveDownloadTaskNotFoundError as exc:raise NotFound(str(exc)) from excreturn dump_response(WorkflowRunArchiveDownloadTaskResponse, task)@@ -187,29 +164,19 @@ class WorkflowRunArchiveDownloadFileApi(Resource):console_ns.models[RedirectResponse.__name__],)@console_ns.response(409, "Download task is not ready")- @setup_required- @login_required- @account_initialization_required- @only_edition_cloud+ @console_account_admission(+ editions=frozenset({DeploymentEdition.CLOUD}),+ allowed_roles=_WORKFLOW_RUN_ARCHIVE_ALLOWED_ROLES,+ )@cloud_edition_billing_paid_plan_required- def get(self, download_id: str):- tenant_id, _ = _current_owner_or_admin_ids()+ def get(self, request_context: RequestContext, download_id: str):try:- task = get_ready_workflow_run_archive_download_task(tenant_id=tenant_id, download_id=download_id)+ presigned_url = application_services().workflow_run_archives.get_download_url(+ request_context,+ download_id=download_id,+ )except WorkflowRunArchiveDownloadTaskNotFoundError as exc:raise NotFound(str(exc)) from excexcept WorkflowRunArchiveDownloadNotReadyError as exc:raise Conflict(str(exc)) from exc-- storage_key = task.storage_key- if storage_key is None:- raise Conflict(f"Workflow run archive download is not ready: {download_id}")-- storage = get_export_storage()- presigned_url = storage.generate_presigned_url(- storage_key,- expires_in=_presigned_url_expires_in(task.expires_at),- filename=task.file_name,- content_type=ARCHIVE_DOWNLOAD_MIME_TYPE,- )return redirect(presigned_url, code=HTTPStatus.FOUND)diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.pyindex d015bd651bb..42927654166 100644--- a/api/extensions/ext_application_services.py+++ b/api/extensions/ext_application_services.py@@ -35,6 +35,7 @@ from repositories.tag_repository import TagRepositoryfrom repositories.trial_app_query_repository import TrialAppQueryRepositoryfrom repositories.trial_app_usage_repository import TrialAppUsageRepositoryfrom repositories.webapp_access_query_repository import WebAppAccessQueryRepository+from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepositoryfrom repositories.workspace_member_query_repository import WorkspaceMemberQueryRepositoryfrom repositories.workspace_query_repository import WorkspaceQueryRepositoryfrom services.account_activation_adapters import (@@ -101,6 +102,12 @@ from services.recommended_app_catalog_gateway import (RemoteRecommendedAppCatalogGateway,)from services.recommended_app_query_service import RecommendedAppQueryService+from services.retention.workflow_run.archive_download_adapters import (+ dispatch_workflow_run_archive_download_task,+ sign_workflow_run_archive_download_url,+)+from services.retention.workflow_run.archive_download_task_cache import WorkflowRunArchiveDownloadTaskCache+from services.retention.workflow_run.archive_log_service import WorkflowRunArchiveServicefrom services.schema_definition_service import SchemaDefinitionServicefrom services.setup_adapters import RedisSetupLock, RegisterServiceAccountProvisionerfrom services.setup_service import SetupService@@ -172,6 +179,7 @@ class ApplicationServices:partner_tenant_bindings: PartnerTenantBindingServicerecommended_app_queries: RecommendedAppQueryServicetrial_app_usage: TrialAppUsageRecorder+ workflow_run_archives: WorkflowRunArchiveServiceworkspace_queries: WorkspaceQueryServiceworkspace_member_queries: WorkspaceMemberQueryServicetags: TagApplicationService@@ -397,6 +405,12 @@ def build_application_services(trial_enabled=trial_app_enabled,),trial_app_usage=TrialAppUsageRepository(session_factory=database_client),+ workflow_run_archives=WorkflowRunArchiveService(+ bundles=WorkflowRunArchiveBundleQueryRepository(session_factory=database_client),+ tasks=WorkflowRunArchiveDownloadTaskCache(redis=redis),+ dispatcher=dispatch_workflow_run_archive_download_task,+ sign_download_url=sign_workflow_run_archive_download_url,+ ),workspace_queries=WorkspaceQueryService(workspaces=workspace_query_repository,plans=DeploymentWorkspacePlanGateway(),diff --git a/api/repositories/workflow_run_archive_repository.py b/api/repositories/workflow_run_archive_repository.pynew file mode 100644index 00000000000..c8296bc59ed--- /dev/null+++ b/api/repositories/workflow_run_archive_repository.py@@ -0,0 +1,98 @@+"""Database repository for workflow-run archive bundle queries."""++from typing import override++from sqlalchemy import select+from sqlalchemy.orm import Session, sessionmaker++from models.workflow import WorkflowRunArchiveBundle+from services.retention.workflow_run.archive_log_service import (+ WorkflowRunArchiveBundleQuery,+ WorkflowRunArchiveBundleRecord,+)+++class WorkflowRunArchiveBundleQueryRepository(WorkflowRunArchiveBundleQuery):+ def __init__(self, *, session_factory: sessionmaker[Session]) -> None:+ self._session_factory = session_factory++ @override+ def list_for_tenant(self, tenant_id: str) -> tuple[WorkflowRunArchiveBundleRecord, ...]:+ stmt = (+ select(+ WorkflowRunArchiveBundle.year,+ WorkflowRunArchiveBundle.month,+ WorkflowRunArchiveBundle.shard,+ WorkflowRunArchiveBundle.bundle_id,+ WorkflowRunArchiveBundle.workflow_run_count,+ WorkflowRunArchiveBundle.row_count,+ WorkflowRunArchiveBundle.archive_bytes,+ WorkflowRunArchiveBundle.archived_at,+ )+ .where(WorkflowRunArchiveBundle.tenant_id == tenant_id)+ .order_by(+ WorkflowRunArchiveBundle.year.desc(),+ WorkflowRunArchiveBundle.month.desc(),+ WorkflowRunArchiveBundle.shard,+ WorkflowRunArchiveBundle.bundle_id,+ )+ )+ with self._session_factory() as session:+ rows = session.execute(stmt).all()++ return tuple(+ WorkflowRunArchiveBundleRecord(+ year=row.year,+ month=row.month,+ shard=row.shard,+ bundle_id=row.bundle_id,+ workflow_run_count=row.workflow_run_count,+ row_count=row.row_count,+ archive_bytes=row.archive_bytes,+ archived_at=row.archived_at,+ )+ for row in rows+ )++ @override+ def list_for_tenant_month(+ self,+ tenant_id: str,+ *,+ year: int,+ month: int,+ ) -> tuple[WorkflowRunArchiveBundleRecord, ...]:+ stmt = (+ select(+ WorkflowRunArchiveBundle.year,+ WorkflowRunArchiveBundle.month,+ WorkflowRunArchiveBundle.shard,+ WorkflowRunArchiveBundle.bundle_id,+ WorkflowRunArchiveBundle.workflow_run_count,+ WorkflowRunArchiveBundle.row_count,+ WorkflowRunArchiveBundle.archive_bytes,+ WorkflowRunArchiveBundle.archived_at,+ )+ .where(+ WorkflowRunArchiveBundle.tenant_id == tenant_id,+ WorkflowRunArchiveBundle.year == year,+ WorkflowRunArchiveBundle.month == month,+ )+ .order_by(WorkflowRunArchiveBundle.shard, WorkflowRunArchiveBundle.bundle_id)+ )+ with self._session_factory() as session:+ rows = session.execute(stmt).all()++ return tuple(+ WorkflowRunArchiveBundleRecord(+ year=row.year,+ month=row.month,+ shard=row.shard,+ bundle_id=row.bundle_id,+ workflow_run_count=row.workflow_run_count,+ row_count=row.row_count,+ archive_bytes=row.archive_bytes,+ archived_at=row.archived_at,+ )+ for row in rows+ )diff --git a/api/services/retention/workflow_run/archive_download_adapters.py b/api/services/retention/workflow_run/archive_download_adapters.pynew file mode 100644index 00000000000..2586bb6d4cc--- /dev/null+++ b/api/services/retention/workflow_run/archive_download_adapters.py@@ -0,0 +1,37 @@+"""Infrastructure adapters used by the workflow-run archive application service."""++from libs.archive_storage import get_export_storage+from services.retention.workflow_run.archive_download_task import WorkflowRunArchiveDownloadTask++_ARCHIVE_DOWNLOAD_MIME_TYPE = "application/zip"+++def dispatch_workflow_run_archive_download_task(+ task: WorkflowRunArchiveDownloadTask,+) -> None:+ """Enqueue ZIP preparation after the application service claims the Celery task id."""+ from tasks.workflow_run_archive_download_tasks import prepare_workflow_run_archive_download_task++ celery_task_id = task.celery_task_id+ if celery_task_id is None:+ raise ValueError("celery_task_id is required before dispatch")++ prepare_workflow_run_archive_download_task.apply_async(+ args=(task.tenant_id, task.download_id),+ task_id=celery_task_id,+ )+++def sign_workflow_run_archive_download_url(+ storage_key: str,+ *,+ expires_in: int,+ filename: str,+) -> str:+ """Create a browser download URL without initializing export storage during app startup."""+ return get_export_storage().generate_presigned_url(+ storage_key,+ expires_in=expires_in,+ filename=filename,+ content_type=_ARCHIVE_DOWNLOAD_MIME_TYPE,+ )diff --git a/api/services/retention/workflow_run/archive_download_preparation.py b/api/services/retention/workflow_run/archive_download_preparation.pyindex cb9200d23da..69aaf002351 100644--- a/api/services/retention/workflow_run/archive_download_preparation.py+++ b/api/services/retention/workflow_run/archive_download_preparation.py@@ -12,31 +12,31 @@ import ioimport loggingimport reimport zipfile-from collections.abc import Sequence+from collections.abc import Callable, Sequenceimport pyarrow as paimport pyarrow.compute as pcimport pyarrow.csv as pa_csvimport pyarrow.parquet as pq-from sqlalchemy import select-from sqlalchemy.orm import Session, sessionmakerfrom core.helper.csv_sanitizer import CSVSanitizer-from extensions.ext_database import db-from libs.archive_storage import ArchiveStorage, get_archive_storage, get_export_storage-from models.workflow import WorkflowRunArchiveBundle+from libs.archive_storage import ArchiveStoragefrom services.retention.workflow_run.archive_bundle_index import (ARCHIVE_BUNDLE_ROOT_PREFIX,ArchiveBundleManifest,ArchiveBundleTableManifestEntry,decode_archive_bundle_manifest,)-from services.retention.workflow_run.archive_download_task_cache import (+from services.retention.workflow_run.archive_download_task import (WorkflowRunArchiveDownloadStatus,WorkflowRunArchiveDownloadTask,- WorkflowRunArchiveDownloadTaskCache,build_archive_download_id,)+from services.retention.workflow_run.archive_log_service import (+ WorkflowRunArchiveBundleQuery,+ WorkflowRunArchiveBundleRecord,+ WorkflowRunArchiveDownloadTaskStore,+)from services.retention.workflow_run.constants import (ARCHIVE_BUNDLE_FORMAT,ARCHIVE_BUNDLE_MANIFEST_NAME,@@ -46,7 +46,6 @@ from services.retention.workflow_run.constants import (logger = logging.getLogger(__name__)ARCHIVE_DOWNLOAD_ROOT_PREFIX = "workflow-runs/downloads/v1/"-ARCHIVE_DOWNLOAD_MIME_TYPE = "application/zip"_CSV_FORMULA_PREFIX_PATTERN = f"^[{re.escape(''.join(CSVSanitizer.FORMULA_CHARS))}]"@@ -60,24 +59,23 @@ class WorkflowRunArchiveDownloadPreparer:without touching long-lived archives."""- archive_storage: ArchiveStorage | None- download_storage: ArchiveStorage | None- cache: WorkflowRunArchiveDownloadTaskCache- session_factory: sessionmaker[Session]+ _archive_storage_provider: Callable[[], ArchiveStorage]+ _download_storage_provider: Callable[[], ArchiveStorage]+ cache: WorkflowRunArchiveDownloadTaskStore+ _bundles: WorkflowRunArchiveBundleQuerydef __init__(self,*,- storage: ArchiveStorage | None = None,- archive_storage: ArchiveStorage | None = None,- download_storage: ArchiveStorage | None = None,- cache: WorkflowRunArchiveDownloadTaskCache | None = None,- session_factory: sessionmaker[Session] | None = None,+ archive_storage_provider: Callable[[], ArchiveStorage],+ download_storage_provider: Callable[[], ArchiveStorage],+ bundles: WorkflowRunArchiveBundleQuery,+ cache: WorkflowRunArchiveDownloadTaskStore,) -> None:- self.archive_storage = archive_storage or storage- self.download_storage = download_storage or storage- self.cache = cache or WorkflowRunArchiveDownloadTaskCache()- self.session_factory = session_factory or sessionmaker(bind=db.engine, expire_on_commit=False)+ self._archive_storage_provider = archive_storage_provider+ self._download_storage_provider = download_storage_provider+ self._bundles = bundles+ self.cache = cachedef prepare(self, *, tenant_id: str, download_id: str) -> WorkflowRunArchiveDownloadTask | None:"""Prepare a ZIP for an existing Redis task and persist terminal task state."""@@ -91,8 +89,8 @@ class WorkflowRunArchiveDownloadPreparer:return taskprocessing_task = self._mark_processing(task)try:- archive_storage = self.archive_storage or get_archive_storage()- download_storage = self.download_storage or get_export_storage()+ archive_storage = self._archive_storage_provider()+ download_storage = self._download_storage_provider()bundles = self._get_task_bundles(processing_task)payload = self._build_zip_payload(archive_storage, processing_task, bundles)storage_key = build_archive_download_storage_key(processing_task)@@ -102,15 +100,19 @@ class WorkflowRunArchiveDownloadPreparer:logger.exception("Failed to prepare workflow run archive download %s", download_id)return self._mark_failed(processing_task, error=str(exc))- def _get_task_bundles(self, task: WorkflowRunArchiveDownloadTask) -> list[WorkflowRunArchiveBundle]:- with self.session_factory() as session:- return _list_task_bundles(session, task)+ def _get_task_bundles(self, task: WorkflowRunArchiveDownloadTask) -> list[WorkflowRunArchiveBundleRecord]:+ indexed_bundles = self._bundles.list_for_tenant_month(+ task.tenant_id,+ year=task.year,+ month=task.month,+ )+ return _select_task_bundles(indexed_bundles, task)def _build_zip_payload(self,storage: ArchiveStorage,task: WorkflowRunArchiveDownloadTask,- bundles: Sequence[WorkflowRunArchiveBundle],+ bundles: Sequence[WorkflowRunArchiveBundleRecord],) -> bytes:zip_root = f"workflow-run-logs-{task.year:04d}-{task.month:02d}"csv_buffers: dict[str, io.BytesIO] = {}@@ -218,17 +220,10 @@ def build_archive_download_storage_key(task: WorkflowRunArchiveDownloadTask) ->)-def _list_task_bundles(session: Session, task: WorkflowRunArchiveDownloadTask) -> list[WorkflowRunArchiveBundle]:- stmt = (- select(WorkflowRunArchiveBundle)- .where(- WorkflowRunArchiveBundle.tenant_id == task.tenant_id,- WorkflowRunArchiveBundle.year == task.year,- WorkflowRunArchiveBundle.month == task.month,- )- .order_by(WorkflowRunArchiveBundle.shard, WorkflowRunArchiveBundle.bundle_id)- )- indexed_bundles = list(session.scalars(stmt))+def _select_task_bundles(+ indexed_bundles: Sequence[WorkflowRunArchiveBundleRecord],+ task: WorkflowRunArchiveDownloadTask,+) -> list[WorkflowRunArchiveBundleRecord]:if task.bundle_refs:requested_refs = [(ref.shard, ref.bundle_id) for ref in task.bundle_refs]else:@@ -260,7 +255,7 @@ def _list_task_bundles(session: Session, task: WorkflowRunArchiveDownloadTask) -def _build_archive_bundle_object_prefix(task: WorkflowRunArchiveDownloadTask,- bundle: WorkflowRunArchiveBundle,+ bundle: WorkflowRunArchiveBundleRecord,) -> str:return (f"{ARCHIVE_BUNDLE_ROOT_PREFIX}tenant_prefix={task.tenant_id[0].lower()}/tenant_id={task.tenant_id}/"@@ -271,7 +266,7 @@ def _build_archive_bundle_object_prefix(def _load_and_validate_manifest(storage: ArchiveStorage,task: WorkflowRunArchiveDownloadTask,- bundle: WorkflowRunArchiveBundle,+ bundle: WorkflowRunArchiveBundleRecord,object_prefix: str,) -> tuple[bytes, ArchiveBundleManifest]:manifest_key = f"{object_prefix}/{ARCHIVE_BUNDLE_MANIFEST_NAME}"@@ -284,7 +279,7 @@ def _load_and_validate_manifest(def _validate_manifest(*,task: WorkflowRunArchiveDownloadTask,- bundle: WorkflowRunArchiveBundle,+ bundle: WorkflowRunArchiveBundleRecord,manifest: ArchiveBundleManifest,object_prefix: str,) -> None:diff --git a/api/services/retention/workflow_run/archive_download_task.py b/api/services/retention/workflow_run/archive_download_task.pynew file mode 100644index 00000000000..a6d569c2387--- /dev/null+++ b/api/services/retention/workflow_run/archive_download_task.py@@ -0,0 +1,121 @@+"""Domain objects and builders for workflow-run archive downloads."""++import datetime+import hashlib+import json+from collections.abc import Sequence+from enum import StrEnum++from pydantic import BaseModel, ConfigDict, Field++ARCHIVE_DOWNLOAD_FORMAT_VERSION = "v1"+DEFAULT_ARCHIVE_DOWNLOAD_TASK_TTL_SECONDS = 24 * 60 * 60+++class WorkflowRunArchiveDownloadStatus(StrEnum):+ """Lifecycle state for an asynchronous archive download request."""++ PENDING = "pending"+ PROCESSING = "processing"+ READY = "ready"+ FAILED = "failed"+++class WorkflowRunArchiveBundleRef(BaseModel):+ """Immutable object-store identity for one bundle included in a download task."""++ model_config = ConfigDict(extra="forbid")++ shard: str+ bundle_id: str+++class WorkflowRunArchiveDownloadTask(BaseModel):+ """Temporary state for a monthly archive download request."""++ model_config = ConfigDict(extra="forbid")++ download_id: str+ tenant_id: str+ requested_by: str+ year: int = Field(ge=1)+ month: int = Field(ge=1, le=12)+ bundle_ids: list[str]+ bundle_refs: list[WorkflowRunArchiveBundleRef] = Field(default_factory=list)+ bundle_count: int = Field(ge=0)+ archive_bytes: int = Field(ge=0)+ status: WorkflowRunArchiveDownloadStatus+ file_name: str | None = None+ storage_key: str | None = None+ file_size_bytes: int | None = Field(default=None, ge=0)+ celery_task_id: str | None = None+ error: str | None = None+ created_at: datetime.datetime+ updated_at: datetime.datetime+ expires_at: datetime.datetime+ started_at: datetime.datetime | None = None+ finished_at: datetime.datetime | None = None+++def build_pending_archive_download_task(+ *,+ tenant_id: str,+ requested_by: str,+ year: int,+ month: int,+ bundle_ids: Sequence[str],+ bundle_refs: Sequence[tuple[str, str]] = (),+ archive_bytes: int,+ download_id: str,+ ttl_seconds: int = DEFAULT_ARCHIVE_DOWNLOAD_TASK_TTL_SECONDS,+ now: datetime.datetime | None = None,+) -> WorkflowRunArchiveDownloadTask:+ """Create the temporary state stored when the console starts an archive download."""+ created_at = now or datetime.datetime.now(datetime.UTC)+ if created_at.tzinfo is None:+ created_at = created_at.replace(tzinfo=datetime.UTC)+ normalized_bundle_ids = list(bundle_ids)+ normalized_bundle_refs = [+ WorkflowRunArchiveBundleRef(shard=shard, bundle_id=bundle_id) for shard, bundle_id in bundle_refs+ ]+ return WorkflowRunArchiveDownloadTask(+ download_id=download_id,+ tenant_id=tenant_id,+ requested_by=requested_by,+ year=year,+ month=month,+ bundle_ids=normalized_bundle_ids,+ bundle_refs=normalized_bundle_refs,+ bundle_count=len(normalized_bundle_ids),+ archive_bytes=archive_bytes,+ status=WorkflowRunArchiveDownloadStatus.PENDING,+ created_at=created_at,+ updated_at=created_at,+ expires_at=created_at + datetime.timedelta(seconds=ttl_seconds),+ )+++def build_archive_download_id(+ *,+ tenant_id: str,+ year: int,+ month: int,+ bundle_refs: Sequence[tuple[str, str]],+ download_format_version: str = ARCHIVE_DOWNLOAD_FORMAT_VERSION,+) -> str:+ """Build a stable id for the exact archive download content."""+ if not bundle_refs:+ raise ValueError("bundle_refs must not be empty")+ normalized_refs = sorted(f"{shard}:{bundle_id}" for shard, bundle_id in bundle_refs)+ payload = json.dumps(+ {+ "tenant_id": tenant_id,+ "year": year,+ "month": month,+ "bundle_refs": normalized_refs,+ "download_format_version": download_format_version,+ },+ sort_keys=True,+ separators=(",", ":"),+ )+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]diff --git a/api/services/retention/workflow_run/archive_download_task_cache.py b/api/services/retention/workflow_run/archive_download_task_cache.pyindex a508ee18804..e26477890a7 100644--- a/api/services/retention/workflow_run/archive_download_task_cache.py+++ b/api/services/retention/workflow_run/archive_download_task_cache.py@@ -1,76 +1,24 @@"""Redis-backed temporary state for workflow-run archive downloads."""import datetime-import hashlib-import jsonimport logging-from collections.abc import Sequence-from enum import StrEnumfrom typing import Any-from pydantic import BaseModel, ConfigDict, Field--from extensions.ext_redis import RedisClientWrapper, redis_client+from extensions.ext_redis import RedisClientWrapper+from services.retention.workflow_run.archive_download_task import WorkflowRunArchiveDownloadTasklogger = logging.getLogger(__name__)-ARCHIVE_DOWNLOAD_FORMAT_VERSION = "v1"-DEFAULT_ARCHIVE_DOWNLOAD_TASK_TTL_SECONDS = 24 * 60 * 60ARCHIVE_DOWNLOAD_TASK_LOCK_TIMEOUT_SECONDS = 30_CACHE_KEY_PREFIX = "workflow_run_archive_download"-class WorkflowRunArchiveDownloadStatus(StrEnum):- """Lifecycle state for an asynchronous archive download request."""-- PENDING = "pending"- PROCESSING = "processing"- READY = "ready"- FAILED = "failed"---class WorkflowRunArchiveBundleRef(BaseModel):- """Immutable object-store identity for one bundle included in a download task."""-- model_config = ConfigDict(extra="forbid")-- shard: str- bundle_id: str---class WorkflowRunArchiveDownloadTask(BaseModel):- """Temporary Redis payload for a monthly archive download request."""-- model_config = ConfigDict(extra="forbid")-- download_id: str- tenant_id: str- requested_by: str- year: int = Field(ge=1)- month: int = Field(ge=1, le=12)- bundle_ids: list[str]- bundle_refs: list[WorkflowRunArchiveBundleRef] = Field(default_factory=list)- bundle_count: int = Field(ge=0)- archive_bytes: int = Field(ge=0)- status: WorkflowRunArchiveDownloadStatus- file_name: str | None = None- storage_key: str | None = None- file_size_bytes: int | None = Field(default=None, ge=0)- celery_task_id: str | None = None- error: str | None = None- created_at: datetime.datetime- updated_at: datetime.datetime- expires_at: datetime.datetime- started_at: datetime.datetime | None = None- finished_at: datetime.datetime | None = None--class WorkflowRunArchiveDownloadTaskCache:"""Store ephemeral archive download task state in Redis with a TTL."""_redis: RedisClientWrapper- def __init__(self, redis: RedisClientWrapper = redis_client) -> None:+ def __init__(self, *, redis: RedisClientWrapper) -> None:self._redis = redisdef get(self, *, tenant_id: str, download_id: str) -> WorkflowRunArchiveDownloadTask | None:@@ -111,67 +59,3 @@ class WorkflowRunArchiveDownloadTaskCache:expires_at_utc = expires_at if expires_at.tzinfo else expires_at.replace(tzinfo=datetime.UTC)remaining = expires_at_utc - datetime.datetime.now(datetime.UTC)return max(int(remaining.total_seconds()), 1)---def build_pending_archive_download_task(- *,- tenant_id: str,- requested_by: str,- year: int,- month: int,- bundle_ids: Sequence[str],- bundle_refs: Sequence[tuple[str, str]] = (),- archive_bytes: int,- download_id: str,- ttl_seconds: int = DEFAULT_ARCHIVE_DOWNLOAD_TASK_TTL_SECONDS,- now: datetime.datetime | None = None,-) -> WorkflowRunArchiveDownloadTask:- """Create the Redis payload stored when the console starts an archive download."""- created_at = now or datetime.datetime.now(datetime.UTC)- if created_at.tzinfo is None:- created_at = created_at.replace(tzinfo=datetime.UTC)- normalized_bundle_ids = list(bundle_ids)- normalized_bundle_refs = [- WorkflowRunArchiveBundleRef(shard=shard, bundle_id=bundle_id) for shard, bundle_id in bundle_refs- ]- return WorkflowRunArchiveDownloadTask(- download_id=download_id,- tenant_id=tenant_id,- requested_by=requested_by,- year=year,- month=month,- bundle_ids=normalized_bundle_ids,- bundle_refs=normalized_bundle_refs,- bundle_count=len(normalized_bundle_ids),- archive_bytes=archive_bytes,- status=WorkflowRunArchiveDownloadStatus.PENDING,- created_at=created_at,- updated_at=created_at,- expires_at=created_at + datetime.timedelta(seconds=ttl_seconds),- )---def build_archive_download_id(- *,- tenant_id: str,- year: int,- month: int,- bundle_refs: Sequence[tuple[str, str]],- download_format_version: str = ARCHIVE_DOWNLOAD_FORMAT_VERSION,-) -> str:- """Build a stable id for the exact archive download content."""- if not bundle_refs:- raise ValueError("bundle_refs must not be empty")- normalized_refs = sorted(f"{shard}:{bundle_id}" for shard, bundle_id in bundle_refs)- payload = json.dumps(- {- "tenant_id": tenant_id,- "year": year,- "month": month,- "bundle_refs": normalized_refs,- "download_format_version": download_format_version,- },- sort_keys=True,- separators=(",", ":"),- )- return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]diff --git a/api/services/retention/workflow_run/archive_log_service.py b/api/services/retention/workflow_run/archive_log_service.pyindex 42377c658cc..070ca9260bc 100644--- a/api/services/retention/workflow_run/archive_log_service.py+++ b/api/services/retention/workflow_run/archive_log_service.py@@ -1,34 +1,69 @@-"""-Console-facing workflow-run archive queries.--The object store remains the recoverable archive source of truth. This module only reads the DB bundle index and writes-temporary Redis download-task state, so console requests never list R2 online.-"""+"""Application service for Console workflow-run archive downloads."""import datetimeimport loggingimport uuid-from collections.abc import Callable+from collections.abc import Sequence+from contextlib import AbstractContextManagerfrom dataclasses import dataclass+from typing import Any, NamedTuple, Protocol-from sqlalchemy import select-from sqlalchemy.orm import Session--from models.workflow import WorkflowRunArchiveBundle-from services.retention.workflow_run.archive_download_task_cache import (+from machinery.context import RequestContext+from services.retention.workflow_run.archive_download_task import (WorkflowRunArchiveDownloadStatus,WorkflowRunArchiveDownloadTask,- WorkflowRunArchiveDownloadTaskCache,build_archive_download_id,build_pending_archive_download_task,)logger = logging.getLogger(__name__)-ArchiveDownloadTaskDispatcher = Callable[- [WorkflowRunArchiveDownloadTask, WorkflowRunArchiveDownloadTaskCache],- WorkflowRunArchiveDownloadTask,-]++class WorkflowRunArchiveBundleRecord(NamedTuple):+ """Persistence-neutral metadata for one immutable archive bundle."""++ year: int+ month: int+ shard: str+ bundle_id: str+ workflow_run_count: int+ row_count: int+ archive_bytes: int+ archived_at: datetime.datetime+++class WorkflowRunArchiveBundleQuery(Protocol):+ def list_for_tenant(self, tenant_id: str) -> Sequence[WorkflowRunArchiveBundleRecord]: ...++ def list_for_tenant_month(+ self,+ tenant_id: str,+ *,+ year: int,+ month: int,+ ) -> Sequence[WorkflowRunArchiveBundleRecord]: ...+++class WorkflowRunArchiveDownloadTaskStore(Protocol):+ def get(self, *, tenant_id: str, download_id: str) -> WorkflowRunArchiveDownloadTask | None: ...++ def save(self, task: WorkflowRunArchiveDownloadTask) -> None: ...++ def lock(self, *, tenant_id: str, download_id: str) -> AbstractContextManager[Any]: ...+++class WorkflowRunArchiveDownloadTaskDispatcher(Protocol):+ def __call__(self, task: WorkflowRunArchiveDownloadTask) -> None: ...+++class WorkflowRunArchiveDownloadUrlSigner(Protocol):+ def __call__(+ self,+ storage_key: str,+ *,+ expires_in: int,+ filename: str,+ ) -> str: ...@dataclass(frozen=True)@@ -47,7 +82,7 @@ class WorkflowRunArchiveMonth:@dataclass(frozen=True)class WorkflowRunArchiveSummary:- """Top-level archive totals shown on the console page."""+ """Top-level archive totals shown on the Console page."""archived_month_count: intworkflow_run_count: int@@ -68,212 +103,161 @@ class WorkflowRunArchiveNotFoundError(Exception):class WorkflowRunArchiveDownloadTaskNotFoundError(Exception):- """Raised when the temporary Redis task has expired or never existed."""+ """Raised when the temporary download task has expired or never existed."""class WorkflowRunArchiveDownloadNotReadyError(Exception):"""Raised when a cached download task has not produced a file yet."""-def list_workflow_run_archives(- session: Session,- tenant_id: str,- *,- cache: WorkflowRunArchiveDownloadTaskCache | None = None,-) -> WorkflowRunArchiveList:- """Return monthly archive metadata for one tenant from the DB bundle index."""- stmt = (- select(WorkflowRunArchiveBundle)- .where(WorkflowRunArchiveBundle.tenant_id == tenant_id)- .order_by(- WorkflowRunArchiveBundle.year.desc(),- WorkflowRunArchiveBundle.month.desc(),- WorkflowRunArchiveBundle.shard,- WorkflowRunArchiveBundle.bundle_id,- )- )- month_bundles: dict[tuple[int, int], list[WorkflowRunArchiveBundle]] = {}- for bundle in session.scalars(stmt):- month_bundles.setdefault((bundle.year, bundle.month), []).append(bundle)-- task_cache = cache or WorkflowRunArchiveDownloadTaskCache()- months: list[WorkflowRunArchiveMonth] = []- for (year, month), bundles in month_bundles.items():- bundle_refs = [(bundle.shard, bundle.bundle_id) for bundle in bundles]- months.append(- WorkflowRunArchiveMonth(- year=year,- month=month,- bundle_count=len(bundles),- workflow_run_count=sum(bundle.workflow_run_count for bundle in bundles),- row_count=sum(bundle.row_count for bundle in bundles),- archive_bytes=sum(bundle.archive_bytes for bundle in bundles),- latest_archived_at=max(bundle.archived_at for bundle in bundles),- download_task=_get_cached_month_download_task(- task_cache,- tenant_id=tenant_id,+class WorkflowRunArchiveService:+ def __init__(+ self,+ *,+ bundles: WorkflowRunArchiveBundleQuery,+ tasks: WorkflowRunArchiveDownloadTaskStore,+ dispatcher: WorkflowRunArchiveDownloadTaskDispatcher,+ sign_download_url: WorkflowRunArchiveDownloadUrlSigner,+ ) -> None:+ self._bundles = bundles+ self._tasks = tasks+ self._dispatcher = dispatcher+ self._sign_download_url = sign_download_url++ def list_archives(self, context: RequestContext) -> WorkflowRunArchiveList:+ """Return monthly archive metadata for the active workspace."""+ tenant_id = self._active_workspace_id(context)+ records_by_month: dict[tuple[int, int], list[WorkflowRunArchiveBundleRecord]] = {}+ for record in self._bundles.list_for_tenant(tenant_id):+ records_by_month.setdefault((record.year, record.month), []).append(record)++ months: list[WorkflowRunArchiveMonth] = []+ for year, month in sorted(records_by_month, reverse=True):+ records = records_by_month[(year, month)]+ bundle_refs = [(record.shard, record.bundle_id) for record in records]+ months.append(+ WorkflowRunArchiveMonth(year=year,month=month,- bundle_refs=bundle_refs,- ),+ bundle_count=len(records),+ workflow_run_count=sum(record.workflow_run_count for record in records),+ row_count=sum(record.row_count for record in records),+ archive_bytes=sum(record.archive_bytes for record in records),+ latest_archived_at=max(record.archived_at for record in records),+ download_task=self._get_cached_month_download_task(+ tenant_id=tenant_id,+ year=year,+ month=month,+ bundle_refs=bundle_refs,+ ),+ ))++ latest_archived_at = max((archive.latest_archived_at for archive in months), default=None)+ return WorkflowRunArchiveList(+ summary=WorkflowRunArchiveSummary(+ archived_month_count=len(months),+ workflow_run_count=sum(archive.workflow_run_count for archive in months),+ archive_bytes=sum(archive.archive_bytes for archive in months),+ latest_archived_at=latest_archived_at,+ ),+ months=months,)- latest_archived_at = max((month.latest_archived_at for month in months), default=None)- return WorkflowRunArchiveList(- summary=WorkflowRunArchiveSummary(- archived_month_count=len(months),- workflow_run_count=sum(month.workflow_run_count for month in months),- archive_bytes=sum(month.archive_bytes for month in months),- latest_archived_at=latest_archived_at,- ),- months=months,- )---def _get_cached_month_download_task(- cache: WorkflowRunArchiveDownloadTaskCache,- *,- tenant_id: str,- year: int,- month: int,- bundle_refs: list[tuple[str, str]],-) -> WorkflowRunArchiveDownloadTask | None:- if not bundle_refs:- return None- download_id = build_archive_download_id(- tenant_id=tenant_id,- year=year,- month=month,- bundle_refs=bundle_refs,- )- try:- return cache.get(tenant_id=tenant_id, download_id=download_id)- except Exception:- logger.warning("Failed to read cached workflow run archive download task: %s", download_id, exc_info=True)- return None---def create_workflow_run_archive_download_task(- session: Session,- *,- tenant_id: str,- requested_by: str,- year: int,- month: int,- cache: WorkflowRunArchiveDownloadTaskCache | None = None,- dispatcher: ArchiveDownloadTaskDispatcher | None = None,-) -> WorkflowRunArchiveDownloadTask:- """- Create or return the idempotent Redis task for downloading one tenant/month archive.-- The task identity is based on the exact ordered bundle set currently indexed for the month. If the month receives a- new bundle later, the next request gets a different download id and prepares a fresh file.- """- bundles = _list_archive_bundles(session, tenant_id=tenant_id, year=year, month=month)- if not bundles:- raise WorkflowRunArchiveNotFoundError(f"Workflow run archive not found: {year:04d}-{month:02d}")-- bundle_refs = [(bundle.shard, bundle.bundle_id) for bundle in bundles]- download_id = build_archive_download_id(- tenant_id=tenant_id,- year=year,- month=month,- bundle_refs=bundle_refs,- )- task = build_pending_archive_download_task(- tenant_id=tenant_id,- requested_by=requested_by,- year=year,- month=month,- bundle_ids=[bundle.bundle_id for bundle in bundles],- bundle_refs=bundle_refs,- archive_bytes=sum(bundle.archive_bytes for bundle in bundles),- download_id=download_id,- )- task_cache = cache or WorkflowRunArchiveDownloadTaskCache()- dispatch = dispatcher or _dispatch_workflow_run_archive_download_task- with task_cache.lock(tenant_id=tenant_id, download_id=download_id):- existing = task_cache.get(tenant_id=tenant_id, download_id=download_id)- if existing is None or existing.status == WorkflowRunArchiveDownloadStatus.FAILED:- task_to_queue = task- elif existing.status == WorkflowRunArchiveDownloadStatus.PENDING and not existing.celery_task_id:- task_to_queue = existing- else:- return existing-- queued_task = task_to_queue.model_copy(- update={"celery_task_id": uuid.uuid4().hex, "updated_at": datetime.datetime.now(datetime.UTC)}++ def create_download(+ self,+ context: RequestContext,+ *,+ year: int,+ month: int,+ ) -> WorkflowRunArchiveDownloadTask:+ """Create or return the idempotent download task for one workspace/month."""+ tenant_id = self._active_workspace_id(context)+ bundles = self._bundles.list_for_tenant_month(tenant_id, year=year, month=month)+ if not bundles:+ raise WorkflowRunArchiveNotFoundError(f"Workflow run archive not found: {year:04d}-{month:02d}")++ bundle_refs = [(bundle.shard, bundle.bundle_id) for bundle in bundles]+ download_id = build_archive_download_id(+ tenant_id=tenant_id,+ year=year,+ month=month,+ bundle_refs=bundle_refs,)- task_cache.save(queued_task)-- return dispatch(queued_task, task_cache)---def get_workflow_run_archive_download_task(- *,- tenant_id: str,- download_id: str,- cache: WorkflowRunArchiveDownloadTaskCache | None = None,-) -> WorkflowRunArchiveDownloadTask:- """Return a cached archive download task or raise when the TTL has expired."""- task_cache = cache or WorkflowRunArchiveDownloadTaskCache()- task = task_cache.get(tenant_id=tenant_id, download_id=download_id)- if task is None:- raise WorkflowRunArchiveDownloadTaskNotFoundError(f"Workflow run archive download not found: {download_id}")- return task---def get_ready_workflow_run_archive_download_task(- *,- tenant_id: str,- download_id: str,- cache: WorkflowRunArchiveDownloadTaskCache | None = None,-) -> WorkflowRunArchiveDownloadTask:- """Return a ready cached archive download task or raise when the file is not available."""- task = get_workflow_run_archive_download_task(tenant_id=tenant_id, download_id=download_id, cache=cache)- if task.status != WorkflowRunArchiveDownloadStatus.READY or not task.storage_key or not task.file_name:- raise WorkflowRunArchiveDownloadNotReadyError(f"Workflow run archive download is not ready: {download_id}")- return task---def _list_archive_bundles(- session: Session,- *,- tenant_id: str,- year: int,- month: int,-) -> list[WorkflowRunArchiveBundle]:- stmt = (- select(WorkflowRunArchiveBundle)- .where(- WorkflowRunArchiveBundle.tenant_id == tenant_id,- WorkflowRunArchiveBundle.year == year,- WorkflowRunArchiveBundle.month == month,+ task = build_pending_archive_download_task(+ tenant_id=tenant_id,+ requested_by=context.account_id,+ year=year,+ month=month,+ bundle_ids=[bundle.bundle_id for bundle in bundles],+ bundle_refs=bundle_refs,+ archive_bytes=sum(bundle.archive_bytes for bundle in bundles),+ download_id=download_id,)- .order_by(WorkflowRunArchiveBundle.shard, WorkflowRunArchiveBundle.bundle_id)- )- return list(session.scalars(stmt))---def _dispatch_workflow_run_archive_download_task(- task: WorkflowRunArchiveDownloadTask,- cache: WorkflowRunArchiveDownloadTaskCache,-) -> WorkflowRunArchiveDownloadTask:- """- Enqueue background ZIP preparation after the caller atomically claimed and cached the Celery id.- """- from tasks.workflow_run_archive_download_tasks import prepare_workflow_run_archive_download_task-- celery_task_id = task.celery_task_id- if celery_task_id is None:- raise ValueError("celery_task_id is required before dispatch")-- try:- prepare_workflow_run_archive_download_task.apply_async(- args=(task.tenant_id, task.download_id),- task_id=celery_task_id,++ with self._tasks.lock(tenant_id=tenant_id, download_id=download_id):+ existing = self._tasks.get(tenant_id=tenant_id, download_id=download_id)+ if existing is None or existing.status == WorkflowRunArchiveDownloadStatus.FAILED:+ task_to_queue = task+ elif existing.status == WorkflowRunArchiveDownloadStatus.PENDING and not existing.celery_task_id:+ task_to_queue = existing+ else:+ return existing++ queued_task = task_to_queue.model_copy(+ update={"celery_task_id": uuid.uuid4().hex, "updated_at": datetime.datetime.now(datetime.UTC)}+ )+ self._tasks.save(queued_task)++ try:+ self._dispatcher(queued_task)+ except Exception:+ return self._record_dispatch_failure(queued_task)+ return queued_task++ def get_download(self, context: RequestContext, *, download_id: str) -> WorkflowRunArchiveDownloadTask:+ """Return a cached download task or raise after its TTL expires."""+ tenant_id = self._active_workspace_id(context)+ task = self._tasks.get(tenant_id=tenant_id, download_id=download_id)+ if task is None:+ raise WorkflowRunArchiveDownloadTaskNotFoundError(f"Workflow run archive download not found: {download_id}")+ return task++ def get_download_url(self, context: RequestContext, *, download_id: str) -> str:+ """Return a short-lived URL for a ready archive download."""+ task = self.get_download(context, download_id=download_id)+ if task.status != WorkflowRunArchiveDownloadStatus.READY or not task.storage_key or not task.file_name:+ raise WorkflowRunArchiveDownloadNotReadyError(f"Workflow run archive download is not ready: {download_id}")+ return self._sign_download_url(+ task.storage_key,+ expires_in=self._presigned_url_expires_in(task.expires_at),+ filename=task.file_name,+ )++ def _get_cached_month_download_task(+ self,+ *,+ tenant_id: str,+ year: int,+ month: int,+ bundle_refs: Sequence[tuple[str, str]],+ ) -> WorkflowRunArchiveDownloadTask | None:+ download_id = build_archive_download_id(+ tenant_id=tenant_id,+ year=year,+ month=month,+ bundle_refs=bundle_refs,)- except Exception:+ try:+ return self._tasks.get(tenant_id=tenant_id, download_id=download_id)+ except Exception:+ logger.warning("Failed to read cached workflow run archive download task: %s", download_id, exc_info=True)+ return None++ def _record_dispatch_failure(+ self,+ task: WorkflowRunArchiveDownloadTask,+ ) -> WorkflowRunArchiveDownloadTask:failure_time = datetime.datetime.now(datetime.UTC)failed_task = task.model_copy(update={@@ -283,16 +267,27 @@ def _dispatch_workflow_run_archive_download_task("finished_at": failure_time,})- with cache.lock(tenant_id=task.tenant_id, download_id=task.download_id):- current = cache.get(tenant_id=task.tenant_id, download_id=task.download_id)+ with self._tasks.lock(tenant_id=task.tenant_id, download_id=task.download_id):+ current = self._tasks.get(tenant_id=task.tenant_id, download_id=task.download_id)if (current is not Noneand current.status == WorkflowRunArchiveDownloadStatus.PENDING- and current.celery_task_id == celery_task_id+ and current.celery_task_id == task.celery_task_id):- cache.save(failed_task)+ self._tasks.save(failed_task)current = failed_tasklogger.exception("Failed to enqueue workflow run archive download task %s", task.download_id)return current or failed_task- return task+ @staticmethod+ def _presigned_url_expires_in(expires_at: datetime.datetime) -> int:+ expires_at_utc = expires_at if expires_at.tzinfo else expires_at.replace(tzinfo=datetime.UTC)+ remaining_seconds = int((expires_at_utc - datetime.datetime.now(datetime.UTC)).total_seconds())+ return max(1, min(3600, remaining_seconds))++ @staticmethod+ def _active_workspace_id(context: RequestContext) -> str:+ workspace_id = context.active_workspace_id+ if workspace_id is None:+ raise RuntimeError("Console account admission did not resolve an active workspace")+ return workspace_iddiff --git a/api/tasks/workflow_run_archive_download_tasks.py b/api/tasks/workflow_run_archive_download_tasks.pyindex 85efbf60e64..c230b427982 100644--- a/api/tasks/workflow_run_archive_download_tasks.py+++ b/api/tasks/workflow_run_archive_download_tasks.py@@ -4,7 +4,12 @@ import loggingfrom celery import shared_task+from core.db.session_factory import get_session_maker+from extensions.ext_redis import redis_client+from libs.archive_storage import get_archive_storage, get_export_storage+from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepositoryfrom services.retention.workflow_run.archive_download_preparation import WorkflowRunArchiveDownloadPreparer+from services.retention.workflow_run.archive_download_task_cache import WorkflowRunArchiveDownloadTaskCachelogger = logging.getLogger(__name__)@@ -15,4 +20,9 @@ WORKFLOW_RUN_ARCHIVE_DOWNLOAD_QUEUE = "workflow_archive"def prepare_workflow_run_archive_download_task(tenant_id: str, download_id: str) -> None:"""Prepare a cached workflow-run archive download in the background."""logger.info("Preparing workflow run archive download: tenant=%s download_id=%s", tenant_id, download_id)- WorkflowRunArchiveDownloadPreparer().prepare(tenant_id=tenant_id, download_id=download_id)+ WorkflowRunArchiveDownloadPreparer(+ archive_storage_provider=get_archive_storage,+ bundles=WorkflowRunArchiveBundleQueryRepository(session_factory=get_session_maker()),+ cache=WorkflowRunArchiveDownloadTaskCache(redis=redis_client),+ download_storage_provider=get_export_storage,+ ).prepare(tenant_id=tenant_id, download_id=download_id)diff --git a/api/tests/unit_tests/controllers/console/test_workflow_run_archive.py b/api/tests/unit_tests/controllers/console/test_workflow_run_archive.pyindex 084654f10ad..ea0fc9c2dc8 100644--- a/api/tests/unit_tests/controllers/console/test_workflow_run_archive.py+++ b/api/tests/unit_tests/controllers/console/test_workflow_run_archive.py@@ -1,15 +1,42 @@+import inspect+from types import SimpleNamespace+from unittest.mock import MagicMock+import pytest-from werkzeug.exceptions import Forbidden+from flask import Flask+from werkzeug.exceptions import Conflict, Forbidden, NotFoundfrom configs import dify_config-from controllers.console import workflow_run_archive+from controllers.console import flask_admission, workflow_run_archivefrom controllers.console.workflow_run_archive import (WorkflowRunArchiveDownloadApi,WorkflowRunArchiveDownloadFileApi,+ WorkflowRunArchiveDownloadPayload,WorkflowRunArchiveDownloadsApi,WorkflowRunArchivesApi,)+from enums import CloudPlan, DeploymentEdition+from libs.login import AccountWithTenant+from machinery.context import RequestContextfrom models import Account, TenantAccountRole+from services.retention.workflow_run.archive_log_service import (+ WorkflowRunArchiveDownloadNotReadyError,+ WorkflowRunArchiveDownloadTaskNotFoundError,+ WorkflowRunArchiveNotFoundError,+)++_CONTEXT = RequestContext(+ request_id="request-1",+ trace_id="trace-1",+ account_id="account-1",+ active_workspace_id="tenant-1",+)+_ENDPOINTS = [+ WorkflowRunArchivesApi.get,+ WorkflowRunArchiveDownloadsApi.post,+ WorkflowRunArchiveDownloadApi.get,+ WorkflowRunArchiveDownloadFileApi.get,+]def _account(role: TenantAccountRole) -> Account:@@ -19,83 +46,209 @@ def _account(role: TenantAccountRole) -> Account:return account- "role",- [TenantAccountRole.EDITOR, TenantAccountRole.NORMAL, TenantAccountRole.DATASET_OPERATOR],-)[email protected]("rbac_enabled", [False, True])-def test_current_owner_or_admin_ids_rejects_non_manager(- monkeypatch: pytest.MonkeyPatch, role: TenantAccountRole, rbac_enabled: bool-) -> None:- current_user = _account(role)- monkeypatch.setattr(dify_config, "RBAC_ENABLED", rbac_enabled)- monkeypatch.setattr(- workflow_run_archive,- "current_account_with_tenant",- lambda: (current_user, "tenant-1"),- )+def _original(method):+ while hasattr(method, "__wrapped__"):+ method = method.__wrapped__+ return method+- with pytest.raises(Forbidden):- workflow_run_archive._current_owner_or_admin_ids()+def _admission_injector(method):+ method = inspect.getclosurevars(method).nonlocals["admitted"]+ while "inject_request_context" not in method.__code__.co_qualname:+ method = method.__wrapped__+ return method++[email protected]("method", _ENDPOINTS)+def test_workflow_run_archive_endpoints_declare_cloud_paid_plan_admission(method) -> None:+ decorator_names = set()+ current = method+ while hasattr(current, "__wrapped__"):+ decorator_names.add(current.__code__.co_qualname.partition(".<locals>")[0])+ current = current.__wrapped__+ admission = _admission_injector(method)+ allowed_roles = inspect.getclosurevars(admission).nonlocals["allowed_roles"][email protected]("role", [TenantAccountRole.OWNER, TenantAccountRole.ADMIN])[email protected]("rbac_enabled", [False, True])-def test_current_owner_or_admin_ids_returns_current_ids_for_manager(- monkeypatch: pytest.MonkeyPatch, role: TenantAccountRole, rbac_enabled: bool+ assert "console_account_admission" in decorator_names+ assert "cloud_edition_billing_paid_plan_required" in decorator_names+ assert "only_edition_cloud" not in decorator_names+ assert allowed_roles == frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN})++[email protected]("method", _ENDPOINTS)+def test_workflow_run_archive_endpoints_reject_non_manager_when_rbac_is_disabled(+ monkeypatch: pytest.MonkeyPatch,+ method,) -> None:- current_user = _account(role)- monkeypatch.setattr(dify_config, "RBAC_ENABLED", rbac_enabled)+ account = _account(TenantAccountRole.NORMAL)+ monkeypatch.setattr(dify_config, "RBAC_ENABLED", False)monkeypatch.setattr(- workflow_run_archive,+ flask_admission,"current_account_with_tenant",- lambda: (current_user, "tenant-1"),+ lambda: AccountWithTenant(account=account, tenant_id="tenant-1"),)+ app = Flask(__name__)- assert workflow_run_archive._current_owner_or_admin_ids() == ("tenant-1", "account-1")+ with app.test_request_context(), pytest.raises(Forbidden):+ _admission_injector(method)(None)@pytest.mark.parametrize(("method", "args"),[(WorkflowRunArchivesApi.get, ()),- (WorkflowRunArchiveDownloadsApi.post, (None,)),+ (WorkflowRunArchiveDownloadsApi.post, ()),(WorkflowRunArchiveDownloadApi.get, ("download-1",)),(WorkflowRunArchiveDownloadFileApi.get, ("download-1",)),],)-def test_workflow_run_archive_endpoints_enforce_fixed_workspace_roles(- monkeypatch: pytest.MonkeyPatch, method, args: tuple[str, ...]+def test_workflow_run_archive_endpoints_are_hidden_outside_cloud(+ monkeypatch: pytest.MonkeyPatch,+ method,+ args: tuple[object, ...],) -> None:- while hasattr(method, "__wrapped__"):- method = method.__wrapped__+ monkeypatch.setattr(dify_config, "DEPLOYMENT_EDITION", DeploymentEdition.COMMUNITY)+ app = Flask(__name__)- def reject_non_manager() -> tuple[str, str]:- raise Forbidden()+ with app.test_request_context(), pytest.raises(NotFound):+ method(None, *args)- monkeypatch.setattr(workflow_run_archive, "_current_owner_or_admin_ids", reject_non_manager)- with pytest.raises(Forbidden):- method(None, *args)+def test_workflow_run_archive_endpoint_allows_admitted_role_when_rbac_is_enabled(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ account = _account(TenantAccountRole.NORMAL)+ monkeypatch.setattr(dify_config, "RBAC_ENABLED", True)+ account_with_tenant = AccountWithTenant(account=account, tenant_id="tenant-1")+ monkeypatch.setattr(flask_admission, "current_account_with_tenant", lambda: account_with_tenant)+ monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", lambda: account_with_tenant)++ def get_billing_info(tenant_id: str, *, exclude_vector_space: bool):+ assert tenant_id == "tenant-1"+ assert exclude_vector_space+ return {+ "enabled": True,+ "subscription": {"plan": CloudPlan.TEAM},+ }++ monkeypatch.setattr(+ "controllers.console.wraps.BillingService.get_info",+ get_billing_info,+ )+ service = MagicMock()+ service.list_archives.return_value = object()+ monkeypatch.setattr(+ workflow_run_archive,+ "application_services",+ lambda: SimpleNamespace(workflow_run_archives=service),+ )+ monkeypatch.setattr(workflow_run_archive, "dump_response", lambda _model, value: value)+ app = Flask(__name__)++ with app.test_request_context():+ _admission_injector(WorkflowRunArchivesApi.get)(None)++ context = service.list_archives.call_args.args[0]+ assert context.account_id == "account-1"+ assert context.active_workspace_id == "tenant-1"+++def test_workflow_run_archive_endpoints_delegate_to_application_service(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ service = MagicMock()+ archives = object()+ task = object()+ service.list_archives.return_value = archives+ service.create_download.return_value = task+ service.get_download.return_value = task+ monkeypatch.setattr(+ workflow_run_archive,+ "application_services",+ lambda: SimpleNamespace(workflow_run_archives=service),+ )+ dump_response = MagicMock(side_effect=lambda _model, value: value)+ monkeypatch.setattr(workflow_run_archive, "dump_response", dump_response)++ assert _original(WorkflowRunArchivesApi.get)(None, _CONTEXT) is archives+ assert _original(WorkflowRunArchiveDownloadsApi.post)(+ None,+ WorkflowRunArchiveDownloadPayload(year=2025, month=3),+ _CONTEXT,+ ) == (task, 202)+ assert _original(WorkflowRunArchiveDownloadApi.get)(None, _CONTEXT, "download-1") is task++ service.list_archives.assert_called_once_with(_CONTEXT)+ service.create_download.assert_called_once_with(_CONTEXT, year=2025, month=3)+ service.get_download.assert_called_once_with(_CONTEXT, download_id="download-1")@pytest.mark.parametrize(- "method",+ ("method", "service_method", "service_error", "http_error", "args"),[- WorkflowRunArchivesApi.get,- WorkflowRunArchiveDownloadsApi.post,- WorkflowRunArchiveDownloadApi.get,- WorkflowRunArchiveDownloadFileApi.get,+ (+ WorkflowRunArchiveDownloadsApi.post,+ "create_download",+ WorkflowRunArchiveNotFoundError("archive missing"),+ NotFound,+ (WorkflowRunArchiveDownloadPayload(year=2025, month=3), _CONTEXT),+ ),+ (+ WorkflowRunArchiveDownloadApi.get,+ "get_download",+ WorkflowRunArchiveDownloadTaskNotFoundError("task missing"),+ NotFound,+ (_CONTEXT, "download-1"),+ ),+ (+ WorkflowRunArchiveDownloadFileApi.get,+ "get_download_url",+ WorkflowRunArchiveDownloadTaskNotFoundError("task missing"),+ NotFound,+ (_CONTEXT, "download-1"),+ ),+ (+ WorkflowRunArchiveDownloadFileApi.get,+ "get_download_url",+ WorkflowRunArchiveDownloadNotReadyError("task pending"),+ Conflict,+ (_CONTEXT, "download-1"),+ ),],)-def test_workflow_run_archive_endpoints_require_cloud_paid_plan(method) -> None:- decorator_names = set()- while hasattr(method, "__wrapped__"):- decorator_names.add(method.__code__.co_qualname.partition(".<locals>")[0])- method = method.__wrapped__+def test_workflow_run_archive_endpoints_translate_service_errors(+ monkeypatch: pytest.MonkeyPatch,+ method,+ service_method: str,+ service_error: Exception,+ http_error: type[Exception],+ args: tuple[object, ...],+) -> None:+ service = MagicMock()+ service.configure_mock(**{f"{service_method}.side_effect": service_error})+ monkeypatch.setattr(+ workflow_run_archive,+ "application_services",+ lambda: SimpleNamespace(workflow_run_archives=service),+ )++ with pytest.raises(http_error):+ _original(method)(None, *args)+++def test_workflow_run_archive_file_redirects_to_signed_url(monkeypatch: pytest.MonkeyPatch) -> None:+ service = MagicMock()+ service.get_download_url.return_value = "https://storage.example.com/archive.zip"+ monkeypatch.setattr(+ workflow_run_archive,+ "application_services",+ lambda: SimpleNamespace(workflow_run_archives=service),+ )+ app = Flask(__name__)++ with app.test_request_context():+ response = _original(WorkflowRunArchiveDownloadFileApi.get)(None, _CONTEXT, "download-1")- assert {- "only_edition_cloud",- "cloud_edition_billing_paid_plan_required",- } <= decorator_names- assert "rbac_permission_required" not in decorator_names+ assert response.status_code == 302+ assert response.location == "https://storage.example.com/archive.zip"diff --git a/api/tests/unit_tests/extensions/test_ext_application_services.py b/api/tests/unit_tests/extensions/test_ext_application_services.pyindex f53ba6db671..ba2bb85b5ed 100644--- a/api/tests/unit_tests/extensions/test_ext_application_services.py+++ b/api/tests/unit_tests/extensions/test_ext_application_services.py@@ -22,6 +22,7 @@ from repositories.account_activation_repository import SQLAlchemyAccountActivatifrom repositories.account_integration_repository import SQLAlchemyAccountIntegrationRepositoryfrom repositories.account_repository import SQLAlchemyAccountRepositoryfrom repositories.app_site_command_repository import AppSiteCommandRepository+from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepositoryfrom services import recommended_app_catalog_gatewayfrom services.account_activation_adapters import (BillingAccountActivationEligibility,@@ -39,6 +40,8 @@ from services.enterprise.enterprise_service import WebAppSettingsfrom services.errors.enterprise import EnterpriseAPIError, EnterpriseAPINotFoundErrorfrom services.init_validation_service import InvalidInitializationPasswordErrorfrom services.partner_tenant_binding_service import PartnerTenantBindingService+from services.retention.workflow_run.archive_download_task_cache import WorkflowRunArchiveDownloadTaskCache+from services.retention.workflow_run.archive_log_service import WorkflowRunArchiveServicefrom services.tag_application_service import TagApplicationServicefrom services.webapp_access_query_service import WebAppAccessUnavailableErrorfrom services.workflow_statistic_query_service import WorkflowStatisticQueryService@@ -182,6 +185,28 @@ def test_build_application_services_wires_tag_boundary(assert isinstance(services.tags, TagApplicationService)+def test_build_application_services_wires_workflow_run_archives(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ redis = MagicMock(spec=RedisClientWrapper)++ services = ext_application_services.build_application_services(+ database_client=sqlite_session_factory,+ deployment_edition=DeploymentEdition.COMMUNITY,+ initialization_password="",+ redis=redis,+ )++ workflow_run_archives = services.workflow_run_archives+ assert isinstance(workflow_run_archives, WorkflowRunArchiveService)+ assert isinstance(workflow_run_archives._bundles, WorkflowRunArchiveBundleQueryRepository)+ assert workflow_run_archives._bundles._session_factory is sqlite_session_factory+ assert isinstance(workflow_run_archives._tasks, WorkflowRunArchiveDownloadTaskCache)+ assert workflow_run_archives._tasks._redis is redis+ assert workflow_run_archives._dispatcher is ext_application_services.dispatch_workflow_run_archive_download_task+ assert workflow_run_archives._sign_download_url is ext_application_services.sign_workflow_run_archive_download_url++def test_build_application_services_wires_app_site_boundary(sqlite_session_factory: sessionmaker[Session],) -> None:diff --git a/api/tests/unit_tests/repositories/test_workflow_run_archive_repository.py b/api/tests/unit_tests/repositories/test_workflow_run_archive_repository.pynew file mode 100644index 00000000000..c6c763a5410--- /dev/null+++ b/api/tests/unit_tests/repositories/test_workflow_run_archive_repository.py@@ -0,0 +1,138 @@+import datetime++from sqlalchemy.orm import Session, sessionmaker++from models.workflow import WorkflowRunArchiveBundle+from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository+from services.retention.workflow_run.archive_log_service import WorkflowRunArchiveBundleRecord+++def _bundle(+ *,+ tenant_id: str,+ year: int,+ month: int,+ shard: str,+ bundle_id: str,+ workflow_run_count: int = 1,+ row_count: int = 9,+ archive_bytes: int = 1024,+ archived_at: datetime.datetime | None = None,+) -> WorkflowRunArchiveBundle:+ return WorkflowRunArchiveBundle(+ tenant_id=tenant_id,+ year=year,+ month=month,+ shard=shard,+ bundle_id=bundle_id,+ workflow_run_count=workflow_run_count,+ row_count=row_count,+ archive_bytes=archive_bytes,+ archived_at=archived_at or datetime.datetime(2026, 6, 25, 8, 0),+ )+++def test_list_for_tenant_returns_ordered_pure_records(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ latest = datetime.datetime(2026, 6, 26, 8, 0)+ previous = datetime.datetime(2026, 6, 25, 8, 0)+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ _bundle(+ tenant_id="tenant-1",+ year=2025,+ month=2,+ shard="00-of-01",+ bundle_id="bundle-c",+ ),+ _bundle(+ tenant_id="tenant-1",+ year=2025,+ month=3,+ shard="01-of-02",+ bundle_id="bundle-b",+ workflow_run_count=60,+ row_count=540,+ archive_bytes=3072,+ archived_at=latest,+ ),+ _bundle(+ tenant_id="tenant-1",+ year=2025,+ month=3,+ shard="00-of-02",+ bundle_id="bundle-a",+ workflow_run_count=40,+ row_count=360,+ archive_bytes=1024,+ archived_at=previous,+ ),+ _bundle(+ tenant_id="tenant-2",+ year=2026,+ month=1,+ shard="00-of-01",+ bundle_id="other-tenant-bundle",+ ),+ ]+ )++ records = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory).list_for_tenant(+ "tenant-1"+ )++ assert records == (+ WorkflowRunArchiveBundleRecord(2025, 3, "00-of-02", "bundle-a", 40, 360, 1024, previous),+ WorkflowRunArchiveBundleRecord(2025, 3, "01-of-02", "bundle-b", 60, 540, 3072, latest),+ WorkflowRunArchiveBundleRecord(2025, 2, "00-of-01", "bundle-c", 1, 9, 1024, previous),+ )+++def test_list_for_tenant_month_filters_and_orders_bundles(+ sqlite_session_factory: sessionmaker[Session],+) -> None:+ archived_at = datetime.datetime(2026, 6, 25, 8, 0)+ with sqlite_session_factory.begin() as session:+ session.add_all(+ [+ _bundle(+ tenant_id="tenant-1",+ year=2025,+ month=3,+ shard="00-of-01",+ bundle_id="bundle-b",+ ),+ _bundle(+ tenant_id="tenant-1",+ year=2025,+ month=3,+ shard="00-of-01",+ bundle_id="bundle-a",+ ),+ _bundle(+ tenant_id="tenant-1",+ year=2025,+ month=2,+ shard="00-of-01",+ bundle_id="previous-month",+ ),+ _bundle(+ tenant_id="tenant-2",+ year=2025,+ month=3,+ shard="00-of-01",+ bundle_id="other-tenant-bundle",+ ),+ ]+ )++ records = WorkflowRunArchiveBundleQueryRepository(session_factory=sqlite_session_factory).list_for_tenant_month(+ "tenant-1", year=2025, month=3+ )++ assert records == (+ WorkflowRunArchiveBundleRecord(2025, 3, "00-of-01", "bundle-a", 1, 9, 1024, archived_at),+ WorkflowRunArchiveBundleRecord(2025, 3, "00-of-01", "bundle-b", 1, 9, 1024, archived_at),+ )diff --git a/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_adapters.py b/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_adapters.pynew file mode 100644index 00000000000..6d69520eedb--- /dev/null+++ b/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_adapters.py@@ -0,0 +1,100 @@+import sys+from types import ModuleType, SimpleNamespace+from unittest.mock import MagicMock++import pytest++from services.retention.workflow_run import archive_download_adapters+from services.retention.workflow_run.archive_download_adapters import (+ dispatch_workflow_run_archive_download_task,+ sign_workflow_run_archive_download_url,+)+from services.retention.workflow_run.archive_download_task import (+ WorkflowRunArchiveDownloadTask,+ build_pending_archive_download_task,+)+++def _task(*, celery_task_id: str | None = "celery-task-1") -> WorkflowRunArchiveDownloadTask:+ return build_pending_archive_download_task(+ tenant_id="tenant-1",+ requested_by="account-1",+ year=2025,+ month=3,+ bundle_ids=["bundle-1"],+ bundle_refs=[("00-of-01", "bundle-1")],+ archive_bytes=1024,+ download_id="download-1",+ ).model_copy(update={"celery_task_id": celery_task_id})+++def _patch_archive_download_task(monkeypatch: pytest.MonkeyPatch, *, apply_async: MagicMock) -> None:+ task_module = ModuleType("tasks.workflow_run_archive_download_tasks")+ task_module.__dict__["prepare_workflow_run_archive_download_task"] = SimpleNamespace(apply_async=apply_async)+ monkeypatch.setitem(sys.modules, task_module.__name__, task_module)+++def test_dispatch_workflow_run_archive_download_task_enqueues_claimed_task(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ apply_async = MagicMock()+ _patch_archive_download_task(monkeypatch, apply_async=apply_async)++ result = dispatch_workflow_run_archive_download_task(_task())++ assert result is None+ apply_async.assert_called_once_with(+ args=("tenant-1", "download-1"),+ task_id="celery-task-1",+ )+++def test_dispatch_workflow_run_archive_download_task_requires_claim_id(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ apply_async = MagicMock()+ _patch_archive_download_task(monkeypatch, apply_async=apply_async)++ with pytest.raises(ValueError, match="celery_task_id is required before dispatch"):+ dispatch_workflow_run_archive_download_task(_task(celery_task_id=None))++ apply_async.assert_not_called()+++def test_dispatch_workflow_run_archive_download_task_propagates_enqueue_error(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ failure = RuntimeError("broker unavailable")+ apply_async = MagicMock(side_effect=failure)+ _patch_archive_download_task(monkeypatch, apply_async=apply_async)++ with pytest.raises(RuntimeError) as raised:+ dispatch_workflow_run_archive_download_task(_task())++ assert raised.value is failure+++def test_sign_workflow_run_archive_download_url_resolves_export_storage_lazily(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ storage = MagicMock()+ storage.generate_presigned_url.return_value = "https://download.example/archive.zip"+ get_export_storage = MagicMock(return_value=storage)+ monkeypatch.setattr(archive_download_adapters, "get_export_storage", get_export_storage)++ assert get_export_storage.call_count == 0++ result = sign_workflow_run_archive_download_url(+ "downloads/archive.zip",+ expires_in=900,+ filename="workflow-run-logs-2025-03.zip",+ )++ assert result == "https://download.example/archive.zip"+ get_export_storage.assert_called_once_with()+ storage.generate_presigned_url.assert_called_once_with(+ "downloads/archive.zip",+ expires_in=900,+ filename="workflow-run-logs-2025-03.zip",+ content_type="application/zip",+ )diff --git a/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_preparation.py b/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_preparation.pyindex d83a7bc978e..9e6689e2756 100644--- a/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_preparation.py+++ b/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_preparation.py@@ -9,22 +9,20 @@ from unittest.mock import MagicMockimport pyarrow as paimport pyarrow.parquet as pq-from sqlalchemy.orm import Session, sessionmakerfrom libs.archive_storage import ArchiveStorage-from models.workflow import WorkflowRunArchiveBundlefrom services.retention.workflow_run.archive_bundle_index import ARCHIVE_BUNDLE_ROOT_PREFIX, ArchiveBundleManifestfrom services.retention.workflow_run.archive_download_preparation import (WorkflowRunArchiveDownloadPreparer,build_archive_download_storage_key,)-from services.retention.workflow_run.archive_download_task_cache import (+from services.retention.workflow_run.archive_download_task import (WorkflowRunArchiveDownloadStatus,WorkflowRunArchiveDownloadTask,- WorkflowRunArchiveDownloadTaskCache,build_archive_download_id,build_pending_archive_download_task,)+from services.retention.workflow_run.archive_log_service import WorkflowRunArchiveBundleRecordfrom services.retention.workflow_run.constants import ARCHIVE_BUNDLE_FORMAT, ARCHIVE_BUNDLE_SCHEMA_VERSIONTENANT_ID = "1251fe32-c0c7-4fe2-a7bd-a8105267faf5"@@ -74,6 +72,25 @@ class FakeTaskCache:self.saved_tasks.append(task)+class FakeBundleQuery:+ def __init__(self, records: tuple[WorkflowRunArchiveBundleRecord, ...]) -> None:+ self.records = records++ def list_for_tenant(self, tenant_id: str) -> tuple[WorkflowRunArchiveBundleRecord, ...]:+ assert tenant_id == TENANT_ID+ return self.records++ def list_for_tenant_month(+ self,+ tenant_id: str,+ *,+ year: int,+ month: int,+ ) -> tuple[WorkflowRunArchiveBundleRecord, ...]:+ assert tenant_id == TENANT_ID+ return tuple(record for record in self.records if record.year == year and record.month == month)++def _object_prefix(bundle_id: str = BUNDLE_ID) -> str:return (f"{ARCHIVE_BUNDLE_ROOT_PREFIX}tenant_prefix=1/tenant_id={TENANT_ID}/"@@ -81,9 +98,8 @@ def _object_prefix(bundle_id: str = BUNDLE_ID) -> str:)-def _bundle(bundle_id: str = BUNDLE_ID) -> WorkflowRunArchiveBundle:- return WorkflowRunArchiveBundle(- tenant_id=TENANT_ID,+def _bundle(bundle_id: str = BUNDLE_ID) -> WorkflowRunArchiveBundleRecord:+ return WorkflowRunArchiveBundleRecord(year=2025,month=3,shard=SHARD,@@ -154,21 +170,17 @@ def _preparer(archive_storage: FakeArchiveStorage | None = None,download_storage: FakeArchiveStorage | None = None,cache: FakeTaskCache,- session_factory: sessionmaker[Session],- bundles: list[WorkflowRunArchiveBundle] | None = None,+ bundles: list[WorkflowRunArchiveBundleRecord] | None = None,) -> WorkflowRunArchiveDownloadPreparer:source_storage = archive_storage or storagetarget_storage = download_storage or storageassert source_storage is not Noneassert target_storage is not None- with session_factory() as session:- session.add_all([_bundle()] if bundles is None else bundles)- session.commit()return WorkflowRunArchiveDownloadPreparer(- archive_storage=cast(ArchiveStorage, source_storage),- download_storage=cast(ArchiveStorage, target_storage),- cache=cast(WorkflowRunArchiveDownloadTaskCache, cache),- session_factory=cast(sessionmaker[Session], session_factory),+ archive_storage_provider=lambda: cast(ArchiveStorage, source_storage),+ bundles=FakeBundleQuery(tuple([_bundle()] if bundles is None else bundles)),+ cache=cache,+ download_storage_provider=lambda: cast(ArchiveStorage, target_storage),)@@ -178,9 +190,7 @@ def _parquet_bytes(records: list[dict[str, object]]) -> bytes:return buffer.getvalue()-def test_prepare_workflow_run_archive_download_builds_csv_zip_and_marks_ready(- sqlite_session_factory: sessionmaker[Session],-) -> None:+def test_prepare_workflow_run_archive_download_builds_csv_zip_and_marks_ready() -> None:bundle_refs = [(SHARD, "bundle-a"), (SHARD, "bundle-b")]task = _task(bundle_refs)first_bundle_payloads = {@@ -217,7 +227,6 @@ def test_prepare_workflow_run_archive_download_builds_csv_zip_and_marks_ready(archive_storage=archive_storage,download_storage=download_storage,cache=cache,- session_factory=sqlite_session_factory,bundles=[_bundle("bundle-a"), _bundle("bundle-b")],)@@ -244,9 +253,7 @@ def test_prepare_workflow_run_archive_download_builds_csv_zip_and_marks_ready(assert '"run-b","failed","safe"' in workflow_runs_csv-def test_prepare_workflow_run_archive_download_marks_failed_on_checksum_mismatch(- sqlite_session_factory: sessionmaker[Session],-) -> None:+def test_prepare_workflow_run_archive_download_marks_failed_on_checksum_mismatch() -> None:task = _task()table_payloads = {"workflow_runs": _parquet_bytes([{"id": "run-a", "status": "succeeded"}])}manifest_data = json.loads(_manifest_bytes(table_payloads).decode("utf-8"))@@ -258,7 +265,7 @@ def test_prepare_workflow_run_archive_download_marks_failed_on_checksum_mismatch})cache = FakeTaskCache(task)- preparer = _preparer(storage=storage, cache=cache, session_factory=sqlite_session_factory)+ preparer = _preparer(storage=storage, cache=cache)result = preparer.prepare(tenant_id=TENANT_ID, download_id=task.download_id)@@ -268,16 +275,33 @@ def test_prepare_workflow_run_archive_download_marks_failed_on_checksum_mismatchassert storage.put_objects == {}-def test_prepare_workflow_run_archive_download_skips_duplicate_worker(- sqlite_session_factory: sessionmaker[Session],-) -> None:+def test_prepare_workflow_run_archive_download_marks_failed_when_storage_is_unavailable() -> None:+ task = _task()+ cache = FakeTaskCache(task)+ archive_storage_provider = MagicMock(side_effect=RuntimeError("archive storage unavailable"))+ download_storage_provider = MagicMock()+ preparer = WorkflowRunArchiveDownloadPreparer(+ archive_storage_provider=archive_storage_provider,+ bundles=FakeBundleQuery((_bundle(),)),+ cache=cache,+ download_storage_provider=download_storage_provider,+ )++ result = preparer.prepare(tenant_id=TENANT_ID, download_id=task.download_id)++ assert result is not None+ assert result.status == WorkflowRunArchiveDownloadStatus.FAILED+ assert result.error == "archive storage unavailable"+ download_storage_provider.assert_not_called()+++def test_prepare_workflow_run_archive_download_skips_duplicate_worker() -> None:task = _task().model_copy(update={"celery_task_id": "celery-task-1"})storage = FakeArchiveStorage({})cache = FakeTaskCache(task)preparer = _preparer(storage=storage,cache=cache,- session_factory=sqlite_session_factory,bundles=[],)nested_results: list[WorkflowRunArchiveDownloadTask | None] = []@@ -298,15 +322,13 @@ def test_prepare_workflow_run_archive_download_skips_duplicate_worker(preparer._build_zip_payload.assert_called_once()-def test_failed_worker_cannot_overwrite_ready_task(- sqlite_session_factory: sessionmaker[Session],-) -> None:+def test_failed_worker_cannot_overwrite_ready_task() -> None:processing_task = _task().model_copy(update={"status": WorkflowRunArchiveDownloadStatus.PROCESSING, "celery_task_id": "celery-task-1"})ready_task = processing_task.model_copy(update={"status": WorkflowRunArchiveDownloadStatus.READY})cache = FakeTaskCache(ready_task)- preparer = _preparer(storage=FakeArchiveStorage({}), cache=cache, session_factory=sqlite_session_factory)+ preparer = _preparer(storage=FakeArchiveStorage({}), cache=cache)result = preparer._mark_failed(processing_task, error="late failure")diff --git a/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_task_cache.py b/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_task_cache.pyindex 79852028615..bafd609f8e8 100644--- a/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_task_cache.py+++ b/api/tests/unit_tests/services/retention/workflow_run/test_archive_download_task_cache.py@@ -3,15 +3,17 @@ from contextlib import nullcontextimport pytest-from services.retention.workflow_run.archive_download_task_cache import (+from services.retention.workflow_run.archive_download_task import (ARCHIVE_DOWNLOAD_FORMAT_VERSION,- ARCHIVE_DOWNLOAD_TASK_LOCK_TIMEOUT_SECONDS,WorkflowRunArchiveDownloadStatus,WorkflowRunArchiveDownloadTask,- WorkflowRunArchiveDownloadTaskCache,build_archive_download_id,build_pending_archive_download_task,)+from services.retention.workflow_run.archive_download_task_cache import (+ ARCHIVE_DOWNLOAD_TASK_LOCK_TIMEOUT_SECONDS,+ WorkflowRunArchiveDownloadTaskCache,+)class FakeRedis:@@ -119,7 +121,7 @@ def test_build_archive_download_id_rejects_empty_bundle_refs() -> None:def test_archive_download_task_cache_round_trips_with_ttl() -> None:redis = FakeRedis()- cache = WorkflowRunArchiveDownloadTaskCache(redis)+ cache = WorkflowRunArchiveDownloadTaskCache(redis=redis)task = build_pending_archive_download_task(tenant_id="tenant-1",requested_by="account-1",@@ -142,7 +144,7 @@ def test_archive_download_task_cache_round_trips_with_ttl() -> None:def test_archive_download_task_cache_uses_per_download_lock() -> None:redis = FakeRedis()- cache = WorkflowRunArchiveDownloadTaskCache(redis)+ cache = WorkflowRunArchiveDownloadTaskCache(redis=redis)with cache.lock(tenant_id="tenant-1", download_id="download-1"):pass@@ -158,7 +160,7 @@ def test_archive_download_task_cache_uses_per_download_lock() -> None:def test_archive_download_task_cache_delete_removes_entry() -> None:redis = FakeRedis()- cache = WorkflowRunArchiveDownloadTaskCache(redis)+ cache = WorkflowRunArchiveDownloadTaskCache(redis=redis)task = WorkflowRunArchiveDownloadTask(download_id="download-1",tenant_id="tenant-1",@@ -183,7 +185,7 @@ def test_archive_download_task_cache_delete_removes_entry() -> None:def test_archive_download_task_cache_ignores_malformed_json() -> None:redis = FakeRedis()- cache = WorkflowRunArchiveDownloadTaskCache(redis)+ cache = WorkflowRunArchiveDownloadTaskCache(redis=redis)redis.setex("workflow_run_archive_download:tenant-1:download-1", 3600, "{")assert cache.get(tenant_id="tenant-1", download_id="download-1") is Nonediff --git a/api/tests/unit_tests/services/retention/workflow_run/test_archive_log_service.py b/api/tests/unit_tests/services/retention/workflow_run/test_archive_log_service.pyindex 8817c71744f..2d0946e7232 100644--- a/api/tests/unit_tests/services/retention/workflow_run/test_archive_log_service.py+++ b/api/tests/unit_tests/services/retention/workflow_run/test_archive_log_service.py@@ -1,54 +1,75 @@import datetimefrom contextlib import nullcontext-from typing import castimport pytest-from sqlalchemy.orm import Session-from models.workflow import WorkflowRunArchiveBundle-from services.retention.workflow_run.archive_download_task_cache import (+from machinery.context import RequestContext+from services.retention.workflow_run.archive_download_task import (WorkflowRunArchiveDownloadStatus,WorkflowRunArchiveDownloadTask,- WorkflowRunArchiveDownloadTaskCache,build_archive_download_id,build_pending_archive_download_task,)from services.retention.workflow_run.archive_log_service import (- ArchiveDownloadTaskDispatcher,+ WorkflowRunArchiveBundleRecord,WorkflowRunArchiveDownloadNotReadyError,+ WorkflowRunArchiveDownloadTaskDispatcher,+ WorkflowRunArchiveDownloadTaskNotFoundError,WorkflowRunArchiveNotFoundError,- create_workflow_run_archive_download_task,- get_ready_workflow_run_archive_download_task,- list_workflow_run_archives,+ WorkflowRunArchiveService,)+_CONTEXT = RequestContext(+ request_id="request-1",+ trace_id="trace-1",+ account_id="account-1",+ active_workspace_id="tenant-1",+)+++class FakeBundleQuery:+ def __init__(self, records: tuple[WorkflowRunArchiveBundleRecord, ...] = ()) -> None:+ self.records = records++ def list_for_tenant(self, tenant_id: str) -> tuple[WorkflowRunArchiveBundleRecord, ...]:+ assert tenant_id == "tenant-1"+ return self.records++ def list_for_tenant_month(+ self,+ tenant_id: str,+ *,+ year: int,+ month: int,+ ) -> tuple[WorkflowRunArchiveBundleRecord, ...]:+ assert tenant_id == "tenant-1"+ return tuple(record for record in self.records if record.year == year and record.month == month)-class FakeTaskCache:- saved_task: WorkflowRunArchiveDownloadTask | None- existing_task: WorkflowRunArchiveDownloadTask | None- tasks_by_download_id: dict[str, WorkflowRunArchiveDownloadTask]+class FakeTaskStore:def __init__(self,+ tasks: dict[str, WorkflowRunArchiveDownloadTask] | None = None,*,- existing_task: WorkflowRunArchiveDownloadTask | None = None,- tasks_by_download_id: dict[str, WorkflowRunArchiveDownloadTask] | None = None,+ fail_reads: bool = False,) -> None:- self.saved_task = None- self.existing_task = existing_task- self.tasks_by_download_id = tasks_by_download_id or {}+ self.tasks = tasks or {}+ self.saved_tasks: list[WorkflowRunArchiveDownloadTask] = []+ self.fail_reads = fail_reads+ self.lock_calls: list[tuple[str, str]] = []def lock(self, *, tenant_id: str, download_id: str):+ self.lock_calls.append((tenant_id, download_id))return nullcontext()def get(self, *, tenant_id: str, download_id: str) -> WorkflowRunArchiveDownloadTask | None:- if self.tasks_by_download_id:- return self.tasks_by_download_id.get(download_id)- return self.existing_task+ if self.fail_reads:+ raise RuntimeError("cache unavailable")+ return self.tasks.get(download_id)def save(self, task: WorkflowRunArchiveDownloadTask) -> None:- self.saved_task = task- self.existing_task = task+ self.saved_tasks.append(task)+ self.tasks[task.download_id] = taskdef _bundle(@@ -61,9 +82,8 @@ def _bundle(workflow_run_count: int = 1,row_count: int = 9,archived_at: datetime.datetime | None = None,-) -> WorkflowRunArchiveBundle:- return WorkflowRunArchiveBundle(- tenant_id="tenant-1",+) -> WorkflowRunArchiveBundleRecord:+ return WorkflowRunArchiveBundleRecord(year=year,month=month,shard=shard,@@ -75,20 +95,68 @@ def _bundle()-def _fake_dispatcher(dispatched_tasks: list[WorkflowRunArchiveDownloadTask]) -> ArchiveDownloadTaskDispatcher:- def dispatch(- task: WorkflowRunArchiveDownloadTask,- cache: WorkflowRunArchiveDownloadTaskCache,- ) -> WorkflowRunArchiveDownloadTask:- dispatched_tasks.append(task)- return task-- return dispatch+def _service(+ *,+ records: tuple[WorkflowRunArchiveBundleRecord, ...] = (),+ tasks: FakeTaskStore | None = None,+ dispatched_tasks: list[WorkflowRunArchiveDownloadTask] | None = None,+ signed_urls: list[tuple[str, int, str]] | None = None,+ dispatcher: WorkflowRunArchiveDownloadTaskDispatcher | None = None,+) -> WorkflowRunArchiveService:+ task_store = tasks or FakeTaskStore()++ def dispatch(task: WorkflowRunArchiveDownloadTask) -> None:+ if dispatched_tasks is not None:+ dispatched_tasks.append(task)++ def sign(storage_key: str, *, expires_in: int, filename: str) -> str:+ if signed_urls is not None:+ signed_urls.append((storage_key, expires_in, filename))+ return "https://storage.example.com/archive.zip"++ return WorkflowRunArchiveService(+ bundles=FakeBundleQuery(records),+ tasks=task_store,+ dispatcher=dispatcher or dispatch,+ sign_download_url=sign,+ )-def test_list_workflow_run_archives_aggregates_month_rows(sqlite_session: Session) -> None:+def test_list_archives_aggregates_month_rows_and_includes_cached_task() -> None:latest = datetime.datetime(2026, 6, 25, 8, 0)previous = datetime.datetime(2026, 6, 24, 8, 0)+ records = (+ _bundle(+ year=2025,+ month=3,+ shard="00-of-01",+ bundle_id="bundle-a",+ workflow_run_count=40,+ row_count=360,+ archive_bytes=1024,+ archived_at=previous,+ ),+ _bundle(+ year=2025,+ month=3,+ shard="00-of-01",+ bundle_id="bundle-b",+ workflow_run_count=60,+ row_count=540,+ archive_bytes=3072,+ archived_at=latest,+ ),+ _bundle(+ year=2025,+ month=2,+ shard="00-of-01",+ bundle_id="bundle-c",+ workflow_run_count=20,+ row_count=180,+ archive_bytes=1024,+ archived_at=previous,+ ),+ )march_download_id = build_archive_download_id(tenant_id="tenant-1",year=2025,@@ -104,54 +172,9 @@ def test_list_workflow_run_archives_aggregates_month_rows(sqlite_session: Sessiobundle_refs=[("00-of-01", "bundle-a"), ("00-of-01", "bundle-b")],archive_bytes=4096,download_id=march_download_id,- ).model_copy(- update={- "status": WorkflowRunArchiveDownloadStatus.READY,- "file_name": "workflow-run-logs-2025-03.zip",- "storage_key": "workflow-run-archive-downloads/tenant-1/2025/03/download.zip",- "file_size_bytes": 8192,- }- )- cache = FakeTaskCache(tasks_by_download_id={march_download_id: ready_task})- sqlite_session.add_all(- [- _bundle(- year=2025,- month=3,- shard="00-of-01",- bundle_id="bundle-a",- workflow_run_count=40,- row_count=360,- archive_bytes=1024,- archived_at=previous,- ),- _bundle(- year=2025,- month=3,- shard="00-of-01",- bundle_id="bundle-b",- workflow_run_count=60,- row_count=540,- archive_bytes=3072,- archived_at=latest,- ),- _bundle(- year=2025,- month=2,- shard="00-of-01",- bundle_id="bundle-c",- workflow_run_count=20,- row_count=180,- archive_bytes=1024,- archived_at=previous,- ),- ]- )- sqlite_session.flush()+ ).model_copy(update={"status": WorkflowRunArchiveDownloadStatus.READY})- result = list_workflow_run_archives(- sqlite_session, "tenant-1", cache=cast(WorkflowRunArchiveDownloadTaskCache, cache)- )+ result = _service(records=records, tasks=FakeTaskStore({march_download_id: ready_task})).list_archives(_CONTEXT)assert result.summary.archived_month_count == 2assert result.summary.workflow_run_count == 120@@ -166,25 +189,27 @@ def test_list_workflow_run_archives_aggregates_month_rows(sqlite_session: Sessioassert result.months[1].download_task is None-def test_create_workflow_run_archive_download_task_creates_stable_pending_task(sqlite_session: Session) -> None:- sqlite_session.add_all(- [- _bundle(shard="01-of-02", bundle_id="bundle-b", archive_bytes=2048),- _bundle(shard="00-of-02", bundle_id="bundle-a", archive_bytes=1024),- ]+def test_list_archives_tolerates_unavailable_task_cache() -> None:+ result = _service(+ records=(_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024),),+ tasks=FakeTaskStore(fail_reads=True),+ ).list_archives(_CONTEXT)++ assert result.months[0].download_task is None+++def test_create_download_creates_stable_pending_task() -> None:+ records = (+ _bundle(shard="00-of-02", bundle_id="bundle-a", archive_bytes=1024),+ _bundle(shard="01-of-02", bundle_id="bundle-b", archive_bytes=2048),)- sqlite_session.flush()- cache = FakeTaskCache()+ task_store = FakeTaskStore()dispatched_tasks: list[WorkflowRunArchiveDownloadTask] = []- task = create_workflow_run_archive_download_task(- sqlite_session,- tenant_id="tenant-1",- requested_by="account-1",+ task = _service(records=records, tasks=task_store, dispatched_tasks=dispatched_tasks).create_download(+ _CONTEXT,year=2025,month=3,- cache=cast(WorkflowRunArchiveDownloadTaskCache, cache),- dispatcher=_fake_dispatcher(dispatched_tasks),)assert task.download_id == build_archive_download_id(@@ -200,48 +225,50 @@ def test_create_workflow_run_archive_download_task_creates_stable_pending_task(s("01-of-02", "bundle-b"),]assert task.archive_bytes == 3072- assert cache.saved_task == dispatched_tasks[0]+ assert task_store.saved_tasks == dispatched_tasksassert task.celery_task_id is not None-def test_create_workflow_run_archive_download_task_returns_existing_task_when_cache_key_exists(- sqlite_session: Session,-) -> None:- sqlite_session.add(_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024))- sqlite_session.flush()- existing_task = build_pending_archive_download_task(+def test_create_download_returns_existing_queued_task() -> None:+ records = (_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024),)+ download_id = build_archive_download_id(+ tenant_id="tenant-1",+ year=2025,+ month=3,+ bundle_refs=[("00-of-01", "bundle-a")],+ )+ existing = build_pending_archive_download_task(tenant_id="tenant-1",requested_by="account-1",year=2025,month=3,bundle_ids=["bundle-a"],archive_bytes=1024,- download_id="existing-download",+ download_id=download_id,).model_copy(update={"celery_task_id": "celery-task-1"})- cache = FakeTaskCache(existing_task=existing_task)+ task_store = FakeTaskStore({download_id: existing})dispatched_tasks: list[WorkflowRunArchiveDownloadTask] = []- task = create_workflow_run_archive_download_task(- sqlite_session,- tenant_id="tenant-1",- requested_by="account-1",+ result = _service(records=records, tasks=task_store, dispatched_tasks=dispatched_tasks).create_download(+ _CONTEXT,year=2025,month=3,- cache=cast(WorkflowRunArchiveDownloadTaskCache, cache),- dispatcher=_fake_dispatcher(dispatched_tasks),)- assert task == existing_task- assert cache.saved_task is None+ assert result == existing+ assert task_store.saved_tasks == []assert dispatched_tasks == []-def test_create_workflow_run_archive_download_task_retries_failed_cached_task(- sqlite_session: Session,-) -> None:- sqlite_session.add(_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024))- sqlite_session.flush()- existing_task = build_pending_archive_download_task(+def test_create_download_retries_failed_task() -> None:+ records = (_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024),)+ download_id = build_archive_download_id(+ tenant_id="tenant-1",+ year=2025,+ month=3,+ bundle_refs=[("00-of-01", "bundle-a")],+ )+ failed = build_pending_archive_download_task(tenant_id="tenant-1",requested_by="account-1",year=2025,@@ -249,107 +276,116 @@ def test_create_workflow_run_archive_download_task_retries_failed_cached_task(bundle_ids=["bundle-a"],bundle_refs=[("00-of-01", "bundle-a")],archive_bytes=1024,- download_id=build_archive_download_id(- tenant_id="tenant-1",- year=2025,- month=3,- bundle_refs=[("00-of-01", "bundle-a")],- ),+ download_id=download_id,).model_copy(update={"status": WorkflowRunArchiveDownloadStatus.FAILED, "error": "failed"})- cache = FakeTaskCache(existing_task=existing_task)+ task_store = FakeTaskStore({download_id: failed})dispatched_tasks: list[WorkflowRunArchiveDownloadTask] = []- task = create_workflow_run_archive_download_task(- sqlite_session,- tenant_id="tenant-1",- requested_by="account-1",+ task = _service(records=records, tasks=task_store, dispatched_tasks=dispatched_tasks).create_download(+ _CONTEXT,year=2025,month=3,- cache=cast(WorkflowRunArchiveDownloadTaskCache, cache),- dispatcher=_fake_dispatcher(dispatched_tasks),)assert task.status == WorkflowRunArchiveDownloadStatus.PENDINGassert task.error is Noneassert task.celery_task_id is not None- assert cache.saved_task == dispatched_tasks[0]+ assert task_store.saved_tasks == dispatched_tasks[email protected]("retry_failed_task", [False, True])-def test_create_workflow_run_archive_download_task_claims_dispatch_once(- retry_failed_task: bool, sqlite_session: Session-) -> None:- sqlite_session.add(_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024))- sqlite_session.flush()- download_id = build_archive_download_id(- tenant_id="tenant-1",+def test_create_download_marks_matching_claim_failed_when_dispatch_fails() -> None:+ records = (_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024),)+ task_store = FakeTaskStore()++ def dispatch(_task: WorkflowRunArchiveDownloadTask) -> None:+ raise RuntimeError("broker unavailable")++ result = _service(records=records, tasks=task_store, dispatcher=dispatch).create_download(+ _CONTEXT,year=2025,month=3,- bundle_refs=[("00-of-01", "bundle-a")],)- existing_task = None- if retry_failed_task:- existing_task = build_pending_archive_download_task(- tenant_id="tenant-1",- requested_by="account-1",- year=2025,- month=3,- bundle_ids=["bundle-a"],- bundle_refs=[("00-of-01", "bundle-a")],- archive_bytes=1024,- download_id=download_id,- ).model_copy(update={"status": WorkflowRunArchiveDownloadStatus.FAILED})- cache = FakeTaskCache(existing_task=existing_task)- dispatched_tasks: list[WorkflowRunArchiveDownloadTask] = []- concurrent_results: list[WorkflowRunArchiveDownloadTask] = []- def dispatch(- task: WorkflowRunArchiveDownloadTask,- task_cache: WorkflowRunArchiveDownloadTaskCache,- ) -> WorkflowRunArchiveDownloadTask:- dispatched_tasks.append(task)- concurrent_results.append(- create_workflow_run_archive_download_task(- sqlite_session,- tenant_id="tenant-1",- requested_by="account-2",- year=2025,- month=3,- cache=task_cache,- dispatcher=dispatch,- )+ assert result.status == WorkflowRunArchiveDownloadStatus.FAILED+ assert result.error == "Failed to enqueue archive download task."+ assert result.finished_at is not None+ assert len(task_store.saved_tasks) == 2+ assert task_store.saved_tasks[0].status == WorkflowRunArchiveDownloadStatus.PENDING+ assert task_store.saved_tasks[1] == result+ assert task_store.lock_calls == [+ ("tenant-1", result.download_id),+ ("tenant-1", result.download_id),+ ]+++def test_create_download_does_not_overwrite_newer_state_when_dispatch_fails() -> None:+ records = (_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024),)+ task_store = FakeTaskStore()++ def dispatch(task: WorkflowRunArchiveDownloadTask) -> None:+ task_store.tasks[task.download_id] = task.model_copy(+ update={+ "status": WorkflowRunArchiveDownloadStatus.READY,+ "storage_key": "downloads/archive.zip",+ "file_name": "archive.zip",+ })- return task+ raise RuntimeError("broker response lost")- result = create_workflow_run_archive_download_task(- sqlite_session,- tenant_id="tenant-1",- requested_by="account-1",+ result = _service(records=records, tasks=task_store, dispatcher=dispatch).create_download(+ _CONTEXT,year=2025,month=3,- cache=cast(WorkflowRunArchiveDownloadTaskCache, cache),+ )++ assert result.status == WorkflowRunArchiveDownloadStatus.READY+ assert len(task_store.saved_tasks) == 1+ assert task_store.lock_calls == [+ ("tenant-1", result.download_id),+ ("tenant-1", result.download_id),+ ]+++def test_create_download_claims_dispatch_once() -> None:+ records = (_bundle(shard="00-of-01", bundle_id="bundle-a", archive_bytes=1024),)+ task_store = FakeTaskStore()+ dispatched_tasks: list[WorkflowRunArchiveDownloadTask] = []+ concurrent_results: list[WorkflowRunArchiveDownloadTask] = []++ def dispatch(task: WorkflowRunArchiveDownloadTask) -> None:+ dispatched_tasks.append(task)+ concurrent_results.append(service.create_download(_CONTEXT, year=2025, month=3))++ def sign_download_url(_storage_key: str, *, expires_in: int, filename: str) -> str:+ assert expires_in > 0+ assert filename+ return "unused"++ service = WorkflowRunArchiveService(+ bundles=FakeBundleQuery(records),+ tasks=task_store,dispatcher=dispatch,+ sign_download_url=sign_download_url,)+ result = service.create_download(_CONTEXT, year=2025, month=3)+assert dispatched_tasks == [result]assert concurrent_results == [result]-def test_create_workflow_run_archive_download_task_rejects_missing_month(sqlite_session: Session) -> None:+def test_create_download_rejects_missing_month() -> None:with pytest.raises(WorkflowRunArchiveNotFoundError):- create_workflow_run_archive_download_task(- sqlite_session,- tenant_id="tenant-1",- requested_by="account-1",- year=2025,- month=3,- cache=cast(WorkflowRunArchiveDownloadTaskCache, FakeTaskCache()),- dispatcher=_fake_dispatcher([]),- )+ _service().create_download(_CONTEXT, year=2025, month=3)+++def test_get_download_rejects_missing_task() -> None:+ with pytest.raises(WorkflowRunArchiveDownloadTaskNotFoundError):+ _service().get_download(_CONTEXT, download_id="missing")-def test_get_ready_workflow_run_archive_download_task_requires_ready_file() -> None:- pending_task = build_pending_archive_download_task(+def test_get_download_url_requires_ready_file() -> None:+ pending = build_pending_archive_download_task(tenant_id="tenant-1",requested_by="account-1",year=2025,@@ -358,29 +394,37 @@ def test_get_ready_workflow_run_archive_download_task_requires_ready_file() -> Narchive_bytes=1024,download_id="download-1",)- cache = FakeTaskCache(existing_task=pending_task)with pytest.raises(WorkflowRunArchiveDownloadNotReadyError):- get_ready_workflow_run_archive_download_task(- tenant_id="tenant-1",- download_id="download-1",- cache=cast(WorkflowRunArchiveDownloadTaskCache, cache),+ _service(tasks=FakeTaskStore({pending.download_id: pending})).get_download_url(+ _CONTEXT,+ download_id=pending.download_id,)- ready_task = pending_task.model_copy(++def test_get_download_url_signs_ready_file_for_at_most_one_hour() -> None:+ ready = build_pending_archive_download_task(+ tenant_id="tenant-1",+ requested_by="account-1",+ year=2025,+ month=3,+ bundle_ids=["bundle-a"],+ archive_bytes=1024,+ download_id="download-1",+ ttl_seconds=7200,+ ).model_copy(update={"status": WorkflowRunArchiveDownloadStatus.READY,"storage_key": "downloads/download-1.zip","file_name": "workflow-run-logs-2025-03.zip",})- cache = FakeTaskCache(existing_task=ready_task)+ signed_urls: list[tuple[str, int, str]] = []- assert (- get_ready_workflow_run_archive_download_task(- tenant_id="tenant-1",- download_id="download-1",- cache=cast(WorkflowRunArchiveDownloadTaskCache, cache),- )- == ready_task- )+ url = _service(+ tasks=FakeTaskStore({ready.download_id: ready}),+ signed_urls=signed_urls,+ ).get_download_url(_CONTEXT, download_id=ready.download_id)++ assert url == "https://storage.example.com/archive.zip"+ assert signed_urls == [("downloads/download-1.zip", 3600, "workflow-run-logs-2025-03.zip")]