Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
feat: support clean workspace related resource (#43803)
wangxiaolei7a02240701aa
diff --git a/api/.importlinter b/api/.importlinterindex 07609d175bb..336ea2ed16c 100644--- a/api/.importlinter+++ b/api/.importlinter@@ -234,6 +234,7 @@ ignore_imports =repositories.knowledge.dataset_api_key_repository -> services.knowledge.api_key_servicerepositories.knowledge.dataset_repository -> services.knowledge.segments.applicationrepositories.knowledge.dataset_repository -> services.knowledge.dataset_access+ repositories.knowledge.dataset_repository -> services.knowledge.entities.datasetsrepositories.knowledge.dataset_repository -> services.knowledge.indexing.estimaterepositories.knowledge.dataset_repository -> services.knowledge.resource_scoperepositories.knowledge.document_repository -> services.knowledge.document_syncdiff --git a/api/controllers/inner_api/workspace/workspace.py b/api/controllers/inner_api/workspace/workspace.pyindex 1414eefe279..2ecc2d7797b 100644--- a/api/controllers/inner_api/workspace/workspace.py+++ b/api/controllers/inner_api/workspace/workspace.py@@ -1,3 +1,5 @@+from uuid import UUID+from flask_restx import Resourcefrom pydantic import BaseModel@@ -144,3 +146,15 @@ class EnterpriseWorkspaceMember(Resource):"role": member.role,},}+++@inner_api_ns.route("/enterprise/workspace/<uuid:workspace_id>/resources")+class EnterpriseWorkspaceResourceDeletion(Resource):+ @setup_required+ @inner_api_only+ @inner_api_ns.doc("delete_enterprise_workspace_resources")+ @inner_api_ns.doc(description="Schedule App, Dataset and Agent cleanup; retain the workspace and memberships")+ @inner_api_ns.doc(responses={202: "Cleanup accepted", 401: "Invalid API key"})+ def delete(self, workspace_id: UUID):+ application_services().workspaces.resource_cleanup.cleanup(str(workspace_id))+ return {"message": "workspace resource cleanup accepted."}, 202diff --git a/api/extensions/application_services/workspace.py b/api/extensions/application_services/workspace.pyindex fb78b00bbb9..5a7019688cf 100644--- a/api/extensions/application_services/workspace.py+++ b/api/extensions/application_services/workspace.py@@ -2,8 +2,13 @@from dataclasses import dataclass+from sqlalchemy.orm import Session, sessionmaker+from extensions.ext_redis import RedisClientWrapperfrom repositories.account.repository import SQLAlchemyAccountRepository+from repositories.app.console_repository import ConsoleAppRepository+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository+from repositories.workspace.resource_cleanup_repository import WorkspaceResourceCleanupRepositoryfrom repositories.workspace.workspace_repository import WorkspaceRepositoryfrom services.account.adapters import RedisInvitationTokenStorefrom services.file_service import FileService@@ -27,6 +32,7 @@ from services.workspace.member_service import (WorkspaceOwnerTransferService,)from services.workspace.provisioning_service import WorkspaceProvisioningService+from services.workspace.resource_cleanup_service import WorkspaceResourceCleanupServicefrom services.workspace.service import WorkspaceQueryService, WorkspaceService@@ -40,6 +46,7 @@ class WorkspaceServices:owner_transfer: WorkspaceOwnerTransferServiceinvitations: WorkspaceInvitationServiceidentity: WorkspaceIdentityGateway+ resource_cleanup: WorkspaceResourceCleanupServicedef build_workspace_membership_services(@@ -66,6 +73,7 @@ def build_workspace_membership_services(def build_workspace_services(*,+ database_client: sessionmaker[Session],workspaces: WorkspaceRepository,accounts: SQLAlchemyAccountRepository,files: FileService,@@ -101,4 +109,10 @@ def build_workspace_services(invitations=WorkspaceInvitationGateway(tokens=invitation_tokens, redis=redis),),identity=WorkspaceIdentityGateway(workspaces=workspaces),+ resource_cleanup=WorkspaceResourceCleanupService(+ session_factory=database_client,+ apps=ConsoleAppRepository(session_factory=database_client),+ datasets=SQLAlchemyDatasetRepository(session_factory=database_client),+ agent_resources=WorkspaceResourceCleanupRepository(session_factory=database_client),+ ),)diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.pyindex fe28573388a..ab8e2e9dc8f 100644--- a/api/extensions/ext_application_services.py+++ b/api/extensions/ext_application_services.py@@ -504,6 +504,7 @@ def build_application_services(activation_accounts=activation_accounts,)workspace_services = build_workspace_services(+ database_client=database_client,workspaces=workspace_repository,accounts=accounts,files=file_service,diff --git a/api/extensions/storage/aliyun_oss_storage.py b/api/extensions/storage/aliyun_oss_storage.pyindex 5e96e7dbab1..6938a469fcb 100644--- a/api/extensions/storage/aliyun_oss_storage.py+++ b/api/extensions/storage/aliyun_oss_storage.py@@ -60,5 +60,12 @@ class AliyunOssStorage(BaseStorage):def delete(self, filename: str):self.client.delete_object(self.__wrapper_folder_filename(filename))+ @override+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:+ prefix = self.__wrapper_folder_filename(path)+ folder_prefix = self.folder.rstrip("/") + "/" if self.folder else ""+ keys = (obj.key.removeprefix(folder_prefix) for obj in aliyun_s3.ObjectIterator(self.client, prefix=prefix))+ return self._filter_scan_keys(keys, files, directories)+def __wrapper_folder_filename(self, filename: str) -> str:return posixpath.join(self.folder, filename) if self.folder else filenamediff --git a/api/extensions/storage/aws_s3_storage.py b/api/extensions/storage/aws_s3_storage.pyindex 1c0de64286e..83cc20d1ef7 100644--- a/api/extensions/storage/aws_s3_storage.py+++ b/api/extensions/storage/aws_s3_storage.py@@ -53,6 +53,19 @@ class AwsS3Storage(BaseStorage):def save(self, filename, data):self.client.put_object(Bucket=self.bucket_name, Key=filename, Body=data)+ @override+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:+ if not files and not directories:+ raise ValueError("At least one of files or directories must be True")+ keys: list[str] = []+ paginator = self.client.get_paginator("list_objects_v2")+ for page in paginator.paginate(Bucket=self.bucket_name, Prefix=path):+ for obj in page.get("Contents", []):+ key = obj["Key"]+ if (key.endswith("/") and directories) or (not key.endswith("/") and files):+ keys.append(key)+ return keys+@overridedef load_once(self, filename: str) -> bytes:try:diff --git a/api/extensions/storage/azure_blob_storage.py b/api/extensions/storage/azure_blob_storage.pyindex a17f61eb9bf..df325225c33 100644--- a/api/extensions/storage/azure_blob_storage.py+++ b/api/extensions/storage/azure_blob_storage.py@@ -91,6 +91,15 @@ class AzureBlobStorage(BaseStorage):blob_container = client.get_container_client(container=self.bucket_name)blob_container.delete_blob(filename)+ @override+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:+ if not self.bucket_name:+ raise ValueError("Azure bucket name is not configured.")+ container = self._sync_client().get_container_client(container=self.bucket_name)+ # ItemPaged follows continuation tokens automatically.+ keys = (blob.name for blob in container.list_blobs(name_starts_with=path))+ return self._filter_scan_keys(keys, files, directories)+def _sync_client(self):if self.account_key == "managedidentity":return BlobServiceClient(account_url=self.account_url, credential=self.credential) # type: ignorediff --git a/api/extensions/storage/baidu_obs_storage.py b/api/extensions/storage/baidu_obs_storage.pyindex 9bb9bb31292..a4f579e551f 100644--- a/api/extensions/storage/baidu_obs_storage.py+++ b/api/extensions/storage/baidu_obs_storage.py@@ -62,3 +62,17 @@ class BaiduObsStorage(BaseStorage):@overridedef delete(self, filename: str):self.client.delete_object(bucket_name=self.bucket_name, key=filename)++ @override+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:+ keys: list[str] = []+ marker = ""+ while True:+ page = self.client.list_objects(bucket_name=self.bucket_name, prefix=path, marker=marker, max_keys=1000)+ keys.extend(obj.key for obj in page.contents)+ if not page.is_truncated:+ break+ if not page.next_marker or page.next_marker == marker:+ raise RuntimeError("BOS object listing did not advance its pagination marker")+ marker = page.next_marker+ return self._filter_scan_keys(keys, files, directories)diff --git a/api/extensions/storage/base_storage.py b/api/extensions/storage/base_storage.pyindex d9c3ff6c4c3..cb5004c9a4e 100644--- a/api/extensions/storage/base_storage.py+++ b/api/extensions/storage/base_storage.py@@ -1,7 +1,7 @@"""Abstract interface for file storage implementations."""from abc import ABC, abstractmethod-from collections.abc import Generator+from collections.abc import Generator, Iterableclass BaseStorage(ABC):@@ -48,3 +48,10 @@ class BaseStorage(ABC):If a storage backend doesn't support scanning, it will raise NotImplementedError."""raise NotImplementedError("This storage backend doesn't support scanning")++ @staticmethod+ def _filter_scan_keys(keys: Iterable[str], files: bool, directories: bool) -> list[str]:+ """Filter recursive object listings; directory entries end with '/'."""+ if not files and not directories:+ raise ValueError("At least one of files or directories must be True")+ return [key for key in keys if (key.endswith("/") and directories) or (not key.endswith("/") and files)]diff --git a/api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.py b/api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.pyindex f09ccd2a28f..59d8ffa373e 100644--- a/api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.py+++ b/api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.py@@ -439,7 +439,9 @@ class ClickZettaVolumeStorage(BaseStorage):return existsexcept Exception as e:logger.warning("Error checking file existence for %s: %s", filename, e)- return False+ # Deletion relies on this result. A failed query is not proof that+ # the file is absent; retain references so callers can retry.+ raise@overridedef delete(self, filename: str):@@ -483,19 +485,27 @@ class ClickZettaVolumeStorage(BaseStorage):"""Scan files and directories in ClickZetta Volume.Args:- path: Path to scan (dataset_id for table volumes)+ path: Path prefix to scan, or a bare dataset_id for a table-volume rootfiles: Include files in resultsdirectories: Include directories in resultsReturns:List of file/directory paths"""+ if not files and not directories:+ raise ValueError("At least one of files or directories must be True")+ requested_path = pathtry:- # For table volumes, path is treated as dataset_id+ # Match save/load/delete: the first path component selects the+ # table (or special USER VOLUME), and the remainder is its file path.dataset_id = None+ logical_root = ""if self._config.volume_type == "table":- dataset_id = path- path = "" # Root of the table volume+ root, separator, relative_path = path.partition("/")+ dataset_id = root.removeprefix(self._config.table_prefix)+ path = relative_path+ if separator:+ logical_root = root + "/"volume_prefix = self._get_volume_sql_prefix(dataset_id)@@ -524,6 +534,10 @@ class ClickZettaVolumeStorage(BaseStorage):if volume_prefix == "USER VOLUME" and file_path.startswith(dify_prefix_with_slash):file_path = file_path[len(dify_prefix_with_slash) :] # Remove prefix+ file_path = logical_root + file_path+ if logical_root and not file_path.startswith(requested_path):+ continue+if files and not file_path.endswith("/") or directories and file_path.endswith("/"):result.append(file_path)@@ -531,5 +545,5 @@ class ClickZettaVolumeStorage(BaseStorage):return resultexcept Exception:- logger.exception("Error scanning path %s", path)- return []+ logger.exception("Error scanning path %s", requested_path)+ raisediff --git a/api/extensions/storage/google_cloud_storage.py b/api/extensions/storage/google_cloud_storage.pyindex 83f199c45a4..18c1af3cde6 100644--- a/api/extensions/storage/google_cloud_storage.py+++ b/api/extensions/storage/google_cloud_storage.py@@ -73,3 +73,8 @@ class GoogleCloudStorage(BaseStorage):def delete(self, filename: str):bucket = self.client.get_bucket(self.bucket_name)bucket.delete_blob(filename)++ @override+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:+ keys = (blob.name for blob in self.client.list_blobs(self.bucket_name, prefix=path))+ return self._filter_scan_keys(keys, files, directories)diff --git a/api/extensions/storage/huawei_obs_storage.py b/api/extensions/storage/huawei_obs_storage.pyindex c840b170861..3c90baa32dc 100644--- a/api/extensions/storage/huawei_obs_storage.py+++ b/api/extensions/storage/huawei_obs_storage.py@@ -1,5 +1,5 @@from collections.abc import Generator-from typing import override+from typing import Any, overridefrom obs import ObsClient@@ -51,7 +51,31 @@ class HuaweiObsStorage(BaseStorage):@overridedef delete(self, filename: str):- self.client.deleteObject(bucketName=self.bucket_name, objectKey=filename)+ response: Any = self.client.deleteObject(bucketName=self.bucket_name, objectKey=filename)+ if response.status == 404 and response.errorCode == "NoSuchKey":+ raise FileNotFoundError(filename)+ if response.status >= 300:+ raise RuntimeError(f"OBS object deletion failed: status={response.status}, code={response.errorCode}")++ @override+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:+ keys: list[str] = []+ marker = ""+ while True:+ # The SDK's decorator annotations do not describe its response model.+ response: Any = self.client.listObjects(+ bucketName=self.bucket_name, prefix=path, marker=marker, max_keys=1000+ )+ if response.status >= 300:+ raise RuntimeError(f"OBS object listing failed: status={response.status}, code={response.errorCode}")+ page = response.body+ keys.extend(obj.key for obj in page.contents or [])+ if not page.is_truncated:+ break+ if not page.next_marker or page.next_marker == marker:+ raise RuntimeError("OBS object listing did not advance its pagination marker")+ marker = page.next_marker+ return self._filter_scan_keys(keys, files, directories)def _get_meta(self, filename):res = self.client.getObjectMetadata(bucketName=self.bucket_name, objectKey=filename)diff --git a/api/extensions/storage/oracle_oci_storage.py b/api/extensions/storage/oracle_oci_storage.pyindex 4097bb0cb09..4f535d3cb18 100644--- a/api/extensions/storage/oracle_oci_storage.py+++ b/api/extensions/storage/oracle_oci_storage.py@@ -64,3 +64,9 @@ class OracleOCIStorage(BaseStorage):@overridedef delete(self, filename: str):self.client.delete_object(Bucket=self.bucket_name, Key=filename)++ @override+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:+ pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=self.bucket_name, Prefix=path)+ keys = (obj["Key"] for page in pages for obj in page.get("Contents", []))+ return self._filter_scan_keys(keys, files, directories)diff --git a/api/extensions/storage/supabase_storage.py b/api/extensions/storage/supabase_storage.pyindex f47153ce370..50c14ac2901 100644--- a/api/extensions/storage/supabase_storage.py+++ b/api/extensions/storage/supabase_storage.py@@ -61,6 +61,41 @@ class SupabaseStorage(BaseStorage):def delete(self, filename: str):self.client.storage.from_(self.bucket_name).remove([filename])+ @override+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:+ bucket = self.client.storage.from_(self.bucket_name)+ # Supabase lists only immediate children, with offset pagination.+ root = path.rstrip("/") if path.endswith("/") else path.rpartition("/")[0]+ pending = [root]+ keys: list[str] = []+ page_size = 100+ while pending:+ folder = pending.pop()+ offset = 0+ while True:+ page = bucket.list(+ path=folder,+ options={+ "limit": page_size,+ "offset": offset,+ "sortBy": {"column": "name", "order": "asc"},+ },+ )+ for entry in page:+ key = f"{folder}/{entry['name']}" if folder else entry["name"]+ if entry.get("id") is None:+ directory_key = key + "/"+ if directory_key.startswith(path) or path.startswith(directory_key):+ pending.append(key)+ if directory_key.startswith(path):+ keys.append(directory_key)+ elif key.startswith(path):+ keys.append(key)+ if len(page) < page_size:+ break+ offset += page_size+ return self._filter_scan_keys(keys, files, directories)+def bucket_exists(self):buckets = self.client.storage.list_buckets()return any(bucket.name == self.bucket_name for bucket in buckets)diff --git a/api/extensions/storage/tencent_cos_storage.py b/api/extensions/storage/tencent_cos_storage.pyindex da34a6e0071..5727a782ebb 100644--- a/api/extensions/storage/tencent_cos_storage.py+++ b/api/extensions/storage/tencent_cos_storage.py@@ -56,3 +56,18 @@ class TencentCosStorage(BaseStorage):@overridedef delete(self, filename: str):self.client.delete_object(Bucket=self.bucket_name, Key=filename)++ @override+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:+ keys: list[str] = []+ marker = ""+ while True:+ page = self.client.list_objects(Bucket=self.bucket_name, Prefix=path, Marker=marker, MaxKeys=1000)+ keys.extend(obj["Key"] for obj in page.get("Contents", []))+ if str(page.get("IsTruncated", "false")).lower() != "true":+ break+ next_marker = page.get("NextMarker")+ if not next_marker or next_marker == marker:+ raise RuntimeError("COS object listing did not advance its pagination marker")+ marker = next_marker+ return self._filter_scan_keys(keys, files, directories)diff --git a/api/extensions/storage/volcengine_tos_storage.py b/api/extensions/storage/volcengine_tos_storage.pyindex 709f198712f..e0a03a06c0b 100644--- a/api/extensions/storage/volcengine_tos_storage.py+++ b/api/extensions/storage/volcengine_tos_storage.py@@ -71,3 +71,19 @@ class VolcengineTosStorage(BaseStorage):if not self.bucket_name:returnself.client.delete_object(bucket=self.bucket_name, key=filename)++ @override+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:+ if not self.bucket_name:+ raise ValueError("VOLCENGINE_TOS_BUCKET_NAME is not set")+ keys: list[str] = []+ marker = ""+ while True:+ page = self.client.list_objects(bucket=self.bucket_name, prefix=path, marker=marker, max_keys=1000)+ keys.extend(obj.key for obj in page.contents)+ if not page.is_truncated:+ break+ if not page.next_marker or page.next_marker == marker:+ raise RuntimeError("TOS object listing did not advance its pagination marker")+ marker = page.next_marker+ return self._filter_scan_keys(keys, files, directories)diff --git a/api/repositories/app/console_repository.py b/api/repositories/app/console_repository.pyindex 9e3dc2fff18..65c1f0c9101 100644--- a/api/repositories/app/console_repository.py+++ b/api/repositories/app/console_repository.py@@ -144,6 +144,11 @@ class ConsoleAppRepository(ConsoleApps, DiscoveryApps, RelatedApps):config = load_app_parameter_config(app, session=session) if include_config else Nonereturn AppDescription(app_summary(app), bool(app.enable_api), config)+ def list_workspace_app_ids(self, tenant_id: str) -> list[str]:+ """List every App row in the workspace, ignoring console visibility and status."""+ with self._session_factory() as session:+ return list(session.scalars(select(App.id).where(App.tenant_id == tenant_id)).all())+@overridedef find_visible_app(self, app_id: str, tenant_id: str) -> AppSummary | None:with self._session_factory() as session:diff --git a/api/repositories/knowledge/dataset_repository.py b/api/repositories/knowledge/dataset_repository.pyindex 83698ae635a..f93015d9a54 100644--- a/api/repositories/knowledge/dataset_repository.py+++ b/api/repositories/knowledge/dataset_repository.py@@ -1,11 +1,12 @@"""SQLAlchemy repository for tenant-owned dataset state."""-from sqlalchemy import and_, exists, literal, select+from sqlalchemy import and_, delete, exists, literal, selectfrom sqlalchemy.orm import Session, sessionmakerfrom models.dataset import Dataset, DatasetPermission, Document-from repositories.knowledge.dataset_read_repository import pipeline_dataset_query+from repositories.knowledge.dataset_read_repository import get_dataset_doc_form, pipeline_dataset_queryfrom services.knowledge.dataset_access import DatasetAccessRecord, DatasetAccessSnapshot+from services.knowledge.entities.datasets import DatasetCleanupRecordfrom services.knowledge.indexing.estimate import DatasetEstimateRecordfrom services.knowledge.resource_scope import DatasetReffrom services.knowledge.segments.application import SegmentDatasetRecord, SegmentDocumentRecord, SegmentScope@@ -66,6 +67,32 @@ class SQLAlchemyDatasetRepository:with self._session_factory() as session:return session.scalar(pipeline_dataset_query(workspace_id, pipeline_id).with_only_columns(Dataset.id))+ def list_workspace_dataset_ids(self, tenant_id: str) -> list[str]:+ with self._session_factory() as session:+ return list(session.scalars(select(Dataset.id).where(Dataset.tenant_id == tenant_id)).all())++ def get_cleanup_record(self, dataset_ref: DatasetRef) -> DatasetCleanupRecord | None:+ """None means the dataset is already gone."""+ with self._session_factory() as session:+ dataset = _get_dataset(session, dataset_ref)+ if dataset is None:+ return None+ return DatasetCleanupRecord(+ id=dataset.id,+ workspace_id=dataset.tenant_id,+ indexing_technique=dataset.indexing_technique,+ index_struct=dataset.index_struct,+ collection_binding_id=dataset.collection_binding_id,+ doc_form=get_dataset_doc_form(dataset, session=session),+ pipeline_id=dataset.pipeline_id,+ )++ def delete(self, dataset_ref: DatasetRef) -> None:+ with self._session_factory.begin() as session:+ session.execute(+ delete(Dataset).where(Dataset.id == dataset_ref.dataset_id, Dataset.tenant_id == dataset_ref.tenant_id)+ )+def get_estimate_record(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord | None:with self._session_factory() as session:dataset = _get_dataset(session, dataset_ref)diff --git a/api/repositories/workspace/resource_cleanup_repository.py b/api/repositories/workspace/resource_cleanup_repository.pynew file mode 100644index 00000000000..2501a2fdefa--- /dev/null+++ b/api/repositories/workspace/resource_cleanup_repository.py@@ -0,0 +1,60 @@+"""Tenant-scoped Agent persistence for workspace resource cleanup.++Each method owns its session and transaction; the caller orchestrates the order.++TODO: Move these methods to Agent, Agent Workspace and Skill repositories once+those domains own sessionmaker-based persistence, then remove this module.+"""++from dataclasses import dataclass++from sqlalchemy import delete, select+from sqlalchemy.orm import Session, sessionmaker++from libs.datetime_utils import naive_utc_now+from models.agent import Agent, AgentStatus, AgentWorkspace, AgentWorkspaceBinding, WorkflowAgentNodeBinding+from models.skill import AgentSkillBinding, AgentSkillBindingSnapshot+++@dataclass(frozen=True, slots=True)+class ArchivedAgents:+ agent_ids: list[str]+ backing_app_ids: list[str]+++class WorkspaceResourceCleanupRepository:+ def __init__(self, session_factory: sessionmaker[Session]) -> None:+ self._session_factory = session_factory++ def archive_agents(self, tenant_id: str) -> ArchivedAgents:+ """Archive every tenant Agent, including archived and backing-App-less ones, for repeated cleanup."""+ with self._session_factory.begin() as session:+ agents = session.scalars(select(Agent).where(Agent.tenant_id == tenant_id)).all()+ now = naive_utc_now()+ for agent in agents:+ if agent.status != AgentStatus.ARCHIVED:+ agent.status = AgentStatus.ARCHIVED+ agent.archived_at = now+ agent.updated_at = now+ return ArchivedAgents(+ agent_ids=[agent.id for agent in agents],+ backing_app_ids=sorted({agent.backing_app_id for agent in agents if agent.backing_app_id}),+ )++ def list_agent_workspace_ids(self, tenant_id: str) -> list[str]:+ with self._session_factory() as session:+ return list(session.scalars(select(AgentWorkspace.id).where(AgentWorkspace.tenant_id == tenant_id)).all())++ def list_agent_workspace_binding_ids(self, tenant_id: str) -> list[str]:+ with self._session_factory() as session:+ return list(+ session.scalars(+ select(AgentWorkspaceBinding.id).where(AgentWorkspaceBinding.tenant_id == tenant_id)+ ).all()+ )++ def delete_agent_bindings(self, tenant_id: str) -> None:+ with self._session_factory.begin() as session:+ session.execute(delete(WorkflowAgentNodeBinding).where(WorkflowAgentNodeBinding.tenant_id == tenant_id))+ session.execute(delete(AgentSkillBinding).where(AgentSkillBinding.tenant_id == tenant_id))+ session.execute(delete(AgentSkillBindingSnapshot).where(AgentSkillBindingSnapshot.tenant_id == tenant_id))diff --git a/api/services/agent/home_snapshot_service.py b/api/services/agent/home_snapshot_service.pyindex a1e4ff5d74d..707a3796028 100644--- a/api/services/agent/home_snapshot_service.py+++ b/api/services/agent/home_snapshot_service.py@@ -136,6 +136,21 @@ class AgentHomeSnapshotService:row.retired_at = nowreturn [row.id for row in rows]+ @classmethod+ def retire_all_for_tenant(cls, *, session: Session, tenant_id: str) -> list[str]:+ """Retire every tenant Home Snapshot in the caller's transaction.++ Includes orphan snapshots whose Agent row is already gone. Returns all+ matched ids, including already-retired ones, so repeated collection converges.+ """+ rows = session.scalars(select(AgentHomeSnapshot).where(AgentHomeSnapshot.tenant_id == tenant_id)).all()+ now = naive_utc_now()+ for row in rows:+ if row.status == AgentWorkingResourceStatus.ACTIVE:+ row.status = AgentWorkingResourceStatus.RETIRED+ row.retired_at = now+ return [row.id for row in rows]+@classmethoddef collect_retired_home_snapshot(cls, *, tenant_id: str, home_snapshot_id: str) -> None:with session_factory.create_session() as session:diff --git a/api/services/app_service.py b/api/services/app_service.pyindex 979929e4066..bfbc32ca50b 100644--- a/api/services/app_service.py+++ b/api/services/app_service.py@@ -4,6 +4,7 @@ from collections.abc import Callable, Mapping, Sequencefrom copy import deepcopyfrom dataclasses import dataclassfrom datetime import datetime+from types import EllipsisTypefrom typing import Any, NotRequired, TypedDict, castimport sqlalchemy as sa@@ -777,9 +778,11 @@ class AppService:return app- def delete_app(self, app: App, *, session: Session) -> None:+ def delete_app(self, app: App, *, session: Session, account_id: str | None | EllipsisType = ...) -> None:"""Delete an App and commit the passed session.+ Omit account_id to use the logged-in actor; pass None for system cleanup.+The transaction releases all of a Workflow App's binding owners acrossdraft and published versions, archives a backing Roster Agent, retiresits resources, and deletes the App. Deleting a Roster Agent's backing@@ -831,7 +834,8 @@ class AppService:AgentSkillBinding.agent_id.in_(agent_ids_to_unbind),))- account_id = current_user.id if current_user else None+ if account_id is ...:+ account_id = current_user.id if current_user else Noneif backing_agent is not None:now = naive_utc_now()backing_agent.status = AgentStatus.ARCHIVEDdiff --git a/api/services/app_storage_cleanup_service.py b/api/services/app_storage_cleanup_service.pynew file mode 100644index 00000000000..e764768fb73--- /dev/null+++ b/api/services/app_storage_cleanup_service.py@@ -0,0 +1,214 @@+"""Remove application-owned storage before deleting the durable database references.++Uploads owned by accounts, subscription-wide trigger requests, tenant telemetry,+and manually named exports do not have exclusive application ownership. They are+deliberately retained rather than deleting another resource's data.+"""++import logging++from sqlalchemy import delete, exists, or_, select+from sqlalchemy.orm import InstrumentedAttribute+from sqlalchemy.sql.selectable import Exists++from core.db.session_factory import session_factory+from extensions.ext_storage import storage+from models import Conversation, EndUser, Message, MessageFile, UploadFile+from models.enums import CreatorUserRole+from models.human_input import HumanInputFormUploadFile+from models.tools import ToolFile+from models.workflow import (+ Workflow,+ WorkflowDraftVariableFile,+ WorkflowNodeExecutionOffload,+ WorkflowPause,+ WorkflowPauseReason,+ WorkflowRun,+)++logger = logging.getLogger(__name__)+_BATCH_SIZE = 500+++def _delete_object(key: str) -> None:+ """Allow redelivery after a storage delete succeeded but the DB commit failed."""+ try:+ storage.delete(key)+ except FileNotFoundError:+ return+ except Exception:+ # Some backends return False from exists() for permission failures too;+ # never use that as evidence that a failed deletion actually succeeded.+ logger.exception("Application storage cleanup failed for key=%s", key)+ raise+++def cleanup_app_storage(tenant_id: str, app_id: str) -> None:+ """Clean exclusively owned files with external I/O outside DB transactions.++ Call before deleting runs, conversations, messages, or end users. A storage+ failure propagates to the app deletion task so its existing retry can still+ resolve the keys. Each successful batch removes its references atomically.+ """+ # The App row is already deleted by the caller. Validate trace ownership+ # against remaining tenant-scoped resources before removing those resources.+ with session_factory.create_session() as session:+ owns_traces = any(+ session.scalar(select(exists().where(model.tenant_id == tenant_id, model.app_id == app_id)))+ for model in (Workflow, WorkflowRun, EndUser)+ )+ _cleanup_pauses(tenant_id, app_id)+ _cleanup_node_offloads(tenant_id, app_id)+ _cleanup_tool_files(tenant_id, app_id)+ _cleanup_end_user_uploads(tenant_id, app_id)+ if owns_traces:+ _cleanup_traces(app_id)+++def _cleanup_pauses(tenant_id: str, app_id: str) -> None:+ run_ids = select(WorkflowRun.id).where(WorkflowRun.tenant_id == tenant_id, WorkflowRun.app_id == app_id)+ while True:+ with session_factory.create_session() as session:+ rows = session.execute(+ select(WorkflowPause.id, WorkflowPause.state_object_key)+ .where(WorkflowPause.workflow_run_id.in_(run_ids))+ .limit(_BATCH_SIZE)+ ).all()+ if not rows:+ return+ for _, key in rows:+ _delete_object(key)+ ids = [row.id for row in rows]+ with session_factory.create_session() as session, session.begin():+ session.execute(delete(WorkflowPauseReason).where(WorkflowPauseReason.pause_id.in_(ids)))+ session.execute(delete(WorkflowPause).where(WorkflowPause.id.in_(ids)))+++def _cleanup_node_offloads(tenant_id: str, app_id: str) -> None:+ while True:+ with session_factory.create_session() as session:+ rows = session.execute(+ select(WorkflowNodeExecutionOffload.id, WorkflowNodeExecutionOffload.file_id, UploadFile.key)+ .outerjoin(+ UploadFile,+ (UploadFile.id == WorkflowNodeExecutionOffload.file_id) & (UploadFile.tenant_id == tenant_id),+ )+ .where(+ WorkflowNodeExecutionOffload.tenant_id == tenant_id,+ WorkflowNodeExecutionOffload.app_id == app_id,+ )+ .limit(_BATCH_SIZE)+ ).all()+ file_ids = [row.file_id for row in rows]+ shared = set(+ session.scalars(+ select(WorkflowNodeExecutionOffload.file_id).where(+ WorkflowNodeExecutionOffload.file_id.in_(file_ids),+ or_(+ WorkflowNodeExecutionOffload.tenant_id != tenant_id,+ WorkflowNodeExecutionOffload.app_id != app_id,+ ),+ )+ )+ )+ if not rows:+ return+ for row in rows:+ if row.key is not None and row.file_id not in shared:+ _delete_object(row.key)+ with session_factory.create_session() as session, session.begin():+ session.execute(+ delete(WorkflowNodeExecutionOffload).where(+ WorkflowNodeExecutionOffload.id.in_([row.id for row in rows]),+ WorkflowNodeExecutionOffload.tenant_id == tenant_id,+ WorkflowNodeExecutionOffload.app_id == app_id,+ )+ )+ session.execute(+ delete(UploadFile).where(+ UploadFile.tenant_id == tenant_id,+ UploadFile.id.in_(+ [row.file_id for row in rows if row.key is not None and row.file_id not in shared]+ ),+ )+ )+++def _other_app_message_reference(file_column: InstrumentedAttribute[str], app_id: str) -> Exists:+ # Include orphan message references conservatively: their ownership cannot be+ # established, so they must not authorize destruction of the underlying file.+ return exists(+ select(MessageFile.id)+ .outerjoin(Message, Message.id == MessageFile.message_id)+ .where(MessageFile.upload_file_id == file_column, or_(Message.app_id != app_id, Message.id.is_(None)))+ )+++def _cleanup_tool_files(tenant_id: str, app_id: str) -> None:+ # Conversation has no tenant column; its app owner plus ToolFile.tenant_id+ # together constrain the complete ownership chain.+ conversation_ids = select(Conversation.id).where(Conversation.app_id == app_id)+ # A file with no conversation can still be owned by an app end user. Account+ # files without a conversation are reusable workspace assets and are retained.+ end_user_ids = select(EndUser.id).where(EndUser.tenant_id == tenant_id, EndUser.app_id == app_id)+ while True:+ with session_factory.create_session() as session:+ rows = session.execute(+ select(ToolFile.id, ToolFile.file_key)+ .where(+ ToolFile.tenant_id == tenant_id,+ or_(+ ToolFile.conversation_id.in_(conversation_ids),+ ToolFile.conversation_id.is_(None) & ToolFile.user_id.in_(end_user_ids),+ ),+ ~_other_app_message_reference(ToolFile.id, app_id),+ )+ .limit(_BATCH_SIZE)+ ).all()+ if not rows:+ return+ for _, key in rows:+ _delete_object(key)+ with session_factory.create_session() as session, session.begin():+ session.execute(+ delete(ToolFile).where(ToolFile.tenant_id == tenant_id, ToolFile.id.in_([r.id for r in rows]))+ )+++def _cleanup_end_user_uploads(tenant_id: str, app_id: str) -> None:+ end_user_ids = select(EndUser.id).where(EndUser.tenant_id == tenant_id, EndUser.app_id == app_id)+ while True:+ with session_factory.create_session() as session:+ rows = session.execute(+ select(UploadFile.id, UploadFile.key)+ .where(+ UploadFile.tenant_id == tenant_id,+ UploadFile.created_by_role == CreatorUserRole.END_USER,+ UploadFile.created_by.in_(end_user_ids),+ ~_other_app_message_reference(UploadFile.id, app_id),+ ~exists().where(WorkflowNodeExecutionOffload.file_id == UploadFile.id),+ ~exists().where(WorkflowDraftVariableFile.upload_file_id == UploadFile.id),+ ~exists().where(HumanInputFormUploadFile.upload_file_id == UploadFile.id),+ )+ .limit(_BATCH_SIZE)+ ).all()+ if not rows:+ return+ for _, key in rows:+ _delete_object(key)+ with session_factory.create_session() as session, session.begin():+ session.execute(+ delete(UploadFile).where(UploadFile.tenant_id == tenant_id, UploadFile.id.in_([row.id for row in rows]))+ )+++def _cleanup_traces(app_id: str) -> None:+ prefix = f"ops_trace/{app_id}/"+ try:+ keys = storage.scan(prefix)+ except FileNotFoundError:+ return+ for key in keys:+ if not key.startswith(prefix):+ raise ValueError(f"Trace storage listing returned a key outside application prefix: {key}")+ _delete_object(key)diff --git a/api/services/dataset_storage_cleanup_service.py b/api/services/dataset_storage_cleanup_service.pynew file mode 100644index 00000000000..6043f73781e--- /dev/null+++ b/api/services/dataset_storage_cleanup_service.py@@ -0,0 +1,68 @@+"""Remove crawl caches whose remaining document references belong to a deleted dataset."""++import json+import logging++from sqlalchemy import select++from core.db.session_factory import session_factory+from extensions.ext_storage import storage+from models.dataset import Document++logger = logging.getLogger(__name__)+++def _crawl_cache_key(data_source_info: str | None) -> str | None:+ try:+ info = json.loads(data_source_info or "{}")+ except (ValueError, TypeError):+ return None+ if not isinstance(info, dict) or info.get("provider") != "firecrawl":+ return None+ job_id = info.get("job_id")+ if not isinstance(job_id, str) or not job_id or any(char in job_id for char in "/\\\x00"):+ return None+ return f"website_files/{job_id}.txt"+++def cleanup_dataset_crawl_cache(tenant_id: str, dataset_id: str) -> None:+ """Delete only unshared Firecrawl caches, before deleting their document references.++ Crawl keys have no tenant namespace, so references in other tenants also prevent deletion.+ Storage failures propagate, preserving the database references for a later retry.+ """+ with session_factory.create_session() as session:+ candidates = {+ key+ for info in session.scalars(+ select(Document.data_source_info).where(+ Document.tenant_id == tenant_id,+ Document.dataset_id == dataset_id,+ Document.data_source_type == "website_crawl",+ )+ )+ if (key := _crawl_cache_key(info)) is not None+ }+ if not candidates:+ return+ references = session.execute(+ select(Document.tenant_id, Document.dataset_id, Document.data_source_info)+ .where(Document.data_source_type == "website_crawl")+ .execution_options(yield_per=500)+ )+ for owner_tenant, owner_dataset, info in references:+ if owner_tenant != tenant_id or owner_dataset != dataset_id:+ key = _crawl_cache_key(info)+ if key is not None:+ candidates.discard(key)+ # Finish the read transaction before accessing object storage.+ for key in sorted(candidates):+ try:+ storage.delete(key)+ except FileNotFoundError:+ pass+ except Exception:+ logger.exception(+ "Failed to delete dataset crawl cache tenant=%s dataset=%s key=%s", tenant_id, dataset_id, key+ )+ raisediff --git a/api/services/knowledge/entities/datasets.py b/api/services/knowledge/entities/datasets.pyindex 21d5d659b99..a769dacac84 100644--- a/api/services/knowledge/entities/datasets.py+++ b/api/services/knowledge/entities/datasets.py@@ -6,7 +6,7 @@ objects retain their stored keys; this boundary does not apply response defaultsfrom collections.abc import Mapping, Sequencefrom datetime import datetime-from typing import NotRequired, TypedDict+from typing import NamedTuple, NotRequired, TypedDictfrom pydantic import JsonValue@@ -67,3 +67,16 @@ class DatasetPage(TypedDict):total: intpage: intlimit: int+++class DatasetCleanupRecord(NamedTuple):+ """Dataset state needed to schedule index cleanup after the row is deleted."""++ id: str+ workspace_id: str+ indexing_technique: str | None+ index_struct: str | None+ collection_binding_id: str | None+ # None means neither the dataset nor any of its documents declares a form.+ doc_form: str | None+ pipeline_id: str | Nonediff --git a/api/services/workspace/resource_cleanup_service.py b/api/services/workspace/resource_cleanup_service.pynew file mode 100644index 00000000000..06bb47fe93b--- /dev/null+++ b/api/services/workspace/resource_cleanup_service.py@@ -0,0 +1,133 @@+"""Schedule existing resource deletion lifecycles for an enterprise workspace.++This service does not delete the Tenant or its memberships, and does not claim+that publishing child tasks means their physical cleanup has completed.+"""++import logging++from sqlalchemy.orm import Session, sessionmaker++from constants.resource_access_token import ResourceAccessTokenResourceType+from extensions.application_services.resource_access_token import build_resource_access_token_cleanup_service+from repositories.app.console_repository import ConsoleAppRepository+from repositories.knowledge import dataset_api_key_bindings+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository+from repositories.workspace.resource_cleanup_repository import WorkspaceResourceCleanupRepository+from services.agent.home_snapshot_service import AgentHomeSnapshotService+from services.agent.workspace_service import AgentWorkspaceService+from services.app_service import AppService+from services.knowledge.resource_scope import DatasetRef+from tasks.clean_dataset_task import clean_dataset_task+from tasks.collect_agent_resources_task import enqueue_agent_resource_collection+from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task++logger = logging.getLogger(__name__)+++class WorkspaceResourceCleanupService:+ def __init__(+ self,+ *,+ session_factory: sessionmaker[Session],+ apps: ConsoleAppRepository,+ datasets: SQLAlchemyDatasetRepository,+ agent_resources: WorkspaceResourceCleanupRepository,+ ) -> None:+ # The session factory only serves existing services that require a+ # caller-owned session; drop it once they own their persistence.+ self._session_factory = session_factory+ self._apps = apps+ self._datasets = datasets+ self._agent_resources = agent_resources++ def cleanup(self, workspace_id: str) -> None:+ """Attempt all resource groups; propagate failures to the caller."""+ failures: list[str] = []+ app_ids = self._apps.list_workspace_app_ids(workspace_id)+ dataset_ids = self._datasets.list_workspace_dataset_ids(workspace_id)++ for app_id in app_ids:+ try:+ self._delete_app(workspace_id, app_id)+ except Exception:+ logger.exception("Workspace App deletion failed: %s:%s", workspace_id, app_id)+ failures.append(f"app:{app_id}")++ for dataset_id in dataset_ids:+ try:+ self._delete_dataset(workspace_id, dataset_id)+ except Exception:+ logger.exception("Workspace Dataset deletion failed: %s:%s", workspace_id, dataset_id)+ failures.append(f"dataset:{dataset_id}")++ try:+ self._retire_agents(workspace_id)+ except Exception:+ logger.exception("Workspace Agent retirement failed: %s", workspace_id)+ failures.append("agents")+ if failures:+ raise RuntimeError(f"Workspace resource deletion failed for {workspace_id}: {', '.join(failures)}")++ def _delete_app(self, workspace_id: str, app_id: str) -> None:+ with self._session_factory() as session:+ app = AppService.get_app_in_workspace(tenant_id=workspace_id, app_id=app_id, session=session)+ if app is not None:+ AppService().delete_app(app, session=session, account_id=None)++ def _delete_dataset(self, workspace_id: str, dataset_id: str) -> None:+ # Inner-API authentication authorizes the whole tenant. No member actor+ # is available here; preserve the normal deletion effects without RBAC.+ dataset_ref = DatasetRef(workspace_id, dataset_id)+ dataset = self._datasets.get_cleanup_record(dataset_ref)+ if dataset is None:+ return+ # Unlike the member deletion signal, also schedule incomplete or+ # empty datasets so their relational data is not skipped.+ clean_dataset_task.delay(+ dataset.id,+ workspace_id,+ dataset.indexing_technique,+ dataset.index_struct,+ dataset.collection_binding_id,+ dataset.doc_form or "text_model",+ dataset.pipeline_id,+ )+ with self._session_factory.begin() as session:+ dataset_api_key_bindings.delete_keys_scoped_only_to(session, dataset.id)+ build_resource_access_token_cleanup_service(session=session).delete_resource_relations(+ tenant_id=workspace_id,+ resource_type=ResourceAccessTokenResourceType.KNOWLEDGE,+ resource_id=dataset.id,+ )+ # Delete the row last so a failed step is retried from the remaining Dataset.+ self._datasets.delete(dataset_ref)++ def _retire_agents(self, workspace_id: str) -> None:+ # Sweep independently of App records: archived, workflow-only and+ # backing-App-less Agents must also be collected on repeated requests.+ archived = self._agent_resources.archive_agents(workspace_id)+ workspace_ids = self._agent_resources.list_agent_workspace_ids(workspace_id)+ binding_ids = self._agent_resources.list_agent_workspace_binding_ids(workspace_id)+ with self._session_factory.begin() as session:+ snapshot_ids = AgentHomeSnapshotService.retire_all_for_tenant(session=session, tenant_id=workspace_id)+ # Both retire calls skip resources that are no longer active.+ for resource_id in workspace_ids:+ AgentWorkspaceService.retire_workspace(+ session=session, tenant_id=workspace_id, workspace_id=resource_id+ )+ for binding_id in binding_ids:+ AgentWorkspaceService.retire_binding(session=session, tenant_id=workspace_id, binding_id=binding_id)+ self._agent_resources.delete_agent_bindings(workspace_id)++ # Backing App records can already be gone; re-publish their idempotent+ # related-data cleanup before collecting the archived Agent aggregates.+ for app_id in archived.backing_app_ids:+ remove_app_and_related_data_task.delay(tenant_id=workspace_id, app_id=app_id)+ enqueue_agent_resource_collection(+ tenant_id=workspace_id,+ binding_ids=binding_ids,+ workspace_ids=workspace_ids,+ home_snapshot_ids=snapshot_ids,+ purge_agent_ids=archived.agent_ids,+ )diff --git a/api/tasks/clean_dataset_task.py b/api/tasks/clean_dataset_task.pyindex 5a4baabefbc..357d30d6d4b 100644--- a/api/tasks/clean_dataset_task.py+++ b/api/tasks/clean_dataset_task.py@@ -24,6 +24,7 @@ from models.dataset import ()from models.model import UploadFilefrom models.workflow import Workflow+from services.dataset_storage_cleanup_service import cleanup_dataset_crawl_cachefrom tasks.refresh_billing_vector_space_task import schedule_billing_vector_space_refreshlogger = logging.getLogger(__name__)@@ -34,9 +35,9 @@ logger = logging.getLogger(__name__)def clean_dataset_task(dataset_id: str,tenant_id: str,- indexing_technique: str,- index_struct: str,- collection_binding_id: str,+ indexing_technique: str | None,+ index_struct: str | None,+ collection_binding_id: str | None,doc_form: str,pipeline_id: str | None = None,):@@ -55,6 +56,8 @@ def clean_dataset_task(start_at = time.perf_counter()vector_cleanup_succeeded = False+ cleanup_dataset_crawl_cache(tenant_id, dataset_id)+with session_factory.create_session() as session:try:dataset = Dataset(diff --git a/api/tasks/remove_app_and_related_data_task.py b/api/tasks/remove_app_and_related_data_task.pyindex f0df4411423..afba447f678 100644--- a/api/tasks/remove_app_and_related_data_task.py+++ b/api/tasks/remove_app_and_related_data_task.py@@ -54,6 +54,7 @@ from models.workflow import ()from repositories.factory import DifyAPIRepositoryFactoryfrom services.api_token_service import ApiTokenCache+from services.app_storage_cleanup_service import cleanup_app_storagelogger = logging.getLogger(__name__)@@ -63,6 +64,9 @@ def remove_app_and_related_data_task(self, tenant_id: str, app_id: str):logger.info(click.style(f"Start deleting app and related data: {tenant_id}:{app_id}", fg="green"))start_at = time.perf_counter()try:+ # Preserve ownership records until storage cleanup succeeds, so retries+ # can still locate files. External I/O runs outside DB transactions.+ cleanup_app_storage(tenant_id, app_id)# Delete related data_delete_app_model_configs(tenant_id, app_id)_delete_app_site(tenant_id, app_id)diff --git a/api/tests/unit_tests/controllers/inner_api/workspace/test_resource_deletion.py b/api/tests/unit_tests/controllers/inner_api/workspace/test_resource_deletion.pynew file mode 100644index 00000000000..4c33d4e2d9e--- /dev/null+++ b/api/tests/unit_tests/controllers/inner_api/workspace/test_resource_deletion.py@@ -0,0 +1,58 @@+from collections.abc import Callable+from inspect import unwrap+from types import SimpleNamespace+from unittest.mock import Mock+from uuid import UUID++import pytest+from flask import Flask++from controllers.inner_api.workspace import workspace as controller+from controllers.inner_api.wraps import InnerApiUnauthorizedError++WORKSPACE_ID = UUID("0dd1d570-cf7b-4bab-9c7f-84ebe6c36985")+++def test_delete_calls_cleanup_with_the_workspace_id(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:+ service = Mock()+ monkeypatch.setattr(+ controller,+ "application_services",+ lambda: SimpleNamespace(workspaces=SimpleNamespace(resource_cleanup=service)),+ )+ with app.test_request_context(method="DELETE"):+ result = unwrap(controller.EnterpriseWorkspaceResourceDeletion.delete)(+ controller.EnterpriseWorkspaceResourceDeletion(), WORKSPACE_ID+ )+ assert result == ({"message": "workspace resource cleanup accepted."}, 202)+ service.cleanup.assert_called_once_with(str(WORKSPACE_ID))+++def test_delete_requires_inner_api_key(+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None]+) -> None:+ config_overrides(INNER_API=True, INNER_API_KEY="secret", DEPLOYMENT_EDITION="CLOUD")+ service = Mock()+ monkeypatch.setattr(+ controller,+ "application_services",+ lambda: SimpleNamespace(workspaces=SimpleNamespace(resource_cleanup=service)),+ )+ with app.test_request_context(method="DELETE"):+ with pytest.raises(InnerApiUnauthorizedError):+ controller.EnterpriseWorkspaceResourceDeletion().delete(WORKSPACE_ID)+ service.cleanup.assert_not_called()+++def test_cleanup_failure_does_not_return_accepted(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:+ service = Mock()+ service.cleanup.side_effect = RuntimeError("broker unavailable")+ monkeypatch.setattr(+ controller,+ "application_services",+ lambda: SimpleNamespace(workspaces=SimpleNamespace(resource_cleanup=service)),+ )+ with app.test_request_context(method="DELETE"), pytest.raises(RuntimeError, match="broker unavailable"):+ unwrap(controller.EnterpriseWorkspaceResourceDeletion.delete)(+ controller.EnterpriseWorkspaceResourceDeletion(), WORKSPACE_ID+ )diff --git a/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py b/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.pyindex 878e5771458..64c5682339b 100644--- a/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py+++ b/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py@@ -45,6 +45,7 @@ def provisioning(monkeypatch: pytest.MonkeyPatch) -> Mock:owner_transfer=Mock(),invitations=Mock(),identity=Mock(),+ resource_cleanup=Mock(),)monkeypatch.setattr(controller, "application_services", lambda: Services(services))return provisioningdiff --git a/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py b/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.pyindex acb3c7c9449..3d79410059b 100644--- a/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py+++ b/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py@@ -3,6 +3,22 @@ from unittest.mock import MagicMockfrom extensions.storage.aws_s3_storage import AwsS3Storage+def test_scan_reads_all_pages_and_filters_directory_markers() -> None:+ storage = AwsS3Storage.__new__(AwsS3Storage)+ storage.bucket_name = "test-bucket"+ storage.client = MagicMock()+ storage.client.get_paginator.return_value.paginate.return_value = [+ {"Contents": [{"Key": "ops_trace/app/"}, {"Key": "ops_trace/app/one.json"}]},+ {"Contents": [{"Key": "ops_trace/app/two.json"}]},+ dict[str, str](),+ ]+ assert storage.scan("ops_trace/app/") == ["ops_trace/app/one.json", "ops_trace/app/two.json"]+ storage.client.get_paginator.assert_called_once_with("list_objects_v2")+ storage.client.get_paginator.return_value.paginate.assert_called_once_with(+ Bucket="test-bucket", Prefix="ops_trace/app/"+ )++def test_generate_presigned_url() -> None:storage = AwsS3Storage.__new__(AwsS3Storage)storage.bucket_name = "test-bucket"diff --git a/api/tests/unit_tests/extensions/storage/test_clickzetta_volume_paths.py b/api/tests/unit_tests/extensions/storage/test_clickzetta_volume_paths.pyindex e98e066d2dd..bc03add50cb 100644--- a/api/tests/unit_tests/extensions/storage/test_clickzetta_volume_paths.py+++ b/api/tests/unit_tests/extensions/storage/test_clickzetta_volume_paths.py@@ -1,5 +1,7 @@"""Unit coverage for ClickZetta configuration and path generation."""+from unittest.mock import MagicMock+import pytestfrom extensions.storage.clickzetta_volume.clickzetta_volume_storage import (@@ -62,3 +64,51 @@ def test_sql_prefix_generation() -> None:storage._config.volume_type = "external"storage._config.volume_name = "my_external_volume"assert storage._get_volume_sql_prefix() == "VOLUME my_external_volume"+++def test_scan_table_prefix_matches_file_storage_path(monkeypatch: pytest.MonkeyPatch) -> None:+ storage = _table_storage()+ execute = MagicMock(return_value=[("app/trace.json",), ("another-app/trace.json",)])+ monkeypatch.setattr(storage, "_execute_sql", execute)+ assert storage.scan("ops_trace/app/") == ["ops_trace/app/trace.json"]+ execute.assert_called_once_with("LIST TABLE VOLUME test_dataset_ops_trace SUBDIRECTORY 'app/'", fetch=True)+++def test_scan_special_table_prefix_restores_logical_root(monkeypatch: pytest.MonkeyPatch) -> None:+ storage = _table_storage()+ execute = MagicMock(return_value=[("dify_km/tenant/file.txt",)])+ monkeypatch.setattr(storage, "_execute_sql", execute)+ assert storage.scan("upload_files/tenant/") == ["upload_files/tenant/file.txt"]+ execute.assert_called_once_with("LIST USER VOLUME SUBDIRECTORY 'dify_km/tenant/'", fetch=True)+++def test_scan_preserves_bare_dataset_root_behavior(monkeypatch: pytest.MonkeyPatch) -> None:+ storage = _table_storage()+ execute = MagicMock(return_value=[("file.txt",)])+ monkeypatch.setattr(storage, "_execute_sql", execute)+ assert storage.scan("12345") == ["file.txt"]+ execute.assert_called_once_with("LIST TABLE VOLUME test_dataset_12345", fetch=True)+++def test_scan_propagates_backend_failure(monkeypatch: pytest.MonkeyPatch) -> None:+ storage = _table_storage()+ monkeypatch.setattr(storage, "_execute_sql", MagicMock(side_effect=RuntimeError("backend unavailable")))+ with pytest.raises(RuntimeError, match="backend unavailable"):+ storage.scan("ops_trace/app/")+++def test_delete_propagates_failed_existence_check(monkeypatch: pytest.MonkeyPatch) -> None:+ storage = _table_storage()+ execute = MagicMock(side_effect=RuntimeError("backend unavailable"))+ monkeypatch.setattr(storage, "_execute_sql", execute)+ with pytest.raises(RuntimeError, match="backend unavailable"):+ storage.delete("ops_trace/app/trace.json")+ assert execute.call_count == 1+++def test_delete_absent_file_remains_idempotent(monkeypatch: pytest.MonkeyPatch) -> None:+ storage = _table_storage()+ execute = MagicMock(return_value=[])+ monkeypatch.setattr(storage, "_execute_sql", execute)+ storage.delete("ops_trace/app/trace.json")+ assert execute.call_count == 1diff --git a/api/tests/unit_tests/services/test_app_storage_cleanup_service.py b/api/tests/unit_tests/services/test_app_storage_cleanup_service.pynew file mode 100644index 00000000000..a9da4d71da8--- /dev/null+++ b/api/tests/unit_tests/services/test_app_storage_cleanup_service.py@@ -0,0 +1,403 @@+from datetime import UTC, datetime+from types import SimpleNamespace+from unittest.mock import MagicMock++import pytest+from botocore.exceptions import ClientError+from sqlalchemy.orm import Session++from extensions.storage.clickzetta_volume.clickzetta_volume_storage import ClickZettaVolumeStorage+from extensions.storage.huawei_obs_storage import HuaweiObsStorage+from extensions.storage.storage_type import StorageType+from graphon.entities.pause_reason import PauseReasonType+from graphon.enums import WorkflowExecutionStatus+from graphon.file import FileTransferMethod, FileType+from models import WorkflowType+from models.enums import (+ ConversationFromSource,+ CreatorUserRole,+ EndUserType,+ ExecutionOffLoadType,+ WorkflowRunTriggeredFrom,+)+from models.model import AppMode, Conversation, EndUser, Message, MessageFile, UploadFile+from models.tools import ToolFile+from models.workflow import WorkflowNodeExecutionOffload, WorkflowPause, WorkflowPauseReason, WorkflowRun+from services import app_storage_cleanup_service as cleanup+++def _offload(session: Session, app_id: str, key: str, tenant_id: str = "tenant-1") -> tuple[str, str]:+ file = UploadFile(+ tenant_id=tenant_id,+ storage_type=StorageType.LOCAL,+ key=key,+ name="node_execution_outputs.json",+ size=2,+ extension="json",+ mime_type="application/json",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="account-1",+ created_at=datetime.now(UTC),+ used=True,+ )+ session.add(file)+ session.flush()+ offload = WorkflowNodeExecutionOffload(+ tenant_id=tenant_id,+ app_id=app_id,+ node_execution_id=None,+ type_=ExecutionOffLoadType.OUTPUTS,+ file_id=file.id,+ )+ session.add(offload)+ session.commit()+ return file.id, offload.id+++def storage(monkeypatch: pytest.MonkeyPatch) -> MagicMock:+ mock = MagicMock()+ mock.scan.return_value = list[str]()+ monkeypatch.setattr(cleanup, "storage", mock)+ return mock+++def test_cleanup_removes_owned_offloads_and_preserves_other_owners(sqlite_session: Session, storage: MagicMock) -> None:+ owned = _offload(sqlite_session, "app-1", "owned.json")+ other_app = _offload(sqlite_session, "app-2", "other-app.json")+ other_tenant = _offload(sqlite_session, "app-1", "other-tenant.json", "tenant-2")++ cleanup.cleanup_app_storage("tenant-1", "app-1")+ cleanup.cleanup_app_storage("tenant-1", "app-1")++ storage.delete.assert_called_once_with("owned.json")+ sqlite_session.expire_all()+ assert sqlite_session.get(UploadFile, owned[0]) is None+ assert sqlite_session.get(WorkflowNodeExecutionOffload, owned[1]) is None+ for file_id, offload_id in (other_app, other_tenant):+ assert sqlite_session.get(UploadFile, file_id) is not None+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None+++def test_storage_failure_retains_references_for_retry(sqlite_session: Session, storage: MagicMock) -> None:+ file_id, offload_id = _offload(sqlite_session, "app-1", "owned.json")+ storage.delete.side_effect = RuntimeError("storage unavailable")+ storage.exists.return_value = True++ with pytest.raises(RuntimeError, match="storage unavailable"):+ cleanup.cleanup_app_storage("tenant-1", "app-1")++ sqlite_session.expire_all()+ assert sqlite_session.get(UploadFile, file_id) is not None+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None++ storage.delete.side_effect = None+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ sqlite_session.expire_all()+ assert sqlite_session.get(UploadFile, file_id) is None+++def test_missing_object_allows_redelivery(sqlite_session: Session, storage: MagicMock) -> None:+ file_id, _ = _offload(sqlite_session, "app-1", "owned.json")+ storage.delete.side_effect = FileNotFoundError()+ storage.exists.return_value = False++ cleanup.cleanup_app_storage("tenant-1", "app-1")++ sqlite_session.expire_all()+ assert sqlite_session.get(UploadFile, file_id) is None+++def test_access_denied_retains_references_even_if_exists_returns_false(+ sqlite_session: Session, storage: MagicMock+) -> None:+ file_id, offload_id = _offload(sqlite_session, "app-1", "owned.json")+ storage.delete.side_effect = ClientError({"Error": {"Code": "AccessDenied"}}, "DeleteObject")+ storage.exists.return_value = False+ with pytest.raises(ClientError):+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ sqlite_session.expire_all()+ assert sqlite_session.get(UploadFile, file_id) is not None+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None+ storage.exists.assert_not_called()+++def test_obs_delete_failure_preserves_retry_references(+ sqlite_session: Session, monkeypatch: pytest.MonkeyPatch+) -> None:+ file_id, offload_id = _offload(sqlite_session, "app-1", "owned.json")+ backend = HuaweiObsStorage.__new__(HuaweiObsStorage)+ backend.bucket_name = "bucket"+ backend.client = MagicMock()+ backend.client.deleteObject.return_value = SimpleNamespace(status=403, errorCode="AccessDenied")+ monkeypatch.setattr(cleanup, "storage", backend)+ with pytest.raises(RuntimeError, match="403"):+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ sqlite_session.expire_all()+ assert sqlite_session.get(UploadFile, file_id) is not None+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None+++def test_clickzetta_failed_existence_check_preserves_retry_references(+ sqlite_session: Session,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ file_id, offload_id = _offload(sqlite_session, "app-1", "owned.json")+ backend = ClickZettaVolumeStorage.__new__(ClickZettaVolumeStorage)+ monkeypatch.setattr(backend, "_config", SimpleNamespace(volume_type="user", dify_prefix="dify"), raising=False)+ monkeypatch.setattr(backend, "_execute_sql", MagicMock(side_effect=RuntimeError("backend unavailable")))+ monkeypatch.setattr(cleanup, "storage", backend)+ with pytest.raises(RuntimeError, match="backend unavailable"):+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ sqlite_session.expire_all()+ assert sqlite_session.get(UploadFile, file_id) is not None+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None+++def test_shared_offload_file_is_retained(sqlite_session: Session, storage: MagicMock) -> None:+ file_id, offload_id = _offload(sqlite_session, "app-1", "shared.json")+ shared = WorkflowNodeExecutionOffload(+ tenant_id="tenant-1",+ app_id="app-2",+ node_execution_id=None,+ type_=ExecutionOffLoadType.INPUTS,+ file_id=file_id,+ )+ sqlite_session.add(shared)+ sqlite_session.commit()++ cleanup.cleanup_app_storage("tenant-1", "app-1")++ storage.delete.assert_not_called()+ sqlite_session.expire_all()+ assert sqlite_session.get(UploadFile, file_id) is not None+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is None+ assert sqlite_session.get(WorkflowNodeExecutionOffload, shared.id) is not None+++def test_trace_cleanup_stays_within_app_prefix(sqlite_session: Session, storage: MagicMock) -> None:+ _end_user(sqlite_session)+ storage.scan.return_value = ["ops_trace/app-1/trace.json"]+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ storage.delete.assert_called_once_with("ops_trace/app-1/trace.json")++ storage.reset_mock()+ storage.scan.return_value = ["ops_trace/app-2/trace.json"]+ with pytest.raises(ValueError, match="outside application prefix"):+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ storage.delete.assert_not_called()+++def _end_user(session: Session) -> EndUser:+ user = EndUser(tenant_id="tenant-1", app_id="app-1", type=EndUserType.SERVICE_API, session_id="session")+ session.add(user)+ session.commit()+ return user+++def test_wrong_tenant_does_not_scan_traces(sqlite_session: Session, storage: MagicMock) -> None:+ _end_user(sqlite_session)+ cleanup.cleanup_app_storage("tenant-2", "app-1")+ storage.scan.assert_not_called()++[email protected]("fails", [False, True])+def test_pause_files_and_reasons_are_removed_only_after_storage_success(+ sqlite_session: Session, storage: MagicMock, fails: bool+) -> None:+ run = WorkflowRun(+ tenant_id="tenant-1",+ app_id="app-1",+ workflow_id="workflow-1",+ type=WorkflowType.CHAT,+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,+ version="1",+ status=WorkflowExecutionStatus.PAUSED,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="account-1",+ )+ sqlite_session.add(run)+ sqlite_session.flush()+ pause = WorkflowPause(workflow_id="workflow-1", workflow_run_id=run.id, state_object_key="pause.json")+ sqlite_session.add(pause)+ sqlite_session.flush()+ reason = WorkflowPauseReason(pause_id=pause.id, type_=PauseReasonType.HITL_REQUIRED)+ sqlite_session.add(reason)+ sqlite_session.commit()+ pause_id, reason_id = pause.id, reason.id+ if fails:+ storage.delete.side_effect = RuntimeError("unavailable")+ storage.exists.return_value = True+ with pytest.raises(RuntimeError):+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ else:+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ sqlite_session.expire_all()+ assert (sqlite_session.get(WorkflowPause, pause_id) is not None) == fails+ assert (sqlite_session.get(WorkflowPauseReason, reason_id) is not None) == fails+ assert sqlite_session.get(WorkflowRun, run.id) is not None++[email protected]("fails", [False, True])+def test_end_user_files_are_cleaned_before_owner_records(+ sqlite_session: Session, storage: MagicMock, fails: bool+) -> None:+ user = _end_user(sqlite_session)+ tool = ToolFile(+ tenant_id="tenant-1",+ user_id=user.id,+ conversation_id=None,+ file_key="tools/owned.json",+ mimetype="application/json",+ )+ sqlite_session.add(tool)+ file_id, offload_id = _offload(sqlite_session, "app-2", "upload_files/owned.txt")+ offload = sqlite_session.get(WorkflowNodeExecutionOffload, offload_id)+ assert offload is not None+ sqlite_session.delete(offload)+ upload = sqlite_session.get(UploadFile, file_id)+ assert upload is not None+ upload.created_by_role = CreatorUserRole.END_USER+ upload.created_by = user.id+ sqlite_session.commit()+ tool_id = tool.id+ if fails:+ storage.delete.side_effect = RuntimeError("unavailable")+ storage.exists.return_value = True+ with pytest.raises(RuntimeError):+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ else:+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ sqlite_session.expire_all()+ assert (sqlite_session.get(ToolFile, tool_id) is not None) == fails+ assert (sqlite_session.get(UploadFile, file_id) is not None) == fails+ assert sqlite_session.get(EndUser, user.id) is not None+++def test_shared_message_files_are_retained(sqlite_session: Session, storage: MagicMock) -> None:+ user = _end_user(sqlite_session)+ tool = ToolFile(+ tenant_id="tenant-1",+ user_id=user.id,+ conversation_id=None,+ file_key="tools/shared.json",+ mimetype="application/json",+ )+ sqlite_session.add(tool)+ file_id, _ = _offload(sqlite_session, "app-2", "upload_files/shared.txt")+ upload = sqlite_session.get(UploadFile, file_id)+ assert upload is not None+ upload.created_by_role = CreatorUserRole.END_USER+ upload.created_by = user.id+ conversation = Conversation(+ app_id="app-2", mode=AppMode.CHAT, name="other app", inputs={}, from_source=ConversationFromSource.API+ )+ sqlite_session.add(conversation)+ sqlite_session.flush()+ message = Message(+ app_id="app-2",+ conversation_id=conversation.id,+ inputs={},+ query="q",+ message={},+ message_unit_price=0,+ answer="a",+ answer_unit_price=0,+ currency="USD",+ from_source=ConversationFromSource.API,+ )+ sqlite_session.add(message)+ sqlite_session.flush()+ for record_id, method in ((tool.id, FileTransferMethod.TOOL_FILE), (file_id, FileTransferMethod.LOCAL_FILE)):+ sqlite_session.add(+ MessageFile(+ message_id=message.id,+ type=FileType.DOCUMENT,+ transfer_method=method,+ upload_file_id=record_id,+ created_by_role=CreatorUserRole.END_USER,+ created_by=user.id,+ )+ )+ sqlite_session.commit()+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ storage.delete.assert_not_called()+ sqlite_session.expire_all()+ assert sqlite_session.get(ToolFile, tool.id) is not None+ assert sqlite_session.get(UploadFile, file_id) is not None+++def test_partial_batch_failure_can_retry_already_deleted_objects(sqlite_session: Session, storage: MagicMock) -> None:+ first = _offload(sqlite_session, "app-1", "first.json")+ second = _offload(sqlite_session, "app-1", "second.json")+ storage.delete.side_effect = [None, RuntimeError("unavailable")]+ storage.exists.return_value = True+ with pytest.raises(RuntimeError):+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ sqlite_session.expire_all()+ for file_id, offload_id in (first, second):+ assert sqlite_session.get(UploadFile, file_id) is not None+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None+ storage.delete.side_effect = [FileNotFoundError(), None]+ storage.exists.return_value = False+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ sqlite_session.expire_all()+ for file_id, offload_id in (first, second):+ assert sqlite_session.get(UploadFile, file_id) is None+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is None+++def test_conversation_owned_tools_are_cleaned_and_account_assets_retained(+ sqlite_session: Session, storage: MagicMock+) -> None:+ conversation = Conversation(+ app_id="app-1", mode=AppMode.CHAT, name="owned", inputs={}, from_source=ConversationFromSource.CONSOLE+ )+ sqlite_session.add(conversation)+ sqlite_session.flush()+ owned = ToolFile(+ tenant_id="tenant-1",+ user_id="account-1",+ conversation_id=conversation.id,+ file_key="tools/owned.json",+ mimetype="application/json",+ )+ workspace = ToolFile(+ tenant_id="tenant-1",+ user_id="account-1",+ conversation_id=None,+ file_key="tools/workspace.json",+ mimetype="application/json",+ )+ foreign = ToolFile(+ tenant_id="tenant-2",+ user_id="account-1",+ conversation_id=conversation.id,+ file_key="tools/foreign.json",+ mimetype="application/json",+ )+ sqlite_session.add_all([owned, workspace, foreign])+ sqlite_session.commit()+ owned_id, workspace_id, foreign_id = owned.id, workspace.id, foreign.id+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ storage.delete.assert_called_once_with("tools/owned.json")+ sqlite_session.expire_all()+ assert sqlite_session.get(ToolFile, owned_id) is None+ assert sqlite_session.get(ToolFile, workspace_id) is not None+ assert sqlite_session.get(ToolFile, foreign_id) is not None+++def test_tool_with_conflicting_conversation_owner_is_retained(sqlite_session: Session, storage: MagicMock) -> None:+ user = _end_user(sqlite_session)+ tool = ToolFile(+ tenant_id="tenant-1",+ user_id=user.id,+ conversation_id="another-app-conversation",+ file_key="tools/shared.json",+ mimetype="application/json",+ )+ sqlite_session.add(tool)+ sqlite_session.commit()+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ storage.delete.assert_not_called()+ assert sqlite_session.get(ToolFile, tool.id) is not Nonediff --git a/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py b/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.pynew file mode 100644index 00000000000..6eb7b78b027--- /dev/null+++ b/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py@@ -0,0 +1,315 @@+from unittest.mock import Mock++import pytest+from flask import Flask+from sqlalchemy.orm import Session, sessionmaker++from models import Dataset+from models.agent import Agent, AgentHomeSnapshot, AgentScope, AgentSource, AgentStatus, AgentWorkingResourceStatus+from repositories.app.console_repository import ConsoleAppRepository+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository+from repositories.workspace.resource_cleanup_repository import WorkspaceResourceCleanupRepository+from services.workspace import resource_cleanup_service as module+++def _service(session_factory: sessionmaker[Session]) -> module.WorkspaceResourceCleanupService:+ return module.WorkspaceResourceCleanupService(+ session_factory=session_factory,+ apps=ConsoleAppRepository(session_factory=session_factory),+ datasets=SQLAlchemyDatasetRepository(session_factory=session_factory),+ agent_resources=WorkspaceResourceCleanupRepository(session_factory=session_factory),+ )+++def test_agent_sweep_covers_all_scopes_and_retains_other_tenants(+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ agents = [+ Agent(+ id="roster",+ tenant_id="target",+ name="Roster",+ scope=AgentScope.ROSTER,+ source=AgentSource.WORKFLOW,+ backing_app_id="missing-app",+ ),+ Agent(+ id="inline", tenant_id="target", name="Inline", scope=AgentScope.WORKFLOW_ONLY, source=AgentSource.WORKFLOW+ ),+ Agent(+ id="archived",+ tenant_id="target",+ name="Archived",+ scope=AgentScope.ROSTER,+ source=AgentSource.WORKFLOW,+ status=AgentStatus.ARCHIVED,+ ),+ Agent(id="other", tenant_id="other-tenant", name="Other", scope=AgentScope.ROSTER, source=AgentSource.WORKFLOW),+ ]+ snapshot = AgentHomeSnapshot(id="home", tenant_id="target", agent_id="inline", snapshot_ref="backend-home")+ sqlite_session.add_all([*agents, snapshot])+ sqlite_session.commit()+ collection = Mock()+ app_cleanup = Mock()+ monkeypatch.setattr(module, "enqueue_agent_resource_collection", collection)+ monkeypatch.setattr(module, "remove_app_and_related_data_task", app_cleanup)++ def assert_committed(**_kwargs: object) -> None:+ sqlite_session.expire_all()+ inline_agent = sqlite_session.get(Agent, "inline")+ assert inline_agent is not None+ assert inline_agent.status == AgentStatus.ARCHIVED+ home_snapshot = sqlite_session.get(AgentHomeSnapshot, "home")+ assert home_snapshot is not None+ assert home_snapshot.status == AgentWorkingResourceStatus.RETIRED++ collection.side_effect = assert_committed+ _service(sqlite_session_factory)._retire_agents("target")+ payload = collection.call_args.kwargs+ assert set(payload["purge_agent_ids"]) == {"roster", "inline", "archived"}+ assert payload["home_snapshot_ids"] == ["home"]+ other_agent = sqlite_session.get(Agent, "other")+ assert other_agent is not None+ assert other_agent.status == AgentStatus.ACTIVE+ app_cleanup.delay.assert_called_once_with(tenant_id="target", app_id="missing-app")+++def test_dataset_deletion_schedules_even_without_index_configuration(+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ sqlite_session.add_all(+ [+ Dataset(id="dataset", tenant_id="target", name="Empty", created_by="owner"),+ Dataset(id="other-dataset", tenant_id="other", name="Other", created_by="owner"),+ ]+ )+ sqlite_session.commit()+ cleanup = Mock()+ monkeypatch.setattr(module, "clean_dataset_task", cleanup)+ monkeypatch.setattr(module.dataset_api_key_bindings, "delete_keys_scoped_only_to", Mock())+ monkeypatch.setattr(module, "build_resource_access_token_cleanup_service", Mock())+ _service(sqlite_session_factory)._delete_dataset("target", "other-dataset")+ cleanup.delay.assert_not_called()+ _service(sqlite_session_factory)._delete_dataset("target", "dataset")+ cleanup.delay.assert_called_once()+ assert cleanup.delay.call_args.args[:2] == ("dataset", "target")+ sqlite_session.expire_all()+ assert sqlite_session.get(Dataset, "dataset") is None+ assert sqlite_session.get(Dataset, "other-dataset") is not None+++def test_agent_collection_failure_propagates_and_retirement_can_be_retried(+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ sqlite_session.add(+ Agent(id="agent", tenant_id="target", name="Agent", scope=AgentScope.ROSTER, source=AgentSource.WORKFLOW)+ )+ sqlite_session.commit()+ collection = Mock(side_effect=RuntimeError("broker unavailable"))+ monkeypatch.setattr(module, "enqueue_agent_resource_collection", collection)+ with pytest.raises(RuntimeError, match="broker unavailable"):+ _service(sqlite_session_factory)._retire_agents("target")+ collection.side_effect = None+ _service(sqlite_session_factory)._retire_agents("target")+ assert collection.call_args.kwargs["purge_agent_ids"] == ["agent"]+++def test_working_resources_are_retired_even_when_the_app_is_missing(+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ from models.agent import (+ AgentConfigVersionKind,+ AgentWorkspace,+ AgentWorkspaceBinding,+ AgentWorkspaceOwnerType,+ )++ workspace = AgentWorkspace(+ id="working",+ tenant_id="target",+ app_id="missing-app",+ owner_type=AgentWorkspaceOwnerType.CONVERSATION,+ owner_id="conversation",+ owner_scope_key="conversation",+ backend_workspace_ref="backend-workspace",+ )+ binding = AgentWorkspaceBinding(+ id="binding",+ tenant_id="target",+ app_id="missing-app",+ workspace_id="working",+ agent_id="missing-agent",+ agent_config_version_id="config",+ agent_config_version_kind=AgentConfigVersionKind.SNAPSHOT,+ backend_binding_ref="backend-binding",+ )+ sqlite_session.add_all([workspace, binding])+ sqlite_session.commit()+ collection = Mock()+ monkeypatch.setattr(module, "enqueue_agent_resource_collection", collection)+ _service(sqlite_session_factory)._retire_agents("target")+ sqlite_session.expire_all()+ working_workspace = sqlite_session.get(AgentWorkspace, "working")+ assert working_workspace is not None+ assert working_workspace.status == AgentWorkingResourceStatus.RETIRED+ working_binding = sqlite_session.get(AgentWorkspaceBinding, "binding")+ assert working_binding is not None+ assert working_binding.status == AgentWorkingResourceStatus.RETIRED+ assert collection.call_args.kwargs["workspace_ids"] == ["working"]+ assert collection.call_args.kwargs["binding_ids"] == ["binding"]+++def test_resource_failure_does_not_skip_the_other_groups(+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ from models import App+ from models.enums import AppStatus+ from models.model import AppMode++ sqlite_session.add_all(+ [+ App(+ id="app",+ tenant_id="target",+ name="App",+ mode=AppMode.WORKFLOW,+ status=AppStatus.NORMAL,+ enable_site=True,+ enable_api=True,+ ),+ Dataset(id="dataset", tenant_id="target", name="Dataset", created_by="owner"),+ ]+ )+ sqlite_session.commit()+ app_service = Mock()+ app_service.return_value.delete_app.side_effect = RuntimeError("app failed")+ monkeypatch.setattr(module, "AppService", app_service)+ datasets = Mock()+ agents = Mock()+ monkeypatch.setattr(module.WorkspaceResourceCleanupService, "_delete_dataset", datasets)+ monkeypatch.setattr(module.WorkspaceResourceCleanupService, "_retire_agents", agents)+ with pytest.raises(RuntimeError, match="app:app"):+ _service(sqlite_session_factory).cleanup("target")+ datasets.assert_called_once_with("target", "dataset")+ agents.assert_called_once_with("target")+++def test_agent_skill_snapshots_and_orphan_homes_are_included(+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch+) -> None:+ from models.skill import AgentSkillBindingSnapshot++ sqlite_session.add_all(+ [+ AgentHomeSnapshot(id="orphan-home", tenant_id="target", agent_id="missing", snapshot_ref="backend"),+ AgentSkillBindingSnapshot(+ id="target-binding",+ tenant_id="target",+ agent_id="missing",+ config_snapshot_id="config",+ skill_id="skill",+ priority=0,+ ),+ AgentSkillBindingSnapshot(+ id="other-binding",+ tenant_id="other",+ agent_id="missing",+ config_snapshot_id="config",+ skill_id="skill",+ priority=0,+ ),+ ]+ )+ sqlite_session.commit()+ collection = Mock()+ monkeypatch.setattr(module, "enqueue_agent_resource_collection", collection)+ _service(sqlite_session_factory)._retire_agents("target")+ sqlite_session.expire_all()+ assert sqlite_session.get(AgentSkillBindingSnapshot, "target-binding") is None+ assert sqlite_session.get(AgentSkillBindingSnapshot, "other-binding") is not None+ orphan_home = sqlite_session.get(AgentHomeSnapshot, "orphan-home")+ assert orphan_home is not None+ assert orphan_home.status == AgentWorkingResourceStatus.RETIRED+ assert collection.call_args.kwargs["home_snapshot_ids"] == ["orphan-home"]+++def test_direct_cleanup_does_not_resolve_a_console_login(+ app: Flask,+ sqlite_session: Session,+ sqlite_session_factory: sessionmaker[Session],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from unittest.mock import MagicMock++ from models import App+ from models.enums import AppStatus+ from models.model import AppMode+ from services import app_service++ sqlite_session.add(+ App(+ id="app",+ tenant_id="target",+ name="App",+ mode=AppMode.WORKFLOW,+ status=AppStatus.NORMAL,+ enable_site=True,+ enable_api=True,+ )+ )+ sqlite_session.commit()+ user = MagicMock()+ user.__bool__.side_effect = AssertionError("inner API must not load a console user")+ monkeypatch.setattr(app_service, "current_user", user)+ monkeypatch.setattr(app_service.app_was_deleted, "send", Mock())+ monkeypatch.setattr(app_service, "build_resource_access_token_cleanup_service", Mock())+ monkeypatch.setattr(app_service, "remove_app_and_related_data_task", Mock())+ retirement = Mock()+ monkeypatch.setattr(app_service, "WorkflowAgentRetirementService", retirement)+ monkeypatch.setattr(app_service, "enqueue_agent_resource_collection", Mock())+ features = Mock()+ features.is_webapp_auth_enabled.return_value = False+ monkeypatch.setattr(app_service, "SystemFeatureService", features)+ monkeypatch.setattr(app_service, "BillingService", Mock())+ monkeypatch.setattr(module, "enqueue_agent_resource_collection", Mock())+ with app.test_request_context(method="DELETE"):+ _service(sqlite_session_factory).cleanup("target")+ sqlite_session.expire_all()+ assert sqlite_session.get(App, "app") is None+ assert retirement.retire_unowned.call_args.kwargs["account_id"] is None+ user.__bool__.assert_not_called()+++def test_tenant_home_snapshot_retirement_is_tenant_scoped_and_idempotent(sqlite_session: Session) -> None:+ from datetime import datetime++ from services.agent.home_snapshot_service import AgentHomeSnapshotService++ retired_at = datetime(2026, 1, 1)+ sqlite_session.add_all(+ [+ AgentHomeSnapshot(id="active", tenant_id="target", agent_id="agent", snapshot_ref="a"),+ AgentHomeSnapshot(+ id="retired",+ tenant_id="target",+ agent_id="missing",+ snapshot_ref="b",+ status=AgentWorkingResourceStatus.RETIRED,+ retired_at=retired_at,+ ),+ AgentHomeSnapshot(id="other", tenant_id="other", agent_id="agent", snapshot_ref="c"),+ ]+ )+ sqlite_session.commit()+ ids = AgentHomeSnapshotService.retire_all_for_tenant(session=sqlite_session, tenant_id="target")+ sqlite_session.commit()+ assert set(ids) == {"active", "retired"}+ active = sqlite_session.get(AgentHomeSnapshot, "active")+ retired = sqlite_session.get(AgentHomeSnapshot, "retired")+ other = sqlite_session.get(AgentHomeSnapshot, "other")+ assert active is not None+ assert active.status == AgentWorkingResourceStatus.RETIRED+ assert retired is not None+ assert retired.retired_at == retired_at+ assert other is not None+ assert other.status == AgentWorkingResourceStatus.ACTIVEdiff --git a/api/tests/unit_tests/tasks/test_clean_dataset_task.py b/api/tests/unit_tests/tasks/test_clean_dataset_task.pyindex 3ac74ec1f4f..6cf16b42ce2 100644--- a/api/tests/unit_tests/tasks/test_clean_dataset_task.py+++ b/api/tests/unit_tests/tasks/test_clean_dataset_task.py@@ -579,3 +579,59 @@ class TestIndexProcessorParameters:)schedule_refresh.assert_not_called()++[email protected]("shared_tenant", [None, "same", "other"])+def test_crawl_cache_cleanup_preserves_shared_jobs(orm_session_maker, tenant_id, dataset_id, shared_tenant):+ from services import dataset_storage_cleanup_service as cleanup++ document = _persist_document(orm_session_maker, tenant_id=tenant_id, dataset_id=dataset_id)+ with orm_session_maker.begin() as session:+ row = session.get(Document, document.id)+ row.data_source_type = "website_crawl"+ row.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})+ if shared_tenant:+ other = _persist_document(+ orm_session_maker,+ tenant_id=tenant_id if shared_tenant == "same" else str(uuid.uuid4()),+ dataset_id=str(uuid.uuid4()),+ )+ with orm_session_maker.begin() as session:+ row = session.get(Document, other.id)+ row.data_source_type = "website_crawl"+ row.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})+ with patch.object(cleanup, "storage", autospec=True) as mock:+ cleanup.cleanup_dataset_crawl_cache(tenant_id, dataset_id)+ if shared_tenant:+ mock.delete.assert_not_called()+ else:+ mock.delete.assert_called_once_with("website_files/job-42.txt")++[email protected]("error", [FileNotFoundError(), PermissionError("denied")])+def test_crawl_cache_failure_preserves_document(orm_session_maker, tenant_id, dataset_id, error):+ from services import dataset_storage_cleanup_service as cleanup++ document = _persist_document(orm_session_maker, tenant_id=tenant_id, dataset_id=dataset_id)+ with orm_session_maker.begin() as session:+ row = session.get(Document, document.id)+ row.data_source_type = "website_crawl"+ row.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})+ with patch.object(cleanup, "storage", autospec=True) as mock:+ mock.delete.side_effect = error+ if isinstance(error, PermissionError):+ with pytest.raises(PermissionError):+ cleanup.cleanup_dataset_crawl_cache(tenant_id, dataset_id)+ else:+ cleanup.cleanup_dataset_crawl_cache(tenant_id, dataset_id)+ with orm_session_maker() as session:+ assert session.get(Document, document.id) is not None+++ "info", ["broken", "[]", '{"provider":"watercrawl","job_id":"job"}', '{"provider":"firecrawl","job_id":"../job"}']+)+def test_crawl_cache_ignores_invalid_or_other_provider_info(info):+ from services.dataset_storage_cleanup_service import _crawl_cache_key++ assert _crawl_cache_key(info) is Nonediff --git a/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py b/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.pyindex e1911e3aa10..35e8c782490 100644--- a/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py+++ b/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py@@ -80,6 +80,7 @@ def test_app_cleanup_removes_agent_bindings_before_workflows(monkeypatch: pytestevents: list[str] = []apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)other_cleanup_names = (+ "cleanup_app_storage","_delete_app_model_configs","_delete_app_site","_delete_app_mcp_servers",