Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

feat: support clean workspace related resource (#43803)

wangxiaolei7a02240701aa
diff --git a/api/.importlinter b/api/.importlinter
index 07609d175bb..336ea2ed16c 100644
--- a/api/.importlinter
+++ b/api/.importlinter
@@ -234,6 +234,7 @@ ignore_imports =
repositories.knowledge.dataset_api_key_repository -> services.knowledge.api_key_service
repositories.knowledge.dataset_repository -> services.knowledge.segments.application
repositories.knowledge.dataset_repository -> services.knowledge.dataset_access
+ repositories.knowledge.dataset_repository -> services.knowledge.entities.datasets
repositories.knowledge.dataset_repository -> services.knowledge.indexing.estimate
repositories.knowledge.dataset_repository -> services.knowledge.resource_scope
repositories.knowledge.document_repository -> services.knowledge.document_sync
diff --git a/api/controllers/inner_api/workspace/workspace.py b/api/controllers/inner_api/workspace/workspace.py
index 1414eefe279..2ecc2d7797b 100644
--- a/api/controllers/inner_api/workspace/workspace.py
+++ b/api/controllers/inner_api/workspace/workspace.py
@@ -1,3 +1,5 @@
+from uuid import UUID
+
from flask_restx import Resource
from pydantic import BaseModel
@@ -144,3 +146,15 @@ class EnterpriseWorkspaceMember(Resource):
"role": member.role,
},
}
+
+
+@inner_api_ns.route("/enterprise/workspace/<uuid:workspace_id>/resources")
+class EnterpriseWorkspaceResourceDeletion(Resource):
+ @setup_required
+ @inner_api_only
+ @inner_api_ns.doc("delete_enterprise_workspace_resources")
+ @inner_api_ns.doc(description="Schedule App, Dataset and Agent cleanup; retain the workspace and memberships")
+ @inner_api_ns.doc(responses={202: "Cleanup accepted", 401: "Invalid API key"})
+ def delete(self, workspace_id: UUID):
+ application_services().workspaces.resource_cleanup.cleanup(str(workspace_id))
+ return {"message": "workspace resource cleanup accepted."}, 202
diff --git a/api/extensions/application_services/workspace.py b/api/extensions/application_services/workspace.py
index fb78b00bbb9..5a7019688cf 100644
--- a/api/extensions/application_services/workspace.py
+++ b/api/extensions/application_services/workspace.py
@@ -2,8 +2,13 @@
from dataclasses import dataclass
+from sqlalchemy.orm import Session, sessionmaker
+
from extensions.ext_redis import RedisClientWrapper
from repositories.account.repository import SQLAlchemyAccountRepository
+from repositories.app.console_repository import ConsoleAppRepository
+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository
+from repositories.workspace.resource_cleanup_repository import WorkspaceResourceCleanupRepository
from repositories.workspace.workspace_repository import WorkspaceRepository
from services.account.adapters import RedisInvitationTokenStore
from services.file_service import FileService
@@ -27,6 +32,7 @@ from services.workspace.member_service import (
WorkspaceOwnerTransferService,
)
from services.workspace.provisioning_service import WorkspaceProvisioningService
+from services.workspace.resource_cleanup_service import WorkspaceResourceCleanupService
from services.workspace.service import WorkspaceQueryService, WorkspaceService
@@ -40,6 +46,7 @@ class WorkspaceServices:
owner_transfer: WorkspaceOwnerTransferService
invitations: WorkspaceInvitationService
identity: WorkspaceIdentityGateway
+ resource_cleanup: WorkspaceResourceCleanupService
def build_workspace_membership_services(
@@ -66,6 +73,7 @@ def build_workspace_membership_services(
def build_workspace_services(
*,
+ database_client: sessionmaker[Session],
workspaces: WorkspaceRepository,
accounts: SQLAlchemyAccountRepository,
files: FileService,
@@ -101,4 +109,10 @@ def build_workspace_services(
invitations=WorkspaceInvitationGateway(tokens=invitation_tokens, redis=redis),
),
identity=WorkspaceIdentityGateway(workspaces=workspaces),
+ resource_cleanup=WorkspaceResourceCleanupService(
+ session_factory=database_client,
+ apps=ConsoleAppRepository(session_factory=database_client),
+ datasets=SQLAlchemyDatasetRepository(session_factory=database_client),
+ agent_resources=WorkspaceResourceCleanupRepository(session_factory=database_client),
+ ),
)
diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.py
index fe28573388a..ab8e2e9dc8f 100644
--- a/api/extensions/ext_application_services.py
+++ b/api/extensions/ext_application_services.py
@@ -504,6 +504,7 @@ def build_application_services(
activation_accounts=activation_accounts,
)
workspace_services = build_workspace_services(
+ database_client=database_client,
workspaces=workspace_repository,
accounts=accounts,
files=file_service,
diff --git a/api/extensions/storage/aliyun_oss_storage.py b/api/extensions/storage/aliyun_oss_storage.py
index 5e96e7dbab1..6938a469fcb 100644
--- a/api/extensions/storage/aliyun_oss_storage.py
+++ b/api/extensions/storage/aliyun_oss_storage.py
@@ -60,5 +60,12 @@ class AliyunOssStorage(BaseStorage):
def delete(self, filename: str):
self.client.delete_object(self.__wrapper_folder_filename(filename))
+ @override
+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+ prefix = self.__wrapper_folder_filename(path)
+ folder_prefix = self.folder.rstrip("/") + "/" if self.folder else ""
+ keys = (obj.key.removeprefix(folder_prefix) for obj in aliyun_s3.ObjectIterator(self.client, prefix=prefix))
+ return self._filter_scan_keys(keys, files, directories)
+
def __wrapper_folder_filename(self, filename: str) -> str:
return posixpath.join(self.folder, filename) if self.folder else filename
diff --git a/api/extensions/storage/aws_s3_storage.py b/api/extensions/storage/aws_s3_storage.py
index 1c0de64286e..83cc20d1ef7 100644
--- a/api/extensions/storage/aws_s3_storage.py
+++ b/api/extensions/storage/aws_s3_storage.py
@@ -53,6 +53,19 @@ class AwsS3Storage(BaseStorage):
def save(self, filename, data):
self.client.put_object(Bucket=self.bucket_name, Key=filename, Body=data)
+ @override
+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+ if not files and not directories:
+ raise ValueError("At least one of files or directories must be True")
+ keys: list[str] = []
+ paginator = self.client.get_paginator("list_objects_v2")
+ for page in paginator.paginate(Bucket=self.bucket_name, Prefix=path):
+ for obj in page.get("Contents", []):
+ key = obj["Key"]
+ if (key.endswith("/") and directories) or (not key.endswith("/") and files):
+ keys.append(key)
+ return keys
+
@override
def load_once(self, filename: str) -> bytes:
try:
diff --git a/api/extensions/storage/azure_blob_storage.py b/api/extensions/storage/azure_blob_storage.py
index a17f61eb9bf..df325225c33 100644
--- a/api/extensions/storage/azure_blob_storage.py
+++ b/api/extensions/storage/azure_blob_storage.py
@@ -91,6 +91,15 @@ class AzureBlobStorage(BaseStorage):
blob_container = client.get_container_client(container=self.bucket_name)
blob_container.delete_blob(filename)
+ @override
+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+ if not self.bucket_name:
+ raise ValueError("Azure bucket name is not configured.")
+ container = self._sync_client().get_container_client(container=self.bucket_name)
+ # ItemPaged follows continuation tokens automatically.
+ keys = (blob.name for blob in container.list_blobs(name_starts_with=path))
+ return self._filter_scan_keys(keys, files, directories)
+
def _sync_client(self):
if self.account_key == "managedidentity":
return BlobServiceClient(account_url=self.account_url, credential=self.credential) # type: ignore
diff --git a/api/extensions/storage/baidu_obs_storage.py b/api/extensions/storage/baidu_obs_storage.py
index 9bb9bb31292..a4f579e551f 100644
--- a/api/extensions/storage/baidu_obs_storage.py
+++ b/api/extensions/storage/baidu_obs_storage.py
@@ -62,3 +62,17 @@ class BaiduObsStorage(BaseStorage):
@override
def delete(self, filename: str):
self.client.delete_object(bucket_name=self.bucket_name, key=filename)
+
+ @override
+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+ keys: list[str] = []
+ marker = ""
+ while True:
+ page = self.client.list_objects(bucket_name=self.bucket_name, prefix=path, marker=marker, max_keys=1000)
+ keys.extend(obj.key for obj in page.contents)
+ if not page.is_truncated:
+ break
+ if not page.next_marker or page.next_marker == marker:
+ raise RuntimeError("BOS object listing did not advance its pagination marker")
+ marker = page.next_marker
+ return self._filter_scan_keys(keys, files, directories)
diff --git a/api/extensions/storage/base_storage.py b/api/extensions/storage/base_storage.py
index d9c3ff6c4c3..cb5004c9a4e 100644
--- a/api/extensions/storage/base_storage.py
+++ b/api/extensions/storage/base_storage.py
@@ -1,7 +1,7 @@
"""Abstract interface for file storage implementations."""
from abc import ABC, abstractmethod
-from collections.abc import Generator
+from collections.abc import Generator, Iterable
class BaseStorage(ABC):
@@ -48,3 +48,10 @@ class BaseStorage(ABC):
If a storage backend doesn't support scanning, it will raise NotImplementedError.
"""
raise NotImplementedError("This storage backend doesn't support scanning")
+
+ @staticmethod
+ def _filter_scan_keys(keys: Iterable[str], files: bool, directories: bool) -> list[str]:
+ """Filter recursive object listings; directory entries end with '/'."""
+ if not files and not directories:
+ raise ValueError("At least one of files or directories must be True")
+ return [key for key in keys if (key.endswith("/") and directories) or (not key.endswith("/") and files)]
diff --git a/api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.py b/api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.py
index f09ccd2a28f..59d8ffa373e 100644
--- a/api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.py
+++ b/api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.py
@@ -439,7 +439,9 @@ class ClickZettaVolumeStorage(BaseStorage):
return exists
except Exception as e:
logger.warning("Error checking file existence for %s: %s", filename, e)
- return False
+ # Deletion relies on this result. A failed query is not proof that
+ # the file is absent; retain references so callers can retry.
+ raise
@override
def delete(self, filename: str):
@@ -483,19 +485,27 @@ class ClickZettaVolumeStorage(BaseStorage):
"""Scan files and directories in ClickZetta Volume.
Args:
- path: Path to scan (dataset_id for table volumes)
+ path: Path prefix to scan, or a bare dataset_id for a table-volume root
files: Include files in results
directories: Include directories in results
Returns:
List of file/directory paths
"""
+ if not files and not directories:
+ raise ValueError("At least one of files or directories must be True")
+ requested_path = path
try:
- # For table volumes, path is treated as dataset_id
+ # Match save/load/delete: the first path component selects the
+ # table (or special USER VOLUME), and the remainder is its file path.
dataset_id = None
+ logical_root = ""
if self._config.volume_type == "table":
- dataset_id = path
- path = "" # Root of the table volume
+ root, separator, relative_path = path.partition("/")
+ dataset_id = root.removeprefix(self._config.table_prefix)
+ path = relative_path
+ if separator:
+ logical_root = root + "/"
volume_prefix = self._get_volume_sql_prefix(dataset_id)
@@ -524,6 +534,10 @@ class ClickZettaVolumeStorage(BaseStorage):
if volume_prefix == "USER VOLUME" and file_path.startswith(dify_prefix_with_slash):
file_path = file_path[len(dify_prefix_with_slash) :] # Remove prefix
+ file_path = logical_root + file_path
+ if logical_root and not file_path.startswith(requested_path):
+ continue
+
if files and not file_path.endswith("/") or directories and file_path.endswith("/"):
result.append(file_path)
@@ -531,5 +545,5 @@ class ClickZettaVolumeStorage(BaseStorage):
return result
except Exception:
- logger.exception("Error scanning path %s", path)
- return []
+ logger.exception("Error scanning path %s", requested_path)
+ raise
diff --git a/api/extensions/storage/google_cloud_storage.py b/api/extensions/storage/google_cloud_storage.py
index 83f199c45a4..18c1af3cde6 100644
--- a/api/extensions/storage/google_cloud_storage.py
+++ b/api/extensions/storage/google_cloud_storage.py
@@ -73,3 +73,8 @@ class GoogleCloudStorage(BaseStorage):
def delete(self, filename: str):
bucket = self.client.get_bucket(self.bucket_name)
bucket.delete_blob(filename)
+
+ @override
+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+ keys = (blob.name for blob in self.client.list_blobs(self.bucket_name, prefix=path))
+ return self._filter_scan_keys(keys, files, directories)
diff --git a/api/extensions/storage/huawei_obs_storage.py b/api/extensions/storage/huawei_obs_storage.py
index c840b170861..3c90baa32dc 100644
--- a/api/extensions/storage/huawei_obs_storage.py
+++ b/api/extensions/storage/huawei_obs_storage.py
@@ -1,5 +1,5 @@
from collections.abc import Generator
-from typing import override
+from typing import Any, override
from obs import ObsClient
@@ -51,7 +51,31 @@ class HuaweiObsStorage(BaseStorage):
@override
def delete(self, filename: str):
- self.client.deleteObject(bucketName=self.bucket_name, objectKey=filename)
+ response: Any = self.client.deleteObject(bucketName=self.bucket_name, objectKey=filename)
+ if response.status == 404 and response.errorCode == "NoSuchKey":
+ raise FileNotFoundError(filename)
+ if response.status >= 300:
+ raise RuntimeError(f"OBS object deletion failed: status={response.status}, code={response.errorCode}")
+
+ @override
+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+ keys: list[str] = []
+ marker = ""
+ while True:
+ # The SDK's decorator annotations do not describe its response model.
+ response: Any = self.client.listObjects(
+ bucketName=self.bucket_name, prefix=path, marker=marker, max_keys=1000
+ )
+ if response.status >= 300:
+ raise RuntimeError(f"OBS object listing failed: status={response.status}, code={response.errorCode}")
+ page = response.body
+ keys.extend(obj.key for obj in page.contents or [])
+ if not page.is_truncated:
+ break
+ if not page.next_marker or page.next_marker == marker:
+ raise RuntimeError("OBS object listing did not advance its pagination marker")
+ marker = page.next_marker
+ return self._filter_scan_keys(keys, files, directories)
def _get_meta(self, filename):
res = self.client.getObjectMetadata(bucketName=self.bucket_name, objectKey=filename)
diff --git a/api/extensions/storage/oracle_oci_storage.py b/api/extensions/storage/oracle_oci_storage.py
index 4097bb0cb09..4f535d3cb18 100644
--- a/api/extensions/storage/oracle_oci_storage.py
+++ b/api/extensions/storage/oracle_oci_storage.py
@@ -64,3 +64,9 @@ class OracleOCIStorage(BaseStorage):
@override
def delete(self, filename: str):
self.client.delete_object(Bucket=self.bucket_name, Key=filename)
+
+ @override
+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+ pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=self.bucket_name, Prefix=path)
+ keys = (obj["Key"] for page in pages for obj in page.get("Contents", []))
+ return self._filter_scan_keys(keys, files, directories)
diff --git a/api/extensions/storage/supabase_storage.py b/api/extensions/storage/supabase_storage.py
index f47153ce370..50c14ac2901 100644
--- a/api/extensions/storage/supabase_storage.py
+++ b/api/extensions/storage/supabase_storage.py
@@ -61,6 +61,41 @@ class SupabaseStorage(BaseStorage):
def delete(self, filename: str):
self.client.storage.from_(self.bucket_name).remove([filename])
+ @override
+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+ bucket = self.client.storage.from_(self.bucket_name)
+ # Supabase lists only immediate children, with offset pagination.
+ root = path.rstrip("/") if path.endswith("/") else path.rpartition("/")[0]
+ pending = [root]
+ keys: list[str] = []
+ page_size = 100
+ while pending:
+ folder = pending.pop()
+ offset = 0
+ while True:
+ page = bucket.list(
+ path=folder,
+ options={
+ "limit": page_size,
+ "offset": offset,
+ "sortBy": {"column": "name", "order": "asc"},
+ },
+ )
+ for entry in page:
+ key = f"{folder}/{entry['name']}" if folder else entry["name"]
+ if entry.get("id") is None:
+ directory_key = key + "/"
+ if directory_key.startswith(path) or path.startswith(directory_key):
+ pending.append(key)
+ if directory_key.startswith(path):
+ keys.append(directory_key)
+ elif key.startswith(path):
+ keys.append(key)
+ if len(page) < page_size:
+ break
+ offset += page_size
+ return self._filter_scan_keys(keys, files, directories)
+
def bucket_exists(self):
buckets = self.client.storage.list_buckets()
return any(bucket.name == self.bucket_name for bucket in buckets)
diff --git a/api/extensions/storage/tencent_cos_storage.py b/api/extensions/storage/tencent_cos_storage.py
index da34a6e0071..5727a782ebb 100644
--- a/api/extensions/storage/tencent_cos_storage.py
+++ b/api/extensions/storage/tencent_cos_storage.py
@@ -56,3 +56,18 @@ class TencentCosStorage(BaseStorage):
@override
def delete(self, filename: str):
self.client.delete_object(Bucket=self.bucket_name, Key=filename)
+
+ @override
+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+ keys: list[str] = []
+ marker = ""
+ while True:
+ page = self.client.list_objects(Bucket=self.bucket_name, Prefix=path, Marker=marker, MaxKeys=1000)
+ keys.extend(obj["Key"] for obj in page.get("Contents", []))
+ if str(page.get("IsTruncated", "false")).lower() != "true":
+ break
+ next_marker = page.get("NextMarker")
+ if not next_marker or next_marker == marker:
+ raise RuntimeError("COS object listing did not advance its pagination marker")
+ marker = next_marker
+ return self._filter_scan_keys(keys, files, directories)
diff --git a/api/extensions/storage/volcengine_tos_storage.py b/api/extensions/storage/volcengine_tos_storage.py
index 709f198712f..e0a03a06c0b 100644
--- a/api/extensions/storage/volcengine_tos_storage.py
+++ b/api/extensions/storage/volcengine_tos_storage.py
@@ -71,3 +71,19 @@ class VolcengineTosStorage(BaseStorage):
if not self.bucket_name:
return
self.client.delete_object(bucket=self.bucket_name, key=filename)
+
+ @override
+ def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+ if not self.bucket_name:
+ raise ValueError("VOLCENGINE_TOS_BUCKET_NAME is not set")
+ keys: list[str] = []
+ marker = ""
+ while True:
+ page = self.client.list_objects(bucket=self.bucket_name, prefix=path, marker=marker, max_keys=1000)
+ keys.extend(obj.key for obj in page.contents)
+ if not page.is_truncated:
+ break
+ if not page.next_marker or page.next_marker == marker:
+ raise RuntimeError("TOS object listing did not advance its pagination marker")
+ marker = page.next_marker
+ return self._filter_scan_keys(keys, files, directories)
diff --git a/api/repositories/app/console_repository.py b/api/repositories/app/console_repository.py
index 9e3dc2fff18..65c1f0c9101 100644
--- a/api/repositories/app/console_repository.py
+++ b/api/repositories/app/console_repository.py
@@ -144,6 +144,11 @@ class ConsoleAppRepository(ConsoleApps, DiscoveryApps, RelatedApps):
config = load_app_parameter_config(app, session=session) if include_config else None
return AppDescription(app_summary(app), bool(app.enable_api), config)
+ def list_workspace_app_ids(self, tenant_id: str) -> list[str]:
+ """List every App row in the workspace, ignoring console visibility and status."""
+ with self._session_factory() as session:
+ return list(session.scalars(select(App.id).where(App.tenant_id == tenant_id)).all())
+
@override
def find_visible_app(self, app_id: str, tenant_id: str) -> AppSummary | None:
with self._session_factory() as session:
diff --git a/api/repositories/knowledge/dataset_repository.py b/api/repositories/knowledge/dataset_repository.py
index 83698ae635a..f93015d9a54 100644
--- a/api/repositories/knowledge/dataset_repository.py
+++ b/api/repositories/knowledge/dataset_repository.py
@@ -1,11 +1,12 @@
"""SQLAlchemy repository for tenant-owned dataset state."""
-from sqlalchemy import and_, exists, literal, select
+from sqlalchemy import and_, delete, exists, literal, select
from sqlalchemy.orm import Session, sessionmaker
from models.dataset import Dataset, DatasetPermission, Document
-from repositories.knowledge.dataset_read_repository import pipeline_dataset_query
+from repositories.knowledge.dataset_read_repository import get_dataset_doc_form, pipeline_dataset_query
from services.knowledge.dataset_access import DatasetAccessRecord, DatasetAccessSnapshot
+from services.knowledge.entities.datasets import DatasetCleanupRecord
from services.knowledge.indexing.estimate import DatasetEstimateRecord
from services.knowledge.resource_scope import DatasetRef
from services.knowledge.segments.application import SegmentDatasetRecord, SegmentDocumentRecord, SegmentScope
@@ -66,6 +67,32 @@ class SQLAlchemyDatasetRepository:
with self._session_factory() as session:
return session.scalar(pipeline_dataset_query(workspace_id, pipeline_id).with_only_columns(Dataset.id))
+ def list_workspace_dataset_ids(self, tenant_id: str) -> list[str]:
+ with self._session_factory() as session:
+ return list(session.scalars(select(Dataset.id).where(Dataset.tenant_id == tenant_id)).all())
+
+ def get_cleanup_record(self, dataset_ref: DatasetRef) -> DatasetCleanupRecord | None:
+ """None means the dataset is already gone."""
+ with self._session_factory() as session:
+ dataset = _get_dataset(session, dataset_ref)
+ if dataset is None:
+ return None
+ return DatasetCleanupRecord(
+ id=dataset.id,
+ workspace_id=dataset.tenant_id,
+ indexing_technique=dataset.indexing_technique,
+ index_struct=dataset.index_struct,
+ collection_binding_id=dataset.collection_binding_id,
+ doc_form=get_dataset_doc_form(dataset, session=session),
+ pipeline_id=dataset.pipeline_id,
+ )
+
+ def delete(self, dataset_ref: DatasetRef) -> None:
+ with self._session_factory.begin() as session:
+ session.execute(
+ delete(Dataset).where(Dataset.id == dataset_ref.dataset_id, Dataset.tenant_id == dataset_ref.tenant_id)
+ )
+
def get_estimate_record(self, dataset_ref: DatasetRef) -> DatasetEstimateRecord | None:
with self._session_factory() as session:
dataset = _get_dataset(session, dataset_ref)
diff --git a/api/repositories/workspace/resource_cleanup_repository.py b/api/repositories/workspace/resource_cleanup_repository.py
new file mode 100644
index 00000000000..2501a2fdefa
--- /dev/null
+++ b/api/repositories/workspace/resource_cleanup_repository.py
@@ -0,0 +1,60 @@
+"""Tenant-scoped Agent persistence for workspace resource cleanup.
+
+Each method owns its session and transaction; the caller orchestrates the order.
+
+TODO: Move these methods to Agent, Agent Workspace and Skill repositories once
+those domains own sessionmaker-based persistence, then remove this module.
+"""
+
+from dataclasses import dataclass
+
+from sqlalchemy import delete, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from libs.datetime_utils import naive_utc_now
+from models.agent import Agent, AgentStatus, AgentWorkspace, AgentWorkspaceBinding, WorkflowAgentNodeBinding
+from models.skill import AgentSkillBinding, AgentSkillBindingSnapshot
+
+
+@dataclass(frozen=True, slots=True)
+class ArchivedAgents:
+ agent_ids: list[str]
+ backing_app_ids: list[str]
+
+
+class WorkspaceResourceCleanupRepository:
+ def __init__(self, session_factory: sessionmaker[Session]) -> None:
+ self._session_factory = session_factory
+
+ def archive_agents(self, tenant_id: str) -> ArchivedAgents:
+ """Archive every tenant Agent, including archived and backing-App-less ones, for repeated cleanup."""
+ with self._session_factory.begin() as session:
+ agents = session.scalars(select(Agent).where(Agent.tenant_id == tenant_id)).all()
+ now = naive_utc_now()
+ for agent in agents:
+ if agent.status != AgentStatus.ARCHIVED:
+ agent.status = AgentStatus.ARCHIVED
+ agent.archived_at = now
+ agent.updated_at = now
+ return ArchivedAgents(
+ agent_ids=[agent.id for agent in agents],
+ backing_app_ids=sorted({agent.backing_app_id for agent in agents if agent.backing_app_id}),
+ )
+
+ def list_agent_workspace_ids(self, tenant_id: str) -> list[str]:
+ with self._session_factory() as session:
+ return list(session.scalars(select(AgentWorkspace.id).where(AgentWorkspace.tenant_id == tenant_id)).all())
+
+ def list_agent_workspace_binding_ids(self, tenant_id: str) -> list[str]:
+ with self._session_factory() as session:
+ return list(
+ session.scalars(
+ select(AgentWorkspaceBinding.id).where(AgentWorkspaceBinding.tenant_id == tenant_id)
+ ).all()
+ )
+
+ def delete_agent_bindings(self, tenant_id: str) -> None:
+ with self._session_factory.begin() as session:
+ session.execute(delete(WorkflowAgentNodeBinding).where(WorkflowAgentNodeBinding.tenant_id == tenant_id))
+ session.execute(delete(AgentSkillBinding).where(AgentSkillBinding.tenant_id == tenant_id))
+ session.execute(delete(AgentSkillBindingSnapshot).where(AgentSkillBindingSnapshot.tenant_id == tenant_id))
diff --git a/api/services/agent/home_snapshot_service.py b/api/services/agent/home_snapshot_service.py
index a1e4ff5d74d..707a3796028 100644
--- a/api/services/agent/home_snapshot_service.py
+++ b/api/services/agent/home_snapshot_service.py
@@ -136,6 +136,21 @@ class AgentHomeSnapshotService:
row.retired_at = now
return [row.id for row in rows]
+ @classmethod
+ def retire_all_for_tenant(cls, *, session: Session, tenant_id: str) -> list[str]:
+ """Retire every tenant Home Snapshot in the caller's transaction.
+
+ Includes orphan snapshots whose Agent row is already gone. Returns all
+ matched ids, including already-retired ones, so repeated collection converges.
+ """
+ rows = session.scalars(select(AgentHomeSnapshot).where(AgentHomeSnapshot.tenant_id == tenant_id)).all()
+ now = naive_utc_now()
+ for row in rows:
+ if row.status == AgentWorkingResourceStatus.ACTIVE:
+ row.status = AgentWorkingResourceStatus.RETIRED
+ row.retired_at = now
+ return [row.id for row in rows]
+
@classmethod
def collect_retired_home_snapshot(cls, *, tenant_id: str, home_snapshot_id: str) -> None:
with session_factory.create_session() as session:
diff --git a/api/services/app_service.py b/api/services/app_service.py
index 979929e4066..bfbc32ca50b 100644
--- a/api/services/app_service.py
+++ b/api/services/app_service.py
@@ -4,6 +4,7 @@ from collections.abc import Callable, Mapping, Sequence
from copy import deepcopy
from dataclasses import dataclass
from datetime import datetime
+from types import EllipsisType
from typing import Any, NotRequired, TypedDict, cast
import sqlalchemy as sa
@@ -777,9 +778,11 @@ class AppService:
return app
- def delete_app(self, app: App, *, session: Session) -> None:
+ def delete_app(self, app: App, *, session: Session, account_id: str | None | EllipsisType = ...) -> None:
"""Delete an App and commit the passed session.
+ Omit account_id to use the logged-in actor; pass None for system cleanup.
+
The transaction releases all of a Workflow App's binding owners across
draft and published versions, archives a backing Roster Agent, retires
its resources, and deletes the App. Deleting a Roster Agent's backing
@@ -831,7 +834,8 @@ class AppService:
AgentSkillBinding.agent_id.in_(agent_ids_to_unbind),
)
)
- account_id = current_user.id if current_user else None
+ if account_id is ...:
+ account_id = current_user.id if current_user else None
if backing_agent is not None:
now = naive_utc_now()
backing_agent.status = AgentStatus.ARCHIVED
diff --git a/api/services/app_storage_cleanup_service.py b/api/services/app_storage_cleanup_service.py
new file mode 100644
index 00000000000..e764768fb73
--- /dev/null
+++ b/api/services/app_storage_cleanup_service.py
@@ -0,0 +1,214 @@
+"""Remove application-owned storage before deleting the durable database references.
+
+Uploads owned by accounts, subscription-wide trigger requests, tenant telemetry,
+and manually named exports do not have exclusive application ownership. They are
+deliberately retained rather than deleting another resource's data.
+"""
+
+import logging
+
+from sqlalchemy import delete, exists, or_, select
+from sqlalchemy.orm import InstrumentedAttribute
+from sqlalchemy.sql.selectable import Exists
+
+from core.db.session_factory import session_factory
+from extensions.ext_storage import storage
+from models import Conversation, EndUser, Message, MessageFile, UploadFile
+from models.enums import CreatorUserRole
+from models.human_input import HumanInputFormUploadFile
+from models.tools import ToolFile
+from models.workflow import (
+ Workflow,
+ WorkflowDraftVariableFile,
+ WorkflowNodeExecutionOffload,
+ WorkflowPause,
+ WorkflowPauseReason,
+ WorkflowRun,
+)
+
+logger = logging.getLogger(__name__)
+_BATCH_SIZE = 500
+
+
+def _delete_object(key: str) -> None:
+ """Allow redelivery after a storage delete succeeded but the DB commit failed."""
+ try:
+ storage.delete(key)
+ except FileNotFoundError:
+ return
+ except Exception:
+ # Some backends return False from exists() for permission failures too;
+ # never use that as evidence that a failed deletion actually succeeded.
+ logger.exception("Application storage cleanup failed for key=%s", key)
+ raise
+
+
+def cleanup_app_storage(tenant_id: str, app_id: str) -> None:
+ """Clean exclusively owned files with external I/O outside DB transactions.
+
+ Call before deleting runs, conversations, messages, or end users. A storage
+ failure propagates to the app deletion task so its existing retry can still
+ resolve the keys. Each successful batch removes its references atomically.
+ """
+ # The App row is already deleted by the caller. Validate trace ownership
+ # against remaining tenant-scoped resources before removing those resources.
+ with session_factory.create_session() as session:
+ owns_traces = any(
+ session.scalar(select(exists().where(model.tenant_id == tenant_id, model.app_id == app_id)))
+ for model in (Workflow, WorkflowRun, EndUser)
+ )
+ _cleanup_pauses(tenant_id, app_id)
+ _cleanup_node_offloads(tenant_id, app_id)
+ _cleanup_tool_files(tenant_id, app_id)
+ _cleanup_end_user_uploads(tenant_id, app_id)
+ if owns_traces:
+ _cleanup_traces(app_id)
+
+
+def _cleanup_pauses(tenant_id: str, app_id: str) -> None:
+ run_ids = select(WorkflowRun.id).where(WorkflowRun.tenant_id == tenant_id, WorkflowRun.app_id == app_id)
+ while True:
+ with session_factory.create_session() as session:
+ rows = session.execute(
+ select(WorkflowPause.id, WorkflowPause.state_object_key)
+ .where(WorkflowPause.workflow_run_id.in_(run_ids))
+ .limit(_BATCH_SIZE)
+ ).all()
+ if not rows:
+ return
+ for _, key in rows:
+ _delete_object(key)
+ ids = [row.id for row in rows]
+ with session_factory.create_session() as session, session.begin():
+ session.execute(delete(WorkflowPauseReason).where(WorkflowPauseReason.pause_id.in_(ids)))
+ session.execute(delete(WorkflowPause).where(WorkflowPause.id.in_(ids)))
+
+
+def _cleanup_node_offloads(tenant_id: str, app_id: str) -> None:
+ while True:
+ with session_factory.create_session() as session:
+ rows = session.execute(
+ select(WorkflowNodeExecutionOffload.id, WorkflowNodeExecutionOffload.file_id, UploadFile.key)
+ .outerjoin(
+ UploadFile,
+ (UploadFile.id == WorkflowNodeExecutionOffload.file_id) & (UploadFile.tenant_id == tenant_id),
+ )
+ .where(
+ WorkflowNodeExecutionOffload.tenant_id == tenant_id,
+ WorkflowNodeExecutionOffload.app_id == app_id,
+ )
+ .limit(_BATCH_SIZE)
+ ).all()
+ file_ids = [row.file_id for row in rows]
+ shared = set(
+ session.scalars(
+ select(WorkflowNodeExecutionOffload.file_id).where(
+ WorkflowNodeExecutionOffload.file_id.in_(file_ids),
+ or_(
+ WorkflowNodeExecutionOffload.tenant_id != tenant_id,
+ WorkflowNodeExecutionOffload.app_id != app_id,
+ ),
+ )
+ )
+ )
+ if not rows:
+ return
+ for row in rows:
+ if row.key is not None and row.file_id not in shared:
+ _delete_object(row.key)
+ with session_factory.create_session() as session, session.begin():
+ session.execute(
+ delete(WorkflowNodeExecutionOffload).where(
+ WorkflowNodeExecutionOffload.id.in_([row.id for row in rows]),
+ WorkflowNodeExecutionOffload.tenant_id == tenant_id,
+ WorkflowNodeExecutionOffload.app_id == app_id,
+ )
+ )
+ session.execute(
+ delete(UploadFile).where(
+ UploadFile.tenant_id == tenant_id,
+ UploadFile.id.in_(
+ [row.file_id for row in rows if row.key is not None and row.file_id not in shared]
+ ),
+ )
+ )
+
+
+def _other_app_message_reference(file_column: InstrumentedAttribute[str], app_id: str) -> Exists:
+ # Include orphan message references conservatively: their ownership cannot be
+ # established, so they must not authorize destruction of the underlying file.
+ return exists(
+ select(MessageFile.id)
+ .outerjoin(Message, Message.id == MessageFile.message_id)
+ .where(MessageFile.upload_file_id == file_column, or_(Message.app_id != app_id, Message.id.is_(None)))
+ )
+
+
+def _cleanup_tool_files(tenant_id: str, app_id: str) -> None:
+ # Conversation has no tenant column; its app owner plus ToolFile.tenant_id
+ # together constrain the complete ownership chain.
+ conversation_ids = select(Conversation.id).where(Conversation.app_id == app_id)
+ # A file with no conversation can still be owned by an app end user. Account
+ # files without a conversation are reusable workspace assets and are retained.
+ end_user_ids = select(EndUser.id).where(EndUser.tenant_id == tenant_id, EndUser.app_id == app_id)
+ while True:
+ with session_factory.create_session() as session:
+ rows = session.execute(
+ select(ToolFile.id, ToolFile.file_key)
+ .where(
+ ToolFile.tenant_id == tenant_id,
+ or_(
+ ToolFile.conversation_id.in_(conversation_ids),
+ ToolFile.conversation_id.is_(None) & ToolFile.user_id.in_(end_user_ids),
+ ),
+ ~_other_app_message_reference(ToolFile.id, app_id),
+ )
+ .limit(_BATCH_SIZE)
+ ).all()
+ if not rows:
+ return
+ for _, key in rows:
+ _delete_object(key)
+ with session_factory.create_session() as session, session.begin():
+ session.execute(
+ delete(ToolFile).where(ToolFile.tenant_id == tenant_id, ToolFile.id.in_([r.id for r in rows]))
+ )
+
+
+def _cleanup_end_user_uploads(tenant_id: str, app_id: str) -> None:
+ end_user_ids = select(EndUser.id).where(EndUser.tenant_id == tenant_id, EndUser.app_id == app_id)
+ while True:
+ with session_factory.create_session() as session:
+ rows = session.execute(
+ select(UploadFile.id, UploadFile.key)
+ .where(
+ UploadFile.tenant_id == tenant_id,
+ UploadFile.created_by_role == CreatorUserRole.END_USER,
+ UploadFile.created_by.in_(end_user_ids),
+ ~_other_app_message_reference(UploadFile.id, app_id),
+ ~exists().where(WorkflowNodeExecutionOffload.file_id == UploadFile.id),
+ ~exists().where(WorkflowDraftVariableFile.upload_file_id == UploadFile.id),
+ ~exists().where(HumanInputFormUploadFile.upload_file_id == UploadFile.id),
+ )
+ .limit(_BATCH_SIZE)
+ ).all()
+ if not rows:
+ return
+ for _, key in rows:
+ _delete_object(key)
+ with session_factory.create_session() as session, session.begin():
+ session.execute(
+ delete(UploadFile).where(UploadFile.tenant_id == tenant_id, UploadFile.id.in_([row.id for row in rows]))
+ )
+
+
+def _cleanup_traces(app_id: str) -> None:
+ prefix = f"ops_trace/{app_id}/"
+ try:
+ keys = storage.scan(prefix)
+ except FileNotFoundError:
+ return
+ for key in keys:
+ if not key.startswith(prefix):
+ raise ValueError(f"Trace storage listing returned a key outside application prefix: {key}")
+ _delete_object(key)
diff --git a/api/services/dataset_storage_cleanup_service.py b/api/services/dataset_storage_cleanup_service.py
new file mode 100644
index 00000000000..6043f73781e
--- /dev/null
+++ b/api/services/dataset_storage_cleanup_service.py
@@ -0,0 +1,68 @@
+"""Remove crawl caches whose remaining document references belong to a deleted dataset."""
+
+import json
+import logging
+
+from sqlalchemy import select
+
+from core.db.session_factory import session_factory
+from extensions.ext_storage import storage
+from models.dataset import Document
+
+logger = logging.getLogger(__name__)
+
+
+def _crawl_cache_key(data_source_info: str | None) -> str | None:
+ try:
+ info = json.loads(data_source_info or "{}")
+ except (ValueError, TypeError):
+ return None
+ if not isinstance(info, dict) or info.get("provider") != "firecrawl":
+ return None
+ job_id = info.get("job_id")
+ if not isinstance(job_id, str) or not job_id or any(char in job_id for char in "/\\\x00"):
+ return None
+ return f"website_files/{job_id}.txt"
+
+
+def cleanup_dataset_crawl_cache(tenant_id: str, dataset_id: str) -> None:
+ """Delete only unshared Firecrawl caches, before deleting their document references.
+
+ Crawl keys have no tenant namespace, so references in other tenants also prevent deletion.
+ Storage failures propagate, preserving the database references for a later retry.
+ """
+ with session_factory.create_session() as session:
+ candidates = {
+ key
+ for info in session.scalars(
+ select(Document.data_source_info).where(
+ Document.tenant_id == tenant_id,
+ Document.dataset_id == dataset_id,
+ Document.data_source_type == "website_crawl",
+ )
+ )
+ if (key := _crawl_cache_key(info)) is not None
+ }
+ if not candidates:
+ return
+ references = session.execute(
+ select(Document.tenant_id, Document.dataset_id, Document.data_source_info)
+ .where(Document.data_source_type == "website_crawl")
+ .execution_options(yield_per=500)
+ )
+ for owner_tenant, owner_dataset, info in references:
+ if owner_tenant != tenant_id or owner_dataset != dataset_id:
+ key = _crawl_cache_key(info)
+ if key is not None:
+ candidates.discard(key)
+ # Finish the read transaction before accessing object storage.
+ for key in sorted(candidates):
+ try:
+ storage.delete(key)
+ except FileNotFoundError:
+ pass
+ except Exception:
+ logger.exception(
+ "Failed to delete dataset crawl cache tenant=%s dataset=%s key=%s", tenant_id, dataset_id, key
+ )
+ raise
diff --git a/api/services/knowledge/entities/datasets.py b/api/services/knowledge/entities/datasets.py
index 21d5d659b99..a769dacac84 100644
--- a/api/services/knowledge/entities/datasets.py
+++ b/api/services/knowledge/entities/datasets.py
@@ -6,7 +6,7 @@ objects retain their stored keys; this boundary does not apply response defaults
from collections.abc import Mapping, Sequence
from datetime import datetime
-from typing import NotRequired, TypedDict
+from typing import NamedTuple, NotRequired, TypedDict
from pydantic import JsonValue
@@ -67,3 +67,16 @@ class DatasetPage(TypedDict):
total: int
page: int
limit: int
+
+
+class DatasetCleanupRecord(NamedTuple):
+ """Dataset state needed to schedule index cleanup after the row is deleted."""
+
+ id: str
+ workspace_id: str
+ indexing_technique: str | None
+ index_struct: str | None
+ collection_binding_id: str | None
+ # None means neither the dataset nor any of its documents declares a form.
+ doc_form: str | None
+ pipeline_id: str | None
diff --git a/api/services/workspace/resource_cleanup_service.py b/api/services/workspace/resource_cleanup_service.py
new file mode 100644
index 00000000000..06bb47fe93b
--- /dev/null
+++ b/api/services/workspace/resource_cleanup_service.py
@@ -0,0 +1,133 @@
+"""Schedule existing resource deletion lifecycles for an enterprise workspace.
+
+This service does not delete the Tenant or its memberships, and does not claim
+that publishing child tasks means their physical cleanup has completed.
+"""
+
+import logging
+
+from sqlalchemy.orm import Session, sessionmaker
+
+from constants.resource_access_token import ResourceAccessTokenResourceType
+from extensions.application_services.resource_access_token import build_resource_access_token_cleanup_service
+from repositories.app.console_repository import ConsoleAppRepository
+from repositories.knowledge import dataset_api_key_bindings
+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository
+from repositories.workspace.resource_cleanup_repository import WorkspaceResourceCleanupRepository
+from services.agent.home_snapshot_service import AgentHomeSnapshotService
+from services.agent.workspace_service import AgentWorkspaceService
+from services.app_service import AppService
+from services.knowledge.resource_scope import DatasetRef
+from tasks.clean_dataset_task import clean_dataset_task
+from tasks.collect_agent_resources_task import enqueue_agent_resource_collection
+from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task
+
+logger = logging.getLogger(__name__)
+
+
+class WorkspaceResourceCleanupService:
+ def __init__(
+ self,
+ *,
+ session_factory: sessionmaker[Session],
+ apps: ConsoleAppRepository,
+ datasets: SQLAlchemyDatasetRepository,
+ agent_resources: WorkspaceResourceCleanupRepository,
+ ) -> None:
+ # The session factory only serves existing services that require a
+ # caller-owned session; drop it once they own their persistence.
+ self._session_factory = session_factory
+ self._apps = apps
+ self._datasets = datasets
+ self._agent_resources = agent_resources
+
+ def cleanup(self, workspace_id: str) -> None:
+ """Attempt all resource groups; propagate failures to the caller."""
+ failures: list[str] = []
+ app_ids = self._apps.list_workspace_app_ids(workspace_id)
+ dataset_ids = self._datasets.list_workspace_dataset_ids(workspace_id)
+
+ for app_id in app_ids:
+ try:
+ self._delete_app(workspace_id, app_id)
+ except Exception:
+ logger.exception("Workspace App deletion failed: %s:%s", workspace_id, app_id)
+ failures.append(f"app:{app_id}")
+
+ for dataset_id in dataset_ids:
+ try:
+ self._delete_dataset(workspace_id, dataset_id)
+ except Exception:
+ logger.exception("Workspace Dataset deletion failed: %s:%s", workspace_id, dataset_id)
+ failures.append(f"dataset:{dataset_id}")
+
+ try:
+ self._retire_agents(workspace_id)
+ except Exception:
+ logger.exception("Workspace Agent retirement failed: %s", workspace_id)
+ failures.append("agents")
+ if failures:
+ raise RuntimeError(f"Workspace resource deletion failed for {workspace_id}: {', '.join(failures)}")
+
+ def _delete_app(self, workspace_id: str, app_id: str) -> None:
+ with self._session_factory() as session:
+ app = AppService.get_app_in_workspace(tenant_id=workspace_id, app_id=app_id, session=session)
+ if app is not None:
+ AppService().delete_app(app, session=session, account_id=None)
+
+ def _delete_dataset(self, workspace_id: str, dataset_id: str) -> None:
+ # Inner-API authentication authorizes the whole tenant. No member actor
+ # is available here; preserve the normal deletion effects without RBAC.
+ dataset_ref = DatasetRef(workspace_id, dataset_id)
+ dataset = self._datasets.get_cleanup_record(dataset_ref)
+ if dataset is None:
+ return
+ # Unlike the member deletion signal, also schedule incomplete or
+ # empty datasets so their relational data is not skipped.
+ clean_dataset_task.delay(
+ dataset.id,
+ workspace_id,
+ dataset.indexing_technique,
+ dataset.index_struct,
+ dataset.collection_binding_id,
+ dataset.doc_form or "text_model",
+ dataset.pipeline_id,
+ )
+ with self._session_factory.begin() as session:
+ dataset_api_key_bindings.delete_keys_scoped_only_to(session, dataset.id)
+ build_resource_access_token_cleanup_service(session=session).delete_resource_relations(
+ tenant_id=workspace_id,
+ resource_type=ResourceAccessTokenResourceType.KNOWLEDGE,
+ resource_id=dataset.id,
+ )
+ # Delete the row last so a failed step is retried from the remaining Dataset.
+ self._datasets.delete(dataset_ref)
+
+ def _retire_agents(self, workspace_id: str) -> None:
+ # Sweep independently of App records: archived, workflow-only and
+ # backing-App-less Agents must also be collected on repeated requests.
+ archived = self._agent_resources.archive_agents(workspace_id)
+ workspace_ids = self._agent_resources.list_agent_workspace_ids(workspace_id)
+ binding_ids = self._agent_resources.list_agent_workspace_binding_ids(workspace_id)
+ with self._session_factory.begin() as session:
+ snapshot_ids = AgentHomeSnapshotService.retire_all_for_tenant(session=session, tenant_id=workspace_id)
+ # Both retire calls skip resources that are no longer active.
+ for resource_id in workspace_ids:
+ AgentWorkspaceService.retire_workspace(
+ session=session, tenant_id=workspace_id, workspace_id=resource_id
+ )
+ for binding_id in binding_ids:
+ AgentWorkspaceService.retire_binding(session=session, tenant_id=workspace_id, binding_id=binding_id)
+ self._agent_resources.delete_agent_bindings(workspace_id)
+
+ # Backing App records can already be gone; re-publish their idempotent
+ # related-data cleanup before collecting the archived Agent aggregates.
+ for app_id in archived.backing_app_ids:
+ remove_app_and_related_data_task.delay(tenant_id=workspace_id, app_id=app_id)
+ enqueue_agent_resource_collection(
+ tenant_id=workspace_id,
+ binding_ids=binding_ids,
+ workspace_ids=workspace_ids,
+ home_snapshot_ids=snapshot_ids,
+ purge_agent_ids=archived.agent_ids,
+ )
diff --git a/api/tasks/clean_dataset_task.py b/api/tasks/clean_dataset_task.py
index 5a4baabefbc..357d30d6d4b 100644
--- a/api/tasks/clean_dataset_task.py
+++ b/api/tasks/clean_dataset_task.py
@@ -24,6 +24,7 @@ from models.dataset import (
)
from models.model import UploadFile
from models.workflow import Workflow
+from services.dataset_storage_cleanup_service import cleanup_dataset_crawl_cache
from tasks.refresh_billing_vector_space_task import schedule_billing_vector_space_refresh
logger = logging.getLogger(__name__)
@@ -34,9 +35,9 @@ logger = logging.getLogger(__name__)
def clean_dataset_task(
dataset_id: str,
tenant_id: str,
- indexing_technique: str,
- index_struct: str,
- collection_binding_id: str,
+ indexing_technique: str | None,
+ index_struct: str | None,
+ collection_binding_id: str | None,
doc_form: str,
pipeline_id: str | None = None,
):
@@ -55,6 +56,8 @@ def clean_dataset_task(
start_at = time.perf_counter()
vector_cleanup_succeeded = False
+ cleanup_dataset_crawl_cache(tenant_id, dataset_id)
+
with session_factory.create_session() as session:
try:
dataset = Dataset(
diff --git a/api/tasks/remove_app_and_related_data_task.py b/api/tasks/remove_app_and_related_data_task.py
index f0df4411423..afba447f678 100644
--- a/api/tasks/remove_app_and_related_data_task.py
+++ b/api/tasks/remove_app_and_related_data_task.py
@@ -54,6 +54,7 @@ from models.workflow import (
)
from repositories.factory import DifyAPIRepositoryFactory
from services.api_token_service import ApiTokenCache
+from services.app_storage_cleanup_service import cleanup_app_storage
logger = logging.getLogger(__name__)
@@ -63,6 +64,9 @@ def remove_app_and_related_data_task(self, tenant_id: str, app_id: str):
logger.info(click.style(f"Start deleting app and related data: {tenant_id}:{app_id}", fg="green"))
start_at = time.perf_counter()
try:
+ # Preserve ownership records until storage cleanup succeeds, so retries
+ # can still locate files. External I/O runs outside DB transactions.
+ cleanup_app_storage(tenant_id, app_id)
# Delete related data
_delete_app_model_configs(tenant_id, app_id)
_delete_app_site(tenant_id, app_id)
diff --git a/api/tests/unit_tests/controllers/inner_api/workspace/test_resource_deletion.py b/api/tests/unit_tests/controllers/inner_api/workspace/test_resource_deletion.py
new file mode 100644
index 00000000000..4c33d4e2d9e
--- /dev/null
+++ b/api/tests/unit_tests/controllers/inner_api/workspace/test_resource_deletion.py
@@ -0,0 +1,58 @@
+from collections.abc import Callable
+from inspect import unwrap
+from types import SimpleNamespace
+from unittest.mock import Mock
+from uuid import UUID
+
+import pytest
+from flask import Flask
+
+from controllers.inner_api.workspace import workspace as controller
+from controllers.inner_api.wraps import InnerApiUnauthorizedError
+
+WORKSPACE_ID = UUID("0dd1d570-cf7b-4bab-9c7f-84ebe6c36985")
+
+
+def test_delete_calls_cleanup_with_the_workspace_id(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
+ service = Mock()
+ monkeypatch.setattr(
+ controller,
+ "application_services",
+ lambda: SimpleNamespace(workspaces=SimpleNamespace(resource_cleanup=service)),
+ )
+ with app.test_request_context(method="DELETE"):
+ result = unwrap(controller.EnterpriseWorkspaceResourceDeletion.delete)(
+ controller.EnterpriseWorkspaceResourceDeletion(), WORKSPACE_ID
+ )
+ assert result == ({"message": "workspace resource cleanup accepted."}, 202)
+ service.cleanup.assert_called_once_with(str(WORKSPACE_ID))
+
+
+def test_delete_requires_inner_api_key(
+ app: Flask, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None]
+) -> None:
+ config_overrides(INNER_API=True, INNER_API_KEY="secret", DEPLOYMENT_EDITION="CLOUD")
+ service = Mock()
+ monkeypatch.setattr(
+ controller,
+ "application_services",
+ lambda: SimpleNamespace(workspaces=SimpleNamespace(resource_cleanup=service)),
+ )
+ with app.test_request_context(method="DELETE"):
+ with pytest.raises(InnerApiUnauthorizedError):
+ controller.EnterpriseWorkspaceResourceDeletion().delete(WORKSPACE_ID)
+ service.cleanup.assert_not_called()
+
+
+def test_cleanup_failure_does_not_return_accepted(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
+ service = Mock()
+ service.cleanup.side_effect = RuntimeError("broker unavailable")
+ monkeypatch.setattr(
+ controller,
+ "application_services",
+ lambda: SimpleNamespace(workspaces=SimpleNamespace(resource_cleanup=service)),
+ )
+ with app.test_request_context(method="DELETE"), pytest.raises(RuntimeError, match="broker unavailable"):
+ unwrap(controller.EnterpriseWorkspaceResourceDeletion.delete)(
+ controller.EnterpriseWorkspaceResourceDeletion(), WORKSPACE_ID
+ )
diff --git a/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py b/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py
index 878e5771458..64c5682339b 100644
--- a/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py
+++ b/api/tests/unit_tests/controllers/inner_api/workspace/test_workspace.py
@@ -45,6 +45,7 @@ def provisioning(monkeypatch: pytest.MonkeyPatch) -> Mock:
owner_transfer=Mock(),
invitations=Mock(),
identity=Mock(),
+ resource_cleanup=Mock(),
)
monkeypatch.setattr(controller, "application_services", lambda: Services(services))
return provisioning
diff --git a/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py b/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py
index acb3c7c9449..3d79410059b 100644
--- a/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py
+++ b/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py
@@ -3,6 +3,22 @@ from unittest.mock import MagicMock
from extensions.storage.aws_s3_storage import AwsS3Storage
+def test_scan_reads_all_pages_and_filters_directory_markers() -> None:
+ storage = AwsS3Storage.__new__(AwsS3Storage)
+ storage.bucket_name = "test-bucket"
+ storage.client = MagicMock()
+ storage.client.get_paginator.return_value.paginate.return_value = [
+ {"Contents": [{"Key": "ops_trace/app/"}, {"Key": "ops_trace/app/one.json"}]},
+ {"Contents": [{"Key": "ops_trace/app/two.json"}]},
+ dict[str, str](),
+ ]
+ assert storage.scan("ops_trace/app/") == ["ops_trace/app/one.json", "ops_trace/app/two.json"]
+ storage.client.get_paginator.assert_called_once_with("list_objects_v2")
+ storage.client.get_paginator.return_value.paginate.assert_called_once_with(
+ Bucket="test-bucket", Prefix="ops_trace/app/"
+ )
+
+
def test_generate_presigned_url() -> None:
storage = AwsS3Storage.__new__(AwsS3Storage)
storage.bucket_name = "test-bucket"
diff --git a/api/tests/unit_tests/extensions/storage/test_clickzetta_volume_paths.py b/api/tests/unit_tests/extensions/storage/test_clickzetta_volume_paths.py
index e98e066d2dd..bc03add50cb 100644
--- a/api/tests/unit_tests/extensions/storage/test_clickzetta_volume_paths.py
+++ b/api/tests/unit_tests/extensions/storage/test_clickzetta_volume_paths.py
@@ -1,5 +1,7 @@
"""Unit coverage for ClickZetta configuration and path generation."""
+from unittest.mock import MagicMock
+
import pytest
from extensions.storage.clickzetta_volume.clickzetta_volume_storage import (
@@ -62,3 +64,51 @@ def test_sql_prefix_generation() -> None:
storage._config.volume_type = "external"
storage._config.volume_name = "my_external_volume"
assert storage._get_volume_sql_prefix() == "VOLUME my_external_volume"
+
+
+def test_scan_table_prefix_matches_file_storage_path(monkeypatch: pytest.MonkeyPatch) -> None:
+ storage = _table_storage()
+ execute = MagicMock(return_value=[("app/trace.json",), ("another-app/trace.json",)])
+ monkeypatch.setattr(storage, "_execute_sql", execute)
+ assert storage.scan("ops_trace/app/") == ["ops_trace/app/trace.json"]
+ execute.assert_called_once_with("LIST TABLE VOLUME test_dataset_ops_trace SUBDIRECTORY 'app/'", fetch=True)
+
+
+def test_scan_special_table_prefix_restores_logical_root(monkeypatch: pytest.MonkeyPatch) -> None:
+ storage = _table_storage()
+ execute = MagicMock(return_value=[("dify_km/tenant/file.txt",)])
+ monkeypatch.setattr(storage, "_execute_sql", execute)
+ assert storage.scan("upload_files/tenant/") == ["upload_files/tenant/file.txt"]
+ execute.assert_called_once_with("LIST USER VOLUME SUBDIRECTORY 'dify_km/tenant/'", fetch=True)
+
+
+def test_scan_preserves_bare_dataset_root_behavior(monkeypatch: pytest.MonkeyPatch) -> None:
+ storage = _table_storage()
+ execute = MagicMock(return_value=[("file.txt",)])
+ monkeypatch.setattr(storage, "_execute_sql", execute)
+ assert storage.scan("12345") == ["file.txt"]
+ execute.assert_called_once_with("LIST TABLE VOLUME test_dataset_12345", fetch=True)
+
+
+def test_scan_propagates_backend_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+ storage = _table_storage()
+ monkeypatch.setattr(storage, "_execute_sql", MagicMock(side_effect=RuntimeError("backend unavailable")))
+ with pytest.raises(RuntimeError, match="backend unavailable"):
+ storage.scan("ops_trace/app/")
+
+
+def test_delete_propagates_failed_existence_check(monkeypatch: pytest.MonkeyPatch) -> None:
+ storage = _table_storage()
+ execute = MagicMock(side_effect=RuntimeError("backend unavailable"))
+ monkeypatch.setattr(storage, "_execute_sql", execute)
+ with pytest.raises(RuntimeError, match="backend unavailable"):
+ storage.delete("ops_trace/app/trace.json")
+ assert execute.call_count == 1
+
+
+def test_delete_absent_file_remains_idempotent(monkeypatch: pytest.MonkeyPatch) -> None:
+ storage = _table_storage()
+ execute = MagicMock(return_value=[])
+ monkeypatch.setattr(storage, "_execute_sql", execute)
+ storage.delete("ops_trace/app/trace.json")
+ assert execute.call_count == 1
diff --git a/api/tests/unit_tests/services/test_app_storage_cleanup_service.py b/api/tests/unit_tests/services/test_app_storage_cleanup_service.py
new file mode 100644
index 00000000000..a9da4d71da8
--- /dev/null
+++ b/api/tests/unit_tests/services/test_app_storage_cleanup_service.py
@@ -0,0 +1,403 @@
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+from botocore.exceptions import ClientError
+from sqlalchemy.orm import Session
+
+from extensions.storage.clickzetta_volume.clickzetta_volume_storage import ClickZettaVolumeStorage
+from extensions.storage.huawei_obs_storage import HuaweiObsStorage
+from extensions.storage.storage_type import StorageType
+from graphon.entities.pause_reason import PauseReasonType
+from graphon.enums import WorkflowExecutionStatus
+from graphon.file import FileTransferMethod, FileType
+from models import WorkflowType
+from models.enums import (
+ ConversationFromSource,
+ CreatorUserRole,
+ EndUserType,
+ ExecutionOffLoadType,
+ WorkflowRunTriggeredFrom,
+)
+from models.model import AppMode, Conversation, EndUser, Message, MessageFile, UploadFile
+from models.tools import ToolFile
+from models.workflow import WorkflowNodeExecutionOffload, WorkflowPause, WorkflowPauseReason, WorkflowRun
+from services import app_storage_cleanup_service as cleanup
+
+
+def _offload(session: Session, app_id: str, key: str, tenant_id: str = "tenant-1") -> tuple[str, str]:
+ file = UploadFile(
+ tenant_id=tenant_id,
+ storage_type=StorageType.LOCAL,
+ key=key,
+ name="node_execution_outputs.json",
+ size=2,
+ extension="json",
+ mime_type="application/json",
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by="account-1",
+ created_at=datetime.now(UTC),
+ used=True,
+ )
+ session.add(file)
+ session.flush()
+ offload = WorkflowNodeExecutionOffload(
+ tenant_id=tenant_id,
+ app_id=app_id,
+ node_execution_id=None,
+ type_=ExecutionOffLoadType.OUTPUTS,
+ file_id=file.id,
+ )
+ session.add(offload)
+ session.commit()
+ return file.id, offload.id
+
+
+def storage(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
+ mock = MagicMock()
+ mock.scan.return_value = list[str]()
+ monkeypatch.setattr(cleanup, "storage", mock)
+ return mock
+
+
+def test_cleanup_removes_owned_offloads_and_preserves_other_owners(sqlite_session: Session, storage: MagicMock) -> None:
+ owned = _offload(sqlite_session, "app-1", "owned.json")
+ other_app = _offload(sqlite_session, "app-2", "other-app.json")
+ other_tenant = _offload(sqlite_session, "app-1", "other-tenant.json", "tenant-2")
+
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+
+ storage.delete.assert_called_once_with("owned.json")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(UploadFile, owned[0]) is None
+ assert sqlite_session.get(WorkflowNodeExecutionOffload, owned[1]) is None
+ for file_id, offload_id in (other_app, other_tenant):
+ assert sqlite_session.get(UploadFile, file_id) is not None
+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None
+
+
+def test_storage_failure_retains_references_for_retry(sqlite_session: Session, storage: MagicMock) -> None:
+ file_id, offload_id = _offload(sqlite_session, "app-1", "owned.json")
+ storage.delete.side_effect = RuntimeError("storage unavailable")
+ storage.exists.return_value = True
+
+ with pytest.raises(RuntimeError, match="storage unavailable"):
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+
+ sqlite_session.expire_all()
+ assert sqlite_session.get(UploadFile, file_id) is not None
+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None
+
+ storage.delete.side_effect = None
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(UploadFile, file_id) is None
+
+
+def test_missing_object_allows_redelivery(sqlite_session: Session, storage: MagicMock) -> None:
+ file_id, _ = _offload(sqlite_session, "app-1", "owned.json")
+ storage.delete.side_effect = FileNotFoundError()
+ storage.exists.return_value = False
+
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+
+ sqlite_session.expire_all()
+ assert sqlite_session.get(UploadFile, file_id) is None
+
+
+def test_access_denied_retains_references_even_if_exists_returns_false(
+ sqlite_session: Session, storage: MagicMock
+) -> None:
+ file_id, offload_id = _offload(sqlite_session, "app-1", "owned.json")
+ storage.delete.side_effect = ClientError({"Error": {"Code": "AccessDenied"}}, "DeleteObject")
+ storage.exists.return_value = False
+ with pytest.raises(ClientError):
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(UploadFile, file_id) is not None
+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None
+ storage.exists.assert_not_called()
+
+
+def test_obs_delete_failure_preserves_retry_references(
+ sqlite_session: Session, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ file_id, offload_id = _offload(sqlite_session, "app-1", "owned.json")
+ backend = HuaweiObsStorage.__new__(HuaweiObsStorage)
+ backend.bucket_name = "bucket"
+ backend.client = MagicMock()
+ backend.client.deleteObject.return_value = SimpleNamespace(status=403, errorCode="AccessDenied")
+ monkeypatch.setattr(cleanup, "storage", backend)
+ with pytest.raises(RuntimeError, match="403"):
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(UploadFile, file_id) is not None
+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None
+
+
+def test_clickzetta_failed_existence_check_preserves_retry_references(
+ sqlite_session: Session,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ file_id, offload_id = _offload(sqlite_session, "app-1", "owned.json")
+ backend = ClickZettaVolumeStorage.__new__(ClickZettaVolumeStorage)
+ monkeypatch.setattr(backend, "_config", SimpleNamespace(volume_type="user", dify_prefix="dify"), raising=False)
+ monkeypatch.setattr(backend, "_execute_sql", MagicMock(side_effect=RuntimeError("backend unavailable")))
+ monkeypatch.setattr(cleanup, "storage", backend)
+ with pytest.raises(RuntimeError, match="backend unavailable"):
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(UploadFile, file_id) is not None
+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None
+
+
+def test_shared_offload_file_is_retained(sqlite_session: Session, storage: MagicMock) -> None:
+ file_id, offload_id = _offload(sqlite_session, "app-1", "shared.json")
+ shared = WorkflowNodeExecutionOffload(
+ tenant_id="tenant-1",
+ app_id="app-2",
+ node_execution_id=None,
+ type_=ExecutionOffLoadType.INPUTS,
+ file_id=file_id,
+ )
+ sqlite_session.add(shared)
+ sqlite_session.commit()
+
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+
+ storage.delete.assert_not_called()
+ sqlite_session.expire_all()
+ assert sqlite_session.get(UploadFile, file_id) is not None
+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is None
+ assert sqlite_session.get(WorkflowNodeExecutionOffload, shared.id) is not None
+
+
+def test_trace_cleanup_stays_within_app_prefix(sqlite_session: Session, storage: MagicMock) -> None:
+ _end_user(sqlite_session)
+ storage.scan.return_value = ["ops_trace/app-1/trace.json"]
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ storage.delete.assert_called_once_with("ops_trace/app-1/trace.json")
+
+ storage.reset_mock()
+ storage.scan.return_value = ["ops_trace/app-2/trace.json"]
+ with pytest.raises(ValueError, match="outside application prefix"):
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ storage.delete.assert_not_called()
+
+
+def _end_user(session: Session) -> EndUser:
+ user = EndUser(tenant_id="tenant-1", app_id="app-1", type=EndUserType.SERVICE_API, session_id="session")
+ session.add(user)
+ session.commit()
+ return user
+
+
+def test_wrong_tenant_does_not_scan_traces(sqlite_session: Session, storage: MagicMock) -> None:
+ _end_user(sqlite_session)
+ cleanup.cleanup_app_storage("tenant-2", "app-1")
+ storage.scan.assert_not_called()
+
+
[email protected]("fails", [False, True])
+def test_pause_files_and_reasons_are_removed_only_after_storage_success(
+ sqlite_session: Session, storage: MagicMock, fails: bool
+) -> None:
+ run = WorkflowRun(
+ tenant_id="tenant-1",
+ app_id="app-1",
+ workflow_id="workflow-1",
+ type=WorkflowType.CHAT,
+ triggered_from=WorkflowRunTriggeredFrom.APP_RUN,
+ version="1",
+ status=WorkflowExecutionStatus.PAUSED,
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by="account-1",
+ )
+ sqlite_session.add(run)
+ sqlite_session.flush()
+ pause = WorkflowPause(workflow_id="workflow-1", workflow_run_id=run.id, state_object_key="pause.json")
+ sqlite_session.add(pause)
+ sqlite_session.flush()
+ reason = WorkflowPauseReason(pause_id=pause.id, type_=PauseReasonType.HITL_REQUIRED)
+ sqlite_session.add(reason)
+ sqlite_session.commit()
+ pause_id, reason_id = pause.id, reason.id
+ if fails:
+ storage.delete.side_effect = RuntimeError("unavailable")
+ storage.exists.return_value = True
+ with pytest.raises(RuntimeError):
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ else:
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ sqlite_session.expire_all()
+ assert (sqlite_session.get(WorkflowPause, pause_id) is not None) == fails
+ assert (sqlite_session.get(WorkflowPauseReason, reason_id) is not None) == fails
+ assert sqlite_session.get(WorkflowRun, run.id) is not None
+
+
[email protected]("fails", [False, True])
+def test_end_user_files_are_cleaned_before_owner_records(
+ sqlite_session: Session, storage: MagicMock, fails: bool
+) -> None:
+ user = _end_user(sqlite_session)
+ tool = ToolFile(
+ tenant_id="tenant-1",
+ user_id=user.id,
+ conversation_id=None,
+ file_key="tools/owned.json",
+ mimetype="application/json",
+ )
+ sqlite_session.add(tool)
+ file_id, offload_id = _offload(sqlite_session, "app-2", "upload_files/owned.txt")
+ offload = sqlite_session.get(WorkflowNodeExecutionOffload, offload_id)
+ assert offload is not None
+ sqlite_session.delete(offload)
+ upload = sqlite_session.get(UploadFile, file_id)
+ assert upload is not None
+ upload.created_by_role = CreatorUserRole.END_USER
+ upload.created_by = user.id
+ sqlite_session.commit()
+ tool_id = tool.id
+ if fails:
+ storage.delete.side_effect = RuntimeError("unavailable")
+ storage.exists.return_value = True
+ with pytest.raises(RuntimeError):
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ else:
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ sqlite_session.expire_all()
+ assert (sqlite_session.get(ToolFile, tool_id) is not None) == fails
+ assert (sqlite_session.get(UploadFile, file_id) is not None) == fails
+ assert sqlite_session.get(EndUser, user.id) is not None
+
+
+def test_shared_message_files_are_retained(sqlite_session: Session, storage: MagicMock) -> None:
+ user = _end_user(sqlite_session)
+ tool = ToolFile(
+ tenant_id="tenant-1",
+ user_id=user.id,
+ conversation_id=None,
+ file_key="tools/shared.json",
+ mimetype="application/json",
+ )
+ sqlite_session.add(tool)
+ file_id, _ = _offload(sqlite_session, "app-2", "upload_files/shared.txt")
+ upload = sqlite_session.get(UploadFile, file_id)
+ assert upload is not None
+ upload.created_by_role = CreatorUserRole.END_USER
+ upload.created_by = user.id
+ conversation = Conversation(
+ app_id="app-2", mode=AppMode.CHAT, name="other app", inputs={}, from_source=ConversationFromSource.API
+ )
+ sqlite_session.add(conversation)
+ sqlite_session.flush()
+ message = Message(
+ app_id="app-2",
+ conversation_id=conversation.id,
+ inputs={},
+ query="q",
+ message={},
+ message_unit_price=0,
+ answer="a",
+ answer_unit_price=0,
+ currency="USD",
+ from_source=ConversationFromSource.API,
+ )
+ sqlite_session.add(message)
+ sqlite_session.flush()
+ for record_id, method in ((tool.id, FileTransferMethod.TOOL_FILE), (file_id, FileTransferMethod.LOCAL_FILE)):
+ sqlite_session.add(
+ MessageFile(
+ message_id=message.id,
+ type=FileType.DOCUMENT,
+ transfer_method=method,
+ upload_file_id=record_id,
+ created_by_role=CreatorUserRole.END_USER,
+ created_by=user.id,
+ )
+ )
+ sqlite_session.commit()
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ storage.delete.assert_not_called()
+ sqlite_session.expire_all()
+ assert sqlite_session.get(ToolFile, tool.id) is not None
+ assert sqlite_session.get(UploadFile, file_id) is not None
+
+
+def test_partial_batch_failure_can_retry_already_deleted_objects(sqlite_session: Session, storage: MagicMock) -> None:
+ first = _offload(sqlite_session, "app-1", "first.json")
+ second = _offload(sqlite_session, "app-1", "second.json")
+ storage.delete.side_effect = [None, RuntimeError("unavailable")]
+ storage.exists.return_value = True
+ with pytest.raises(RuntimeError):
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ sqlite_session.expire_all()
+ for file_id, offload_id in (first, second):
+ assert sqlite_session.get(UploadFile, file_id) is not None
+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is not None
+ storage.delete.side_effect = [FileNotFoundError(), None]
+ storage.exists.return_value = False
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ sqlite_session.expire_all()
+ for file_id, offload_id in (first, second):
+ assert sqlite_session.get(UploadFile, file_id) is None
+ assert sqlite_session.get(WorkflowNodeExecutionOffload, offload_id) is None
+
+
+def test_conversation_owned_tools_are_cleaned_and_account_assets_retained(
+ sqlite_session: Session, storage: MagicMock
+) -> None:
+ conversation = Conversation(
+ app_id="app-1", mode=AppMode.CHAT, name="owned", inputs={}, from_source=ConversationFromSource.CONSOLE
+ )
+ sqlite_session.add(conversation)
+ sqlite_session.flush()
+ owned = ToolFile(
+ tenant_id="tenant-1",
+ user_id="account-1",
+ conversation_id=conversation.id,
+ file_key="tools/owned.json",
+ mimetype="application/json",
+ )
+ workspace = ToolFile(
+ tenant_id="tenant-1",
+ user_id="account-1",
+ conversation_id=None,
+ file_key="tools/workspace.json",
+ mimetype="application/json",
+ )
+ foreign = ToolFile(
+ tenant_id="tenant-2",
+ user_id="account-1",
+ conversation_id=conversation.id,
+ file_key="tools/foreign.json",
+ mimetype="application/json",
+ )
+ sqlite_session.add_all([owned, workspace, foreign])
+ sqlite_session.commit()
+ owned_id, workspace_id, foreign_id = owned.id, workspace.id, foreign.id
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ storage.delete.assert_called_once_with("tools/owned.json")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(ToolFile, owned_id) is None
+ assert sqlite_session.get(ToolFile, workspace_id) is not None
+ assert sqlite_session.get(ToolFile, foreign_id) is not None
+
+
+def test_tool_with_conflicting_conversation_owner_is_retained(sqlite_session: Session, storage: MagicMock) -> None:
+ user = _end_user(sqlite_session)
+ tool = ToolFile(
+ tenant_id="tenant-1",
+ user_id=user.id,
+ conversation_id="another-app-conversation",
+ file_key="tools/shared.json",
+ mimetype="application/json",
+ )
+ sqlite_session.add(tool)
+ sqlite_session.commit()
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ storage.delete.assert_not_called()
+ assert sqlite_session.get(ToolFile, tool.id) is not None
diff --git a/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py b/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py
new file mode 100644
index 00000000000..6eb7b78b027
--- /dev/null
+++ b/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py
@@ -0,0 +1,315 @@
+from unittest.mock import Mock
+
+import pytest
+from flask import Flask
+from sqlalchemy.orm import Session, sessionmaker
+
+from models import Dataset
+from models.agent import Agent, AgentHomeSnapshot, AgentScope, AgentSource, AgentStatus, AgentWorkingResourceStatus
+from repositories.app.console_repository import ConsoleAppRepository
+from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository
+from repositories.workspace.resource_cleanup_repository import WorkspaceResourceCleanupRepository
+from services.workspace import resource_cleanup_service as module
+
+
+def _service(session_factory: sessionmaker[Session]) -> module.WorkspaceResourceCleanupService:
+ return module.WorkspaceResourceCleanupService(
+ session_factory=session_factory,
+ apps=ConsoleAppRepository(session_factory=session_factory),
+ datasets=SQLAlchemyDatasetRepository(session_factory=session_factory),
+ agent_resources=WorkspaceResourceCleanupRepository(session_factory=session_factory),
+ )
+
+
+def test_agent_sweep_covers_all_scopes_and_retains_other_tenants(
+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ agents = [
+ Agent(
+ id="roster",
+ tenant_id="target",
+ name="Roster",
+ scope=AgentScope.ROSTER,
+ source=AgentSource.WORKFLOW,
+ backing_app_id="missing-app",
+ ),
+ Agent(
+ id="inline", tenant_id="target", name="Inline", scope=AgentScope.WORKFLOW_ONLY, source=AgentSource.WORKFLOW
+ ),
+ Agent(
+ id="archived",
+ tenant_id="target",
+ name="Archived",
+ scope=AgentScope.ROSTER,
+ source=AgentSource.WORKFLOW,
+ status=AgentStatus.ARCHIVED,
+ ),
+ Agent(id="other", tenant_id="other-tenant", name="Other", scope=AgentScope.ROSTER, source=AgentSource.WORKFLOW),
+ ]
+ snapshot = AgentHomeSnapshot(id="home", tenant_id="target", agent_id="inline", snapshot_ref="backend-home")
+ sqlite_session.add_all([*agents, snapshot])
+ sqlite_session.commit()
+ collection = Mock()
+ app_cleanup = Mock()
+ monkeypatch.setattr(module, "enqueue_agent_resource_collection", collection)
+ monkeypatch.setattr(module, "remove_app_and_related_data_task", app_cleanup)
+
+ def assert_committed(**_kwargs: object) -> None:
+ sqlite_session.expire_all()
+ inline_agent = sqlite_session.get(Agent, "inline")
+ assert inline_agent is not None
+ assert inline_agent.status == AgentStatus.ARCHIVED
+ home_snapshot = sqlite_session.get(AgentHomeSnapshot, "home")
+ assert home_snapshot is not None
+ assert home_snapshot.status == AgentWorkingResourceStatus.RETIRED
+
+ collection.side_effect = assert_committed
+ _service(sqlite_session_factory)._retire_agents("target")
+ payload = collection.call_args.kwargs
+ assert set(payload["purge_agent_ids"]) == {"roster", "inline", "archived"}
+ assert payload["home_snapshot_ids"] == ["home"]
+ other_agent = sqlite_session.get(Agent, "other")
+ assert other_agent is not None
+ assert other_agent.status == AgentStatus.ACTIVE
+ app_cleanup.delay.assert_called_once_with(tenant_id="target", app_id="missing-app")
+
+
+def test_dataset_deletion_schedules_even_without_index_configuration(
+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ sqlite_session.add_all(
+ [
+ Dataset(id="dataset", tenant_id="target", name="Empty", created_by="owner"),
+ Dataset(id="other-dataset", tenant_id="other", name="Other", created_by="owner"),
+ ]
+ )
+ sqlite_session.commit()
+ cleanup = Mock()
+ monkeypatch.setattr(module, "clean_dataset_task", cleanup)
+ monkeypatch.setattr(module.dataset_api_key_bindings, "delete_keys_scoped_only_to", Mock())
+ monkeypatch.setattr(module, "build_resource_access_token_cleanup_service", Mock())
+ _service(sqlite_session_factory)._delete_dataset("target", "other-dataset")
+ cleanup.delay.assert_not_called()
+ _service(sqlite_session_factory)._delete_dataset("target", "dataset")
+ cleanup.delay.assert_called_once()
+ assert cleanup.delay.call_args.args[:2] == ("dataset", "target")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(Dataset, "dataset") is None
+ assert sqlite_session.get(Dataset, "other-dataset") is not None
+
+
+def test_agent_collection_failure_propagates_and_retirement_can_be_retried(
+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ sqlite_session.add(
+ Agent(id="agent", tenant_id="target", name="Agent", scope=AgentScope.ROSTER, source=AgentSource.WORKFLOW)
+ )
+ sqlite_session.commit()
+ collection = Mock(side_effect=RuntimeError("broker unavailable"))
+ monkeypatch.setattr(module, "enqueue_agent_resource_collection", collection)
+ with pytest.raises(RuntimeError, match="broker unavailable"):
+ _service(sqlite_session_factory)._retire_agents("target")
+ collection.side_effect = None
+ _service(sqlite_session_factory)._retire_agents("target")
+ assert collection.call_args.kwargs["purge_agent_ids"] == ["agent"]
+
+
+def test_working_resources_are_retired_even_when_the_app_is_missing(
+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from models.agent import (
+ AgentConfigVersionKind,
+ AgentWorkspace,
+ AgentWorkspaceBinding,
+ AgentWorkspaceOwnerType,
+ )
+
+ workspace = AgentWorkspace(
+ id="working",
+ tenant_id="target",
+ app_id="missing-app",
+ owner_type=AgentWorkspaceOwnerType.CONVERSATION,
+ owner_id="conversation",
+ owner_scope_key="conversation",
+ backend_workspace_ref="backend-workspace",
+ )
+ binding = AgentWorkspaceBinding(
+ id="binding",
+ tenant_id="target",
+ app_id="missing-app",
+ workspace_id="working",
+ agent_id="missing-agent",
+ agent_config_version_id="config",
+ agent_config_version_kind=AgentConfigVersionKind.SNAPSHOT,
+ backend_binding_ref="backend-binding",
+ )
+ sqlite_session.add_all([workspace, binding])
+ sqlite_session.commit()
+ collection = Mock()
+ monkeypatch.setattr(module, "enqueue_agent_resource_collection", collection)
+ _service(sqlite_session_factory)._retire_agents("target")
+ sqlite_session.expire_all()
+ working_workspace = sqlite_session.get(AgentWorkspace, "working")
+ assert working_workspace is not None
+ assert working_workspace.status == AgentWorkingResourceStatus.RETIRED
+ working_binding = sqlite_session.get(AgentWorkspaceBinding, "binding")
+ assert working_binding is not None
+ assert working_binding.status == AgentWorkingResourceStatus.RETIRED
+ assert collection.call_args.kwargs["workspace_ids"] == ["working"]
+ assert collection.call_args.kwargs["binding_ids"] == ["binding"]
+
+
+def test_resource_failure_does_not_skip_the_other_groups(
+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from models import App
+ from models.enums import AppStatus
+ from models.model import AppMode
+
+ sqlite_session.add_all(
+ [
+ App(
+ id="app",
+ tenant_id="target",
+ name="App",
+ mode=AppMode.WORKFLOW,
+ status=AppStatus.NORMAL,
+ enable_site=True,
+ enable_api=True,
+ ),
+ Dataset(id="dataset", tenant_id="target", name="Dataset", created_by="owner"),
+ ]
+ )
+ sqlite_session.commit()
+ app_service = Mock()
+ app_service.return_value.delete_app.side_effect = RuntimeError("app failed")
+ monkeypatch.setattr(module, "AppService", app_service)
+ datasets = Mock()
+ agents = Mock()
+ monkeypatch.setattr(module.WorkspaceResourceCleanupService, "_delete_dataset", datasets)
+ monkeypatch.setattr(module.WorkspaceResourceCleanupService, "_retire_agents", agents)
+ with pytest.raises(RuntimeError, match="app:app"):
+ _service(sqlite_session_factory).cleanup("target")
+ datasets.assert_called_once_with("target", "dataset")
+ agents.assert_called_once_with("target")
+
+
+def test_agent_skill_snapshots_and_orphan_homes_are_included(
+ sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from models.skill import AgentSkillBindingSnapshot
+
+ sqlite_session.add_all(
+ [
+ AgentHomeSnapshot(id="orphan-home", tenant_id="target", agent_id="missing", snapshot_ref="backend"),
+ AgentSkillBindingSnapshot(
+ id="target-binding",
+ tenant_id="target",
+ agent_id="missing",
+ config_snapshot_id="config",
+ skill_id="skill",
+ priority=0,
+ ),
+ AgentSkillBindingSnapshot(
+ id="other-binding",
+ tenant_id="other",
+ agent_id="missing",
+ config_snapshot_id="config",
+ skill_id="skill",
+ priority=0,
+ ),
+ ]
+ )
+ sqlite_session.commit()
+ collection = Mock()
+ monkeypatch.setattr(module, "enqueue_agent_resource_collection", collection)
+ _service(sqlite_session_factory)._retire_agents("target")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(AgentSkillBindingSnapshot, "target-binding") is None
+ assert sqlite_session.get(AgentSkillBindingSnapshot, "other-binding") is not None
+ orphan_home = sqlite_session.get(AgentHomeSnapshot, "orphan-home")
+ assert orphan_home is not None
+ assert orphan_home.status == AgentWorkingResourceStatus.RETIRED
+ assert collection.call_args.kwargs["home_snapshot_ids"] == ["orphan-home"]
+
+
+def test_direct_cleanup_does_not_resolve_a_console_login(
+ app: Flask,
+ sqlite_session: Session,
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from unittest.mock import MagicMock
+
+ from models import App
+ from models.enums import AppStatus
+ from models.model import AppMode
+ from services import app_service
+
+ sqlite_session.add(
+ App(
+ id="app",
+ tenant_id="target",
+ name="App",
+ mode=AppMode.WORKFLOW,
+ status=AppStatus.NORMAL,
+ enable_site=True,
+ enable_api=True,
+ )
+ )
+ sqlite_session.commit()
+ user = MagicMock()
+ user.__bool__.side_effect = AssertionError("inner API must not load a console user")
+ monkeypatch.setattr(app_service, "current_user", user)
+ monkeypatch.setattr(app_service.app_was_deleted, "send", Mock())
+ monkeypatch.setattr(app_service, "build_resource_access_token_cleanup_service", Mock())
+ monkeypatch.setattr(app_service, "remove_app_and_related_data_task", Mock())
+ retirement = Mock()
+ monkeypatch.setattr(app_service, "WorkflowAgentRetirementService", retirement)
+ monkeypatch.setattr(app_service, "enqueue_agent_resource_collection", Mock())
+ features = Mock()
+ features.is_webapp_auth_enabled.return_value = False
+ monkeypatch.setattr(app_service, "SystemFeatureService", features)
+ monkeypatch.setattr(app_service, "BillingService", Mock())
+ monkeypatch.setattr(module, "enqueue_agent_resource_collection", Mock())
+ with app.test_request_context(method="DELETE"):
+ _service(sqlite_session_factory).cleanup("target")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(App, "app") is None
+ assert retirement.retire_unowned.call_args.kwargs["account_id"] is None
+ user.__bool__.assert_not_called()
+
+
+def test_tenant_home_snapshot_retirement_is_tenant_scoped_and_idempotent(sqlite_session: Session) -> None:
+ from datetime import datetime
+
+ from services.agent.home_snapshot_service import AgentHomeSnapshotService
+
+ retired_at = datetime(2026, 1, 1)
+ sqlite_session.add_all(
+ [
+ AgentHomeSnapshot(id="active", tenant_id="target", agent_id="agent", snapshot_ref="a"),
+ AgentHomeSnapshot(
+ id="retired",
+ tenant_id="target",
+ agent_id="missing",
+ snapshot_ref="b",
+ status=AgentWorkingResourceStatus.RETIRED,
+ retired_at=retired_at,
+ ),
+ AgentHomeSnapshot(id="other", tenant_id="other", agent_id="agent", snapshot_ref="c"),
+ ]
+ )
+ sqlite_session.commit()
+ ids = AgentHomeSnapshotService.retire_all_for_tenant(session=sqlite_session, tenant_id="target")
+ sqlite_session.commit()
+ assert set(ids) == {"active", "retired"}
+ active = sqlite_session.get(AgentHomeSnapshot, "active")
+ retired = sqlite_session.get(AgentHomeSnapshot, "retired")
+ other = sqlite_session.get(AgentHomeSnapshot, "other")
+ assert active is not None
+ assert active.status == AgentWorkingResourceStatus.RETIRED
+ assert retired is not None
+ assert retired.retired_at == retired_at
+ assert other is not None
+ assert other.status == AgentWorkingResourceStatus.ACTIVE
diff --git a/api/tests/unit_tests/tasks/test_clean_dataset_task.py b/api/tests/unit_tests/tasks/test_clean_dataset_task.py
index 3ac74ec1f4f..6cf16b42ce2 100644
--- a/api/tests/unit_tests/tasks/test_clean_dataset_task.py
+++ b/api/tests/unit_tests/tasks/test_clean_dataset_task.py
@@ -579,3 +579,59 @@ class TestIndexProcessorParameters:
)
schedule_refresh.assert_not_called()
+
+
[email protected]("shared_tenant", [None, "same", "other"])
+def test_crawl_cache_cleanup_preserves_shared_jobs(orm_session_maker, tenant_id, dataset_id, shared_tenant):
+ from services import dataset_storage_cleanup_service as cleanup
+
+ document = _persist_document(orm_session_maker, tenant_id=tenant_id, dataset_id=dataset_id)
+ with orm_session_maker.begin() as session:
+ row = session.get(Document, document.id)
+ row.data_source_type = "website_crawl"
+ row.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})
+ if shared_tenant:
+ other = _persist_document(
+ orm_session_maker,
+ tenant_id=tenant_id if shared_tenant == "same" else str(uuid.uuid4()),
+ dataset_id=str(uuid.uuid4()),
+ )
+ with orm_session_maker.begin() as session:
+ row = session.get(Document, other.id)
+ row.data_source_type = "website_crawl"
+ row.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})
+ with patch.object(cleanup, "storage", autospec=True) as mock:
+ cleanup.cleanup_dataset_crawl_cache(tenant_id, dataset_id)
+ if shared_tenant:
+ mock.delete.assert_not_called()
+ else:
+ mock.delete.assert_called_once_with("website_files/job-42.txt")
+
+
[email protected]("error", [FileNotFoundError(), PermissionError("denied")])
+def test_crawl_cache_failure_preserves_document(orm_session_maker, tenant_id, dataset_id, error):
+ from services import dataset_storage_cleanup_service as cleanup
+
+ document = _persist_document(orm_session_maker, tenant_id=tenant_id, dataset_id=dataset_id)
+ with orm_session_maker.begin() as session:
+ row = session.get(Document, document.id)
+ row.data_source_type = "website_crawl"
+ row.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})
+ with patch.object(cleanup, "storage", autospec=True) as mock:
+ mock.delete.side_effect = error
+ if isinstance(error, PermissionError):
+ with pytest.raises(PermissionError):
+ cleanup.cleanup_dataset_crawl_cache(tenant_id, dataset_id)
+ else:
+ cleanup.cleanup_dataset_crawl_cache(tenant_id, dataset_id)
+ with orm_session_maker() as session:
+ assert session.get(Document, document.id) is not None
+
+
+ "info", ["broken", "[]", '{"provider":"watercrawl","job_id":"job"}', '{"provider":"firecrawl","job_id":"../job"}']
+)
+def test_crawl_cache_ignores_invalid_or_other_provider_info(info):
+ from services.dataset_storage_cleanup_service import _crawl_cache_key
+
+ assert _crawl_cache_key(info) is None
diff --git a/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py b/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py
index e1911e3aa10..35e8c782490 100644
--- a/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py
+++ b/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py
@@ -80,6 +80,7 @@ def test_app_cleanup_removes_agent_bindings_before_workflows(monkeypatch: pytest
events: list[str] = []
apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY)
other_cleanup_names = (
+ "cleanup_app_storage",
"_delete_app_model_configs",
"_delete_app_site",
"_delete_app_mcp_servers",