Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(dataset): batch segment and document cleanup with retries (#38519)
Alexey Shabalin638ed4eb5565
diff --git a/api/providers/vdb/vdb-elasticsearch/src/dify_vdb_elasticsearch/elasticsearch_vector.py b/api/providers/vdb/vdb-elasticsearch/src/dify_vdb_elasticsearch/elasticsearch_vector.pyindex d1c8ced881a..8254f2a4bf7 100644--- a/api/providers/vdb/vdb-elasticsearch/src/dify_vdb_elasticsearch/elasticsearch_vector.py+++ b/api/providers/vdb/vdb-elasticsearch/src/dify_vdb_elasticsearch/elasticsearch_vector.py@@ -5,7 +5,7 @@ from typing import Any, cast, overridefrom urllib.parse import urlparsefrom elasticsearch import ConnectionError as ElasticsearchConnectionError-from elasticsearch import Elasticsearch+from elasticsearch import Elasticsearch, NotFoundErrorfrom flask import current_appfrom packaging.version import parse as parse_versionfrom pydantic import BaseModel, model_validator@@ -195,8 +195,15 @@ class ElasticSearchVector(BaseVector):self.delete_by_ids(ids)@override- def delete(self):- self._client.indices.delete(index=self._collection_name)+ def delete(self) -> None:+ try:+ self._client.indices.delete(index=self._collection_name)+ except NotFoundError as exc:+ error = exc.body.get("error") if isinstance(exc.body, dict) else None+ if not isinstance(error, dict) or error.get("type") != "index_not_found_exception":+ raise+ # A previous attempt may have deleted the index before a relational commit failed.+ logger.debug("Elasticsearch index already absent: %s", self._collection_name)@overridedef search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:diff --git a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.pyindex 7e0c9850925..8c626673abd 100644--- a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py+++ b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py@@ -6,6 +6,7 @@ from types import SimpleNamespacefrom unittest.mock import MagicMock, patchimport pytest+from elasticsearch import NotFoundErrorfrom models.dataset import Dataset@@ -27,19 +28,24 @@ def _build_fake_elasticsearch_modules():elasticsearch.Elasticsearch = Elasticsearchelasticsearch.ConnectionError = ConnectionError+ elasticsearch.__dict__["NotFoundError"] = NotFoundErrorreturn {"elasticsearch": elasticsearch}@pytest.fixturedef elasticsearch_ja_module(monkeypatch: pytest.MonkeyPatch):- for name, module in _build_fake_elasticsearch_modules().items():- monkeypatch.setitem(sys.modules, name, module)-import dify_vdb_elasticsearch.elasticsearch_ja_vector as ja_moduleimport dify_vdb_elasticsearch.elasticsearch_vector as base_module- importlib.reload(base_module)- return importlib.reload(ja_module)+ try:+ with monkeypatch.context() as module_patch:+ for name, fake_module in _build_fake_elasticsearch_modules().items():+ module_patch.setitem(sys.modules, name, fake_module)+ importlib.reload(base_module)+ yield importlib.reload(ja_module)+ finally:+ importlib.reload(base_module)+ importlib.reload(ja_module)def test_create_collection_cache_hit(elasticsearch_ja_module, monkeypatch: pytest.MonkeyPatch):diff --git a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.pyindex 80e85f75b71..59110786988 100644--- a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py+++ b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py@@ -6,6 +6,8 @@ from types import SimpleNamespacefrom unittest.mock import MagicMock, patchimport pytest+from elastic_transport import ApiResponseMeta, HttpHeaders, NodeConfig, TransportApiResponse+from elasticsearch import ApiError, Elasticsearch, NotFoundErrorfrom pydantic import ValidationErrorfrom core.rag.models.document import Document@@ -36,17 +38,23 @@ def _build_fake_elasticsearch_modules():elasticsearch.Elasticsearch = Elasticsearchelasticsearch.ConnectionError = ConnectionError+ elasticsearch.__dict__["NotFoundError"] = NotFoundErrorreturn {"elasticsearch": elasticsearch}@pytest.fixturedef elasticsearch_module(monkeypatch: pytest.MonkeyPatch):- for name, module in _build_fake_elasticsearch_modules().items():- monkeypatch.setitem(sys.modules, name, module)-import dify_vdb_elasticsearch.elasticsearch_vector as module- return importlib.reload(module)+ try:+ with monkeypatch.context() as module_patch:+ for name, fake_module in _build_fake_elasticsearch_modules().items():+ module_patch.setitem(sys.modules, name, fake_module)+ yield importlib.reload(module)+ finally:+ importlib.reload(module)+ if "dify_vdb_elasticsearch.elasticsearch_ja_vector" in sys.modules:+ importlib.reload(sys.modules["dify_vdb_elasticsearch.elasticsearch_ja_vector"])def _regular_config(module, **overrides):@@ -228,6 +236,69 @@ def test_crud_methods_and_get_type(elasticsearch_module):assert vector.get_type() == elasticsearch_module.VectorType.ELASTICSEARCH[email protected]("japanese", [False, True], ids=["elasticsearch", "elasticsearch-ja"])+ ("status", "body", "missing_index"),+ [+ pytest.param(+ 404,+ {"error": {"type": "index_not_found_exception", "reason": "no such index [collection_1]"}, "status": 404},+ True,+ id="missing-index",+ ),+ pytest.param(+ 404,+ {"error": {"type": "resource_not_found_exception", "reason": "resource missing"}, "status": 404},+ False,+ id="other-not-found",+ ),+ pytest.param(404, "Not Found", False, id="proxy-not-found"),+ pytest.param(404, "index_not_found_exception", False, id="unstructured-missing-index"),+ pytest.param(+ 404,+ {"error": "index_not_found_exception", "status": 404},+ False,+ id="string-error-missing-index",+ ),+ pytest.param(403, {"error": {"type": "security_exception"}, "status": 403}, False, id="permission-denied"),+ pytest.param(500, {"error": {"type": "index_not_found_exception"}, "status": 500}, False, id="server-error"),+ ],+)+def test_delete_only_ignores_missing_index(elasticsearch_module, japanese, status, body, missing_index):+ if japanese:+ import dify_vdb_elasticsearch.elasticsearch_ja_vector as ja_module++ vector_class = importlib.reload(ja_module).ElasticSearchJaVector+ else:+ vector_class = elasticsearch_module.ElasticSearchVector+ vector = vector_class.__new__(vector_class)+ vector._collection_name = "collection_1"+ response = TransportApiResponse(+ ApiResponseMeta(+ status=status,+ http_version="1.1",+ headers=HttpHeaders({"x-elastic-product": "Elasticsearch"}),+ duration=0,+ node=NodeConfig("http", "localhost", 9200),+ ),+ body,+ )+ with Elasticsearch("http://localhost:9200", max_retries=0) as client:+ vector._client = client+ # Keep SDK error parsing real; replace only the HTTP transport boundary.+ with patch.object(client.transport, "perform_request", return_value=response) as request:+ if missing_index:+ vector.delete()+ vector.delete()+ else:+ with pytest.raises(ApiError) as raised:+ vector.delete()+ assert raised.value.status_code == status+ assert raised.value.body == body+ assert request.call_count == (2 if missing_index else 1)+ assert request.call_args.args[:2] == ("DELETE", "/collection_1")++def test_search_by_vector_and_full_text(elasticsearch_module):vector = elasticsearch_module.ElasticSearchVector.__new__(elasticsearch_module.ElasticSearchVector)vector._collection_name = "collection_1"diff --git a/api/tasks/clean_dataset_task.py b/api/tasks/clean_dataset_task.pyindex 357d30d6d4b..e0751ae2566 100644--- a/api/tasks/clean_dataset_task.py+++ b/api/tasks/clean_dataset_task.py@@ -1,9 +1,11 @@import loggingimport time+from collections.abc import Callable, Sequenceimport clickfrom celery import shared_taskfrom sqlalchemy import delete, select+from sqlalchemy.orm import Sessionfrom core.db.session_factory import session_factoryfrom core.rag.index_processor.index_processor import IndexProcessorFactory@@ -29,10 +31,16 @@ from tasks.refresh_billing_vector_space_task import schedule_billing_vector_spaclogger = logging.getLogger(__name__)+# Keep committed batches on failure, but preserve dependent rows until a retry+# finishes their prerequisite cleanup.+_DELETE_BATCH_SIZE = 1000+_MAX_RETRIES = 3+_RETRY_DELAY_SECONDS = 30-# Add import statement for ValueError-@shared_task(queue="dataset")++@shared_task(queue="dataset", bind=True, max_retries=_MAX_RETRIES, default_retry_delay=_RETRY_DELAY_SECONDS)def clean_dataset_task(+ self,dataset_id: str,tenant_id: str,indexing_technique: str | None,@@ -40,7 +48,9 @@ def clean_dataset_task(collection_binding_id: str | None,doc_form: str,pipeline_id: str | None = None,-):+ *,+ index_cleanup_done: bool = False,+) -> None:"""Clean dataset when dataset deleted.:param dataset_id: dataset id@@ -49,171 +59,209 @@ def clean_dataset_task(:param index_struct: index struct dict:param collection_binding_id: collection binding id:param doc_form: dataset form+ :param index_cleanup_done: preserve successful index cleanup across explicit retriesUsage: clean_dataset_task.delay(dataset_id, tenant_id, indexing_technique, index_struct)"""logger.info(click.style(f"Start clean dataset when dataset deleted: {dataset_id}", fg="green"))start_at = time.perf_counter()- vector_cleanup_succeeded = False- cleanup_dataset_crawl_cache(tenant_id, dataset_id)+ # Enhanced validation: Check if doc_form is None, empty string, or contains only whitespace.+ # Treating all invalid values as the default paragraph index type keeps the vector+ # database cleanup reachable for empty/broken datasets.+ if doc_form is None or (isinstance(doc_form, str) and not doc_form.strip()):+ from core.rag.index_processor.constant.index_type import IndexStructureType- with session_factory.create_session() as session:- try:- dataset = Dataset(- id=dataset_id,- tenant_id=tenant_id,- indexing_technique=indexing_technique,- index_struct=index_struct,- collection_binding_id=collection_binding_id,- )- documents = session.scalars(select(Document).where(Document.dataset_id == dataset_id)).all()- segments = session.scalars(select(DocumentSegment).where(DocumentSegment.dataset_id == dataset_id)).all()- # Use JOIN to fetch attachments with bindings in a single query- attachments_with_bindings = session.execute(- select(SegmentAttachmentBinding, UploadFile)- .join(UploadFile, UploadFile.id == SegmentAttachmentBinding.attachment_id)- .where(- SegmentAttachmentBinding.tenant_id == tenant_id,- SegmentAttachmentBinding.dataset_id == dataset_id,- )- ).all()-- # Enhanced validation: Check if doc_form is None, empty string, or contains only whitespace- # This ensures all invalid doc_form values are properly handled- if doc_form is None or (isinstance(doc_form, str) and not doc_form.strip()):- # Use default paragraph index type for empty/invalid datasets to enable vector database cleanup- from core.rag.index_processor.constant.index_type import IndexStructureType-- doc_form = IndexStructureType.PARAGRAPH_INDEX- logger.info(- click.style(- f"Invalid doc_form detected, using default index type for cleanup: {doc_form}",- fg="yellow",- )- )-- # Add exception handling around IndexProcessorFactory.clean() to prevent single point of failure- # This ensures Document/Segment deletion can continue even if vector database cleanup fails- try:- index_processor = IndexProcessorFactory(doc_form).init_index_processor()- index_processor.clean(dataset, None, with_keywords=True, delete_child_chunks=True, session=session)- vector_cleanup_succeeded = True- logger.info(click.style(f"Successfully cleaned vector database for dataset: {dataset_id}", fg="green"))- except Exception:- logger.exception(click.style(f"Failed to clean vector database for dataset {dataset_id}", fg="red"))- # Continue with document and segment deletion even if vector cleanup fails- logger.info(- click.style(f"Continuing with document and segment deletion for dataset: {dataset_id}", fg="yellow")- )-- if documents is None or len(documents) == 0:- logger.info(click.style(f"No documents found for dataset: {dataset_id}", fg="green"))- else:- logger.info(click.style(f"Cleaning documents for dataset: {dataset_id}", fg="green"))-- for document in documents:- session.delete(document)-- segment_ids = [segment.id for segment in segments]- for segment in segments:- image_upload_file_ids = get_image_upload_file_ids(segment.content)- image_files = session.scalars(- select(UploadFile).where(UploadFile.id.in_(image_upload_file_ids))- ).all()- for image_file in image_files:- if image_file is None:- continue- try:- storage.delete(image_file.key)- except Exception:- logger.exception(- "Delete image_files failed when storage deleted, \- image_upload_file_is: %s",- image_file.id,- )- stmt = delete(UploadFile).where(UploadFile.id.in_(image_upload_file_ids))- session.execute(stmt)-- segment_delete_stmt = delete(DocumentSegment).where(DocumentSegment.id.in_(segment_ids))- session.execute(segment_delete_stmt)- # delete segment attachments- if attachments_with_bindings:- attachment_ids = [attachment_file.id for _, attachment_file in attachments_with_bindings]- binding_ids = [binding.id for binding, _ in attachments_with_bindings]- for binding, attachment_file in attachments_with_bindings:- try:- storage.delete(attachment_file.key)- except Exception:- logger.exception(- "Delete attachment_file failed when storage deleted, \- attachment_file_id: %s",- binding.attachment_id,- )- attachment_file_delete_stmt = delete(UploadFile).where(UploadFile.id.in_(attachment_ids))- session.execute(attachment_file_delete_stmt)-- binding_delete_stmt = delete(SegmentAttachmentBinding).where(- SegmentAttachmentBinding.id.in_(binding_ids)- )- session.execute(binding_delete_stmt)-- session.execute(delete(DatasetProcessRule).where(DatasetProcessRule.dataset_id == dataset_id))- session.execute(delete(DatasetQuery).where(DatasetQuery.dataset_id == dataset_id))- session.execute(delete(AppDatasetJoin).where(AppDatasetJoin.dataset_id == dataset_id))- # delete dataset metadata- session.execute(delete(DatasetMetadata).where(DatasetMetadata.dataset_id == dataset_id))- session.execute(delete(DatasetMetadataBinding).where(DatasetMetadataBinding.dataset_id == dataset_id))- # delete pipeline and workflow- if pipeline_id:- session.execute(delete(Pipeline).where(Pipeline.id == pipeline_id))- session.execute(- delete(Workflow).where(- Workflow.tenant_id == tenant_id,- Workflow.app_id == pipeline_id,- Workflow.type == WorkflowType.RAG_PIPELINE,- )- )- # delete files- if documents:- file_ids = []- for document in documents:- if document.data_source_type == "upload_file":- if document.data_source_info:- data_source_info = document.data_source_info_dict- if data_source_info and "upload_file_id" in data_source_info:- file_id = data_source_info["upload_file_id"]- file_ids.append(file_id)- files = session.scalars(select(UploadFile).where(UploadFile.id.in_(file_ids))).all()- for file in files:- storage.delete(file.key)-- file_delete_stmt = delete(UploadFile).where(UploadFile.id.in_(file_ids))- session.execute(file_delete_stmt)-- session.commit()- if vector_cleanup_succeeded:+ doc_form = IndexStructureType.PARAGRAPH_INDEX+ logger.info(+ click.style(f"Invalid doc_form detected, using default index type for cleanup: {doc_form}", fg="yellow")+ )++ dataset = Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ indexing_technique=indexing_technique,+ index_struct=index_struct,+ collection_binding_id=collection_binding_id,+ )++ try:+ cleanup_dataset_crawl_cache(tenant_id, dataset_id)++ with session_factory.create_session() as session:+ if not index_cleanup_done:+ # Index processors also delete relational child rows, so the whole step must succeed first.+ _run_step(session, dataset_id, "dataset index", lambda: _clean_index(session, dataset, doc_form))+ index_cleanup_done = Trueschedule_billing_vector_space_refresh(dataset.tenant_id)- end_at = time.perf_counter()- logger.info(- click.style(- f"Cleaned dataset when dataset deleted: {dataset_id} latency: {end_at - start_at}",- fg="green",- )++ # Stop on failure so documents and their resources outlive any remaining segments.+ _run_step(session, dataset_id, "segments", lambda: _delete_segments(session, dataset_id))+ _run_step(session, dataset_id, "documents", lambda: _delete_documents(session, dataset_id))+ _run_step(session, dataset_id, "attachments", lambda: _delete_attachments(session, tenant_id, dataset_id))+ _run_step(+ session,+ dataset_id,+ "dataset metadata",+ lambda: _delete_dataset_scoped_rows(session, tenant_id, dataset_id, pipeline_id),)++ except Exception as exc:+ if self.request.retries >= _MAX_RETRIES:+ logger.exception("Dataset cleanup retry budget exhausted, dataset_id=%s", dataset_id)+ raise++ logger.warning(+ "Dataset cleanup failed, scheduling retry %d/%d, dataset_id=%s",+ self.request.retries + 1,+ _MAX_RETRIES,+ dataset_id,+ exc_info=True,+ )+ # The session is closed before retrying. Normalize arguments for positional callers too.+ raise self.retry(+ exc=exc,+ args=(),+ kwargs={+ "dataset_id": dataset_id,+ "tenant_id": tenant_id,+ "indexing_technique": indexing_technique,+ "index_struct": index_struct,+ "collection_binding_id": collection_binding_id,+ "doc_form": doc_form,+ "pipeline_id": pipeline_id,+ "index_cleanup_done": index_cleanup_done,+ },+ countdown=_RETRY_DELAY_SECONDS * (2**self.request.retries),+ ) from exc++ end_at = time.perf_counter()+ logger.info(+ click.style(+ f"Cleaned dataset when dataset deleted: {dataset_id} latency: {end_at - start_at}",+ fg="green",+ )+ )+++def _run_step(session: Session, dataset_id: str, name: str, step: Callable[[], None]) -> None:+ """Roll back uncommitted work on failure and stop dependent steps until retry."""+ try:+ step()+ except Exception:+ logger.exception(click.style(f"Failed to clean {name} for dataset {dataset_id}", fg="red"))+ try:+ session.rollback()+ except Exception:+ logger.exception("Failed to rollback database session")+ raise+++def _clean_index(session: Session, dataset: Dataset, doc_form: str) -> None:+ """Clean the dataset index and commit any relational rows removed by its processor."""+ index_processor = IndexProcessorFactory(doc_form).init_index_processor()+ index_processor.clean(dataset, None, with_keywords=True, delete_child_chunks=True, session=session)+ session.commit()+ logger.info(click.style(f"Successfully cleaned vector database for dataset: {dataset.id}", fg="green"))+++def _delete_segments(session: Session, dataset_id: str) -> None:+ """Delete document segments in batches, removing their inline image files first."""+ while True:+ segments = session.scalars(+ select(DocumentSegment).where(DocumentSegment.dataset_id == dataset_id).limit(_DELETE_BATCH_SIZE)+ ).all()+ if not segments:+ break+ image_upload_file_ids: list[str] = []+ for segment in segments:+ image_upload_file_ids.extend(get_image_upload_file_ids(segment.content))+ _delete_upload_files(session, image_upload_file_ids)+ session.execute(delete(DocumentSegment).where(DocumentSegment.id.in_([segment.id for segment in segments])))+ session.commit()+++def _delete_documents(session: Session, dataset_id: str) -> None:+ """Delete documents in batches, removing their uploaded source files first."""+ while True:+ documents = session.scalars(+ select(Document).where(Document.dataset_id == dataset_id).limit(_DELETE_BATCH_SIZE)+ ).all()+ if not documents:+ break+ file_ids: list[str] = []+ for document in documents:+ if document.data_source_type == "upload_file" and document.data_source_info:+ data_source_info = document.data_source_info_dict+ if data_source_info and "upload_file_id" in data_source_info:+ file_ids.append(data_source_info["upload_file_id"])+ _delete_upload_files(session, file_ids)+ session.execute(delete(Document).where(Document.id.in_([document.id for document in documents])))+ session.commit()+++def _delete_upload_files(session: Session, file_ids: Sequence[str]) -> None:+ """Delete blobs before their lookup rows so failures retain storage keys for retry."""+ if not file_ids:+ return+ files = session.scalars(select(UploadFile).where(UploadFile.id.in_(file_ids))).all()+ for file in files:+ try:+ storage.delete(file.key)+ except FileNotFoundError:+ logger.debug("Upload file already absent from storage, upload_file_id: %s", file.id)+ session.execute(delete(UploadFile).where(UploadFile.id.in_(file_ids)))+++def _delete_attachments(session: Session, tenant_id: str, dataset_id: str) -> None:+ """Delete segment attachments (upload files + bindings) for the dataset."""+ # Inline-image cleanup may already have deleted the file; its binding still needs cleanup.+ attachments_with_bindings = session.execute(+ select(SegmentAttachmentBinding, UploadFile)+ .outerjoin(UploadFile, UploadFile.id == SegmentAttachmentBinding.attachment_id)+ .where(+ SegmentAttachmentBinding.tenant_id == tenant_id,+ SegmentAttachmentBinding.dataset_id == dataset_id,+ )+ ).all()+ if not attachments_with_bindings:+ return+ attachment_ids = [+ attachment_file.id for _, attachment_file in attachments_with_bindings if attachment_file is not None+ ]+ binding_ids = [binding.id for binding, _ in attachments_with_bindings]+ for binding, attachment_file in attachments_with_bindings:+ if attachment_file is None:+ continue+ try:+ storage.delete(attachment_file.key)except Exception:- # Add rollback to prevent dirty session state in case of exceptions- # This ensures the database session is properly cleaned up- try:- session.rollback()- logger.info(click.style(f"Rolled back database session for dataset: {dataset_id}", fg="yellow"))- except Exception:- logger.exception("Failed to rollback database session")-- logger.exception("Cleaned dataset when dataset deleted failed")- finally:- # Explicitly close the session for test expectations and safety- try:- session.close()- except Exception:- logger.exception("Failed to close database session")+ logger.exception(+ "Delete attachment_file failed when storage deleted, attachment_file_id: %s",+ binding.attachment_id,+ )+ if attachment_ids:+ session.execute(delete(UploadFile).where(UploadFile.id.in_(attachment_ids)))+ session.execute(delete(SegmentAttachmentBinding).where(SegmentAttachmentBinding.id.in_(binding_ids)))+ session.commit()+++def _delete_dataset_scoped_rows(session: Session, tenant_id: str, dataset_id: str, pipeline_id: str | None) -> None:+ """Delete the remaining small dataset-scoped tables (and the pipeline/workflow)."""+ session.execute(delete(DatasetProcessRule).where(DatasetProcessRule.dataset_id == dataset_id))+ session.execute(delete(DatasetQuery).where(DatasetQuery.dataset_id == dataset_id))+ session.execute(delete(AppDatasetJoin).where(AppDatasetJoin.dataset_id == dataset_id))+ # delete dataset metadata+ session.execute(delete(DatasetMetadata).where(DatasetMetadata.dataset_id == dataset_id))+ session.execute(delete(DatasetMetadataBinding).where(DatasetMetadataBinding.dataset_id == dataset_id))+ # delete pipeline and workflow+ if pipeline_id:+ session.execute(delete(Pipeline).where(Pipeline.id == pipeline_id))+ session.execute(+ delete(Workflow).where(+ Workflow.tenant_id == tenant_id,+ Workflow.app_id == pipeline_id,+ Workflow.type == WorkflowType.RAG_PIPELINE,+ )+ )+ session.commit()diff --git a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.pyindex d6cdd01c0b9..dc4fc8d7c09 100644--- a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py+++ b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py@@ -523,19 +523,10 @@ class TestCleanDatasetTask:# Should be called 4 times (once for each test case)assert mock_factory.call_count == 4- def test_clean_dataset_task_error_handling_and_rollback(+ def test_index_cleanup_failure_preserves_documents_and_segments(self, db_session_with_containers: Session, mock_external_service_dependencies- ):- """- Test error handling and rollback mechanism when database operations fail.-- This test verifies that the task can properly:- 1. Handle database operation failures gracefully- 2. Rollback database session to prevent dirty state- 3. Continue cleanup operations even if some parts fail- 4. Log appropriate error messages- 5. Maintain database session integrity- """+ ) -> None:+ """An incomplete index cleanup must preserve its dependent records for retry."""# Create test dataaccount, tenant = self._create_test_account_and_tenant(db_session_with_containers)dataset = self._create_test_dataset(db_session_with_containers, account, tenant)@@ -544,37 +535,31 @@ class TestCleanDatasetTask:# Mock IndexProcessorFactory to raise an exceptionmock_index_processor = mock_external_service_dependencies["index_processor"]- mock_index_processor.clean.side_effect = Exception("Vector database cleanup failed")-- # Execute the task - it should handle the exception gracefully- clean_dataset_task(- dataset_id=dataset.id,- tenant_id=tenant.id,- indexing_technique=dataset.indexing_technique,- index_struct=dataset.index_struct,- collection_binding_id=dataset.collection_binding_id,- doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),- )+ mock_index_processor.clean.side_effect = RuntimeError("Vector database cleanup failed")- # Verify results - even with vector cleanup failure, documents and segments should be deleted+ # A direct call propagates the original exception instead of publishing a retry.+ with pytest.raises(RuntimeError, match="Vector database cleanup failed"):+ clean_dataset_task(+ dataset_id=dataset.id,+ tenant_id=tenant.id,+ indexing_technique=dataset.indexing_technique,+ index_struct=dataset.index_struct,+ collection_binding_id=dataset.collection_binding_id,+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),+ )- # Check that documents were still deleted despite vector cleanup failureremaining_documents = db_session_with_containers.scalars(select(Document).where(Document.dataset_id == dataset.id)).all()- assert len(remaining_documents) == 0+ assert len(remaining_documents) == 1- # Check that segments were still deleted despite vector cleanup failureremaining_segments = db_session_with_containers.scalars(select(DocumentSegment).where(DocumentSegment.dataset_id == dataset.id)).all()- assert len(remaining_segments) == 0+ assert len(remaining_segments) == 1- # Verify that index processor was called and failedmock_index_processor.clean.assert_called_once()-- # Verify that the task continued with cleanup despite the error- # This demonstrates the resilience of the cleanup process+ mock_external_service_dependencies["storage"].delete.assert_not_called()def test_clean_dataset_task_with_image_file_references(self, db_session_with_containers: Session, mock_external_service_dependencies@@ -809,74 +794,54 @@ class TestCleanDatasetTask:print(f"Total cleanup time: {cleanup_duration:.3f} seconds")print(f"Average time per document: {cleanup_duration / len(documents):.3f} seconds")- def test_clean_dataset_task_storage_exception_handling(+ def test_source_storage_failure_preserves_upload_file_and_document(self, db_session_with_containers: Session, mock_external_service_dependencies- ):- """- Test dataset cleanup when storage operations fail.-- This test verifies that the task can properly:- 1. Handle storage deletion failures gracefully- 2. Continue cleanup process despite storage errors- 3. Log appropriate error messages for storage failures- 4. Maintain database consistency even with storage issues- 5. Provide meaningful error reporting- """+ ) -> None:+ """Keep source storage keys and documents available for a cleanup retry."""# Create test dataaccount, tenant = self._create_test_account_and_tenant(db_session_with_containers)dataset = self._create_test_dataset(db_session_with_containers, account, tenant)document = self._create_test_document(db_session_with_containers, account, tenant, dataset)self._create_test_segment(db_session_with_containers, account, tenant, dataset, document)upload_file = self._create_test_upload_file(db_session_with_containers, account, tenant)+ upload_file_id = upload_file.id+ upload_file_key = upload_file.key+ document_id = document.id# Update document with file referenceimport json- document.data_source_info = json.dumps({"upload_file_id": upload_file.id})+ document.data_source_info = json.dumps({"upload_file_id": upload_file_id})db_session_with_containers.commit()# Mock storage to raise exceptionsmock_storage = mock_external_service_dependencies["storage"]- mock_storage.delete.side_effect = Exception("Storage service unavailable")-- # Execute the task - it should handle storage failures gracefully- clean_dataset_task(- dataset_id=dataset.id,- tenant_id=tenant.id,- indexing_technique=dataset.indexing_technique,- index_struct=dataset.index_struct,- collection_binding_id=dataset.collection_binding_id,- doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),- )+ mock_storage.delete.side_effect = RuntimeError("Storage service unavailable")- # Verify results- # Note: When storage operations fail, database deletions may be rolled back by implementation.- # This test focuses on ensuring the task handles the exception and continues execution/logging.+ with pytest.raises(RuntimeError, match="Storage service unavailable"):+ clean_dataset_task(+ dataset_id=dataset.id,+ tenant_id=tenant.id,+ indexing_technique=dataset.indexing_technique,+ index_struct=dataset.index_struct,+ collection_binding_id=dataset.collection_binding_id,+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),+ )- # Check that upload file was still deleted from database despite storage failure- # Note: When storage operations fail, the upload file may not be deleted- # This demonstrates that the cleanup process continues even with storage errors- remaining_files = db_session_with_containers.scalars(- select(UploadFile).where(UploadFile.id == upload_file.id)- ).all()- # The upload file should still be deleted from the database even if storage cleanup fails- # However, this depends on the specific implementation of clean_dataset_task- if len(remaining_files) > 0:- print(f"Warning: Upload file {upload_file.id} was not deleted despite storage failure")- print("This demonstrates that the cleanup process continues even with storage errors")- # We don't assert here as the behavior depends on the specific implementation+ db_session_with_containers.expire_all()+ remaining_file = db_session_with_containers.get(UploadFile, upload_file_id)+ assert remaining_file is not None+ assert remaining_file.key == upload_file_key+ remaining_document = db_session_with_containers.get(Document, document_id)+ assert remaining_document is not None+ assert remaining_document.data_source_info_dict == {"upload_file_id": upload_file_id}- # Verify that storage.delete was called- mock_storage.delete.assert_called_once()+ mock_storage.delete.assert_called_once_with(upload_file_key)# Verify that index processor was called successfullymock_index_processor = mock_external_service_dependencies["index_processor"]mock_index_processor.clean.assert_called_once()- # This test demonstrates that the cleanup process continues- # even when external storage operations fail, ensuring data- # consistency in the database-def test_clean_dataset_task_edge_cases_and_boundary_conditions(self, db_session_with_containers: Session, mock_external_service_dependencies):diff --git a/api/tests/unit_tests/tasks/test_clean_dataset_task.py b/api/tests/unit_tests/tasks/test_clean_dataset_task.pyindex 6cf16b42ce2..33e000371dc 100644--- a/api/tests/unit_tests/tasks/test_clean_dataset_task.py+++ b/api/tests/unit_tests/tasks/test_clean_dataset_task.py@@ -13,12 +13,18 @@ This module tests the dataset cleanup task functionality including:import jsonimport uuid-from collections.abc import Iterator+from collections.abc import Iterator, Sequencefrom datetime import UTC, datetime+from http.server import BaseHTTPRequestHandler, HTTPServer+from threading import Thread+from typing import overridefrom unittest.mock import MagicMock, patchimport pytest+from celery.exceptions import Retry+from flask import Flaskfrom sqlalchemy import Engine, event+from sqlalchemy.exc import OperationalErrorfrom sqlalchemy.orm import Session, sessionmakerfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType@@ -26,6 +32,7 @@ from extensions.storage.storage_type import StorageTypefrom models.base import TypeBasefrom models.dataset import (AppDatasetJoin,+ ChildChunk,DatasetMetadata,DatasetMetadataBinding,DatasetProcessRule,@@ -78,6 +85,7 @@ def orm_session_maker(models = (Document,DocumentSegment,+ ChildChunk,SegmentAttachmentBinding,UploadFile,DatasetProcessRule,@@ -125,6 +133,55 @@ def mock_get_image_upload_file_ids() -> Iterator[MagicMock]:yield mock_func+def elasticsearch_delete_endpoint() -> Iterator[tuple[int, list[int], list[tuple[str, int]]]]:+ """Serve real SDK requests with Elasticsearch's missing-index response shape."""+ pytest.importorskip("elasticsearch")+ pytest.importorskip("dify_vdb_elasticsearch.elasticsearch_vector")+ responses: list[int] = []+ requests: list[tuple[str, int]] = []++ class Handler(BaseHTTPRequestHandler):+ def respond(self, status: int, payload: dict[str, object]) -> None:+ body = json.dumps(payload).encode()+ self.send_response(status)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.send_header("X-Elastic-Product", "Elasticsearch")+ self.end_headers()+ if self.command != "HEAD":+ self.wfile.write(body)++ def do_HEAD(self) -> None:+ self.respond(200, {})++ def do_GET(self) -> None:+ self.respond(200, {"version": {"number": "8.14.0"}})++ def do_DELETE(self) -> None:+ status = responses.pop(0) if responses else 500+ requests.append((self.path, status))+ if status == 200:+ self.respond(status, {"acknowledged": True})+ else:+ error = {"type": "index_not_found_exception", "reason": "no such index [cleanup-index]"}+ self.respond(status, {"error": {**error, "root_cause": [error]}, "status": status})++ @override+ def log_message(self, format: str, *args: object) -> None:+ pass++ server = HTTPServer(("127.0.0.1", 0), Handler)+ thread = Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True)+ thread.start()+ try:+ yield server.server_port, responses, requests+ finally:+ server.shutdown()+ thread.join(timeout=1)+ server.server_close()++def _run_clean_dataset(*,dataset_id: str,@@ -218,6 +275,29 @@ def _persist_attachment(return binding, attachment_file+def _persist_segment(+ session_maker: sessionmaker[Session],+ *,+ dataset_id: str,+ tenant_id: str,+ document_id: str,+ position: int = 1,+) -> DocumentSegment:+ segment = DocumentSegment(+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_id=document_id,+ position=position,+ content="segment",+ word_count=1,+ tokens=1,+ created_by=str(uuid.uuid4()),+ )+ with session_maker.begin() as session:+ session.add(segment)+ return segment++# ============================================================================# Test Basic Cleanup# ============================================================================@@ -267,15 +347,17 @@ class TestErrorHandling:# Acttry:- _run_clean_dataset(- dataset_id=dataset_id,- tenant_id=tenant_id,- collection_binding_id=collection_binding_id,- )+ with patch.object(clean_dataset_task, "retry", side_effect=Retry()) as retry, pytest.raises(Retry):+ _run_clean_dataset(+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ collection_binding_id=collection_binding_id,+ )finally:event.remove(orm_session_maker.class_, "before_commit", fail_commit)event.remove(orm_session_maker.class_, "after_rollback", fail_rollback)+ retry.assert_called_once()# Assert: rollback happened before its hook failed, and the closed task# session did not prevent a new real session from reading the row.with orm_session_maker() as session:@@ -372,6 +454,82 @@ class TestPipelineAndWorkflowDeletion:class TestSegmentAttachmentCleanup:"""Test cases for segment attachment cleanup."""+ @pytest.mark.parametrize("storage_fails", [False, True])+ def test_inline_image_attachment_binding_is_deleted_after_its_file(+ self,+ storage_fails: bool,+ dataset_id: str,+ tenant_id: str,+ collection_binding_id: str,+ orm_session_maker: sessionmaker[Session],+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ ) -> None:+ """Inline images, source files and attachments can share an uploaded file."""+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+ segment = _persist_segment(+ orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id+ )+ binding, attachment_file = _persist_attachment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+ segment.content = f""+ document.data_source_type = "upload_file"+ document.data_source_info = json.dumps({"upload_file_id": attachment_file.id})+ binding.document_id = document.id+ binding.segment_id = segment.id+ with orm_session_maker.begin() as session:+ session.add_all([document, segment, binding])+ unrelated_attachments = [+ _persist_attachment(orm_session_maker, dataset_id=str(uuid.uuid4()), tenant_id=tenant_id),+ _persist_attachment(orm_session_maker, dataset_id=dataset_id, tenant_id=str(uuid.uuid4())),+ ]+ if storage_fails:+ failure = TimeoutError("shared source and inline image storage unavailable")+ mock_storage.delete.side_effect = failure+ with pytest.raises(Retry) as retried:+ clean_dataset_task.apply(+ kwargs={+ "dataset_id": dataset_id,+ "tenant_id": tenant_id,+ "indexing_technique": IndexTechniqueType.HIGH_QUALITY,+ "index_struct": None,+ "collection_binding_id": collection_binding_id,+ "doc_form": IndexStructureType.PARAGRAPH_INDEX,+ },+ throw=True,+ )+ assert retried.value.exc is failure+ with orm_session_maker() as session:+ assert session.get(Document, document.id) is not None+ assert session.get(DocumentSegment, segment.id) is not None+ assert session.get(SegmentAttachmentBinding, binding.id) is not None+ retained_file = session.get(UploadFile, attachment_file.id)+ assert retained_file is not None+ assert retained_file.key == attachment_file.key+ # celery-stubs omits the signature stored on Retry at runtime.+ retry_signature = retried.value.sig # pyrefly: ignore[missing-attribute]+ assert retry_signature is not None+ assert retry_signature.kwargs["index_cleanup_done"] is True+ mock_storage.delete.side_effect = None+ assert retry_signature.apply(throw=True).successful()+ else:+ _run_clean_dataset(+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ collection_binding_id=collection_binding_id,+ )++ assert mock_storage.delete.call_count == (2 if storage_fails else 1)+ mock_storage.delete.assert_called_with(attachment_file.key)+ mock_index_processor_factory["processor"].clean.assert_called_once()+ with orm_session_maker() as session:+ assert session.get(Document, document.id) is None+ assert session.get(DocumentSegment, segment.id) is None+ assert session.get(UploadFile, attachment_file.id) is None+ assert session.get(SegmentAttachmentBinding, binding.id) is None+ for other_binding, other_file in unrelated_attachments:+ assert session.get(SegmentAttachmentBinding, other_binding.id) is not None+ assert session.get(UploadFile, other_file.id) is not None+def test_clean_dataset_task_with_attachments(self,dataset_id: str,@@ -500,6 +658,103 @@ class TestEdgeCases:class TestIndexProcessorParameters:"""Test cases for IndexProcessor clean method parameters."""+ @pytest.mark.parametrize("commit_fails", [False, True])+ def test_missing_elasticsearch_index_allows_parent_child_cleanup_after_retry(+ self,+ commit_fails: bool,+ app: Flask,+ monkeypatch: pytest.MonkeyPatch,+ elasticsearch_delete_endpoint: tuple[int, list[int], list[tuple[str, int]]],+ dataset_id: str,+ tenant_id: str,+ collection_binding_id: str,+ orm_session_maker: sessionmaker[Session],+ mock_storage: MagicMock,+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ port, responses, requests = elasticsearch_delete_endpoint+ expected_statuses = [200, 404] if commit_fails else [404]+ responses.extend(expected_statuses)+ for key, value in {+ "ELASTICSEARCH_USE_CLOUD": False,+ "ELASTICSEARCH_HOST": "http://127.0.0.1",+ "ELASTICSEARCH_PORT": port,+ "ELASTICSEARCH_USERNAME": "elastic",+ "ELASTICSEARCH_PASSWORD": "test",+ "ELASTICSEARCH_REQUEST_TIMEOUT": 2,+ "ELASTICSEARCH_MAX_RETRIES": 0,+ "ELASTICSEARCH_RETRY_ON_TIMEOUT": False,+ }.items():+ monkeypatch.setitem(app.config, key, value)+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+ segment = _persist_segment(+ orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id+ )+ child = ChildChunk(+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_id=document.id,+ segment_id=segment.id,+ position=1,+ content="child",+ word_count=1,+ created_by=str(uuid.uuid4()),+ )+ with orm_session_maker.begin() as session:+ session.add(child)+ task_kwargs = {+ "dataset_id": dataset_id,+ "tenant_id": tenant_id,+ "indexing_technique": IndexTechniqueType.HIGH_QUALITY,+ "index_struct": json.dumps({"type": "elasticsearch", "vector_store": {"class_prefix": "cleanup-index"}}),+ "collection_binding_id": collection_binding_id,+ "doc_form": IndexStructureType.PARENT_CHILD_INDEX,+ }+ failure = OperationalError("commit", {}, RuntimeError("temporary index commit failure"))+ injected_failures: list[OperationalError] = []++ def fail_first_index_commit(session: Session) -> None:+ if requests == [("/cleanup-index", 200)] and not injected_failures:+ assert session.get(ChildChunk, child.id) is None+ injected_failures.append(failure)+ raise failure++ with (+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,+ caplog.at_level("INFO", logger="tasks.clean_dataset_task"),+ ):+ if commit_fails:+ event.listen(orm_session_maker, "before_commit", fail_first_index_commit)+ try:+ with pytest.raises(Retry) as retried:+ clean_dataset_task.apply(kwargs=task_kwargs, throw=True)+ finally:+ event.remove(orm_session_maker, "before_commit", fail_first_index_commit)+ assert injected_failures == [failure]+ assert retried.value.exc is failure+ with orm_session_maker() as session:+ assert session.get(ChildChunk, child.id) is not None+ assert session.get(DocumentSegment, segment.id) is not None+ assert session.get(Document, document.id) is not None+ schedule_refresh.assert_not_called()+ assert not any("Cleaned dataset when dataset deleted:" in message for message in caplog.messages)+ # celery-stubs omits the signature stored on Retry at runtime.+ retry_signature = retried.value.sig # pyrefly: ignore[missing-attribute]+ assert retry_signature is not None+ assert retry_signature.kwargs["index_cleanup_done"] is False+ result = retry_signature.apply(throw=True)+ else:+ result = clean_dataset_task.apply(kwargs=task_kwargs, throw=True)+ assert result.successful()+ schedule_refresh.assert_called_once_with(tenant_id)++ assert requests == [("/cleanup-index", status) for status in expected_statuses]+ with orm_session_maker() as session:+ assert session.get(ChildChunk, child.id) is None+ assert session.get(DocumentSegment, segment.id) is None+ assert session.get(Document, document.id) is None+ mock_storage.delete.assert_not_called()+def test_clean_dataset_task_passes_correct_parameters_to_index_processor(self,dataset_id: str,@@ -556,7 +811,7 @@ class TestIndexProcessorParameters:assert call_args[1]["delete_child_chunks"] is Trueschedule_refresh.assert_called_once_with(tenant_id)- def test_vector_cleanup_failure_does_not_schedule_billing_refresh(+ def test_vector_cleanup_failure_preserves_documents_and_does_not_schedule_billing_refresh(self,dataset_id: str,tenant_id: str,@@ -566,9 +821,17 @@ class TestIndexProcessorParameters:mock_index_processor_factory: dict[str, MagicMock],mock_get_image_upload_file_ids: MagicMock,) -> None:- mock_index_processor_factory["processor"].clean.side_effect = RuntimeError("vector cleanup failed")+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+ segment = _persist_segment(+ orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id+ )+ failure = RuntimeError("vector cleanup failed")+ mock_index_processor_factory["processor"].clean.side_effect = failure- with patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh:+ with (+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,+ pytest.raises(RuntimeError, match="vector cleanup failed") as raised,+ ):clean_dataset_task(dataset_id=dataset_id,tenant_id=tenant_id,@@ -578,7 +841,531 @@ class TestIndexProcessorParameters:doc_form=IndexStructureType.PARAGRAPH_INDEX,)+ assert raised.value is failureschedule_refresh.assert_not_called()+ with orm_session_maker() as session:+ assert session.get(Document, document.id) is not None+ assert session.get(DocumentSegment, segment.id) is not None++ @pytest.mark.parametrize("failure_phase", ["flush", "commit"])+ def test_index_database_failure_preserves_parent_and_child_rows_until_retry(+ self,+ failure_phase: str,+ dataset_id: str,+ tenant_id: str,+ collection_binding_id: str,+ orm_session_maker: sessionmaker[Session],+ sqlite_engine: Engine,+ mock_storage: MagicMock,+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ """An index rollback must not strand its child rows after parent deletion."""+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+ segment = _persist_segment(+ orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id+ )+ child = ChildChunk(+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_id=document.id,+ segment_id=segment.id,+ position=1,+ content="child",+ word_count=1,+ created_by=str(uuid.uuid4()),+ )+ with orm_session_maker.begin() as session:+ session.add(child)+ failure = OperationalError(failure_phase, {}, RuntimeError("temporary index database failure"))+ child_deletes: list[str] = []+ injected_failures: list[OperationalError] = []++ def record_child_delete(statement: str, **_kwargs: object) -> None:+ if statement.lower().startswith("delete from child_chunks"):+ child_deletes.append(statement)++ def fail_first_index_commit(_session: Session) -> None:+ if failure_phase == "commit" and child_deletes and not injected_failures:+ injected_failures.append(failure)+ raise failure++ original_flush = Session.flush++ def fail_first_index_flush(session: Session, objects: Sequence[object] | None = None) -> None:+ if failure_phase == "flush" and child_deletes and not injected_failures:+ injected_failures.append(failure)+ raise failure+ original_flush(session, objects)++ with (+ patch("core.rag.index_processor.processor.parent_child_index_processor.Vector", autospec=True) as vector,+ patch.object(Session, "flush", autospec=True, side_effect=fail_first_index_flush),+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,+ caplog.at_level("INFO", logger="tasks.clean_dataset_task"),+ ):+ event.listen(sqlite_engine, "after_cursor_execute", record_child_delete, named=True)+ event.listen(orm_session_maker, "before_commit", fail_first_index_commit)+ try:+ with pytest.raises(Retry) as retried:+ clean_dataset_task.apply(+ kwargs={+ "dataset_id": dataset_id,+ "tenant_id": tenant_id,+ "indexing_technique": IndexTechniqueType.HIGH_QUALITY,+ "index_struct": None,+ "collection_binding_id": collection_binding_id,+ "doc_form": IndexStructureType.PARENT_CHILD_INDEX,+ },+ throw=True,+ )+ finally:+ event.remove(sqlite_engine, "after_cursor_execute", record_child_delete)+ event.remove(orm_session_maker, "before_commit", fail_first_index_commit)++ assert len(child_deletes) == 1+ assert injected_failures == [failure]+ assert retried.value.exc is failure+ # celery-stubs omits the signature stored on Retry at runtime.+ retry_signature = retried.value.sig # pyrefly: ignore[missing-attribute]+ assert retry_signature is not None+ assert retry_signature.kwargs["index_cleanup_done"] is False+ with orm_session_maker() as session:+ assert session.get(ChildChunk, child.id) is not None+ assert session.get(DocumentSegment, segment.id) is not None+ assert session.get(Document, document.id) is not None+ schedule_refresh.assert_not_called()+ assert not any("Cleaned dataset when dataset deleted:" in message for message in caplog.messages)++ assert retry_signature.apply(throw=True).successful()+ assert vector.return_value.delete.call_count == 2+ schedule_refresh.assert_called_once_with(tenant_id)++ with orm_session_maker() as session:+ assert session.get(ChildChunk, child.id) is None+ assert session.get(DocumentSegment, segment.id) is None+ assert session.get(Document, document.id) is None+ mock_storage.delete.assert_not_called()+++# ============================================================================+# Test Step Isolation and Batching+# ============================================================================+++class TestStepIsolation:+ """Committed batches survive failure, while dependent cleanup waits for retry."""++ def test_source_file_failure_keeps_batch_references_and_retries_missing_blob(+ self,+ orm_session_maker: sessionmaker[Session],+ dataset_id: str,+ tenant_id: str,+ collection_binding_id: str,+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ """Retry a partially deleted source-file batch without losing its storage keys."""+ documents = [_persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id) for _ in range(2)]+ source_files = [+ UploadFile(+ tenant_id=tenant_id,+ storage_type=StorageType.LOCAL,+ key=f"upload_files/{uuid.uuid4()}.pdf",+ name="source.pdf",+ size=10,+ extension="pdf",+ mime_type="application/pdf",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by=str(uuid.uuid4()),+ created_at=datetime.now(UTC),+ used=False,+ )+ for _ in documents+ ]+ with orm_session_maker.begin() as session:+ for document, source_file in zip(documents, source_files, strict=True):+ document.data_source_type = "upload_file"+ document.data_source_info = json.dumps({"upload_file_id": source_file.id})+ session.add_all([document, source_file])++ remaining_blobs = {source_file.key for source_file in source_files}+ attempted_keys: list[str] = []+ failure = TimeoutError("source storage unavailable")++ def delete_blob(key: str) -> None:+ attempted_keys.append(key)+ if len(attempted_keys) == 2:+ raise failure+ if key not in remaining_blobs:+ raise FileNotFoundError(key)+ remaining_blobs.remove(key)++ mock_storage.delete.side_effect = delete_blob+ with (+ patch("tasks.clean_dataset_task._DELETE_BATCH_SIZE", 2),+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,+ caplog.at_level("INFO", logger="tasks.clean_dataset_task"),+ ):+ with pytest.raises(Retry) as retried:+ clean_dataset_task.apply(+ kwargs={+ "dataset_id": dataset_id,+ "tenant_id": tenant_id,+ "indexing_technique": IndexTechniqueType.HIGH_QUALITY,+ "index_struct": None,+ "collection_binding_id": collection_binding_id,+ "doc_form": IndexStructureType.PARAGRAPH_INDEX,+ },+ throw=True,+ )++ assert len(attempted_keys) == 2+ assert remaining_blobs == {attempted_keys[1]}+ with orm_session_maker() as session:+ for document, source_file in zip(documents, source_files, strict=True):+ retained_document = session.get(Document, document.id)+ retained_file = session.get(UploadFile, source_file.id)+ assert retained_document is not None+ assert retained_document.data_source_info_dict["upload_file_id"] == source_file.id+ assert retained_file is not None+ assert retained_file.key == source_file.key+ assert retried.value.exc is failure+ assert not any("Cleaned dataset when dataset deleted:" in message for message in caplog.messages)+ # celery-stubs omits the signature stored on Retry at runtime.+ retry_signature = retried.value.sig # pyrefly: ignore[missing-attribute]+ assert retry_signature is not None+ assert retry_signature.kwargs["index_cleanup_done"] is True+ schedule_refresh.assert_called_once_with(tenant_id)++ assert retry_signature.apply(throw=True).successful()+ mock_index_processor_factory["processor"].clean.assert_called_once()+ schedule_refresh.assert_called_once_with(tenant_id)++ assert not remaining_blobs+ assert all(attempted_keys.count(source_file.key) == 2 for source_file in source_files)+ with orm_session_maker() as session:+ assert all(session.get(Document, document.id) is None for document in documents)+ assert all(session.get(UploadFile, source_file.id) is None for source_file in source_files)++ def test_cleanup_commits_more_than_once(+ self,+ orm_session_maker: sessionmaker[Session],+ sqlite_session_factory: sessionmaker[Session],+ dataset_id: str,+ tenant_id: str,+ collection_binding_id: str,+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,+ ):+ """A single commit at the end is what strands data when any step fails."""+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+ _persist_segment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id)++ commits: list[int] = []++ def _count_commit(_session: Session) -> None:+ commits.append(1)++ event.listen(sqlite_session_factory, "after_commit", _count_commit)+ try:+ _run_clean_dataset(+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ collection_binding_id=collection_binding_id,+ )+ finally:+ event.remove(sqlite_session_factory, "after_commit", _count_commit)++ assert len(commits) > 1, (+ f"Cleanup still runs as one transaction ({len(commits)} commit); a failure anywhere rolls back everything"+ )++ def test_document_failure_keeps_segments_deleted_and_preserves_metadata(+ self,+ orm_session_maker: sessionmaker[Session],+ dataset_id: str,+ tenant_id: str,+ collection_binding_id: str,+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,+ ) -> None:+ """A document failure must preserve later cleanup inputs for the retry."""+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+ segment = _persist_segment(+ orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id+ )+ metadata = DatasetMetadata(+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ type="string",+ name="meta",+ created_by=str(uuid.uuid4()),+ )+ with orm_session_maker.begin() as session:+ session.add(metadata)++ with (+ patch("tasks.clean_dataset_task._delete_documents", side_effect=RuntimeError("deadlock on documents")),+ pytest.raises(RuntimeError, match="deadlock on documents"),+ ):+ _run_clean_dataset(+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ collection_binding_id=collection_binding_id,+ )++ with orm_session_maker() as session:+ assert session.get(DocumentSegment, segment.id) is None, (+ "The committed segment deletion was rolled back by a later failure"+ )+ assert session.get(DatasetMetadata, metadata.id) is not None+ assert session.get(Document, document.id) is not None, (+ "Sanity check: the failing step should not have deleted its own rows"+ )++ @pytest.mark.parametrize("call_style", ["positional", "keywords"])+ def test_second_segment_batch_failure_preserves_cleanup_inputs_and_resumes(+ self,+ call_style: str,+ orm_session_maker: sessionmaker[Session],+ sqlite_engine: Engine,+ dataset_id: str,+ tenant_id: str,+ pipeline_id: str,+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+ segments = [+ _persist_segment(+ orm_session_maker,+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ document_id=document.id,+ position=position,+ )+ for position in range(5)+ ]+ binding, attachment_file = _persist_attachment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+ metadata = DatasetMetadata(+ tenant_id=tenant_id, dataset_id=dataset_id, type="string", name="meta", created_by=str(uuid.uuid4())+ )+ with orm_session_maker.begin() as session:+ session.add(metadata)+ task_kwargs = {+ "dataset_id": dataset_id,+ "tenant_id": tenant_id,+ "indexing_technique": None,+ "index_struct": None,+ "collection_binding_id": None,+ "doc_form": IndexStructureType.PARAGRAPH_INDEX,+ "pipeline_id": pipeline_id,+ }+ failure = RuntimeError("second segment batch commit failed")+ segment_deletes: list[str] = []++ def record_segment_delete(statement: str, **_kwargs: object) -> None:+ if statement.lower().startswith("delete from document_segments"):+ segment_deletes.append(statement)++ def fail_second_segment_commit(_session: Session) -> None:+ if len(segment_deletes) == 2:+ raise failure++ retry_task = clean_dataset_task.retry++ def retry_after_transaction_ends(+ *, exc: Exception, args: tuple[object, ...], kwargs: dict[str, object], countdown: int+ ) -> Retry:+ cleanup_session = mock_index_processor_factory["processor"].clean.call_args.kwargs["session"]+ assert not cleanup_session.in_transaction()+ return retry_task(exc=exc, args=args, kwargs=kwargs, countdown=countdown)++ with (+ patch("tasks.clean_dataset_task._DELETE_BATCH_SIZE", 2),+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,+ patch.object(clean_dataset_task, "retry", side_effect=retry_after_transaction_ends),+ caplog.at_level("INFO", logger="tasks.clean_dataset_task"),+ ):+ event.listen(sqlite_engine, "after_cursor_execute", record_segment_delete, named=True)+ event.listen(orm_session_maker, "before_commit", fail_second_segment_commit)+ try:+ with pytest.raises(Retry) as retried:+ clean_dataset_task.apply(+ args=tuple(task_kwargs.values()) if call_style == "positional" else (),+ kwargs=task_kwargs if call_style == "keywords" else {},+ throw=True,+ )+ finally:+ event.remove(sqlite_engine, "after_cursor_execute", record_segment_delete)+ event.remove(orm_session_maker, "before_commit", fail_second_segment_commit)++ assert len(segment_deletes) == 2+ assert not any("Cleaned dataset when dataset deleted:" in message for message in caplog.messages)+ with orm_session_maker() as session:+ assert sum(session.get(DocumentSegment, segment.id) is not None for segment in segments) == 3+ assert session.get(Document, document.id) is not None+ assert session.get(SegmentAttachmentBinding, binding.id) is not None+ assert session.get(UploadFile, attachment_file.id) is not None+ assert session.get(DatasetMetadata, metadata.id) is not None+ mock_storage.delete.assert_not_called()+ schedule_refresh.assert_called_once_with(tenant_id)+ assert retried.value.exc is failure+ assert retried.value.when == 30+ retry_signature = retried.value.sig+ assert retry_signature is not None+ assert retry_signature.args == ()+ assert retry_signature.kwargs == {**task_kwargs, "index_cleanup_done": True}+ assert retry_signature.apply(throw=True).successful()+ mock_index_processor_factory["processor"].clean.assert_called_once()+ schedule_refresh.assert_called_once_with(tenant_id)++ with orm_session_maker() as session:+ assert all(session.get(DocumentSegment, segment.id) is None for segment in segments)+ assert session.get(Document, document.id) is None+ assert session.get(SegmentAttachmentBinding, binding.id) is None+ assert session.get(UploadFile, attachment_file.id) is None+ assert session.get(DatasetMetadata, metadata.id) is None+ mock_storage.delete.assert_called_once_with(attachment_file.key)++ @pytest.mark.parametrize(("retries", "delay"), [(1, 60), (2, 120), (3, None)])+ def test_crawl_cache_failure_retries_before_index_cleanup_and_stops_at_limit(+ self,+ retries: int,+ delay: int | None,+ dataset_id: str,+ tenant_id: str,+ collection_binding_id: str,+ mock_index_processor_factory: dict[str, MagicMock],+ caplog: pytest.LogCaptureFixture,+ ) -> None:+ failure = RuntimeError("crawl cache unavailable")+ with (+ patch("tasks.clean_dataset_task.cleanup_dataset_crawl_cache", side_effect=failure),+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,+ caplog.at_level("INFO", logger="tasks.clean_dataset_task"),+ pytest.raises(Retry if delay is not None else RuntimeError) as raised,+ ):+ clean_dataset_task.apply(+ kwargs={+ "dataset_id": dataset_id,+ "tenant_id": tenant_id,+ "indexing_technique": IndexTechniqueType.HIGH_QUALITY,+ "index_struct": '{"type": "paragraph"}',+ "collection_binding_id": collection_binding_id,+ "doc_form": IndexStructureType.PARAGRAPH_INDEX,+ },+ retries=retries,+ throw=True,+ )++ if delay is None:+ assert raised.value is failure+ else:+ assert isinstance(raised.value, Retry)+ assert raised.value.exc is failure+ assert raised.value.when == delay+ # celery-stubs omits the signature stored on Retry at runtime.+ retry_signature = raised.value.sig # pyrefly: ignore[missing-attribute]+ assert retry_signature is not None+ assert retry_signature.kwargs["index_cleanup_done"] is False+ assert not any("Cleaned dataset when dataset deleted:" in message for message in caplog.messages)+ mock_index_processor_factory["processor"].clean.assert_not_called()+ schedule_refresh.assert_not_called()+++class TestBatchedDeletes:+ """Large child tables are deleted in bounded batches."""++ def test_text_segments_use_bounded_transactions_without_image_file_queries(+ self,+ orm_session_maker: sessionmaker[Session],+ sqlite_engine: Engine,+ dataset_id: str,+ tenant_id: str,+ collection_binding_id: str,+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ ) -> None:+ """Text-only cleanup must not run no-op file SQL or retain the full dataset."""+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+ segments = [+ _persist_segment(+ orm_session_maker,+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ document_id=document.id,+ position=position,+ )+ for position in range(5)+ ]+ file_queries: list[str] = []+ committed_segment_loads: list[int] = []+ loaded_segment_ids: list[str] = []++ def record_file_query(statement: str, **_kwargs: object) -> None:+ if "from upload_files" in statement.lower():+ file_queries.append(statement)++ def record_segment_load(_session: Session, instance: object) -> None:+ if isinstance(instance, DocumentSegment):+ loaded_segment_ids.append(instance.id)++ def record_commit(_session: Session) -> None:+ if loaded_segment_ids:+ committed_segment_loads.append(len(loaded_segment_ids))+ loaded_segment_ids.clear()++ event.listen(sqlite_engine, "before_cursor_execute", record_file_query, named=True)+ event.listen(orm_session_maker, "loaded_as_persistent", record_segment_load)+ event.listen(orm_session_maker, "after_commit", record_commit)+ try:+ with patch("tasks.clean_dataset_task._DELETE_BATCH_SIZE", 2):+ _run_clean_dataset(+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ collection_binding_id=collection_binding_id,+ )+ finally:+ event.remove(sqlite_engine, "before_cursor_execute", record_file_query)+ event.remove(orm_session_maker, "loaded_as_persistent", record_segment_load)+ event.remove(orm_session_maker, "after_commit", record_commit)++ assert file_queries == [], "Text-only segments issued unnecessary UploadFile SQL"+ assert sum(committed_segment_loads) == len(segments)+ assert max(committed_segment_loads) <= 2, "Segment cleanup exceeded the transaction batch size"+ mock_storage.delete.assert_not_called()+ with orm_session_maker() as session:+ assert all(session.get(DocumentSegment, segment.id) is None for segment in segments)++ def test_documents_are_deleted_across_multiple_batches(+ self,+ orm_session_maker: sessionmaker[Session],+ dataset_id: str,+ tenant_id: str,+ collection_binding_id: str,+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,+ ):+ """With a batch size smaller than the row count the loop must still drain the+ table, rather than stopping after the first batch."""+ documents = [_persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id) for _ in range(5)]++ with patch("tasks.clean_dataset_task._DELETE_BATCH_SIZE", 2):+ _run_clean_dataset(+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ collection_binding_id=collection_binding_id,+ )++ with orm_session_maker() as session:+ remaining = [document.id for document in documents if session.get(Document, document.id) is not None]+ assert not remaining, f"Batched delete left {len(remaining)} of 5 documents behind"@pytest.mark.parametrize("shared_tenant", [None, "same", "other"])