Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(dataset): batch segment and document cleanup with retries (#38519)

Alexey Shabalin638ed4eb5565
diff --git a/api/providers/vdb/vdb-elasticsearch/src/dify_vdb_elasticsearch/elasticsearch_vector.py b/api/providers/vdb/vdb-elasticsearch/src/dify_vdb_elasticsearch/elasticsearch_vector.py
index d1c8ced881a..8254f2a4bf7 100644
--- a/api/providers/vdb/vdb-elasticsearch/src/dify_vdb_elasticsearch/elasticsearch_vector.py
+++ b/api/providers/vdb/vdb-elasticsearch/src/dify_vdb_elasticsearch/elasticsearch_vector.py
@@ -5,7 +5,7 @@ from typing import Any, cast, override
from urllib.parse import urlparse
from elasticsearch import ConnectionError as ElasticsearchConnectionError
-from elasticsearch import Elasticsearch
+from elasticsearch import Elasticsearch, NotFoundError
from flask import current_app
from packaging.version import parse as parse_version
from pydantic import BaseModel, model_validator
@@ -195,8 +195,15 @@ class ElasticSearchVector(BaseVector):
self.delete_by_ids(ids)
@override
- def delete(self):
- self._client.indices.delete(index=self._collection_name)
+ def delete(self) -> None:
+ try:
+ self._client.indices.delete(index=self._collection_name)
+ except NotFoundError as exc:
+ error = exc.body.get("error") if isinstance(exc.body, dict) else None
+ if not isinstance(error, dict) or error.get("type") != "index_not_found_exception":
+ raise
+ # A previous attempt may have deleted the index before a relational commit failed.
+ logger.debug("Elasticsearch index already absent: %s", self._collection_name)
@override
def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
diff --git a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py
index 7e0c9850925..8c626673abd 100644
--- a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py
+++ b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_ja_vector.py
@@ -6,6 +6,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
+from elasticsearch import NotFoundError
from models.dataset import Dataset
@@ -27,19 +28,24 @@ def _build_fake_elasticsearch_modules():
elasticsearch.Elasticsearch = Elasticsearch
elasticsearch.ConnectionError = ConnectionError
+ elasticsearch.__dict__["NotFoundError"] = NotFoundError
return {"elasticsearch": elasticsearch}
@pytest.fixture
def elasticsearch_ja_module(monkeypatch: pytest.MonkeyPatch):
- for name, module in _build_fake_elasticsearch_modules().items():
- monkeypatch.setitem(sys.modules, name, module)
-
import dify_vdb_elasticsearch.elasticsearch_ja_vector as ja_module
import dify_vdb_elasticsearch.elasticsearch_vector as base_module
- importlib.reload(base_module)
- return importlib.reload(ja_module)
+ try:
+ with monkeypatch.context() as module_patch:
+ for name, fake_module in _build_fake_elasticsearch_modules().items():
+ module_patch.setitem(sys.modules, name, fake_module)
+ importlib.reload(base_module)
+ yield importlib.reload(ja_module)
+ finally:
+ importlib.reload(base_module)
+ importlib.reload(ja_module)
def test_create_collection_cache_hit(elasticsearch_ja_module, monkeypatch: pytest.MonkeyPatch):
diff --git a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py
index 80e85f75b71..59110786988 100644
--- a/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py
+++ b/api/providers/vdb/vdb-elasticsearch/tests/unit_tests/test_elasticsearch_vector.py
@@ -6,6 +6,8 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
+from elastic_transport import ApiResponseMeta, HttpHeaders, NodeConfig, TransportApiResponse
+from elasticsearch import ApiError, Elasticsearch, NotFoundError
from pydantic import ValidationError
from core.rag.models.document import Document
@@ -36,17 +38,23 @@ def _build_fake_elasticsearch_modules():
elasticsearch.Elasticsearch = Elasticsearch
elasticsearch.ConnectionError = ConnectionError
+ elasticsearch.__dict__["NotFoundError"] = NotFoundError
return {"elasticsearch": elasticsearch}
@pytest.fixture
def elasticsearch_module(monkeypatch: pytest.MonkeyPatch):
- for name, module in _build_fake_elasticsearch_modules().items():
- monkeypatch.setitem(sys.modules, name, module)
-
import dify_vdb_elasticsearch.elasticsearch_vector as module
- return importlib.reload(module)
+ try:
+ with monkeypatch.context() as module_patch:
+ for name, fake_module in _build_fake_elasticsearch_modules().items():
+ module_patch.setitem(sys.modules, name, fake_module)
+ yield importlib.reload(module)
+ finally:
+ importlib.reload(module)
+ if "dify_vdb_elasticsearch.elasticsearch_ja_vector" in sys.modules:
+ importlib.reload(sys.modules["dify_vdb_elasticsearch.elasticsearch_ja_vector"])
def _regular_config(module, **overrides):
@@ -228,6 +236,69 @@ def test_crud_methods_and_get_type(elasticsearch_module):
assert vector.get_type() == elasticsearch_module.VectorType.ELASTICSEARCH
[email protected]("japanese", [False, True], ids=["elasticsearch", "elasticsearch-ja"])
+ ("status", "body", "missing_index"),
+ [
+ pytest.param(
+ 404,
+ {"error": {"type": "index_not_found_exception", "reason": "no such index [collection_1]"}, "status": 404},
+ True,
+ id="missing-index",
+ ),
+ pytest.param(
+ 404,
+ {"error": {"type": "resource_not_found_exception", "reason": "resource missing"}, "status": 404},
+ False,
+ id="other-not-found",
+ ),
+ pytest.param(404, "Not Found", False, id="proxy-not-found"),
+ pytest.param(404, "index_not_found_exception", False, id="unstructured-missing-index"),
+ pytest.param(
+ 404,
+ {"error": "index_not_found_exception", "status": 404},
+ False,
+ id="string-error-missing-index",
+ ),
+ pytest.param(403, {"error": {"type": "security_exception"}, "status": 403}, False, id="permission-denied"),
+ pytest.param(500, {"error": {"type": "index_not_found_exception"}, "status": 500}, False, id="server-error"),
+ ],
+)
+def test_delete_only_ignores_missing_index(elasticsearch_module, japanese, status, body, missing_index):
+ if japanese:
+ import dify_vdb_elasticsearch.elasticsearch_ja_vector as ja_module
+
+ vector_class = importlib.reload(ja_module).ElasticSearchJaVector
+ else:
+ vector_class = elasticsearch_module.ElasticSearchVector
+ vector = vector_class.__new__(vector_class)
+ vector._collection_name = "collection_1"
+ response = TransportApiResponse(
+ ApiResponseMeta(
+ status=status,
+ http_version="1.1",
+ headers=HttpHeaders({"x-elastic-product": "Elasticsearch"}),
+ duration=0,
+ node=NodeConfig("http", "localhost", 9200),
+ ),
+ body,
+ )
+ with Elasticsearch("http://localhost:9200", max_retries=0) as client:
+ vector._client = client
+ # Keep SDK error parsing real; replace only the HTTP transport boundary.
+ with patch.object(client.transport, "perform_request", return_value=response) as request:
+ if missing_index:
+ vector.delete()
+ vector.delete()
+ else:
+ with pytest.raises(ApiError) as raised:
+ vector.delete()
+ assert raised.value.status_code == status
+ assert raised.value.body == body
+ assert request.call_count == (2 if missing_index else 1)
+ assert request.call_args.args[:2] == ("DELETE", "/collection_1")
+
+
def test_search_by_vector_and_full_text(elasticsearch_module):
vector = elasticsearch_module.ElasticSearchVector.__new__(elasticsearch_module.ElasticSearchVector)
vector._collection_name = "collection_1"
diff --git a/api/tasks/clean_dataset_task.py b/api/tasks/clean_dataset_task.py
index 357d30d6d4b..e0751ae2566 100644
--- a/api/tasks/clean_dataset_task.py
+++ b/api/tasks/clean_dataset_task.py
@@ -1,9 +1,11 @@
import logging
import time
+from collections.abc import Callable, Sequence
import click
from celery import shared_task
from sqlalchemy import delete, select
+from sqlalchemy.orm import Session
from core.db.session_factory import session_factory
from core.rag.index_processor.index_processor import IndexProcessorFactory
@@ -29,10 +31,16 @@ from tasks.refresh_billing_vector_space_task import schedule_billing_vector_spac
logger = logging.getLogger(__name__)
+# Keep committed batches on failure, but preserve dependent rows until a retry
+# finishes their prerequisite cleanup.
+_DELETE_BATCH_SIZE = 1000
+_MAX_RETRIES = 3
+_RETRY_DELAY_SECONDS = 30
-# Add import statement for ValueError
-@shared_task(queue="dataset")
+
+@shared_task(queue="dataset", bind=True, max_retries=_MAX_RETRIES, default_retry_delay=_RETRY_DELAY_SECONDS)
def clean_dataset_task(
+ self,
dataset_id: str,
tenant_id: str,
indexing_technique: str | None,
@@ -40,7 +48,9 @@ def clean_dataset_task(
collection_binding_id: str | None,
doc_form: str,
pipeline_id: str | None = None,
-):
+ *,
+ index_cleanup_done: bool = False,
+) -> None:
"""
Clean dataset when dataset deleted.
:param dataset_id: dataset id
@@ -49,171 +59,209 @@ def clean_dataset_task(
:param index_struct: index struct dict
:param collection_binding_id: collection binding id
:param doc_form: dataset form
+ :param index_cleanup_done: preserve successful index cleanup across explicit retries
Usage: clean_dataset_task.delay(dataset_id, tenant_id, indexing_technique, index_struct)
"""
logger.info(click.style(f"Start clean dataset when dataset deleted: {dataset_id}", fg="green"))
start_at = time.perf_counter()
- vector_cleanup_succeeded = False
- cleanup_dataset_crawl_cache(tenant_id, dataset_id)
+ # Enhanced validation: Check if doc_form is None, empty string, or contains only whitespace.
+ # Treating all invalid values as the default paragraph index type keeps the vector
+ # database cleanup reachable for empty/broken datasets.
+ if doc_form is None or (isinstance(doc_form, str) and not doc_form.strip()):
+ from core.rag.index_processor.constant.index_type import IndexStructureType
- with session_factory.create_session() as session:
- try:
- dataset = Dataset(
- id=dataset_id,
- tenant_id=tenant_id,
- indexing_technique=indexing_technique,
- index_struct=index_struct,
- collection_binding_id=collection_binding_id,
- )
- documents = session.scalars(select(Document).where(Document.dataset_id == dataset_id)).all()
- segments = session.scalars(select(DocumentSegment).where(DocumentSegment.dataset_id == dataset_id)).all()
- # Use JOIN to fetch attachments with bindings in a single query
- attachments_with_bindings = session.execute(
- select(SegmentAttachmentBinding, UploadFile)
- .join(UploadFile, UploadFile.id == SegmentAttachmentBinding.attachment_id)
- .where(
- SegmentAttachmentBinding.tenant_id == tenant_id,
- SegmentAttachmentBinding.dataset_id == dataset_id,
- )
- ).all()
-
- # Enhanced validation: Check if doc_form is None, empty string, or contains only whitespace
- # This ensures all invalid doc_form values are properly handled
- if doc_form is None or (isinstance(doc_form, str) and not doc_form.strip()):
- # Use default paragraph index type for empty/invalid datasets to enable vector database cleanup
- from core.rag.index_processor.constant.index_type import IndexStructureType
-
- doc_form = IndexStructureType.PARAGRAPH_INDEX
- logger.info(
- click.style(
- f"Invalid doc_form detected, using default index type for cleanup: {doc_form}",
- fg="yellow",
- )
- )
-
- # Add exception handling around IndexProcessorFactory.clean() to prevent single point of failure
- # This ensures Document/Segment deletion can continue even if vector database cleanup fails
- try:
- index_processor = IndexProcessorFactory(doc_form).init_index_processor()
- index_processor.clean(dataset, None, with_keywords=True, delete_child_chunks=True, session=session)
- vector_cleanup_succeeded = True
- logger.info(click.style(f"Successfully cleaned vector database for dataset: {dataset_id}", fg="green"))
- except Exception:
- logger.exception(click.style(f"Failed to clean vector database for dataset {dataset_id}", fg="red"))
- # Continue with document and segment deletion even if vector cleanup fails
- logger.info(
- click.style(f"Continuing with document and segment deletion for dataset: {dataset_id}", fg="yellow")
- )
-
- if documents is None or len(documents) == 0:
- logger.info(click.style(f"No documents found for dataset: {dataset_id}", fg="green"))
- else:
- logger.info(click.style(f"Cleaning documents for dataset: {dataset_id}", fg="green"))
-
- for document in documents:
- session.delete(document)
-
- segment_ids = [segment.id for segment in segments]
- for segment in segments:
- image_upload_file_ids = get_image_upload_file_ids(segment.content)
- image_files = session.scalars(
- select(UploadFile).where(UploadFile.id.in_(image_upload_file_ids))
- ).all()
- for image_file in image_files:
- if image_file is None:
- continue
- try:
- storage.delete(image_file.key)
- except Exception:
- logger.exception(
- "Delete image_files failed when storage deleted, \
- image_upload_file_is: %s",
- image_file.id,
- )
- stmt = delete(UploadFile).where(UploadFile.id.in_(image_upload_file_ids))
- session.execute(stmt)
-
- segment_delete_stmt = delete(DocumentSegment).where(DocumentSegment.id.in_(segment_ids))
- session.execute(segment_delete_stmt)
- # delete segment attachments
- if attachments_with_bindings:
- attachment_ids = [attachment_file.id for _, attachment_file in attachments_with_bindings]
- binding_ids = [binding.id for binding, _ in attachments_with_bindings]
- for binding, attachment_file in attachments_with_bindings:
- try:
- storage.delete(attachment_file.key)
- except Exception:
- logger.exception(
- "Delete attachment_file failed when storage deleted, \
- attachment_file_id: %s",
- binding.attachment_id,
- )
- attachment_file_delete_stmt = delete(UploadFile).where(UploadFile.id.in_(attachment_ids))
- session.execute(attachment_file_delete_stmt)
-
- binding_delete_stmt = delete(SegmentAttachmentBinding).where(
- SegmentAttachmentBinding.id.in_(binding_ids)
- )
- session.execute(binding_delete_stmt)
-
- session.execute(delete(DatasetProcessRule).where(DatasetProcessRule.dataset_id == dataset_id))
- session.execute(delete(DatasetQuery).where(DatasetQuery.dataset_id == dataset_id))
- session.execute(delete(AppDatasetJoin).where(AppDatasetJoin.dataset_id == dataset_id))
- # delete dataset metadata
- session.execute(delete(DatasetMetadata).where(DatasetMetadata.dataset_id == dataset_id))
- session.execute(delete(DatasetMetadataBinding).where(DatasetMetadataBinding.dataset_id == dataset_id))
- # delete pipeline and workflow
- if pipeline_id:
- session.execute(delete(Pipeline).where(Pipeline.id == pipeline_id))
- session.execute(
- delete(Workflow).where(
- Workflow.tenant_id == tenant_id,
- Workflow.app_id == pipeline_id,
- Workflow.type == WorkflowType.RAG_PIPELINE,
- )
- )
- # delete files
- if documents:
- file_ids = []
- for document in documents:
- if document.data_source_type == "upload_file":
- if document.data_source_info:
- data_source_info = document.data_source_info_dict
- if data_source_info and "upload_file_id" in data_source_info:
- file_id = data_source_info["upload_file_id"]
- file_ids.append(file_id)
- files = session.scalars(select(UploadFile).where(UploadFile.id.in_(file_ids))).all()
- for file in files:
- storage.delete(file.key)
-
- file_delete_stmt = delete(UploadFile).where(UploadFile.id.in_(file_ids))
- session.execute(file_delete_stmt)
-
- session.commit()
- if vector_cleanup_succeeded:
+ doc_form = IndexStructureType.PARAGRAPH_INDEX
+ logger.info(
+ click.style(f"Invalid doc_form detected, using default index type for cleanup: {doc_form}", fg="yellow")
+ )
+
+ dataset = Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ indexing_technique=indexing_technique,
+ index_struct=index_struct,
+ collection_binding_id=collection_binding_id,
+ )
+
+ try:
+ cleanup_dataset_crawl_cache(tenant_id, dataset_id)
+
+ with session_factory.create_session() as session:
+ if not index_cleanup_done:
+ # Index processors also delete relational child rows, so the whole step must succeed first.
+ _run_step(session, dataset_id, "dataset index", lambda: _clean_index(session, dataset, doc_form))
+ index_cleanup_done = True
schedule_billing_vector_space_refresh(dataset.tenant_id)
- end_at = time.perf_counter()
- logger.info(
- click.style(
- f"Cleaned dataset when dataset deleted: {dataset_id} latency: {end_at - start_at}",
- fg="green",
- )
+
+ # Stop on failure so documents and their resources outlive any remaining segments.
+ _run_step(session, dataset_id, "segments", lambda: _delete_segments(session, dataset_id))
+ _run_step(session, dataset_id, "documents", lambda: _delete_documents(session, dataset_id))
+ _run_step(session, dataset_id, "attachments", lambda: _delete_attachments(session, tenant_id, dataset_id))
+ _run_step(
+ session,
+ dataset_id,
+ "dataset metadata",
+ lambda: _delete_dataset_scoped_rows(session, tenant_id, dataset_id, pipeline_id),
)
+
+ except Exception as exc:
+ if self.request.retries >= _MAX_RETRIES:
+ logger.exception("Dataset cleanup retry budget exhausted, dataset_id=%s", dataset_id)
+ raise
+
+ logger.warning(
+ "Dataset cleanup failed, scheduling retry %d/%d, dataset_id=%s",
+ self.request.retries + 1,
+ _MAX_RETRIES,
+ dataset_id,
+ exc_info=True,
+ )
+ # The session is closed before retrying. Normalize arguments for positional callers too.
+ raise self.retry(
+ exc=exc,
+ args=(),
+ kwargs={
+ "dataset_id": dataset_id,
+ "tenant_id": tenant_id,
+ "indexing_technique": indexing_technique,
+ "index_struct": index_struct,
+ "collection_binding_id": collection_binding_id,
+ "doc_form": doc_form,
+ "pipeline_id": pipeline_id,
+ "index_cleanup_done": index_cleanup_done,
+ },
+ countdown=_RETRY_DELAY_SECONDS * (2**self.request.retries),
+ ) from exc
+
+ end_at = time.perf_counter()
+ logger.info(
+ click.style(
+ f"Cleaned dataset when dataset deleted: {dataset_id} latency: {end_at - start_at}",
+ fg="green",
+ )
+ )
+
+
+def _run_step(session: Session, dataset_id: str, name: str, step: Callable[[], None]) -> None:
+ """Roll back uncommitted work on failure and stop dependent steps until retry."""
+ try:
+ step()
+ except Exception:
+ logger.exception(click.style(f"Failed to clean {name} for dataset {dataset_id}", fg="red"))
+ try:
+ session.rollback()
+ except Exception:
+ logger.exception("Failed to rollback database session")
+ raise
+
+
+def _clean_index(session: Session, dataset: Dataset, doc_form: str) -> None:
+ """Clean the dataset index and commit any relational rows removed by its processor."""
+ index_processor = IndexProcessorFactory(doc_form).init_index_processor()
+ index_processor.clean(dataset, None, with_keywords=True, delete_child_chunks=True, session=session)
+ session.commit()
+ logger.info(click.style(f"Successfully cleaned vector database for dataset: {dataset.id}", fg="green"))
+
+
+def _delete_segments(session: Session, dataset_id: str) -> None:
+ """Delete document segments in batches, removing their inline image files first."""
+ while True:
+ segments = session.scalars(
+ select(DocumentSegment).where(DocumentSegment.dataset_id == dataset_id).limit(_DELETE_BATCH_SIZE)
+ ).all()
+ if not segments:
+ break
+ image_upload_file_ids: list[str] = []
+ for segment in segments:
+ image_upload_file_ids.extend(get_image_upload_file_ids(segment.content))
+ _delete_upload_files(session, image_upload_file_ids)
+ session.execute(delete(DocumentSegment).where(DocumentSegment.id.in_([segment.id for segment in segments])))
+ session.commit()
+
+
+def _delete_documents(session: Session, dataset_id: str) -> None:
+ """Delete documents in batches, removing their uploaded source files first."""
+ while True:
+ documents = session.scalars(
+ select(Document).where(Document.dataset_id == dataset_id).limit(_DELETE_BATCH_SIZE)
+ ).all()
+ if not documents:
+ break
+ file_ids: list[str] = []
+ for document in documents:
+ if document.data_source_type == "upload_file" and document.data_source_info:
+ data_source_info = document.data_source_info_dict
+ if data_source_info and "upload_file_id" in data_source_info:
+ file_ids.append(data_source_info["upload_file_id"])
+ _delete_upload_files(session, file_ids)
+ session.execute(delete(Document).where(Document.id.in_([document.id for document in documents])))
+ session.commit()
+
+
+def _delete_upload_files(session: Session, file_ids: Sequence[str]) -> None:
+ """Delete blobs before their lookup rows so failures retain storage keys for retry."""
+ if not file_ids:
+ return
+ files = session.scalars(select(UploadFile).where(UploadFile.id.in_(file_ids))).all()
+ for file in files:
+ try:
+ storage.delete(file.key)
+ except FileNotFoundError:
+ logger.debug("Upload file already absent from storage, upload_file_id: %s", file.id)
+ session.execute(delete(UploadFile).where(UploadFile.id.in_(file_ids)))
+
+
+def _delete_attachments(session: Session, tenant_id: str, dataset_id: str) -> None:
+ """Delete segment attachments (upload files + bindings) for the dataset."""
+ # Inline-image cleanup may already have deleted the file; its binding still needs cleanup.
+ attachments_with_bindings = session.execute(
+ select(SegmentAttachmentBinding, UploadFile)
+ .outerjoin(UploadFile, UploadFile.id == SegmentAttachmentBinding.attachment_id)
+ .where(
+ SegmentAttachmentBinding.tenant_id == tenant_id,
+ SegmentAttachmentBinding.dataset_id == dataset_id,
+ )
+ ).all()
+ if not attachments_with_bindings:
+ return
+ attachment_ids = [
+ attachment_file.id for _, attachment_file in attachments_with_bindings if attachment_file is not None
+ ]
+ binding_ids = [binding.id for binding, _ in attachments_with_bindings]
+ for binding, attachment_file in attachments_with_bindings:
+ if attachment_file is None:
+ continue
+ try:
+ storage.delete(attachment_file.key)
except Exception:
- # Add rollback to prevent dirty session state in case of exceptions
- # This ensures the database session is properly cleaned up
- try:
- session.rollback()
- logger.info(click.style(f"Rolled back database session for dataset: {dataset_id}", fg="yellow"))
- except Exception:
- logger.exception("Failed to rollback database session")
-
- logger.exception("Cleaned dataset when dataset deleted failed")
- finally:
- # Explicitly close the session for test expectations and safety
- try:
- session.close()
- except Exception:
- logger.exception("Failed to close database session")
+ logger.exception(
+ "Delete attachment_file failed when storage deleted, attachment_file_id: %s",
+ binding.attachment_id,
+ )
+ if attachment_ids:
+ session.execute(delete(UploadFile).where(UploadFile.id.in_(attachment_ids)))
+ session.execute(delete(SegmentAttachmentBinding).where(SegmentAttachmentBinding.id.in_(binding_ids)))
+ session.commit()
+
+
+def _delete_dataset_scoped_rows(session: Session, tenant_id: str, dataset_id: str, pipeline_id: str | None) -> None:
+ """Delete the remaining small dataset-scoped tables (and the pipeline/workflow)."""
+ session.execute(delete(DatasetProcessRule).where(DatasetProcessRule.dataset_id == dataset_id))
+ session.execute(delete(DatasetQuery).where(DatasetQuery.dataset_id == dataset_id))
+ session.execute(delete(AppDatasetJoin).where(AppDatasetJoin.dataset_id == dataset_id))
+ # delete dataset metadata
+ session.execute(delete(DatasetMetadata).where(DatasetMetadata.dataset_id == dataset_id))
+ session.execute(delete(DatasetMetadataBinding).where(DatasetMetadataBinding.dataset_id == dataset_id))
+ # delete pipeline and workflow
+ if pipeline_id:
+ session.execute(delete(Pipeline).where(Pipeline.id == pipeline_id))
+ session.execute(
+ delete(Workflow).where(
+ Workflow.tenant_id == tenant_id,
+ Workflow.app_id == pipeline_id,
+ Workflow.type == WorkflowType.RAG_PIPELINE,
+ )
+ )
+ session.commit()
diff --git a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py
index d6cdd01c0b9..dc4fc8d7c09 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_clean_dataset_task.py
@@ -523,19 +523,10 @@ class TestCleanDatasetTask:
# Should be called 4 times (once for each test case)
assert mock_factory.call_count == 4
- def test_clean_dataset_task_error_handling_and_rollback(
+ def test_index_cleanup_failure_preserves_documents_and_segments(
self, db_session_with_containers: Session, mock_external_service_dependencies
- ):
- """
- Test error handling and rollback mechanism when database operations fail.
-
- This test verifies that the task can properly:
- 1. Handle database operation failures gracefully
- 2. Rollback database session to prevent dirty state
- 3. Continue cleanup operations even if some parts fail
- 4. Log appropriate error messages
- 5. Maintain database session integrity
- """
+ ) -> None:
+ """An incomplete index cleanup must preserve its dependent records for retry."""
# Create test data
account, tenant = self._create_test_account_and_tenant(db_session_with_containers)
dataset = self._create_test_dataset(db_session_with_containers, account, tenant)
@@ -544,37 +535,31 @@ class TestCleanDatasetTask:
# Mock IndexProcessorFactory to raise an exception
mock_index_processor = mock_external_service_dependencies["index_processor"]
- mock_index_processor.clean.side_effect = Exception("Vector database cleanup failed")
-
- # Execute the task - it should handle the exception gracefully
- clean_dataset_task(
- dataset_id=dataset.id,
- tenant_id=tenant.id,
- indexing_technique=dataset.indexing_technique,
- index_struct=dataset.index_struct,
- collection_binding_id=dataset.collection_binding_id,
- doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),
- )
+ mock_index_processor.clean.side_effect = RuntimeError("Vector database cleanup failed")
- # Verify results - even with vector cleanup failure, documents and segments should be deleted
+ # A direct call propagates the original exception instead of publishing a retry.
+ with pytest.raises(RuntimeError, match="Vector database cleanup failed"):
+ clean_dataset_task(
+ dataset_id=dataset.id,
+ tenant_id=tenant.id,
+ indexing_technique=dataset.indexing_technique,
+ index_struct=dataset.index_struct,
+ collection_binding_id=dataset.collection_binding_id,
+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),
+ )
- # Check that documents were still deleted despite vector cleanup failure
remaining_documents = db_session_with_containers.scalars(
select(Document).where(Document.dataset_id == dataset.id)
).all()
- assert len(remaining_documents) == 0
+ assert len(remaining_documents) == 1
- # Check that segments were still deleted despite vector cleanup failure
remaining_segments = db_session_with_containers.scalars(
select(DocumentSegment).where(DocumentSegment.dataset_id == dataset.id)
).all()
- assert len(remaining_segments) == 0
+ assert len(remaining_segments) == 1
- # Verify that index processor was called and failed
mock_index_processor.clean.assert_called_once()
-
- # Verify that the task continued with cleanup despite the error
- # This demonstrates the resilience of the cleanup process
+ mock_external_service_dependencies["storage"].delete.assert_not_called()
def test_clean_dataset_task_with_image_file_references(
self, db_session_with_containers: Session, mock_external_service_dependencies
@@ -809,74 +794,54 @@ class TestCleanDatasetTask:
print(f"Total cleanup time: {cleanup_duration:.3f} seconds")
print(f"Average time per document: {cleanup_duration / len(documents):.3f} seconds")
- def test_clean_dataset_task_storage_exception_handling(
+ def test_source_storage_failure_preserves_upload_file_and_document(
self, db_session_with_containers: Session, mock_external_service_dependencies
- ):
- """
- Test dataset cleanup when storage operations fail.
-
- This test verifies that the task can properly:
- 1. Handle storage deletion failures gracefully
- 2. Continue cleanup process despite storage errors
- 3. Log appropriate error messages for storage failures
- 4. Maintain database consistency even with storage issues
- 5. Provide meaningful error reporting
- """
+ ) -> None:
+ """Keep source storage keys and documents available for a cleanup retry."""
# Create test data
account, tenant = self._create_test_account_and_tenant(db_session_with_containers)
dataset = self._create_test_dataset(db_session_with_containers, account, tenant)
document = self._create_test_document(db_session_with_containers, account, tenant, dataset)
self._create_test_segment(db_session_with_containers, account, tenant, dataset, document)
upload_file = self._create_test_upload_file(db_session_with_containers, account, tenant)
+ upload_file_id = upload_file.id
+ upload_file_key = upload_file.key
+ document_id = document.id
# Update document with file reference
import json
- document.data_source_info = json.dumps({"upload_file_id": upload_file.id})
+ document.data_source_info = json.dumps({"upload_file_id": upload_file_id})
db_session_with_containers.commit()
# Mock storage to raise exceptions
mock_storage = mock_external_service_dependencies["storage"]
- mock_storage.delete.side_effect = Exception("Storage service unavailable")
-
- # Execute the task - it should handle storage failures gracefully
- clean_dataset_task(
- dataset_id=dataset.id,
- tenant_id=tenant.id,
- indexing_technique=dataset.indexing_technique,
- index_struct=dataset.index_struct,
- collection_binding_id=dataset.collection_binding_id,
- doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),
- )
+ mock_storage.delete.side_effect = RuntimeError("Storage service unavailable")
- # Verify results
- # Note: When storage operations fail, database deletions may be rolled back by implementation.
- # This test focuses on ensuring the task handles the exception and continues execution/logging.
+ with pytest.raises(RuntimeError, match="Storage service unavailable"):
+ clean_dataset_task(
+ dataset_id=dataset.id,
+ tenant_id=tenant.id,
+ indexing_technique=dataset.indexing_technique,
+ index_struct=dataset.index_struct,
+ collection_binding_id=dataset.collection_binding_id,
+ doc_form=get_dataset_doc_form(dataset, session=db_session_with_containers),
+ )
- # Check that upload file was still deleted from database despite storage failure
- # Note: When storage operations fail, the upload file may not be deleted
- # This demonstrates that the cleanup process continues even with storage errors
- remaining_files = db_session_with_containers.scalars(
- select(UploadFile).where(UploadFile.id == upload_file.id)
- ).all()
- # The upload file should still be deleted from the database even if storage cleanup fails
- # However, this depends on the specific implementation of clean_dataset_task
- if len(remaining_files) > 0:
- print(f"Warning: Upload file {upload_file.id} was not deleted despite storage failure")
- print("This demonstrates that the cleanup process continues even with storage errors")
- # We don't assert here as the behavior depends on the specific implementation
+ db_session_with_containers.expire_all()
+ remaining_file = db_session_with_containers.get(UploadFile, upload_file_id)
+ assert remaining_file is not None
+ assert remaining_file.key == upload_file_key
+ remaining_document = db_session_with_containers.get(Document, document_id)
+ assert remaining_document is not None
+ assert remaining_document.data_source_info_dict == {"upload_file_id": upload_file_id}
- # Verify that storage.delete was called
- mock_storage.delete.assert_called_once()
+ mock_storage.delete.assert_called_once_with(upload_file_key)
# Verify that index processor was called successfully
mock_index_processor = mock_external_service_dependencies["index_processor"]
mock_index_processor.clean.assert_called_once()
- # This test demonstrates that the cleanup process continues
- # even when external storage operations fail, ensuring data
- # consistency in the database
-
def test_clean_dataset_task_edge_cases_and_boundary_conditions(
self, db_session_with_containers: Session, mock_external_service_dependencies
):
diff --git a/api/tests/unit_tests/tasks/test_clean_dataset_task.py b/api/tests/unit_tests/tasks/test_clean_dataset_task.py
index 6cf16b42ce2..33e000371dc 100644
--- a/api/tests/unit_tests/tasks/test_clean_dataset_task.py
+++ b/api/tests/unit_tests/tasks/test_clean_dataset_task.py
@@ -13,12 +13,18 @@ This module tests the dataset cleanup task functionality including:
import json
import uuid
-from collections.abc import Iterator
+from collections.abc import Iterator, Sequence
from datetime import UTC, datetime
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from threading import Thread
+from typing import override
from unittest.mock import MagicMock, patch
import pytest
+from celery.exceptions import Retry
+from flask import Flask
from sqlalchemy import Engine, event
+from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session, sessionmaker
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
@@ -26,6 +32,7 @@ from extensions.storage.storage_type import StorageType
from models.base import TypeBase
from models.dataset import (
AppDatasetJoin,
+ ChildChunk,
DatasetMetadata,
DatasetMetadataBinding,
DatasetProcessRule,
@@ -78,6 +85,7 @@ def orm_session_maker(
models = (
Document,
DocumentSegment,
+ ChildChunk,
SegmentAttachmentBinding,
UploadFile,
DatasetProcessRule,
@@ -125,6 +133,55 @@ def mock_get_image_upload_file_ids() -> Iterator[MagicMock]:
yield mock_func
+def elasticsearch_delete_endpoint() -> Iterator[tuple[int, list[int], list[tuple[str, int]]]]:
+ """Serve real SDK requests with Elasticsearch's missing-index response shape."""
+ pytest.importorskip("elasticsearch")
+ pytest.importorskip("dify_vdb_elasticsearch.elasticsearch_vector")
+ responses: list[int] = []
+ requests: list[tuple[str, int]] = []
+
+ class Handler(BaseHTTPRequestHandler):
+ def respond(self, status: int, payload: dict[str, object]) -> None:
+ body = json.dumps(payload).encode()
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.send_header("X-Elastic-Product", "Elasticsearch")
+ self.end_headers()
+ if self.command != "HEAD":
+ self.wfile.write(body)
+
+ def do_HEAD(self) -> None:
+ self.respond(200, {})
+
+ def do_GET(self) -> None:
+ self.respond(200, {"version": {"number": "8.14.0"}})
+
+ def do_DELETE(self) -> None:
+ status = responses.pop(0) if responses else 500
+ requests.append((self.path, status))
+ if status == 200:
+ self.respond(status, {"acknowledged": True})
+ else:
+ error = {"type": "index_not_found_exception", "reason": "no such index [cleanup-index]"}
+ self.respond(status, {"error": {**error, "root_cause": [error]}, "status": status})
+
+ @override
+ def log_message(self, format: str, *args: object) -> None:
+ pass
+
+ server = HTTPServer(("127.0.0.1", 0), Handler)
+ thread = Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True)
+ thread.start()
+ try:
+ yield server.server_port, responses, requests
+ finally:
+ server.shutdown()
+ thread.join(timeout=1)
+ server.server_close()
+
+
def _run_clean_dataset(
*,
dataset_id: str,
@@ -218,6 +275,29 @@ def _persist_attachment(
return binding, attachment_file
+def _persist_segment(
+ session_maker: sessionmaker[Session],
+ *,
+ dataset_id: str,
+ tenant_id: str,
+ document_id: str,
+ position: int = 1,
+) -> DocumentSegment:
+ segment = DocumentSegment(
+ tenant_id=tenant_id,
+ dataset_id=dataset_id,
+ document_id=document_id,
+ position=position,
+ content="segment",
+ word_count=1,
+ tokens=1,
+ created_by=str(uuid.uuid4()),
+ )
+ with session_maker.begin() as session:
+ session.add(segment)
+ return segment
+
+
# ============================================================================
# Test Basic Cleanup
# ============================================================================
@@ -267,15 +347,17 @@ class TestErrorHandling:
# Act
try:
- _run_clean_dataset(
- dataset_id=dataset_id,
- tenant_id=tenant_id,
- collection_binding_id=collection_binding_id,
- )
+ with patch.object(clean_dataset_task, "retry", side_effect=Retry()) as retry, pytest.raises(Retry):
+ _run_clean_dataset(
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ collection_binding_id=collection_binding_id,
+ )
finally:
event.remove(orm_session_maker.class_, "before_commit", fail_commit)
event.remove(orm_session_maker.class_, "after_rollback", fail_rollback)
+ retry.assert_called_once()
# Assert: rollback happened before its hook failed, and the closed task
# session did not prevent a new real session from reading the row.
with orm_session_maker() as session:
@@ -372,6 +454,82 @@ class TestPipelineAndWorkflowDeletion:
class TestSegmentAttachmentCleanup:
"""Test cases for segment attachment cleanup."""
+ @pytest.mark.parametrize("storage_fails", [False, True])
+ def test_inline_image_attachment_binding_is_deleted_after_its_file(
+ self,
+ storage_fails: bool,
+ dataset_id: str,
+ tenant_id: str,
+ collection_binding_id: str,
+ orm_session_maker: sessionmaker[Session],
+ mock_storage: MagicMock,
+ mock_index_processor_factory: dict[str, MagicMock],
+ ) -> None:
+ """Inline images, source files and attachments can share an uploaded file."""
+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
+ segment = _persist_segment(
+ orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id
+ )
+ binding, attachment_file = _persist_attachment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
+ segment.content = f"![image](https://example.com/files/{attachment_file.id}/file-preview)"
+ document.data_source_type = "upload_file"
+ document.data_source_info = json.dumps({"upload_file_id": attachment_file.id})
+ binding.document_id = document.id
+ binding.segment_id = segment.id
+ with orm_session_maker.begin() as session:
+ session.add_all([document, segment, binding])
+ unrelated_attachments = [
+ _persist_attachment(orm_session_maker, dataset_id=str(uuid.uuid4()), tenant_id=tenant_id),
+ _persist_attachment(orm_session_maker, dataset_id=dataset_id, tenant_id=str(uuid.uuid4())),
+ ]
+ if storage_fails:
+ failure = TimeoutError("shared source and inline image storage unavailable")
+ mock_storage.delete.side_effect = failure
+ with pytest.raises(Retry) as retried:
+ clean_dataset_task.apply(
+ kwargs={
+ "dataset_id": dataset_id,
+ "tenant_id": tenant_id,
+ "indexing_technique": IndexTechniqueType.HIGH_QUALITY,
+ "index_struct": None,
+ "collection_binding_id": collection_binding_id,
+ "doc_form": IndexStructureType.PARAGRAPH_INDEX,
+ },
+ throw=True,
+ )
+ assert retried.value.exc is failure
+ with orm_session_maker() as session:
+ assert session.get(Document, document.id) is not None
+ assert session.get(DocumentSegment, segment.id) is not None
+ assert session.get(SegmentAttachmentBinding, binding.id) is not None
+ retained_file = session.get(UploadFile, attachment_file.id)
+ assert retained_file is not None
+ assert retained_file.key == attachment_file.key
+ # celery-stubs omits the signature stored on Retry at runtime.
+ retry_signature = retried.value.sig # pyrefly: ignore[missing-attribute]
+ assert retry_signature is not None
+ assert retry_signature.kwargs["index_cleanup_done"] is True
+ mock_storage.delete.side_effect = None
+ assert retry_signature.apply(throw=True).successful()
+ else:
+ _run_clean_dataset(
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ collection_binding_id=collection_binding_id,
+ )
+
+ assert mock_storage.delete.call_count == (2 if storage_fails else 1)
+ mock_storage.delete.assert_called_with(attachment_file.key)
+ mock_index_processor_factory["processor"].clean.assert_called_once()
+ with orm_session_maker() as session:
+ assert session.get(Document, document.id) is None
+ assert session.get(DocumentSegment, segment.id) is None
+ assert session.get(UploadFile, attachment_file.id) is None
+ assert session.get(SegmentAttachmentBinding, binding.id) is None
+ for other_binding, other_file in unrelated_attachments:
+ assert session.get(SegmentAttachmentBinding, other_binding.id) is not None
+ assert session.get(UploadFile, other_file.id) is not None
+
def test_clean_dataset_task_with_attachments(
self,
dataset_id: str,
@@ -500,6 +658,103 @@ class TestEdgeCases:
class TestIndexProcessorParameters:
"""Test cases for IndexProcessor clean method parameters."""
+ @pytest.mark.parametrize("commit_fails", [False, True])
+ def test_missing_elasticsearch_index_allows_parent_child_cleanup_after_retry(
+ self,
+ commit_fails: bool,
+ app: Flask,
+ monkeypatch: pytest.MonkeyPatch,
+ elasticsearch_delete_endpoint: tuple[int, list[int], list[tuple[str, int]]],
+ dataset_id: str,
+ tenant_id: str,
+ collection_binding_id: str,
+ orm_session_maker: sessionmaker[Session],
+ mock_storage: MagicMock,
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ port, responses, requests = elasticsearch_delete_endpoint
+ expected_statuses = [200, 404] if commit_fails else [404]
+ responses.extend(expected_statuses)
+ for key, value in {
+ "ELASTICSEARCH_USE_CLOUD": False,
+ "ELASTICSEARCH_HOST": "http://127.0.0.1",
+ "ELASTICSEARCH_PORT": port,
+ "ELASTICSEARCH_USERNAME": "elastic",
+ "ELASTICSEARCH_PASSWORD": "test",
+ "ELASTICSEARCH_REQUEST_TIMEOUT": 2,
+ "ELASTICSEARCH_MAX_RETRIES": 0,
+ "ELASTICSEARCH_RETRY_ON_TIMEOUT": False,
+ }.items():
+ monkeypatch.setitem(app.config, key, value)
+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
+ segment = _persist_segment(
+ orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id
+ )
+ child = ChildChunk(
+ tenant_id=tenant_id,
+ dataset_id=dataset_id,
+ document_id=document.id,
+ segment_id=segment.id,
+ position=1,
+ content="child",
+ word_count=1,
+ created_by=str(uuid.uuid4()),
+ )
+ with orm_session_maker.begin() as session:
+ session.add(child)
+ task_kwargs = {
+ "dataset_id": dataset_id,
+ "tenant_id": tenant_id,
+ "indexing_technique": IndexTechniqueType.HIGH_QUALITY,
+ "index_struct": json.dumps({"type": "elasticsearch", "vector_store": {"class_prefix": "cleanup-index"}}),
+ "collection_binding_id": collection_binding_id,
+ "doc_form": IndexStructureType.PARENT_CHILD_INDEX,
+ }
+ failure = OperationalError("commit", {}, RuntimeError("temporary index commit failure"))
+ injected_failures: list[OperationalError] = []
+
+ def fail_first_index_commit(session: Session) -> None:
+ if requests == [("/cleanup-index", 200)] and not injected_failures:
+ assert session.get(ChildChunk, child.id) is None
+ injected_failures.append(failure)
+ raise failure
+
+ with (
+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,
+ caplog.at_level("INFO", logger="tasks.clean_dataset_task"),
+ ):
+ if commit_fails:
+ event.listen(orm_session_maker, "before_commit", fail_first_index_commit)
+ try:
+ with pytest.raises(Retry) as retried:
+ clean_dataset_task.apply(kwargs=task_kwargs, throw=True)
+ finally:
+ event.remove(orm_session_maker, "before_commit", fail_first_index_commit)
+ assert injected_failures == [failure]
+ assert retried.value.exc is failure
+ with orm_session_maker() as session:
+ assert session.get(ChildChunk, child.id) is not None
+ assert session.get(DocumentSegment, segment.id) is not None
+ assert session.get(Document, document.id) is not None
+ schedule_refresh.assert_not_called()
+ assert not any("Cleaned dataset when dataset deleted:" in message for message in caplog.messages)
+ # celery-stubs omits the signature stored on Retry at runtime.
+ retry_signature = retried.value.sig # pyrefly: ignore[missing-attribute]
+ assert retry_signature is not None
+ assert retry_signature.kwargs["index_cleanup_done"] is False
+ result = retry_signature.apply(throw=True)
+ else:
+ result = clean_dataset_task.apply(kwargs=task_kwargs, throw=True)
+ assert result.successful()
+ schedule_refresh.assert_called_once_with(tenant_id)
+
+ assert requests == [("/cleanup-index", status) for status in expected_statuses]
+ with orm_session_maker() as session:
+ assert session.get(ChildChunk, child.id) is None
+ assert session.get(DocumentSegment, segment.id) is None
+ assert session.get(Document, document.id) is None
+ mock_storage.delete.assert_not_called()
+
def test_clean_dataset_task_passes_correct_parameters_to_index_processor(
self,
dataset_id: str,
@@ -556,7 +811,7 @@ class TestIndexProcessorParameters:
assert call_args[1]["delete_child_chunks"] is True
schedule_refresh.assert_called_once_with(tenant_id)
- def test_vector_cleanup_failure_does_not_schedule_billing_refresh(
+ def test_vector_cleanup_failure_preserves_documents_and_does_not_schedule_billing_refresh(
self,
dataset_id: str,
tenant_id: str,
@@ -566,9 +821,17 @@ class TestIndexProcessorParameters:
mock_index_processor_factory: dict[str, MagicMock],
mock_get_image_upload_file_ids: MagicMock,
) -> None:
- mock_index_processor_factory["processor"].clean.side_effect = RuntimeError("vector cleanup failed")
+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
+ segment = _persist_segment(
+ orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id
+ )
+ failure = RuntimeError("vector cleanup failed")
+ mock_index_processor_factory["processor"].clean.side_effect = failure
- with patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh:
+ with (
+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,
+ pytest.raises(RuntimeError, match="vector cleanup failed") as raised,
+ ):
clean_dataset_task(
dataset_id=dataset_id,
tenant_id=tenant_id,
@@ -578,7 +841,531 @@ class TestIndexProcessorParameters:
doc_form=IndexStructureType.PARAGRAPH_INDEX,
)
+ assert raised.value is failure
schedule_refresh.assert_not_called()
+ with orm_session_maker() as session:
+ assert session.get(Document, document.id) is not None
+ assert session.get(DocumentSegment, segment.id) is not None
+
+ @pytest.mark.parametrize("failure_phase", ["flush", "commit"])
+ def test_index_database_failure_preserves_parent_and_child_rows_until_retry(
+ self,
+ failure_phase: str,
+ dataset_id: str,
+ tenant_id: str,
+ collection_binding_id: str,
+ orm_session_maker: sessionmaker[Session],
+ sqlite_engine: Engine,
+ mock_storage: MagicMock,
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """An index rollback must not strand its child rows after parent deletion."""
+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
+ segment = _persist_segment(
+ orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id
+ )
+ child = ChildChunk(
+ tenant_id=tenant_id,
+ dataset_id=dataset_id,
+ document_id=document.id,
+ segment_id=segment.id,
+ position=1,
+ content="child",
+ word_count=1,
+ created_by=str(uuid.uuid4()),
+ )
+ with orm_session_maker.begin() as session:
+ session.add(child)
+ failure = OperationalError(failure_phase, {}, RuntimeError("temporary index database failure"))
+ child_deletes: list[str] = []
+ injected_failures: list[OperationalError] = []
+
+ def record_child_delete(statement: str, **_kwargs: object) -> None:
+ if statement.lower().startswith("delete from child_chunks"):
+ child_deletes.append(statement)
+
+ def fail_first_index_commit(_session: Session) -> None:
+ if failure_phase == "commit" and child_deletes and not injected_failures:
+ injected_failures.append(failure)
+ raise failure
+
+ original_flush = Session.flush
+
+ def fail_first_index_flush(session: Session, objects: Sequence[object] | None = None) -> None:
+ if failure_phase == "flush" and child_deletes and not injected_failures:
+ injected_failures.append(failure)
+ raise failure
+ original_flush(session, objects)
+
+ with (
+ patch("core.rag.index_processor.processor.parent_child_index_processor.Vector", autospec=True) as vector,
+ patch.object(Session, "flush", autospec=True, side_effect=fail_first_index_flush),
+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,
+ caplog.at_level("INFO", logger="tasks.clean_dataset_task"),
+ ):
+ event.listen(sqlite_engine, "after_cursor_execute", record_child_delete, named=True)
+ event.listen(orm_session_maker, "before_commit", fail_first_index_commit)
+ try:
+ with pytest.raises(Retry) as retried:
+ clean_dataset_task.apply(
+ kwargs={
+ "dataset_id": dataset_id,
+ "tenant_id": tenant_id,
+ "indexing_technique": IndexTechniqueType.HIGH_QUALITY,
+ "index_struct": None,
+ "collection_binding_id": collection_binding_id,
+ "doc_form": IndexStructureType.PARENT_CHILD_INDEX,
+ },
+ throw=True,
+ )
+ finally:
+ event.remove(sqlite_engine, "after_cursor_execute", record_child_delete)
+ event.remove(orm_session_maker, "before_commit", fail_first_index_commit)
+
+ assert len(child_deletes) == 1
+ assert injected_failures == [failure]
+ assert retried.value.exc is failure
+ # celery-stubs omits the signature stored on Retry at runtime.
+ retry_signature = retried.value.sig # pyrefly: ignore[missing-attribute]
+ assert retry_signature is not None
+ assert retry_signature.kwargs["index_cleanup_done"] is False
+ with orm_session_maker() as session:
+ assert session.get(ChildChunk, child.id) is not None
+ assert session.get(DocumentSegment, segment.id) is not None
+ assert session.get(Document, document.id) is not None
+ schedule_refresh.assert_not_called()
+ assert not any("Cleaned dataset when dataset deleted:" in message for message in caplog.messages)
+
+ assert retry_signature.apply(throw=True).successful()
+ assert vector.return_value.delete.call_count == 2
+ schedule_refresh.assert_called_once_with(tenant_id)
+
+ with orm_session_maker() as session:
+ assert session.get(ChildChunk, child.id) is None
+ assert session.get(DocumentSegment, segment.id) is None
+ assert session.get(Document, document.id) is None
+ mock_storage.delete.assert_not_called()
+
+
+# ============================================================================
+# Test Step Isolation and Batching
+# ============================================================================
+
+
+class TestStepIsolation:
+ """Committed batches survive failure, while dependent cleanup waits for retry."""
+
+ def test_source_file_failure_keeps_batch_references_and_retries_missing_blob(
+ self,
+ orm_session_maker: sessionmaker[Session],
+ dataset_id: str,
+ tenant_id: str,
+ collection_binding_id: str,
+ mock_storage: MagicMock,
+ mock_index_processor_factory: dict[str, MagicMock],
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """Retry a partially deleted source-file batch without losing its storage keys."""
+ documents = [_persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id) for _ in range(2)]
+ source_files = [
+ UploadFile(
+ tenant_id=tenant_id,
+ storage_type=StorageType.LOCAL,
+ key=f"upload_files/{uuid.uuid4()}.pdf",
+ name="source.pdf",
+ size=10,
+ extension="pdf",
+ mime_type="application/pdf",
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by=str(uuid.uuid4()),
+ created_at=datetime.now(UTC),
+ used=False,
+ )
+ for _ in documents
+ ]
+ with orm_session_maker.begin() as session:
+ for document, source_file in zip(documents, source_files, strict=True):
+ document.data_source_type = "upload_file"
+ document.data_source_info = json.dumps({"upload_file_id": source_file.id})
+ session.add_all([document, source_file])
+
+ remaining_blobs = {source_file.key for source_file in source_files}
+ attempted_keys: list[str] = []
+ failure = TimeoutError("source storage unavailable")
+
+ def delete_blob(key: str) -> None:
+ attempted_keys.append(key)
+ if len(attempted_keys) == 2:
+ raise failure
+ if key not in remaining_blobs:
+ raise FileNotFoundError(key)
+ remaining_blobs.remove(key)
+
+ mock_storage.delete.side_effect = delete_blob
+ with (
+ patch("tasks.clean_dataset_task._DELETE_BATCH_SIZE", 2),
+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,
+ caplog.at_level("INFO", logger="tasks.clean_dataset_task"),
+ ):
+ with pytest.raises(Retry) as retried:
+ clean_dataset_task.apply(
+ kwargs={
+ "dataset_id": dataset_id,
+ "tenant_id": tenant_id,
+ "indexing_technique": IndexTechniqueType.HIGH_QUALITY,
+ "index_struct": None,
+ "collection_binding_id": collection_binding_id,
+ "doc_form": IndexStructureType.PARAGRAPH_INDEX,
+ },
+ throw=True,
+ )
+
+ assert len(attempted_keys) == 2
+ assert remaining_blobs == {attempted_keys[1]}
+ with orm_session_maker() as session:
+ for document, source_file in zip(documents, source_files, strict=True):
+ retained_document = session.get(Document, document.id)
+ retained_file = session.get(UploadFile, source_file.id)
+ assert retained_document is not None
+ assert retained_document.data_source_info_dict["upload_file_id"] == source_file.id
+ assert retained_file is not None
+ assert retained_file.key == source_file.key
+ assert retried.value.exc is failure
+ assert not any("Cleaned dataset when dataset deleted:" in message for message in caplog.messages)
+ # celery-stubs omits the signature stored on Retry at runtime.
+ retry_signature = retried.value.sig # pyrefly: ignore[missing-attribute]
+ assert retry_signature is not None
+ assert retry_signature.kwargs["index_cleanup_done"] is True
+ schedule_refresh.assert_called_once_with(tenant_id)
+
+ assert retry_signature.apply(throw=True).successful()
+ mock_index_processor_factory["processor"].clean.assert_called_once()
+ schedule_refresh.assert_called_once_with(tenant_id)
+
+ assert not remaining_blobs
+ assert all(attempted_keys.count(source_file.key) == 2 for source_file in source_files)
+ with orm_session_maker() as session:
+ assert all(session.get(Document, document.id) is None for document in documents)
+ assert all(session.get(UploadFile, source_file.id) is None for source_file in source_files)
+
+ def test_cleanup_commits_more_than_once(
+ self,
+ orm_session_maker: sessionmaker[Session],
+ sqlite_session_factory: sessionmaker[Session],
+ dataset_id: str,
+ tenant_id: str,
+ collection_binding_id: str,
+ mock_storage: MagicMock,
+ mock_index_processor_factory: dict[str, MagicMock],
+ mock_get_image_upload_file_ids: MagicMock,
+ ):
+ """A single commit at the end is what strands data when any step fails."""
+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
+ _persist_segment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id)
+
+ commits: list[int] = []
+
+ def _count_commit(_session: Session) -> None:
+ commits.append(1)
+
+ event.listen(sqlite_session_factory, "after_commit", _count_commit)
+ try:
+ _run_clean_dataset(
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ collection_binding_id=collection_binding_id,
+ )
+ finally:
+ event.remove(sqlite_session_factory, "after_commit", _count_commit)
+
+ assert len(commits) > 1, (
+ f"Cleanup still runs as one transaction ({len(commits)} commit); a failure anywhere rolls back everything"
+ )
+
+ def test_document_failure_keeps_segments_deleted_and_preserves_metadata(
+ self,
+ orm_session_maker: sessionmaker[Session],
+ dataset_id: str,
+ tenant_id: str,
+ collection_binding_id: str,
+ mock_storage: MagicMock,
+ mock_index_processor_factory: dict[str, MagicMock],
+ mock_get_image_upload_file_ids: MagicMock,
+ ) -> None:
+ """A document failure must preserve later cleanup inputs for the retry."""
+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
+ segment = _persist_segment(
+ orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id
+ )
+ metadata = DatasetMetadata(
+ tenant_id=tenant_id,
+ dataset_id=dataset_id,
+ type="string",
+ name="meta",
+ created_by=str(uuid.uuid4()),
+ )
+ with orm_session_maker.begin() as session:
+ session.add(metadata)
+
+ with (
+ patch("tasks.clean_dataset_task._delete_documents", side_effect=RuntimeError("deadlock on documents")),
+ pytest.raises(RuntimeError, match="deadlock on documents"),
+ ):
+ _run_clean_dataset(
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ collection_binding_id=collection_binding_id,
+ )
+
+ with orm_session_maker() as session:
+ assert session.get(DocumentSegment, segment.id) is None, (
+ "The committed segment deletion was rolled back by a later failure"
+ )
+ assert session.get(DatasetMetadata, metadata.id) is not None
+ assert session.get(Document, document.id) is not None, (
+ "Sanity check: the failing step should not have deleted its own rows"
+ )
+
+ @pytest.mark.parametrize("call_style", ["positional", "keywords"])
+ def test_second_segment_batch_failure_preserves_cleanup_inputs_and_resumes(
+ self,
+ call_style: str,
+ orm_session_maker: sessionmaker[Session],
+ sqlite_engine: Engine,
+ dataset_id: str,
+ tenant_id: str,
+ pipeline_id: str,
+ mock_storage: MagicMock,
+ mock_index_processor_factory: dict[str, MagicMock],
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
+ segments = [
+ _persist_segment(
+ orm_session_maker,
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ document_id=document.id,
+ position=position,
+ )
+ for position in range(5)
+ ]
+ binding, attachment_file = _persist_attachment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
+ metadata = DatasetMetadata(
+ tenant_id=tenant_id, dataset_id=dataset_id, type="string", name="meta", created_by=str(uuid.uuid4())
+ )
+ with orm_session_maker.begin() as session:
+ session.add(metadata)
+ task_kwargs = {
+ "dataset_id": dataset_id,
+ "tenant_id": tenant_id,
+ "indexing_technique": None,
+ "index_struct": None,
+ "collection_binding_id": None,
+ "doc_form": IndexStructureType.PARAGRAPH_INDEX,
+ "pipeline_id": pipeline_id,
+ }
+ failure = RuntimeError("second segment batch commit failed")
+ segment_deletes: list[str] = []
+
+ def record_segment_delete(statement: str, **_kwargs: object) -> None:
+ if statement.lower().startswith("delete from document_segments"):
+ segment_deletes.append(statement)
+
+ def fail_second_segment_commit(_session: Session) -> None:
+ if len(segment_deletes) == 2:
+ raise failure
+
+ retry_task = clean_dataset_task.retry
+
+ def retry_after_transaction_ends(
+ *, exc: Exception, args: tuple[object, ...], kwargs: dict[str, object], countdown: int
+ ) -> Retry:
+ cleanup_session = mock_index_processor_factory["processor"].clean.call_args.kwargs["session"]
+ assert not cleanup_session.in_transaction()
+ return retry_task(exc=exc, args=args, kwargs=kwargs, countdown=countdown)
+
+ with (
+ patch("tasks.clean_dataset_task._DELETE_BATCH_SIZE", 2),
+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,
+ patch.object(clean_dataset_task, "retry", side_effect=retry_after_transaction_ends),
+ caplog.at_level("INFO", logger="tasks.clean_dataset_task"),
+ ):
+ event.listen(sqlite_engine, "after_cursor_execute", record_segment_delete, named=True)
+ event.listen(orm_session_maker, "before_commit", fail_second_segment_commit)
+ try:
+ with pytest.raises(Retry) as retried:
+ clean_dataset_task.apply(
+ args=tuple(task_kwargs.values()) if call_style == "positional" else (),
+ kwargs=task_kwargs if call_style == "keywords" else {},
+ throw=True,
+ )
+ finally:
+ event.remove(sqlite_engine, "after_cursor_execute", record_segment_delete)
+ event.remove(orm_session_maker, "before_commit", fail_second_segment_commit)
+
+ assert len(segment_deletes) == 2
+ assert not any("Cleaned dataset when dataset deleted:" in message for message in caplog.messages)
+ with orm_session_maker() as session:
+ assert sum(session.get(DocumentSegment, segment.id) is not None for segment in segments) == 3
+ assert session.get(Document, document.id) is not None
+ assert session.get(SegmentAttachmentBinding, binding.id) is not None
+ assert session.get(UploadFile, attachment_file.id) is not None
+ assert session.get(DatasetMetadata, metadata.id) is not None
+ mock_storage.delete.assert_not_called()
+ schedule_refresh.assert_called_once_with(tenant_id)
+ assert retried.value.exc is failure
+ assert retried.value.when == 30
+ retry_signature = retried.value.sig
+ assert retry_signature is not None
+ assert retry_signature.args == ()
+ assert retry_signature.kwargs == {**task_kwargs, "index_cleanup_done": True}
+ assert retry_signature.apply(throw=True).successful()
+ mock_index_processor_factory["processor"].clean.assert_called_once()
+ schedule_refresh.assert_called_once_with(tenant_id)
+
+ with orm_session_maker() as session:
+ assert all(session.get(DocumentSegment, segment.id) is None for segment in segments)
+ assert session.get(Document, document.id) is None
+ assert session.get(SegmentAttachmentBinding, binding.id) is None
+ assert session.get(UploadFile, attachment_file.id) is None
+ assert session.get(DatasetMetadata, metadata.id) is None
+ mock_storage.delete.assert_called_once_with(attachment_file.key)
+
+ @pytest.mark.parametrize(("retries", "delay"), [(1, 60), (2, 120), (3, None)])
+ def test_crawl_cache_failure_retries_before_index_cleanup_and_stops_at_limit(
+ self,
+ retries: int,
+ delay: int | None,
+ dataset_id: str,
+ tenant_id: str,
+ collection_binding_id: str,
+ mock_index_processor_factory: dict[str, MagicMock],
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ failure = RuntimeError("crawl cache unavailable")
+ with (
+ patch("tasks.clean_dataset_task.cleanup_dataset_crawl_cache", side_effect=failure),
+ patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh,
+ caplog.at_level("INFO", logger="tasks.clean_dataset_task"),
+ pytest.raises(Retry if delay is not None else RuntimeError) as raised,
+ ):
+ clean_dataset_task.apply(
+ kwargs={
+ "dataset_id": dataset_id,
+ "tenant_id": tenant_id,
+ "indexing_technique": IndexTechniqueType.HIGH_QUALITY,
+ "index_struct": '{"type": "paragraph"}',
+ "collection_binding_id": collection_binding_id,
+ "doc_form": IndexStructureType.PARAGRAPH_INDEX,
+ },
+ retries=retries,
+ throw=True,
+ )
+
+ if delay is None:
+ assert raised.value is failure
+ else:
+ assert isinstance(raised.value, Retry)
+ assert raised.value.exc is failure
+ assert raised.value.when == delay
+ # celery-stubs omits the signature stored on Retry at runtime.
+ retry_signature = raised.value.sig # pyrefly: ignore[missing-attribute]
+ assert retry_signature is not None
+ assert retry_signature.kwargs["index_cleanup_done"] is False
+ assert not any("Cleaned dataset when dataset deleted:" in message for message in caplog.messages)
+ mock_index_processor_factory["processor"].clean.assert_not_called()
+ schedule_refresh.assert_not_called()
+
+
+class TestBatchedDeletes:
+ """Large child tables are deleted in bounded batches."""
+
+ def test_text_segments_use_bounded_transactions_without_image_file_queries(
+ self,
+ orm_session_maker: sessionmaker[Session],
+ sqlite_engine: Engine,
+ dataset_id: str,
+ tenant_id: str,
+ collection_binding_id: str,
+ mock_storage: MagicMock,
+ mock_index_processor_factory: dict[str, MagicMock],
+ ) -> None:
+ """Text-only cleanup must not run no-op file SQL or retain the full dataset."""
+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
+ segments = [
+ _persist_segment(
+ orm_session_maker,
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ document_id=document.id,
+ position=position,
+ )
+ for position in range(5)
+ ]
+ file_queries: list[str] = []
+ committed_segment_loads: list[int] = []
+ loaded_segment_ids: list[str] = []
+
+ def record_file_query(statement: str, **_kwargs: object) -> None:
+ if "from upload_files" in statement.lower():
+ file_queries.append(statement)
+
+ def record_segment_load(_session: Session, instance: object) -> None:
+ if isinstance(instance, DocumentSegment):
+ loaded_segment_ids.append(instance.id)
+
+ def record_commit(_session: Session) -> None:
+ if loaded_segment_ids:
+ committed_segment_loads.append(len(loaded_segment_ids))
+ loaded_segment_ids.clear()
+
+ event.listen(sqlite_engine, "before_cursor_execute", record_file_query, named=True)
+ event.listen(orm_session_maker, "loaded_as_persistent", record_segment_load)
+ event.listen(orm_session_maker, "after_commit", record_commit)
+ try:
+ with patch("tasks.clean_dataset_task._DELETE_BATCH_SIZE", 2):
+ _run_clean_dataset(
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ collection_binding_id=collection_binding_id,
+ )
+ finally:
+ event.remove(sqlite_engine, "before_cursor_execute", record_file_query)
+ event.remove(orm_session_maker, "loaded_as_persistent", record_segment_load)
+ event.remove(orm_session_maker, "after_commit", record_commit)
+
+ assert file_queries == [], "Text-only segments issued unnecessary UploadFile SQL"
+ assert sum(committed_segment_loads) == len(segments)
+ assert max(committed_segment_loads) <= 2, "Segment cleanup exceeded the transaction batch size"
+ mock_storage.delete.assert_not_called()
+ with orm_session_maker() as session:
+ assert all(session.get(DocumentSegment, segment.id) is None for segment in segments)
+
+ def test_documents_are_deleted_across_multiple_batches(
+ self,
+ orm_session_maker: sessionmaker[Session],
+ dataset_id: str,
+ tenant_id: str,
+ collection_binding_id: str,
+ mock_storage: MagicMock,
+ mock_index_processor_factory: dict[str, MagicMock],
+ mock_get_image_upload_file_ids: MagicMock,
+ ):
+ """With a batch size smaller than the row count the loop must still drain the
+ table, rather than stopping after the first batch."""
+ documents = [_persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id) for _ in range(5)]
+
+ with patch("tasks.clean_dataset_task._DELETE_BATCH_SIZE", 2):
+ _run_clean_dataset(
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ collection_binding_id=collection_binding_id,
+ )
+
+ with orm_session_maker() as session:
+ remaining = [document.id for document in documents if session.get(Document, document.id) is not None]
+ assert not remaining, f"Batched delete left {len(remaining)} of 5 documents behind"
@pytest.mark.parametrize("shared_tenant", [None, "same", "other"])