Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(api): discard index entries of documents deleted during indexing (#44364)
Tianyu Cai06b8640c4aed
diff --git a/api/repositories/knowledge/document_repository.py b/api/repositories/knowledge/document_repository.pyindex d5d9f8724f3..e3c3cc11a38 100644--- a/api/repositories/knowledge/document_repository.py+++ b/api/repositories/knowledge/document_repository.py@@ -11,6 +11,7 @@ from models import Accountfrom models.dataset import Dataset, DatasetProcessRule, Document, Pipelinefrom models.enums import IndexingStatusfrom repositories.knowledge.dataset_read_repository import get_pipeline_dataset+from repositories.knowledge.dataset_repository import _get_datasetfrom services.knowledge.document_sync import SyncDocumentRecordfrom services.knowledge.indexing.errors import DocumentIsDeletedPausedError, DocumentIsPausedErrorfrom services.knowledge.indexing.estimate import EstimateDocumentRecord, StoredSource@@ -119,6 +120,14 @@ class SQLAlchemyDocumentRepository:assert dataset is not Nonereturn dataset, document+ def get_indexing_dataset(self, dataset_ref: DatasetRef) -> Dataset | None:+ """Detached dataset for index cleanup after the document row may already be gone.++ None means the dataset was deleted too, so dataset cleanup owns its whole index.+ """+ with self._session_factory() as session:+ return _get_dataset(session, dataset_ref)+def get_indexing_user(self, ref: DocumentRef) -> Account:with self._session_factory() as session:document = require_indexing_document(session, ref)diff --git a/api/services/knowledge/dataset_service.py b/api/services/knowledge/dataset_service.pyindex 481822f9485..e5f48fecb28 100644--- a/api/services/knowledge/dataset_service.py+++ b/api/services/knowledge/dataset_service.py@@ -1942,23 +1942,28 @@ class DocumentService:@staticmethoddef delete_document(document, session: Session):- # trigger document_was_deleted signalfile_id = Noneif document.data_source_type == DataSourceType.UPLOAD_FILE:if document.data_source_info:data_source_info = document.data_source_info_dictif data_source_info and "upload_file_id" in data_source_info:file_id = data_source_info["upload_file_id"]- document_was_deleted.send(- document.id,- dataset_id=document.dataset_id,- doc_form=document.doc_form,- file_id=file_id,- )+ # Read everything cleanup needs before the row is deleted and expired.+ document_id, dataset_id, doc_form = document.id, document.dataset_id, document.doc_formsession.delete(document)session.commit()+ # Dispatch cleanup only after the deletion commits, as bulk deletion does. An+ # in-flight indexing run either sees the document gone and discards what it+ # wrote, or finished writing before the commit, so cleanup still finds it.+ document_was_deleted.send(+ document_id,+ dataset_id=dataset_id,+ doc_form=doc_form,+ file_id=file_id,+ )+@staticmethoddef delete_documents(dataset_ref: DatasetRef,diff --git a/api/services/knowledge/indexing/adapters/cleanup.py b/api/services/knowledge/indexing/adapters/cleanup.pyindex e6ef60a5cb4..a8f195124fe 100644--- a/api/services/knowledge/indexing/adapters/cleanup.py+++ b/api/services/knowledge/indexing/adapters/cleanup.py@@ -15,6 +15,52 @@ from repositories.knowledge.dataset_read_repository import get_dataset_keyword_tfrom repositories.knowledge.keyword_table_repository import persist_keyword_table+def delete_index_entries(+ dataset: Dataset,+ *,+ vector_node_ids: Sequence[str],+ vector_type: str | None,+ keyword_node_ids: Sequence[str],+ new_session: Callable[[], Session],+) -> None:+ """Delete index entries by node id; the single owner of how each index type is removed.++ High-quality datasets delete ``vector_node_ids`` from the vector store; other datasets+ delete ``keyword_node_ids`` from the keyword table under the dataset keyword lock.+ """+ if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:+ if vector_node_ids:+ Vector(dataset, session=None, vector_type=vector_type).delete_by_ids(list(vector_node_ids))+ return+ if not keyword_node_ids:+ return++ # The keyword adapter owns its lock and storage I/O; callbacks only do SQL.+ def read() -> tuple[str, str | None]:+ with new_session() as session:+ row = get_dataset_keyword_table(dataset, session=session)+ return (row.data_source_type, row.keyword_table) if row else (dify_config.KEYWORD_DATA_SOURCE_TYPE, None)++ def write(storage_type: str, data: str, keywords: Mapping[str, Sequence[str]]) -> None:+ with new_session() as session, session.begin():+ persist_keyword_table(+ session,+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ storage_type=storage_type,+ data=data,+ keywords=keywords,+ )++ Jieba(dataset).update_texts(+ [],+ read=read,+ write=write,+ lock=redis_client.lock(f"keyword_indexing_lock_{dataset.id}", timeout=600),+ deleted_ids=keyword_node_ids,+ )++def clean_document_indexes(*, dataset_id: str, document_ids: Sequence[str], doc_form: str | None, new_session: Callable[[], Session]) -> str | None:@@ -68,36 +114,13 @@ def clean_document_indexes(if not node_ids and not summary_ids and not child_ids:return None- if high_quality:- if node_ids:- Vector(dataset, session=None, vector_type=vector_type).delete_by_ids(node_ids)- elif body_node_ids:- # The keyword adapter owns its lock and storage I/O; callbacks only do SQL.- def read() -> tuple[str, str | None]:- with new_session() as session:- row = get_dataset_keyword_table(dataset, session=session)- return (- (row.data_source_type, row.keyword_table) if row else (dify_config.KEYWORD_DATA_SOURCE_TYPE, None)- )-- def write(storage_type: str, data: str, keywords: Mapping[str, Sequence[str]]) -> None:- with new_session() as session, session.begin():- persist_keyword_table(- session,- tenant_id=tenant_id,- dataset_id=dataset_id,- storage_type=storage_type,- data=data,- keywords=keywords,- )-- Jieba(dataset).update_texts(- [],- read=read,- write=write,- lock=redis_client.lock(f"keyword_indexing_lock_{dataset_id}", timeout=600),- deleted_ids=body_node_ids,- )+ delete_index_entries(+ dataset,+ vector_node_ids=node_ids,+ vector_type=vector_type,+ keyword_node_ids=body_node_ids,+ new_session=new_session,+ )with new_session() as session, session.begin():if summary_ids:diff --git a/api/services/knowledge/indexing/adapters/execution.py b/api/services/knowledge/indexing/adapters/execution.pyindex 80f81b90b01..08254c7828a 100644--- a/api/services/knowledge/indexing/adapters/execution.py+++ b/api/services/knowledge/indexing/adapters/execution.py@@ -33,6 +33,7 @@ from models.enums import DataSourceTypefrom repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom repositories.knowledge.segment_repository import SQLAlchemySegmentRepositoryfrom repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository+from services.knowledge.indexing.adapters.cleanup import delete_index_entriesfrom services.knowledge.indexing.adapters.sources import (CompositeStoredSourceResolver,FileSourceAdapter,@@ -174,6 +175,38 @@ class IndexingExecutionAdapter:with ThreadPoolExecutor(max_workers=1) as executor:executor.submit(propagate_context(self._process_chunk), app, document.ref, chunks, True).result()+ def discard(self, document: IndexingDocument, chunks: list[Document]) -> None:+ """Remove the index entries ``load`` writes for ``chunks`` through ``delete_index_entries``.++ Mirrors ``_process_chunk``: child vectors for parent-child documents, body vectors+ otherwise, and body keywords for economy datasets. Some ids may never have been+ written; each store handles that as it does when document cleanup issues the same call.+ """+ dataset = self._documents.get_indexing_dataset(document.ref.dataset)+ if dataset is None:+ return+ if document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:+ node_ids = [child.metadata["doc_id"] for chunk in chunks for child in chunk.children or []]+ else:+ node_ids = [chunk.metadata["doc_id"] for chunk in chunks]+ if not node_ids:+ return+ vector_type = None+ if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:+ with self._session_factory() as session:+ vector_type = Vector.resolve_vector_type(dataset, session=session)+ writes_keywords = (+ dataset.indexing_technique == IndexTechniqueType.ECONOMY+ and document.doc_form != IndexStructureType.PARENT_CHILD_INDEX+ )+ delete_index_entries(+ dataset,+ vector_node_ids=node_ids,+ vector_type=vector_type,+ keyword_node_ids=node_ids if writes_keywords else [],+ new_session=self._session_factory,+ )+def _process_chunk(self, app: Flask, ref: DocumentRef, chunks: list[Document], keywords: bool) -> None:with app.app_context():self.check_paused(ref)diff --git a/api/services/knowledge/indexing/execution.py b/api/services/knowledge/indexing/execution.pyindex dba53de6dbb..97a9fec071d 100644--- a/api/services/knowledge/indexing/execution.py+++ b/api/services/knowledge/indexing/execution.py@@ -69,6 +69,10 @@ class IndexingBackend(Protocol):def load(self, document: IndexingDocument, chunks: list[Document]) -> None: ...+ def discard(self, document: IndexingDocument, chunks: list[Document]) -> None:+ """Remove index entries that ``load`` may have written for a document deleted meanwhile."""+ ...+class DocumentIndexingService:"""Coordinate committed phases; no caller session is held across execution."""@@ -102,6 +106,11 @@ class DocumentIndexingService:self._run(ref, start="indexing")def _run(self, ref: DocumentRef, *, start: Literal["parsing", "splitting", "indexing"]) -> None:+ # None: no index write has started, so a deletion leaves nothing behind.+ # Set: load may have written these chunks; document cleanup snapshots segment+ # node ids before deleting them and cannot see entries written after its snapshot,+ # so whichever way this run stops, the entries are discarded if the document is gone.+ loading: tuple[IndexingDocument, list[Document]] | None = Nonetry:document = self._documents.get_indexing_document(ref)if document is None:@@ -126,13 +135,44 @@ class DocumentIndexingService:started = self._clock()self._backend.check_paused(ref)+ loading = (document, chunks)self._backend.load(document, chunks)self._backend.check_paused(ref)self._documents.complete_indexing(ref, tokens=total_tokens, latency=self._clock() - started)except DocumentIsPausedError:+ # The pause flag is checked before document existence, so a paused document+ # deleted mid-write surfaces here rather than as a deletion.+ self._discard_if_deleted(ref, loading)raiseexcept DocumentIsDeletedPausedError:logger.info("Document deleted during indexing: %s", ref)+ self._discard(ref, loading)except Exception as error:logger.exception("Document indexing failed: %s", ref)self._documents.fail_indexing(ref, self._backend.describe_error(error))+ self._discard_if_deleted(ref, loading)++ def _discard(self, ref: DocumentRef, loading: tuple[IndexingDocument, list[Document]] | None) -> None:+ """Best-effort removal of entries ``load`` may have written; failures never mask the outcome."""+ if loading is None:+ return+ try:+ self._backend.discard(*loading)+ except Exception:+ logger.exception("Failed to discard index entries of document deleted during indexing: %s", ref)++ def _discard_if_deleted(self, ref: DocumentRef, loading: tuple[IndexingDocument, list[Document]] | None) -> None:+ """Discard written entries when indexing stopped for another reason and the document is gone."""+ if loading is None:+ return+ try:+ self._documents.get_indexing_document(ref)+ except DocumentIsDeletedPausedError:+ logger.info("Document deleted during indexing: %s", ref)+ self._discard(ref, loading)+ except DocumentIsPausedError:+ return+ except Exception:+ logger.warning(+ "Could not check whether document still exists after indexing stopped: %s", ref, exc_info=True+ )diff --git a/api/tests/unit_tests/services/knowledge/indexing/adapters/test_execution.py b/api/tests/unit_tests/services/knowledge/indexing/adapters/test_execution.pyindex 40d54d800c4..a9b7358de25 100644--- a/api/tests/unit_tests/services/knowledge/indexing/adapters/test_execution.py+++ b/api/tests/unit_tests/services/knowledge/indexing/adapters/test_execution.py@@ -1,7 +1,7 @@import jsonimport threading-from collections.abc import Generator, Iterator-from contextlib import contextmanager+from collections.abc import Callable, Generator, Iterator, Mapping, Sequence+from contextlib import AbstractContextManager, contextmanagerfrom dataclasses import replacefrom datetime import datetimefrom unittest.mock import MagicMock, patch@@ -14,6 +14,7 @@ from werkzeug.exceptions import NotFoundfrom core.errors.error import LLMBadRequestError, ProviderTokenNotInitError, QuotaExceededErrorfrom core.plugin.impl.exc import PluginDaemonClientSideError+from core.rag.datasource.keyword.jieba.jieba import Jiebafrom core.rag.datasource.vdb.vector_factory import Vectorfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom core.rag.models.document import AttachmentDocument, ChildDocument@@ -25,6 +26,7 @@ from models.model import UploadFilefrom repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom repositories.knowledge.segment_repository import SQLAlchemySegmentRepositoryfrom repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository+from services.knowledge.indexing.adapters.cleanup import clean_document_indexesfrom services.knowledge.indexing.adapters.execution import IndexingExecutionAdapterfrom services.knowledge.indexing.errors import DocumentIsPausedErrorfrom services.knowledge.indexing.execution import DocumentIndexingService, IndexingDocument@@ -33,6 +35,7 @@ from tests.helpers.unit.config_override import apply_config_overridesfrom tests.unit_tests.repositories.knowledge.test_document_repository import _dataset, _documentMODULE = "services.knowledge.indexing.adapters.execution"+CLEANUP_MODULE = "services.knowledge.indexing.adapters.cleanup"BackendFixture = tuple[@@ -526,3 +529,187 @@ def test_partial_worker_failure_retries_only_incomplete_segments_and_keeps_totalassert row.indexing_status == IndexingStatus.COMPLETEDassert row.tokens == 30assert row.error is None+++def _delete_document_row(session_factory: sessionmaker[Session]) -> None:+ with session_factory.begin() as session:+ document = session.get(Document, "document-1")+ assert document is not None+ session.delete(document)+++def test_discard_removes_keywords_written_for_a_deleted_document(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session]+) -> None:+ adapter, _, segments, _, job, redis = backend+ table = {"existing": ["old-node"], "hello": ["old-node", "node"], "world": ["node"]}+ with sqlite_session_factory.begin() as session:+ session.add(+ DatasetKeywordTable(+ dataset_id="dataset-1",+ data_source_type="database",+ keyword_table=json.dumps({"__type__": "keyword_table", "__data__": {"table": table}}),+ )+ )+ _delete_document_row(sqlite_session_factory)++ with patch(f"{CLEANUP_MODULE}.redis_client", redis):+ adapter.discard(job, [IndexDocument(page_content="hello world", metadata={"doc_id": "node"})])++ assert redis.lock.call_args.args == ("keyword_indexing_lock_dataset-1",)+ _, encoded = segments.get_keyword_table(job.ref.dataset)+ assert encoded is not None+ assert json.loads(encoded)["__data__"]["table"] == {"existing": ["old-node"], "hello": ["old-node"]}++[email protected]("doc_form", [IndexStructureType.PARAGRAPH_INDEX, IndexStructureType.PARENT_CHILD_INDEX])+def test_discard_deletes_the_vectors_load_writes(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], doc_form: IndexStructureType+) -> None:+ adapter, documents, _, _, job, _ = backend+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ document = session.get(Document, "document-1")+ assert dataset is not None+ assert document is not None+ dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY+ document.doc_form = doc_form+ job = replace(documents.get_indexing_document(job.ref), processing_rule=job.processing_rule)+ _delete_document_row(sqlite_session_factory)+ child = ChildDocument(page_content="child", metadata={"doc_id": "child-node", "doc_hash": "child-hash"})+ chunk = IndexDocument(+ page_content="parent",+ metadata={"doc_id": "node", "doc_hash": "hash"},+ children=[child] if doc_form == IndexStructureType.PARENT_CHILD_INDEX else None,+ )++ with (+ patch(f"{MODULE}.Vector.resolve_vector_type", return_value="qdrant"),+ patch(f"{CLEANUP_MODULE}.Vector") as vector,+ ):+ adapter.discard(job, [chunk])++ expected = ["child-node"] if doc_form == IndexStructureType.PARENT_CHILD_INDEX else ["node"]+ assert vector.call_args.kwargs["vector_type"] == "qdrant"+ vector.return_value.delete_by_ids.assert_called_once_with(expected)+++def test_discard_leaves_index_cleanup_to_dataset_deletion(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session]+) -> None:+ adapter, _, _, _, job, redis = backend+ _delete_document_row(sqlite_session_factory)+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ assert dataset is not None+ session.delete(dataset)++ with patch(f"{CLEANUP_MODULE}.Vector") as vector, patch(f"{CLEANUP_MODULE}.redis_client", redis):+ adapter.discard(job, [IndexDocument(page_content="hello", metadata={"doc_id": "node"})])++ vector.assert_not_called()+ redis.lock.assert_not_called()++[email protected]("paused_first", [False, True])+def test_document_deleted_during_its_keyword_write_leaves_no_orphan_keywords(+ backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], paused_first: bool+) -> None:+ adapter, documents, segments, _, job, redis = backend+ chunk = IndexDocument(page_content="hello world", metadata={"doc_id": "node", "doc_hash": "hash"})+ segments.save_for_indexing(job, [chunk], [5])+ service = DocumentIndexingService(documents=documents, segments=segments, backend=adapter)+ write_keywords = Jieba.update_texts++ def cleanup_wins_the_race(+ self: Jieba,+ texts: Sequence[IndexDocument],+ *,+ read: Callable[[], tuple[str, str | None]],+ write: Callable[[str, str, Mapping[str, Sequence[str]]], None],+ lock: AbstractContextManager[object],+ keywords_list: Sequence[Sequence[str] | None] | None = None,+ replace_existing: bool = False,+ deleted_ids: Sequence[str] = (),+ ) -> None:+ if texts:+ # The user deletes the document after the write passed its existence check;+ # document cleanup snapshots segment node ids before this write lands.+ if paused_first:+ redis.get.return_value = "True" # the pause flag outlives the deleted document+ _delete_document_row(sqlite_session_factory)+ clean_document_indexes(+ dataset_id="dataset-1",+ document_ids=["document-1"],+ doc_form=job.doc_form,+ new_session=sqlite_session_factory,+ )+ write_keywords(+ self,+ texts,+ read=read,+ write=write,+ lock=lock,+ keywords_list=keywords_list,+ replace_existing=replace_existing,+ deleted_ids=deleted_ids,+ )++ with (+ Flask(__name__).app_context(),+ patch.object(Jieba, "update_texts", cleanup_wins_the_race),+ patch(f"{CLEANUP_MODULE}.redis_client", redis),+ ):+ if paused_first:+ with pytest.raises(DocumentIsPausedError):+ service.run_in_indexing_status(job.ref)+ else:+ service.run_in_indexing_status(job.ref)++ _, encoded = segments.get_keyword_table(job.ref.dataset)+ assert encoded is not None+ assert all("node" not in node_ids for node_ids in json.loads(encoded)["__data__"]["table"].values())+++ ("indexing_technique", "chunk"),+ [+ # parent-child chunks without children wrote no vectors+ (IndexTechniqueType.HIGH_QUALITY, IndexDocument(page_content="parent", metadata={"doc_id": "node"})),+ # economy parent-child datasets never write keywords+ (+ IndexTechniqueType.ECONOMY,+ IndexDocument(+ page_content="parent",+ metadata={"doc_id": "node"},+ children=[ChildDocument(page_content="child", metadata={"doc_id": "child-node"})],+ ),+ ),+ ],+)+def test_discard_touches_no_index_when_load_wrote_nothing(+ backend: BackendFixture,+ sqlite_session_factory: sessionmaker[Session],+ indexing_technique: IndexTechniqueType,+ chunk: IndexDocument,+) -> None:+ adapter, documents, _, _, job, redis = backend+ with sqlite_session_factory.begin() as session:+ dataset = session.get(Dataset, "dataset-1")+ document = session.get(Document, "document-1")+ assert dataset is not None+ assert document is not None+ dataset.indexing_technique = indexing_technique+ document.doc_form = IndexStructureType.PARENT_CHILD_INDEX+ job = replace(documents.get_indexing_document(job.ref), processing_rule=job.processing_rule)+ _delete_document_row(sqlite_session_factory)++ with (+ patch(f"{MODULE}.Vector.resolve_vector_type", return_value="qdrant"),+ patch(f"{CLEANUP_MODULE}.Vector") as vector,+ patch(f"{CLEANUP_MODULE}.redis_client", redis),+ ):+ adapter.discard(job, [chunk])++ vector.assert_not_called()+ redis.lock.assert_not_called()diff --git a/api/tests/unit_tests/services/knowledge/indexing/test_execution.py b/api/tests/unit_tests/services/knowledge/indexing/test_execution.pyindex 630e109a33b..11b66811910 100644--- a/api/tests/unit_tests/services/knowledge/indexing/test_execution.py+++ b/api/tests/unit_tests/services/knowledge/indexing/test_execution.py@@ -126,3 +126,99 @@ def test_indexing_persists_the_backend_error_description(execution: ExecutionFixports.backend.describe_error.assert_called_once_with(error)ports.documents.fail_indexing.assert_called_once_with(document.ref, "source file missing")ports.documents.complete_indexing.assert_not_called()++[email protected]("start", ["parsing", "indexing"])+def test_deletion_after_index_write_discards_the_written_entries(execution: ExecutionFixture, start: str) -> None:+ service, ports, document, chunks = execution+ ports.documents.complete_indexing.side_effect = DocumentIsDeletedPausedError()+ if start == "parsing":+ service.run([document.ref])+ else:+ service.run_in_indexing_status(document.ref)+ loaded = chunks if start == "parsing" else chunks[1:]+ calls = [call[0] for call in ports.mock_calls]+ assert calls.index("backend.load") < calls.index("backend.discard")+ ports.backend.discard.assert_called_once_with(document, loaded)+ ports.documents.fail_indexing.assert_not_called()+++def test_deletion_detected_inside_load_discards_the_chunks_handed_to_load(execution: ExecutionFixture) -> None:+ service, ports, document, chunks = execution+ ports.backend.load.side_effect = DocumentIsDeletedPausedError()+ service.run([document.ref])+ ports.backend.discard.assert_called_once_with(document, chunks)+ ports.documents.complete_indexing.assert_not_called()++[email protected]("stage", ["get_indexing_document", "mark_splitting"])+def test_deletion_before_index_write_has_nothing_to_discard(execution: ExecutionFixture, stage: str) -> None:+ service, ports, document, _ = execution+ operation = {+ "get_indexing_document": ports.documents.get_indexing_document,+ "mark_splitting": ports.documents.mark_splitting,+ }[stage]+ operation.side_effect = DocumentIsDeletedPausedError()+ service.run([document.ref])+ ports.backend.load.assert_not_called()+ ports.backend.discard.assert_not_called()+++def test_discard_failure_is_logged_without_failing_the_deleted_document(+ execution: ExecutionFixture, caplog: pytest.LogCaptureFixture+) -> None:+ service, ports, document, _ = execution+ ports.documents.complete_indexing.side_effect = DocumentIsDeletedPausedError()+ ports.backend.discard.side_effect = RuntimeError("vector store unavailable")+ service.run([document.ref])+ ports.documents.fail_indexing.assert_not_called()+ assert "Failed to discard index entries of document deleted during indexing" in caplog.text+++ ("existence_check", "discarded"), [(DocumentIsDeletedPausedError, True), (DocumentIsPausedError, False)]+)+def test_pause_after_index_write_discards_only_when_the_document_was_deleted(+ execution: ExecutionFixture, existence_check: type[Exception], discarded: bool+) -> None:+ service, ports, document, chunks = execution+ # The pause flag survives deletion and is checked before document existence.+ ports.backend.check_paused.side_effect = [None, None, DocumentIsPausedError()]+ ports.documents.get_indexing_document.side_effect = [document, existence_check()]+ with pytest.raises(DocumentIsPausedError):+ service.run([document.ref])+ if discarded:+ ports.backend.discard.assert_called_once_with(document, chunks)+ else:+ ports.backend.discard.assert_not_called()+ ports.documents.complete_indexing.assert_not_called()++[email protected]("document_deleted", [True, False])+def test_failed_index_write_discards_entries_only_for_a_deleted_document(+ execution: ExecutionFixture, document_deleted: bool+) -> None:+ service, ports, document, chunks = execution+ ports.backend.load.side_effect = RuntimeError("vector store unavailable")+ ports.documents.get_indexing_document.side_effect = [+ document,+ DocumentIsDeletedPausedError() if document_deleted else document,+ ]+ service.run([document.ref])+ ports.documents.fail_indexing.assert_called_once_with(document.ref, "vector store unavailable")+ if document_deleted:+ ports.backend.discard.assert_called_once_with(document, chunks)+ else:+ ports.backend.discard.assert_not_called()+++def test_unverifiable_existence_after_a_pause_keeps_entries_and_the_pause(+ execution: ExecutionFixture, caplog: pytest.LogCaptureFixture+) -> None:+ service, ports, document, _ = execution+ ports.backend.check_paused.side_effect = [None, None, DocumentIsPausedError()]+ ports.documents.get_indexing_document.side_effect = [document, RuntimeError("database unavailable")]+ with pytest.raises(DocumentIsPausedError):+ service.run([document.ref])+ ports.backend.discard.assert_not_called()+ assert "Could not check whether document still exists after indexing stopped" in caplog.textdiff --git a/api/tests/unit_tests/services/knowledge/test_dataset_service.py b/api/tests/unit_tests/services/knowledge/test_dataset_service.pyindex b1c5ff0c181..1dd2b98d504 100644--- a/api/tests/unit_tests/services/knowledge/test_dataset_service.py+++ b/api/tests/unit_tests/services/knowledge/test_dataset_service.py@@ -1969,6 +1969,31 @@ class TestDatasetServiceDocumentCases:event.remove(sqlite_session, "after_commit", count_commit)assert commits == 0+ def test_delete_document_dispatches_cleanup_after_the_deletion_commits(self, sqlite_session: Session) -> None:+ dataset = _dataset_row()+ document = _document_row(document_id="doc-1", data_source_info=json.dumps({"upload_file_id": "file-1"}))+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()+ visible_when_dispatched: list[bool] = []++ def record_visibility(*_args: object) -> None:+ # Cleanup and in-flight indexing run in other sessions; they must not see the row.+ with Session(sqlite_session.get_bind()) as other_session:+ visible_when_dispatched.append(+ other_session.get(_dataset_service_document_Document, "doc-1") is not None+ )++ with _dataset_service_document_patch(+ "events.event_handlers.clean_when_document_deleted.clean_document_task"+ ) as clean_task:+ clean_task.delay.side_effect = record_visibility+ _dataset_service_document_DocumentService.delete_document(document, sqlite_session)++ clean_task.delay.assert_called_once_with(+ "doc-1", dataset.id, _dataset_service_document_IndexStructureType.PARAGRAPH_INDEX, "file-1"+ )+ assert visible_when_dispatched == [False]+def test_rename_document_raises_when_dataset_is_missing(self, sqlite_session: Session):with _dataset_service_document_patch("services.knowledge.dataset_service.current_user", _dataset_service_document_account()