Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
import json
import threading
from collections.abc import Callable, Generator, Iterator, Mapping, Sequence
from contextlib import AbstractContextManager, contextmanager
from dataclasses import replace
from datetime import datetime
from unittest.mock import MagicMock, patch
import pytest
from flask import Flask
from sqlalchemy import Connection, Engine, event, inspect, select
from sqlalchemy.orm import Session, sessionmaker
from werkzeug.exceptions import NotFound
from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError, QuotaExceededError
from core.plugin.impl.exc import PluginDaemonClientSideError
from core.rag.datasource.keyword.jieba.jieba import Jieba
from core.rag.datasource.vdb.vector_factory import Vector
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from core.rag.models.document import AttachmentDocument, ChildDocument
from core.rag.models.document import Document as IndexDocument
from extensions.storage.storage_type import StorageType
from models.dataset import Dataset, DatasetKeywordTable, DatasetProcessRule, Document, DocumentSegment
from models.enums import CreatorUserRole, IndexingStatus, ProcessRuleMode, SegmentStatus
from models.model import UploadFile
from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository
from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository
from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository
from services.knowledge.indexing.adapters.cleanup import clean_document_indexes
from services.knowledge.indexing.adapters.execution import IndexingExecutionAdapter
from services.knowledge.indexing.errors import DocumentIsPausedError
from services.knowledge.indexing.execution import DocumentIndexingService, IndexingDocument
from services.knowledge.resource_scope import DatasetRef
from tests.helpers.unit.config_override import apply_config_overrides
from tests.unit_tests.repositories.knowledge.test_document_repository import _dataset, _document
MODULE = "services.knowledge.indexing.adapters.execution"
CLEANUP_MODULE = "services.knowledge.indexing.adapters.cleanup"
BackendFixture = tuple[
IndexingExecutionAdapter,
SQLAlchemyDocumentRepository,
SQLAlchemySegmentRepository,
MagicMock,
IndexingDocument,
MagicMock,
]
@pytest.fixture
def backend(sqlite_session_factory: sessionmaker[Session]) -> Iterator[BackendFixture]:
with sqlite_session_factory.begin() as session:
session.add_all(
[_dataset("dataset-1", "workspace-1"), _document("document-1", status=IndexingStatus.SPLITTING)]
)
documents = SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory)
segments = SQLAlchemySegmentRepository(session_factory=sqlite_session_factory)
sources = MagicMock()
adapter = IndexingExecutionAdapter(
session_factory=sqlite_session_factory,
documents=documents,
segments=segments,
uploads=SQLAlchemyKnowledgeUploadRepository(session_factory=sqlite_session_factory),
sources=sources,
)
job = replace(
documents.get_indexing_document(DatasetRef("workspace-1", "dataset-1").document("document-1")),
processing_rule={"mode": "automatic", "rules": {}},
)
with patch(f"{MODULE}.redis_client") as redis:
redis.get.return_value = None
yield adapter, documents, segments, sources, job, redis
@pytest.fixture
def active_transactions(sqlite_engine: Engine) -> Iterator[set[Connection]]:
active: set[Connection] = set()
begin, finish = active.add, active.discard
event.listen(sqlite_engine, "begin", begin)
event.listen(sqlite_engine, "commit", finish)
event.listen(sqlite_engine, "rollback", finish)
try:
yield active
finally:
event.remove(sqlite_engine, "begin", begin)
event.remove(sqlite_engine, "commit", finish)
event.remove(sqlite_engine, "rollback", finish)
@pytest.mark.parametrize("doc_form", list(IndexStructureType))
def test_vector_io_releases_transactions_and_preserves_chunk_structure(
backend: BackendFixture,
sqlite_session_factory: sessionmaker[Session],
active_transactions: set[Connection],
monkeypatch: pytest.MonkeyPatch,
doc_form: IndexStructureType,
) -> None:
adapter, _, segments, _, job, _ = backend
apply_config_overrides(monkeypatch, VECTOR_STORE="qdrant", VECTOR_STORE_WHITELIST_ENABLE=True)
files = [
UploadFile(
tenant_id=tenant_id,
storage_type=StorageType.LOCAL,
key=f"image-{tenant_id}",
name="image.png",
size=1,
extension="png",
mime_type="image/png",
created_by_role=CreatorUserRole.ACCOUNT,
created_by="account-1",
created_at=datetime(2026, 1, 1),
used=True,
)
for tenant_id in ("workspace-1", "workspace-2")
]
with sqlite_session_factory.begin() as session:
dataset = session.get(Dataset, "dataset-1")
document = session.get(Document, "document-1")
assert dataset is not None
assert document is not None
dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
dataset.is_multimodal = True
document.doc_form = doc_form
session.add_all(files)
child = ChildDocument(page_content="child", metadata={"doc_id": "child-node", "doc_hash": "child-hash"})
chunk = IndexDocument(
page_content="parent",
metadata={"doc_id": "node", "doc_hash": "hash"},
children=[child] if doc_form == IndexStructureType.PARENT_CHILD_INDEX else None,
attachments=[
AttachmentDocument(page_content="image", metadata={"doc_id": file.id, "doc_type": "image"})
for file in files
],
)
segments.save_for_indexing(job, [chunk], [5])
vector_backend = MagicMock()
vector_factory = MagicMock()
vector_factory.return_value.init_vector.return_value = vector_backend
def external_call(*_args, **_kwargs) -> list[list[float]]:
assert not active_transactions, "Index I/O must not hold a database transaction"
return [[1.0]]
def get_vector_factory(vector_type: str) -> MagicMock:
assert not active_transactions, "Vector initialization must follow the whitelist read transaction"
assert vector_type == "qdrant"
return vector_factory
def load_image(key: str) -> bytes:
assert not active_transactions
assert key == files[0].key
return b"image"
vector_backend.create.side_effect = external_call
with (
patch.object(Vector, "get_vector_factory", side_effect=get_vector_factory),
patch(
"core.rag.datasource.vdb.vector_factory._LazyEmbeddings.embed_documents", side_effect=external_call
) as embed,
patch(
"core.rag.datasource.vdb.vector_factory._LazyEmbeddings.embed_multimodal_documents",
side_effect=external_call,
) as embed_images,
patch("core.rag.datasource.vdb.vector_factory.storage") as storage,
):
storage.load_once.side_effect = load_image
adapter._process_chunk(Flask(__name__), job.ref, [chunk], False)
embed.assert_called_once_with(["child" if doc_form == IndexStructureType.PARENT_CHILD_INDEX else "parent"])
assert embed_images.call_args.args[0] == [{"content": "aW1hZ2U=", "content_type": "image", "file_id": files[0].id}]
storage.load_once.assert_called_once_with(files[0].key)
assert vector_backend.create.call_count == 2
indexed_text = vector_backend.create.call_args_list[0].kwargs["texts"][0]
assert indexed_text.metadata["doc_id"] == (
"child-node" if doc_form == IndexStructureType.PARENT_CHILD_INDEX else "node"
)
assert segments.resume_indexing(job) == ([], 5)
@pytest.mark.parametrize("storage_type", ["database", "file"])
def test_keyword_io_releases_transactions_and_commits_before_unlocking(
backend: BackendFixture,
sqlite_session_factory: sessionmaker[Session],
active_transactions: set[Connection],
storage_type: str,
) -> None:
adapter, _, segments, _, job, redis = backend
original = json.dumps({"__type__": "keyword_table", "__data__": {"table": {"existing": ["old-node"]}}})
with sqlite_session_factory.begin() as session:
session.add(DatasetKeywordTable(dataset_id="dataset-1", data_source_type=storage_type, keyword_table=original))
chunk = IndexDocument(page_content="hello", metadata={"doc_id": "node", "doc_hash": "hash"})
segments.save_for_indexing(job, [chunk], [5])
locked = False
@contextmanager
def dataset_lock() -> Generator[None]:
nonlocal locked
locked = True
yield
assert not active_transactions
# A separate session must see the write before the next lock holder can read.
with sqlite_session_factory() as session:
segment = session.scalar(select(DocumentSegment))
assert segment is not None
assert segment.keywords == ["hello"]
locked = False
redis.lock.side_effect = lambda *_args, **_kwargs: dataset_lock()
def external_call(*_args, **_kwargs) -> None:
assert locked
assert not active_transactions, "Keyword I/O must not hold a database transaction"
def load_keywords(*_args) -> bytes:
external_call()
return original.encode()
def file_exists(*_args) -> bool:
external_call()
return True
def extract_keywords(*_args) -> set[str]:
external_call()
return {"hello"}
with (
patch("core.rag.datasource.keyword.jieba.jieba.redis_client", redis),
patch("core.rag.datasource.keyword.jieba.jieba.storage") as storage,
patch("core.rag.datasource.keyword.jieba.jieba.JiebaKeywordTableHandler") as handler,
):
storage.load_once.side_effect = load_keywords
storage.exists.side_effect = file_exists
storage.delete.side_effect = external_call
storage.save.side_effect = external_call
handler.return_value.extract_keywords.side_effect = extract_keywords
adapter._process_chunk(Flask(__name__), job.ref, [chunk], True)
assert not locked
assert redis.lock.call_args.args == ("keyword_indexing_lock_dataset-1",)
_, encoded = segments.get_keyword_table(job.ref.dataset)
if storage_type == "file":
storage.save.assert_called_once()
encoded = storage.save.call_args.args[1]
else:
storage.save.assert_not_called()
assert encoded is not None
assert json.loads(encoded)["__data__"]["table"] == {"existing": ["old-node"], "hello": ["node"]}
assert segments.resume_indexing(job) == ([], 5)
def test_extract_attaches_document_metadata_and_owns_local_session(backend: BackendFixture) -> None:
adapter, _, _, sources, job, _ = backend
text = IndexDocument(page_content="source")
with patch(f"{MODULE}.IndexProcessorFactory") as factory:
processor = factory.return_value.init_index_processor.return_value
def extract(setting: object, *, session: Session, process_rule_mode: str) -> list[IndexDocument]:
assert setting is sources.resolve_for_indexing.return_value
assert not session.in_transaction()
assert process_rule_mode == "automatic"
return [text]
processor.extract.side_effect = extract
assert adapter.extract(job) == [text]
sources.resolve_for_indexing.assert_called_once_with(job.source)
assert text.metadata == {"dataset_id": "dataset-1", "document_id": "document-1"}
@pytest.mark.parametrize(
("error", "message"),
[
(ProviderTokenNotInitError(), "Provider Token Not Init"),
(ProviderTokenNotInitError("configure an embedding provider"), "configure an embedding provider"),
(LLMBadRequestError("invalid model input"), "invalid model input"),
(QuotaExceededError(), "Quota Exceeded"),
(PluginDaemonClientSideError("plugin unavailable"), "plugin unavailable"),
(NotFound("source file missing"), "source file missing"),
(RuntimeError("index unavailable"), "index unavailable"),
],
)
def test_indexing_preserves_provider_and_source_error_descriptions(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], error: Exception, message: str
) -> None:
adapter, documents, segments, sources, job, _ = backend
sources.resolve_for_indexing.side_effect = error
with sqlite_session_factory.begin() as session:
rule = DatasetProcessRule(
dataset_id=job.ref.dataset.dataset_id, mode=ProcessRuleMode.AUTOMATIC, rules="{}", created_by="account-1"
)
session.add(rule)
session.flush()
document = session.get(Document, job.ref.document_id)
assert document is not None
document.dataset_process_rule_id = rule.id
DocumentIndexingService(documents=documents, segments=segments, backend=adapter).run([job.ref])
sources.resolve_for_indexing.assert_called_once()
with sqlite_session_factory() as session:
document = session.get(Document, job.ref.document_id)
assert document is not None
assert document.indexing_status == IndexingStatus.ERROR
assert document.error == message
assert document.completed_at is None
@pytest.mark.parametrize("provider", [None, "embedding-provider"])
def test_transform_selects_embedding_model_and_preserves_processor_options(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], provider: str | None
) -> None:
adapter, documents, _, _, job, _ = backend
with sqlite_session_factory.begin() as session:
dataset = session.get(Dataset, "dataset-1")
assert dataset is not None
dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
dataset.embedding_model_provider = provider
dataset.embedding_model = "embedding-model"
with (
patch.object(documents, "get_indexing_user", return_value=MagicMock()) as user,
patch(f"{MODULE}.ModelManager.for_tenant") as manager,
patch(f"{MODULE}.IndexProcessorFactory") as factory,
):
processor = factory.return_value.init_index_processor.return_value
processor.transform.return_value = list[IndexDocument]()
adapter.transform(job, [])
selected = (
manager.return_value.get_model_instance if provider else manager.return_value.get_default_model_instance
)
selected.assert_called_once()
assert processor.transform.call_args.kwargs["embedding_model_instance"] is selected.return_value
assert processor.transform.call_args.kwargs["process_rule"] == job.processing_rule
assert processor.transform.call_args.args == ([], user.return_value)
def test_hash_groups_load_worker_owned_inputs_before_completion(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session]
) -> None:
adapter, _, segments, _, job, _ = backend
with sqlite_session_factory.begin() as session:
persisted_dataset = session.get(Dataset, "dataset-1")
assert persisted_dataset is not None
persisted_dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
chunks = [
IndexDocument(page_content=content, metadata={"doc_id": f"node-{i}", "doc_hash": f"hash-{i}"})
for i, content in enumerate(["same", "other", "same"])
]
segments.save_for_indexing(job, chunks, [3, 5, 7])
calls: list[tuple[Dataset, list[str]]] = []
main_thread = threading.get_ident()
def vector(dataset: Dataset, *, session: Session | None, vector_type: str) -> MagicMock:
assert threading.get_ident() != main_thread
assert inspect(dataset).detached
assert session is None
assert vector_type == "qdrant"
def create(group: list[IndexDocument]) -> None:
with sqlite_session_factory() as reader:
persisted = reader.scalars(select(DocumentSegment)).all()
assert len(persisted) == 3
assert all(
row.status == SegmentStatus.INDEXING
for row in persisted
if row.index_node_id in {chunk.metadata["doc_id"] for chunk in group}
)
calls.append((dataset, [chunk.page_content for chunk in group]))
backend = MagicMock()
backend.create.side_effect = create
return backend
with Flask(__name__).app_context(), patch(f"{MODULE}.Vector", side_effect=vector) as factory:
factory.resolve_vector_type.return_value = "qdrant"
adapter.load(job, chunks)
assert sum(group.count("same") for _, group in calls) == 2
assert sum("same" in group for _, group in calls) == 1
assert len({id(dataset) for dataset, _ in calls}) == len(calls)
assert segments.resume_indexing(job) == ([], 15)
@pytest.mark.parametrize("technique", ["economy", "high_quality"])
def test_worker_failure_marks_document_error_and_keeps_retryable_segments(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], technique: str
) -> None:
adapter, documents, segments, _, job, _ = backend
with sqlite_session_factory.begin() as session:
persisted_dataset = session.get(Dataset, "dataset-1")
assert persisted_dataset is not None
persisted_dataset.indexing_technique = IndexTechniqueType(technique)
chunks = [IndexDocument(page_content="text", metadata={"doc_id": "node", "doc_hash": "hash"})]
segments.save_for_indexing(job, chunks, [9])
with (
Flask(__name__).app_context(),
patch(f"{MODULE}.Jieba") as keyword,
patch(f"{MODULE}.Vector") as vector,
):
keyword.return_value.update_texts.side_effect = RuntimeError("worker failed")
vector.return_value.create.side_effect = RuntimeError("worker failed")
DocumentIndexingService(documents=documents, segments=segments, backend=adapter).run_in_indexing_status(job.ref)
with sqlite_session_factory() as session:
row = session.get(Document, job.ref.document_id)
assert row is not None
assert row.indexing_status == IndexingStatus.ERROR
assert row.error == "worker failed"
assert len(segments.resume_indexing(job)[0]) == 1
def test_worker_pause_does_not_mark_segments_complete(backend: BackendFixture) -> None:
adapter, _, segments, _, job, redis = backend
chunks = [IndexDocument(page_content="text", metadata={"doc_id": "node", "doc_hash": "hash"})]
segments.save_for_indexing(job, chunks, [9])
redis.get.return_value = b"paused"
with Flask(__name__).app_context(), pytest.raises(DocumentIsPausedError):
adapter.load(job, chunks)
assert len(segments.resume_indexing(job)[0]) == 1
@pytest.mark.parametrize("technique", ["economy", "high_quality"])
@pytest.mark.parametrize("doc_form", ["text_model", "qa_model", "hierarchical_model"])
def test_full_indexing_commits_phases_through_real_repositories(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], technique: str, doc_form: str
) -> None:
adapter, documents, segments, _, job, _ = backend
with sqlite_session_factory.begin() as session:
dataset = session.get(Dataset, "dataset-1")
assert dataset is not None
dataset.indexing_technique = IndexTechniqueType(technique)
row = session.get(Document, "document-1")
assert row is not None
row.doc_form = IndexStructureType(doc_form)
rule = DatasetProcessRule(
dataset_id="dataset-1", mode=ProcessRuleMode.AUTOMATIC, rules="{}", created_by="account-1"
)
rule.id = "rule-1"
row.dataset_process_rule_id = rule.id
session.add(rule)
chunk = IndexDocument(page_content="text", metadata={"doc_id": "node", "doc_hash": "hash"})
with (
Flask(__name__).app_context(),
patch.object(documents, "get_indexing_user", return_value=MagicMock()),
patch(f"{MODULE}.IndexProcessorFactory") as factory,
patch(f"{MODULE}.ModelManager.for_tenant"),
patch(f"{MODULE}.calculate_segment_token_counts", return_value=[19]),
patch(f"{MODULE}.Jieba"),
patch(f"{MODULE}.Vector"),
):
processor = factory.return_value.init_index_processor.return_value
processor.extract.return_value = [chunk]
processor.transform.return_value = [chunk]
DocumentIndexingService(documents=documents, segments=segments, backend=adapter).run([job.ref])
with sqlite_session_factory() as session:
row = session.get(Document, "document-1")
assert row is not None
assert row.indexing_status == IndexingStatus.COMPLETED
assert row.tokens == 19
assert row.parsing_completed_at is not None
assert row.splitting_completed_at is not None
assert row.completed_at is not None
@pytest.mark.parametrize("change", ["paused", "deleted"])
def test_document_changed_during_worker_io_cannot_be_completed(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], change: str
) -> None:
adapter, documents, segments, _, job, _ = backend
chunks = [IndexDocument(page_content="text", metadata={"doc_id": "node", "doc_hash": "hash"})]
segments.save_for_indexing(job, chunks, [9])
def change_document(*_args, **_kwargs) -> None:
with sqlite_session_factory.begin() as writer:
row = writer.get(Document, "document-1")
assert row is not None
if change == "paused":
row.is_paused = True
else:
writer.delete(row)
service = DocumentIndexingService(documents=documents, segments=segments, backend=adapter)
with Flask(__name__).app_context(), patch(f"{MODULE}.Jieba") as keyword:
keyword.return_value.update_texts.side_effect = change_document
if change == "paused":
with pytest.raises(DocumentIsPausedError):
service.run_in_indexing_status(job.ref)
else:
service.run_in_indexing_status(job.ref)
with sqlite_session_factory() as session:
segment = session.scalar(select(DocumentSegment))
assert segment is not None
assert segment.status == SegmentStatus.INDEXING
def test_partial_worker_failure_retries_only_incomplete_segments_and_keeps_total(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session]
) -> None:
adapter, documents, segments, _, job, _ = backend
with sqlite_session_factory.begin() as session:
persisted_dataset = session.get(Dataset, "dataset-1")
assert persisted_dataset is not None
persisted_dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
chunks = [
IndexDocument(page_content=content, metadata={"doc_id": content, "doc_hash": content})
for content in ["good", "bad"]
]
segments.save_for_indexing(job, chunks, [13, 17])
service = DocumentIndexingService(documents=documents, segments=segments, backend=adapter)
def load(group: list[IndexDocument]) -> None:
if group[0].page_content == "bad":
raise RuntimeError("one shard failed")
with (
Flask(__name__).app_context(),
patch(f"{MODULE}.Vector") as factory,
patch(f"{MODULE}.helper.generate_text_hash", side_effect=lambda content: "0" if content == "good" else "1"),
):
vector = factory.return_value
vector.create.side_effect = load
service.run_in_indexing_status(job.ref)
pending, tokens = segments.resume_indexing(job)
assert [chunk.page_content for chunk in pending] == ["bad"]
assert tokens == 30
vector.create.reset_mock()
vector.create.side_effect = None
service.run_in_indexing_status(job.ref)
vector.create.assert_called_once()
assert [chunk.page_content for chunk in vector.create.call_args.args[0]] == ["bad"]
with sqlite_session_factory() as session:
row = session.get(Document, "document-1")
assert row is not None
assert row.indexing_status == IndexingStatus.COMPLETED
assert row.tokens == 30
assert row.error is None
def _delete_document_row(session_factory: sessionmaker[Session]) -> None:
with session_factory.begin() as session:
document = session.get(Document, "document-1")
assert document is not None
session.delete(document)
def test_discard_removes_keywords_written_for_a_deleted_document(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session]
) -> None:
adapter, _, segments, _, job, redis = backend
table = {"existing": ["old-node"], "hello": ["old-node", "node"], "world": ["node"]}
with sqlite_session_factory.begin() as session:
session.add(
DatasetKeywordTable(
dataset_id="dataset-1",
data_source_type="database",
keyword_table=json.dumps({"__type__": "keyword_table", "__data__": {"table": table}}),
)
)
_delete_document_row(sqlite_session_factory)
with patch(f"{CLEANUP_MODULE}.redis_client", redis):
adapter.discard(job, [IndexDocument(page_content="hello world", metadata={"doc_id": "node"})])
assert redis.lock.call_args.args == ("keyword_indexing_lock_dataset-1",)
_, encoded = segments.get_keyword_table(job.ref.dataset)
assert encoded is not None
assert json.loads(encoded)["__data__"]["table"] == {"existing": ["old-node"], "hello": ["old-node"]}
@pytest.mark.parametrize("doc_form", [IndexStructureType.PARAGRAPH_INDEX, IndexStructureType.PARENT_CHILD_INDEX])
def test_discard_deletes_the_vectors_load_writes(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], doc_form: IndexStructureType
) -> None:
adapter, documents, _, _, job, _ = backend
with sqlite_session_factory.begin() as session:
dataset = session.get(Dataset, "dataset-1")
document = session.get(Document, "document-1")
assert dataset is not None
assert document is not None
dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
document.doc_form = doc_form
job = replace(documents.get_indexing_document(job.ref), processing_rule=job.processing_rule)
_delete_document_row(sqlite_session_factory)
child = ChildDocument(page_content="child", metadata={"doc_id": "child-node", "doc_hash": "child-hash"})
chunk = IndexDocument(
page_content="parent",
metadata={"doc_id": "node", "doc_hash": "hash"},
children=[child] if doc_form == IndexStructureType.PARENT_CHILD_INDEX else None,
)
with (
patch(f"{MODULE}.Vector.resolve_vector_type", return_value="qdrant"),
patch(f"{CLEANUP_MODULE}.Vector") as vector,
):
adapter.discard(job, [chunk])
expected = ["child-node"] if doc_form == IndexStructureType.PARENT_CHILD_INDEX else ["node"]
assert vector.call_args.kwargs["vector_type"] == "qdrant"
vector.return_value.delete_by_ids.assert_called_once_with(expected)
def test_discard_leaves_index_cleanup_to_dataset_deletion(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session]
) -> None:
adapter, _, _, _, job, redis = backend
_delete_document_row(sqlite_session_factory)
with sqlite_session_factory.begin() as session:
dataset = session.get(Dataset, "dataset-1")
assert dataset is not None
session.delete(dataset)
with patch(f"{CLEANUP_MODULE}.Vector") as vector, patch(f"{CLEANUP_MODULE}.redis_client", redis):
adapter.discard(job, [IndexDocument(page_content="hello", metadata={"doc_id": "node"})])
vector.assert_not_called()
redis.lock.assert_not_called()
@pytest.mark.parametrize("paused_first", [False, True])
def test_document_deleted_during_its_keyword_write_leaves_no_orphan_keywords(
backend: BackendFixture, sqlite_session_factory: sessionmaker[Session], paused_first: bool
) -> None:
adapter, documents, segments, _, job, redis = backend
chunk = IndexDocument(page_content="hello world", metadata={"doc_id": "node", "doc_hash": "hash"})
segments.save_for_indexing(job, [chunk], [5])
service = DocumentIndexingService(documents=documents, segments=segments, backend=adapter)
write_keywords = Jieba.update_texts
def cleanup_wins_the_race(
self: Jieba,
texts: Sequence[IndexDocument],
*,
read: Callable[[], tuple[str, str | None]],
write: Callable[[str, str, Mapping[str, Sequence[str]]], None],
lock: AbstractContextManager[object],
keywords_list: Sequence[Sequence[str] | None] | None = None,
replace_existing: bool = False,
deleted_ids: Sequence[str] = (),
) -> None:
if texts:
# The user deletes the document after the write passed its existence check;
# document cleanup snapshots segment node ids before this write lands.
if paused_first:
redis.get.return_value = "True" # the pause flag outlives the deleted document
_delete_document_row(sqlite_session_factory)
clean_document_indexes(
dataset_id="dataset-1",
document_ids=["document-1"],
doc_form=job.doc_form,
new_session=sqlite_session_factory,
)
write_keywords(
self,
texts,
read=read,
write=write,
lock=lock,
keywords_list=keywords_list,
replace_existing=replace_existing,
deleted_ids=deleted_ids,
)
with (
Flask(__name__).app_context(),
patch.object(Jieba, "update_texts", cleanup_wins_the_race),
patch(f"{CLEANUP_MODULE}.redis_client", redis),
):
if paused_first:
with pytest.raises(DocumentIsPausedError):
service.run_in_indexing_status(job.ref)
else:
service.run_in_indexing_status(job.ref)
_, encoded = segments.get_keyword_table(job.ref.dataset)
assert encoded is not None
assert all("node" not in node_ids for node_ids in json.loads(encoded)["__data__"]["table"].values())
@pytest.mark.parametrize(
("indexing_technique", "chunk"),
[
# parent-child chunks without children wrote no vectors
(IndexTechniqueType.HIGH_QUALITY, IndexDocument(page_content="parent", metadata={"doc_id": "node"})),
# economy parent-child datasets never write keywords
(
IndexTechniqueType.ECONOMY,
IndexDocument(
page_content="parent",
metadata={"doc_id": "node"},
children=[ChildDocument(page_content="child", metadata={"doc_id": "child-node"})],
),
),
],
)
def test_discard_touches_no_index_when_load_wrote_nothing(
backend: BackendFixture,
sqlite_session_factory: sessionmaker[Session],
indexing_technique: IndexTechniqueType,
chunk: IndexDocument,
) -> None:
adapter, documents, _, _, job, redis = backend
with sqlite_session_factory.begin() as session:
dataset = session.get(Dataset, "dataset-1")
document = session.get(Document, "document-1")
assert dataset is not None
assert document is not None
dataset.indexing_technique = indexing_technique
document.doc_form = IndexStructureType.PARENT_CHILD_INDEX
job = replace(documents.get_indexing_document(job.ref), processing_rule=job.processing_rule)
_delete_document_row(sqlite_session_factory)
with (
patch(f"{MODULE}.Vector.resolve_vector_type", return_value="qdrant"),
patch(f"{CLEANUP_MODULE}.Vector") as vector,
patch(f"{CLEANUP_MODULE}.redis_client", redis),
):
adapter.discard(job, [chunk])
vector.assert_not_called()
redis.lock.assert_not_called()