Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: migrate RAG indexing sessions and ORM models to SQLite (#40513)
Asuka Minato3f0a9c5060c0
diff --git a/api/tests/unit_tests/core/rag/indexing/test_index_processor.py b/api/tests/unit_tests/core/rag/indexing/test_index_processor.pyindex 51d817d8f53..436d9b1280b 100644--- a/api/tests/unit_tests/core/rag/indexing/test_index_processor.py+++ b/api/tests/unit_tests/core/rag/indexing/test_index_processor.py@@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patchfrom sqlalchemy import eventfrom sqlalchemy.orm import Session, sessionmaker+from core.rag.index_processor import index_processor as index_processor_modulefrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom core.rag.index_processor.index_processor import IndexProcessorfrom core.workflow.nodes.knowledge_index.protocols import Preview, PreviewItem@@ -237,12 +238,10 @@ class TestIndexProcessor:"core.rag.index_processor.index_processor.current_app",SimpleNamespace(_get_current_object=lambda: flask_app),),- patch(- "core.rag.index_processor.index_processor.session_factory",- SimpleNamespace(create_session=sqlite_session_factory),- ),- patch(- "core.rag.index_processor.index_processor.ParagraphIndexProcessor.generate_summary",+ patch.object(index_processor_module.session_factory, "create_session", sqlite_session_factory),+ patch.object(+ index_processor_module.ParagraphIndexProcessor,+ "generate_summary",side_effect=generate_summary,) as generate_summary,):diff --git a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.pyindex ecb2985dc4b..460a8491125 100644--- a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py+++ b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py@@ -49,11 +49,13 @@ for the full indexing pipeline are handled separately in the integration test suimport jsonimport uuid-from types import SimpleNamespace+from datetime import UTC, datetimefrom typing import Anyfrom unittest.mock import MagicMock, Mock, patchimport pytest+from sqlalchemy import event+from sqlalchemy.orm import Session, sessionmakerfrom sqlalchemy.orm.exc import ObjectDeletedErrorfrom core.entities.knowledge_entities import PreviewDetail@@ -66,12 +68,13 @@ from core.indexing_runner import (from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom core.rag.models.document import ChildDocument, Documentfrom enums import DeploymentEdition+from extensions.storage.storage_type import StorageTypefrom graphon.model_runtime.entities.model_entities import ModelTypefrom libs.datetime_utils import naive_utc_now-from models.dataset import Dataset, DatasetProcessRule, DocumentSegment+from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegmentfrom models.dataset import Document as DatasetDocument-from models.enums import SegmentStatus-from models.model import Account+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, SegmentStatus+from models.model import Account, UploadFilefrom services.vector_space_admission_service import VectorSpaceAdmissionError# ============================================================================@@ -156,6 +159,33 @@ def create_mock_dataset_document()+def persist_indexing_scope(session: Session) -> tuple[Dataset, DatasetDocument]:+ tenant_id = str(uuid.uuid4())+ created_by = str(uuid.uuid4())+ dataset = Dataset(+ id=str(uuid.uuid4()),+ tenant_id=tenant_id,+ name="Dataset",+ description="",+ created_by=created_by,+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ )+ document = DatasetDocument(+ id=str(uuid.uuid4()),+ tenant_id=tenant_id,+ dataset_id=dataset.id,+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch",+ name="Document",+ created_from=DocumentCreatedFrom.API,+ created_by=created_by,+ )+ session.add_all([dataset, document])+ session.commit()+ return dataset, document++def create_sample_documents(count: int = 3,include_children: bool = False,@@ -263,14 +293,14 @@ class TestIndexingRunnerExtract:"""@pytest.fixture- def mock_dependencies(self):+ def mock_dependencies(self, sqlite_session: Session):"""Mock all external dependencies for extract tests."""with (patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,patch("core.indexing_runner.storage") as mock_storage,):yield {- "session": MagicMock(),+ "session": sqlite_session,"factory": mock_factory,"storage": mock_storage,}@@ -329,16 +359,28 @@ class TestIndexingRunnerExtract:),]mock_processor.extract.return_value = extracted_docs+ file_id = json.loads(sample_dataset_document.data_source_info)["upload_file_id"]+ upload_file = UploadFile(+ tenant_id=sample_dataset_document.tenant_id,+ storage_type=StorageType.LOCAL,+ key="uploads/test.pdf",+ name="test.pdf",+ size=10,+ extension="pdf",+ mime_type="application/pdf",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="user-id",+ created_at=datetime.now(UTC),+ used=True,+ )+ upload_file.id = file_id+ mock_dependencies["session"].add(upload_file)+ mock_dependencies["session"].commit()- # Mock the entire _extract method to avoid ExtractSetting validation- # This is necessary because ExtractSetting uses Pydantic validation- with patch.object(runner, "_update_document_index_status"):- with patch("core.indexing_runner.select"):- with patch("core.indexing_runner.ExtractSetting"):- # Act: Call the extract method- result = runner._extract(- mock_processor, sample_dataset_document, sample_process_rule, mock_dependencies["session"]- )+ with patch.object(runner, "_update_document_index_status"), patch("core.indexing_runner.ExtractSetting"):+ result = runner._extract(+ mock_processor, sample_dataset_document, sample_process_rule, mock_dependencies["session"]+ )# Assert: Verify the extraction resultsassert len(result) == 2, "Should extract 2 documents from the PDF"@@ -460,13 +502,13 @@ class TestIndexingRunnerTransform:"""@pytest.fixture- def mock_dependencies(self):+ def mock_dependencies(self, unbound_session: Session):"""Mock all external dependencies for transform tests."""with (patch("core.indexing_runner.ModelManager.for_tenant") as mock_model_manager,):yield {- "session": MagicMock(),+ "session": unbound_session,"model_manager": mock_model_manager,}@@ -625,7 +667,7 @@ class TestIndexingRunnerLoad:"""@pytest.fixture- def mock_dependencies(self):+ def mock_dependencies(self, unbound_session: Session):"""Mock all external dependencies for load tests."""with (patch("core.indexing_runner.ModelManager.for_tenant") as mock_model_manager,@@ -635,7 +677,7 @@ class TestIndexingRunnerLoad:):mock_app._get_current_object = Mock(return_value=Mock())yield {- "session": MagicMock(),+ "session": unbound_session,"model_manager": mock_model_manager,"app": mock_app,"thread": mock_thread,@@ -830,7 +872,7 @@ class TestIndexingRunnerRun:"""@pytest.fixture- def mock_dependencies(self):+ def mock_dependencies(self, sqlite_session: Session):"""Mock all external dependencies for run tests."""with (patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,@@ -839,7 +881,7 @@ class TestIndexingRunnerRun:patch("core.indexing_runner.threading.Thread") as mock_thread,):yield {- "session": MagicMock(),+ "session": sqlite_session,"factory": mock_factory,"model_manager": mock_model_manager,"storage": mock_storage,@@ -847,22 +889,46 @@ class TestIndexingRunnerRun:}@pytest.fixture- def sample_dataset_documents(self):+ def sample_dataset_documents(self, sqlite_session: Session):"""Create sample dataset documents for testing."""docs = []for i in range(2):- docs.append(- DatasetDocument(- id=str(uuid.uuid4()),- dataset_id=str(uuid.uuid4()),- tenant_id=str(uuid.uuid4()),- doc_form=IndexStructureType.PARAGRAPH_INDEX,- doc_language="English",- data_source_type="upload_file",- data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),- dataset_process_rule_id=str(uuid.uuid4()),- )+ tenant_id = str(uuid.uuid4())+ dataset_id = str(uuid.uuid4())+ account = Account(name=f"Account {i}", email=f"account-{i}@example.com")+ account.id = str(uuid.uuid4())+ dataset = Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ name=f"Dataset {i}",+ description="",+ created_by=account.id,+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ )+ process_rule = DatasetProcessRule(+ dataset_id=dataset_id,+ mode="automatic",+ rules="{}",+ created_by=account.id,+ )+ document = DatasetDocument(+ id=str(uuid.uuid4()),+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ position=1,+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ doc_language="English",+ data_source_type=DataSourceType.UPLOAD_FILE,+ data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),+ dataset_process_rule_id=process_rule.id,+ batch="batch",+ name=f"Document {i}",+ created_from=DocumentCreatedFrom.API,+ created_by=account.id,)+ sqlite_session.add_all([account, dataset, process_rule, document])+ docs.append(document)+ sqlite_session.commit()return docsdef test_run_in_indexing_status_loads_child_chunks_with_caller_session(@@ -871,9 +937,15 @@ class TestIndexingRunnerRun:runner = IndexingRunner()dataset_document = sample_dataset_documents[0]dataset_document.doc_form = IndexStructureType.PARENT_CHILD_INDEX- dataset = Dataset()+ session = mock_dependencies["session"]+ dataset = session.get(Dataset, dataset_document.dataset_id)+ assert dataset is not None+ process_rule = session.get(DatasetProcessRule, dataset_document.dataset_process_rule_id)+ assert process_rule is not None+ process_rule.mode = "hierarchical"+ process_rule.rules = json.dumps({"parent_mode": "paragraph"})segment = DocumentSegment(- tenant_id="tenant-id",+ tenant_id=dataset_document.tenant_id,dataset_id=dataset_document.dataset_id,document_id=dataset_document.id,position=1,@@ -885,29 +957,39 @@ class TestIndexingRunnerRun:index_node_id="parent-node",index_node_hash="parent-hash",)- child_chunks = [SimpleNamespace(content="child", index_node_id="child-node", index_node_hash="child-hash")]- session = mock_dependencies["session"]- session.get.side_effect = lambda model, _: dataset_document if model is DatasetDocument else dataset- session.scalars.return_value.all.return_value = [segment]+ child_chunk = ChildChunk(+ tenant_id=dataset_document.tenant_id,+ dataset_id=dataset_document.dataset_id,+ document_id=dataset_document.id,+ segment_id=segment.id,+ position=1,+ content="child",+ word_count=1,+ created_by=dataset_document.created_by,+ index_node_id="child-node",+ index_node_hash="child-hash",+ )+ session.add_all([segment, child_chunk])+ session.commit()with (- patch.object(DocumentSegment, "get_child_chunks", return_value=child_chunks) as get_child_chunks,patch.object(runner, "_load") as load,):runner.run_in_indexing_status(dataset_document, session)- get_child_chunks.assert_called_once_with(session=session)assert load.call_args.kwargs["documents"][0].children[0].page_content == "child"assert load.call_args.kwargs["total_tokens"] == 12def test_run_in_indexing_status_uses_tokens_from_all_segments(self, mock_dependencies, sample_dataset_documents):runner = IndexingRunner()dataset_document = sample_dataset_documents[0]- dataset = Dataset()+ session = mock_dependencies["session"]+ dataset = session.get(Dataset, dataset_document.dataset_id)+ assert dataset is not Nonecompleted_segment = DocumentSegment(- tenant_id="tenant-id",- dataset_id="dataset-id",- document_id="document-id",+ tenant_id=dataset_document.tenant_id,+ dataset_id=dataset_document.dataset_id,+ document_id=dataset_document.id,position=1,content="",word_count=0,@@ -928,9 +1010,8 @@ class TestIndexingRunnerRun:index_node_id="pending-node",index_node_hash="pending-hash",)- session = mock_dependencies["session"]- session.get.side_effect = lambda model, _: dataset_document if model is DatasetDocument else dataset- session.scalars.return_value.all.return_value = [completed_segment, incomplete_segment]+ session.add_all([completed_segment, incomplete_segment])+ session.commit()with patch.object(runner, "_load") as load:runner.run_in_indexing_status(dataset_document, session)@@ -944,23 +1025,13 @@ class TestIndexingRunnerRun:# Arrangerunner = IndexingRunner()doc = sample_dataset_documents[0]-- # Mock database queries- mock_dataset = Dataset(- id=doc.dataset_id,- tenant_id=doc.tenant_id,- indexing_technique=IndexTechniqueType.ECONOMY,- )-- mock_current_user = Account(name="Test Account", email="[email protected]")-- get_dispatch = {"Document": doc, "Dataset": mock_dataset, "Account": mock_current_user}- mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)-- mock_process_rule = DatasetProcessRule(- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"- )- mock_dependencies["session"].scalar.return_value = mock_process_rule+ session = mock_dependencies["session"]+ mock_dataset = session.get(Dataset, doc.dataset_id)+ mock_current_user = session.get(Account, doc.created_by)+ assert mock_dataset is not None+ assert mock_current_user is not None+ mock_dataset.indexing_technique = IndexTechniqueType.ECONOMY+ session.commit()# Mock processormock_processor = MagicMock()@@ -979,30 +1050,13 @@ class TestIndexingRunnerRun:mock_thread_instance = MagicMock()mock_dependencies["thread"].return_value = mock_thread_instance- # Mock all internal methods that interact with database- with (- patch.object(runner, "_extract", return_value=[Document(page_content="Test", metadata={})]),- patch.object(- runner,- "_transform",- return_value=[Document(page_content="Chunk", metadata={"doc_id": "c1", "doc_hash": "h1"})],- ),- patch.object(runner, "_load_segments"),- patch.object(runner, "_load"),- ):- # Act- mock_dependencies["session"].commit.reset_mock()- runner.run([doc], mock_dependencies["session"])+ commits = 0- # Assert - verify the methods were called- # Since we're mocking the internal methods, we just verify no exceptions were raised- set_tenant_id.assert_called_once_with(- mock_current_user,- mock_dataset.tenant_id,- session=mock_dependencies["session"],- )+ def record_commit(_session) -> None:+ nonlocal commits+ commits += 1- mock_dependencies["session"].commit.reset_mock()+ event.listen(session, "after_commit", record_commit)with (patch.object(runner, "_extract", return_value=[Document(page_content="Test", metadata={})]) as mock_extract,patch.object(@@ -1014,14 +1068,16 @@ class TestIndexingRunnerRun:patch.object(runner, "_load") as mock_load,):# Act- runner.run([doc], mock_dependencies["session"])+ runner.run([doc], session)+ event.remove(session, "after_commit", record_commit)# Assert - verify the methods were calledmock_extract.assert_called_once()mock_transform.assert_called_once()mock_load_segments.assert_called_once()mock_load.assert_called_once()- assert mock_dependencies["session"].commit.call_count == 2+ assert commits == 2+ set_tenant_id.assert_called_once_with(mock_current_user, mock_dataset.tenant_id, session=session)mock_processor = MagicMock()mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor@@ -1038,26 +1094,15 @@ class TestIndexingRunnerRun:):runner = IndexingRunner()dataset_document = sample_dataset_documents[0]- dataset = Dataset(- id=dataset_document.dataset_id,- tenant_id=dataset_document.tenant_id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- )- current_user = Account(name="Test Account", email="[email protected]")+ session = mock_dependencies["session"]+ dataset = session.get(Dataset, dataset_document.dataset_id)+ current_user = session.get(Account, dataset_document.created_by)+ assert dataset is not None+ assert current_user is not Nonetransformed_documents = [Document(page_content="first", metadata={"doc_id": "first", "doc_hash": "hash-first"}),Document(page_content="second", metadata={"doc_id": "second", "doc_hash": "hash-second"}),]- model_dispatch = {- DatasetDocument: dataset_document,- Dataset: dataset,- Account: current_user,- }- mock_dependencies["session"].get.side_effect = lambda model, _: model_dispatch.get(model)- process_rule = DatasetProcessRule(- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"- )- mock_dependencies["session"].scalar.return_value = process_rulewith (patch.object(runner, "_extract", return_value=[Document(page_content="source", metadata={})]),@@ -1093,22 +1138,11 @@ class TestIndexingRunnerRun:runner = IndexingRunner(enforce_vector_space_admission=True)dataset_document = sample_dataset_documents[0]dataset_document.need_summary = False- dataset = Dataset(- id=dataset_document.dataset_id,- tenant_id=dataset_document.tenant_id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- )- current_user = Account(name="Test Account", email="[email protected]")- model_dispatch = {- DatasetDocument: dataset_document,- Dataset: dataset,- Account: current_user,- }- mock_dependencies["session"].get.side_effect = lambda model, _: model_dispatch.get(model)- process_rule = DatasetProcessRule(- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"- )- mock_dependencies["session"].scalar.return_value = process_rule+ session = mock_dependencies["session"]+ dataset = session.get(Dataset, dataset_document.dataset_id)+ current_user = session.get(Account, dataset_document.created_by)+ assert dataset is not None+ assert current_user is not Nonetransformed_documents = [Document(page_content="Chunk", metadata={"doc_id": "c1", "doc_hash": "h1"})]admission_error = VectorSpaceAdmissionError("estimated storage exceeds capacity")admission_service = Mock()@@ -1151,28 +1185,15 @@ class TestIndexingRunnerRun:):runner = IndexingRunner()dataset_document = sample_dataset_documents[0]- dataset_document.created_by = "user-1"- dataset = Dataset(- id=dataset_document.dataset_id,- tenant_id=dataset_document.tenant_id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- )- current_user = Account(name="Test Account", email="[email protected]")+ session = mock_dependencies["session"]+ dataset = session.get(Dataset, dataset_document.dataset_id)+ current_user = session.get(Account, dataset_document.created_by)+ assert dataset is not None+ assert current_user is not Nonetransformed_documents = [Document(page_content="first", metadata={"doc_id": "first", "doc_hash": "hash-first"}),Document(page_content="second", metadata={"doc_id": "second", "doc_hash": "hash-second"}),]- model_dispatch = {- DatasetDocument: dataset_document,- Dataset: dataset,- Account: current_user,- }- mock_dependencies["session"].get.side_effect = lambda model, _: model_dispatch.get(model)- mock_dependencies["session"].scalars.return_value.all.return_value = []- process_rule = DatasetProcessRule(- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"- )- mock_dependencies["session"].scalar.return_value = process_rulewith (patch.object(runner, "_extract", return_value=[Document(page_content="source", metadata={})]),@@ -1207,19 +1228,6 @@ class TestIndexingRunnerRun:runner = IndexingRunner()doc = sample_dataset_documents[0]- # Mock database- mock_dataset = Dataset(- tenant_id=doc.tenant_id,- )-- get_dispatch = {"Document": doc, "Dataset": mock_dataset}- mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)-- mock_process_rule = DatasetProcessRule(- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"- )- mock_dependencies["session"].scalar.return_value = mock_process_rule-mock_processor = MagicMock()mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processormock_processor.extract.side_effect = ProviderTokenNotInitError("Token not initialized")@@ -1228,9 +1236,8 @@ class TestIndexingRunnerRun:with patch.object(runner, "_extract", side_effect=ProviderTokenNotInitError("Token not initialized")):runner.run([doc], mock_dependencies["session"])- # Assert- # Verify document status was updated to error- assert mock_dependencies["session"].flush.called+ mock_dependencies["session"].refresh(doc)+ assert doc.indexing_status == "error"def test_run_handles_object_deleted_error(self, mock_dependencies, sample_dataset_documents):"""Test run handles ObjectDeletedError gracefully."""@@ -1238,19 +1245,6 @@ class TestIndexingRunnerRun:runner = IndexingRunner()doc = sample_dataset_documents[0]- # Mock database- mock_dataset = Dataset(- tenant_id=doc.tenant_id,- )-- get_dispatch = {"Document": doc, "Dataset": mock_dataset}- mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)-- mock_process_rule = DatasetProcessRule(- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"- )- mock_dependencies["session"].scalar.return_value = mock_process_rule-mock_processor = MagicMock()mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor@@ -1268,28 +1262,11 @@ class TestIndexingRunnerRun:# Arrangerunner = IndexingRunner()docs = sample_dataset_documents-- # Mock database- mock_dataset = Dataset(- indexing_technique=IndexTechniqueType.ECONOMY,- )- mock_current_user = Account(name="Test Account", email="[email protected]")-- doc_map = {doc.id: doc for doc in docs}- model_dispatch = {"Dataset": mock_dataset, "Account": mock_current_user}-- def get_side_effect(model_class, id):- name = model_class.__name__- if name == "Document":- return doc_map.get(id)- return model_dispatch.get(name)-- mock_dependencies["session"].get.side_effect = get_side_effect-- mock_process_rule = DatasetProcessRule(- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"- )- mock_dependencies["session"].scalar.return_value = mock_process_rule+ for doc in docs:+ dataset = mock_dependencies["session"].get(Dataset, doc.dataset_id)+ assert dataset is not None+ dataset.indexing_technique = IndexTechniqueType.ECONOMY+ mock_dependencies["session"].commit()mock_processor = MagicMock()mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor@@ -1329,13 +1306,13 @@ class TestIndexingRunnerRetryLogic:"""@pytest.fixture- def mock_dependencies(self):+ def mock_dependencies(self, sqlite_session: Session):"""Mock all external dependencies."""with (patch("core.indexing_runner.redis_client") as mock_redis,):yield {- "session": MagicMock(),+ "session": sqlite_session,"redis": mock_redis,}@@ -1360,41 +1337,62 @@ class TestIndexingRunnerRetryLogic:def test_update_document_index_status_success(self, mock_dependencies):"""Test successful document status update."""- # Arrangedocument_id = str(uuid.uuid4())- mock_document = DatasetDocument(id=document_id)-- mock_dependencies["session"].scalar.return_value = 0- mock_dependencies["session"].get.return_value = mock_document+ session = mock_dependencies["session"]+ document = DatasetDocument(+ id=document_id,+ tenant_id=str(uuid.uuid4()),+ dataset_id=str(uuid.uuid4()),+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch",+ name="Document",+ created_from=DocumentCreatedFrom.API,+ created_by=str(uuid.uuid4()),+ is_paused=False,+ )+ session.add(document)+ session.commit()# ActIndexingRunner._update_document_index_status(document_id,"completed",{"tokens": 100, "completed_at": naive_utc_now()},- session=mock_dependencies["session"],+ session=session,)- # Assert- mock_dependencies["session"].flush.assert_called()+ session.refresh(document)+ assert document.indexing_status == "completed"+ assert document.tokens == 100def test_update_document_index_status_paused(self, mock_dependencies):"""Test document status update when document is paused."""- # Arrangedocument_id = str(uuid.uuid4())- mock_dependencies["session"].scalar.return_value = 1+ session = mock_dependencies["session"]+ document = DatasetDocument(+ id=document_id,+ tenant_id=str(uuid.uuid4()),+ dataset_id=str(uuid.uuid4()),+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch",+ name="Paused document",+ created_from=DocumentCreatedFrom.API,+ created_by=str(uuid.uuid4()),+ is_paused=True,+ )+ session.add(document)+ session.commit()# Act & Assertwith pytest.raises(DocumentIsPausedError):- IndexingRunner._update_document_index_status(document_id, "completed", session=mock_dependencies["session"])+ IndexingRunner._update_document_index_status(document_id, "completed", session=session)def test_update_document_index_status_deleted(self, mock_dependencies):"""Test document status update when document is deleted."""# Arrangedocument_id = str(uuid.uuid4())- mock_dependencies["session"].scalar.return_value = 0- mock_dependencies["session"].get.return_value = None-# Act & Assertwith pytest.raises(DocumentIsDeletedPausedError):IndexingRunner._update_document_index_status(document_id, "completed", session=mock_dependencies["session"])@@ -1610,13 +1608,13 @@ class TestIndexingRunnerLoadSegments:"""@pytest.fixture- def mock_dependencies(self):+ def mock_dependencies(self, unbound_session: Session):"""Mock all external dependencies."""with (patch("core.indexing_runner.DatasetDocumentStore") as mock_docstore,):yield {- "session": MagicMock(),+ "session": unbound_session,"docstore": mock_docstore,}@@ -1774,13 +1772,13 @@ class TestIndexingRunnerEstimate:"""@pytest.fixture- def mock_dependencies(self):+ def mock_dependencies(self, sqlite_session: Session):"""Mock all external dependencies."""with (patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,):yield {- "session": MagicMock(),+ "session": sqlite_session,"factory": mock_factory,}@@ -1813,7 +1811,8 @@ class TestIndexingRunnerEstimate:mock_processor = MagicMock()mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processorphase_events: list[str] = []- mock_dependencies["session"].commit.side_effect = lambda: phase_events.append("commit")+ session = mock_dependencies["session"]+ event.listen(session, "after_commit", lambda _session: phase_events.append("commit"))preview_doc = Document(page_content="",@@ -1825,8 +1824,23 @@ class TestIndexingRunnerEstimate:phase_events.append("summary") or [PreviewDetail(content=preview_doc.page_content)])- image_file = SimpleNamespace(key="image_files/tenant-1/source-file-1/image.png")- mock_dependencies["session"].scalar.return_value = image_file+ image_file = UploadFile(+ tenant_id=tenant_id,+ storage_type=StorageType.LOCAL,+ key="image_files/tenant-1/source-file-1/image.png",+ name="image.png",+ size=10,+ extension="png",+ mime_type="image/png",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="user-id",+ created_at=datetime.now(UTC),+ used=True,+ )+ image_file.id = "image-1"+ session.add(image_file)+ session.commit()+ phase_events.clear()with (patch("core.indexing_runner.get_image_upload_file_ids", return_value=["image-1"]),@@ -1844,12 +1858,12 @@ class TestIndexingRunnerEstimate:"summary_index_setting": {"enable": True},},doc_form=IndexStructureType.PARAGRAPH_INDEX,- session=mock_dependencies["session"],+ session=session,)assert result.total_segments == 1mock_storage.delete.assert_called_once_with(image_file.key)- mock_dependencies["session"].delete.assert_called_once_with(image_file)+ assert session.get(UploadFile, image_file.id) is Noneassert phase_events == ["commit", "summary"]@@ -1863,13 +1877,13 @@ class TestIndexingRunnerProcessChunk:"""@pytest.fixture- def mock_dependencies(self):+ def mock_dependencies(self, sqlite_session: Session):"""Mock all external dependencies."""with (patch("core.indexing_runner.redis_client") as mock_redis,):yield {- "session": MagicMock(),+ "session": sqlite_session,"redis": mock_redis,}@@ -1881,7 +1895,12 @@ class TestIndexingRunnerProcessChunk:app.app_context.return_value.__exit__ = MagicMock()return app- def test_process_chunk_loads_index_and_completes_segments(self, mock_dependencies, mock_flask_app):+ def test_process_chunk_loads_index_and_completes_segments(+ self,+ mock_dependencies,+ mock_flask_app,+ sqlite_session_factory: sessionmaker[Session],+ ):"""Test process chunk loads the index and completes segments without counting tokens."""# Arrangefrom core.indexing_runner import IndexingRunner@@ -1893,33 +1912,36 @@ class TestIndexingRunnerProcessChunk:Document(page_content="Chunk 2", metadata={"doc_id": "c2"}),]- mock_dataset = Dataset(- id=str(uuid.uuid4()),- )-- mock_dataset_document = DatasetDocument(id=str(uuid.uuid4()))+ session = mock_dependencies["session"]+ mock_dataset, mock_dataset_document = persist_indexing_scope(session)+ segments = [+ DocumentSegment(+ tenant_id=mock_dataset.tenant_id,+ dataset_id=mock_dataset.id,+ document_id=mock_dataset_document.id,+ position=position,+ content=f"Chunk {position}",+ word_count=1,+ tokens=1,+ created_by=mock_dataset.created_by,+ index_node_id=doc.metadata["doc_id"],+ status=SegmentStatus.INDEXING,+ )+ for position, doc in enumerate(chunk_documents, start=1)+ ]+ session.add_all(segments)+ session.commit()mock_dependencies["redis"].get.return_value = None- # Mock database update for segment status- mock_dependencies["session"].execute.return_value = None- mock_dependencies["session"].get.side_effect = lambda model, _id: {- Dataset: mock_dataset,- DatasetDocument: mock_dataset_document,- }.get(model)-# Create a proper context manager mockmock_context = MagicMock()mock_context.__enter__ = MagicMock(return_value=None)mock_context.__exit__ = MagicMock(return_value=None)mock_flask_app.app_context.return_value = mock_context- session_context = MagicMock()- session_context.__enter__.return_value = mock_dependencies["session"]- session_context.__exit__.return_value = None-with (- patch("core.indexing_runner.session_factory.create_session", return_value=session_context),+ patch("core.indexing_runner.session_factory.create_session", sqlite_session_factory),patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,):mock_factory.return_value.init_index_processor.return_value = mock_processor@@ -1936,10 +1958,15 @@ class TestIndexingRunnerProcessChunk:# Assertassert result is Nonemock_processor.load.assert_called_once()- mock_dependencies["session"].execute.assert_called_once()- mock_dependencies["session"].commit.assert_called_once()-- def test_process_chunk_detects_pause(self, mock_dependencies, mock_flask_app):+ session.expire_all()+ assert all(session.get(DocumentSegment, segment.id).status == SegmentStatus.COMPLETED for segment in segments)++ def test_process_chunk_detects_pause(+ self,+ mock_dependencies,+ mock_flask_app,+ sqlite_session_factory: sessionmaker[Session],+ ):"""Test process chunk detects document pause."""# Arrangefrom core.indexing_runner import IndexingRunner@@ -1947,27 +1974,17 @@ class TestIndexingRunnerProcessChunk:runner = IndexingRunner()chunk_documents = [Document(page_content="Chunk", metadata={"doc_id": "c1"})]- mock_dataset = Dataset()- mock_dataset_document = DatasetDocument(id=str(uuid.uuid4()))+ mock_dataset, mock_dataset_document = persist_indexing_scope(mock_dependencies["session"])# Mock Redis to return paused statusmock_dependencies["redis"].get.return_value = "1"- mock_dependencies["session"].get.side_effect = lambda model, _id: {- Dataset: mock_dataset,- DatasetDocument: mock_dataset_document,- }.get(model)-# Create a proper context manager mockmock_context = MagicMock()mock_context.__enter__ = MagicMock(return_value=None)mock_context.__exit__ = MagicMock(return_value=None)mock_flask_app.app_context.return_value = mock_context- session_context = MagicMock()- session_context.__enter__.return_value = mock_dependencies["session"]- session_context.__exit__.return_value = None-- with patch("core.indexing_runner.session_factory.create_session", return_value=session_context):+ with patch("core.indexing_runner.session_factory.create_session", sqlite_session_factory):# Act & Assert - the method creates its own app_context and sessionwith pytest.raises(DocumentIsPausedError):runner._process_chunk(