Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: migrate RAG indexing sessions and ORM models to SQLite (#40513)

Asuka Minato3f0a9c5060c0
diff --git a/api/tests/unit_tests/core/rag/indexing/test_index_processor.py b/api/tests/unit_tests/core/rag/indexing/test_index_processor.py
index 51d817d8f53..436d9b1280b 100644
--- a/api/tests/unit_tests/core/rag/indexing/test_index_processor.py
+++ b/api/tests/unit_tests/core/rag/indexing/test_index_processor.py
@@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patch
from sqlalchemy import event
from sqlalchemy.orm import Session, sessionmaker
+from core.rag.index_processor import index_processor as index_processor_module
from core.rag.index_processor.constant.index_type import IndexTechniqueType
from core.rag.index_processor.index_processor import IndexProcessor
from core.workflow.nodes.knowledge_index.protocols import Preview, PreviewItem
@@ -237,12 +238,10 @@ class TestIndexProcessor:
"core.rag.index_processor.index_processor.current_app",
SimpleNamespace(_get_current_object=lambda: flask_app),
),
- patch(
- "core.rag.index_processor.index_processor.session_factory",
- SimpleNamespace(create_session=sqlite_session_factory),
- ),
- patch(
- "core.rag.index_processor.index_processor.ParagraphIndexProcessor.generate_summary",
+ patch.object(index_processor_module.session_factory, "create_session", sqlite_session_factory),
+ patch.object(
+ index_processor_module.ParagraphIndexProcessor,
+ "generate_summary",
side_effect=generate_summary,
) as generate_summary,
):
diff --git a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
index ecb2985dc4b..460a8491125 100644
--- a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
+++ b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
@@ -49,11 +49,13 @@ for the full indexing pipeline are handled separately in the integration test su
import json
import uuid
-from types import SimpleNamespace
+from datetime import UTC, datetime
from typing import Any
from unittest.mock import MagicMock, Mock, patch
import pytest
+from sqlalchemy import event
+from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.orm.exc import ObjectDeletedError
from core.entities.knowledge_entities import PreviewDetail
@@ -66,12 +68,13 @@ from core.indexing_runner import (
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from core.rag.models.document import ChildDocument, Document
from enums import DeploymentEdition
+from extensions.storage.storage_type import StorageType
from graphon.model_runtime.entities.model_entities import ModelType
from libs.datetime_utils import naive_utc_now
-from models.dataset import Dataset, DatasetProcessRule, DocumentSegment
+from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentSegment
from models.dataset import Document as DatasetDocument
-from models.enums import SegmentStatus
-from models.model import Account
+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, SegmentStatus
+from models.model import Account, UploadFile
from services.vector_space_admission_service import VectorSpaceAdmissionError
# ============================================================================
@@ -156,6 +159,33 @@ def create_mock_dataset_document(
)
+def persist_indexing_scope(session: Session) -> tuple[Dataset, DatasetDocument]:
+ tenant_id = str(uuid.uuid4())
+ created_by = str(uuid.uuid4())
+ dataset = Dataset(
+ id=str(uuid.uuid4()),
+ tenant_id=tenant_id,
+ name="Dataset",
+ description="",
+ created_by=created_by,
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ )
+ document = DatasetDocument(
+ id=str(uuid.uuid4()),
+ tenant_id=tenant_id,
+ dataset_id=dataset.id,
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch="batch",
+ name="Document",
+ created_from=DocumentCreatedFrom.API,
+ created_by=created_by,
+ )
+ session.add_all([dataset, document])
+ session.commit()
+ return dataset, document
+
+
def create_sample_documents(
count: int = 3,
include_children: bool = False,
@@ -263,14 +293,14 @@ class TestIndexingRunnerExtract:
"""
@pytest.fixture
- def mock_dependencies(self):
+ def mock_dependencies(self, sqlite_session: Session):
"""Mock all external dependencies for extract tests."""
with (
patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,
patch("core.indexing_runner.storage") as mock_storage,
):
yield {
- "session": MagicMock(),
+ "session": sqlite_session,
"factory": mock_factory,
"storage": mock_storage,
}
@@ -329,16 +359,28 @@ class TestIndexingRunnerExtract:
),
]
mock_processor.extract.return_value = extracted_docs
+ file_id = json.loads(sample_dataset_document.data_source_info)["upload_file_id"]
+ upload_file = UploadFile(
+ tenant_id=sample_dataset_document.tenant_id,
+ storage_type=StorageType.LOCAL,
+ key="uploads/test.pdf",
+ name="test.pdf",
+ size=10,
+ extension="pdf",
+ mime_type="application/pdf",
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by="user-id",
+ created_at=datetime.now(UTC),
+ used=True,
+ )
+ upload_file.id = file_id
+ mock_dependencies["session"].add(upload_file)
+ mock_dependencies["session"].commit()
- # Mock the entire _extract method to avoid ExtractSetting validation
- # This is necessary because ExtractSetting uses Pydantic validation
- with patch.object(runner, "_update_document_index_status"):
- with patch("core.indexing_runner.select"):
- with patch("core.indexing_runner.ExtractSetting"):
- # Act: Call the extract method
- result = runner._extract(
- mock_processor, sample_dataset_document, sample_process_rule, mock_dependencies["session"]
- )
+ with patch.object(runner, "_update_document_index_status"), patch("core.indexing_runner.ExtractSetting"):
+ result = runner._extract(
+ mock_processor, sample_dataset_document, sample_process_rule, mock_dependencies["session"]
+ )
# Assert: Verify the extraction results
assert len(result) == 2, "Should extract 2 documents from the PDF"
@@ -460,13 +502,13 @@ class TestIndexingRunnerTransform:
"""
@pytest.fixture
- def mock_dependencies(self):
+ def mock_dependencies(self, unbound_session: Session):
"""Mock all external dependencies for transform tests."""
with (
patch("core.indexing_runner.ModelManager.for_tenant") as mock_model_manager,
):
yield {
- "session": MagicMock(),
+ "session": unbound_session,
"model_manager": mock_model_manager,
}
@@ -625,7 +667,7 @@ class TestIndexingRunnerLoad:
"""
@pytest.fixture
- def mock_dependencies(self):
+ def mock_dependencies(self, unbound_session: Session):
"""Mock all external dependencies for load tests."""
with (
patch("core.indexing_runner.ModelManager.for_tenant") as mock_model_manager,
@@ -635,7 +677,7 @@ class TestIndexingRunnerLoad:
):
mock_app._get_current_object = Mock(return_value=Mock())
yield {
- "session": MagicMock(),
+ "session": unbound_session,
"model_manager": mock_model_manager,
"app": mock_app,
"thread": mock_thread,
@@ -830,7 +872,7 @@ class TestIndexingRunnerRun:
"""
@pytest.fixture
- def mock_dependencies(self):
+ def mock_dependencies(self, sqlite_session: Session):
"""Mock all external dependencies for run tests."""
with (
patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,
@@ -839,7 +881,7 @@ class TestIndexingRunnerRun:
patch("core.indexing_runner.threading.Thread") as mock_thread,
):
yield {
- "session": MagicMock(),
+ "session": sqlite_session,
"factory": mock_factory,
"model_manager": mock_model_manager,
"storage": mock_storage,
@@ -847,22 +889,46 @@ class TestIndexingRunnerRun:
}
@pytest.fixture
- def sample_dataset_documents(self):
+ def sample_dataset_documents(self, sqlite_session: Session):
"""Create sample dataset documents for testing."""
docs = []
for i in range(2):
- docs.append(
- DatasetDocument(
- id=str(uuid.uuid4()),
- dataset_id=str(uuid.uuid4()),
- tenant_id=str(uuid.uuid4()),
- doc_form=IndexStructureType.PARAGRAPH_INDEX,
- doc_language="English",
- data_source_type="upload_file",
- data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),
- dataset_process_rule_id=str(uuid.uuid4()),
- )
+ tenant_id = str(uuid.uuid4())
+ dataset_id = str(uuid.uuid4())
+ account = Account(name=f"Account {i}", email=f"account-{i}@example.com")
+ account.id = str(uuid.uuid4())
+ dataset = Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ name=f"Dataset {i}",
+ description="",
+ created_by=account.id,
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ )
+ process_rule = DatasetProcessRule(
+ dataset_id=dataset_id,
+ mode="automatic",
+ rules="{}",
+ created_by=account.id,
+ )
+ document = DatasetDocument(
+ id=str(uuid.uuid4()),
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ position=1,
+ doc_form=IndexStructureType.PARAGRAPH_INDEX,
+ doc_language="English",
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),
+ dataset_process_rule_id=process_rule.id,
+ batch="batch",
+ name=f"Document {i}",
+ created_from=DocumentCreatedFrom.API,
+ created_by=account.id,
)
+ sqlite_session.add_all([account, dataset, process_rule, document])
+ docs.append(document)
+ sqlite_session.commit()
return docs
def test_run_in_indexing_status_loads_child_chunks_with_caller_session(
@@ -871,9 +937,15 @@ class TestIndexingRunnerRun:
runner = IndexingRunner()
dataset_document = sample_dataset_documents[0]
dataset_document.doc_form = IndexStructureType.PARENT_CHILD_INDEX
- dataset = Dataset()
+ session = mock_dependencies["session"]
+ dataset = session.get(Dataset, dataset_document.dataset_id)
+ assert dataset is not None
+ process_rule = session.get(DatasetProcessRule, dataset_document.dataset_process_rule_id)
+ assert process_rule is not None
+ process_rule.mode = "hierarchical"
+ process_rule.rules = json.dumps({"parent_mode": "paragraph"})
segment = DocumentSegment(
- tenant_id="tenant-id",
+ tenant_id=dataset_document.tenant_id,
dataset_id=dataset_document.dataset_id,
document_id=dataset_document.id,
position=1,
@@ -885,29 +957,39 @@ class TestIndexingRunnerRun:
index_node_id="parent-node",
index_node_hash="parent-hash",
)
- child_chunks = [SimpleNamespace(content="child", index_node_id="child-node", index_node_hash="child-hash")]
- session = mock_dependencies["session"]
- session.get.side_effect = lambda model, _: dataset_document if model is DatasetDocument else dataset
- session.scalars.return_value.all.return_value = [segment]
+ child_chunk = ChildChunk(
+ tenant_id=dataset_document.tenant_id,
+ dataset_id=dataset_document.dataset_id,
+ document_id=dataset_document.id,
+ segment_id=segment.id,
+ position=1,
+ content="child",
+ word_count=1,
+ created_by=dataset_document.created_by,
+ index_node_id="child-node",
+ index_node_hash="child-hash",
+ )
+ session.add_all([segment, child_chunk])
+ session.commit()
with (
- patch.object(DocumentSegment, "get_child_chunks", return_value=child_chunks) as get_child_chunks,
patch.object(runner, "_load") as load,
):
runner.run_in_indexing_status(dataset_document, session)
- get_child_chunks.assert_called_once_with(session=session)
assert load.call_args.kwargs["documents"][0].children[0].page_content == "child"
assert load.call_args.kwargs["total_tokens"] == 12
def test_run_in_indexing_status_uses_tokens_from_all_segments(self, mock_dependencies, sample_dataset_documents):
runner = IndexingRunner()
dataset_document = sample_dataset_documents[0]
- dataset = Dataset()
+ session = mock_dependencies["session"]
+ dataset = session.get(Dataset, dataset_document.dataset_id)
+ assert dataset is not None
completed_segment = DocumentSegment(
- tenant_id="tenant-id",
- dataset_id="dataset-id",
- document_id="document-id",
+ tenant_id=dataset_document.tenant_id,
+ dataset_id=dataset_document.dataset_id,
+ document_id=dataset_document.id,
position=1,
content="",
word_count=0,
@@ -928,9 +1010,8 @@ class TestIndexingRunnerRun:
index_node_id="pending-node",
index_node_hash="pending-hash",
)
- session = mock_dependencies["session"]
- session.get.side_effect = lambda model, _: dataset_document if model is DatasetDocument else dataset
- session.scalars.return_value.all.return_value = [completed_segment, incomplete_segment]
+ session.add_all([completed_segment, incomplete_segment])
+ session.commit()
with patch.object(runner, "_load") as load:
runner.run_in_indexing_status(dataset_document, session)
@@ -944,23 +1025,13 @@ class TestIndexingRunnerRun:
# Arrange
runner = IndexingRunner()
doc = sample_dataset_documents[0]
-
- # Mock database queries
- mock_dataset = Dataset(
- id=doc.dataset_id,
- tenant_id=doc.tenant_id,
- indexing_technique=IndexTechniqueType.ECONOMY,
- )
-
- mock_current_user = Account(name="Test Account", email="[email protected]")
-
- get_dispatch = {"Document": doc, "Dataset": mock_dataset, "Account": mock_current_user}
- mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)
-
- mock_process_rule = DatasetProcessRule(
- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
- )
- mock_dependencies["session"].scalar.return_value = mock_process_rule
+ session = mock_dependencies["session"]
+ mock_dataset = session.get(Dataset, doc.dataset_id)
+ mock_current_user = session.get(Account, doc.created_by)
+ assert mock_dataset is not None
+ assert mock_current_user is not None
+ mock_dataset.indexing_technique = IndexTechniqueType.ECONOMY
+ session.commit()
# Mock processor
mock_processor = MagicMock()
@@ -979,30 +1050,13 @@ class TestIndexingRunnerRun:
mock_thread_instance = MagicMock()
mock_dependencies["thread"].return_value = mock_thread_instance
- # Mock all internal methods that interact with database
- with (
- patch.object(runner, "_extract", return_value=[Document(page_content="Test", metadata={})]),
- patch.object(
- runner,
- "_transform",
- return_value=[Document(page_content="Chunk", metadata={"doc_id": "c1", "doc_hash": "h1"})],
- ),
- patch.object(runner, "_load_segments"),
- patch.object(runner, "_load"),
- ):
- # Act
- mock_dependencies["session"].commit.reset_mock()
- runner.run([doc], mock_dependencies["session"])
+ commits = 0
- # Assert - verify the methods were called
- # Since we're mocking the internal methods, we just verify no exceptions were raised
- set_tenant_id.assert_called_once_with(
- mock_current_user,
- mock_dataset.tenant_id,
- session=mock_dependencies["session"],
- )
+ def record_commit(_session) -> None:
+ nonlocal commits
+ commits += 1
- mock_dependencies["session"].commit.reset_mock()
+ event.listen(session, "after_commit", record_commit)
with (
patch.object(runner, "_extract", return_value=[Document(page_content="Test", metadata={})]) as mock_extract,
patch.object(
@@ -1014,14 +1068,16 @@ class TestIndexingRunnerRun:
patch.object(runner, "_load") as mock_load,
):
# Act
- runner.run([doc], mock_dependencies["session"])
+ runner.run([doc], session)
+ event.remove(session, "after_commit", record_commit)
# Assert - verify the methods were called
mock_extract.assert_called_once()
mock_transform.assert_called_once()
mock_load_segments.assert_called_once()
mock_load.assert_called_once()
- assert mock_dependencies["session"].commit.call_count == 2
+ assert commits == 2
+ set_tenant_id.assert_called_once_with(mock_current_user, mock_dataset.tenant_id, session=session)
mock_processor = MagicMock()
mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor
@@ -1038,26 +1094,15 @@ class TestIndexingRunnerRun:
):
runner = IndexingRunner()
dataset_document = sample_dataset_documents[0]
- dataset = Dataset(
- id=dataset_document.dataset_id,
- tenant_id=dataset_document.tenant_id,
- indexing_technique=IndexTechniqueType.HIGH_QUALITY,
- )
- current_user = Account(name="Test Account", email="[email protected]")
+ session = mock_dependencies["session"]
+ dataset = session.get(Dataset, dataset_document.dataset_id)
+ current_user = session.get(Account, dataset_document.created_by)
+ assert dataset is not None
+ assert current_user is not None
transformed_documents = [
Document(page_content="first", metadata={"doc_id": "first", "doc_hash": "hash-first"}),
Document(page_content="second", metadata={"doc_id": "second", "doc_hash": "hash-second"}),
]
- model_dispatch = {
- DatasetDocument: dataset_document,
- Dataset: dataset,
- Account: current_user,
- }
- mock_dependencies["session"].get.side_effect = lambda model, _: model_dispatch.get(model)
- process_rule = DatasetProcessRule(
- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
- )
- mock_dependencies["session"].scalar.return_value = process_rule
with (
patch.object(runner, "_extract", return_value=[Document(page_content="source", metadata={})]),
@@ -1093,22 +1138,11 @@ class TestIndexingRunnerRun:
runner = IndexingRunner(enforce_vector_space_admission=True)
dataset_document = sample_dataset_documents[0]
dataset_document.need_summary = False
- dataset = Dataset(
- id=dataset_document.dataset_id,
- tenant_id=dataset_document.tenant_id,
- indexing_technique=IndexTechniqueType.HIGH_QUALITY,
- )
- current_user = Account(name="Test Account", email="[email protected]")
- model_dispatch = {
- DatasetDocument: dataset_document,
- Dataset: dataset,
- Account: current_user,
- }
- mock_dependencies["session"].get.side_effect = lambda model, _: model_dispatch.get(model)
- process_rule = DatasetProcessRule(
- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
- )
- mock_dependencies["session"].scalar.return_value = process_rule
+ session = mock_dependencies["session"]
+ dataset = session.get(Dataset, dataset_document.dataset_id)
+ current_user = session.get(Account, dataset_document.created_by)
+ assert dataset is not None
+ assert current_user is not None
transformed_documents = [Document(page_content="Chunk", metadata={"doc_id": "c1", "doc_hash": "h1"})]
admission_error = VectorSpaceAdmissionError("estimated storage exceeds capacity")
admission_service = Mock()
@@ -1151,28 +1185,15 @@ class TestIndexingRunnerRun:
):
runner = IndexingRunner()
dataset_document = sample_dataset_documents[0]
- dataset_document.created_by = "user-1"
- dataset = Dataset(
- id=dataset_document.dataset_id,
- tenant_id=dataset_document.tenant_id,
- indexing_technique=IndexTechniqueType.HIGH_QUALITY,
- )
- current_user = Account(name="Test Account", email="[email protected]")
+ session = mock_dependencies["session"]
+ dataset = session.get(Dataset, dataset_document.dataset_id)
+ current_user = session.get(Account, dataset_document.created_by)
+ assert dataset is not None
+ assert current_user is not None
transformed_documents = [
Document(page_content="first", metadata={"doc_id": "first", "doc_hash": "hash-first"}),
Document(page_content="second", metadata={"doc_id": "second", "doc_hash": "hash-second"}),
]
- model_dispatch = {
- DatasetDocument: dataset_document,
- Dataset: dataset,
- Account: current_user,
- }
- mock_dependencies["session"].get.side_effect = lambda model, _: model_dispatch.get(model)
- mock_dependencies["session"].scalars.return_value.all.return_value = []
- process_rule = DatasetProcessRule(
- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
- )
- mock_dependencies["session"].scalar.return_value = process_rule
with (
patch.object(runner, "_extract", return_value=[Document(page_content="source", metadata={})]),
@@ -1207,19 +1228,6 @@ class TestIndexingRunnerRun:
runner = IndexingRunner()
doc = sample_dataset_documents[0]
- # Mock database
- mock_dataset = Dataset(
- tenant_id=doc.tenant_id,
- )
-
- get_dispatch = {"Document": doc, "Dataset": mock_dataset}
- mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)
-
- mock_process_rule = DatasetProcessRule(
- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
- )
- mock_dependencies["session"].scalar.return_value = mock_process_rule
-
mock_processor = MagicMock()
mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor
mock_processor.extract.side_effect = ProviderTokenNotInitError("Token not initialized")
@@ -1228,9 +1236,8 @@ class TestIndexingRunnerRun:
with patch.object(runner, "_extract", side_effect=ProviderTokenNotInitError("Token not initialized")):
runner.run([doc], mock_dependencies["session"])
- # Assert
- # Verify document status was updated to error
- assert mock_dependencies["session"].flush.called
+ mock_dependencies["session"].refresh(doc)
+ assert doc.indexing_status == "error"
def test_run_handles_object_deleted_error(self, mock_dependencies, sample_dataset_documents):
"""Test run handles ObjectDeletedError gracefully."""
@@ -1238,19 +1245,6 @@ class TestIndexingRunnerRun:
runner = IndexingRunner()
doc = sample_dataset_documents[0]
- # Mock database
- mock_dataset = Dataset(
- tenant_id=doc.tenant_id,
- )
-
- get_dispatch = {"Document": doc, "Dataset": mock_dataset}
- mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)
-
- mock_process_rule = DatasetProcessRule(
- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
- )
- mock_dependencies["session"].scalar.return_value = mock_process_rule
-
mock_processor = MagicMock()
mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor
@@ -1268,28 +1262,11 @@ class TestIndexingRunnerRun:
# Arrange
runner = IndexingRunner()
docs = sample_dataset_documents
-
- # Mock database
- mock_dataset = Dataset(
- indexing_technique=IndexTechniqueType.ECONOMY,
- )
- mock_current_user = Account(name="Test Account", email="[email protected]")
-
- doc_map = {doc.id: doc for doc in docs}
- model_dispatch = {"Dataset": mock_dataset, "Account": mock_current_user}
-
- def get_side_effect(model_class, id):
- name = model_class.__name__
- if name == "Document":
- return doc_map.get(id)
- return model_dispatch.get(name)
-
- mock_dependencies["session"].get.side_effect = get_side_effect
-
- mock_process_rule = DatasetProcessRule(
- dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
- )
- mock_dependencies["session"].scalar.return_value = mock_process_rule
+ for doc in docs:
+ dataset = mock_dependencies["session"].get(Dataset, doc.dataset_id)
+ assert dataset is not None
+ dataset.indexing_technique = IndexTechniqueType.ECONOMY
+ mock_dependencies["session"].commit()
mock_processor = MagicMock()
mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor
@@ -1329,13 +1306,13 @@ class TestIndexingRunnerRetryLogic:
"""
@pytest.fixture
- def mock_dependencies(self):
+ def mock_dependencies(self, sqlite_session: Session):
"""Mock all external dependencies."""
with (
patch("core.indexing_runner.redis_client") as mock_redis,
):
yield {
- "session": MagicMock(),
+ "session": sqlite_session,
"redis": mock_redis,
}
@@ -1360,41 +1337,62 @@ class TestIndexingRunnerRetryLogic:
def test_update_document_index_status_success(self, mock_dependencies):
"""Test successful document status update."""
- # Arrange
document_id = str(uuid.uuid4())
- mock_document = DatasetDocument(id=document_id)
-
- mock_dependencies["session"].scalar.return_value = 0
- mock_dependencies["session"].get.return_value = mock_document
+ session = mock_dependencies["session"]
+ document = DatasetDocument(
+ id=document_id,
+ tenant_id=str(uuid.uuid4()),
+ dataset_id=str(uuid.uuid4()),
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch="batch",
+ name="Document",
+ created_from=DocumentCreatedFrom.API,
+ created_by=str(uuid.uuid4()),
+ is_paused=False,
+ )
+ session.add(document)
+ session.commit()
# Act
IndexingRunner._update_document_index_status(
document_id,
"completed",
{"tokens": 100, "completed_at": naive_utc_now()},
- session=mock_dependencies["session"],
+ session=session,
)
- # Assert
- mock_dependencies["session"].flush.assert_called()
+ session.refresh(document)
+ assert document.indexing_status == "completed"
+ assert document.tokens == 100
def test_update_document_index_status_paused(self, mock_dependencies):
"""Test document status update when document is paused."""
- # Arrange
document_id = str(uuid.uuid4())
- mock_dependencies["session"].scalar.return_value = 1
+ session = mock_dependencies["session"]
+ document = DatasetDocument(
+ id=document_id,
+ tenant_id=str(uuid.uuid4()),
+ dataset_id=str(uuid.uuid4()),
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch="batch",
+ name="Paused document",
+ created_from=DocumentCreatedFrom.API,
+ created_by=str(uuid.uuid4()),
+ is_paused=True,
+ )
+ session.add(document)
+ session.commit()
# Act & Assert
with pytest.raises(DocumentIsPausedError):
- IndexingRunner._update_document_index_status(document_id, "completed", session=mock_dependencies["session"])
+ IndexingRunner._update_document_index_status(document_id, "completed", session=session)
def test_update_document_index_status_deleted(self, mock_dependencies):
"""Test document status update when document is deleted."""
# Arrange
document_id = str(uuid.uuid4())
- mock_dependencies["session"].scalar.return_value = 0
- mock_dependencies["session"].get.return_value = None
-
# Act & Assert
with pytest.raises(DocumentIsDeletedPausedError):
IndexingRunner._update_document_index_status(document_id, "completed", session=mock_dependencies["session"])
@@ -1610,13 +1608,13 @@ class TestIndexingRunnerLoadSegments:
"""
@pytest.fixture
- def mock_dependencies(self):
+ def mock_dependencies(self, unbound_session: Session):
"""Mock all external dependencies."""
with (
patch("core.indexing_runner.DatasetDocumentStore") as mock_docstore,
):
yield {
- "session": MagicMock(),
+ "session": unbound_session,
"docstore": mock_docstore,
}
@@ -1774,13 +1772,13 @@ class TestIndexingRunnerEstimate:
"""
@pytest.fixture
- def mock_dependencies(self):
+ def mock_dependencies(self, sqlite_session: Session):
"""Mock all external dependencies."""
with (
patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,
):
yield {
- "session": MagicMock(),
+ "session": sqlite_session,
"factory": mock_factory,
}
@@ -1813,7 +1811,8 @@ class TestIndexingRunnerEstimate:
mock_processor = MagicMock()
mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor
phase_events: list[str] = []
- mock_dependencies["session"].commit.side_effect = lambda: phase_events.append("commit")
+ session = mock_dependencies["session"]
+ event.listen(session, "after_commit", lambda _session: phase_events.append("commit"))
preview_doc = Document(
page_content="![image](http://files.local/files/image-1/file-preview)",
@@ -1825,8 +1824,23 @@ class TestIndexingRunnerEstimate:
phase_events.append("summary") or [PreviewDetail(content=preview_doc.page_content)]
)
- image_file = SimpleNamespace(key="image_files/tenant-1/source-file-1/image.png")
- mock_dependencies["session"].scalar.return_value = image_file
+ image_file = UploadFile(
+ tenant_id=tenant_id,
+ storage_type=StorageType.LOCAL,
+ key="image_files/tenant-1/source-file-1/image.png",
+ name="image.png",
+ size=10,
+ extension="png",
+ mime_type="image/png",
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by="user-id",
+ created_at=datetime.now(UTC),
+ used=True,
+ )
+ image_file.id = "image-1"
+ session.add(image_file)
+ session.commit()
+ phase_events.clear()
with (
patch("core.indexing_runner.get_image_upload_file_ids", return_value=["image-1"]),
@@ -1844,12 +1858,12 @@ class TestIndexingRunnerEstimate:
"summary_index_setting": {"enable": True},
},
doc_form=IndexStructureType.PARAGRAPH_INDEX,
- session=mock_dependencies["session"],
+ session=session,
)
assert result.total_segments == 1
mock_storage.delete.assert_called_once_with(image_file.key)
- mock_dependencies["session"].delete.assert_called_once_with(image_file)
+ assert session.get(UploadFile, image_file.id) is None
assert phase_events == ["commit", "summary"]
@@ -1863,13 +1877,13 @@ class TestIndexingRunnerProcessChunk:
"""
@pytest.fixture
- def mock_dependencies(self):
+ def mock_dependencies(self, sqlite_session: Session):
"""Mock all external dependencies."""
with (
patch("core.indexing_runner.redis_client") as mock_redis,
):
yield {
- "session": MagicMock(),
+ "session": sqlite_session,
"redis": mock_redis,
}
@@ -1881,7 +1895,12 @@ class TestIndexingRunnerProcessChunk:
app.app_context.return_value.__exit__ = MagicMock()
return app
- def test_process_chunk_loads_index_and_completes_segments(self, mock_dependencies, mock_flask_app):
+ def test_process_chunk_loads_index_and_completes_segments(
+ self,
+ mock_dependencies,
+ mock_flask_app,
+ sqlite_session_factory: sessionmaker[Session],
+ ):
"""Test process chunk loads the index and completes segments without counting tokens."""
# Arrange
from core.indexing_runner import IndexingRunner
@@ -1893,33 +1912,36 @@ class TestIndexingRunnerProcessChunk:
Document(page_content="Chunk 2", metadata={"doc_id": "c2"}),
]
- mock_dataset = Dataset(
- id=str(uuid.uuid4()),
- )
-
- mock_dataset_document = DatasetDocument(id=str(uuid.uuid4()))
+ session = mock_dependencies["session"]
+ mock_dataset, mock_dataset_document = persist_indexing_scope(session)
+ segments = [
+ DocumentSegment(
+ tenant_id=mock_dataset.tenant_id,
+ dataset_id=mock_dataset.id,
+ document_id=mock_dataset_document.id,
+ position=position,
+ content=f"Chunk {position}",
+ word_count=1,
+ tokens=1,
+ created_by=mock_dataset.created_by,
+ index_node_id=doc.metadata["doc_id"],
+ status=SegmentStatus.INDEXING,
+ )
+ for position, doc in enumerate(chunk_documents, start=1)
+ ]
+ session.add_all(segments)
+ session.commit()
mock_dependencies["redis"].get.return_value = None
- # Mock database update for segment status
- mock_dependencies["session"].execute.return_value = None
- mock_dependencies["session"].get.side_effect = lambda model, _id: {
- Dataset: mock_dataset,
- DatasetDocument: mock_dataset_document,
- }.get(model)
-
# Create a proper context manager mock
mock_context = MagicMock()
mock_context.__enter__ = MagicMock(return_value=None)
mock_context.__exit__ = MagicMock(return_value=None)
mock_flask_app.app_context.return_value = mock_context
- session_context = MagicMock()
- session_context.__enter__.return_value = mock_dependencies["session"]
- session_context.__exit__.return_value = None
-
with (
- patch("core.indexing_runner.session_factory.create_session", return_value=session_context),
+ patch("core.indexing_runner.session_factory.create_session", sqlite_session_factory),
patch("core.indexing_runner.IndexProcessorFactory") as mock_factory,
):
mock_factory.return_value.init_index_processor.return_value = mock_processor
@@ -1936,10 +1958,15 @@ class TestIndexingRunnerProcessChunk:
# Assert
assert result is None
mock_processor.load.assert_called_once()
- mock_dependencies["session"].execute.assert_called_once()
- mock_dependencies["session"].commit.assert_called_once()
-
- def test_process_chunk_detects_pause(self, mock_dependencies, mock_flask_app):
+ session.expire_all()
+ assert all(session.get(DocumentSegment, segment.id).status == SegmentStatus.COMPLETED for segment in segments)
+
+ def test_process_chunk_detects_pause(
+ self,
+ mock_dependencies,
+ mock_flask_app,
+ sqlite_session_factory: sessionmaker[Session],
+ ):
"""Test process chunk detects document pause."""
# Arrange
from core.indexing_runner import IndexingRunner
@@ -1947,27 +1974,17 @@ class TestIndexingRunnerProcessChunk:
runner = IndexingRunner()
chunk_documents = [Document(page_content="Chunk", metadata={"doc_id": "c1"})]
- mock_dataset = Dataset()
- mock_dataset_document = DatasetDocument(id=str(uuid.uuid4()))
+ mock_dataset, mock_dataset_document = persist_indexing_scope(mock_dependencies["session"])
# Mock Redis to return paused status
mock_dependencies["redis"].get.return_value = "1"
- mock_dependencies["session"].get.side_effect = lambda model, _id: {
- Dataset: mock_dataset,
- DatasetDocument: mock_dataset_document,
- }.get(model)
-
# Create a proper context manager mock
mock_context = MagicMock()
mock_context.__enter__ = MagicMock(return_value=None)
mock_context.__exit__ = MagicMock(return_value=None)
mock_flask_app.app_context.return_value = mock_context
- session_context = MagicMock()
- session_context.__enter__.return_value = mock_dependencies["session"]
- session_context.__exit__.return_value = None
-
- with patch("core.indexing_runner.session_factory.create_session", return_value=session_context):
+ with patch("core.indexing_runner.session_factory.create_session", sqlite_session_factory):
# Act & Assert - the method creates its own app_context and session
with pytest.raises(DocumentIsPausedError):
runner._process_chunk(