Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: use real ORM models in RAG indexing tests (#40017)
Asuka Minatoac0c909d1154
diff --git a/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py b/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.pyindex 6a89e6ec5c8..589d29d8240 100644--- a/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py+++ b/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py@@ -158,11 +158,12 @@ class _SimpleRetrievalSegment:class TestRetrievalServiceInternals:@pytest.fixturedef internal_dataset(self) -> Dataset:- dataset = Mock(spec=Dataset)- dataset.id = "dataset-id"- dataset.tenant_id = "tenant-id"- dataset.is_multimodal = False- dataset.doc_form = IndexStructureType.PARENT_CHILD_INDEX+ dataset = Dataset(+ id="dataset-id",+ tenant_id="tenant-id",+ is_multimodal=False,+ chunk_structure=IndexStructureType.PARENT_CHILD_INDEX,+ )return dataset@pytest.fixture@@ -264,7 +265,7 @@ class TestRetrievalServiceInternals:@patch("core.rag.datasource.retrieval_service.Session")def test_get_dataset_queries_by_id(self, mock_session_class):- expected_dataset = Mock(spec=Dataset)+ expected_dataset = Dataset()mock_session = Mock()mock_session.scalar.return_value = expected_datasetmock_session_class.return_value.__enter__.return_value = mock_sessiondiff --git a/api/tests/unit_tests/core/rag/docstore/test_dataset_docstore.py b/api/tests/unit_tests/core/rag/docstore/test_dataset_docstore.pyindex b7c51c77427..366bbcbd501 100644--- a/api/tests/unit_tests/core/rag/docstore/test_dataset_docstore.py+++ b/api/tests/unit_tests/core/rag/docstore/test_dataset_docstore.py@@ -22,9 +22,10 @@ USER_ID = "00000000-0000-0000-0000-000000000004"def _dataset() -> Dataset:- dataset = MagicMock(spec=Dataset)- dataset.id = DATASET_ID- dataset.tenant_id = TENANT_ID+ dataset = Dataset(+ id=DATASET_ID,+ tenant_id=TENANT_ID,+ )return dataset@@ -36,7 +37,25 @@ def _persist_segment(content: str = "Test content",tokens: int = 5,) -> DocumentSegment:- segment = DocumentSegment(+ segment = _segment(+ index_node_id=index_node_id,+ index_node_hash=index_node_hash,+ content=content,+ tokens=tokens,+ )+ session.add(segment)+ session.flush()+ return segment+++def _segment(+ *,+ index_node_id: str = "doc-1",+ index_node_hash: str = "hash-1",+ content: str = "Test content",+ tokens: int = 5,+) -> DocumentSegment:+ return DocumentSegment(tenant_id=TENANT_ID,dataset_id=DATASET_ID,document_id=DOCUMENT_ID,@@ -48,9 +67,6 @@ def _persist_segment(index_node_id=index_node_id,index_node_hash=index_node_hash,)- session.add(segment)- session.flush()- return segmentclass TestDatasetDocumentStoreInit:@@ -59,8 +75,9 @@ class TestDatasetDocumentStoreInit:def test_init_with_all_parameters(self):"""Test initialization with dataset, user_id, and document_id."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )store = DatasetDocumentStore(dataset=mock_dataset,@@ -77,8 +94,9 @@ class TestDatasetDocumentStoreInit:def test_init_without_document_id(self):"""Test initialization without document_id."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )store = DatasetDocumentStore(dataset=mock_dataset,@@ -95,8 +113,9 @@ class TestDatasetDocumentStoreSerialization:def test_to_dict(self):"""Test serialization to dictionary."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )store = DatasetDocumentStore(dataset=mock_dataset,@@ -129,15 +148,11 @@ class TestDatasetDocumentStoreDocs:def test_docs_returns_document_dict(self):"""Test that docs property returns a dictionary of documents."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )- mock_segment = MagicMock(spec=DocumentSegment)- mock_segment.index_node_id = "node-1"- mock_segment.index_node_hash = "hash-1"- mock_segment.document_id = "doc-1"- mock_segment.dataset_id = "test-dataset-id"- mock_segment.content = "Test content"+ mock_segment = _segment(index_node_id="node-1")mock_session = MagicMock()mock_session.scalars.return_value.all.return_value = [mock_segment]@@ -155,8 +170,9 @@ class TestDatasetDocumentStoreDocs:def test_docs_empty_dataset(self):"""Test docs property with no segments."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_session = MagicMock()mock_session.scalars.return_value.all.return_value = []@@ -326,8 +342,9 @@ class TestDatasetDocumentStoreExists:def test_document_exists_returns_true(self):"""Test document_exists returns True when segment exists."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_segment = MagicMock()mock_session = MagicMock()@@ -345,8 +362,9 @@ class TestDatasetDocumentStoreExists:def test_document_exists_returns_false(self):"""Test document_exists returns False when segment doesn't exist."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_session = MagicMock()with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):@@ -366,15 +384,11 @@ class TestDatasetDocumentStoreGetDocument:def test_get_document_success(self):"""Test getting a document successfully."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )- mock_segment = MagicMock(spec=DocumentSegment)- mock_segment.index_node_id = "node-1"- mock_segment.index_node_hash = "hash-1"- mock_segment.document_id = "doc-1"- mock_segment.dataset_id = "test-dataset-id"- mock_segment.content = "Test content"+ mock_segment = _segment(index_node_id="node-1")mock_session = MagicMock()with patch.object(DatasetDocumentStore, "get_document_segment", return_value=mock_segment):@@ -391,8 +405,9 @@ class TestDatasetDocumentStoreGetDocument:def test_get_document_returns_none_when_not_found(self):"""Test get_document returns None when not found and raise_error=False."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_session = MagicMock()with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):@@ -408,8 +423,9 @@ class TestDatasetDocumentStoreGetDocument:def test_get_document_raises_when_not_found(self):"""Test get_document raises ValueError when not found and raise_error=True."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_session = MagicMock()with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):@@ -428,8 +444,9 @@ class TestDatasetDocumentStoreDeleteDocument:def test_delete_document_success(self):"""Test deleting a document successfully."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_segment = MagicMock()mock_session = MagicMock()@@ -448,8 +465,9 @@ class TestDatasetDocumentStoreDeleteDocument:def test_delete_document_returns_none_when_not_found(self):"""Test delete_document returns None when not found and raise_error=False."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_session = MagicMock()with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):@@ -465,8 +483,9 @@ class TestDatasetDocumentStoreDeleteDocument:def test_delete_document_raises_when_not_found(self):"""Test delete_document raises ValueError when not found and raise_error=True."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_session = MagicMock()with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):@@ -485,8 +504,9 @@ class TestDatasetDocumentStoreHashOperations:def test_set_document_hash_success(self):"""Test setting document hash successfully."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_segment = MagicMock()mock_segment.index_node_hash = "old-hash"@@ -506,8 +526,9 @@ class TestDatasetDocumentStoreHashOperations:def test_set_document_hash_returns_none_when_not_found(self):"""Test set_document_hash returns None when segment not found."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_session = MagicMock()with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):@@ -523,8 +544,9 @@ class TestDatasetDocumentStoreHashOperations:def test_get_document_hash_success(self):"""Test getting document hash successfully."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_segment = MagicMock()mock_segment.index_node_hash = "test-hash"@@ -543,8 +565,9 @@ class TestDatasetDocumentStoreHashOperations:def test_get_document_hash_returns_none_when_not_found(self):"""Test get_document_hash returns None when segment not found."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_session = MagicMock()with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):@@ -564,10 +587,11 @@ class TestDatasetDocumentStoreSegment:def test_get_document_segment_returns_segment(self):"""Test getting a document segment."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )- mock_segment = MagicMock(spec=DocumentSegment)+ mock_segment = _segment()mock_session = MagicMock()mock_session.scalar.return_value = mock_segment@@ -584,8 +608,9 @@ class TestDatasetDocumentStoreSegment:def test_get_document_segment_returns_none(self):"""Test getting a non-existent document segment."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"+ mock_dataset = Dataset(+ id="test-dataset-id",+ )mock_session = MagicMock()mock_session.scalar.return_value = None@@ -606,9 +631,10 @@ class TestDatasetDocumentStoreMultimodelBinding:def test_add_multimodel_documents_binding_with_attachments(self):"""Test adding multimodel document bindings."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"- mock_dataset.tenant_id = "tenant-1"+ mock_dataset = Dataset(+ id="test-dataset-id",+ tenant_id="tenant-1",+ )mock_attachment = MagicMock(spec=AttachmentDocument)mock_attachment.metadata = {"doc_id": "attachment-1"}@@ -628,9 +654,10 @@ class TestDatasetDocumentStoreMultimodelBinding:def test_add_multimodel_documents_binding_without_attachments(self):"""Test adding bindings with None attachments."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"- mock_dataset.tenant_id = "tenant-1"+ mock_dataset = Dataset(+ id="test-dataset-id",+ tenant_id="tenant-1",+ )mock_session = MagicMock()@@ -647,9 +674,10 @@ class TestDatasetDocumentStoreMultimodelBinding:def test_add_multimodel_documents_binding_with_empty_list(self):"""Test adding bindings with empty list."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"- mock_dataset.tenant_id = "tenant-1"+ mock_dataset = Dataset(+ id="test-dataset-id",+ tenant_id="tenant-1",+ )mock_session = MagicMock()@@ -666,9 +694,10 @@ class TestDatasetDocumentStoreMultimodelBinding:def test_add_multimodel_documents_binding_with_none_document_id(self):"""Test that no bindings are added when document_id is None."""- mock_dataset = MagicMock(spec=Dataset)- mock_dataset.id = "test-dataset-id"- mock_dataset.tenant_id = "tenant-1"+ mock_dataset = Dataset(+ id="test-dataset-id",+ tenant_id="tenant-1",+ )mock_attachment = MagicMock(spec=AttachmentDocument)mock_attachment.metadata = {"doc_id": "attachment-1"}diff --git a/api/tests/unit_tests/core/rag/embedding/test_token_counter.py b/api/tests/unit_tests/core/rag/embedding/test_token_counter.pyindex 0c134a94c90..733f55fa16c 100644--- a/api/tests/unit_tests/core/rag/embedding/test_token_counter.py+++ b/api/tests/unit_tests/core/rag/embedding/test_token_counter.py@@ -1,4 +1,4 @@-from unittest.mock import Mock, patch+from unittest.mock import patchfrom core.rag.embedding.token_counter import calculate_segment_token_countsfrom core.rag.index_processor.constant.index_type import IndexTechniqueType@@ -7,11 +7,12 @@ from models.dataset import Datasetdef test_high_quality_counts_each_document_once() -> None:- dataset = Mock(spec=Dataset)- dataset.tenant_id = "tenant-1"- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY- dataset.embedding_model_provider = "provider"- dataset.embedding_model = "model"+ dataset = Dataset(+ tenant_id="tenant-1",+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ embedding_model_provider="provider",+ embedding_model="model",+ )documents = [Document(page_content="first", metadata={}),Document(page_content="second", metadata={}),@@ -31,8 +32,9 @@ def test_high_quality_counts_each_document_once() -> None:def test_economy_returns_zero_without_loading_model() -> None:- dataset = Mock(spec=Dataset)- dataset.indexing_technique = IndexTechniqueType.ECONOMY+ dataset = Dataset(+ indexing_technique=IndexTechniqueType.ECONOMY,+ )documents = [Document(page_content="first", metadata={}),Document(page_content="second", metadata={}),@@ -46,8 +48,9 @@ def test_economy_returns_zero_without_loading_model() -> None:def test_empty_documents_return_without_loading_model() -> None:- dataset = Mock(spec=Dataset)- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY+ dataset = Dataset(+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ )with patch("core.rag.embedding.token_counter.ModelManager.for_tenant") as model_manager_factory:result = calculate_segment_token_counts(dataset=dataset, documents=[])diff --git a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.pyindex ef1b38f49d7..c129e51c5ae 100644--- a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py+++ b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py@@ -83,10 +83,10 @@ def create_mock_dataset(indexing_technique: str = IndexTechniqueType.HIGH_QUALITY,embedding_provider: str = "openai",embedding_model: str = "text-embedding-ada-002",-) -> Mock:- """Create a mock Dataset object with configurable parameters.+) -> Dataset:+ """Create a Dataset object with configurable parameters.- This helper function creates a properly configured mock Dataset object that can be+ This helper function creates a properly configured Dataset object that can beused across multiple tests, ensuring consistency in test data.Args:@@ -97,18 +97,19 @@ def create_mock_dataset(embedding_model: The embedding model name.Returns:- Mock: A configured mock Dataset object with all required attributes.+ Dataset: A configured Dataset object with all required attributes.Example:>>> dataset = create_mock_dataset(indexing_technique="economy")>>> assert dataset.indexing_technique == "economy""""- dataset = Mock(spec=Dataset)- dataset.id = dataset_id or str(uuid.uuid4())- dataset.tenant_id = tenant_id or str(uuid.uuid4())- dataset.indexing_technique = indexing_technique- dataset.embedding_model_provider = embedding_provider- dataset.embedding_model = embedding_model+ dataset = Dataset(+ id=dataset_id or str(uuid.uuid4()),+ tenant_id=tenant_id or str(uuid.uuid4()),+ indexing_technique=indexing_technique,+ embedding_model_provider=embedding_provider,+ embedding_model=embedding_model,+ )return dataset@@ -119,10 +120,10 @@ def create_mock_dataset_document(doc_form: str = IndexStructureType.PARAGRAPH_INDEX,data_source_type: str = "upload_file",doc_language: str = "English",-) -> Mock:- """Create a mock DatasetDocument object with configurable parameters.+) -> DatasetDocument:+ """Create a DatasetDocument object with configurable parameters.- This helper function creates a properly configured mock DatasetDocument object,+ This helper function creates a properly configured DatasetDocument object,reducing boilerplate code in individual tests.Args:@@ -134,23 +135,23 @@ def create_mock_dataset_document(doc_language: The document language.Returns:- Mock: A configured mock DatasetDocument object with all required attributes.+ DatasetDocument: A configured DatasetDocument object with all required attributes.Example:>>> doc = create_mock_dataset_document(doc_form=IndexStructureType.QA_INDEX)>>> assert doc.doc_form == IndexStructureType.QA_INDEX"""- doc = Mock(spec=DatasetDocument)- doc.id = document_id or str(uuid.uuid4())- doc.dataset_id = dataset_id or str(uuid.uuid4())- doc.tenant_id = tenant_id or str(uuid.uuid4())- doc.doc_form = doc_form- doc.doc_language = doc_language- doc.data_source_type = data_source_type- doc.data_source_info_dict = {"upload_file_id": str(uuid.uuid4())}- doc.dataset_process_rule_id = str(uuid.uuid4())- doc.created_by = str(uuid.uuid4())- return doc+ return DatasetDocument(+ id=document_id or str(uuid.uuid4()),+ dataset_id=dataset_id or str(uuid.uuid4()),+ tenant_id=tenant_id or str(uuid.uuid4()),+ doc_form=doc_form,+ doc_language=doc_language,+ data_source_type=data_source_type,+ data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),+ dataset_process_rule_id=str(uuid.uuid4()),+ created_by=str(uuid.uuid4()),+ )def create_sample_documents(@@ -275,14 +276,14 @@ class TestIndexingRunnerExtract:@pytest.fixturedef sample_dataset_document(self):"""Create a sample dataset document for testing."""- doc = Mock(spec=DatasetDocument)- doc.id = str(uuid.uuid4())- doc.dataset_id = str(uuid.uuid4())- doc.tenant_id = str(uuid.uuid4())- doc.doc_form = IndexStructureType.PARAGRAPH_INDEX- doc.data_source_type = "upload_file"- doc.data_source_info_dict = {"upload_file_id": str(uuid.uuid4())}- return doc+ return DatasetDocument(+ id=str(uuid.uuid4()),+ dataset_id=str(uuid.uuid4()),+ tenant_id=str(uuid.uuid4()),+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ data_source_type="upload_file",+ data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),+ )@pytest.fixturedef sample_process_rule(self):@@ -353,12 +354,14 @@ class TestIndexingRunnerExtract:# Arrangerunner = IndexingRunner()sample_dataset_document.data_source_type = "notion_import"- sample_dataset_document.data_source_info_dict = {- "credential_id": str(uuid.uuid4()),- "notion_workspace_id": "workspace123",- "notion_page_id": "page123",- "type": "page",- }+ sample_dataset_document.data_source_info = json.dumps(+ {+ "credential_id": str(uuid.uuid4()),+ "notion_workspace_id": "workspace123",+ "notion_page_id": "page123",+ "type": "page",+ }+ )mock_processor = MagicMock()mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor@@ -383,13 +386,15 @@ class TestIndexingRunnerExtract:# Arrangerunner = IndexingRunner()sample_dataset_document.data_source_type = "website_crawl"- sample_dataset_document.data_source_info_dict = {- "provider": "firecrawl",- "url": "https://example.com",- "job_id": "job123",- "mode": "crawl",- "only_main_content": True,- }+ sample_dataset_document.data_source_info = json.dumps(+ {+ "provider": "firecrawl",+ "url": "https://example.com",+ "job_id": "job123",+ "mode": "crawl",+ "only_main_content": True,+ }+ )mock_processor = MagicMock()mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor@@ -415,7 +420,7 @@ class TestIndexingRunnerExtract:"""Test extraction fails when upload file is missing."""# Arrangerunner = IndexingRunner()- sample_dataset_document.data_source_info_dict = {}+ sample_dataset_document.data_source_info = "{}"mock_processor = MagicMock()mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor@@ -466,12 +471,13 @@ class TestIndexingRunnerTransform:@pytest.fixturedef sample_dataset(self):"""Create a sample dataset for testing."""- dataset = Mock(spec=Dataset)- dataset.id = str(uuid.uuid4())- dataset.tenant_id = str(uuid.uuid4())- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY- dataset.embedding_model_provider = "openai"- dataset.embedding_model = "text-embedding-ada-002"+ dataset = Dataset(+ id=str(uuid.uuid4()),+ tenant_id=str(uuid.uuid4()),+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ embedding_model_provider="openai",+ embedding_model="text-embedding-ada-002",+ )return dataset@pytest.fixture@@ -637,22 +643,23 @@ class TestIndexingRunnerLoad:@pytest.fixturedef sample_dataset(self):"""Create a sample dataset for testing."""- dataset = Mock(spec=Dataset)- dataset.id = str(uuid.uuid4())- dataset.tenant_id = str(uuid.uuid4())- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY- dataset.embedding_model_provider = "openai"- dataset.embedding_model = "text-embedding-ada-002"+ dataset = Dataset(+ id=str(uuid.uuid4()),+ tenant_id=str(uuid.uuid4()),+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ embedding_model_provider="openai",+ embedding_model="text-embedding-ada-002",+ )return dataset@pytest.fixturedef sample_dataset_document(self):"""Create a sample dataset document for testing."""- doc = Mock(spec=DatasetDocument)- doc.id = str(uuid.uuid4())- doc.dataset_id = str(uuid.uuid4())- doc.doc_form = IndexStructureType.PARAGRAPH_INDEX- return doc+ return DatasetDocument(+ id=str(uuid.uuid4()),+ dataset_id=str(uuid.uuid4()),+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ )@pytest.fixturedef sample_documents(self):@@ -842,16 +849,18 @@ class TestIndexingRunnerRun:"""Create sample dataset documents for testing."""docs = []for i in range(2):- doc = Mock(spec=DatasetDocument)- doc.id = str(uuid.uuid4())- doc.dataset_id = str(uuid.uuid4())- doc.tenant_id = str(uuid.uuid4())- doc.doc_form = IndexStructureType.PARAGRAPH_INDEX- doc.doc_language = "English"- doc.data_source_type = "upload_file"- doc.data_source_info_dict = {"upload_file_id": str(uuid.uuid4())}- doc.dataset_process_rule_id = str(uuid.uuid4())- docs.append(doc)+ docs.append(+ DatasetDocument(+ id=str(uuid.uuid4()),+ dataset_id=str(uuid.uuid4()),+ tenant_id=str(uuid.uuid4()),+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ doc_language="English",+ data_source_type="upload_file",+ data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),+ dataset_process_rule_id=str(uuid.uuid4()),+ )+ )return docsdef test_run_in_indexing_status_loads_child_chunks_with_caller_session(@@ -860,44 +869,63 @@ class TestIndexingRunnerRun:runner = IndexingRunner()dataset_document = sample_dataset_documents[0]dataset_document.doc_form = IndexStructureType.PARENT_CHILD_INDEX- dataset = Mock(spec=Dataset)- segment = Mock(spec=DocumentSegment)- segment.status = "waiting"- segment.content = "parent"- segment.index_node_id = "parent-node"- segment.index_node_hash = "parent-hash"- segment.document_id = dataset_document.id- segment.dataset_id = dataset_document.dataset_id- segment.tokens = 12- segment.get_child_chunks.return_value = [- SimpleNamespace(content="child", index_node_id="child-node", index_node_hash="child-hash")- ]+ dataset = Dataset()+ segment = DocumentSegment(+ tenant_id="tenant-id",+ dataset_id=dataset_document.dataset_id,+ document_id=dataset_document.id,+ position=1,+ content="parent",+ word_count=0,+ tokens=12,+ created_by="account-id",+ status="waiting",+ index_node_id="parent-node",+ index_node_hash="parent-hash",+ )+ child_chunks = [SimpleNamespace(content="child", index_node_id="child-node", index_node_hash="child-hash")]session = mock_dependencies["session"]session.get.side_effect = lambda model, _: dataset_document if model is DatasetDocument else datasetsession.scalars.return_value.all.return_value = [segment]- with patch.object(runner, "_load") as load:+ with (+ patch.object(DocumentSegment, "get_child_chunks", return_value=child_chunks) as get_child_chunks,+ patch.object(runner, "_load") as load,+ ):runner.run_in_indexing_status(dataset_document, session)- segment.get_child_chunks.assert_called_once_with(session=session)+ get_child_chunks.assert_called_once_with(session=session)assert load.call_args.kwargs["documents"][0].children[0].page_content == "child"assert load.call_args.kwargs["total_tokens"] == 12def test_run_in_indexing_status_uses_tokens_from_all_segments(self, mock_dependencies, sample_dataset_documents):runner = IndexingRunner()dataset_document = sample_dataset_documents[0]- dataset = Mock(spec=Dataset)- completed_segment = Mock(spec=DocumentSegment)- completed_segment.status = SegmentStatus.COMPLETED- completed_segment.tokens = 10- incomplete_segment = Mock(spec=DocumentSegment)- incomplete_segment.status = SegmentStatus.WAITING- incomplete_segment.tokens = 20- incomplete_segment.content = "pending"- incomplete_segment.index_node_id = "pending-node"- incomplete_segment.index_node_hash = "pending-hash"- incomplete_segment.document_id = dataset_document.id- incomplete_segment.dataset_id = dataset_document.dataset_id+ dataset = Dataset()+ completed_segment = DocumentSegment(+ tenant_id="tenant-id",+ dataset_id="dataset-id",+ document_id="document-id",+ position=1,+ content="",+ word_count=0,+ tokens=10,+ created_by="account-id",+ status=SegmentStatus.COMPLETED,+ )+ incomplete_segment = DocumentSegment(+ tenant_id="tenant-id",+ dataset_id=dataset_document.dataset_id,+ document_id=dataset_document.id,+ position=1,+ content="pending",+ word_count=0,+ tokens=20,+ created_by="account-id",+ status=SegmentStatus.WAITING,+ index_node_id="pending-node",+ index_node_hash="pending-hash",+ )session = mock_dependencies["session"]session.get.side_effect = lambda model, _: dataset_document if model is DatasetDocument else datasetsession.scalars.return_value.all.return_value = [completed_segment, incomplete_segment]@@ -908,25 +936,28 @@ class TestIndexingRunnerRun:assert load.call_args.kwargs["documents"][0].page_content == "pending"assert load.call_args.kwargs["total_tokens"] == 30- def test_run_success_single_document(self, mock_dependencies, sample_dataset_documents):+ @patch.object(Account, "set_tenant_id_with_session", autospec=True)+ def test_run_success_single_document(self, set_tenant_id, mock_dependencies, sample_dataset_documents):"""Test successful run with single document."""# Arrangerunner = IndexingRunner()doc = sample_dataset_documents[0]# Mock database queries- mock_dataset = Mock(spec=Dataset)- mock_dataset.id = doc.dataset_id- mock_dataset.tenant_id = doc.tenant_id- mock_dataset.indexing_technique = IndexTechniqueType.ECONOMY+ mock_dataset = Dataset(+ id=doc.dataset_id,+ tenant_id=doc.tenant_id,+ indexing_technique=IndexTechniqueType.ECONOMY,+ )- mock_current_user = Mock(spec=Account)+ mock_current_user = Account(name="Test Account", email="[email protected]")get_dispatch = {"Document": doc, "Dataset": mock_dataset, "Account": mock_current_user}mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)- mock_process_rule = Mock(spec=DatasetProcessRule)- mock_process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}+ mock_process_rule = DatasetProcessRule(+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"+ )mock_dependencies["session"].scalar.return_value = mock_process_rule# Mock processor@@ -963,7 +994,8 @@ class TestIndexingRunnerRun:# Assert - verify the methods were called# Since we're mocking the internal methods, we just verify no exceptions were raised- mock_current_user.set_tenant_id_with_session.assert_called_once_with(+ set_tenant_id.assert_called_once_with(+ mock_current_user,mock_dataset.tenant_id,session=mock_dependencies["session"],)@@ -998,14 +1030,18 @@ class TestIndexingRunnerRun:with pytest.raises(DocumentIsPausedError):runner.run([doc], mock_dependencies["session"])- def test_run_counts_each_transformed_document_once(self, mock_dependencies, sample_dataset_documents):+ @patch.object(Account, "set_tenant_id_with_session", autospec=True)+ def test_run_counts_each_transformed_document_once(+ self, set_tenant_id, mock_dependencies, sample_dataset_documents+ ):runner = IndexingRunner()dataset_document = sample_dataset_documents[0]- dataset = Mock(spec=Dataset)- dataset.id = dataset_document.dataset_id- dataset.tenant_id = dataset_document.tenant_id- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY- current_user = Mock(spec=Account)+ dataset = Dataset(+ id=dataset_document.dataset_id,+ tenant_id=dataset_document.tenant_id,+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ )+ current_user = Account(name="Test Account", email="[email protected]")transformed_documents = [Document(page_content="first", metadata={"doc_id": "first", "doc_hash": "hash-first"}),Document(page_content="second", metadata={"doc_id": "second", "doc_hash": "hash-second"}),@@ -1016,8 +1052,9 @@ class TestIndexingRunnerRun:Account: current_user,}mock_dependencies["session"].get.side_effect = lambda model, _: model_dispatch.get(model)- process_rule = Mock(spec=DatasetProcessRule)- process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}+ process_rule = DatasetProcessRule(+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"+ )mock_dependencies["session"].scalar.return_value = process_rulewith (@@ -1041,18 +1078,25 @@ class TestIndexingRunnerRun:token_counts=[11, 22],)assert load.call_args.kwargs["total_tokens"] == 33+ set_tenant_id.assert_called_once_with(+ current_user,+ dataset.tenant_id,+ session=mock_dependencies["session"],+ )+ @patch.object(Account, "set_tenant_id_with_session", autospec=True)def test_run_in_splitting_status_counts_each_transformed_document_once(- self, mock_dependencies, sample_dataset_documents+ self, set_tenant_id, mock_dependencies, sample_dataset_documents):runner = IndexingRunner()dataset_document = sample_dataset_documents[0]dataset_document.created_by = "user-1"- dataset = Mock(spec=Dataset)- dataset.id = dataset_document.dataset_id- dataset.tenant_id = dataset_document.tenant_id- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY- current_user = Mock(spec=Account)+ dataset = Dataset(+ id=dataset_document.dataset_id,+ tenant_id=dataset_document.tenant_id,+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ )+ current_user = Account(name="Test Account", email="[email protected]")transformed_documents = [Document(page_content="first", metadata={"doc_id": "first", "doc_hash": "hash-first"}),Document(page_content="second", metadata={"doc_id": "second", "doc_hash": "hash-second"}),@@ -1064,8 +1108,9 @@ class TestIndexingRunnerRun:}mock_dependencies["session"].get.side_effect = lambda model, _: model_dispatch.get(model)mock_dependencies["session"].scalars.return_value.all.return_value = []- process_rule = Mock(spec=DatasetProcessRule)- process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}+ process_rule = DatasetProcessRule(+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"+ )mock_dependencies["session"].scalar.return_value = process_rulewith (@@ -1089,6 +1134,11 @@ class TestIndexingRunnerRun:token_counts=[11, 22],)assert load.call_args.kwargs["total_tokens"] == 33+ set_tenant_id.assert_called_once_with(+ current_user,+ dataset.tenant_id,+ session=mock_dependencies["session"],+ )def test_run_handles_provider_token_error(self, mock_dependencies, sample_dataset_documents):"""Test run handles ProviderTokenNotInitError and updates document status."""@@ -1097,14 +1147,16 @@ class TestIndexingRunnerRun:doc = sample_dataset_documents[0]# Mock database- mock_dataset = Mock(spec=Dataset)- mock_dataset.tenant_id = doc.tenant_id+ mock_dataset = Dataset(+ tenant_id=doc.tenant_id,+ )get_dispatch = {"Document": doc, "Dataset": mock_dataset}mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)- mock_process_rule = Mock(spec=DatasetProcessRule)- mock_process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}+ mock_process_rule = DatasetProcessRule(+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"+ )mock_dependencies["session"].scalar.return_value = mock_process_rulemock_processor = MagicMock()@@ -1126,14 +1178,16 @@ class TestIndexingRunnerRun:doc = sample_dataset_documents[0]# Mock database- mock_dataset = Mock(spec=Dataset)- mock_dataset.tenant_id = doc.tenant_id+ mock_dataset = Dataset(+ tenant_id=doc.tenant_id,+ )get_dispatch = {"Document": doc, "Dataset": mock_dataset}mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)- mock_process_rule = Mock(spec=DatasetProcessRule)- mock_process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}+ mock_process_rule = DatasetProcessRule(+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"+ )mock_dependencies["session"].scalar.return_value = mock_process_rulemock_processor = MagicMock()@@ -1147,16 +1201,18 @@ class TestIndexingRunnerRun:# Assert - should not raise, just log warning# No exception should be raised- def test_run_processes_multiple_documents(self, mock_dependencies, sample_dataset_documents):+ @patch.object(Account, "set_tenant_id_with_session", autospec=True)+ def test_run_processes_multiple_documents(self, set_tenant_id, mock_dependencies, sample_dataset_documents):"""Test run processes multiple documents sequentially."""# Arrangerunner = IndexingRunner()docs = sample_dataset_documents# Mock database- mock_dataset = Mock(spec=Dataset)- mock_dataset.indexing_technique = IndexTechniqueType.ECONOMY- mock_current_user = Mock(spec=Account)+ mock_dataset = Dataset(+ indexing_technique=IndexTechniqueType.ECONOMY,+ )+ mock_current_user = Account(name="Test Account", email="[email protected]")doc_map = {doc.id: doc for doc in docs}model_dispatch = {"Dataset": mock_dataset, "Account": mock_current_user}@@ -1169,8 +1225,9 @@ class TestIndexingRunnerRun:mock_dependencies["session"].get.side_effect = get_side_effect- mock_process_rule = Mock(spec=DatasetProcessRule)- mock_process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}+ mock_process_rule = DatasetProcessRule(+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"+ )mock_dependencies["session"].scalar.return_value = mock_process_rulemock_processor = MagicMock()@@ -1197,7 +1254,7 @@ class TestIndexingRunnerRun:# Assert# Verify extract was called for each documentassert mock_extract.call_count == len(docs)- assert mock_current_user.set_tenant_id_with_session.call_count == len(docs)+ assert set_tenant_id.call_count == len(docs)class TestIndexingRunnerRetryLogic:@@ -1244,8 +1301,7 @@ class TestIndexingRunnerRetryLogic:"""Test successful document status update."""# Arrangedocument_id = str(uuid.uuid4())- mock_document = Mock(spec=DatasetDocument)- mock_document.id = document_id+ mock_document = DatasetDocument(id=document_id)mock_dependencies["session"].scalar.return_value = 0mock_dependencies["session"].get.return_value = mock_document@@ -1296,23 +1352,29 @@ class TestIndexingRunnerDocumentCleaning:@pytest.fixturedef sample_process_rule_automatic(self):"""Create automatic processing rule."""- rule = Mock(spec=DatasetProcessRule)- rule.mode = "automatic"- rule.rules = None+ rule = DatasetProcessRule(+ dataset_id="dataset-id",+ mode="automatic",+ rules=None,+ created_by="account-id",+ )return rule@pytest.fixturedef sample_process_rule_custom(self):"""Create custom processing rule."""- rule = Mock(spec=DatasetProcessRule)- rule.mode = "custom"- rule.rules = json.dumps(- {- "pre_processing_rules": [- {"id": "remove_extra_spaces", "enabled": True},- {"id": "remove_urls_emails", "enabled": True},- ]- }+ rule = DatasetProcessRule(+ dataset_id="dataset-id",+ mode="custom",+ rules=json.dumps(+ {+ "pre_processing_rules": [+ {"id": "remove_extra_spaces", "enabled": True},+ {"id": "remove_urls_emails", "enabled": True},+ ]+ }+ ),+ created_by="account-id",)return rule@@ -1500,20 +1562,21 @@ class TestIndexingRunnerLoadSegments:@pytest.fixturedef sample_dataset(self):"""Create sample dataset."""- dataset = Mock(spec=Dataset)- dataset.id = str(uuid.uuid4())- dataset.tenant_id = str(uuid.uuid4())+ dataset = Dataset(+ id=str(uuid.uuid4()),+ tenant_id=str(uuid.uuid4()),+ )return dataset@pytest.fixturedef sample_dataset_document(self):"""Create sample dataset document."""- doc = Mock(spec=DatasetDocument)- doc.id = str(uuid.uuid4())- doc.dataset_id = str(uuid.uuid4())- doc.created_by = str(uuid.uuid4())- doc.doc_form = IndexStructureType.PARAGRAPH_INDEX- return doc+ return DatasetDocument(+ id=str(uuid.uuid4()),+ dataset_id=str(uuid.uuid4()),+ created_by=str(uuid.uuid4()),+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ )@pytest.fixturedef sample_documents(self):@@ -1769,11 +1832,11 @@ class TestIndexingRunnerProcessChunk:Document(page_content="Chunk 2", metadata={"doc_id": "c2"}),]- mock_dataset = Mock(spec=Dataset)- mock_dataset.id = str(uuid.uuid4())+ mock_dataset = Dataset(+ id=str(uuid.uuid4()),+ )- mock_dataset_document = Mock(spec=DatasetDocument)- mock_dataset_document.id = str(uuid.uuid4())+ mock_dataset_document = DatasetDocument(id=str(uuid.uuid4()))mock_dependencies["redis"].get.return_value = None@@ -1823,9 +1886,8 @@ class TestIndexingRunnerProcessChunk:runner = IndexingRunner()chunk_documents = [Document(page_content="Chunk", metadata={"doc_id": "c1"})]- mock_dataset = Mock(spec=Dataset)- mock_dataset_document = Mock(spec=DatasetDocument)- mock_dataset_document.id = str(uuid.uuid4())+ mock_dataset = Dataset()+ mock_dataset_document = DatasetDocument(id=str(uuid.uuid4()))# Mock Redis to return paused statusmock_dependencies["redis"].get.return_value = "1"diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.pyindex 9e504dd1b9b..f9922643e96 100644--- a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py+++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py@@ -252,24 +252,25 @@ class TestRetrievalService:@pytest.fixturedef mock_dataset(self) -> Dataset:"""- Create a mock Dataset object for testing.+ Create a Dataset object for testing.Returns:- Dataset: Mock dataset with standard configuration- """- dataset = Mock(spec=Dataset)- dataset.id = str(uuid4())- dataset.tenant_id = str(uuid4())- dataset.name = "test_dataset"- dataset.indexing_technique = "high_quality"- dataset.embedding_model = "text-embedding-ada-002"- dataset.embedding_model_provider = "openai"- dataset.retrieval_model = {- "search_method": RetrievalMethod.SEMANTIC_SEARCH,- "reranking_enable": False,- "top_k": 4,- "score_threshold_enabled": False,- }+ Dataset with standard configuration.+ """+ dataset = Dataset(+ id=str(uuid4()),+ tenant_id=str(uuid4()),+ name="test_dataset",+ indexing_technique="high_quality",+ embedding_model="text-embedding-ada-002",+ embedding_model_provider="openai",+ retrieval_model={+ "search_method": RetrievalMethod.SEMANTIC_SEARCH,+ "reranking_enable": False,+ "top_k": 4,+ "score_threshold_enabled": False,+ },+ )return dataset@pytest.fixture@@ -1834,10 +1835,11 @@ class TestRetrievalService:all_documents = []# Create second dataset- mock_dataset2 = Mock(spec=Dataset)- mock_dataset2.id = str(uuid4())- mock_dataset2.indexing_technique = "high_quality"- mock_dataset2.provider = "dify"+ mock_dataset2 = Dataset(+ id=str(uuid4()),+ indexing_technique="high_quality",+ provider="dify",+ )# Act - Call with dataset_count = 2with _patched_retriever_session() as rerank_session:@@ -2207,36 +2209,33 @@ def create_mock_dataset_methods(tenant_id: str | None = None,provider: str = "dify",indexing_technique: str = "high_quality",- available_document_count: int = 10,-) -> Mock:+) -> Dataset:"""- Create a mock Dataset object for testing.+ Create a Dataset object for testing.Args:dataset_id: Unique identifier for the datasettenant_id: Tenant ID for the datasetprovider: Provider type ("dify" or "external")indexing_technique: Indexing technique ("high_quality" or "economy")- available_document_count: Number of available documents-Returns:- Mock: A properly configured Dataset mock+ A configured Dataset model."""- dataset = Mock(spec=Dataset)- dataset.id = dataset_id or str(uuid4())- dataset.tenant_id = tenant_id or str(uuid4())- dataset.name = "test_dataset"- dataset.provider = provider- dataset.indexing_technique = indexing_technique- dataset.available_document_count = available_document_count- dataset.embedding_model = "text-embedding-ada-002"- dataset.embedding_model_provider = "openai"- dataset.retrieval_model = {- "search_method": "semantic_search",- "reranking_enable": False,- "top_k": 4,- "score_threshold_enabled": False,- }+ dataset = Dataset(+ id=dataset_id or str(uuid4()),+ tenant_id=tenant_id or str(uuid4()),+ name="test_dataset",+ provider=provider,+ indexing_technique=indexing_technique,+ embedding_model="text-embedding-ada-002",+ embedding_model_provider="openai",+ retrieval_model={+ "search_method": "semantic_search",+ "reranking_enable": False,+ "top_k": 4,+ "score_threshold_enabled": False,+ },+ )return dataset@@ -3740,12 +3739,13 @@ class TestProcessMetadataFilterFunc:class TestKnowledgeRetrievalRegression:@pytest.fixturedef mock_dataset(self) -> Dataset:- dataset = Mock(spec=Dataset)- dataset.id = str(uuid4())- dataset.tenant_id = str(uuid4())- dataset.name = "test_dataset"- dataset.indexing_technique = "high_quality"- dataset.provider = "dify"+ dataset = Dataset(+ id=str(uuid4()),+ tenant_id=str(uuid4()),+ name="test_dataset",+ indexing_technique="high_quality",+ provider="dify",+ )return datasetdef test_multiple_retrieve_reranking_with_app_context(self, mock_dataset):@@ -3760,10 +3760,11 @@ class TestKnowledgeRetrievalRegression:tenant_id = str(uuid4())# second dataset to ensure dataset_count > 1 reranking branch- secondary_dataset = Mock(spec=Dataset)- secondary_dataset.id = str(uuid4())- secondary_dataset.provider = "dify"- secondary_dataset.indexing_technique = "high_quality"+ secondary_dataset = Dataset(+ id=str(uuid4()),+ provider="dify",+ indexing_technique="high_quality",+ )# retriever returns 1 doc into internal list (all_documents_item)document = Document(diff --git a/api/tests/unit_tests/models/test_dataset_models.py b/api/tests/unit_tests/models/test_dataset_models.pyindex 0a7a0722933..b2e45e63e43 100644--- a/api/tests/unit_tests/models/test_dataset_models.py+++ b/api/tests/unit_tests/models/test_dataset_models.py@@ -97,8 +97,8 @@ class TestDatasetModelValidation:name="Test Dataset",data_source_type=DataSourceType.UPLOAD_FILE,created_by=str(uuid4()),+ id=str(uuid4()),)- dataset.id = str(uuid4())account = Mock()process_rule = Mock()session = Mock()@@ -118,8 +118,8 @@ class TestDatasetModelValidation:name="Test Dataset",data_source_type=DataSourceType.UPLOAD_FILE,created_by=str(uuid4()),+ id=str(uuid4()),)- dataset.id = str(uuid4())keyword_table = Mock()session = Mock()session.scalar.return_value = keyword_table@@ -137,8 +137,8 @@ class TestDatasetModelValidation:created_by=str(uuid4()),provider="vendor",built_in_field_enabled=False,+ id=str(uuid4()),)- dataset.id = str(uuid4())account = Mock(name="account", name_value="Ada")account.name = "Ada"session = Mock()@@ -274,9 +274,13 @@ class TestDatasetModelValidation:created_by=str(uuid4()),provider="external",)- binding = Mock(spec=ExternalKnowledgeBindings)- binding.external_knowledge_id = "knowledge-1"- binding.external_knowledge_api_id = str(uuid4())+ binding = ExternalKnowledgeBindings(+ tenant_id="tenant-id",+ external_knowledge_api_id=str(uuid4()),+ dataset_id="dataset-id",+ external_knowledge_id="knowledge-1",+ created_by="account-id",+ )session = Mock()session.scalar.side_effect = [binding, None]@@ -686,7 +690,7 @@ class TestDocumentModelRelationships:created_from=DocumentCreatedFrom.WEB,created_by=str(uuid4()),)- dataset = Mock(spec=Dataset)+ dataset = Dataset()session = Mock()session.get.return_value = dataset@@ -752,20 +756,31 @@ class TestDocumentSegmentIndexing:tokens=5,created_by=str(uuid4()),)- document = Mock(spec=Document)+ document = Document()process_rule = Mock(mode="hierarchical", rules_dict={"parent_mode": "paragraph"})- document.get_dataset_process_rule.return_value = process_rule- child_chunk = Mock(spec=ChildChunk)+ child_chunk = ChildChunk(+ tenant_id="tenant-id",+ dataset_id="dataset-id",+ document_id="document-id",+ segment_id="segment-id",+ position=1,+ content="",+ word_count=0,+ created_by="account-id",+ )session = Mock()session.get.return_value = documentsession.scalars.return_value.all.return_value = [child_chunk]- with patch("models.dataset.Rule.model_validate", return_value=Mock(parent_mode="paragraph")):+ with (+ patch.object(Document, "get_dataset_process_rule", return_value=process_rule) as get_process_rule,+ patch("models.dataset.Rule.model_validate", return_value=Mock(parent_mode="paragraph")),+ ):result = segment.get_child_chunks(session=session)assert result == [child_chunk]session.get.assert_called_once_with(Document, segment.document_id)- document.get_dataset_process_rule.assert_called_once_with(session=session)+ get_process_rule.assert_called_once_with(session=session)session.scalars.assert_called_once()def test_get_child_chunks_includes_full_doc_unless_explicitly_hidden(self):@@ -779,17 +794,29 @@ class TestDocumentSegmentIndexing:tokens=5,created_by=str(uuid4()),)- document = Mock(spec=Document)- document.get_dataset_process_rule.return_value = Mock(+ document = Document()+ process_rule = Mock(mode="hierarchical",rules_dict={"parent_mode": ParentMode.FULL_DOC},)session = Mock()session.get.return_value = document- child_chunk = Mock(spec=ChildChunk)+ child_chunk = ChildChunk(+ tenant_id="tenant-id",+ dataset_id="dataset-id",+ document_id="document-id",+ segment_id="segment-id",+ position=1,+ content="",+ word_count=0,+ created_by="account-id",+ )session.scalars.return_value.all.return_value = [child_chunk]- with patch("models.dataset.Rule.model_validate", return_value=Mock(parent_mode=ParentMode.FULL_DOC)):+ with (+ patch.object(Document, "get_dataset_process_rule", return_value=process_rule),+ patch("models.dataset.Rule.model_validate", return_value=Mock(parent_mode=ParentMode.FULL_DOC)),+ ):result = segment.get_child_chunks(session=session)response_result = segment.get_child_chunks(session=session, include_full_doc=False)@@ -808,8 +835,8 @@ class TestDocumentSegmentIndexing:tokens=5,created_by=str(uuid4()),)- dataset = Mock(spec=Dataset)- document = Mock(spec=Document)+ dataset = Dataset()+ document = Document()session = Mock()session.get.side_effect = [dataset, document]@@ -1369,8 +1396,8 @@ class TestModelIntegration:data_source_type=DataSourceType.UPLOAD_FILE,created_by=created_by,indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ id=dataset_id,)- dataset.id = dataset_id# Create documentdocument = Document(@@ -1383,8 +1410,8 @@ class TestModelIntegration:created_from=DocumentCreatedFrom.WEB,created_by=created_by,word_count=100,+ id=document_id,)- document.id = document_id# Create segmentsegment = DocumentSegment(diff --git a/api/tests/unit_tests/services/test_summary_index_service.py b/api/tests/unit_tests/services/test_summary_index_service.pyindex 4af3b4cdad0..1eb1baa40a3 100644--- a/api/tests/unit_tests/services/test_summary_index_service.py+++ b/api/tests/unit_tests/services/test_summary_index_service.py@@ -63,23 +63,24 @@ def _segment(*, has_document: bool = True) -> MagicMock:return segment-def _summary_record(*, summary_content: str = "summary", node_id: str | None = None) -> MagicMock:- record = MagicMock(spec=summary_module.DocumentSegmentSummary, name="summary_record")+def _summary_record(*, summary_content: str = "summary", node_id: str | None = None) -> DocumentSegmentSummary:+ record = summary_module.DocumentSegmentSummary(+ dataset_id="dataset-1",+ document_id="doc-1",+ chunk_id="seg-1",+ summary_content=summary_content,+ summary_index_node_id=node_id,+ summary_index_node_hash=None,+ tokens=None,+ status=SummaryStatus.GENERATING,+ error=None,+ enabled=True,+ disabled_at=None,+ disabled_by=None,+ )record.id = "sum-1"- record.dataset_id = "dataset-1"- record.document_id = "doc-1"- record.chunk_id = "seg-1"- record.summary_content = summary_content- record.summary_index_node_id = node_id- record.summary_index_node_hash = None- record.tokens = None- record.status = SummaryStatus.GENERATING- record.error = None- record.enabled = Truerecord.created_at = datetime(2024, 1, 1, tzinfo=UTC)record.updated_at = datetime(2024, 1, 1, tzinfo=UTC)- record.disabled_at = None- record.disabled_by = Nonereturn record@@ -625,9 +626,7 @@ def test_generate_and_vectorize_summary_creates_missing_record_and_logs_usage(def test_generate_summaries_for_document_skip_conditions(monkeypatch: pytest.MonkeyPatch) -> None:dataset = _dataset(indexing_technique=IndexTechniqueType.ECONOMY)- document = MagicMock(spec=summary_module.DatasetDocument)- document.id = "doc-1"- document.doc_form = IndexStructureType.PARAGRAPH_INDEX+ document = summary_module.DatasetDocument(id="doc-1", doc_form=IndexStructureType.PARAGRAPH_INDEX)assert SummaryIndexService.generate_summaries_for_document(dataset, document, {"enable": True}) == []dataset = _dataset()@@ -639,9 +638,7 @@ def test_generate_summaries_for_document_skip_conditions(monkeypatch: pytest.Mondef test_generate_summaries_for_document_runs_and_handles_errors(monkeypatch: pytest.MonkeyPatch) -> None:dataset = _dataset()- document = MagicMock(spec=summary_module.DatasetDocument)- document.id = "doc-1"- document.doc_form = IndexStructureType.PARAGRAPH_INDEX+ document = summary_module.DatasetDocument(id="doc-1", doc_form=IndexStructureType.PARAGRAPH_INDEX)seg1 = _segment()seg2 = _segment()@@ -671,9 +668,7 @@ def test_generate_summaries_for_document_runs_and_handles_errors(monkeypatch: pydef test_generate_summaries_for_document_no_segments_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:dataset = _dataset()- document = MagicMock(spec=summary_module.DatasetDocument)- document.id = "doc-1"- document.doc_form = IndexStructureType.PARAGRAPH_INDEX+ document = summary_module.DatasetDocument(id="doc-1", doc_form=IndexStructureType.PARAGRAPH_INDEX)session = MagicMock()session.scalars.return_value.all.return_value = []@@ -690,9 +685,7 @@ def test_generate_summaries_for_document_applies_segment_ids_and_only_parent_chumonkeypatch: pytest.MonkeyPatch,) -> None:dataset = _dataset()- document = MagicMock(spec=summary_module.DatasetDocument)- document.id = "doc-1"- document.doc_form = IndexStructureType.PARAGRAPH_INDEX+ document = summary_module.DatasetDocument(id="doc-1", doc_form=IndexStructureType.PARAGRAPH_INDEX)seg = _segment()session = MagicMock()diff --git a/api/tests/unit_tests/tasks/test_dataset_indexing_task.py b/api/tests/unit_tests/tasks/test_dataset_indexing_task.pyindex 39ff472ab7e..ff2cf2f92e3 100644--- a/api/tests/unit_tests/tasks/test_dataset_indexing_task.py+++ b/api/tests/unit_tests/tasks/test_dataset_indexing_task.py@@ -192,31 +192,33 @@ def mock_db_session():@pytest.fixturedef mock_dataset(dataset_id, tenant_id):- """Create a mock Dataset object."""- dataset = Mock(spec=Dataset)- dataset.id = dataset_id- dataset.tenant_id = tenant_id- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY- dataset.embedding_model_provider = "openai"- dataset.embedding_model = "text-embedding-ada-002"+ """Create a Dataset model object."""+ dataset = Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ embedding_model_provider="openai",+ embedding_model="text-embedding-ada-002",+ )return dataset@pytest.fixturedef mock_documents(document_ids, dataset_id):- """Create mock Document objects."""+ """Create Document model objects."""documents = []for doc_id in document_ids:- doc = Mock(spec=Document)- doc.id = doc_id- doc.dataset_id = dataset_id- doc.indexing_status = "waiting"- doc.error = None- doc.stopped_at = None- doc.processing_started_at = None- # optional attribute used in some code paths- doc.doc_form = IndexStructureType.PARAGRAPH_INDEX- documents.append(doc)+ documents.append(+ Document(+ id=doc_id,+ dataset_id=dataset_id,+ indexing_status="waiting",+ error=None,+ stopped_at=None,+ processing_started_at=None,+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ )+ )return documents@@ -397,7 +399,7 @@ class TestBatchProcessing:# Arrange - Create actual document objects that can be modifiedmock_documents = []for doc_id in document_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -439,7 +441,7 @@ class TestBatchProcessing:mock_documents = []for doc_id in document_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -479,7 +481,7 @@ class TestBatchProcessing:mock_documents = []for doc_id in document_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -548,7 +550,7 @@ class TestProgressTracking:# Arrange - Create actual document objectsmock_documents = []for doc_id in document_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -584,7 +586,7 @@ class TestProgressTracking:# Arrange - Create actual document objectsmock_documents = []for doc_id in document_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -677,7 +679,7 @@ class TestErrorHandling:# Arrange - Create actual document objectsmock_documents = []for doc_id in document_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -911,7 +913,7 @@ class TestAdvancedScenarios:# The new code uses .all() which will only return existing documentsmock_documents = []for i, doc_id in enumerate([document_ids[0], document_ids[2]]): # Skip middle one- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -1002,7 +1004,7 @@ class TestAdvancedScenarios:# Arrangemock_documents = []for doc_id in document_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -1141,7 +1143,7 @@ class TestAdvancedScenarios:mock_documents = []for doc_id in large_batch_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -1183,7 +1185,7 @@ class TestIntegration:# Arrange - Create actual document objectsmock_documents = []for doc_id in document_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -1221,7 +1223,7 @@ class TestIntegration:# Arrange - Create actual document objectsmock_documents = []for doc_id in document_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"@@ -1357,7 +1359,7 @@ class TestPerformanceScenarios:mock_documents = []for doc_id in document_ids:- doc = MagicMock(spec=Document)+ doc = Document()doc.id = doc_iddoc.dataset_id = dataset_iddoc.indexing_status = "waiting"