Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: use real ORM models in RAG indexing tests (#40017)

Asuka Minatoac0c909d1154
diff --git a/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py b/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py
index 6a89e6ec5c8..589d29d8240 100644
--- a/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py
+++ b/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py
@@ -158,11 +158,12 @@ class _SimpleRetrievalSegment:
class TestRetrievalServiceInternals:
@pytest.fixture
def internal_dataset(self) -> Dataset:
- dataset = Mock(spec=Dataset)
- dataset.id = "dataset-id"
- dataset.tenant_id = "tenant-id"
- dataset.is_multimodal = False
- dataset.doc_form = IndexStructureType.PARENT_CHILD_INDEX
+ dataset = Dataset(
+ id="dataset-id",
+ tenant_id="tenant-id",
+ is_multimodal=False,
+ chunk_structure=IndexStructureType.PARENT_CHILD_INDEX,
+ )
return dataset
@pytest.fixture
@@ -264,7 +265,7 @@ class TestRetrievalServiceInternals:
@patch("core.rag.datasource.retrieval_service.Session")
def test_get_dataset_queries_by_id(self, mock_session_class):
- expected_dataset = Mock(spec=Dataset)
+ expected_dataset = Dataset()
mock_session = Mock()
mock_session.scalar.return_value = expected_dataset
mock_session_class.return_value.__enter__.return_value = mock_session
diff --git a/api/tests/unit_tests/core/rag/docstore/test_dataset_docstore.py b/api/tests/unit_tests/core/rag/docstore/test_dataset_docstore.py
index b7c51c77427..366bbcbd501 100644
--- a/api/tests/unit_tests/core/rag/docstore/test_dataset_docstore.py
+++ b/api/tests/unit_tests/core/rag/docstore/test_dataset_docstore.py
@@ -22,9 +22,10 @@ USER_ID = "00000000-0000-0000-0000-000000000004"
def _dataset() -> Dataset:
- dataset = MagicMock(spec=Dataset)
- dataset.id = DATASET_ID
- dataset.tenant_id = TENANT_ID
+ dataset = Dataset(
+ id=DATASET_ID,
+ tenant_id=TENANT_ID,
+ )
return dataset
@@ -36,7 +37,25 @@ def _persist_segment(
content: str = "Test content",
tokens: int = 5,
) -> DocumentSegment:
- segment = DocumentSegment(
+ segment = _segment(
+ index_node_id=index_node_id,
+ index_node_hash=index_node_hash,
+ content=content,
+ tokens=tokens,
+ )
+ session.add(segment)
+ session.flush()
+ return segment
+
+
+def _segment(
+ *,
+ index_node_id: str = "doc-1",
+ index_node_hash: str = "hash-1",
+ content: str = "Test content",
+ tokens: int = 5,
+) -> DocumentSegment:
+ return DocumentSegment(
tenant_id=TENANT_ID,
dataset_id=DATASET_ID,
document_id=DOCUMENT_ID,
@@ -48,9 +67,6 @@ def _persist_segment(
index_node_id=index_node_id,
index_node_hash=index_node_hash,
)
- session.add(segment)
- session.flush()
- return segment
class TestDatasetDocumentStoreInit:
@@ -59,8 +75,9 @@ class TestDatasetDocumentStoreInit:
def test_init_with_all_parameters(self):
"""Test initialization with dataset, user_id, and document_id."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
store = DatasetDocumentStore(
dataset=mock_dataset,
@@ -77,8 +94,9 @@ class TestDatasetDocumentStoreInit:
def test_init_without_document_id(self):
"""Test initialization without document_id."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
store = DatasetDocumentStore(
dataset=mock_dataset,
@@ -95,8 +113,9 @@ class TestDatasetDocumentStoreSerialization:
def test_to_dict(self):
"""Test serialization to dictionary."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
store = DatasetDocumentStore(
dataset=mock_dataset,
@@ -129,15 +148,11 @@ class TestDatasetDocumentStoreDocs:
def test_docs_returns_document_dict(self):
"""Test that docs property returns a dictionary of documents."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
- mock_segment = MagicMock(spec=DocumentSegment)
- mock_segment.index_node_id = "node-1"
- mock_segment.index_node_hash = "hash-1"
- mock_segment.document_id = "doc-1"
- mock_segment.dataset_id = "test-dataset-id"
- mock_segment.content = "Test content"
+ mock_segment = _segment(index_node_id="node-1")
mock_session = MagicMock()
mock_session.scalars.return_value.all.return_value = [mock_segment]
@@ -155,8 +170,9 @@ class TestDatasetDocumentStoreDocs:
def test_docs_empty_dataset(self):
"""Test docs property with no segments."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_session = MagicMock()
mock_session.scalars.return_value.all.return_value = []
@@ -326,8 +342,9 @@ class TestDatasetDocumentStoreExists:
def test_document_exists_returns_true(self):
"""Test document_exists returns True when segment exists."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_segment = MagicMock()
mock_session = MagicMock()
@@ -345,8 +362,9 @@ class TestDatasetDocumentStoreExists:
def test_document_exists_returns_false(self):
"""Test document_exists returns False when segment doesn't exist."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_session = MagicMock()
with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):
@@ -366,15 +384,11 @@ class TestDatasetDocumentStoreGetDocument:
def test_get_document_success(self):
"""Test getting a document successfully."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
- mock_segment = MagicMock(spec=DocumentSegment)
- mock_segment.index_node_id = "node-1"
- mock_segment.index_node_hash = "hash-1"
- mock_segment.document_id = "doc-1"
- mock_segment.dataset_id = "test-dataset-id"
- mock_segment.content = "Test content"
+ mock_segment = _segment(index_node_id="node-1")
mock_session = MagicMock()
with patch.object(DatasetDocumentStore, "get_document_segment", return_value=mock_segment):
@@ -391,8 +405,9 @@ class TestDatasetDocumentStoreGetDocument:
def test_get_document_returns_none_when_not_found(self):
"""Test get_document returns None when not found and raise_error=False."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_session = MagicMock()
with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):
@@ -408,8 +423,9 @@ class TestDatasetDocumentStoreGetDocument:
def test_get_document_raises_when_not_found(self):
"""Test get_document raises ValueError when not found and raise_error=True."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_session = MagicMock()
with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):
@@ -428,8 +444,9 @@ class TestDatasetDocumentStoreDeleteDocument:
def test_delete_document_success(self):
"""Test deleting a document successfully."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_segment = MagicMock()
mock_session = MagicMock()
@@ -448,8 +465,9 @@ class TestDatasetDocumentStoreDeleteDocument:
def test_delete_document_returns_none_when_not_found(self):
"""Test delete_document returns None when not found and raise_error=False."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_session = MagicMock()
with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):
@@ -465,8 +483,9 @@ class TestDatasetDocumentStoreDeleteDocument:
def test_delete_document_raises_when_not_found(self):
"""Test delete_document raises ValueError when not found and raise_error=True."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_session = MagicMock()
with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):
@@ -485,8 +504,9 @@ class TestDatasetDocumentStoreHashOperations:
def test_set_document_hash_success(self):
"""Test setting document hash successfully."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_segment = MagicMock()
mock_segment.index_node_hash = "old-hash"
@@ -506,8 +526,9 @@ class TestDatasetDocumentStoreHashOperations:
def test_set_document_hash_returns_none_when_not_found(self):
"""Test set_document_hash returns None when segment not found."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_session = MagicMock()
with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):
@@ -523,8 +544,9 @@ class TestDatasetDocumentStoreHashOperations:
def test_get_document_hash_success(self):
"""Test getting document hash successfully."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_segment = MagicMock()
mock_segment.index_node_hash = "test-hash"
@@ -543,8 +565,9 @@ class TestDatasetDocumentStoreHashOperations:
def test_get_document_hash_returns_none_when_not_found(self):
"""Test get_document_hash returns None when segment not found."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_session = MagicMock()
with patch.object(DatasetDocumentStore, "get_document_segment", return_value=None):
@@ -564,10 +587,11 @@ class TestDatasetDocumentStoreSegment:
def test_get_document_segment_returns_segment(self):
"""Test getting a document segment."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
- mock_segment = MagicMock(spec=DocumentSegment)
+ mock_segment = _segment()
mock_session = MagicMock()
mock_session.scalar.return_value = mock_segment
@@ -584,8 +608,9 @@ class TestDatasetDocumentStoreSegment:
def test_get_document_segment_returns_none(self):
"""Test getting a non-existent document segment."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ )
mock_session = MagicMock()
mock_session.scalar.return_value = None
@@ -606,9 +631,10 @@ class TestDatasetDocumentStoreMultimodelBinding:
def test_add_multimodel_documents_binding_with_attachments(self):
"""Test adding multimodel document bindings."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
- mock_dataset.tenant_id = "tenant-1"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ tenant_id="tenant-1",
+ )
mock_attachment = MagicMock(spec=AttachmentDocument)
mock_attachment.metadata = {"doc_id": "attachment-1"}
@@ -628,9 +654,10 @@ class TestDatasetDocumentStoreMultimodelBinding:
def test_add_multimodel_documents_binding_without_attachments(self):
"""Test adding bindings with None attachments."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
- mock_dataset.tenant_id = "tenant-1"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ tenant_id="tenant-1",
+ )
mock_session = MagicMock()
@@ -647,9 +674,10 @@ class TestDatasetDocumentStoreMultimodelBinding:
def test_add_multimodel_documents_binding_with_empty_list(self):
"""Test adding bindings with empty list."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
- mock_dataset.tenant_id = "tenant-1"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ tenant_id="tenant-1",
+ )
mock_session = MagicMock()
@@ -666,9 +694,10 @@ class TestDatasetDocumentStoreMultimodelBinding:
def test_add_multimodel_documents_binding_with_none_document_id(self):
"""Test that no bindings are added when document_id is None."""
- mock_dataset = MagicMock(spec=Dataset)
- mock_dataset.id = "test-dataset-id"
- mock_dataset.tenant_id = "tenant-1"
+ mock_dataset = Dataset(
+ id="test-dataset-id",
+ tenant_id="tenant-1",
+ )
mock_attachment = MagicMock(spec=AttachmentDocument)
mock_attachment.metadata = {"doc_id": "attachment-1"}
diff --git a/api/tests/unit_tests/core/rag/embedding/test_token_counter.py b/api/tests/unit_tests/core/rag/embedding/test_token_counter.py
index 0c134a94c90..733f55fa16c 100644
--- a/api/tests/unit_tests/core/rag/embedding/test_token_counter.py
+++ b/api/tests/unit_tests/core/rag/embedding/test_token_counter.py
@@ -1,4 +1,4 @@
-from unittest.mock import Mock, patch
+from unittest.mock import patch
from core.rag.embedding.token_counter import calculate_segment_token_counts
from core.rag.index_processor.constant.index_type import IndexTechniqueType
@@ -7,11 +7,12 @@ from models.dataset import Dataset
def test_high_quality_counts_each_document_once() -> None:
- dataset = Mock(spec=Dataset)
- dataset.tenant_id = "tenant-1"
- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
- dataset.embedding_model_provider = "provider"
- dataset.embedding_model = "model"
+ dataset = Dataset(
+ tenant_id="tenant-1",
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ embedding_model_provider="provider",
+ embedding_model="model",
+ )
documents = [
Document(page_content="first", metadata={}),
Document(page_content="second", metadata={}),
@@ -31,8 +32,9 @@ def test_high_quality_counts_each_document_once() -> None:
def test_economy_returns_zero_without_loading_model() -> None:
- dataset = Mock(spec=Dataset)
- dataset.indexing_technique = IndexTechniqueType.ECONOMY
+ dataset = Dataset(
+ indexing_technique=IndexTechniqueType.ECONOMY,
+ )
documents = [
Document(page_content="first", metadata={}),
Document(page_content="second", metadata={}),
@@ -46,8 +48,9 @@ def test_economy_returns_zero_without_loading_model() -> None:
def test_empty_documents_return_without_loading_model() -> None:
- dataset = Mock(spec=Dataset)
- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
+ dataset = Dataset(
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ )
with patch("core.rag.embedding.token_counter.ModelManager.for_tenant") as model_manager_factory:
result = calculate_segment_token_counts(dataset=dataset, documents=[])
diff --git a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
index ef1b38f49d7..c129e51c5ae 100644
--- a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
+++ b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
@@ -83,10 +83,10 @@ def create_mock_dataset(
indexing_technique: str = IndexTechniqueType.HIGH_QUALITY,
embedding_provider: str = "openai",
embedding_model: str = "text-embedding-ada-002",
-) -> Mock:
- """Create a mock Dataset object with configurable parameters.
+) -> Dataset:
+ """Create a Dataset object with configurable parameters.
- This helper function creates a properly configured mock Dataset object that can be
+ This helper function creates a properly configured Dataset object that can be
used across multiple tests, ensuring consistency in test data.
Args:
@@ -97,18 +97,19 @@ def create_mock_dataset(
embedding_model: The embedding model name.
Returns:
- Mock: A configured mock Dataset object with all required attributes.
+ Dataset: A configured Dataset object with all required attributes.
Example:
>>> dataset = create_mock_dataset(indexing_technique="economy")
>>> assert dataset.indexing_technique == "economy"
"""
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_id or str(uuid.uuid4())
- dataset.tenant_id = tenant_id or str(uuid.uuid4())
- dataset.indexing_technique = indexing_technique
- dataset.embedding_model_provider = embedding_provider
- dataset.embedding_model = embedding_model
+ dataset = Dataset(
+ id=dataset_id or str(uuid.uuid4()),
+ tenant_id=tenant_id or str(uuid.uuid4()),
+ indexing_technique=indexing_technique,
+ embedding_model_provider=embedding_provider,
+ embedding_model=embedding_model,
+ )
return dataset
@@ -119,10 +120,10 @@ def create_mock_dataset_document(
doc_form: str = IndexStructureType.PARAGRAPH_INDEX,
data_source_type: str = "upload_file",
doc_language: str = "English",
-) -> Mock:
- """Create a mock DatasetDocument object with configurable parameters.
+) -> DatasetDocument:
+ """Create a DatasetDocument object with configurable parameters.
- This helper function creates a properly configured mock DatasetDocument object,
+ This helper function creates a properly configured DatasetDocument object,
reducing boilerplate code in individual tests.
Args:
@@ -134,23 +135,23 @@ def create_mock_dataset_document(
doc_language: The document language.
Returns:
- Mock: A configured mock DatasetDocument object with all required attributes.
+ DatasetDocument: A configured DatasetDocument object with all required attributes.
Example:
>>> doc = create_mock_dataset_document(doc_form=IndexStructureType.QA_INDEX)
>>> assert doc.doc_form == IndexStructureType.QA_INDEX
"""
- doc = Mock(spec=DatasetDocument)
- doc.id = document_id or str(uuid.uuid4())
- doc.dataset_id = dataset_id or str(uuid.uuid4())
- doc.tenant_id = tenant_id or str(uuid.uuid4())
- doc.doc_form = doc_form
- doc.doc_language = doc_language
- doc.data_source_type = data_source_type
- doc.data_source_info_dict = {"upload_file_id": str(uuid.uuid4())}
- doc.dataset_process_rule_id = str(uuid.uuid4())
- doc.created_by = str(uuid.uuid4())
- return doc
+ return DatasetDocument(
+ id=document_id or str(uuid.uuid4()),
+ dataset_id=dataset_id or str(uuid.uuid4()),
+ tenant_id=tenant_id or str(uuid.uuid4()),
+ doc_form=doc_form,
+ doc_language=doc_language,
+ data_source_type=data_source_type,
+ data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),
+ dataset_process_rule_id=str(uuid.uuid4()),
+ created_by=str(uuid.uuid4()),
+ )
def create_sample_documents(
@@ -275,14 +276,14 @@ class TestIndexingRunnerExtract:
@pytest.fixture
def sample_dataset_document(self):
"""Create a sample dataset document for testing."""
- doc = Mock(spec=DatasetDocument)
- doc.id = str(uuid.uuid4())
- doc.dataset_id = str(uuid.uuid4())
- doc.tenant_id = str(uuid.uuid4())
- doc.doc_form = IndexStructureType.PARAGRAPH_INDEX
- doc.data_source_type = "upload_file"
- doc.data_source_info_dict = {"upload_file_id": str(uuid.uuid4())}
- return doc
+ return DatasetDocument(
+ id=str(uuid.uuid4()),
+ dataset_id=str(uuid.uuid4()),
+ tenant_id=str(uuid.uuid4()),
+ doc_form=IndexStructureType.PARAGRAPH_INDEX,
+ data_source_type="upload_file",
+ data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),
+ )
@pytest.fixture
def sample_process_rule(self):
@@ -353,12 +354,14 @@ class TestIndexingRunnerExtract:
# Arrange
runner = IndexingRunner()
sample_dataset_document.data_source_type = "notion_import"
- sample_dataset_document.data_source_info_dict = {
- "credential_id": str(uuid.uuid4()),
- "notion_workspace_id": "workspace123",
- "notion_page_id": "page123",
- "type": "page",
- }
+ sample_dataset_document.data_source_info = json.dumps(
+ {
+ "credential_id": str(uuid.uuid4()),
+ "notion_workspace_id": "workspace123",
+ "notion_page_id": "page123",
+ "type": "page",
+ }
+ )
mock_processor = MagicMock()
mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor
@@ -383,13 +386,15 @@ class TestIndexingRunnerExtract:
# Arrange
runner = IndexingRunner()
sample_dataset_document.data_source_type = "website_crawl"
- sample_dataset_document.data_source_info_dict = {
- "provider": "firecrawl",
- "url": "https://example.com",
- "job_id": "job123",
- "mode": "crawl",
- "only_main_content": True,
- }
+ sample_dataset_document.data_source_info = json.dumps(
+ {
+ "provider": "firecrawl",
+ "url": "https://example.com",
+ "job_id": "job123",
+ "mode": "crawl",
+ "only_main_content": True,
+ }
+ )
mock_processor = MagicMock()
mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor
@@ -415,7 +420,7 @@ class TestIndexingRunnerExtract:
"""Test extraction fails when upload file is missing."""
# Arrange
runner = IndexingRunner()
- sample_dataset_document.data_source_info_dict = {}
+ sample_dataset_document.data_source_info = "{}"
mock_processor = MagicMock()
mock_dependencies["factory"].return_value.init_index_processor.return_value = mock_processor
@@ -466,12 +471,13 @@ class TestIndexingRunnerTransform:
@pytest.fixture
def sample_dataset(self):
"""Create a sample dataset for testing."""
- dataset = Mock(spec=Dataset)
- dataset.id = str(uuid.uuid4())
- dataset.tenant_id = str(uuid.uuid4())
- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
- dataset.embedding_model_provider = "openai"
- dataset.embedding_model = "text-embedding-ada-002"
+ dataset = Dataset(
+ id=str(uuid.uuid4()),
+ tenant_id=str(uuid.uuid4()),
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ embedding_model_provider="openai",
+ embedding_model="text-embedding-ada-002",
+ )
return dataset
@pytest.fixture
@@ -637,22 +643,23 @@ class TestIndexingRunnerLoad:
@pytest.fixture
def sample_dataset(self):
"""Create a sample dataset for testing."""
- dataset = Mock(spec=Dataset)
- dataset.id = str(uuid.uuid4())
- dataset.tenant_id = str(uuid.uuid4())
- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
- dataset.embedding_model_provider = "openai"
- dataset.embedding_model = "text-embedding-ada-002"
+ dataset = Dataset(
+ id=str(uuid.uuid4()),
+ tenant_id=str(uuid.uuid4()),
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ embedding_model_provider="openai",
+ embedding_model="text-embedding-ada-002",
+ )
return dataset
@pytest.fixture
def sample_dataset_document(self):
"""Create a sample dataset document for testing."""
- doc = Mock(spec=DatasetDocument)
- doc.id = str(uuid.uuid4())
- doc.dataset_id = str(uuid.uuid4())
- doc.doc_form = IndexStructureType.PARAGRAPH_INDEX
- return doc
+ return DatasetDocument(
+ id=str(uuid.uuid4()),
+ dataset_id=str(uuid.uuid4()),
+ doc_form=IndexStructureType.PARAGRAPH_INDEX,
+ )
@pytest.fixture
def sample_documents(self):
@@ -842,16 +849,18 @@ class TestIndexingRunnerRun:
"""Create sample dataset documents for testing."""
docs = []
for i in range(2):
- doc = Mock(spec=DatasetDocument)
- doc.id = str(uuid.uuid4())
- doc.dataset_id = str(uuid.uuid4())
- doc.tenant_id = str(uuid.uuid4())
- doc.doc_form = IndexStructureType.PARAGRAPH_INDEX
- doc.doc_language = "English"
- doc.data_source_type = "upload_file"
- doc.data_source_info_dict = {"upload_file_id": str(uuid.uuid4())}
- doc.dataset_process_rule_id = str(uuid.uuid4())
- docs.append(doc)
+ docs.append(
+ DatasetDocument(
+ id=str(uuid.uuid4()),
+ dataset_id=str(uuid.uuid4()),
+ tenant_id=str(uuid.uuid4()),
+ doc_form=IndexStructureType.PARAGRAPH_INDEX,
+ doc_language="English",
+ data_source_type="upload_file",
+ data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),
+ dataset_process_rule_id=str(uuid.uuid4()),
+ )
+ )
return docs
def test_run_in_indexing_status_loads_child_chunks_with_caller_session(
@@ -860,44 +869,63 @@ class TestIndexingRunnerRun:
runner = IndexingRunner()
dataset_document = sample_dataset_documents[0]
dataset_document.doc_form = IndexStructureType.PARENT_CHILD_INDEX
- dataset = Mock(spec=Dataset)
- segment = Mock(spec=DocumentSegment)
- segment.status = "waiting"
- segment.content = "parent"
- segment.index_node_id = "parent-node"
- segment.index_node_hash = "parent-hash"
- segment.document_id = dataset_document.id
- segment.dataset_id = dataset_document.dataset_id
- segment.tokens = 12
- segment.get_child_chunks.return_value = [
- SimpleNamespace(content="child", index_node_id="child-node", index_node_hash="child-hash")
- ]
+ dataset = Dataset()
+ segment = DocumentSegment(
+ tenant_id="tenant-id",
+ dataset_id=dataset_document.dataset_id,
+ document_id=dataset_document.id,
+ position=1,
+ content="parent",
+ word_count=0,
+ tokens=12,
+ created_by="account-id",
+ status="waiting",
+ index_node_id="parent-node",
+ index_node_hash="parent-hash",
+ )
+ child_chunks = [SimpleNamespace(content="child", index_node_id="child-node", index_node_hash="child-hash")]
session = mock_dependencies["session"]
session.get.side_effect = lambda model, _: dataset_document if model is DatasetDocument else dataset
session.scalars.return_value.all.return_value = [segment]
- with patch.object(runner, "_load") as load:
+ with (
+ patch.object(DocumentSegment, "get_child_chunks", return_value=child_chunks) as get_child_chunks,
+ patch.object(runner, "_load") as load,
+ ):
runner.run_in_indexing_status(dataset_document, session)
- segment.get_child_chunks.assert_called_once_with(session=session)
+ get_child_chunks.assert_called_once_with(session=session)
assert load.call_args.kwargs["documents"][0].children[0].page_content == "child"
assert load.call_args.kwargs["total_tokens"] == 12
def test_run_in_indexing_status_uses_tokens_from_all_segments(self, mock_dependencies, sample_dataset_documents):
runner = IndexingRunner()
dataset_document = sample_dataset_documents[0]
- dataset = Mock(spec=Dataset)
- completed_segment = Mock(spec=DocumentSegment)
- completed_segment.status = SegmentStatus.COMPLETED
- completed_segment.tokens = 10
- incomplete_segment = Mock(spec=DocumentSegment)
- incomplete_segment.status = SegmentStatus.WAITING
- incomplete_segment.tokens = 20
- incomplete_segment.content = "pending"
- incomplete_segment.index_node_id = "pending-node"
- incomplete_segment.index_node_hash = "pending-hash"
- incomplete_segment.document_id = dataset_document.id
- incomplete_segment.dataset_id = dataset_document.dataset_id
+ dataset = Dataset()
+ completed_segment = DocumentSegment(
+ tenant_id="tenant-id",
+ dataset_id="dataset-id",
+ document_id="document-id",
+ position=1,
+ content="",
+ word_count=0,
+ tokens=10,
+ created_by="account-id",
+ status=SegmentStatus.COMPLETED,
+ )
+ incomplete_segment = DocumentSegment(
+ tenant_id="tenant-id",
+ dataset_id=dataset_document.dataset_id,
+ document_id=dataset_document.id,
+ position=1,
+ content="pending",
+ word_count=0,
+ tokens=20,
+ created_by="account-id",
+ status=SegmentStatus.WAITING,
+ index_node_id="pending-node",
+ index_node_hash="pending-hash",
+ )
session = mock_dependencies["session"]
session.get.side_effect = lambda model, _: dataset_document if model is DatasetDocument else dataset
session.scalars.return_value.all.return_value = [completed_segment, incomplete_segment]
@@ -908,25 +936,28 @@ class TestIndexingRunnerRun:
assert load.call_args.kwargs["documents"][0].page_content == "pending"
assert load.call_args.kwargs["total_tokens"] == 30
- def test_run_success_single_document(self, mock_dependencies, sample_dataset_documents):
+ @patch.object(Account, "set_tenant_id_with_session", autospec=True)
+ def test_run_success_single_document(self, set_tenant_id, mock_dependencies, sample_dataset_documents):
"""Test successful run with single document."""
# Arrange
runner = IndexingRunner()
doc = sample_dataset_documents[0]
# Mock database queries
- mock_dataset = Mock(spec=Dataset)
- mock_dataset.id = doc.dataset_id
- mock_dataset.tenant_id = doc.tenant_id
- mock_dataset.indexing_technique = IndexTechniqueType.ECONOMY
+ mock_dataset = Dataset(
+ id=doc.dataset_id,
+ tenant_id=doc.tenant_id,
+ indexing_technique=IndexTechniqueType.ECONOMY,
+ )
- mock_current_user = Mock(spec=Account)
+ mock_current_user = Account(name="Test Account", email="[email protected]")
get_dispatch = {"Document": doc, "Dataset": mock_dataset, "Account": mock_current_user}
mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)
- mock_process_rule = Mock(spec=DatasetProcessRule)
- mock_process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}
+ mock_process_rule = DatasetProcessRule(
+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
+ )
mock_dependencies["session"].scalar.return_value = mock_process_rule
# Mock processor
@@ -963,7 +994,8 @@ class TestIndexingRunnerRun:
# Assert - verify the methods were called
# Since we're mocking the internal methods, we just verify no exceptions were raised
- mock_current_user.set_tenant_id_with_session.assert_called_once_with(
+ set_tenant_id.assert_called_once_with(
+ mock_current_user,
mock_dataset.tenant_id,
session=mock_dependencies["session"],
)
@@ -998,14 +1030,18 @@ class TestIndexingRunnerRun:
with pytest.raises(DocumentIsPausedError):
runner.run([doc], mock_dependencies["session"])
- def test_run_counts_each_transformed_document_once(self, mock_dependencies, sample_dataset_documents):
+ @patch.object(Account, "set_tenant_id_with_session", autospec=True)
+ def test_run_counts_each_transformed_document_once(
+ self, set_tenant_id, mock_dependencies, sample_dataset_documents
+ ):
runner = IndexingRunner()
dataset_document = sample_dataset_documents[0]
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_document.dataset_id
- dataset.tenant_id = dataset_document.tenant_id
- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
- current_user = Mock(spec=Account)
+ dataset = Dataset(
+ id=dataset_document.dataset_id,
+ tenant_id=dataset_document.tenant_id,
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ )
+ current_user = Account(name="Test Account", email="[email protected]")
transformed_documents = [
Document(page_content="first", metadata={"doc_id": "first", "doc_hash": "hash-first"}),
Document(page_content="second", metadata={"doc_id": "second", "doc_hash": "hash-second"}),
@@ -1016,8 +1052,9 @@ class TestIndexingRunnerRun:
Account: current_user,
}
mock_dependencies["session"].get.side_effect = lambda model, _: model_dispatch.get(model)
- process_rule = Mock(spec=DatasetProcessRule)
- process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}
+ process_rule = DatasetProcessRule(
+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
+ )
mock_dependencies["session"].scalar.return_value = process_rule
with (
@@ -1041,18 +1078,25 @@ class TestIndexingRunnerRun:
token_counts=[11, 22],
)
assert load.call_args.kwargs["total_tokens"] == 33
+ set_tenant_id.assert_called_once_with(
+ current_user,
+ dataset.tenant_id,
+ session=mock_dependencies["session"],
+ )
+ @patch.object(Account, "set_tenant_id_with_session", autospec=True)
def test_run_in_splitting_status_counts_each_transformed_document_once(
- self, mock_dependencies, sample_dataset_documents
+ self, set_tenant_id, mock_dependencies, sample_dataset_documents
):
runner = IndexingRunner()
dataset_document = sample_dataset_documents[0]
dataset_document.created_by = "user-1"
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_document.dataset_id
- dataset.tenant_id = dataset_document.tenant_id
- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
- current_user = Mock(spec=Account)
+ dataset = Dataset(
+ id=dataset_document.dataset_id,
+ tenant_id=dataset_document.tenant_id,
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ )
+ current_user = Account(name="Test Account", email="[email protected]")
transformed_documents = [
Document(page_content="first", metadata={"doc_id": "first", "doc_hash": "hash-first"}),
Document(page_content="second", metadata={"doc_id": "second", "doc_hash": "hash-second"}),
@@ -1064,8 +1108,9 @@ class TestIndexingRunnerRun:
}
mock_dependencies["session"].get.side_effect = lambda model, _: model_dispatch.get(model)
mock_dependencies["session"].scalars.return_value.all.return_value = []
- process_rule = Mock(spec=DatasetProcessRule)
- process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}
+ process_rule = DatasetProcessRule(
+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
+ )
mock_dependencies["session"].scalar.return_value = process_rule
with (
@@ -1089,6 +1134,11 @@ class TestIndexingRunnerRun:
token_counts=[11, 22],
)
assert load.call_args.kwargs["total_tokens"] == 33
+ set_tenant_id.assert_called_once_with(
+ current_user,
+ dataset.tenant_id,
+ session=mock_dependencies["session"],
+ )
def test_run_handles_provider_token_error(self, mock_dependencies, sample_dataset_documents):
"""Test run handles ProviderTokenNotInitError and updates document status."""
@@ -1097,14 +1147,16 @@ class TestIndexingRunnerRun:
doc = sample_dataset_documents[0]
# Mock database
- mock_dataset = Mock(spec=Dataset)
- mock_dataset.tenant_id = doc.tenant_id
+ mock_dataset = Dataset(
+ tenant_id=doc.tenant_id,
+ )
get_dispatch = {"Document": doc, "Dataset": mock_dataset}
mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)
- mock_process_rule = Mock(spec=DatasetProcessRule)
- mock_process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}
+ mock_process_rule = DatasetProcessRule(
+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
+ )
mock_dependencies["session"].scalar.return_value = mock_process_rule
mock_processor = MagicMock()
@@ -1126,14 +1178,16 @@ class TestIndexingRunnerRun:
doc = sample_dataset_documents[0]
# Mock database
- mock_dataset = Mock(spec=Dataset)
- mock_dataset.tenant_id = doc.tenant_id
+ mock_dataset = Dataset(
+ tenant_id=doc.tenant_id,
+ )
get_dispatch = {"Document": doc, "Dataset": mock_dataset}
mock_dependencies["session"].get.side_effect = lambda model, id: get_dispatch.get(model.__name__)
- mock_process_rule = Mock(spec=DatasetProcessRule)
- mock_process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}
+ mock_process_rule = DatasetProcessRule(
+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
+ )
mock_dependencies["session"].scalar.return_value = mock_process_rule
mock_processor = MagicMock()
@@ -1147,16 +1201,18 @@ class TestIndexingRunnerRun:
# Assert - should not raise, just log warning
# No exception should be raised
- def test_run_processes_multiple_documents(self, mock_dependencies, sample_dataset_documents):
+ @patch.object(Account, "set_tenant_id_with_session", autospec=True)
+ def test_run_processes_multiple_documents(self, set_tenant_id, mock_dependencies, sample_dataset_documents):
"""Test run processes multiple documents sequentially."""
# Arrange
runner = IndexingRunner()
docs = sample_dataset_documents
# Mock database
- mock_dataset = Mock(spec=Dataset)
- mock_dataset.indexing_technique = IndexTechniqueType.ECONOMY
- mock_current_user = Mock(spec=Account)
+ mock_dataset = Dataset(
+ indexing_technique=IndexTechniqueType.ECONOMY,
+ )
+ mock_current_user = Account(name="Test Account", email="[email protected]")
doc_map = {doc.id: doc for doc in docs}
model_dispatch = {"Dataset": mock_dataset, "Account": mock_current_user}
@@ -1169,8 +1225,9 @@ class TestIndexingRunnerRun:
mock_dependencies["session"].get.side_effect = get_side_effect
- mock_process_rule = Mock(spec=DatasetProcessRule)
- mock_process_rule.to_dict.return_value = {"mode": "automatic", "rules": {}}
+ mock_process_rule = DatasetProcessRule(
+ dataset_id="dataset-id", mode="automatic", rules="{}", created_by="account-id"
+ )
mock_dependencies["session"].scalar.return_value = mock_process_rule
mock_processor = MagicMock()
@@ -1197,7 +1254,7 @@ class TestIndexingRunnerRun:
# Assert
# Verify extract was called for each document
assert mock_extract.call_count == len(docs)
- assert mock_current_user.set_tenant_id_with_session.call_count == len(docs)
+ assert set_tenant_id.call_count == len(docs)
class TestIndexingRunnerRetryLogic:
@@ -1244,8 +1301,7 @@ class TestIndexingRunnerRetryLogic:
"""Test successful document status update."""
# Arrange
document_id = str(uuid.uuid4())
- mock_document = Mock(spec=DatasetDocument)
- mock_document.id = document_id
+ mock_document = DatasetDocument(id=document_id)
mock_dependencies["session"].scalar.return_value = 0
mock_dependencies["session"].get.return_value = mock_document
@@ -1296,23 +1352,29 @@ class TestIndexingRunnerDocumentCleaning:
@pytest.fixture
def sample_process_rule_automatic(self):
"""Create automatic processing rule."""
- rule = Mock(spec=DatasetProcessRule)
- rule.mode = "automatic"
- rule.rules = None
+ rule = DatasetProcessRule(
+ dataset_id="dataset-id",
+ mode="automatic",
+ rules=None,
+ created_by="account-id",
+ )
return rule
@pytest.fixture
def sample_process_rule_custom(self):
"""Create custom processing rule."""
- rule = Mock(spec=DatasetProcessRule)
- rule.mode = "custom"
- rule.rules = json.dumps(
- {
- "pre_processing_rules": [
- {"id": "remove_extra_spaces", "enabled": True},
- {"id": "remove_urls_emails", "enabled": True},
- ]
- }
+ rule = DatasetProcessRule(
+ dataset_id="dataset-id",
+ mode="custom",
+ rules=json.dumps(
+ {
+ "pre_processing_rules": [
+ {"id": "remove_extra_spaces", "enabled": True},
+ {"id": "remove_urls_emails", "enabled": True},
+ ]
+ }
+ ),
+ created_by="account-id",
)
return rule
@@ -1500,20 +1562,21 @@ class TestIndexingRunnerLoadSegments:
@pytest.fixture
def sample_dataset(self):
"""Create sample dataset."""
- dataset = Mock(spec=Dataset)
- dataset.id = str(uuid.uuid4())
- dataset.tenant_id = str(uuid.uuid4())
+ dataset = Dataset(
+ id=str(uuid.uuid4()),
+ tenant_id=str(uuid.uuid4()),
+ )
return dataset
@pytest.fixture
def sample_dataset_document(self):
"""Create sample dataset document."""
- doc = Mock(spec=DatasetDocument)
- doc.id = str(uuid.uuid4())
- doc.dataset_id = str(uuid.uuid4())
- doc.created_by = str(uuid.uuid4())
- doc.doc_form = IndexStructureType.PARAGRAPH_INDEX
- return doc
+ return DatasetDocument(
+ id=str(uuid.uuid4()),
+ dataset_id=str(uuid.uuid4()),
+ created_by=str(uuid.uuid4()),
+ doc_form=IndexStructureType.PARAGRAPH_INDEX,
+ )
@pytest.fixture
def sample_documents(self):
@@ -1769,11 +1832,11 @@ class TestIndexingRunnerProcessChunk:
Document(page_content="Chunk 2", metadata={"doc_id": "c2"}),
]
- mock_dataset = Mock(spec=Dataset)
- mock_dataset.id = str(uuid.uuid4())
+ mock_dataset = Dataset(
+ id=str(uuid.uuid4()),
+ )
- mock_dataset_document = Mock(spec=DatasetDocument)
- mock_dataset_document.id = str(uuid.uuid4())
+ mock_dataset_document = DatasetDocument(id=str(uuid.uuid4()))
mock_dependencies["redis"].get.return_value = None
@@ -1823,9 +1886,8 @@ class TestIndexingRunnerProcessChunk:
runner = IndexingRunner()
chunk_documents = [Document(page_content="Chunk", metadata={"doc_id": "c1"})]
- mock_dataset = Mock(spec=Dataset)
- mock_dataset_document = Mock(spec=DatasetDocument)
- mock_dataset_document.id = str(uuid.uuid4())
+ mock_dataset = Dataset()
+ mock_dataset_document = DatasetDocument(id=str(uuid.uuid4()))
# Mock Redis to return paused status
mock_dependencies["redis"].get.return_value = "1"
diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py
index 9e504dd1b9b..f9922643e96 100644
--- a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py
+++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py
@@ -252,24 +252,25 @@ class TestRetrievalService:
@pytest.fixture
def mock_dataset(self) -> Dataset:
"""
- Create a mock Dataset object for testing.
+ Create a Dataset object for testing.
Returns:
- Dataset: Mock dataset with standard configuration
- """
- dataset = Mock(spec=Dataset)
- dataset.id = str(uuid4())
- dataset.tenant_id = str(uuid4())
- dataset.name = "test_dataset"
- dataset.indexing_technique = "high_quality"
- dataset.embedding_model = "text-embedding-ada-002"
- dataset.embedding_model_provider = "openai"
- dataset.retrieval_model = {
- "search_method": RetrievalMethod.SEMANTIC_SEARCH,
- "reranking_enable": False,
- "top_k": 4,
- "score_threshold_enabled": False,
- }
+ Dataset with standard configuration.
+ """
+ dataset = Dataset(
+ id=str(uuid4()),
+ tenant_id=str(uuid4()),
+ name="test_dataset",
+ indexing_technique="high_quality",
+ embedding_model="text-embedding-ada-002",
+ embedding_model_provider="openai",
+ retrieval_model={
+ "search_method": RetrievalMethod.SEMANTIC_SEARCH,
+ "reranking_enable": False,
+ "top_k": 4,
+ "score_threshold_enabled": False,
+ },
+ )
return dataset
@pytest.fixture
@@ -1834,10 +1835,11 @@ class TestRetrievalService:
all_documents = []
# Create second dataset
- mock_dataset2 = Mock(spec=Dataset)
- mock_dataset2.id = str(uuid4())
- mock_dataset2.indexing_technique = "high_quality"
- mock_dataset2.provider = "dify"
+ mock_dataset2 = Dataset(
+ id=str(uuid4()),
+ indexing_technique="high_quality",
+ provider="dify",
+ )
# Act - Call with dataset_count = 2
with _patched_retriever_session() as rerank_session:
@@ -2207,36 +2209,33 @@ def create_mock_dataset_methods(
tenant_id: str | None = None,
provider: str = "dify",
indexing_technique: str = "high_quality",
- available_document_count: int = 10,
-) -> Mock:
+) -> Dataset:
"""
- Create a mock Dataset object for testing.
+ Create a Dataset object for testing.
Args:
dataset_id: Unique identifier for the dataset
tenant_id: Tenant ID for the dataset
provider: Provider type ("dify" or "external")
indexing_technique: Indexing technique ("high_quality" or "economy")
- available_document_count: Number of available documents
-
Returns:
- Mock: A properly configured Dataset mock
+ A configured Dataset model.
"""
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_id or str(uuid4())
- dataset.tenant_id = tenant_id or str(uuid4())
- dataset.name = "test_dataset"
- dataset.provider = provider
- dataset.indexing_technique = indexing_technique
- dataset.available_document_count = available_document_count
- dataset.embedding_model = "text-embedding-ada-002"
- dataset.embedding_model_provider = "openai"
- dataset.retrieval_model = {
- "search_method": "semantic_search",
- "reranking_enable": False,
- "top_k": 4,
- "score_threshold_enabled": False,
- }
+ dataset = Dataset(
+ id=dataset_id or str(uuid4()),
+ tenant_id=tenant_id or str(uuid4()),
+ name="test_dataset",
+ provider=provider,
+ indexing_technique=indexing_technique,
+ embedding_model="text-embedding-ada-002",
+ embedding_model_provider="openai",
+ retrieval_model={
+ "search_method": "semantic_search",
+ "reranking_enable": False,
+ "top_k": 4,
+ "score_threshold_enabled": False,
+ },
+ )
return dataset
@@ -3740,12 +3739,13 @@ class TestProcessMetadataFilterFunc:
class TestKnowledgeRetrievalRegression:
@pytest.fixture
def mock_dataset(self) -> Dataset:
- dataset = Mock(spec=Dataset)
- dataset.id = str(uuid4())
- dataset.tenant_id = str(uuid4())
- dataset.name = "test_dataset"
- dataset.indexing_technique = "high_quality"
- dataset.provider = "dify"
+ dataset = Dataset(
+ id=str(uuid4()),
+ tenant_id=str(uuid4()),
+ name="test_dataset",
+ indexing_technique="high_quality",
+ provider="dify",
+ )
return dataset
def test_multiple_retrieve_reranking_with_app_context(self, mock_dataset):
@@ -3760,10 +3760,11 @@ class TestKnowledgeRetrievalRegression:
tenant_id = str(uuid4())
# second dataset to ensure dataset_count > 1 reranking branch
- secondary_dataset = Mock(spec=Dataset)
- secondary_dataset.id = str(uuid4())
- secondary_dataset.provider = "dify"
- secondary_dataset.indexing_technique = "high_quality"
+ secondary_dataset = Dataset(
+ id=str(uuid4()),
+ provider="dify",
+ indexing_technique="high_quality",
+ )
# retriever returns 1 doc into internal list (all_documents_item)
document = Document(
diff --git a/api/tests/unit_tests/models/test_dataset_models.py b/api/tests/unit_tests/models/test_dataset_models.py
index 0a7a0722933..b2e45e63e43 100644
--- a/api/tests/unit_tests/models/test_dataset_models.py
+++ b/api/tests/unit_tests/models/test_dataset_models.py
@@ -97,8 +97,8 @@ class TestDatasetModelValidation:
name="Test Dataset",
data_source_type=DataSourceType.UPLOAD_FILE,
created_by=str(uuid4()),
+ id=str(uuid4()),
)
- dataset.id = str(uuid4())
account = Mock()
process_rule = Mock()
session = Mock()
@@ -118,8 +118,8 @@ class TestDatasetModelValidation:
name="Test Dataset",
data_source_type=DataSourceType.UPLOAD_FILE,
created_by=str(uuid4()),
+ id=str(uuid4()),
)
- dataset.id = str(uuid4())
keyword_table = Mock()
session = Mock()
session.scalar.return_value = keyword_table
@@ -137,8 +137,8 @@ class TestDatasetModelValidation:
created_by=str(uuid4()),
provider="vendor",
built_in_field_enabled=False,
+ id=str(uuid4()),
)
- dataset.id = str(uuid4())
account = Mock(name="account", name_value="Ada")
account.name = "Ada"
session = Mock()
@@ -274,9 +274,13 @@ class TestDatasetModelValidation:
created_by=str(uuid4()),
provider="external",
)
- binding = Mock(spec=ExternalKnowledgeBindings)
- binding.external_knowledge_id = "knowledge-1"
- binding.external_knowledge_api_id = str(uuid4())
+ binding = ExternalKnowledgeBindings(
+ tenant_id="tenant-id",
+ external_knowledge_api_id=str(uuid4()),
+ dataset_id="dataset-id",
+ external_knowledge_id="knowledge-1",
+ created_by="account-id",
+ )
session = Mock()
session.scalar.side_effect = [binding, None]
@@ -686,7 +690,7 @@ class TestDocumentModelRelationships:
created_from=DocumentCreatedFrom.WEB,
created_by=str(uuid4()),
)
- dataset = Mock(spec=Dataset)
+ dataset = Dataset()
session = Mock()
session.get.return_value = dataset
@@ -752,20 +756,31 @@ class TestDocumentSegmentIndexing:
tokens=5,
created_by=str(uuid4()),
)
- document = Mock(spec=Document)
+ document = Document()
process_rule = Mock(mode="hierarchical", rules_dict={"parent_mode": "paragraph"})
- document.get_dataset_process_rule.return_value = process_rule
- child_chunk = Mock(spec=ChildChunk)
+ child_chunk = ChildChunk(
+ tenant_id="tenant-id",
+ dataset_id="dataset-id",
+ document_id="document-id",
+ segment_id="segment-id",
+ position=1,
+ content="",
+ word_count=0,
+ created_by="account-id",
+ )
session = Mock()
session.get.return_value = document
session.scalars.return_value.all.return_value = [child_chunk]
- with patch("models.dataset.Rule.model_validate", return_value=Mock(parent_mode="paragraph")):
+ with (
+ patch.object(Document, "get_dataset_process_rule", return_value=process_rule) as get_process_rule,
+ patch("models.dataset.Rule.model_validate", return_value=Mock(parent_mode="paragraph")),
+ ):
result = segment.get_child_chunks(session=session)
assert result == [child_chunk]
session.get.assert_called_once_with(Document, segment.document_id)
- document.get_dataset_process_rule.assert_called_once_with(session=session)
+ get_process_rule.assert_called_once_with(session=session)
session.scalars.assert_called_once()
def test_get_child_chunks_includes_full_doc_unless_explicitly_hidden(self):
@@ -779,17 +794,29 @@ class TestDocumentSegmentIndexing:
tokens=5,
created_by=str(uuid4()),
)
- document = Mock(spec=Document)
- document.get_dataset_process_rule.return_value = Mock(
+ document = Document()
+ process_rule = Mock(
mode="hierarchical",
rules_dict={"parent_mode": ParentMode.FULL_DOC},
)
session = Mock()
session.get.return_value = document
- child_chunk = Mock(spec=ChildChunk)
+ child_chunk = ChildChunk(
+ tenant_id="tenant-id",
+ dataset_id="dataset-id",
+ document_id="document-id",
+ segment_id="segment-id",
+ position=1,
+ content="",
+ word_count=0,
+ created_by="account-id",
+ )
session.scalars.return_value.all.return_value = [child_chunk]
- with patch("models.dataset.Rule.model_validate", return_value=Mock(parent_mode=ParentMode.FULL_DOC)):
+ with (
+ patch.object(Document, "get_dataset_process_rule", return_value=process_rule),
+ patch("models.dataset.Rule.model_validate", return_value=Mock(parent_mode=ParentMode.FULL_DOC)),
+ ):
result = segment.get_child_chunks(session=session)
response_result = segment.get_child_chunks(session=session, include_full_doc=False)
@@ -808,8 +835,8 @@ class TestDocumentSegmentIndexing:
tokens=5,
created_by=str(uuid4()),
)
- dataset = Mock(spec=Dataset)
- document = Mock(spec=Document)
+ dataset = Dataset()
+ document = Document()
session = Mock()
session.get.side_effect = [dataset, document]
@@ -1369,8 +1396,8 @@ class TestModelIntegration:
data_source_type=DataSourceType.UPLOAD_FILE,
created_by=created_by,
indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ id=dataset_id,
)
- dataset.id = dataset_id
# Create document
document = Document(
@@ -1383,8 +1410,8 @@ class TestModelIntegration:
created_from=DocumentCreatedFrom.WEB,
created_by=created_by,
word_count=100,
+ id=document_id,
)
- document.id = document_id
# Create segment
segment = DocumentSegment(
diff --git a/api/tests/unit_tests/services/test_summary_index_service.py b/api/tests/unit_tests/services/test_summary_index_service.py
index 4af3b4cdad0..1eb1baa40a3 100644
--- a/api/tests/unit_tests/services/test_summary_index_service.py
+++ b/api/tests/unit_tests/services/test_summary_index_service.py
@@ -63,23 +63,24 @@ def _segment(*, has_document: bool = True) -> MagicMock:
return segment
-def _summary_record(*, summary_content: str = "summary", node_id: str | None = None) -> MagicMock:
- record = MagicMock(spec=summary_module.DocumentSegmentSummary, name="summary_record")
+def _summary_record(*, summary_content: str = "summary", node_id: str | None = None) -> DocumentSegmentSummary:
+ record = summary_module.DocumentSegmentSummary(
+ dataset_id="dataset-1",
+ document_id="doc-1",
+ chunk_id="seg-1",
+ summary_content=summary_content,
+ summary_index_node_id=node_id,
+ summary_index_node_hash=None,
+ tokens=None,
+ status=SummaryStatus.GENERATING,
+ error=None,
+ enabled=True,
+ disabled_at=None,
+ disabled_by=None,
+ )
record.id = "sum-1"
- record.dataset_id = "dataset-1"
- record.document_id = "doc-1"
- record.chunk_id = "seg-1"
- record.summary_content = summary_content
- record.summary_index_node_id = node_id
- record.summary_index_node_hash = None
- record.tokens = None
- record.status = SummaryStatus.GENERATING
- record.error = None
- record.enabled = True
record.created_at = datetime(2024, 1, 1, tzinfo=UTC)
record.updated_at = datetime(2024, 1, 1, tzinfo=UTC)
- record.disabled_at = None
- record.disabled_by = None
return record
@@ -625,9 +626,7 @@ def test_generate_and_vectorize_summary_creates_missing_record_and_logs_usage(
def test_generate_summaries_for_document_skip_conditions(monkeypatch: pytest.MonkeyPatch) -> None:
dataset = _dataset(indexing_technique=IndexTechniqueType.ECONOMY)
- document = MagicMock(spec=summary_module.DatasetDocument)
- document.id = "doc-1"
- document.doc_form = IndexStructureType.PARAGRAPH_INDEX
+ document = summary_module.DatasetDocument(id="doc-1", doc_form=IndexStructureType.PARAGRAPH_INDEX)
assert SummaryIndexService.generate_summaries_for_document(dataset, document, {"enable": True}) == []
dataset = _dataset()
@@ -639,9 +638,7 @@ def test_generate_summaries_for_document_skip_conditions(monkeypatch: pytest.Mon
def test_generate_summaries_for_document_runs_and_handles_errors(monkeypatch: pytest.MonkeyPatch) -> None:
dataset = _dataset()
- document = MagicMock(spec=summary_module.DatasetDocument)
- document.id = "doc-1"
- document.doc_form = IndexStructureType.PARAGRAPH_INDEX
+ document = summary_module.DatasetDocument(id="doc-1", doc_form=IndexStructureType.PARAGRAPH_INDEX)
seg1 = _segment()
seg2 = _segment()
@@ -671,9 +668,7 @@ def test_generate_summaries_for_document_runs_and_handles_errors(monkeypatch: py
def test_generate_summaries_for_document_no_segments_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None:
dataset = _dataset()
- document = MagicMock(spec=summary_module.DatasetDocument)
- document.id = "doc-1"
- document.doc_form = IndexStructureType.PARAGRAPH_INDEX
+ document = summary_module.DatasetDocument(id="doc-1", doc_form=IndexStructureType.PARAGRAPH_INDEX)
session = MagicMock()
session.scalars.return_value.all.return_value = []
@@ -690,9 +685,7 @@ def test_generate_summaries_for_document_applies_segment_ids_and_only_parent_chu
monkeypatch: pytest.MonkeyPatch,
) -> None:
dataset = _dataset()
- document = MagicMock(spec=summary_module.DatasetDocument)
- document.id = "doc-1"
- document.doc_form = IndexStructureType.PARAGRAPH_INDEX
+ document = summary_module.DatasetDocument(id="doc-1", doc_form=IndexStructureType.PARAGRAPH_INDEX)
seg = _segment()
session = MagicMock()
diff --git a/api/tests/unit_tests/tasks/test_dataset_indexing_task.py b/api/tests/unit_tests/tasks/test_dataset_indexing_task.py
index 39ff472ab7e..ff2cf2f92e3 100644
--- a/api/tests/unit_tests/tasks/test_dataset_indexing_task.py
+++ b/api/tests/unit_tests/tasks/test_dataset_indexing_task.py
@@ -192,31 +192,33 @@ def mock_db_session():
@pytest.fixture
def mock_dataset(dataset_id, tenant_id):
- """Create a mock Dataset object."""
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_id
- dataset.tenant_id = tenant_id
- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
- dataset.embedding_model_provider = "openai"
- dataset.embedding_model = "text-embedding-ada-002"
+ """Create a Dataset model object."""
+ dataset = Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ embedding_model_provider="openai",
+ embedding_model="text-embedding-ada-002",
+ )
return dataset
@pytest.fixture
def mock_documents(document_ids, dataset_id):
- """Create mock Document objects."""
+ """Create Document model objects."""
documents = []
for doc_id in document_ids:
- doc = Mock(spec=Document)
- doc.id = doc_id
- doc.dataset_id = dataset_id
- doc.indexing_status = "waiting"
- doc.error = None
- doc.stopped_at = None
- doc.processing_started_at = None
- # optional attribute used in some code paths
- doc.doc_form = IndexStructureType.PARAGRAPH_INDEX
- documents.append(doc)
+ documents.append(
+ Document(
+ id=doc_id,
+ dataset_id=dataset_id,
+ indexing_status="waiting",
+ error=None,
+ stopped_at=None,
+ processing_started_at=None,
+ doc_form=IndexStructureType.PARAGRAPH_INDEX,
+ )
+ )
return documents
@@ -397,7 +399,7 @@ class TestBatchProcessing:
# Arrange - Create actual document objects that can be modified
mock_documents = []
for doc_id in document_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -439,7 +441,7 @@ class TestBatchProcessing:
mock_documents = []
for doc_id in document_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -479,7 +481,7 @@ class TestBatchProcessing:
mock_documents = []
for doc_id in document_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -548,7 +550,7 @@ class TestProgressTracking:
# Arrange - Create actual document objects
mock_documents = []
for doc_id in document_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -584,7 +586,7 @@ class TestProgressTracking:
# Arrange - Create actual document objects
mock_documents = []
for doc_id in document_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -677,7 +679,7 @@ class TestErrorHandling:
# Arrange - Create actual document objects
mock_documents = []
for doc_id in document_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -911,7 +913,7 @@ class TestAdvancedScenarios:
# The new code uses .all() which will only return existing documents
mock_documents = []
for i, doc_id in enumerate([document_ids[0], document_ids[2]]): # Skip middle one
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -1002,7 +1004,7 @@ class TestAdvancedScenarios:
# Arrange
mock_documents = []
for doc_id in document_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -1141,7 +1143,7 @@ class TestAdvancedScenarios:
mock_documents = []
for doc_id in large_batch_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -1183,7 +1185,7 @@ class TestIntegration:
# Arrange - Create actual document objects
mock_documents = []
for doc_id in document_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -1221,7 +1223,7 @@ class TestIntegration:
# Arrange - Create actual document objects
mock_documents = []
for doc_id in document_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"
@@ -1357,7 +1359,7 @@ class TestPerformanceScenarios:
mock_documents = []
for doc_id in document_ids:
- doc = MagicMock(spec=Document)
+ doc = Document()
doc.id = doc_id
doc.dataset_id = dataset_id
doc.indexing_status = "waiting"