Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: use real ORM models in dataset service tests (#40018)

Asuka Minato479a5613a829
diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py
index eccf5ed56d1..51feb9bdb70 100644
--- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py
+++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py
@@ -32,7 +32,7 @@ class TestDataSourceContentPreviewApi:
payload = self._valid_payload()
- pipeline = MagicMock(spec=Pipeline)
+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
node_id = "node-1"
account = make_account()
@@ -72,7 +72,7 @@ class TestDataSourceContentPreviewApi:
# datasource_type missing
}
- pipeline = MagicMock(spec=Pipeline)
+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
account = make_account()
with (
@@ -92,7 +92,7 @@ class TestDataSourceContentPreviewApi:
"credential_id": None,
}
- pipeline = MagicMock(spec=Pipeline)
+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
account = make_account()
service_instance = MagicMock()
diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_import.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_import.py
index 248c6c210a8..48b70e32f61 100644
--- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_import.py
+++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_import.py
@@ -213,7 +213,7 @@ class TestRagPipelineImportCheckDependenciesApi:
api = RagPipelineImportCheckDependenciesApi()
method = unwrap(api.get)
- pipeline = MagicMock(spec=Pipeline)
+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
result = CheckDependenciesResult()
service = MagicMock()
@@ -235,7 +235,7 @@ class TestRagPipelineImportCheckDependenciesApi:
api = RagPipelineImportCheckDependenciesApi()
method = unwrap(api.get)
- pipeline = MagicMock(spec=Pipeline)
+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
dependency = PluginDependency(
type=PluginDependencyType.Marketplace,
value=PluginDependency.Marketplace(
@@ -276,7 +276,7 @@ class TestRagPipelineExportApi:
api = RagPipelineExportApi()
method = unwrap(api.get)
- pipeline = MagicMock(spec=Pipeline)
+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
service = MagicMock()
service.export_rag_pipeline_dsl.return_value = "yaml: data"
diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py
index 541c4b3f392..68b1ad0dd60 100644
--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py
+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py
@@ -1272,18 +1272,20 @@ class TestDatasetApiKeyApi:
def test_get_api_keys_success(self, app: Flask):
api = DatasetApiKeyApi()
method = unwrap(api.get)
- mock_key_1 = MagicMock(spec=ApiToken)
- mock_key_1.id = "key-1"
- mock_key_1.type = "dataset"
- mock_key_1.token = "ds-abc"
- mock_key_1.last_used_at = None
- mock_key_1.created_at = None
- mock_key_2 = MagicMock(spec=ApiToken)
- mock_key_2.id = "key-2"
- mock_key_2.type = "dataset"
- mock_key_2.token = "ds-def"
- mock_key_2.last_used_at = None
- mock_key_2.created_at = None
+ mock_key_1 = ApiToken(
+ id="key-1",
+ type="dataset",
+ token="ds-abc",
+ last_used_at=None,
+ created_at=None,
+ )
+ mock_key_2 = ApiToken(
+ id="key-2",
+ type="dataset",
+ token="ds-def",
+ last_used_at=None,
+ created_at=None,
+ )
session = MagicMock()
session.scalars.return_value.all.return_value = [mock_key_1, mock_key_2]
with app.test_request_context("/"):
diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.py
index 73c028da03b..22597a21f9b 100644
--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.py
+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.py
@@ -1,3 +1,4 @@
+from datetime import datetime
from inspect import unwrap
from types import SimpleNamespace
from typing import Any, cast
@@ -42,16 +43,16 @@ def _segment():
word_count=1,
tokens=1,
created_by="u1",
+ answer="a",
+ keywords=["test"],
+ index_node_id="n1",
+ index_node_hash="h",
+ status=SegmentStatus.COMPLETED,
+ updated_by="u1",
)
segment.id = "seg-1"
- segment.answer = "a"
- segment.keywords = ["test"]
- segment.index_node_id = "n1"
- segment.index_node_hash = "h"
- segment.status = SegmentStatus.COMPLETED
segment.created_at = naive_utc_now()
segment.updated_at = naive_utc_now()
- segment.updated_by = "u1"
return segment
@@ -65,9 +66,9 @@ def _child_chunk():
content="child",
word_count=1,
created_by="u1",
+ type=SegmentType.CUSTOMIZED,
)
child_chunk.id = "cc-1"
- child_chunk.type = SegmentType.CUSTOMIZED
child_chunk.created_at = naive_utc_now()
child_chunk.updated_at = naive_utc_now()
return child_chunk
@@ -532,8 +533,19 @@ class TestDatasetDocumentSegmentBatchImportApi:
api = DatasetDocumentSegmentBatchImportApi()
method = unwrap(api.post)
payload = {"upload_file_id": "file-1"}
- upload_file = MagicMock(spec=UploadFile)
- upload_file.name = "test.csv"
+ upload_file = UploadFile(
+ tenant_id="tenant-id",
+ storage_type="opendal",
+ key="test-key",
+ name="test.csv",
+ size=0,
+ extension="txt",
+ mime_type="text/plain",
+ created_by_role="account",
+ created_by="account-id",
+ created_at=datetime.now(),
+ used=False,
+ )
user = MagicMock(id="u1")
session = MagicMock()
session.scalar.return_value = upload_file
@@ -1079,8 +1091,20 @@ class TestSegmentOperationCases:
user = MagicMock(is_dataset_editor=True)
dataset = MagicMock()
document = MagicMock()
- upload_file = MagicMock(spec=UploadFile, extension="csv", id="file-1")
- upload_file.name = "test.csv"
+ upload_file = UploadFile(
+ tenant_id="tenant-id",
+ storage_type="opendal",
+ key="test-key",
+ name="test.csv",
+ size=0,
+ extension="csv",
+ mime_type="text/csv",
+ created_by_role="account",
+ created_by="account-id",
+ created_at=datetime.now(),
+ used=False,
+ )
+ upload_file.id = "file-1"
payload = {"upload_file_id": "file-1"}
session = MagicMock()
session.scalar.return_value = upload_file
diff --git a/api/tests/unit_tests/controllers/console/datasets/test_wraps.py b/api/tests/unit_tests/controllers/console/datasets/test_wraps.py
index 80ee65a7094..ab7a7978991 100644
--- a/api/tests/unit_tests/controllers/console/datasets/test_wraps.py
+++ b/api/tests/unit_tests/controllers/console/datasets/test_wraps.py
@@ -39,9 +39,11 @@ class TestGetRagPipeline:
get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=session_factory.return_value)
def test_pipeline_found_and_injected(self, mocker: MockerFixture):
- pipeline = Mock(spec=Pipeline)
+ pipeline = Pipeline(
+ tenant_id="tenant-1",
+ name="Test Pipeline",
+ )
pipeline.id = "pipeline-1"
- pipeline.tenant_id = "tenant-1"
@get_rag_pipeline
def dummy_view(**kwargs):
@@ -64,7 +66,7 @@ class TestGetRagPipeline:
get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=session_factory.return_value)
def test_load_rag_pipeline_uses_provided_session(self, mocker: MockerFixture):
- pipeline = Mock(spec=Pipeline)
+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
session = Mock(spec=Session)
mocker.patch(
@@ -82,7 +84,7 @@ class TestGetRagPipeline:
get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=session)
def test_pipeline_id_removed_from_kwargs(self, mocker: MockerFixture):
- pipeline = Mock(spec=Pipeline)
+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
@get_rag_pipeline
def dummy_view(**kwargs):
@@ -105,7 +107,7 @@ class TestGetRagPipeline:
assert result == "ok"
def test_pipeline_id_cast_to_string(self, mocker: MockerFixture):
- pipeline = Mock(spec=Pipeline)
+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")
@get_rag_pipeline
def dummy_view(**kwargs):
diff --git a/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py b/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py
index 20730e2e8a9..827950fae2f 100644
--- a/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py
+++ b/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py
@@ -1,3 +1,4 @@
+from datetime import datetime
from unittest.mock import MagicMock, patch
import httpx
@@ -219,9 +220,19 @@ class TestDatasourceFileManager:
@patch("core.datasource.datasource_file_manager.storage")
def test_get_file_binary(self, mock_storage, mock_db):
# Setup
- mock_upload_file = MagicMock(spec=UploadFile)
- mock_upload_file.key = "some_key"
- mock_upload_file.mime_type = "image/png"
+ mock_upload_file = UploadFile(
+ tenant_id="tenant-id",
+ storage_type="opendal",
+ key="some_key",
+ name="test.txt",
+ size=0,
+ extension="txt",
+ mime_type="image/png",
+ created_by_role="account",
+ created_by="account-id",
+ created_at=datetime.now(),
+ used=False,
+ )
mock_db.session.get.return_value = mock_upload_file
@@ -241,12 +252,22 @@ class TestDatasourceFileManager:
@patch("core.datasource.datasource_file_manager.storage")
def test_get_file_binary_by_message_file_id(self, mock_storage, mock_db):
# Setup
- mock_message_file = MagicMock(spec=MessageFile)
- mock_message_file.url = "http://localhost/files/tools/tool_id.png"
+ mock_message_file = MessageFile(
+ message_id="message-id",
+ type="image",
+ transfer_method="local_file",
+ created_by_role="user",
+ created_by="user-id",
+ url="http://localhost/files/tools/tool_id.png",
+ )
- mock_tool_file = MagicMock(spec=ToolFile)
- mock_tool_file.file_key = "tool_key"
- mock_tool_file.mimetype = "image/png"
+ mock_tool_file = ToolFile(
+ user_id="user-id",
+ tenant_id="tenant-id",
+ conversation_id="conversation-id",
+ file_key="tool_key",
+ mimetype="image/png",
+ )
def mock_get(model, id):
if model == MessageFile:
@@ -268,13 +289,23 @@ class TestDatasourceFileManager:
@patch("core.datasource.datasource_file_manager.storage")
def test_get_file_binary_by_message_file_id_with_extension(self, mock_storage, mock_db):
# Test that it correctly parses tool_id even with extension in URL
- mock_message_file = MagicMock(spec=MessageFile)
- mock_message_file.url = "http://localhost/files/tools/abcdef.png"
+ mock_message_file = MessageFile(
+ message_id="message-id",
+ type="image",
+ transfer_method="local_file",
+ created_by_role="user",
+ created_by="user-id",
+ url="http://localhost/files/tools/abcdef.png",
+ )
- mock_tool_file = MagicMock(spec=ToolFile)
+ mock_tool_file = ToolFile(
+ user_id="user-id",
+ tenant_id="tenant-id",
+ conversation_id="conversation-id",
+ file_key="tk",
+ mimetype="image/png",
+ )
mock_tool_file.id = "abcdef"
- mock_tool_file.file_key = "tk"
- mock_tool_file.mimetype = "image/png"
def mock_get(model, id):
if model == MessageFile:
@@ -295,8 +326,14 @@ class TestDatasourceFileManager:
assert DatasourceFileManager.get_file_binary_by_message_file_id("none") is None
# Case 2: Message file found but tool file not found
- mock_message_file = MagicMock(spec=MessageFile)
- mock_message_file.url = None
+ mock_message_file = MessageFile(
+ message_id="message-id",
+ type="image",
+ transfer_method="local_file",
+ created_by_role="user",
+ created_by="user-id",
+ url=None,
+ )
def mock_get_v2(model, id):
if model == MessageFile:
@@ -310,9 +347,19 @@ class TestDatasourceFileManager:
@patch("core.datasource.datasource_file_manager.storage")
def test_get_file_generator_by_upload_file_id(self, mock_storage, mock_db):
# Setup
- mock_upload_file = MagicMock(spec=UploadFile)
- mock_upload_file.key = "upload_key"
- mock_upload_file.mime_type = "text/plain"
+ mock_upload_file = UploadFile(
+ tenant_id="tenant-id",
+ storage_type="opendal",
+ key="upload_key",
+ name="test.txt",
+ size=0,
+ extension="txt",
+ mime_type="text/plain",
+ created_by_role="account",
+ created_by="account-id",
+ created_at=datetime.now(),
+ used=False,
+ )
mock_db.session.get.return_value = mock_upload_file
diff --git a/api/tests/unit_tests/core/datasource/utils/test_message_transformer.py b/api/tests/unit_tests/core/datasource/utils/test_message_transformer.py
index 0fca43cd0b1..30ecb5b4003 100644
--- a/api/tests/unit_tests/core/datasource/utils/test_message_transformer.py
+++ b/api/tests/unit_tests/core/datasource/utils/test_message_transformer.py
@@ -40,9 +40,14 @@ class TestDatasourceFileMessageTransformer:
def test_transform_image_message_success(self, mock_guess_ext, mock_tool_file_manager_cls):
# Setup
mock_manager = mock_tool_file_manager_cls.return_value
- mock_tool_file = MagicMock(spec=ToolFile)
+ mock_tool_file = ToolFile(
+ user_id="user-id",
+ tenant_id="tenant-id",
+ conversation_id="conversation-id",
+ file_key="test-key",
+ mimetype="image/png",
+ )
mock_tool_file.id = "file_id_123"
- mock_tool_file.mimetype = "image/png"
mock_manager.create_file_by_url.return_value = mock_tool_file
mock_guess_ext.return_value = ".png"
@@ -101,9 +106,14 @@ class TestDatasourceFileMessageTransformer:
def test_transform_blob_message_image(self, mock_guess_ext, mock_tool_file_manager_cls):
# Setup
mock_manager = mock_tool_file_manager_cls.return_value
- mock_tool_file = MagicMock(spec=ToolFile)
+ mock_tool_file = ToolFile(
+ user_id="user-id",
+ tenant_id="tenant-id",
+ conversation_id="conversation-id",
+ file_key="test-key",
+ mimetype="image/jpeg",
+ )
mock_tool_file.id = "blob_id_456"
- mock_tool_file.mimetype = "image/jpeg"
mock_manager.create_file_by_raw.return_value = mock_tool_file
mock_guess_ext.return_value = ".jpg"
@@ -137,9 +147,14 @@ class TestDatasourceFileMessageTransformer:
):
# Setup
mock_manager = mock_tool_file_manager_cls.return_value
- mock_tool_file = MagicMock(spec=ToolFile)
+ mock_tool_file = ToolFile(
+ user_id="user-id",
+ tenant_id="tenant-id",
+ conversation_id="conversation-id",
+ file_key="test-key",
+ mimetype="application/pdf",
+ )
mock_tool_file.id = "blob_id_789"
- mock_tool_file.mimetype = "application/pdf"
mock_manager.create_file_by_raw.return_value = mock_tool_file
mock_guess_type.return_value = ("application/pdf", None)
mock_guess_ext.return_value = ".pdf"
@@ -291,7 +306,13 @@ class TestDatasourceFileMessageTransformer:
# This tests line 70 where filename might be None
with patch("core.datasource.utils.message_transformer.ToolFileManager") as mock_tool_file_manager_cls:
mock_manager = mock_tool_file_manager_cls.return_value
- mock_tool_file = MagicMock(spec=ToolFile)
+ mock_tool_file = ToolFile(
+ user_id="user-id",
+ tenant_id="tenant-id",
+ conversation_id="conversation-id",
+ file_key="test-key",
+ mimetype="text/plain",
+ )
mock_tool_file.id = "blob_id_no_name"
mock_tool_file.mimetype = "application/octet-stream"
mock_manager.create_file_by_raw.return_value = mock_tool_file
diff --git a/api/tests/unit_tests/core/moderation/api/test_api.py b/api/tests/unit_tests/core/moderation/api/test_api.py
index f075deefe72..73f6709c438 100644
--- a/api/tests/unit_tests/core/moderation/api/test_api.py
+++ b/api/tests/unit_tests/core/moderation/api/test_api.py
@@ -50,7 +50,9 @@ class TestApiModeration:
@patch("core.moderation.api.api.ApiModeration._get_api_based_extension")
def test_validate_config_success(self, mock_get_extension, api_config):
- mock_get_extension.return_value = MagicMock(spec=APIBasedExtension)
+ mock_get_extension.return_value = APIBasedExtension(
+ tenant_id="tenant-id", name="Test Extension", api_endpoint="https://example.com", api_key="test-key"
+ )
ApiModeration.validate_config("test-tenant-id", api_config)
mock_get_extension.assert_called_once_with("test-tenant-id", "test-extension-id", db.session)
@@ -136,9 +138,12 @@ class TestApiModeration:
@patch("core.moderation.api.api.decrypt_token")
@patch("core.moderation.api.api.APIBasedExtensionRequestor")
def test_get_config_by_requestor_success(self, mock_requestor_cls, mock_decrypt, mock_get_ext, api_moderation):
- mock_ext = MagicMock(spec=APIBasedExtension)
- mock_ext.api_endpoint = "http://api.test"
- mock_ext.api_key = "encrypted-key"
+ mock_ext = APIBasedExtension(
+ tenant_id="tenant-id",
+ name="Test Extension",
+ api_endpoint="http://api.test",
+ api_key="encrypted-key",
+ )
mock_get_ext.return_value = mock_ext
mock_decrypt.return_value = "decrypted-key"
diff --git a/api/tests/unit_tests/services/controller_api.py b/api/tests/unit_tests/services/controller_api.py
index 48805da884e..b106f34964a 100644
--- a/api/tests/unit_tests/services/controller_api.py
+++ b/api/tests/unit_tests/services/controller_api.py
@@ -192,34 +192,27 @@ class ControllerApiTestDataFactory:
tenant_id: str = "tenant-123",
permission: DatasetPermissionEnum = DatasetPermissionEnum.ONLY_ME,
**kwargs,
- ) -> Mock:
+ ) -> Dataset:
"""
- Create a mock Dataset instance.
+ Create a Dataset instance.
Args:
dataset_id: Unique identifier for the dataset
name: Name of the dataset
tenant_id: Tenant identifier
permission: Dataset permission level
- **kwargs: Additional attributes to set on the mock
+ **kwargs: Additional mapped attributes for the dataset
Returns:
- Mock object configured as a Dataset instance
+ Configured Dataset instance
"""
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_id
- dataset.name = name
- dataset.tenant_id = tenant_id
- dataset.permission = permission
- dataset.to_dict.return_value = {
- "id": dataset_id,
- "name": name,
- "tenant_id": tenant_id,
- "permission": permission.value,
- }
- for key, value in kwargs.items():
- setattr(dataset, key, value)
- return dataset
+ return Dataset(
+ id=dataset_id,
+ name=name,
+ tenant_id=tenant_id,
+ permission=permission,
+ **kwargs,
+ )
@staticmethod
def create_user_mock(
@@ -240,15 +233,14 @@ class ControllerApiTestDataFactory:
Returns:
Mock object configured as a user/account instance
"""
- user = Mock()
- user.id = user_id
- user.current_tenant_id = tenant_id
- user.is_dataset_editor = is_dataset_editor
- user.has_edit_permission = True
- user.is_dataset_operator = False
- for key, value in kwargs.items():
- setattr(user, key, value)
- return user
+ return Mock(
+ id=user_id,
+ current_tenant_id=tenant_id,
+ is_dataset_editor=is_dataset_editor,
+ has_edit_permission=True,
+ is_dataset_operator=False,
+ **kwargs,
+ )
@staticmethod
def create_paginated_response(items, total, page=1, per_page=20):
@@ -822,19 +814,18 @@ class TestExternalDatasetApi:
@pytest.fixture
def mock_current_account_context(self, app: Flask) -> Iterator[Mock]:
"""Provide the wrapper auth context required by HTTP-client controller tests."""
- mock_user = Account(name="Test User", email="[email protected]")
+ mock_user = Account(
+ name="Test User",
+ status=AccountStatus.ACTIVE,
+ )
mock_user.id = "user-123"
- mock_user.status = AccountStatus.ACTIVE
mock_user.role = TenantAccountRole.EDITOR
def load_user_from_request_context() -> None:
g._login_user = mock_user
- setattr( # noqa: B010
- app,
- "login_manager",
- SimpleNamespace(load_user_from_request_context=load_user_from_request_context),
- )
+ app.login_manager = SimpleNamespace(load_user_from_request_context=load_user_from_request_context)
with (
patch("controllers.console.wraps.current_account_with_tenant") as mock_get_user,
diff --git a/api/tests/unit_tests/services/dataset_service_test_helpers.py b/api/tests/unit_tests/services/dataset_service_test_helpers.py
index d3dfe2fce79..d08913f537d 100644
--- a/api/tests/unit_tests/services/dataset_service_test_helpers.py
+++ b/api/tests/unit_tests/services/dataset_service_test_helpers.py
@@ -6,6 +6,7 @@ document, and segment service test modules that exercise
"""
import json
+from datetime import datetime
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock, Mock, create_autospec, patch
@@ -169,25 +170,23 @@ class DatasetServiceUnitDataFactory:
enable_api: bool = False,
summary_index_setting: dict[str, Any] | None = None,
**kwargs,
- ) -> Mock:
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_id
- dataset.tenant_id = tenant_id
- dataset.permission = permission
- dataset.created_by = created_by
- dataset.indexing_technique = indexing_technique
- dataset.embedding_model_provider = embedding_model_provider
- dataset.embedding_model = embedding_model
- dataset.built_in_field_enabled = built_in_field_enabled
- dataset.doc_form = doc_form
- dataset.get_doc_form.return_value = doc_form
- dataset.enable_api = enable_api
- dataset.updated_by = None
- dataset.updated_at = None
- dataset.summary_index_setting = summary_index_setting
- for key, value in kwargs.items():
- setattr(dataset, key, value)
- return dataset
+ ) -> Dataset:
+ return Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ permission=permission,
+ created_by=created_by,
+ indexing_technique=indexing_technique,
+ embedding_model_provider=embedding_model_provider,
+ embedding_model=embedding_model,
+ built_in_field_enabled=built_in_field_enabled,
+ chunk_structure=doc_form,
+ enable_api=enable_api,
+ updated_by=None,
+ updated_at=None,
+ summary_index_setting=summary_index_setting,
+ **kwargs,
+ )
@staticmethod
def create_user_mock(
@@ -196,14 +195,12 @@ class DatasetServiceUnitDataFactory:
role: str = TenantAccountRole.OWNER,
**kwargs,
) -> SimpleNamespace:
- user = SimpleNamespace(
+ return SimpleNamespace(
id=user_id,
current_tenant_id=tenant_id,
current_role=role,
+ **kwargs,
)
- for key, value in kwargs.items():
- setattr(user, key, value)
- return user
@staticmethod
def create_document_mock(
@@ -224,34 +221,45 @@ class DatasetServiceUnitDataFactory:
doc_metadata: dict[str, Any] | None = None,
name: str = "Document",
**kwargs,
- ) -> Mock:
- document = Mock(spec=Document)
- document.id = document_id
- document.dataset_id = dataset_id
- document.tenant_id = tenant_id
- document.indexing_status = indexing_status
- document.is_paused = is_paused
- document.paused_by = None
- document.paused_at = None
- document.archived = archived
- document.enabled = enabled
- document.data_source_type = data_source_type
- document.data_source_info_dict = data_source_info_dict or {}
- document.data_source_info = data_source_info
- document.doc_form = doc_form
- document.need_summary = need_summary
- document.position = position
- document.doc_metadata = doc_metadata
- document.name = name
- for key, value in kwargs.items():
- setattr(document, key, value)
- return document
+ ) -> Document:
+ return Document(
+ id=document_id,
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ indexing_status=indexing_status,
+ is_paused=is_paused,
+ paused_by=None,
+ paused_at=None,
+ archived=archived,
+ enabled=enabled,
+ data_source_type=data_source_type,
+ data_source_info=data_source_info
+ if data_source_info is not None
+ else json.dumps(data_source_info_dict or {}),
+ doc_form=doc_form,
+ need_summary=need_summary,
+ position=position,
+ doc_metadata=doc_metadata,
+ name=name,
+ **kwargs,
+ )
@staticmethod
- def create_upload_file_mock(file_id: str = "file-123", name: str = "upload.txt") -> Mock:
- upload_file = Mock(spec=UploadFile)
+ def create_upload_file_mock(file_id: str = "file-123", name: str = "upload.txt") -> UploadFile:
+ upload_file = UploadFile(
+ tenant_id="tenant-id",
+ storage_type="opendal",
+ key="test-key",
+ name=name,
+ size=0,
+ extension="txt",
+ mime_type="text/plain",
+ created_by_role="account",
+ created_by="account-id",
+ created_at=datetime.now(),
+ used=False,
+ )
upload_file.id = file_id
- upload_file.name = name
return upload_file
@@ -281,21 +289,20 @@ def _make_dataset(
tenant_id: str = "tenant-1",
data_source_type: str | None = None,
indexing_technique: str | None = "economy",
- latest_process_rule=None,
-) -> Mock:
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_id
- dataset.tenant_id = tenant_id
- dataset.data_source_type = data_source_type
- dataset.indexing_technique = indexing_technique
- dataset.latest_process_rule = latest_process_rule
- dataset.get_latest_process_rule.return_value = latest_process_rule
- dataset.embedding_model_provider = "provider"
- dataset.embedding_model = "embedding-model"
- dataset.summary_index_setting = None
- dataset.retrieval_model = None
- dataset.collection_binding_id = None
- return dataset
+ doc_form: str = IndexStructureType.PARAGRAPH_INDEX,
+) -> Dataset:
+ return Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ data_source_type=data_source_type,
+ indexing_technique=indexing_technique,
+ chunk_structure=doc_form,
+ embedding_model_provider="provider",
+ embedding_model="embedding-model",
+ summary_index_setting=None,
+ retrieval_model=None,
+ collection_binding_id=None,
+ )
def _make_document(
@@ -310,31 +317,29 @@ def _make_document(
enabled: bool = True,
archived: bool = False,
indexing_status: str = "completed",
- display_status: str = "available",
-) -> Mock:
- document = Mock(spec=Document)
- document.id = document_id
- document.dataset_id = dataset_id
- document.tenant_id = tenant_id
- document.batch = batch
- document.doc_form = doc_form
- document.word_count = word_count
- document.name = name
- document.enabled = enabled
- document.archived = archived
- document.indexing_status = indexing_status
- document.display_status = display_status
- document.data_source_type = "upload_file"
- document.data_source_info = "{}"
- document.completed_at = SimpleNamespace()
- document.processing_started_at = "started"
- document.parsing_completed_at = "parsed"
- document.cleaning_completed_at = "cleaned"
- document.splitting_completed_at = "split"
- document.updated_at = None
- document.created_from = None
- document.dataset_process_rule_id = "process-rule-1"
- return document
+) -> Document:
+ return Document(
+ id=document_id,
+ dataset_id=dataset_id,
+ tenant_id=tenant_id,
+ batch=batch,
+ doc_form=doc_form,
+ word_count=word_count,
+ name=name,
+ enabled=enabled,
+ archived=archived,
+ indexing_status=indexing_status,
+ data_source_type="upload_file",
+ data_source_info="{}",
+ completed_at=SimpleNamespace(),
+ processing_started_at="started",
+ parsing_completed_at="parsed",
+ cleaning_completed_at="cleaned",
+ splitting_completed_at="split",
+ updated_at=None,
+ created_from=None,
+ dataset_process_rule_id="process-rule-1",
+ )
def _make_segment(
@@ -347,21 +352,26 @@ def _make_segment(
index_node_id: str = "node-1",
dataset_id: str = "dataset-1",
document_id: str = "doc-1",
-) -> Mock:
- segment = Mock(spec=DocumentSegment)
+) -> DocumentSegment:
+ segment = DocumentSegment(
+ tenant_id="tenant-id",
+ dataset_id=dataset_id,
+ document_id=document_id,
+ position=1,
+ content=content,
+ word_count=word_count,
+ tokens=0,
+ created_by="account-id",
+ enabled=enabled,
+ keywords=keywords or [],
+ answer=None,
+ index_node_id=index_node_id,
+ disabled_at=None,
+ disabled_by=None,
+ status="completed",
+ error=None,
+ )
segment.id = segment_id
- segment.dataset_id = dataset_id
- segment.document_id = document_id
- segment.content = content
- segment.word_count = word_count
- segment.enabled = enabled
- segment.keywords = keywords or []
- segment.answer = None
- segment.index_node_id = index_node_id
- segment.disabled_at = None
- segment.disabled_by = None
- segment.status = "completed"
- segment.error = None
return segment
diff --git a/api/tests/unit_tests/services/document_service_validation.py b/api/tests/unit_tests/services/document_service_validation.py
index 8fad09e4d4c..bb0a6fd5948 100644
--- a/api/tests/unit_tests/services/document_service_validation.py
+++ b/api/tests/unit_tests/services/document_service_validation.py
@@ -156,9 +156,9 @@ class DocumentValidationTestDataFactory:
embedding_model_provider: str = "openai",
embedding_model: str = "text-embedding-ada-002",
**kwargs,
- ) -> Mock:
+ ) -> Dataset:
"""
- Create a mock Dataset with specified attributes.
+ Create a Dataset with specified attributes.
Args:
dataset_id: Unique identifier for the dataset
@@ -167,22 +167,20 @@ class DocumentValidationTestDataFactory:
indexing_technique: Indexing technique
embedding_model_provider: Embedding model provider
embedding_model: Embedding model name
- **kwargs: Additional attributes to set on the mock
+ **kwargs: Additional mapped attributes for the dataset
Returns:
- Mock object configured as a Dataset instance
- """
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_id
- dataset.tenant_id = tenant_id
- dataset.doc_form = doc_form
- dataset.get_doc_form.return_value = doc_form
- dataset.indexing_technique = indexing_technique
- dataset.embedding_model_provider = embedding_model_provider
- dataset.embedding_model = embedding_model
- for key, value in kwargs.items():
- setattr(dataset, key, value)
- return dataset
+ Configured Dataset instance
+ """
+ return Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ chunk_structure=doc_form,
+ indexing_technique=indexing_technique,
+ embedding_model_provider=embedding_model_provider,
+ embedding_model=embedding_model,
+ **kwargs,
+ )
@staticmethod
def create_knowledge_config_mock(
@@ -205,14 +203,14 @@ class DocumentValidationTestDataFactory:
Returns:
Mock object configured as a KnowledgeConfig instance
"""
- config = Mock(spec=KnowledgeConfig)
- config.data_source = data_source
- config.process_rule = process_rule
- config.doc_form = doc_form
- config.indexing_technique = indexing_technique
- for key, value in kwargs.items():
- setattr(config, key, value)
- return config
+ return Mock(
+ spec=KnowledgeConfig,
+ data_source=data_source,
+ process_rule=process_rule,
+ doc_form=doc_form,
+ indexing_technique=indexing_technique,
+ **kwargs,
+ )
@staticmethod
def create_data_source_mock(
@@ -352,6 +350,7 @@ class TestDatasetServiceCheckDocForm:
dataset = DocumentValidationTestDataFactory.create_dataset_mock(doc_form=None)
doc_form = IndexStructureType.PARAGRAPH_INDEX
session = Mock()
+ session.scalar.return_value = None
# Act (should not raise)
DatasetService.check_doc_form(dataset, doc_form, session=session)
diff --git a/api/tests/unit_tests/services/hit_service.py b/api/tests/unit_tests/services/hit_service.py
index 2a456dc4b9d..6ef10ad0726 100644
--- a/api/tests/unit_tests/services/hit_service.py
+++ b/api/tests/unit_tests/services/hit_service.py
@@ -16,7 +16,7 @@ from sqlalchemy.orm import Session
from core.rag.models.document import Document
from core.rag.retrieval.retrieval_methods import RetrievalMethod
-from models import Account
+from models import Account, Tenant
from models.dataset import Dataset, DatasetQuery
from services.hit_testing_service import HitTestingService
@@ -41,27 +41,27 @@ class HitTestingTestDataFactory:
provider: str = "vendor",
retrieval_model: dict[str, Any] | None = None,
**kwargs,
- ) -> Mock:
+ ) -> Dataset:
"""
- Create a mock dataset with specified attributes.
+ Create a dataset with specified attributes.
Args:
dataset_id: Unique identifier for the dataset
tenant_id: Tenant identifier
provider: Dataset provider (vendor, external, etc.)
retrieval_model: Optional retrieval model configuration
- **kwargs: Additional attributes to set on the mock
+ **kwargs: Additional mapped attributes for the dataset
Returns:
- Mock object configured as a Dataset instance
- """
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_id
- dataset.tenant_id = tenant_id
- dataset.provider = provider
- dataset.retrieval_model = retrieval_model
- for key, value in kwargs.items():
- setattr(dataset, key, value)
+ Configured Dataset instance
+ """
+ dataset = Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ provider=provider,
+ retrieval_model=retrieval_model,
+ **kwargs,
+ )
return dataset
@staticmethod
@@ -69,7 +69,7 @@ class HitTestingTestDataFactory:
user_id: str = "user-789",
tenant_id: str = "tenant-123",
**kwargs,
- ) -> Mock:
+ ) -> Account:
"""
Create a mock user (Account) with specified attributes.
@@ -81,12 +81,10 @@ class HitTestingTestDataFactory:
Returns:
Mock object configured as an Account instance
"""
- user = Mock(spec=Account)
+ user = Account(name="Test User", email="[email protected]", **kwargs)
user.id = user_id
- user.current_tenant_id = tenant_id
- user.name = "Test User"
- for key, value in kwargs.items():
- setattr(user, key, value)
+ user._current_tenant = Tenant(name="Test Tenant")
+ user._current_tenant.id = tenant_id
return user
@staticmethod
@@ -106,12 +104,7 @@ class HitTestingTestDataFactory:
Returns:
Mock object configured as a Document instance
"""
- document = Mock(spec=Document)
- document.page_content = content
- document.metadata = metadata or {}
- for key, value in kwargs.items():
- setattr(document, key, value)
- return document
+ return Mock(spec=Document, page_content=content, metadata=metadata or {}, **kwargs)
@staticmethod
def create_retrieval_record_mock(
diff --git a/api/tests/unit_tests/services/test_dataset_service_dataset.py b/api/tests/unit_tests/services/test_dataset_service_dataset.py
index 1fcc81039ca..b8e7023d4f7 100644
--- a/api/tests/unit_tests/services/test_dataset_service_dataset.py
+++ b/api/tests/unit_tests/services/test_dataset_service_dataset.py
@@ -34,11 +34,10 @@ class TestDatasetServiceValidation:
def test_check_doc_form_allows_matching_or_missing_dataset_doc_form(self, dataset_doc_form, incoming_doc_form):
dataset = DatasetServiceUnitDataFactory.create_dataset_mock(doc_form=dataset_doc_form)
session = MagicMock()
+ session.scalar.return_value = None
DatasetService.check_doc_form(dataset, incoming_doc_form, session=session)
- dataset.get_doc_form.assert_called_once_with(session=session)
-
def test_check_doc_form_rejects_mismatched_doc_form(self):
dataset = DatasetServiceUnitDataFactory.create_dataset_mock(doc_form="qa_model")
session = MagicMock()
@@ -46,8 +45,6 @@ class TestDatasetServiceValidation:
with pytest.raises(ValueError, match="doc_form is different"):
DatasetService.check_doc_form(dataset, "text_model", session=session)
- dataset.get_doc_form.assert_called_once_with(session=session)
-
def test_check_dataset_model_setting_skips_non_high_quality_datasets(self):
dataset = DatasetServiceUnitDataFactory.create_dataset_mock(indexing_technique="economy")
diff --git a/api/tests/unit_tests/services/test_dataset_service_document.py b/api/tests/unit_tests/services/test_dataset_service_document.py
index 6dd58afdd74..cb0d50e503e 100644
--- a/api/tests/unit_tests/services/test_dataset_service_document.py
+++ b/api/tests/unit_tests/services/test_dataset_service_document.py
@@ -158,10 +158,12 @@ class TestDocumentServiceMutations:
def test_delete_documents_limits_query_and_cleanup_to_dataset_ref(self):
session = MagicMock()
- dataset = _make_dataset(dataset_id="dataset-1", tenant_id="tenant-1")
- dataset.doc_form = "paragraph_index"
+ dataset = _make_dataset(
+ dataset_id="dataset-1",
+ tenant_id="tenant-1",
+ doc_form=IndexStructureType.PARAGRAPH_INDEX,
+ )
document = _make_document(document_id="doc-1", dataset_id=dataset.id, tenant_id=dataset.tenant_id)
- document.data_source_info_dict = {}
with (
patch("services.dataset_service.batch_clean_document_task") as clean_task,
@@ -345,7 +347,6 @@ class TestDocumentServiceMutations:
document_id="doc-1",
data_source_info_dict={"mode": "crawl"},
)
- document.data_source_info = "{}"
with (
patch("services.dataset_service.redis_client") as mock_redis,
@@ -768,13 +769,14 @@ class TestDocumentServiceCreateValidation:
],
)
def test_data_source_args_validate_requires_source_specific_info(self, data_source_type, field_name, message):
- info_list = SimpleNamespace(
- data_source_type=data_source_type,
- file_info_list=object(),
- notion_info_list=object(),
- website_info_list=object(),
- )
- setattr(info_list, field_name, None)
+ info_values = {
+ "data_source_type": data_source_type,
+ "file_info_list": object(),
+ "notion_info_list": object(),
+ "website_info_list": object(),
+ }
+ info_values[field_name] = None
+ info_list = SimpleNamespace(**info_values)
knowledge_config = SimpleNamespace(data_source=SimpleNamespace(info_list=info_list))
with pytest.raises(ValueError, match=message):
@@ -970,7 +972,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
)
def test_save_document_with_dataset_id_requires_existing_process_rule_for_custom_mode(self, account_context):
- dataset = _make_dataset(latest_process_rule=None)
+ dataset = _make_dataset()
knowledge_config = _make_upload_knowledge_config(
file_ids=["file-1"],
process_rule=ProcessRule(mode="custom"),
@@ -978,6 +980,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)):
session = MagicMock()
+ session.scalar.return_value = None
with pytest.raises(ValueError, match="No process rule found"):
DocumentService.save_document_with_dataset_id(
dataset,
@@ -986,7 +989,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
session=session,
)
- dataset.get_latest_process_rule.assert_called_once_with(session=session)
+ session.scalar.assert_called_once()
def test_save_document_with_dataset_id_rejects_invalid_indexing_technique(self, account_context):
dataset = _make_dataset(indexing_technique=None)
@@ -1897,7 +1900,7 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
self, account_context
):
session = MagicMock()
- dataset = _make_dataset(latest_process_rule=None)
+ dataset = _make_dataset()
knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"], process_rule=None)
created_process_rule = SimpleNamespace(id="rule-fallback")
created_document = _make_document(document_id="doc-created", name="file.txt")
@@ -1915,6 +1918,7 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
mock_redis.lock.return_value = _make_lock_context()
process_rule_cls.AUTOMATIC_RULES = DatasetProcessRule.AUTOMATIC_RULES
process_rule_cls.return_value = created_process_rule
+ session.scalar.return_value = None
session.scalars.return_value.all.side_effect = [[SimpleNamespace(id="file-1", name="file.txt")], []]
DocumentService.save_document_with_dataset_id(
@@ -1924,7 +1928,7 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
session=session,
)
- dataset.get_latest_process_rule.assert_called_once_with(session=session)
+ session.scalar.assert_called_once()
assert process_rule_cls.call_args.kwargs == {
"dataset_id": "dataset-1",
"mode": "automatic",
diff --git a/api/tests/unit_tests/services/test_external_dataset_service.py b/api/tests/unit_tests/services/test_external_dataset_service.py
index 62dfdda6b80..629fc07ec7b 100644
--- a/api/tests/unit_tests/services/test_external_dataset_service.py
+++ b/api/tests/unit_tests/services/test_external_dataset_service.py
@@ -9,7 +9,7 @@ import json
import re
from datetime import datetime
from typing import Any
-from unittest.mock import MagicMock, Mock, patch
+from unittest.mock import MagicMock, patch
import pytest
@@ -35,28 +35,29 @@ class ExternalDatasetServiceTestDataFactory:
tenant_id: str = "tenant-123",
name: str = "Test API",
settings: dict[str, Any] | None = None,
- **kwargs,
- ) -> Mock:
- """Create a mock ExternalKnowledgeApis object."""
- api = Mock(spec=ExternalKnowledgeApis)
+ description: str = "Test description",
+ created_by: str = "user-123",
+ updated_by: str = "user-123",
+ created_at: datetime = datetime(2024, 1, 1, 12, 0),
+ updated_at: datetime = datetime(2024, 1, 1, 12, 0),
+ ) -> ExternalKnowledgeApis:
+ """Create an ExternalKnowledgeApis object."""
+ api = ExternalKnowledgeApis(
+ name=name,
+ description=description,
+ tenant_id=tenant_id,
+ settings="{}",
+ created_by=created_by,
+ updated_by=updated_by,
+ )
api.id = api_id
- api.tenant_id = tenant_id
- api.name = name
- api.description = kwargs.get("description", "Test description")
if settings is None:
settings = {"endpoint": "https://api.example.com", "api_key": "test-key-123"}
api.settings = json.dumps(settings, ensure_ascii=False)
- api.settings_dict = settings
- api.created_by = kwargs.get("created_by", "user-123")
- api.updated_by = kwargs.get("updated_by", "user-123")
- api.created_at = kwargs.get("created_at", datetime(2024, 1, 1, 12, 0))
- api.updated_at = kwargs.get("updated_at", datetime(2024, 1, 1, 12, 0))
-
- for key, value in kwargs.items():
- if key not in ["description", "created_by", "updated_by", "created_at", "updated_at"]:
- setattr(api, key, value)
+ api.created_at = created_at
+ api.updated_at = updated_at
return api
@@ -66,23 +67,20 @@ class ExternalDatasetServiceTestDataFactory:
tenant_id: str = "tenant-123",
name: str = "Test Dataset",
provider: str = "external",
- **kwargs,
- ) -> Mock:
- """Create a mock Dataset object."""
- dataset = Mock(spec=Dataset)
- dataset.id = dataset_id
- dataset.tenant_id = tenant_id
- dataset.name = name
- dataset.provider = provider
- dataset.description = kwargs.get("description", "")
- dataset.retrieval_model = kwargs.get("retrieval_model", {})
- dataset.created_by = kwargs.get("created_by", "user-123")
-
- for key, value in kwargs.items():
- if key not in ["description", "retrieval_model", "created_by"]:
- setattr(dataset, key, value)
-
- return dataset
+ description: str = "",
+ retrieval_model: dict[str, Any] | None = None,
+ created_by: str = "user-123",
+ ) -> Dataset:
+ """Create a Dataset object."""
+ return Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ name=name,
+ provider=provider,
+ description=description,
+ retrieval_model=retrieval_model or {},
+ created_by=created_by,
+ )
@staticmethod
def create_external_knowledge_binding_mock(
@@ -91,20 +89,17 @@ class ExternalDatasetServiceTestDataFactory:
dataset_id: str = "dataset-123",
external_knowledge_api_id: str = "api-123",
external_knowledge_id: str = "knowledge-123",
- **kwargs,
- ) -> Mock:
- """Create a mock ExternalKnowledgeBindings object."""
- binding = Mock(spec=ExternalKnowledgeBindings)
+ created_by: str = "user-123",
+ ) -> ExternalKnowledgeBindings:
+ """Create an ExternalKnowledgeBindings object."""
+ binding = ExternalKnowledgeBindings(
+ tenant_id=tenant_id,
+ external_knowledge_api_id=external_knowledge_api_id,
+ dataset_id=dataset_id,
+ external_knowledge_id=external_knowledge_id,
+ created_by=created_by,
+ )
binding.id = binding_id
- binding.tenant_id = tenant_id
- binding.dataset_id = dataset_id
- binding.external_knowledge_api_id = external_knowledge_api_id
- binding.external_knowledge_id = external_knowledge_id
- binding.created_by = kwargs.get("created_by", "user-123")
-
- for key, value in kwargs.items():
- if key != "created_by":
- setattr(binding, key, value)
return binding