Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: use real ORM models in dataset service tests (#40018)
Asuka Minato479a5613a829
diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.pyindex eccf5ed56d1..51feb9bdb70 100644--- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py+++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py@@ -32,7 +32,7 @@ class TestDataSourceContentPreviewApi:payload = self._valid_payload()- pipeline = MagicMock(spec=Pipeline)+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")node_id = "node-1"account = make_account()@@ -72,7 +72,7 @@ class TestDataSourceContentPreviewApi:# datasource_type missing}- pipeline = MagicMock(spec=Pipeline)+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")account = make_account()with (@@ -92,7 +92,7 @@ class TestDataSourceContentPreviewApi:"credential_id": None,}- pipeline = MagicMock(spec=Pipeline)+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")account = make_account()service_instance = MagicMock()diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_import.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_import.pyindex 248c6c210a8..48b70e32f61 100644--- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_import.py+++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_import.py@@ -213,7 +213,7 @@ class TestRagPipelineImportCheckDependenciesApi:api = RagPipelineImportCheckDependenciesApi()method = unwrap(api.get)- pipeline = MagicMock(spec=Pipeline)+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")result = CheckDependenciesResult()service = MagicMock()@@ -235,7 +235,7 @@ class TestRagPipelineImportCheckDependenciesApi:api = RagPipelineImportCheckDependenciesApi()method = unwrap(api.get)- pipeline = MagicMock(spec=Pipeline)+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")dependency = PluginDependency(type=PluginDependencyType.Marketplace,value=PluginDependency.Marketplace(@@ -276,7 +276,7 @@ class TestRagPipelineExportApi:api = RagPipelineExportApi()method = unwrap(api.get)- pipeline = MagicMock(spec=Pipeline)+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")service = MagicMock()service.export_rag_pipeline_dsl.return_value = "yaml: data"diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets.pyindex 541c4b3f392..68b1ad0dd60 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py@@ -1272,18 +1272,20 @@ class TestDatasetApiKeyApi:def test_get_api_keys_success(self, app: Flask):api = DatasetApiKeyApi()method = unwrap(api.get)- mock_key_1 = MagicMock(spec=ApiToken)- mock_key_1.id = "key-1"- mock_key_1.type = "dataset"- mock_key_1.token = "ds-abc"- mock_key_1.last_used_at = None- mock_key_1.created_at = None- mock_key_2 = MagicMock(spec=ApiToken)- mock_key_2.id = "key-2"- mock_key_2.type = "dataset"- mock_key_2.token = "ds-def"- mock_key_2.last_used_at = None- mock_key_2.created_at = None+ mock_key_1 = ApiToken(+ id="key-1",+ type="dataset",+ token="ds-abc",+ last_used_at=None,+ created_at=None,+ )+ mock_key_2 = ApiToken(+ id="key-2",+ type="dataset",+ token="ds-def",+ last_used_at=None,+ created_at=None,+ )session = MagicMock()session.scalars.return_value.all.return_value = [mock_key_1, mock_key_2]with app.test_request_context("/"):diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.pyindex 73c028da03b..22597a21f9b 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_segments.py@@ -1,3 +1,4 @@+from datetime import datetimefrom inspect import unwrapfrom types import SimpleNamespacefrom typing import Any, cast@@ -42,16 +43,16 @@ def _segment():word_count=1,tokens=1,created_by="u1",+ answer="a",+ keywords=["test"],+ index_node_id="n1",+ index_node_hash="h",+ status=SegmentStatus.COMPLETED,+ updated_by="u1",)segment.id = "seg-1"- segment.answer = "a"- segment.keywords = ["test"]- segment.index_node_id = "n1"- segment.index_node_hash = "h"- segment.status = SegmentStatus.COMPLETEDsegment.created_at = naive_utc_now()segment.updated_at = naive_utc_now()- segment.updated_by = "u1"return segment@@ -65,9 +66,9 @@ def _child_chunk():content="child",word_count=1,created_by="u1",+ type=SegmentType.CUSTOMIZED,)child_chunk.id = "cc-1"- child_chunk.type = SegmentType.CUSTOMIZEDchild_chunk.created_at = naive_utc_now()child_chunk.updated_at = naive_utc_now()return child_chunk@@ -532,8 +533,19 @@ class TestDatasetDocumentSegmentBatchImportApi:api = DatasetDocumentSegmentBatchImportApi()method = unwrap(api.post)payload = {"upload_file_id": "file-1"}- upload_file = MagicMock(spec=UploadFile)- upload_file.name = "test.csv"+ upload_file = UploadFile(+ tenant_id="tenant-id",+ storage_type="opendal",+ key="test-key",+ name="test.csv",+ size=0,+ extension="txt",+ mime_type="text/plain",+ created_by_role="account",+ created_by="account-id",+ created_at=datetime.now(),+ used=False,+ )user = MagicMock(id="u1")session = MagicMock()session.scalar.return_value = upload_file@@ -1079,8 +1091,20 @@ class TestSegmentOperationCases:user = MagicMock(is_dataset_editor=True)dataset = MagicMock()document = MagicMock()- upload_file = MagicMock(spec=UploadFile, extension="csv", id="file-1")- upload_file.name = "test.csv"+ upload_file = UploadFile(+ tenant_id="tenant-id",+ storage_type="opendal",+ key="test-key",+ name="test.csv",+ size=0,+ extension="csv",+ mime_type="text/csv",+ created_by_role="account",+ created_by="account-id",+ created_at=datetime.now(),+ used=False,+ )+ upload_file.id = "file-1"payload = {"upload_file_id": "file-1"}session = MagicMock()session.scalar.return_value = upload_filediff --git a/api/tests/unit_tests/controllers/console/datasets/test_wraps.py b/api/tests/unit_tests/controllers/console/datasets/test_wraps.pyindex 80ee65a7094..ab7a7978991 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_wraps.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_wraps.py@@ -39,9 +39,11 @@ class TestGetRagPipeline:get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=session_factory.return_value)def test_pipeline_found_and_injected(self, mocker: MockerFixture):- pipeline = Mock(spec=Pipeline)+ pipeline = Pipeline(+ tenant_id="tenant-1",+ name="Test Pipeline",+ )pipeline.id = "pipeline-1"- pipeline.tenant_id = "tenant-1"@get_rag_pipelinedef dummy_view(**kwargs):@@ -64,7 +66,7 @@ class TestGetRagPipeline:get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=session_factory.return_value)def test_load_rag_pipeline_uses_provided_session(self, mocker: MockerFixture):- pipeline = Mock(spec=Pipeline)+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")session = Mock(spec=Session)mocker.patch(@@ -82,7 +84,7 @@ class TestGetRagPipeline:get_pipeline_by_id.assert_called_once_with("pipeline-1", "tenant-1", session=session)def test_pipeline_id_removed_from_kwargs(self, mocker: MockerFixture):- pipeline = Mock(spec=Pipeline)+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")@get_rag_pipelinedef dummy_view(**kwargs):@@ -105,7 +107,7 @@ class TestGetRagPipeline:assert result == "ok"def test_pipeline_id_cast_to_string(self, mocker: MockerFixture):- pipeline = Mock(spec=Pipeline)+ pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline")@get_rag_pipelinedef dummy_view(**kwargs):diff --git a/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py b/api/tests/unit_tests/core/datasource/test_datasource_file_manager.pyindex 20730e2e8a9..827950fae2f 100644--- a/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py+++ b/api/tests/unit_tests/core/datasource/test_datasource_file_manager.py@@ -1,3 +1,4 @@+from datetime import datetimefrom unittest.mock import MagicMock, patchimport httpx@@ -219,9 +220,19 @@ class TestDatasourceFileManager:@patch("core.datasource.datasource_file_manager.storage")def test_get_file_binary(self, mock_storage, mock_db):# Setup- mock_upload_file = MagicMock(spec=UploadFile)- mock_upload_file.key = "some_key"- mock_upload_file.mime_type = "image/png"+ mock_upload_file = UploadFile(+ tenant_id="tenant-id",+ storage_type="opendal",+ key="some_key",+ name="test.txt",+ size=0,+ extension="txt",+ mime_type="image/png",+ created_by_role="account",+ created_by="account-id",+ created_at=datetime.now(),+ used=False,+ )mock_db.session.get.return_value = mock_upload_file@@ -241,12 +252,22 @@ class TestDatasourceFileManager:@patch("core.datasource.datasource_file_manager.storage")def test_get_file_binary_by_message_file_id(self, mock_storage, mock_db):# Setup- mock_message_file = MagicMock(spec=MessageFile)- mock_message_file.url = "http://localhost/files/tools/tool_id.png"+ mock_message_file = MessageFile(+ message_id="message-id",+ type="image",+ transfer_method="local_file",+ created_by_role="user",+ created_by="user-id",+ url="http://localhost/files/tools/tool_id.png",+ )- mock_tool_file = MagicMock(spec=ToolFile)- mock_tool_file.file_key = "tool_key"- mock_tool_file.mimetype = "image/png"+ mock_tool_file = ToolFile(+ user_id="user-id",+ tenant_id="tenant-id",+ conversation_id="conversation-id",+ file_key="tool_key",+ mimetype="image/png",+ )def mock_get(model, id):if model == MessageFile:@@ -268,13 +289,23 @@ class TestDatasourceFileManager:@patch("core.datasource.datasource_file_manager.storage")def test_get_file_binary_by_message_file_id_with_extension(self, mock_storage, mock_db):# Test that it correctly parses tool_id even with extension in URL- mock_message_file = MagicMock(spec=MessageFile)- mock_message_file.url = "http://localhost/files/tools/abcdef.png"+ mock_message_file = MessageFile(+ message_id="message-id",+ type="image",+ transfer_method="local_file",+ created_by_role="user",+ created_by="user-id",+ url="http://localhost/files/tools/abcdef.png",+ )- mock_tool_file = MagicMock(spec=ToolFile)+ mock_tool_file = ToolFile(+ user_id="user-id",+ tenant_id="tenant-id",+ conversation_id="conversation-id",+ file_key="tk",+ mimetype="image/png",+ )mock_tool_file.id = "abcdef"- mock_tool_file.file_key = "tk"- mock_tool_file.mimetype = "image/png"def mock_get(model, id):if model == MessageFile:@@ -295,8 +326,14 @@ class TestDatasourceFileManager:assert DatasourceFileManager.get_file_binary_by_message_file_id("none") is None# Case 2: Message file found but tool file not found- mock_message_file = MagicMock(spec=MessageFile)- mock_message_file.url = None+ mock_message_file = MessageFile(+ message_id="message-id",+ type="image",+ transfer_method="local_file",+ created_by_role="user",+ created_by="user-id",+ url=None,+ )def mock_get_v2(model, id):if model == MessageFile:@@ -310,9 +347,19 @@ class TestDatasourceFileManager:@patch("core.datasource.datasource_file_manager.storage")def test_get_file_generator_by_upload_file_id(self, mock_storage, mock_db):# Setup- mock_upload_file = MagicMock(spec=UploadFile)- mock_upload_file.key = "upload_key"- mock_upload_file.mime_type = "text/plain"+ mock_upload_file = UploadFile(+ tenant_id="tenant-id",+ storage_type="opendal",+ key="upload_key",+ name="test.txt",+ size=0,+ extension="txt",+ mime_type="text/plain",+ created_by_role="account",+ created_by="account-id",+ created_at=datetime.now(),+ used=False,+ )mock_db.session.get.return_value = mock_upload_filediff --git a/api/tests/unit_tests/core/datasource/utils/test_message_transformer.py b/api/tests/unit_tests/core/datasource/utils/test_message_transformer.pyindex 0fca43cd0b1..30ecb5b4003 100644--- a/api/tests/unit_tests/core/datasource/utils/test_message_transformer.py+++ b/api/tests/unit_tests/core/datasource/utils/test_message_transformer.py@@ -40,9 +40,14 @@ class TestDatasourceFileMessageTransformer:def test_transform_image_message_success(self, mock_guess_ext, mock_tool_file_manager_cls):# Setupmock_manager = mock_tool_file_manager_cls.return_value- mock_tool_file = MagicMock(spec=ToolFile)+ mock_tool_file = ToolFile(+ user_id="user-id",+ tenant_id="tenant-id",+ conversation_id="conversation-id",+ file_key="test-key",+ mimetype="image/png",+ )mock_tool_file.id = "file_id_123"- mock_tool_file.mimetype = "image/png"mock_manager.create_file_by_url.return_value = mock_tool_filemock_guess_ext.return_value = ".png"@@ -101,9 +106,14 @@ class TestDatasourceFileMessageTransformer:def test_transform_blob_message_image(self, mock_guess_ext, mock_tool_file_manager_cls):# Setupmock_manager = mock_tool_file_manager_cls.return_value- mock_tool_file = MagicMock(spec=ToolFile)+ mock_tool_file = ToolFile(+ user_id="user-id",+ tenant_id="tenant-id",+ conversation_id="conversation-id",+ file_key="test-key",+ mimetype="image/jpeg",+ )mock_tool_file.id = "blob_id_456"- mock_tool_file.mimetype = "image/jpeg"mock_manager.create_file_by_raw.return_value = mock_tool_filemock_guess_ext.return_value = ".jpg"@@ -137,9 +147,14 @@ class TestDatasourceFileMessageTransformer:):# Setupmock_manager = mock_tool_file_manager_cls.return_value- mock_tool_file = MagicMock(spec=ToolFile)+ mock_tool_file = ToolFile(+ user_id="user-id",+ tenant_id="tenant-id",+ conversation_id="conversation-id",+ file_key="test-key",+ mimetype="application/pdf",+ )mock_tool_file.id = "blob_id_789"- mock_tool_file.mimetype = "application/pdf"mock_manager.create_file_by_raw.return_value = mock_tool_filemock_guess_type.return_value = ("application/pdf", None)mock_guess_ext.return_value = ".pdf"@@ -291,7 +306,13 @@ class TestDatasourceFileMessageTransformer:# This tests line 70 where filename might be Nonewith patch("core.datasource.utils.message_transformer.ToolFileManager") as mock_tool_file_manager_cls:mock_manager = mock_tool_file_manager_cls.return_value- mock_tool_file = MagicMock(spec=ToolFile)+ mock_tool_file = ToolFile(+ user_id="user-id",+ tenant_id="tenant-id",+ conversation_id="conversation-id",+ file_key="test-key",+ mimetype="text/plain",+ )mock_tool_file.id = "blob_id_no_name"mock_tool_file.mimetype = "application/octet-stream"mock_manager.create_file_by_raw.return_value = mock_tool_filediff --git a/api/tests/unit_tests/core/moderation/api/test_api.py b/api/tests/unit_tests/core/moderation/api/test_api.pyindex f075deefe72..73f6709c438 100644--- a/api/tests/unit_tests/core/moderation/api/test_api.py+++ b/api/tests/unit_tests/core/moderation/api/test_api.py@@ -50,7 +50,9 @@ class TestApiModeration:@patch("core.moderation.api.api.ApiModeration._get_api_based_extension")def test_validate_config_success(self, mock_get_extension, api_config):- mock_get_extension.return_value = MagicMock(spec=APIBasedExtension)+ mock_get_extension.return_value = APIBasedExtension(+ tenant_id="tenant-id", name="Test Extension", api_endpoint="https://example.com", api_key="test-key"+ )ApiModeration.validate_config("test-tenant-id", api_config)mock_get_extension.assert_called_once_with("test-tenant-id", "test-extension-id", db.session)@@ -136,9 +138,12 @@ class TestApiModeration:@patch("core.moderation.api.api.decrypt_token")@patch("core.moderation.api.api.APIBasedExtensionRequestor")def test_get_config_by_requestor_success(self, mock_requestor_cls, mock_decrypt, mock_get_ext, api_moderation):- mock_ext = MagicMock(spec=APIBasedExtension)- mock_ext.api_endpoint = "http://api.test"- mock_ext.api_key = "encrypted-key"+ mock_ext = APIBasedExtension(+ tenant_id="tenant-id",+ name="Test Extension",+ api_endpoint="http://api.test",+ api_key="encrypted-key",+ )mock_get_ext.return_value = mock_extmock_decrypt.return_value = "decrypted-key"diff --git a/api/tests/unit_tests/services/controller_api.py b/api/tests/unit_tests/services/controller_api.pyindex 48805da884e..b106f34964a 100644--- a/api/tests/unit_tests/services/controller_api.py+++ b/api/tests/unit_tests/services/controller_api.py@@ -192,34 +192,27 @@ class ControllerApiTestDataFactory:tenant_id: str = "tenant-123",permission: DatasetPermissionEnum = DatasetPermissionEnum.ONLY_ME,**kwargs,- ) -> Mock:+ ) -> Dataset:"""- Create a mock Dataset instance.+ Create a Dataset instance.Args:dataset_id: Unique identifier for the datasetname: Name of the datasettenant_id: Tenant identifierpermission: Dataset permission level- **kwargs: Additional attributes to set on the mock+ **kwargs: Additional mapped attributes for the datasetReturns:- Mock object configured as a Dataset instance+ Configured Dataset instance"""- dataset = Mock(spec=Dataset)- dataset.id = dataset_id- dataset.name = name- dataset.tenant_id = tenant_id- dataset.permission = permission- dataset.to_dict.return_value = {- "id": dataset_id,- "name": name,- "tenant_id": tenant_id,- "permission": permission.value,- }- for key, value in kwargs.items():- setattr(dataset, key, value)- return dataset+ return Dataset(+ id=dataset_id,+ name=name,+ tenant_id=tenant_id,+ permission=permission,+ **kwargs,+ )@staticmethoddef create_user_mock(@@ -240,15 +233,14 @@ class ControllerApiTestDataFactory:Returns:Mock object configured as a user/account instance"""- user = Mock()- user.id = user_id- user.current_tenant_id = tenant_id- user.is_dataset_editor = is_dataset_editor- user.has_edit_permission = True- user.is_dataset_operator = False- for key, value in kwargs.items():- setattr(user, key, value)- return user+ return Mock(+ id=user_id,+ current_tenant_id=tenant_id,+ is_dataset_editor=is_dataset_editor,+ has_edit_permission=True,+ is_dataset_operator=False,+ **kwargs,+ )@staticmethoddef create_paginated_response(items, total, page=1, per_page=20):@@ -822,19 +814,18 @@ class TestExternalDatasetApi:@pytest.fixturedef mock_current_account_context(self, app: Flask) -> Iterator[Mock]:"""Provide the wrapper auth context required by HTTP-client controller tests."""- mock_user = Account(name="Test User", email="[email protected]")+ mock_user = Account(+ name="Test User",+ email="[email protected]",+ status=AccountStatus.ACTIVE,+ )mock_user.id = "user-123"- mock_user.status = AccountStatus.ACTIVEmock_user.role = TenantAccountRole.EDITORdef load_user_from_request_context() -> None:g._login_user = mock_user- setattr( # noqa: B010- app,- "login_manager",- SimpleNamespace(load_user_from_request_context=load_user_from_request_context),- )+ app.login_manager = SimpleNamespace(load_user_from_request_context=load_user_from_request_context)with (patch("controllers.console.wraps.current_account_with_tenant") as mock_get_user,diff --git a/api/tests/unit_tests/services/dataset_service_test_helpers.py b/api/tests/unit_tests/services/dataset_service_test_helpers.pyindex d3dfe2fce79..d08913f537d 100644--- a/api/tests/unit_tests/services/dataset_service_test_helpers.py+++ b/api/tests/unit_tests/services/dataset_service_test_helpers.py@@ -6,6 +6,7 @@ document, and segment service test modules that exercise"""import json+from datetime import datetimefrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import MagicMock, Mock, create_autospec, patch@@ -169,25 +170,23 @@ class DatasetServiceUnitDataFactory:enable_api: bool = False,summary_index_setting: dict[str, Any] | None = None,**kwargs,- ) -> Mock:- dataset = Mock(spec=Dataset)- dataset.id = dataset_id- dataset.tenant_id = tenant_id- dataset.permission = permission- dataset.created_by = created_by- dataset.indexing_technique = indexing_technique- dataset.embedding_model_provider = embedding_model_provider- dataset.embedding_model = embedding_model- dataset.built_in_field_enabled = built_in_field_enabled- dataset.doc_form = doc_form- dataset.get_doc_form.return_value = doc_form- dataset.enable_api = enable_api- dataset.updated_by = None- dataset.updated_at = None- dataset.summary_index_setting = summary_index_setting- for key, value in kwargs.items():- setattr(dataset, key, value)- return dataset+ ) -> Dataset:+ return Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ permission=permission,+ created_by=created_by,+ indexing_technique=indexing_technique,+ embedding_model_provider=embedding_model_provider,+ embedding_model=embedding_model,+ built_in_field_enabled=built_in_field_enabled,+ chunk_structure=doc_form,+ enable_api=enable_api,+ updated_by=None,+ updated_at=None,+ summary_index_setting=summary_index_setting,+ **kwargs,+ )@staticmethoddef create_user_mock(@@ -196,14 +195,12 @@ class DatasetServiceUnitDataFactory:role: str = TenantAccountRole.OWNER,**kwargs,) -> SimpleNamespace:- user = SimpleNamespace(+ return SimpleNamespace(id=user_id,current_tenant_id=tenant_id,current_role=role,+ **kwargs,)- for key, value in kwargs.items():- setattr(user, key, value)- return user@staticmethoddef create_document_mock(@@ -224,34 +221,45 @@ class DatasetServiceUnitDataFactory:doc_metadata: dict[str, Any] | None = None,name: str = "Document",**kwargs,- ) -> Mock:- document = Mock(spec=Document)- document.id = document_id- document.dataset_id = dataset_id- document.tenant_id = tenant_id- document.indexing_status = indexing_status- document.is_paused = is_paused- document.paused_by = None- document.paused_at = None- document.archived = archived- document.enabled = enabled- document.data_source_type = data_source_type- document.data_source_info_dict = data_source_info_dict or {}- document.data_source_info = data_source_info- document.doc_form = doc_form- document.need_summary = need_summary- document.position = position- document.doc_metadata = doc_metadata- document.name = name- for key, value in kwargs.items():- setattr(document, key, value)- return document+ ) -> Document:+ return Document(+ id=document_id,+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ indexing_status=indexing_status,+ is_paused=is_paused,+ paused_by=None,+ paused_at=None,+ archived=archived,+ enabled=enabled,+ data_source_type=data_source_type,+ data_source_info=data_source_info+ if data_source_info is not None+ else json.dumps(data_source_info_dict or {}),+ doc_form=doc_form,+ need_summary=need_summary,+ position=position,+ doc_metadata=doc_metadata,+ name=name,+ **kwargs,+ )@staticmethod- def create_upload_file_mock(file_id: str = "file-123", name: str = "upload.txt") -> Mock:- upload_file = Mock(spec=UploadFile)+ def create_upload_file_mock(file_id: str = "file-123", name: str = "upload.txt") -> UploadFile:+ upload_file = UploadFile(+ tenant_id="tenant-id",+ storage_type="opendal",+ key="test-key",+ name=name,+ size=0,+ extension="txt",+ mime_type="text/plain",+ created_by_role="account",+ created_by="account-id",+ created_at=datetime.now(),+ used=False,+ )upload_file.id = file_id- upload_file.name = namereturn upload_file@@ -281,21 +289,20 @@ def _make_dataset(tenant_id: str = "tenant-1",data_source_type: str | None = None,indexing_technique: str | None = "economy",- latest_process_rule=None,-) -> Mock:- dataset = Mock(spec=Dataset)- dataset.id = dataset_id- dataset.tenant_id = tenant_id- dataset.data_source_type = data_source_type- dataset.indexing_technique = indexing_technique- dataset.latest_process_rule = latest_process_rule- dataset.get_latest_process_rule.return_value = latest_process_rule- dataset.embedding_model_provider = "provider"- dataset.embedding_model = "embedding-model"- dataset.summary_index_setting = None- dataset.retrieval_model = None- dataset.collection_binding_id = None- return dataset+ doc_form: str = IndexStructureType.PARAGRAPH_INDEX,+) -> Dataset:+ return Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ data_source_type=data_source_type,+ indexing_technique=indexing_technique,+ chunk_structure=doc_form,+ embedding_model_provider="provider",+ embedding_model="embedding-model",+ summary_index_setting=None,+ retrieval_model=None,+ collection_binding_id=None,+ )def _make_document(@@ -310,31 +317,29 @@ def _make_document(enabled: bool = True,archived: bool = False,indexing_status: str = "completed",- display_status: str = "available",-) -> Mock:- document = Mock(spec=Document)- document.id = document_id- document.dataset_id = dataset_id- document.tenant_id = tenant_id- document.batch = batch- document.doc_form = doc_form- document.word_count = word_count- document.name = name- document.enabled = enabled- document.archived = archived- document.indexing_status = indexing_status- document.display_status = display_status- document.data_source_type = "upload_file"- document.data_source_info = "{}"- document.completed_at = SimpleNamespace()- document.processing_started_at = "started"- document.parsing_completed_at = "parsed"- document.cleaning_completed_at = "cleaned"- document.splitting_completed_at = "split"- document.updated_at = None- document.created_from = None- document.dataset_process_rule_id = "process-rule-1"- return document+) -> Document:+ return Document(+ id=document_id,+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ batch=batch,+ doc_form=doc_form,+ word_count=word_count,+ name=name,+ enabled=enabled,+ archived=archived,+ indexing_status=indexing_status,+ data_source_type="upload_file",+ data_source_info="{}",+ completed_at=SimpleNamespace(),+ processing_started_at="started",+ parsing_completed_at="parsed",+ cleaning_completed_at="cleaned",+ splitting_completed_at="split",+ updated_at=None,+ created_from=None,+ dataset_process_rule_id="process-rule-1",+ )def _make_segment(@@ -347,21 +352,26 @@ def _make_segment(index_node_id: str = "node-1",dataset_id: str = "dataset-1",document_id: str = "doc-1",-) -> Mock:- segment = Mock(spec=DocumentSegment)+) -> DocumentSegment:+ segment = DocumentSegment(+ tenant_id="tenant-id",+ dataset_id=dataset_id,+ document_id=document_id,+ position=1,+ content=content,+ word_count=word_count,+ tokens=0,+ created_by="account-id",+ enabled=enabled,+ keywords=keywords or [],+ answer=None,+ index_node_id=index_node_id,+ disabled_at=None,+ disabled_by=None,+ status="completed",+ error=None,+ )segment.id = segment_id- segment.dataset_id = dataset_id- segment.document_id = document_id- segment.content = content- segment.word_count = word_count- segment.enabled = enabled- segment.keywords = keywords or []- segment.answer = None- segment.index_node_id = index_node_id- segment.disabled_at = None- segment.disabled_by = None- segment.status = "completed"- segment.error = Nonereturn segmentdiff --git a/api/tests/unit_tests/services/document_service_validation.py b/api/tests/unit_tests/services/document_service_validation.pyindex 8fad09e4d4c..bb0a6fd5948 100644--- a/api/tests/unit_tests/services/document_service_validation.py+++ b/api/tests/unit_tests/services/document_service_validation.py@@ -156,9 +156,9 @@ class DocumentValidationTestDataFactory:embedding_model_provider: str = "openai",embedding_model: str = "text-embedding-ada-002",**kwargs,- ) -> Mock:+ ) -> Dataset:"""- Create a mock Dataset with specified attributes.+ Create a Dataset with specified attributes.Args:dataset_id: Unique identifier for the dataset@@ -167,22 +167,20 @@ class DocumentValidationTestDataFactory:indexing_technique: Indexing techniqueembedding_model_provider: Embedding model providerembedding_model: Embedding model name- **kwargs: Additional attributes to set on the mock+ **kwargs: Additional mapped attributes for the datasetReturns:- Mock object configured as a Dataset instance- """- dataset = Mock(spec=Dataset)- dataset.id = dataset_id- dataset.tenant_id = tenant_id- dataset.doc_form = doc_form- dataset.get_doc_form.return_value = doc_form- dataset.indexing_technique = indexing_technique- dataset.embedding_model_provider = embedding_model_provider- dataset.embedding_model = embedding_model- for key, value in kwargs.items():- setattr(dataset, key, value)- return dataset+ Configured Dataset instance+ """+ return Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ chunk_structure=doc_form,+ indexing_technique=indexing_technique,+ embedding_model_provider=embedding_model_provider,+ embedding_model=embedding_model,+ **kwargs,+ )@staticmethoddef create_knowledge_config_mock(@@ -205,14 +203,14 @@ class DocumentValidationTestDataFactory:Returns:Mock object configured as a KnowledgeConfig instance"""- config = Mock(spec=KnowledgeConfig)- config.data_source = data_source- config.process_rule = process_rule- config.doc_form = doc_form- config.indexing_technique = indexing_technique- for key, value in kwargs.items():- setattr(config, key, value)- return config+ return Mock(+ spec=KnowledgeConfig,+ data_source=data_source,+ process_rule=process_rule,+ doc_form=doc_form,+ indexing_technique=indexing_technique,+ **kwargs,+ )@staticmethoddef create_data_source_mock(@@ -352,6 +350,7 @@ class TestDatasetServiceCheckDocForm:dataset = DocumentValidationTestDataFactory.create_dataset_mock(doc_form=None)doc_form = IndexStructureType.PARAGRAPH_INDEXsession = Mock()+ session.scalar.return_value = None# Act (should not raise)DatasetService.check_doc_form(dataset, doc_form, session=session)diff --git a/api/tests/unit_tests/services/hit_service.py b/api/tests/unit_tests/services/hit_service.pyindex 2a456dc4b9d..6ef10ad0726 100644--- a/api/tests/unit_tests/services/hit_service.py+++ b/api/tests/unit_tests/services/hit_service.py@@ -16,7 +16,7 @@ from sqlalchemy.orm import Sessionfrom core.rag.models.document import Documentfrom core.rag.retrieval.retrieval_methods import RetrievalMethod-from models import Account+from models import Account, Tenantfrom models.dataset import Dataset, DatasetQueryfrom services.hit_testing_service import HitTestingService@@ -41,27 +41,27 @@ class HitTestingTestDataFactory:provider: str = "vendor",retrieval_model: dict[str, Any] | None = None,**kwargs,- ) -> Mock:+ ) -> Dataset:"""- Create a mock dataset with specified attributes.+ Create a dataset with specified attributes.Args:dataset_id: Unique identifier for the datasettenant_id: Tenant identifierprovider: Dataset provider (vendor, external, etc.)retrieval_model: Optional retrieval model configuration- **kwargs: Additional attributes to set on the mock+ **kwargs: Additional mapped attributes for the datasetReturns:- Mock object configured as a Dataset instance- """- dataset = Mock(spec=Dataset)- dataset.id = dataset_id- dataset.tenant_id = tenant_id- dataset.provider = provider- dataset.retrieval_model = retrieval_model- for key, value in kwargs.items():- setattr(dataset, key, value)+ Configured Dataset instance+ """+ dataset = Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ provider=provider,+ retrieval_model=retrieval_model,+ **kwargs,+ )return dataset@staticmethod@@ -69,7 +69,7 @@ class HitTestingTestDataFactory:user_id: str = "user-789",tenant_id: str = "tenant-123",**kwargs,- ) -> Mock:+ ) -> Account:"""Create a mock user (Account) with specified attributes.@@ -81,12 +81,10 @@ class HitTestingTestDataFactory:Returns:Mock object configured as an Account instance"""- user = Mock(spec=Account)+ user = Account(name="Test User", email="[email protected]", **kwargs)user.id = user_id- user.current_tenant_id = tenant_id- user.name = "Test User"- for key, value in kwargs.items():- setattr(user, key, value)+ user._current_tenant = Tenant(name="Test Tenant")+ user._current_tenant.id = tenant_idreturn user@staticmethod@@ -106,12 +104,7 @@ class HitTestingTestDataFactory:Returns:Mock object configured as a Document instance"""- document = Mock(spec=Document)- document.page_content = content- document.metadata = metadata or {}- for key, value in kwargs.items():- setattr(document, key, value)- return document+ return Mock(spec=Document, page_content=content, metadata=metadata or {}, **kwargs)@staticmethoddef create_retrieval_record_mock(diff --git a/api/tests/unit_tests/services/test_dataset_service_dataset.py b/api/tests/unit_tests/services/test_dataset_service_dataset.pyindex 1fcc81039ca..b8e7023d4f7 100644--- a/api/tests/unit_tests/services/test_dataset_service_dataset.py+++ b/api/tests/unit_tests/services/test_dataset_service_dataset.py@@ -34,11 +34,10 @@ class TestDatasetServiceValidation:def test_check_doc_form_allows_matching_or_missing_dataset_doc_form(self, dataset_doc_form, incoming_doc_form):dataset = DatasetServiceUnitDataFactory.create_dataset_mock(doc_form=dataset_doc_form)session = MagicMock()+ session.scalar.return_value = NoneDatasetService.check_doc_form(dataset, incoming_doc_form, session=session)- dataset.get_doc_form.assert_called_once_with(session=session)-def test_check_doc_form_rejects_mismatched_doc_form(self):dataset = DatasetServiceUnitDataFactory.create_dataset_mock(doc_form="qa_model")session = MagicMock()@@ -46,8 +45,6 @@ class TestDatasetServiceValidation:with pytest.raises(ValueError, match="doc_form is different"):DatasetService.check_doc_form(dataset, "text_model", session=session)- dataset.get_doc_form.assert_called_once_with(session=session)-def test_check_dataset_model_setting_skips_non_high_quality_datasets(self):dataset = DatasetServiceUnitDataFactory.create_dataset_mock(indexing_technique="economy")diff --git a/api/tests/unit_tests/services/test_dataset_service_document.py b/api/tests/unit_tests/services/test_dataset_service_document.pyindex 6dd58afdd74..cb0d50e503e 100644--- a/api/tests/unit_tests/services/test_dataset_service_document.py+++ b/api/tests/unit_tests/services/test_dataset_service_document.py@@ -158,10 +158,12 @@ class TestDocumentServiceMutations:def test_delete_documents_limits_query_and_cleanup_to_dataset_ref(self):session = MagicMock()- dataset = _make_dataset(dataset_id="dataset-1", tenant_id="tenant-1")- dataset.doc_form = "paragraph_index"+ dataset = _make_dataset(+ dataset_id="dataset-1",+ tenant_id="tenant-1",+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ )document = _make_document(document_id="doc-1", dataset_id=dataset.id, tenant_id=dataset.tenant_id)- document.data_source_info_dict = {}with (patch("services.dataset_service.batch_clean_document_task") as clean_task,@@ -345,7 +347,6 @@ class TestDocumentServiceMutations:document_id="doc-1",data_source_info_dict={"mode": "crawl"},)- document.data_source_info = "{}"with (patch("services.dataset_service.redis_client") as mock_redis,@@ -768,13 +769,14 @@ class TestDocumentServiceCreateValidation:],)def test_data_source_args_validate_requires_source_specific_info(self, data_source_type, field_name, message):- info_list = SimpleNamespace(- data_source_type=data_source_type,- file_info_list=object(),- notion_info_list=object(),- website_info_list=object(),- )- setattr(info_list, field_name, None)+ info_values = {+ "data_source_type": data_source_type,+ "file_info_list": object(),+ "notion_info_list": object(),+ "website_info_list": object(),+ }+ info_values[field_name] = None+ info_list = SimpleNamespace(**info_values)knowledge_config = SimpleNamespace(data_source=SimpleNamespace(info_list=info_list))with pytest.raises(ValueError, match=message):@@ -970,7 +972,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:)def test_save_document_with_dataset_id_requires_existing_process_rule_for_custom_mode(self, account_context):- dataset = _make_dataset(latest_process_rule=None)+ dataset = _make_dataset()knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"],process_rule=ProcessRule(mode="custom"),@@ -978,6 +980,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)):session = MagicMock()+ session.scalar.return_value = Nonewith pytest.raises(ValueError, match="No process rule found"):DocumentService.save_document_with_dataset_id(dataset,@@ -986,7 +989,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:session=session,)- dataset.get_latest_process_rule.assert_called_once_with(session=session)+ session.scalar.assert_called_once()def test_save_document_with_dataset_id_rejects_invalid_indexing_technique(self, account_context):dataset = _make_dataset(indexing_technique=None)@@ -1897,7 +1900,7 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:self, account_context):session = MagicMock()- dataset = _make_dataset(latest_process_rule=None)+ dataset = _make_dataset()knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"], process_rule=None)created_process_rule = SimpleNamespace(id="rule-fallback")created_document = _make_document(document_id="doc-created", name="file.txt")@@ -1915,6 +1918,7 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:mock_redis.lock.return_value = _make_lock_context()process_rule_cls.AUTOMATIC_RULES = DatasetProcessRule.AUTOMATIC_RULESprocess_rule_cls.return_value = created_process_rule+ session.scalar.return_value = Nonesession.scalars.return_value.all.side_effect = [[SimpleNamespace(id="file-1", name="file.txt")], []]DocumentService.save_document_with_dataset_id(@@ -1924,7 +1928,7 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:session=session,)- dataset.get_latest_process_rule.assert_called_once_with(session=session)+ session.scalar.assert_called_once()assert process_rule_cls.call_args.kwargs == {"dataset_id": "dataset-1","mode": "automatic",diff --git a/api/tests/unit_tests/services/test_external_dataset_service.py b/api/tests/unit_tests/services/test_external_dataset_service.pyindex 62dfdda6b80..629fc07ec7b 100644--- a/api/tests/unit_tests/services/test_external_dataset_service.py+++ b/api/tests/unit_tests/services/test_external_dataset_service.py@@ -9,7 +9,7 @@ import jsonimport refrom datetime import datetimefrom typing import Any-from unittest.mock import MagicMock, Mock, patch+from unittest.mock import MagicMock, patchimport pytest@@ -35,28 +35,29 @@ class ExternalDatasetServiceTestDataFactory:tenant_id: str = "tenant-123",name: str = "Test API",settings: dict[str, Any] | None = None,- **kwargs,- ) -> Mock:- """Create a mock ExternalKnowledgeApis object."""- api = Mock(spec=ExternalKnowledgeApis)+ description: str = "Test description",+ created_by: str = "user-123",+ updated_by: str = "user-123",+ created_at: datetime = datetime(2024, 1, 1, 12, 0),+ updated_at: datetime = datetime(2024, 1, 1, 12, 0),+ ) -> ExternalKnowledgeApis:+ """Create an ExternalKnowledgeApis object."""+ api = ExternalKnowledgeApis(+ name=name,+ description=description,+ tenant_id=tenant_id,+ settings="{}",+ created_by=created_by,+ updated_by=updated_by,+ )api.id = api_id- api.tenant_id = tenant_id- api.name = name- api.description = kwargs.get("description", "Test description")if settings is None:settings = {"endpoint": "https://api.example.com", "api_key": "test-key-123"}api.settings = json.dumps(settings, ensure_ascii=False)- api.settings_dict = settings- api.created_by = kwargs.get("created_by", "user-123")- api.updated_by = kwargs.get("updated_by", "user-123")- api.created_at = kwargs.get("created_at", datetime(2024, 1, 1, 12, 0))- api.updated_at = kwargs.get("updated_at", datetime(2024, 1, 1, 12, 0))-- for key, value in kwargs.items():- if key not in ["description", "created_by", "updated_by", "created_at", "updated_at"]:- setattr(api, key, value)+ api.created_at = created_at+ api.updated_at = updated_atreturn api@@ -66,23 +67,20 @@ class ExternalDatasetServiceTestDataFactory:tenant_id: str = "tenant-123",name: str = "Test Dataset",provider: str = "external",- **kwargs,- ) -> Mock:- """Create a mock Dataset object."""- dataset = Mock(spec=Dataset)- dataset.id = dataset_id- dataset.tenant_id = tenant_id- dataset.name = name- dataset.provider = provider- dataset.description = kwargs.get("description", "")- dataset.retrieval_model = kwargs.get("retrieval_model", {})- dataset.created_by = kwargs.get("created_by", "user-123")-- for key, value in kwargs.items():- if key not in ["description", "retrieval_model", "created_by"]:- setattr(dataset, key, value)-- return dataset+ description: str = "",+ retrieval_model: dict[str, Any] | None = None,+ created_by: str = "user-123",+ ) -> Dataset:+ """Create a Dataset object."""+ return Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ name=name,+ provider=provider,+ description=description,+ retrieval_model=retrieval_model or {},+ created_by=created_by,+ )@staticmethoddef create_external_knowledge_binding_mock(@@ -91,20 +89,17 @@ class ExternalDatasetServiceTestDataFactory:dataset_id: str = "dataset-123",external_knowledge_api_id: str = "api-123",external_knowledge_id: str = "knowledge-123",- **kwargs,- ) -> Mock:- """Create a mock ExternalKnowledgeBindings object."""- binding = Mock(spec=ExternalKnowledgeBindings)+ created_by: str = "user-123",+ ) -> ExternalKnowledgeBindings:+ """Create an ExternalKnowledgeBindings object."""+ binding = ExternalKnowledgeBindings(+ tenant_id=tenant_id,+ external_knowledge_api_id=external_knowledge_api_id,+ dataset_id=dataset_id,+ external_knowledge_id=external_knowledge_id,+ created_by=created_by,+ )binding.id = binding_id- binding.tenant_id = tenant_id- binding.dataset_id = dataset_id- binding.external_knowledge_api_id = external_knowledge_api_id- binding.external_knowledge_id = external_knowledge_id- binding.created_by = kwargs.get("created_by", "user-123")-- for key, value in kwargs.items():- if key != "created_by":- setattr(binding, key, value)return binding