Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: migrate dataset service sessions and ORM models to SQLite (#40534)

Asuka Minato2400c6c59d7a
diff --git a/api/tests/unit_tests/services/test_dataset_service_dataset.py b/api/tests/unit_tests/services/test_dataset_service_dataset.py
index 5487da2aa22..0d855f1d6ea 100644
--- a/api/tests/unit_tests/services/test_dataset_service_dataset.py
+++ b/api/tests/unit_tests/services/test_dataset_service_dataset.py
@@ -1,87 +1,189 @@
-"""Unit tests for DatasetService and dataset-related collaborators."""
-
-from sqlalchemy.orm import Session
-
-from models.dataset import DatasetPermission
+"""SQLite-backed tests for dataset-level operations in :mod:`services.dataset_service`.
+
+Mapped objects in this module are real SQLAlchemy models. Provider runtimes,
+RBAC clients, Celery tasks, and model-manager results remain mocked at their
+external boundaries.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+from sqlalchemy import event, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
+from core.rag.index_processor.constant.index_type import IndexTechniqueType
+from graphon.model_runtime.entities.model_entities import ModelFeature, ModelType
+from models import Account
+from models.account import Tenant
+from models.dataset import (
+ Dataset,
+ DatasetCollectionBinding,
+ DatasetPermission,
+ DatasetPermissionEnum,
+ ExternalKnowledgeApis,
+ ExternalKnowledgeBindings,
+ Pipeline,
+)
+from services.dataset_service import DatasetCollectionBindingService, DatasetPermissionService, DatasetService
+from services.entities.knowledge_entities.rag_pipeline_entities import (
+ IconInfo,
+ RagPipelineDatasetCreateEntity,
+)
+from services.errors.account import NoPermissionError
+from services.errors.dataset import DatasetNameDuplicateError
from .dataset_service_test_helpers import (
- DatasetNameDuplicateError,
- DatasetPermissionEnum,
- DatasetPermissionService,
- DatasetService,
- DatasetServiceUnitDataFactory,
- LLMBadRequestError,
MagicMock,
- ModelFeature,
- ModelType,
- NoPermissionError,
- PipelineIconInfo,
- ProviderTokenNotInitError,
- RagPipelineDatasetCreateEntity,
- SimpleNamespace,
TenantAccountRole,
_make_knowledge_configuration,
_make_retrieval_model,
- json,
- patch,
- pytest,
)
-class TestDatasetServiceValidation:
- """Unit tests for DatasetService validation helpers."""
+def _account(
+ *,
+ account_id: str = "user-1",
+ tenant_id: str = "tenant-1",
+ role: TenantAccountRole = TenantAccountRole.OWNER,
+) -> Account:
+ account = Account(name=f"User {account_id}", email=f"{account_id}@example.com")
+ account.id = account_id
+ account.role = role
+ tenant = Tenant(name=f"Tenant {tenant_id}")
+ tenant.id = tenant_id
+ account._current_tenant = tenant
+ return account
+
+
+def _dataset(
+ *,
+ dataset_id: str = "dataset-1",
+ tenant_id: str = "tenant-1",
+ name: str = "Dataset",
+ maintainer: str = "user-1",
+ permission: DatasetPermissionEnum = DatasetPermissionEnum.ALL_TEAM,
+ provider: str = "vendor",
+ indexing_technique: str = IndexTechniqueType.ECONOMY,
+ chunk_structure: str | None = "text_model",
+) -> Dataset:
+ return Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ name=name,
+ description="",
+ provider=provider,
+ created_by=maintainer,
+ maintainer=maintainer,
+ permission=permission,
+ indexing_technique=indexing_technique,
+ chunk_structure=chunk_structure,
+ embedding_model_provider="provider",
+ embedding_model="embedding-model",
+ )
+
+
+def _external_api(*, api_id: str = "api-1", tenant_id: str = "tenant-1") -> ExternalKnowledgeApis:
+ api = ExternalKnowledgeApis(
+ name="External API",
+ description="",
+ tenant_id=tenant_id,
+ settings="{}",
+ created_by="user-1",
+ updated_by="user-1",
+ )
+ api.id = api_id
+ return api
+
+
+def _binding(
+ *,
+ binding_id: str = "binding-1",
+ tenant_id: str = "tenant-1",
+ dataset_id: str = "dataset-1",
+ api_id: str = "api-1",
+ knowledge_id: str = "knowledge-1",
+) -> ExternalKnowledgeBindings:
+ binding = ExternalKnowledgeBindings(
+ tenant_id=tenant_id,
+ dataset_id=dataset_id,
+ external_knowledge_api_id=api_id,
+ external_knowledge_id=knowledge_id,
+ created_by="user-1",
+ )
+ binding.id = binding_id
+ return binding
+
+class TestDatasetServiceValidation:
@pytest.mark.parametrize(
("dataset_doc_form", "incoming_doc_form"),
[(None, "text_model"), ("text_model", "text_model")],
)
- def test_check_doc_form_allows_matching_or_missing_dataset_doc_form(self, dataset_doc_form, incoming_doc_form):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(doc_form=dataset_doc_form)
- session = MagicMock()
- session.scalar.return_value = None
+ def test_check_doc_form_allows_matching_or_missing_dataset_doc_form(
+ self,
+ sqlite_session: Session,
+ dataset_doc_form: str | None,
+ incoming_doc_form: str,
+ ) -> None:
+ dataset = _dataset(chunk_structure=dataset_doc_form)
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
- DatasetService.check_doc_form(dataset, incoming_doc_form, session=session)
+ DatasetService.check_doc_form(dataset, incoming_doc_form, session=sqlite_session)
- def test_check_doc_form_rejects_mismatched_doc_form(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(doc_form="qa_model")
- session = MagicMock()
+ def test_check_doc_form_rejects_mismatched_doc_form(self, sqlite_session: Session) -> None:
+ dataset = _dataset(chunk_structure="qa_model")
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
with pytest.raises(ValueError, match="doc_form is different"):
- DatasetService.check_doc_form(dataset, "text_model", session=session)
+ DatasetService.check_doc_form(dataset, "text_model", session=sqlite_session)
@pytest.mark.parametrize("operator_check", [False, True])
def test_dataset_permission_checks_ignore_foreign_tenant_binding(
self, sqlite_session: Session, operator_check: bool
) -> None:
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
+ dataset = _dataset(
dataset_id="dataset-1",
tenant_id="tenant-1",
permission=DatasetPermissionEnum.PARTIAL_TEAM,
maintainer="owner-1",
)
- user = DatasetServiceUnitDataFactory.create_user_mock(
- user_id="user-1",
+ user = _account(
+ account_id="user-1",
tenant_id="tenant-1",
role=TenantAccountRole.NORMAL,
)
- sqlite_session.add(DatasetPermission(dataset_id=dataset.id, account_id=user.id, tenant_id="tenant-2"))
+ sqlite_session.add_all(
+ [
+ dataset,
+ DatasetPermission(dataset_id=dataset.id, account_id=user.id, tenant_id="tenant-2"),
+ ]
+ )
+ sqlite_session.commit()
- with pytest.raises(NoPermissionError):
- if operator_check:
+ if operator_check:
+ with pytest.raises(NoPermissionError):
DatasetService.check_dataset_operator_permission(user, dataset, session=sqlite_session)
- else:
+ else:
+ with pytest.raises(NoPermissionError):
DatasetService.check_dataset_permission(dataset, user, sqlite_session)
- def test_check_dataset_model_setting_skips_non_high_quality_datasets(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(indexing_technique="economy")
+ def test_check_dataset_model_setting_skips_non_high_quality_datasets(self) -> None:
+ dataset = _dataset(indexing_technique=IndexTechniqueType.ECONOMY)
with patch("services.dataset_service.ModelManager") as model_manager_cls:
DatasetService.check_dataset_model_setting(dataset)
model_manager_cls.assert_not_called()
- def test_check_dataset_model_setting_validates_embedding_model_for_high_quality_dataset(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(indexing_technique="high_quality")
+ def test_check_dataset_model_setting_validates_high_quality_embedding(self) -> None:
+ dataset = _dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY)
with patch("services.dataset_service.ModelManager") as model_manager_cls:
DatasetService.check_dataset_model_setting(dataset)
@@ -93,57 +195,25 @@ class TestDatasetServiceValidation:
model=dataset.embedding_model,
)
- def test_check_dataset_model_setting_wraps_llm_bad_request_error(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(indexing_technique="high_quality")
-
- with patch("services.dataset_service.ModelManager") as model_manager_cls:
- model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = LLMBadRequestError()
-
- with pytest.raises(ValueError, match="No Embedding Model available"):
- DatasetService.check_dataset_model_setting(dataset)
-
- def test_check_dataset_model_setting_wraps_provider_token_error(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(indexing_technique="high_quality")
+ @pytest.mark.parametrize(
+ ("error", "message"),
+ [
+ (LLMBadRequestError(), "No Embedding Model available"),
+ (ProviderTokenNotInitError("token missing"), "token missing"),
+ ],
+ )
+ def test_check_dataset_model_setting_wraps_provider_errors(self, error: Exception, message: str) -> None:
+ dataset = _dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY)
with patch("services.dataset_service.ModelManager") as model_manager_cls:
- model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = ProviderTokenNotInitError(
- "token missing"
- )
-
- with pytest.raises(ValueError, match="The dataset is unavailable, due to: token missing"):
+ model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = error
+ with pytest.raises(ValueError, match=message):
DatasetService.check_dataset_model_setting(dataset)
- def test_check_embedding_model_setting_wraps_provider_token_error_description(self):
- with patch("services.dataset_service.ModelManager") as model_manager_cls:
- model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = ProviderTokenNotInitError(
- "provider setup"
- )
-
- with pytest.raises(ValueError, match="provider setup"):
- DatasetService.check_embedding_model_setting("tenant-1", "provider", "embedding-model")
-
- def test_check_reranking_model_setting_uses_rerank_model_type(self):
- with patch("services.dataset_service.ModelManager") as model_manager_cls:
- DatasetService.check_reranking_model_setting("tenant-1", "provider", "reranker")
-
- model_manager_cls.for_tenant.return_value.get_model_instance.assert_called_once_with(
- tenant_id="tenant-1",
- provider="provider",
- model_type=ModelType.RERANK,
- model="reranker",
- )
-
- def test_check_reranking_model_setting_wraps_bad_request(self):
- with patch("services.dataset_service.ModelManager") as model_manager_cls:
- model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = LLMBadRequestError()
-
- with pytest.raises(ValueError, match="No Rerank Model available"):
- DatasetService.check_reranking_model_setting("tenant-1", "provider", "reranker")
-
- def test_check_is_multimodal_model_returns_true_when_model_supports_vision(self):
- model_schema = SimpleNamespace(features=[ModelFeature.VISION])
+ @pytest.mark.parametrize(("features", "expected"), [([ModelFeature.VISION], True), ([], False)])
+ def test_check_is_multimodal_model_reads_runtime_schema(self, features: list[ModelFeature], expected: bool) -> None:
model_type_instance = MagicMock()
- model_type_instance.get_model_schema.return_value = model_schema
+ model_type_instance.get_model_schema.return_value = SimpleNamespace(features=features)
model_instance = SimpleNamespace(
model_type_instance=model_type_instance,
model_name="embedding-model",
@@ -152,512 +222,339 @@ class TestDatasetServiceValidation:
with patch("services.dataset_service.ModelManager") as model_manager_cls:
model_manager_cls.for_tenant.return_value.get_model_instance.return_value = model_instance
-
- result = DatasetService.check_is_multimodal_model("tenant-1", "provider", "embedding-model")
-
- assert result is True
-
- def test_check_is_multimodal_model_returns_false_when_vision_feature_is_absent(self):
- model_schema = SimpleNamespace(features=[])
- model_type_instance = MagicMock()
- model_type_instance.get_model_schema.return_value = model_schema
- model_instance = SimpleNamespace(
- model_type_instance=model_type_instance,
- model_name="embedding-model",
- credentials={"api_key": "secret"},
- )
-
- with patch("services.dataset_service.ModelManager") as model_manager_cls:
- model_manager_cls.for_tenant.return_value.get_model_instance.return_value = model_instance
-
result = DatasetService.check_is_multimodal_model("tenant-1", "provider", "embedding-model")
- assert result is False
+ assert result is expected
- def test_check_is_multimodal_model_raises_when_schema_is_missing(self):
+ def test_check_is_multimodal_model_rejects_missing_schema(self) -> None:
model_type_instance = MagicMock()
model_type_instance.get_model_schema.return_value = None
model_instance = SimpleNamespace(
model_type_instance=model_type_instance,
model_name="embedding-model",
- credentials={"api_key": "secret"},
+ credentials={},
)
with patch("services.dataset_service.ModelManager") as model_manager_cls:
model_manager_cls.for_tenant.return_value.get_model_instance.return_value = model_instance
-
with pytest.raises(ValueError, match="Model schema not found"):
DatasetService.check_is_multimodal_model("tenant-1", "provider", "embedding-model")
- def test_check_is_multimodal_model_wraps_bad_request_error(self):
+ @pytest.mark.parametrize(
+ ("method", "error", "message"),
+ [
+ (
+ DatasetService.check_embedding_model_setting,
+ ProviderTokenNotInitError("provider setup"),
+ "provider setup",
+ ),
+ (DatasetService.check_reranking_model_setting, LLMBadRequestError(), "No Rerank Model available"),
+ ],
+ )
+ def test_direct_model_setting_checks_wrap_runtime_errors(
+ self, method: Callable[[str, str, str], None], error: Exception, message: str
+ ) -> None:
with patch("services.dataset_service.ModelManager") as model_manager_cls:
- model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = LLMBadRequestError()
-
- with pytest.raises(ValueError, match="No Model available"):
- DatasetService.check_is_multimodal_model("tenant-1", "provider", "embedding-model")
+ model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = error
+ with pytest.raises(ValueError, match=message):
+ method("tenant-1", "provider", "model")
class TestDatasetServiceRetrieval:
- def test_get_dataset_for_tenant_scopes_query_by_dataset_and_tenant(self):
- session = MagicMock()
- expected_dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- dataset_id="dataset-1", tenant_id="tenant-1"
- )
- session.scalar.return_value = expected_dataset
-
- result = DatasetService.get_dataset_for_tenant("dataset-1", "tenant-1", session=session)
-
- assert result is expected_dataset
- statement = session.scalar.call_args.args[0].compile()
- assert "datasets.id" in str(statement)
- assert "datasets.tenant_id" in str(statement)
- assert "dataset-1" in statement.params.values()
- assert "tenant-1" in statement.params.values()
-
-
-class TestDatasetServiceRetrievalPermissions:
- """Unit tests for dataset list permission branching."""
-
- def test_get_datasets_filters_by_maintainer_and_rbac_overrides(self):
- session = MagicMock()
- explicit_session = MagicMock()
- explicit_session.scalars.return_value.all.return_value = []
- user = DatasetServiceUnitDataFactory.create_user_mock(role=TenantAccountRole.NORMAL)
+ def test_get_dataset_for_tenant_rejects_cross_tenant_row(self, sqlite_session: Session) -> None:
+ owned = _dataset()
+ foreign = _dataset(dataset_id="dataset-2", tenant_id="tenant-2", name="Foreign")
+ sqlite_session.add_all([owned, foreign])
+ sqlite_session.commit()
+
+ assert DatasetService.get_dataset_for_tenant(owned.id, "tenant-1", session=sqlite_session) is owned
+ assert DatasetService.get_dataset_for_tenant(foreign.id, "tenant-1", session=sqlite_session) is None
+
+ def test_get_datasets_applies_rbac_resource_scope_and_maintainer_override(self, sqlite_session: Session) -> None:
+ user = _account(role=TenantAccountRole.NORMAL)
+ accessible = _dataset(dataset_id="accessible", name="Accessible", maintainer="other")
+ owned = _dataset(dataset_id="owned", name="Owned", maintainer=user.id)
+ hidden = _dataset(dataset_id="hidden", name="Hidden", maintainer="other")
+ foreign = _dataset(dataset_id="foreign", tenant_id="tenant-2", name="Foreign")
+ sqlite_session.add_all([accessible, owned, hidden, foreign])
+ sqlite_session.commit()
with (
- patch("services.dataset_service.paginate_query") as mock_paginate,
patch("services.dataset_service.dify_config.RBAC_ENABLED", True),
patch(
"services.dataset_service.enterprise_rbac_service.RBACService.MyPermissions.get",
return_value=SimpleNamespace(workspace=SimpleNamespace(permission_keys=[])),
),
):
- mock_paginate.return_value = SimpleNamespace(items=[], total=0)
- DatasetService.get_datasets(
+ datasets, total = DatasetService.get_datasets(
page=1,
per_page=20,
- session=explicit_session,
+ session=sqlite_session,
tenant_id="tenant-1",
user=user,
- accessible_dataset_ids=["dataset-shared"],
+ accessible_dataset_ids=[accessible.id],
include_own_datasets=True,
)
- explicit_session.scalars.assert_called_once()
- session.scalars.assert_not_called()
- select_stmt = mock_paginate.call_args.args[0]
- visibility_clause = str(select_stmt._where_criteria[1])
- assert "maintainer" in visibility_clause
- assert "IN" in visibility_clause
+ assert total == 2
+ assert {dataset.id for dataset in datasets} == {accessible.id, owned.id}
- def test_get_datasets_filters_only_by_rbac_overrides_without_manage_own_permission(self):
- session = MagicMock()
- session.scalars.return_value.all.return_value = []
- user = DatasetServiceUnitDataFactory.create_user_mock(role=TenantAccountRole.NORMAL)
+ def test_get_datasets_without_user_keeps_only_team_visible_rows(self, sqlite_session: Session) -> None:
+ shared = _dataset(dataset_id="shared", name="Shared", permission=DatasetPermissionEnum.ALL_TEAM)
+ private = _dataset(dataset_id="private", name="Private", permission=DatasetPermissionEnum.ONLY_ME)
+ sqlite_session.add_all([shared, private])
+ sqlite_session.commit()
- with (
- patch("services.dataset_service.paginate_query") as mock_paginate,
- patch("services.dataset_service.dify_config.RBAC_ENABLED", True),
- patch(
- "services.dataset_service.enterprise_rbac_service.RBACService.MyPermissions.get",
- return_value=SimpleNamespace(workspace=SimpleNamespace(permission_keys=[])),
- ),
- ):
- mock_paginate.return_value = SimpleNamespace(items=[], total=0)
- DatasetService.get_datasets(
+ with patch("services.dataset_service.dify_config.RBAC_ENABLED", False):
+ datasets, total = DatasetService.get_datasets(
page=1,
per_page=20,
- session=session,
+ session=sqlite_session,
tenant_id="tenant-1",
- user=user,
- accessible_dataset_ids=["dataset-shared"],
)
- select_stmt = mock_paginate.call_args.args[0]
- visibility_clause = str(select_stmt._where_criteria[1])
- assert "maintainer" not in visibility_clause
- assert "IN" in visibility_clause
+ assert total == 1
+ assert [dataset.id for dataset in datasets] == [shared.id]
- def test_get_datasets_by_ids_applies_rbac_visibility(self):
- session = MagicMock()
- user = DatasetServiceUnitDataFactory.create_user_mock(role=TenantAccountRole.NORMAL)
+ def test_get_datasets_by_ids_intersects_requested_and_accessible_ids(self, sqlite_session: Session) -> None:
+ user = _account(role=TenantAccountRole.NORMAL)
+ accessible = _dataset(dataset_id="accessible", name="Accessible", maintainer="other")
+ owned = _dataset(dataset_id="owned", name="Owned", maintainer=user.id)
+ hidden = _dataset(dataset_id="hidden", name="Hidden", maintainer="other")
+ sqlite_session.add_all([accessible, owned, hidden])
+ sqlite_session.commit()
- with (
- patch("services.dataset_service.paginate_query") as mock_paginate,
- patch("services.dataset_service.dify_config.RBAC_ENABLED", True),
- ):
- mock_paginate.return_value = SimpleNamespace(items=[], total=0)
- DatasetService.get_datasets_by_ids(
- ["dataset-requested", "dataset-shared"],
+ with patch("services.dataset_service.dify_config.RBAC_ENABLED", True):
+ datasets, total = DatasetService.get_datasets_by_ids(
+ [accessible.id, owned.id, hidden.id],
"tenant-1",
user=user,
- accessible_dataset_ids=["dataset-shared", "dataset-not-requested"],
+ accessible_dataset_ids=[accessible.id, "not-requested"],
include_own_datasets=True,
- session=session,
+ session=sqlite_session,
)
- select_stmt = mock_paginate.call_args.args[0]
- visibility_clause = str(select_stmt._where_criteria[-1])
- assert "maintainer" in visibility_clause
- assert "IN" in visibility_clause
- visibility_params = select_stmt._where_criteria[-1].compile().params
- assert ["dataset-shared"] in visibility_params.values()
- list_params = [value for value in visibility_params.values() if isinstance(value, list)]
- assert all("dataset-not-requested" not in value for value in list_params)
-
- def test_get_datasets_rbac_include_all_uses_workspace_permission(self):
- session = MagicMock()
- session.scalars.return_value.all.return_value = []
+ assert total == 2
+ assert {dataset.id for dataset in datasets} == {accessible.id, owned.id}
- user = DatasetServiceUnitDataFactory.create_user_mock(role=TenantAccountRole.NORMAL)
- mock_permissions = SimpleNamespace(workspace=SimpleNamespace(permission_keys=["dataset.create_and_management"]))
+ def test_get_datasets_rbac_without_user_returns_no_rows(self, sqlite_session: Session) -> None:
+ sqlite_session.add(_dataset())
+ sqlite_session.commit()
- with (
- patch("services.dataset_service.paginate_query") as mock_paginate,
- patch("services.dataset_service.dify_config.RBAC_ENABLED", True),
- patch(
- "services.dataset_service.enterprise_rbac_service.RBACService.MyPermissions.get",
- return_value=mock_permissions,
- ),
- ):
- mock_paginate.return_value = SimpleNamespace(items=[], total=0)
- DatasetService.get_datasets(
+ with patch("services.dataset_service.dify_config.RBAC_ENABLED", True):
+ datasets, total = DatasetService.get_datasets(
page=1,
per_page=20,
- session=session,
+ session=sqlite_session,
tenant_id="tenant-1",
- user=user,
- include_all=True,
)
- session.scalars.assert_called_once()
- mock_paginate.assert_called_once()
- select_stmt = mock_paginate.call_args.args[0]
- assert len(select_stmt._where_criteria) == 1
+ assert datasets == []
+ assert total == 0
- def test_get_datasets_rbac_without_user_returns_empty_result(self):
- session = MagicMock()
- session.scalars.return_value.all.return_value = []
+ def test_get_datasets_rbac_include_all_requires_workspace_permission(self, sqlite_session: Session) -> None:
+ user = _account(role=TenantAccountRole.NORMAL)
+ sqlite_session.add_all(
+ [
+ _dataset(dataset_id="one", name="One", maintainer="other"),
+ _dataset(dataset_id="two", name="Two", maintainer="other"),
+ ]
+ )
+ sqlite_session.commit()
with (
- patch("services.dataset_service.paginate_query") as mock_paginate,
patch("services.dataset_service.dify_config.RBAC_ENABLED", True),
+ patch(
+ "services.dataset_service.enterprise_rbac_service.RBACService.MyPermissions.get",
+ return_value=SimpleNamespace(
+ workspace=SimpleNamespace(permission_keys=["dataset.create_and_management"])
+ ),
+ ),
):
- mock_paginate.return_value = SimpleNamespace(items=[], total=0)
- DatasetService.get_datasets(page=1, per_page=20, session=session, tenant_id="tenant-1", user=None)
-
- session.scalars.assert_not_called()
- mock_paginate.assert_called_once()
- select_stmt = mock_paginate.call_args.args[0]
- assert len(select_stmt._where_criteria) == 2
-
- def test_get_datasets_legacy_owner_include_all_keeps_full_access(self):
- session = MagicMock()
- session.scalars.return_value.all.return_value = []
-
- user = DatasetServiceUnitDataFactory.create_user_mock(role=TenantAccountRole.OWNER)
-
- with (
- patch("services.dataset_service.paginate_query") as mock_paginate,
- patch("services.dataset_service.dify_config.RBAC_ENABLED", False),
- ):
- mock_paginate.return_value = SimpleNamespace(items=[], total=0)
- DatasetService.get_datasets(
+ datasets, total = DatasetService.get_datasets(
page=1,
per_page=20,
- session=session,
+ session=sqlite_session,
tenant_id="tenant-1",
user=user,
include_all=True,
)
- session.scalars.assert_called_once()
- mock_paginate.assert_called_once()
- select_stmt = mock_paginate.call_args.args[0]
- assert len(select_stmt._where_criteria) == 1
+ assert total == 2
+ assert {dataset.id for dataset in datasets} == {"one", "two"}
class TestDatasetServiceCreationAndUpdate:
- """Unit tests for dataset creation and update helpers."""
-
- def test_create_empty_dataset_raises_when_name_already_exists(self):
- session = MagicMock()
- account = SimpleNamespace(id="user-1")
-
- session.scalar.return_value = object()
+ def test_create_empty_dataset_rejects_duplicate_name(self, sqlite_session: Session) -> None:
+ sqlite_session.add(_dataset(name="Existing"))
+ sqlite_session.commit()
- with pytest.raises(DatasetNameDuplicateError, match="Dataset with name Dataset already exists"):
- DatasetService.create_empty_dataset("tenant-1", "Dataset", None, "economy", account, session=session)
+ with pytest.raises(DatasetNameDuplicateError, match="already exists"):
+ DatasetService.create_empty_dataset(
+ "tenant-1",
+ "Existing",
+ "",
+ IndexTechniqueType.ECONOMY,
+ _account(),
+ session=sqlite_session,
+ )
- def test_create_empty_dataset_uses_default_embedding_model_for_high_quality_dataset(self):
- session = MagicMock()
- account = SimpleNamespace(id="user-1")
- default_embedding_model = SimpleNamespace(provider="provider", model_name="default-embedding")
+ def test_create_empty_dataset_persists_default_embedding_and_creator_scope(
+ self,
+ sqlite_session: Session,
+ sqlite_session_factory: sessionmaker[Session],
+ ) -> None:
+ embedding_model = SimpleNamespace(provider="provider", model_name="default-embedding")
with (
- patch("services.dataset_service.select"),
- patch(
- "services.dataset_service.Dataset",
- side_effect=lambda **kwargs: SimpleNamespace(id="dataset-1", **kwargs),
- ),
patch("services.dataset_service.ModelManager") as model_manager_cls,
- patch.object(DatasetService, "check_embedding_model_setting") as check_embedding,
+ patch("services.dataset_service.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"),
):
- session.scalar.return_value = None
- model_manager_cls.for_tenant.return_value.get_default_model_instance.return_value = default_embedding_model
-
+ model_manager_cls.for_tenant.return_value.get_default_model_instance.return_value = embedding_model
dataset = DatasetService.create_empty_dataset(
- tenant_id="tenant-1",
- name="Dataset",
- description="Description",
- indexing_technique="high_quality",
- account=account,
- session=session,
+ "tenant-1",
+ "Created",
+ "Description",
+ IndexTechniqueType.HIGH_QUALITY,
+ _account(),
+ session=sqlite_session,
)
- assert dataset.embedding_model_provider == "provider"
- assert dataset.embedding_model == "default-embedding"
- assert dataset.permission == DatasetPermissionEnum.ONLY_ME
- assert dataset.provider == "vendor"
- model_manager_cls.for_tenant.return_value.get_default_model_instance.assert_called_once_with(
- tenant_id="tenant-1",
- model_type=ModelType.TEXT_EMBEDDING,
- )
- check_embedding.assert_not_called()
- session.commit.assert_called_once()
-
- def test_create_empty_dataset_creates_external_binding_for_high_quality_dataset(self):
- session = MagicMock()
- account = SimpleNamespace(id="user-1")
- retrieval_model = _make_retrieval_model()
- embedding_model = SimpleNamespace(provider="provider", model_name="embedding-model")
+ with sqlite_session_factory() as observer:
+ persisted = observer.get(Dataset, dataset.id)
+ assert persisted is not None
+ assert persisted.tenant_id == "tenant-1"
+ assert persisted.maintainer == "user-1"
+ assert persisted.embedding_model == "default-embedding"
- with (
- patch("services.dataset_service.select"),
- patch(
- "services.dataset_service.Dataset",
- side_effect=lambda **kwargs: SimpleNamespace(id="dataset-1", **kwargs),
- ),
- patch(
- "services.dataset_service.ExternalKnowledgeBindings",
- side_effect=lambda **kwargs: SimpleNamespace(**kwargs),
- ) as binding_cls,
- patch("services.dataset_service.ModelManager") as model_manager_cls,
- patch("services.dataset_service.ExternalDatasetService.get_external_knowledge_api", return_value=object()),
- patch.object(DatasetService, "check_embedding_model_setting") as check_embedding,
- patch.object(DatasetService, "check_reranking_model_setting") as check_reranking,
- ):
- session.scalar.return_value = None
- model_manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model
+ def test_create_empty_external_dataset_persists_tenant_owned_binding(self, sqlite_session: Session) -> None:
+ sqlite_session.add_all([_external_api(), _external_api(api_id="api-foreign", tenant_id="tenant-2")])
+ sqlite_session.commit()
+ with patch("services.dataset_service.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"):
dataset = DatasetService.create_empty_dataset(
- tenant_id="tenant-1",
- name="External Dataset",
- description="Description",
- indexing_technique="high_quality",
- account=account,
- permission=DatasetPermissionEnum.ALL_TEAM,
+ "tenant-1",
+ "External",
+ "",
+ IndexTechniqueType.ECONOMY,
+ _account(),
provider="external",
external_knowledge_api_id="api-1",
external_knowledge_id="knowledge-1",
- embedding_model_provider="provider",
- embedding_model_name="embedding-model",
- retrieval_model=retrieval_model,
- summary_index_setting={"enable": True},
- session=session,
+ retrieval_model=_make_retrieval_model(),
+ session=sqlite_session,
)
- assert dataset.embedding_model_provider == "provider"
- assert dataset.embedding_model == "embedding-model"
- assert dataset.retrieval_model == retrieval_model.model_dump()
- assert dataset.summary_index_setting == {"enable": True}
- check_embedding.assert_called_once_with("tenant-1", "provider", "embedding-model")
- check_reranking.assert_called_once_with("tenant-1", "rerank-provider", "rerank-model")
- binding_cls.assert_called_once_with(
- tenant_id="tenant-1",
- dataset_id="dataset-1",
- external_knowledge_api_id="api-1",
- external_knowledge_id="knowledge-1",
- created_by="user-1",
+ binding = sqlite_session.scalar(
+ select(ExternalKnowledgeBindings).where(ExternalKnowledgeBindings.dataset_id == dataset.id)
)
- assert session.add.call_count == 2
- session.commit.assert_called_once()
+ assert binding is not None
+ assert binding.tenant_id == dataset.tenant_id
+ assert binding.external_knowledge_api_id == "api-1"
+
+ def test_create_empty_external_dataset_rejects_foreign_api(self, sqlite_session: Session) -> None:
+ sqlite_session.add(_external_api(api_id="api-foreign", tenant_id="tenant-2"))
+ sqlite_session.commit()
- def test_create_empty_rag_pipeline_dataset_raises_for_duplicate_name(self):
- session = MagicMock()
+ with (
+ pytest.raises(ValueError, match="api template not found"),
+ patch("services.dataset_service.enterprise_rbac_service.try_sync_creator_access_policy_member_bindings"),
+ ):
+ DatasetService.create_empty_dataset(
+ "tenant-1",
+ "External",
+ "",
+ IndexTechniqueType.ECONOMY,
+ _account(),
+ provider="external",
+ external_knowledge_api_id="api-foreign",
+ external_knowledge_id="knowledge-1",
+ session=sqlite_session,
+ )
+
+ def test_create_empty_rag_pipeline_dataset_generates_incremental_name(self, sqlite_session: Session) -> None:
+ sqlite_session.add(_dataset(name="Untitled 1"))
+ sqlite_session.commit()
entity = RagPipelineDatasetCreateEntity(
- name="Existing Dataset",
- description="Description",
- icon_info=PipelineIconInfo(icon="book", icon_background="#fff"),
+ name="",
+ description="Pipeline dataset",
+ icon_info=IconInfo(icon_type="emoji", icon="📚", icon_background="#FFFFFF", icon_url=None),
permission=DatasetPermissionEnum.ALL_TEAM,
)
- session.scalar.return_value = object()
+ with patch("services.dataset_service.current_user", _account()):
+ dataset = DatasetService.create_empty_rag_pipeline_dataset("tenant-1", entity, sqlite_session)
- with pytest.raises(DatasetNameDuplicateError, match="Existing Dataset already exists"):
- DatasetService.create_empty_rag_pipeline_dataset("tenant-1", entity, session)
+ assert dataset.name == "Untitled 2"
+ pipeline = sqlite_session.get(Pipeline, dataset.pipeline_id)
+ assert pipeline is not None
+ assert pipeline.tenant_id == dataset.tenant_id
- def test_create_empty_rag_pipeline_dataset_generates_name_and_creates_dataset(self):
- session = MagicMock()
+ def test_create_empty_rag_pipeline_dataset_rejects_duplicate_name(self, sqlite_session: Session) -> None:
+ sqlite_session.add(_dataset(name="Existing"))
+ sqlite_session.commit()
entity = RagPipelineDatasetCreateEntity(
- name="",
- description="Description",
- icon_info=PipelineIconInfo(icon="book", icon_background="#fff"),
+ name="Existing",
+ description="",
+ icon_info=IconInfo(icon_type="emoji", icon="📚", icon_background="#FFFFFF", icon_url=None),
permission=DatasetPermissionEnum.ALL_TEAM,
)
- pipeline = SimpleNamespace(id="pipeline-1")
-
- def pipeline_factory(**kwargs):
- pipeline.__dict__.update(kwargs)
- return pipeline
-
- def dataset_factory(**kwargs):
- return SimpleNamespace(id="dataset-1", **kwargs)
with (
- patch("services.dataset_service.select"),
- patch("services.dataset_service.current_user", SimpleNamespace(id="user-1")),
- patch("services.dataset_service.generate_incremental_name", return_value="Untitled 2") as generate_name,
- patch("services.dataset_service.Pipeline", side_effect=pipeline_factory),
- patch("services.dataset_service.Dataset", side_effect=dataset_factory),
+ patch("services.dataset_service.current_user", _account()),
+ pytest.raises(DatasetNameDuplicateError, match="already exists"),
):
- session.scalars.return_value.all.return_value = [
- SimpleNamespace(name="Untitled"),
- SimpleNamespace(name="Untitled 1"),
- ]
-
- dataset = DatasetService.create_empty_rag_pipeline_dataset("tenant-1", entity, session)
+ DatasetService.create_empty_rag_pipeline_dataset("tenant-1", entity, sqlite_session)
- assert entity.name == "Untitled 2"
- assert dataset.pipeline_id == "pipeline-1"
- assert dataset.runtime_mode == "rag_pipeline"
- generate_name.assert_called_once_with(["Untitled", "Untitled 1"], "Untitled")
- session.commit.assert_called_once()
-
- def test_create_empty_rag_pipeline_dataset_requires_current_user_id(self):
- session = MagicMock()
+ def test_create_empty_rag_pipeline_dataset_requires_authenticated_account(self, sqlite_session: Session) -> None:
+ account = _account()
+ account.id = ""
entity = RagPipelineDatasetCreateEntity(
name="Dataset",
- description="Description",
- icon_info=PipelineIconInfo(icon="book", icon_background="#fff"),
+ description="",
+ icon_info=IconInfo(icon_type="emoji", icon="📚", icon_background="#FFFFFF", icon_url=None),
permission=DatasetPermissionEnum.ALL_TEAM,
)
with (
- patch("services.dataset_service.current_user", SimpleNamespace(id=None)),
+ patch("services.dataset_service.current_user", account),
+ pytest.raises(ValueError, match="Current user or current user id not found"),
):
- session.scalar.return_value = None
-
- with pytest.raises(ValueError, match="Current user or current user id not found"):
- DatasetService.create_empty_rag_pipeline_dataset("tenant-1", entity, session)
-
- def test_update_dataset_raises_when_dataset_is_missing(self):
- session = MagicMock()
- with patch.object(DatasetService, "get_dataset", return_value=None):
- with pytest.raises(ValueError, match="Dataset not found"):
- DatasetService.update_dataset("dataset-1", {}, SimpleNamespace(id="user-1"), session=session)
+ DatasetService.create_empty_rag_pipeline_dataset("tenant-1", entity, sqlite_session)
- def test_update_dataset_raises_when_new_name_conflicts(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1", tenant_id="tenant-1")
- dataset.name = "Old Dataset"
+ def test_update_dataset_rejects_missing_and_duplicate_rows(self, sqlite_session: Session) -> None:
+ current = _dataset(name="Current")
+ duplicate = _dataset(dataset_id="dataset-2", name="Duplicate")
+ sqlite_session.add_all([current, duplicate])
+ sqlite_session.commit()
- with (
- patch.object(DatasetService, "get_dataset", return_value=dataset),
- patch.object(DatasetService, "_has_dataset_same_name", return_value=True),
- ):
- with pytest.raises(ValueError, match="Dataset name already exists"):
- DatasetService.update_dataset(
- "dataset-1", {"name": "New Dataset"}, SimpleNamespace(id="user-1"), session=MagicMock()
- )
-
- def test_update_dataset_routes_external_datasets_to_external_helper(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1", tenant_id="tenant-1")
- dataset.provider = "external"
- user = DatasetServiceUnitDataFactory.create_user_mock()
-
- with (
- patch.object(DatasetService, "get_dataset", return_value=dataset),
- patch.object(DatasetService, "check_dataset_permission") as check_permission,
- patch.object(DatasetService, "_update_external_dataset", return_value="updated") as update_external,
- ):
- session = MagicMock()
- result = DatasetService.update_dataset("dataset-1", {"name": dataset.name}, user, session=session)
-
- assert result == "updated"
- check_permission.assert_called_once()
- assert check_permission.call_args.args[:2] == (dataset, user)
- assert len(check_permission.call_args.args) == 3
- update_external.assert_called_once_with(dataset, {"name": dataset.name}, user, session)
-
- def test_update_dataset_routes_internal_datasets_to_internal_helper(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1", tenant_id="tenant-1")
- dataset.provider = "vendor"
- user = DatasetServiceUnitDataFactory.create_user_mock()
-
- with (
- patch.object(DatasetService, "get_dataset", return_value=dataset),
- patch.object(DatasetService, "check_dataset_permission") as check_permission,
- patch.object(DatasetService, "_update_internal_dataset", return_value="updated") as update_internal,
- ):
- session = MagicMock()
- result = DatasetService.update_dataset("dataset-1", {"name": dataset.name}, user, session=session)
-
- assert result == "updated"
- check_permission.assert_called_once()
- assert check_permission.call_args.args[:2] == (dataset, user)
- assert len(check_permission.call_args.args) == 3
- update_internal.assert_called_once_with(dataset, {"name": dataset.name}, user, session)
-
- def test_has_dataset_same_name_returns_true_when_query_matches(self):
- session = MagicMock()
- session.scalar.return_value = object()
-
- result = DatasetService._has_dataset_same_name("tenant-1", "dataset-1", "Dataset", session)
-
- assert result is True
+ with pytest.raises(ValueError, match="Dataset not found"):
+ DatasetService.update_dataset("missing", {}, _account(), session=sqlite_session)
+ with pytest.raises(ValueError, match="Dataset name already exists"):
+ DatasetService.update_dataset(current.id, {"name": duplicate.name}, _account(), session=sqlite_session)
- def test_update_external_dataset_updates_dataset_and_binding(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- user = SimpleNamespace(id="user-1")
- now = object()
+ @pytest.mark.parametrize(
+ ("provider", "helper_name"),
+ [("external", "_update_external_dataset"), ("vendor", "_update_internal_dataset")],
+ )
+ def test_update_dataset_routes_by_provider(self, sqlite_session: Session, provider: str, helper_name: str) -> None:
+ dataset = _dataset(provider=provider)
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
with (
- patch.object(DatasetService, "_update_external_knowledge_binding") as update_binding,
- patch(
- "services.dataset_service.ExternalDatasetService.get_external_knowledge_api", return_value=object()
- ) as get_external_knowledge_api,
- patch("services.dataset_service.naive_utc_now", return_value=now),
+ patch.object(DatasetService, "check_dataset_permission"),
+ patch.object(DatasetService, helper_name, return_value=dataset) as update_helper,
):
- result = DatasetService._update_external_dataset(
- dataset,
- {
- "external_retrieval_model": {"top_k": 3},
- "summary_index_setting": {"enable": True},
- "name": "Updated Dataset",
- "description": "Updated description",
- "permission": DatasetPermissionEnum.PARTIAL_TEAM,
- "external_knowledge_id": "knowledge-1",
- "external_knowledge_api_id": "api-1",
- },
- user,
- session,
+ result = DatasetService.update_dataset(
+ dataset.id,
+ {"name": dataset.name},
+ _account(),
+ session=sqlite_session,
)
assert result is dataset
- assert dataset.retrieval_model == {"top_k": 3}
- assert dataset.summary_index_setting == {"enable": True}
- assert dataset.name == "Updated Dataset"
- assert dataset.description == "Updated description"
- assert dataset.permission == DatasetPermissionEnum.PARTIAL_TEAM
- assert dataset.updated_by == "user-1"
- assert dataset.updated_at is now
- get_external_knowledge_api.assert_called_once_with("api-1", dataset.tenant_id, session=session)
- update_binding.assert_called_once_with("dataset-1", "knowledge-1", "api-1", session)
- session.add.assert_called_once_with(dataset)
- # flush() (not commit()) preserves the caller-managed transaction (#39191)
- session.flush.assert_called_once()
- session.commit.assert_not_called()
+ update_helper.assert_called_once()
@pytest.mark.parametrize(
("payload", "message"),
@@ -666,206 +563,144 @@ class TestDatasetServiceCreationAndUpdate:
({"external_knowledge_id": "knowledge-1"}, "External knowledge api id is required"),
],
)
- def test_update_external_dataset_requires_external_binding_fields(self, payload, message):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
-
+ def test_update_external_dataset_requires_binding_fields(
+ self, unbound_session: Session, payload: dict[str, str], message: str
+ ) -> None:
with pytest.raises(ValueError, match=message):
- DatasetService._update_external_dataset(dataset, payload, SimpleNamespace(id="user-1"), MagicMock())
-
- def test_update_external_dataset_rejects_cross_tenant_external_api_id(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
+ DatasetService._update_external_dataset(_dataset(provider="external"), payload, _account(), unbound_session)
- with (
- patch(
- "services.dataset_service.ExternalDatasetService.get_external_knowledge_api",
- side_effect=ValueError("api template not found"),
- ) as get_external_knowledge_api,
- patch.object(DatasetService, "_update_external_knowledge_binding") as update_binding,
- ):
- with pytest.raises(ValueError, match="api template not found"):
- DatasetService._update_external_dataset(
- dataset,
- {
- "external_knowledge_id": "knowledge-1",
- "external_knowledge_api_id": "foreign-api",
- },
- SimpleNamespace(id="user-1"),
- session,
- )
-
- get_external_knowledge_api.assert_called_once_with("foreign-api", dataset.tenant_id, session=session)
- update_binding.assert_not_called()
- session.commit.assert_not_called()
-
- def test_update_external_knowledge_binding_updates_changed_binding_values(self):
- binding = SimpleNamespace(external_knowledge_id="old-knowledge", external_knowledge_api_id="old-api")
- session = MagicMock()
- session.scalar.return_value = binding
- session.add = MagicMock()
- DatasetService._update_external_knowledge_binding("dataset-1", "new-knowledge", "new-api", session)
-
- assert binding.external_knowledge_id == "new-knowledge"
- assert binding.external_knowledge_api_id == "new-api"
- session.add.assert_called_once_with(binding)
-
- def test_update_external_knowledge_binding_raises_for_missing_binding(self):
- session = MagicMock()
- session.scalar.return_value = None
- with pytest.raises(ValueError, match="External knowledge binding not found"):
- DatasetService._update_external_knowledge_binding("dataset-1", "knowledge-1", "api-1", session)
-
- def test_update_internal_dataset_updates_fields_and_dispatches_regeneration_tasks(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- user = SimpleNamespace(id="user-1")
- now = object()
- update_payload = {
- "name": "Updated Dataset",
- "description": None,
- "partial_member_list": [{"user_id": "member-1"}],
- "external_knowledge_api_id": "api-1",
- "external_knowledge_id": "knowledge-1",
- "external_retrieval_model": {"top_k": 2},
- "retrieval_model": {"top_k": 4},
- "summary_index_setting": {"enable": True},
- "icon_info": {"icon": "book"},
- }
-
- with (
- patch.object(DatasetService, "_handle_indexing_technique_change", return_value="update"),
- patch.object(DatasetService, "_update_pipeline_knowledge_base_node_data") as update_pipeline,
- patch("services.dataset_service.naive_utc_now", return_value=now),
- patch("services.dataset_service.deal_dataset_vector_index_task") as vector_task,
- patch("services.dataset_service.regenerate_summary_index_task") as regenerate_task,
- ):
- result = DatasetService._update_internal_dataset(dataset, update_payload.copy(), user, session)
-
- assert result is dataset
- updated_values = session.execute.call_args.args[0].compile().params
- assert updated_values["name"] == "Updated Dataset"
- assert updated_values["description"] is None
- assert updated_values["retrieval_model"] == {"top_k": 4}
- assert updated_values["summary_index_setting"] == {"enable": True}
- assert updated_values["icon_info"] == {"icon": "book"}
- assert updated_values["updated_by"] == "user-1"
- assert updated_values["updated_at"] is now
- assert "partial_member_list" not in updated_values
- assert "external_knowledge_api_id" not in updated_values
- assert "external_knowledge_id" not in updated_values
- assert "external_retrieval_model" not in updated_values
- # flush() (not commit()) preserves the caller-managed transaction (#39191)
- session.flush.assert_called_once()
- session.commit.assert_not_called()
- session.refresh.assert_called_once_with(dataset)
- update_pipeline.assert_called_once_with(dataset, "user-1", session)
- vector_task.delay.assert_called_once_with("dataset-1", "update")
- regenerate_task.delay.assert_called_once_with(
- "dataset-1",
- regenerate_reason="embedding_model_changed",
- regenerate_vectors_only=True,
+ def test_update_external_dataset_flushes_dataset_and_binding_without_committing(
+ self, sqlite_session: Session
+ ) -> None:
+ dataset = _dataset(provider="external")
+ api = _external_api()
+ binding = _binding()
+ sqlite_session.add_all([dataset, api, binding])
+ sqlite_session.commit()
+ transaction_events: list[str] = []
+ event.listen(sqlite_session, "after_commit", lambda _session: transaction_events.append("commit"))
+
+ updated = DatasetService._update_external_dataset(
+ dataset,
+ {
+ "name": "Updated",
+ "description": "Changed",
+ "external_knowledge_id": "knowledge-2",
+ "external_knowledge_api_id": api.id,
+ },
+ _account(),
+ sqlite_session,
)
- def test_update_pipeline_knowledge_base_node_data_returns_early_for_non_pipeline_dataset(self):
- session = MagicMock()
- dataset = SimpleNamespace(runtime_mode="workflow", pipeline_id="pipeline-1")
-
- DatasetService._update_pipeline_knowledge_base_node_data(dataset, "user-1", session)
-
- session.get.assert_not_called()
-
- def test_update_pipeline_knowledge_base_node_data_returns_when_pipeline_is_missing(self):
- session = MagicMock()
- dataset = SimpleNamespace(runtime_mode="rag_pipeline", pipeline_id="pipeline-1")
+ assert updated is dataset
+ assert dataset.name == "Updated"
+ assert binding.external_knowledge_id == "knowledge-2"
+ assert transaction_events == []
- session.get.return_value = None
+ def test_update_external_dataset_rejects_cross_tenant_api(self, sqlite_session: Session) -> None:
+ dataset = _dataset(provider="external")
+ foreign_api = _external_api(api_id="api-foreign", tenant_id="tenant-2")
+ sqlite_session.add_all([dataset, foreign_api, _binding()])
+ sqlite_session.commit()
- DatasetService._update_pipeline_knowledge_base_node_data(dataset, "user-1", session)
+ with pytest.raises(ValueError, match="api template not found"):
+ DatasetService._update_external_dataset(
+ dataset,
+ {
+ "external_knowledge_id": "knowledge-2",
+ "external_knowledge_api_id": foreign_api.id,
+ },
+ _account(),
+ sqlite_session,
+ )
- session.commit.assert_not_called()
+ def test_update_external_knowledge_binding_rejects_missing_row(self, sqlite_session: Session) -> None:
+ with pytest.raises(ValueError, match="binding not found"):
+ DatasetService._update_external_knowledge_binding(
+ "missing-dataset",
+ "knowledge-2",
+ "api-2",
+ sqlite_session,
+ )
- def test_update_pipeline_knowledge_base_node_data_updates_published_and_draft_workflows(self):
- session = MagicMock()
- dataset = SimpleNamespace(
- id="dataset-1",
- runtime_mode="rag_pipeline",
- pipeline_id="pipeline-1",
- embedding_model="embedding-model",
- embedding_model_provider="provider",
- retrieval_model={"top_k": 5},
- chunk_structure="paragraph",
- indexing_technique="high_quality",
- keyword_number=8,
- summary_index_setting={"enable": True},
- )
- pipeline = SimpleNamespace(id="pipeline-1", tenant_id="tenant-1")
- published_workflow = SimpleNamespace(
- graph=json.dumps({"nodes": [{"data": {"type": "knowledge-index"}}, {"data": {"type": "start"}}]}),
- type="chat",
- features={"feature": True},
- environment_variables=[],
- conversation_variables=[],
- rag_pipeline_variables=[],
- )
- draft_workflow = SimpleNamespace(graph=json.dumps({"nodes": [{"data": {"type": "knowledge-index"}}]}))
- new_workflow = SimpleNamespace(id="workflow-1")
- rag_pipeline_service = MagicMock()
- rag_pipeline_service.get_published_workflow.return_value = published_workflow
- rag_pipeline_service.get_draft_workflow.return_value = draft_workflow
+ def test_update_internal_dataset_executes_real_update_without_committing(self, sqlite_session: Session) -> None:
+ dataset = _dataset(name="Before")
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
+ transaction_events: list[str] = []
+ event.listen(sqlite_session, "after_commit", lambda _session: transaction_events.append("commit"))
with (
- patch("services.dataset_service.RagPipelineService", return_value=rag_pipeline_service),
- patch("services.dataset_service.Workflow.new", return_value=new_workflow) as workflow_new,
+ patch.object(DatasetService, "_handle_indexing_technique_change", return_value="update"),
+ patch.object(DatasetService, "_update_pipeline_knowledge_base_node_data"),
+ patch("services.dataset_service.deal_dataset_vector_index_task.delay") as vector_task,
+ patch("services.dataset_service.regenerate_summary_index_task.delay") as summary_task,
):
- session.get.return_value = pipeline
-
- DatasetService._update_pipeline_knowledge_base_node_data(dataset, "user-1", session)
-
- published_graph = json.loads(workflow_new.call_args.kwargs["graph"])
- assert published_graph["nodes"][0]["data"]["embedding_model"] == "embedding-model"
- assert published_graph["nodes"][0]["data"]["summary_index_setting"] == {"enable": True}
- assert json.loads(draft_workflow.graph)["nodes"][0]["data"]["embedding_model_provider"] == "provider"
- session.add.assert_any_call(new_workflow)
- session.add.assert_any_call(draft_workflow)
- session.commit.assert_called_once()
+ updated = DatasetService._update_internal_dataset(
+ dataset,
+ {"name": "After", "description": "Changed"},
+ _account(),
+ sqlite_session,
+ )
- def test_update_pipeline_knowledge_base_node_data_rolls_back_when_update_fails(self):
- session = MagicMock()
- dataset = SimpleNamespace(runtime_mode="rag_pipeline", pipeline_id="pipeline-1")
- pipeline = SimpleNamespace(id="pipeline-1", tenant_id="tenant-1")
- rag_pipeline_service = MagicMock()
- rag_pipeline_service.get_published_workflow.side_effect = RuntimeError("boom")
+ assert updated.name == "After"
+ assert updated.description == "Changed"
+ assert transaction_events == []
+ vector_task.assert_called_once_with(dataset.id, "update")
+ summary_task.assert_called_once()
- with (
- patch("services.dataset_service.RagPipelineService", return_value=rag_pipeline_service),
- ):
- session.get.return_value = pipeline
+ def test_update_pipeline_node_data_returns_for_non_pipeline_or_missing_pipeline(
+ self, sqlite_session: Session
+ ) -> None:
+ ordinary = _dataset()
+ missing_pipeline = _dataset(dataset_id="pipeline-dataset", name="Pipeline Dataset")
+ missing_pipeline.runtime_mode = "rag_pipeline"
+ missing_pipeline.pipeline_id = "missing-pipeline"
+ sqlite_session.add_all([ordinary, missing_pipeline])
+ sqlite_session.commit()
+
+ DatasetService._update_pipeline_knowledge_base_node_data(ordinary, "user-1", sqlite_session)
+ DatasetService._update_pipeline_knowledge_base_node_data(missing_pipeline, "user-1", sqlite_session)
+
+ def test_update_pipeline_node_data_rolls_back_real_session_on_failure(self, sqlite_session: Session) -> None:
+ pipeline = Pipeline(tenant_id="tenant-1", name="Pipeline", description="", created_by="user-1")
+ pipeline.id = "pipeline-1"
+ dataset = _dataset()
+ dataset.runtime_mode = "rag_pipeline"
+ dataset.pipeline_id = pipeline.id
+ sqlite_session.add_all([pipeline, dataset])
+ sqlite_session.commit()
+ sqlite_session.begin()
+ transaction_events: list[str] = []
+ event.listen(
+ sqlite_session,
+ "after_soft_rollback",
+ lambda _session, _previous_transaction: transaction_events.append("rollback"),
+ )
+ with patch("services.dataset_service.RagPipelineService") as service_cls:
+ service_cls.return_value.get_published_workflow.side_effect = RuntimeError("boom")
with pytest.raises(RuntimeError, match="boom"):
- DatasetService._update_pipeline_knowledge_base_node_data(dataset, "user-1", session)
+ DatasetService._update_pipeline_knowledge_base_node_data(dataset, "user-1", sqlite_session)
- session.rollback.assert_called_once()
+ assert transaction_events == ["rollback"]
- def test_handle_indexing_technique_change_returns_none_without_indexing_technique(self):
- filtered_data: dict[str, object] = {}
- dataset = SimpleNamespace(indexing_technique="economy")
- session = MagicMock()
-
- result = DatasetService._handle_indexing_technique_change(dataset, {}, filtered_data, session)
- assert result is None
- assert filtered_data == {}
+class TestDatasetServiceEmbeddingSettings:
+ def test_handle_indexing_technique_change_returns_none_without_requested_change(
+ self, unbound_session: Session
+ ) -> None:
+ assert DatasetService._handle_indexing_technique_change(_dataset(), {}, {}, unbound_session) is None
- def test_handle_indexing_technique_change_switches_to_economy(self):
+ def test_handle_indexing_technique_change_switches_to_economy(self, unbound_session: Session) -> None:
+ dataset = _dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY)
filtered_data: dict[str, object] = {}
- dataset = SimpleNamespace(indexing_technique="high_quality")
- session = MagicMock()
result = DatasetService._handle_indexing_technique_change(
dataset,
- {"indexing_technique": "economy"},
+ {"indexing_technique": IndexTechniqueType.ECONOMY},
filtered_data,
- session,
+ unbound_session,
)
assert result == "remove"
@@ -875,662 +710,432 @@ class TestDatasetServiceCreationAndUpdate:
"collection_binding_id": None,
}
- def test_handle_indexing_technique_change_switches_to_high_quality(self):
+ def test_handle_indexing_technique_change_delegates_high_quality_configuration(
+ self, unbound_session: Session
+ ) -> None:
+ dataset = _dataset(indexing_technique=IndexTechniqueType.ECONOMY)
filtered_data: dict[str, object] = {}
- dataset = SimpleNamespace(indexing_technique="economy")
- session = MagicMock()
- with patch.object(DatasetService, "_configure_embedding_model_for_high_quality") as configure_embedding:
+ with patch.object(DatasetService, "_configure_embedding_model_for_high_quality") as configure:
result = DatasetService._handle_indexing_technique_change(
dataset,
- {"indexing_technique": "high_quality"},
+ {"indexing_technique": IndexTechniqueType.HIGH_QUALITY},
filtered_data,
- session,
+ unbound_session,
)
assert result == "add"
- configure_embedding.assert_called_once_with({"indexing_technique": "high_quality"}, filtered_data, session)
-
- def test_handle_indexing_technique_change_delegates_when_technique_is_unchanged(self):
- filtered_data: dict[str, object] = {}
- dataset = SimpleNamespace(indexing_technique="high_quality")
- session = MagicMock()
-
- with patch.object(
- DatasetService,
- "_handle_embedding_model_update_when_technique_unchanged",
- return_value="update",
- ) as update_embedding:
- result = DatasetService._handle_indexing_technique_change(
- dataset,
- {"indexing_technique": "high_quality"},
- filtered_data,
- session,
- )
-
- assert result == "update"
- update_embedding.assert_called_once_with(
- dataset,
- {"indexing_technique": "high_quality"},
- filtered_data,
- session,
- )
-
- def test_configure_embedding_model_for_high_quality_updates_filtered_data(self):
- class FakeAccount:
- pass
-
- current_user = FakeAccount()
- current_user.current_tenant_id = "tenant-1"
- embedding_model = SimpleNamespace(provider="provider", model_name="embedding-model")
- filtered_data: dict[str, object] = {}
- session = MagicMock()
-
- with (
- patch("services.dataset_service.Account", FakeAccount),
- patch("services.dataset_service.current_user", current_user),
- patch("services.dataset_service.ModelManager") as model_manager_cls,
- patch(
- "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",
- return_value=SimpleNamespace(id="binding-1"),
- ),
- ):
- model_manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model
-
- DatasetService._configure_embedding_model_for_high_quality(
- {"embedding_model_provider": "provider", "embedding_model": "embedding-model"},
- filtered_data,
- session,
- )
+ configure.assert_called_once()
- assert filtered_data == {
- "embedding_model": "embedding-model",
- "embedding_model_provider": "provider",
- "collection_binding_id": "binding-1",
- }
+ def test_handle_unchanged_indexing_preserves_existing_model(self, unbound_session: Session) -> None:
+ dataset = _dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY)
- @pytest.mark.parametrize(
- ("error", "message"),
- [
- (LLMBadRequestError(), "No Embedding Model available"),
- (ProviderTokenNotInitError("provider setup"), "provider setup"),
- ],
- )
- def test_configure_embedding_model_for_high_quality_wraps_model_errors(self, error, message):
- class FakeAccount:
- pass
-
- current_user = FakeAccount()
- current_user.current_tenant_id = "tenant-1"
- session = MagicMock()
-
- with (
- patch("services.dataset_service.Account", FakeAccount),
- patch("services.dataset_service.current_user", current_user),
- patch("services.dataset_service.ModelManager") as model_manager_cls,
- ):
- model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = error
-
- with pytest.raises(ValueError, match=message):
- DatasetService._configure_embedding_model_for_high_quality(
- {"embedding_model_provider": "provider", "embedding_model": "embedding-model"},
- {},
- session,
- )
-
- def test_handle_embedding_model_update_when_technique_unchanged_preserves_existing_settings(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- embedding_model_provider="provider",
- embedding_model="embedding-model",
- )
- filtered_data: dict[str, object] = {}
- session = MagicMock()
-
- with patch.object(DatasetService, "_preserve_existing_embedding_settings") as preserve_settings:
+ with patch.object(DatasetService, "_preserve_existing_embedding_settings") as preserve:
result = DatasetService._handle_embedding_model_update_when_technique_unchanged(
dataset,
{},
- filtered_data,
- session,
+ {},
+ unbound_session,
)
assert result is None
- preserve_settings.assert_called_once_with(dataset, filtered_data)
+ preserve.assert_called_once()
- def test_handle_embedding_model_update_when_technique_unchanged_updates_when_model_is_provided(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- embedding_model_provider="provider",
- embedding_model="embedding-model",
- )
- session = MagicMock()
+ def test_update_embedding_model_settings_delegates_changed_model(self, unbound_session: Session) -> None:
+ dataset = _dataset()
- with patch.object(DatasetService, "_update_embedding_model_settings", return_value="update") as update_settings:
- result = DatasetService._handle_embedding_model_update_when_technique_unchanged(
+ with patch.object(DatasetService, "_apply_new_embedding_settings") as apply_settings:
+ result = DatasetService._update_embedding_model_settings(
dataset,
- {"embedding_model_provider": "provider-two", "embedding_model": "embedding-model-two"},
+ {"embedding_model_provider": "provider-2", "embedding_model": "model-2"},
{},
- session,
+ unbound_session,
)
assert result == "update"
- update_settings.assert_called_once_with(
- dataset,
- {"embedding_model_provider": "provider-two", "embedding_model": "embedding-model-two"},
- {},
- session,
- )
+ apply_settings.assert_called_once()
- def test_preserve_existing_embedding_settings_keeps_current_binding(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- embedding_model_provider="provider",
- embedding_model="embedding-model",
- collection_binding_id="binding-1",
- )
- filtered_data = {"embedding_model_provider": "", "embedding_model": ""}
-
- DatasetService._preserve_existing_embedding_settings(dataset, filtered_data)
-
- assert filtered_data == {
- "embedding_model_provider": "provider",
- "embedding_model": "embedding-model",
- "collection_binding_id": "binding-1",
- }
+ def test_configure_high_quality_wraps_provider_error(self, unbound_session: Session) -> None:
+ account = _account()
+ with (
+ patch("services.dataset_service.current_user", account),
+ patch("services.dataset_service.ModelManager") as model_manager_cls,
+ ):
+ model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = LLMBadRequestError()
+ with pytest.raises(ValueError, match="No Embedding Model available"):
+ DatasetService._configure_embedding_model_for_high_quality(
+ {"embedding_model_provider": "provider", "embedding_model": "model"},
+ {},
+ unbound_session,
+ )
- def test_preserve_existing_embedding_settings_removes_empty_placeholders_without_existing_values(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- embedding_model_provider=None,
- embedding_model=None,
- collection_binding_id=None,
- )
- filtered_data = {"embedding_model_provider": "", "embedding_model": ""}
+ def test_preserve_existing_embedding_settings(self) -> None:
+ dataset = _dataset()
+ filtered_data: dict[str, object] = {}
DatasetService._preserve_existing_embedding_settings(dataset, filtered_data)
- assert filtered_data == {}
-
- def test_update_embedding_model_settings_returns_update_for_changed_values(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- embedding_model_provider="provider",
- embedding_model="embedding-model",
- )
- session = MagicMock()
-
- with patch.object(DatasetService, "_apply_new_embedding_settings") as apply_settings:
- result = DatasetService._update_embedding_model_settings(
- dataset,
- {"embedding_model_provider": "provider-two", "embedding_model": "embedding-model-two"},
- {},
- session,
- )
-
- assert result == "update"
- apply_settings.assert_called_once_with(
- dataset,
- {"embedding_model_provider": "provider-two", "embedding_model": "embedding-model-two"},
- {},
- session,
- )
+ assert filtered_data["embedding_model_provider"] == dataset.embedding_model_provider
+ assert filtered_data["embedding_model"] == dataset.embedding_model
- def test_update_embedding_model_settings_returns_none_for_unchanged_values(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- embedding_model_provider="provider",
- embedding_model="embedding-model",
- )
- session = MagicMock()
+ def test_update_embedding_model_settings_returns_none_when_unchanged(self, unbound_session: Session) -> None:
+ dataset = _dataset()
result = DatasetService._update_embedding_model_settings(
dataset,
- {"embedding_model_provider": "provider", "embedding_model": "embedding-model"},
+ {
+ "embedding_model_provider": dataset.embedding_model_provider,
+ "embedding_model": dataset.embedding_model,
+ },
{},
- session,
+ unbound_session,
)
assert result is None
- def test_update_embedding_model_settings_wraps_bad_request_errors(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- embedding_model_provider="provider",
- embedding_model="embedding-model",
- )
- session = MagicMock()
-
- with patch.object(DatasetService, "_apply_new_embedding_settings", side_effect=LLMBadRequestError()):
- with pytest.raises(ValueError, match="No Embedding Model available"):
- DatasetService._update_embedding_model_settings(
- dataset,
- {"embedding_model_provider": "provider-two", "embedding_model": "embedding-model-two"},
- {},
- session,
- )
-
- def test_apply_new_embedding_settings_updates_binding_for_new_model(self):
- class FakeAccount:
- pass
-
- current_user = FakeAccount()
- current_user.current_tenant_id = "tenant-1"
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(collection_binding_id="binding-1")
+ def test_apply_new_embedding_settings_uses_real_collection_binding(self, unbound_session: Session) -> None:
+ dataset = _dataset()
filtered_data: dict[str, object] = {}
- session = MagicMock()
+ account = _account()
+ embedding_model = SimpleNamespace(provider="new-provider", model_name="new-model")
+ collection_binding = DatasetCollectionBinding(
+ provider_name="new-provider",
+ model_name="new-model",
+ type="dataset",
+ collection_name="collection",
+ )
+ collection_binding.id = "collection-binding-1"
with (
- patch("services.dataset_service.Account", FakeAccount),
- patch("services.dataset_service.current_user", current_user),
+ patch("services.dataset_service.current_user", account),
patch("services.dataset_service.ModelManager") as model_manager_cls,
- patch(
- "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",
- return_value=SimpleNamespace(id="binding-2"),
+ patch.object(
+ DatasetCollectionBindingService,
+ "get_dataset_collection_binding",
+ return_value=collection_binding,
),
):
- model_manager_cls.for_tenant.return_value.get_model_instance.return_value = SimpleNamespace(
- provider="provider-two",
- model_name="embedding-model-two",
- )
-
+ model_manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model
DatasetService._apply_new_embedding_settings(
dataset,
- {"embedding_model_provider": "provider-two", "embedding_model": "embedding-model-two"},
+ {"embedding_model_provider": "new-provider", "embedding_model": "new-model"},
filtered_data,
- session,
+ unbound_session,
)
- assert filtered_data == {
- "embedding_model": "embedding-model-two",
- "embedding_model_provider": "provider-two",
- "collection_binding_id": "binding-2",
- }
-
- def test_apply_new_embedding_settings_preserves_existing_values_when_provider_token_is_missing(self):
- class FakeAccount:
- pass
+ assert filtered_data["embedding_model_provider"] == "new-provider"
+ assert filtered_data["embedding_model"] == "new-model"
+ assert filtered_data["collection_binding_id"] == collection_binding.id
- current_user = FakeAccount()
- current_user.current_tenant_id = "tenant-1"
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- embedding_model_provider="provider",
- embedding_model="embedding-model",
- collection_binding_id="binding-1",
- )
+ def test_apply_new_embedding_settings_preserves_existing_values_when_token_missing(
+ self, unbound_session: Session
+ ) -> None:
+ dataset = _dataset()
filtered_data: dict[str, object] = {}
- session = MagicMock()
with (
- patch("services.dataset_service.Account", FakeAccount),
- patch("services.dataset_service.current_user", current_user),
+ patch("services.dataset_service.current_user", _account()),
patch("services.dataset_service.ModelManager") as model_manager_cls,
):
model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = ProviderTokenNotInitError(
- "token missing"
+ "missing"
)
-
DatasetService._apply_new_embedding_settings(
dataset,
- {"embedding_model_provider": "provider-two", "embedding_model": "embedding-model-two"},
+ {"embedding_model_provider": "provider-2", "embedding_model": "model-2"},
filtered_data,
- session,
+ unbound_session,
)
- assert filtered_data == {
- "embedding_model_provider": "provider",
- "embedding_model": "embedding-model",
- "collection_binding_id": "binding-1",
- }
+ assert filtered_data["embedding_model_provider"] == dataset.embedding_model_provider
+ assert filtered_data["embedding_model"] == dataset.embedding_model
@pytest.mark.parametrize(
- ("summary_index_setting", "expected"),
+ ("summary_setting", "expected"),
[
(None, False),
({"enable": False}, False),
({"enable": True, "model_name": "old-model", "model_provider_name": "provider"}, False),
- ({"enable": True, "model_name": "new-model", "model_provider_name": "provider-two"}, True),
+ ({"enable": True, "model_name": "new-model", "model_provider_name": "provider"}, True),
],
)
- def test_check_summary_index_setting_model_changed(self, summary_index_setting, expected):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- dataset_id="dataset-1",
- summary_index_setting={"enable": True, "model_name": "old-model", "model_provider_name": "provider"},
- )
-
- result = DatasetService._check_summary_index_setting_model_changed(
- dataset,
- {"summary_index_setting": summary_index_setting} if summary_index_setting is not None else {},
- )
+ def test_check_summary_index_setting_model_changed(
+ self, summary_setting: dict[str, object] | None, expected: bool
+ ) -> None:
+ dataset = _dataset()
+ dataset.summary_index_setting = {
+ "enable": True,
+ "model_name": "old-model",
+ "model_provider_name": "provider",
+ }
- assert result is expected
+ data = {} if summary_setting is None else {"summary_index_setting": summary_setting}
+ assert DatasetService._check_summary_index_setting_model_changed(dataset, data) is expected
class TestDatasetServiceRagPipelineSettings:
- """Unit tests for rag-pipeline dataset setting updates."""
-
- def test_update_rag_pipeline_dataset_settings_requires_current_tenant(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- knowledge_configuration = _make_knowledge_configuration()
-
- with patch("services.dataset_service.current_user", SimpleNamespace(current_tenant_id=None)):
- with pytest.raises(ValueError, match="Current user or current tenant not found"):
- DatasetService.update_rag_pipeline_dataset_settings(dataset, knowledge_configuration, session=session)
-
- def test_update_rag_pipeline_dataset_settings_without_published_high_quality_updates_embedding_settings(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- session.merge.return_value = dataset
- knowledge_configuration = _make_knowledge_configuration(summary_index_setting={"enable": True})
- embedding_model = SimpleNamespace(provider="provider", model_name="embedding-model")
+ def test_requires_current_tenant(self, unbound_session: Session) -> None:
+ account = _account()
+ account._current_tenant = None
with (
- patch("services.dataset_service.current_user", SimpleNamespace(current_tenant_id="tenant-1")),
- patch("services.dataset_service.ModelManager") as model_manager_cls,
- patch.object(DatasetService, "check_is_multimodal_model", return_value=True) as check_multimodal,
- patch(
- "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",
- return_value=SimpleNamespace(id="binding-1"),
- ),
+ patch("services.dataset_service.current_user", account),
+ pytest.raises(ValueError, match="Current user or current tenant not found"),
):
- model_manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model
-
- DatasetService.update_rag_pipeline_dataset_settings(dataset, knowledge_configuration, session=session)
-
- assert dataset.chunk_structure == "paragraph"
- assert dataset.indexing_technique == "high_quality"
- assert dataset.embedding_model == "embedding-model"
- assert dataset.embedding_model_provider == "provider"
- assert dataset.collection_binding_id == "binding-1"
- assert dataset.is_multimodal is True
- assert dataset.retrieval_model == knowledge_configuration.retrieval_model.model_dump()
- assert dataset.summary_index_setting == {"enable": True}
- check_multimodal.assert_called_once_with("tenant-1", "provider", "embedding-model")
- session.add.assert_called_once_with(dataset)
- session.commit.assert_not_called()
-
- def test_update_rag_pipeline_dataset_settings_without_published_economy_updates_keyword_number(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- session.merge.return_value = dataset
- knowledge_configuration = _make_knowledge_configuration(
- indexing_technique="economy",
- embedding_model_provider="",
- embedding_model="",
- keyword_number=12,
- )
-
- with patch("services.dataset_service.current_user", SimpleNamespace(current_tenant_id="tenant-1")):
- DatasetService.update_rag_pipeline_dataset_settings(dataset, knowledge_configuration, session=session)
-
- assert dataset.indexing_technique == "economy"
- assert dataset.keyword_number == 12
- assert dataset.retrieval_model == knowledge_configuration.retrieval_model.model_dump()
- session.add.assert_called_once_with(dataset)
-
- def test_update_rag_pipeline_dataset_settings_with_published_rejects_chunk_structure_changes(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- dataset.chunk_structure = "paragraph"
- session.merge.return_value = dataset
- knowledge_configuration = _make_knowledge_configuration(chunk_structure="sentence")
-
- with patch("services.dataset_service.current_user", SimpleNamespace(current_tenant_id="tenant-1")):
- with pytest.raises(ValueError, match="Chunk structure is not allowed to be updated"):
- DatasetService.update_rag_pipeline_dataset_settings(
- dataset, knowledge_configuration, has_published=True, session=session
- )
-
- def test_update_rag_pipeline_dataset_settings_with_published_rejects_switch_to_economy(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- dataset.chunk_structure = "paragraph"
- dataset.indexing_technique = "high_quality"
- session.merge.return_value = dataset
- knowledge_configuration = _make_knowledge_configuration(
- indexing_technique="economy",
- embedding_model_provider="",
- embedding_model="",
- )
-
- with patch("services.dataset_service.current_user", SimpleNamespace(current_tenant_id="tenant-1")):
- with pytest.raises(
- ValueError,
- match="Knowledge base indexing technique is not allowed to be updated to economy",
- ):
- DatasetService.update_rag_pipeline_dataset_settings(
- dataset, knowledge_configuration, has_published=True, session=session
- )
-
- def test_update_rag_pipeline_dataset_settings_with_published_adds_high_quality_index(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- dataset.chunk_structure = "paragraph"
- dataset.indexing_technique = "economy"
- session.merge.return_value = dataset
- knowledge_configuration = _make_knowledge_configuration()
- embedding_model = SimpleNamespace(provider="provider", model_name="embedding-model")
-
- with (
- patch("services.dataset_service.current_user", SimpleNamespace(current_tenant_id="tenant-1")),
- patch("services.dataset_service.ModelManager") as model_manager_cls,
- patch.object(DatasetService, "check_is_multimodal_model", return_value=False),
- patch(
- "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",
- return_value=SimpleNamespace(id="binding-1"),
- ),
- patch("services.dataset_service.deal_dataset_index_update_task") as update_task,
- ):
- model_manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model
-
DatasetService.update_rag_pipeline_dataset_settings(
- dataset, knowledge_configuration, has_published=True, session=session
+ _dataset(),
+ _make_knowledge_configuration(),
+ session=unbound_session,
)
- assert dataset.indexing_technique == "high_quality"
- assert dataset.embedding_model == "embedding-model"
- assert dataset.embedding_model_provider == "provider"
- assert dataset.collection_binding_id == "binding-1"
- assert dataset.is_multimodal is False
- assert dataset.retrieval_model == knowledge_configuration.retrieval_model.model_dump()
- session.add.assert_called_once_with(dataset)
- session.commit.assert_called_once()
- update_task.delay.assert_called_once_with("dataset-1", "add")
-
- def test_update_rag_pipeline_dataset_settings_with_published_updates_changed_embedding_model(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- dataset.chunk_structure = "paragraph"
- dataset.indexing_technique = "high_quality"
- dataset.embedding_model_provider = "provider"
- dataset.embedding_model = "embedding-model"
- session.merge.return_value = dataset
- knowledge_configuration = _make_knowledge_configuration(
- embedding_model_provider="provider-two",
- embedding_model="embedding-model-two",
- summary_index_setting={"enable": True},
+ def test_unpublished_high_quality_settings_use_real_merged_dataset(self, sqlite_session: Session) -> None:
+ dataset = _dataset(indexing_technique=IndexTechniqueType.ECONOMY)
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
+ account = _account()
+ embedding_model = SimpleNamespace(provider="provider-2", model_name="embedding-2")
+ collection_binding = DatasetCollectionBinding(
+ provider_name="provider-2",
+ model_name="embedding-2",
+ type="dataset",
+ collection_name="collection",
)
+ collection_binding.id = "collection-binding-2"
with (
- patch("services.dataset_service.current_user", SimpleNamespace(current_tenant_id="tenant-1")),
+ patch("services.dataset_service.current_user", account),
patch("services.dataset_service.ModelManager") as model_manager_cls,
patch.object(DatasetService, "check_is_multimodal_model", return_value=True),
- patch(
- "services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",
- return_value=SimpleNamespace(id="binding-2"),
+ patch.object(
+ DatasetCollectionBindingService,
+ "get_dataset_collection_binding",
+ return_value=collection_binding,
),
- patch("services.dataset_service.deal_dataset_index_update_task") as update_task,
):
- model_manager_cls.for_tenant.return_value.get_model_instance.return_value = SimpleNamespace(
- provider="provider-two",
- model_name="embedding-model-two",
- )
-
+ model_manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model
DatasetService.update_rag_pipeline_dataset_settings(
- dataset, knowledge_configuration, has_published=True, session=session
+ dataset,
+ _make_knowledge_configuration(
+ embedding_model_provider="provider-2",
+ embedding_model="embedding-2",
+ summary_index_setting={"enable": True},
+ ),
+ session=sqlite_session,
)
- assert dataset.embedding_model_provider == "provider-two"
- assert dataset.embedding_model == "embedding-model-two"
- assert dataset.collection_binding_id == "binding-2"
- assert dataset.is_multimodal is True
- assert dataset.summary_index_setting == {"enable": True}
- session.add.assert_called_once_with(dataset)
- session.commit.assert_called_once()
- update_task.delay.assert_called_once_with("dataset-1", "update")
-
- def test_update_rag_pipeline_dataset_settings_with_published_skips_embedding_update_when_token_is_missing(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- dataset.chunk_structure = "paragraph"
- dataset.indexing_technique = "high_quality"
- dataset.embedding_model_provider = "provider"
- dataset.embedding_model = "embedding-model"
- session.merge.return_value = dataset
- knowledge_configuration = _make_knowledge_configuration(
- embedding_model_provider="provider-two",
- embedding_model="embedding-model-two",
- )
+ persisted = sqlite_session.get(Dataset, dataset.id)
+ assert persisted is not None
+ assert persisted.indexing_technique == IndexTechniqueType.HIGH_QUALITY
+ assert persisted.embedding_model == "embedding-2"
+ assert persisted.collection_binding_id == collection_binding.id
+ assert persisted.is_multimodal is True
- with (
- patch("services.dataset_service.current_user", SimpleNamespace(current_tenant_id="tenant-1")),
- patch("services.dataset_service.ModelManager") as model_manager_cls,
- patch("services.dataset_service.deal_dataset_index_update_task") as update_task,
- ):
- model_manager_cls.for_tenant.return_value.get_model_instance.side_effect = ProviderTokenNotInitError(
- "token missing"
- )
+ def test_unpublished_economy_settings_update_keyword_number(self, sqlite_session: Session) -> None:
+ dataset = _dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY)
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
+ with patch("services.dataset_service.current_user", _account()):
DatasetService.update_rag_pipeline_dataset_settings(
- dataset, knowledge_configuration, has_published=True, session=session
+ dataset,
+ _make_knowledge_configuration(
+ indexing_technique=IndexTechniqueType.ECONOMY,
+ keyword_number=17,
+ ),
+ session=sqlite_session,
)
- assert dataset.embedding_model_provider == "provider"
- assert dataset.embedding_model == "embedding-model"
- assert dataset.retrieval_model == knowledge_configuration.retrieval_model.model_dump()
- session.add.assert_called_once_with(dataset)
- session.commit.assert_called_once()
- update_task.delay.assert_called_once_with("dataset-1", "update")
-
- def test_update_rag_pipeline_dataset_settings_with_published_updates_economy_keyword_number(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- dataset.chunk_structure = "paragraph"
- dataset.indexing_technique = "economy"
- dataset.keyword_number = 5
- session.merge.return_value = dataset
- knowledge_configuration = _make_knowledge_configuration(
- indexing_technique="economy",
- embedding_model_provider="",
- embedding_model="",
- keyword_number=9,
- )
+ persisted = sqlite_session.get(Dataset, dataset.id)
+ assert persisted is not None
+ assert persisted.indexing_technique == IndexTechniqueType.ECONOMY
+ assert persisted.keyword_number == 17
+
+ def test_published_economy_settings_commit_keyword_change_and_dispatch_no_task(
+ self, sqlite_session: Session
+ ) -> None:
+ dataset = _dataset(indexing_technique=IndexTechniqueType.ECONOMY, chunk_structure="paragraph")
+ dataset.keyword_number = 4
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
with (
- patch("services.dataset_service.current_user", SimpleNamespace(current_tenant_id="tenant-1")),
- patch("services.dataset_service.deal_dataset_index_update_task") as update_task,
+ patch("services.dataset_service.current_user", _account()),
+ patch("services.dataset_service.deal_dataset_index_update_task.delay") as update_task,
):
DatasetService.update_rag_pipeline_dataset_settings(
- dataset, knowledge_configuration, has_published=True, session=session
+ dataset,
+ _make_knowledge_configuration(
+ chunk_structure="paragraph",
+ indexing_technique=IndexTechniqueType.ECONOMY,
+ keyword_number=12,
+ ),
+ has_published=True,
+ session=sqlite_session,
)
- assert dataset.keyword_number == 9
- assert dataset.retrieval_model == knowledge_configuration.retrieval_model.model_dump()
- session.add.assert_called_once_with(dataset)
- session.commit.assert_called_once()
- update_task.delay.assert_not_called()
-
+ assert sqlite_session.get(Dataset, dataset.id).keyword_number == 12
+ update_task.assert_not_called()
-class TestDatasetServicePermissionsAndLifecycle:
- """Unit tests for dataset permissions, deletion, and metadata helpers."""
-
- def test_check_dataset_operator_permission_validates_required_arguments(self):
- session = MagicMock()
-
- with pytest.raises(ValueError, match="Dataset not found"):
- DatasetService.check_dataset_operator_permission(
- user=SimpleNamespace(id="user-1"),
- dataset=None,
- session=session,
- )
+ @pytest.mark.parametrize(
+ ("dataset", "configuration", "message"),
+ [
+ (
+ _dataset(chunk_structure="paragraph"),
+ _make_knowledge_configuration(chunk_structure="sentence"),
+ "Chunk structure is not allowed",
+ ),
+ (
+ _dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY, chunk_structure="paragraph"),
+ _make_knowledge_configuration(
+ chunk_structure="paragraph",
+ indexing_technique=IndexTechniqueType.ECONOMY,
+ ),
+ "not allowed to be updated to economy",
+ ),
+ ],
+ )
+ def test_published_settings_reject_incompatible_changes(
+ self,
+ sqlite_session: Session,
+ dataset: Dataset,
+ configuration,
+ message: str,
+ ) -> None:
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
- with pytest.raises(ValueError, match="User not found"):
- DatasetService.check_dataset_operator_permission(
- user=None,
- dataset=SimpleNamespace(id="dataset-1"),
- session=session,
+ with (
+ patch("services.dataset_service.current_user", _account()),
+ pytest.raises(ValueError, match=message),
+ ):
+ DatasetService.update_rag_pipeline_dataset_settings(
+ dataset,
+ configuration,
+ has_published=True,
+ session=sqlite_session,
)
-class TestDatasetCollectionBindingService:
- """Unit tests for dataset collection binding lookups and creation."""
-
+class TestDatasetPermissions:
+ def test_check_dataset_permission_enforces_tenant_and_partial_members(self, sqlite_session: Session) -> None:
+ dataset = _dataset(permission=DatasetPermissionEnum.PARTIAL_TEAM, maintainer="owner")
+ permitted_user = _account(account_id="permitted", role=TenantAccountRole.NORMAL)
+ denied_user = _account(account_id="denied", role=TenantAccountRole.NORMAL)
+ foreign_user = _account(account_id="foreign", tenant_id="tenant-2", role=TenantAccountRole.NORMAL)
+ sqlite_session.add_all(
+ [
+ dataset,
+ DatasetPermission(
+ tenant_id=dataset.tenant_id,
+ dataset_id=dataset.id,
+ account_id=permitted_user.id,
+ ),
+ ]
+ )
+ sqlite_session.commit()
-class TestDatasetPermissionService:
- """Unit tests for dataset partial-member management helpers."""
+ DatasetService.check_dataset_permission(dataset, permitted_user, sqlite_session)
+ with pytest.raises(NoPermissionError):
+ DatasetService.check_dataset_permission(dataset, denied_user, sqlite_session)
+ with pytest.raises(NoPermissionError):
+ DatasetService.check_dataset_permission(dataset, foreign_user, sqlite_session)
- def test_update_partial_member_list_does_not_rollback_caller_session_on_exception(self):
- session = MagicMock()
- session.add_all.side_effect = RuntimeError("boom")
+ def test_dataset_operator_cannot_change_permission_or_member_list(self, sqlite_session: Session) -> None:
+ dataset = _dataset(permission=DatasetPermissionEnum.PARTIAL_TEAM)
+ operator = _account(role=TenantAccountRole.DATASET_OPERATOR)
+ sqlite_session.add_all(
+ [
+ dataset,
+ DatasetPermission(tenant_id="tenant-1", dataset_id=dataset.id, account_id="member-1"),
+ ]
+ )
+ sqlite_session.commit()
- with pytest.raises(RuntimeError, match="boom"):
- DatasetPermissionService.update_partial_member_list(
- "tenant-1",
- "dataset-1",
- [{"user_id": "user-1"}],
- session,
+ with pytest.raises(NoPermissionError, match="cannot change"):
+ DatasetPermissionService.check_permission(
+ operator,
+ dataset,
+ DatasetPermissionEnum.ALL_TEAM,
+ None,
+ session=sqlite_session,
)
- session.rollback.assert_not_called()
-
- def test_check_permission_requires_dataset_editor(self):
- user = SimpleNamespace(is_dataset_editor=False, is_dataset_operator=False)
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock()
- session = MagicMock()
+ def test_non_editor_cannot_change_dataset_permissions(self, sqlite_session: Session) -> None:
+ user = _account(role=TenantAccountRole.NORMAL)
with pytest.raises(NoPermissionError, match="does not have permission"):
- DatasetPermissionService.check_permission(user, dataset, "all_team", [], session=session)
-
- def test_check_permission_prevents_dataset_operator_from_changing_permission_mode(self):
- user = SimpleNamespace(is_dataset_editor=True, is_dataset_operator=True)
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(permission="all_team")
- session = MagicMock()
-
- with pytest.raises(NoPermissionError, match="cannot change the dataset permissions"):
- DatasetPermissionService.check_permission(user, dataset, "only_me", [], session=session)
-
- def test_check_permission_requires_partial_member_list_for_partial_members_mode(self):
- user = SimpleNamespace(is_dataset_editor=True, is_dataset_operator=True)
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(permission="partial_members")
- session = MagicMock()
+ DatasetPermissionService.check_permission(
+ user,
+ _dataset(),
+ DatasetPermissionEnum.ALL_TEAM,
+ None,
+ session=sqlite_session,
+ )
- with pytest.raises(ValueError, match="Partial member list is required"):
- DatasetPermissionService.check_permission(user, dataset, "partial_members", [], session=session)
+ def test_dataset_operator_can_keep_unchanged_partial_member_list(self, sqlite_session: Session) -> None:
+ dataset = _dataset(permission=DatasetPermissionEnum.PARTIAL_TEAM)
+ operator = _account(role=TenantAccountRole.DATASET_OPERATOR)
+ sqlite_session.add_all(
+ [
+ dataset,
+ DatasetPermission(tenant_id="tenant-1", dataset_id=dataset.id, account_id="member-1"),
+ ]
+ )
+ sqlite_session.commit()
- def test_check_permission_rejects_dataset_operator_member_list_changes(self):
- user = SimpleNamespace(is_dataset_editor=True, is_dataset_operator=True)
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- dataset_id="dataset-1", permission="partial_members"
+ DatasetPermissionService.check_permission(
+ operator,
+ dataset,
+ DatasetPermissionEnum.PARTIAL_TEAM,
+ [{"user_id": "member-1"}],
+ session=sqlite_session,
)
- session = MagicMock()
+ with pytest.raises(ValueError, match="cannot change"):
+ DatasetPermissionService.check_permission(
+ operator,
+ dataset,
+ DatasetPermissionEnum.PARTIAL_TEAM,
+ [{"user_id": "member-2"}],
+ session=sqlite_session,
+ )
- with patch.object(DatasetPermissionService, "get_dataset_partial_member_list", return_value=["user-1"]):
- with pytest.raises(ValueError, match="cannot change the dataset permissions"):
- DatasetPermissionService.check_permission(
- user, dataset, "partial_members", [{"user_id": "user-2"}], session=session
+ def test_update_partial_member_list_flush_failure_does_not_rollback_caller_session(
+ self, sqlite_session: Session
+ ) -> None:
+ transaction_events: list[str] = []
+
+ def fail_flush(_session, _flush_context, _instances) -> None:
+ raise RuntimeError("flush failed")
+
+ event.listen(sqlite_session, "before_flush", fail_flush)
+ event.listen(sqlite_session, "after_rollback", lambda _session: transaction_events.append("rollback"))
+ try:
+ with pytest.raises(RuntimeError, match="flush failed"):
+ DatasetPermissionService.update_partial_member_list(
+ "tenant-1",
+ "dataset-1",
+ [{"user_id": "member-1"}],
+ sqlite_session,
)
+ finally:
+ event.remove(sqlite_session, "before_flush", fail_flush)
- def test_check_permission_allows_dataset_operator_when_member_list_is_unchanged(self):
- user = SimpleNamespace(is_dataset_editor=True, is_dataset_operator=True)
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- dataset_id="dataset-1", permission="partial_members"
- )
- session = MagicMock()
+ assert transaction_events == []
- with patch.object(DatasetPermissionService, "get_dataset_partial_member_list", return_value=["user-1"]):
- DatasetPermissionService.check_permission(
- user, dataset, "partial_members", [{"user_id": "user-1"}], session=session
- )
+ def test_clear_partial_member_list_execute_failure_does_not_rollback_caller_session(
+ self, sqlite_session: Session
+ ) -> None:
+ transaction_events: list[str] = []
- def test_clear_partial_member_list_does_not_rollback_caller_session_on_exception(self):
- session = MagicMock()
- session.execute.side_effect = RuntimeError("boom")
+ def fail_execute(_orm_execute_state) -> None:
+ raise RuntimeError("execute failed")
- with pytest.raises(RuntimeError, match="boom"):
- DatasetPermissionService.clear_partial_member_list("dataset-1", session)
+ event.listen(sqlite_session, "do_orm_execute", fail_execute)
+ event.listen(sqlite_session, "after_rollback", lambda _session: transaction_events.append("rollback"))
+ try:
+ with pytest.raises(RuntimeError, match="execute failed"):
+ DatasetPermissionService.clear_partial_member_list("dataset-1", sqlite_session)
+ finally:
+ event.remove(sqlite_session, "do_orm_execute", fail_execute)
- session.rollback.assert_not_called()
+ assert transaction_events == []