Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: migrate console dataset sessions and ORM models to SQLite (#40521)
Asuka Minato9398482b5346
diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets.pyindex ada7de0ce20..b87cdc3ecaa 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets.py@@ -8,6 +8,7 @@ from unittest.mock import ANY, MagicMock, PropertyMock, call, patchimport pytestfrom flask import Flask+from sqlalchemy.orm import Sessionfrom werkzeug.exceptions import BadRequest, Forbidden, NotFoundimport services@@ -44,7 +45,7 @@ from core.rag.index_processor.constant.index_type import IndexStructureTypefrom core.rag.retrieval.retrieval_methods import RetrievalMethodfrom extensions.storage.storage_type import StorageTypefrom models.account import Account, TenantAccountRole-from models.dataset import Dataset, DatasetQuery, Document+from models.dataset import AppDatasetJoin, Dataset, DatasetPermission, DatasetQuery, Document, DocumentSegmentfrom models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatusfrom models.model import ApiToken, App, AppMode, IconType, UploadFilefrom services.dataset_ref_service import DatasetRef@@ -170,7 +171,29 @@ def make_document_status(**overrides) -> Document:return Document(**base)-class TestDatasetList:+def make_document_segment(*, position: int, completed: bool) -> DocumentSegment:+ return DocumentSegment(+ tenant_id="tenant-1",+ dataset_id="dataset-1",+ document_id="doc-1",+ position=position,+ content=f"segment {position}",+ word_count=2,+ tokens=2,+ created_by="account-1",+ completed_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC) if completed else None,+ )+++class _UsesSQLiteSession:+ session: Session++ @pytest.fixture(autouse=True)+ def _inject_sqlite_session(self, sqlite_session: Session) -> None:+ self.session = sqlite_session+++class TestDatasetList(_UsesSQLiteSession):def _mock_user(self):user = make_account()return user@@ -185,7 +208,7 @@ class TestDatasetList:patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),):- resp, status = method(api, MagicMock(), "tenant-1", current_user)+ resp, status = method(api, self.session, "tenant-1", current_user)assert status == 200assert resp["total"] == 1assert resp["data"][0]["embedding_available"] is True@@ -201,7 +224,7 @@ class TestDatasetList:method = unwrap(api.get)current_user = self._mock_user()dataset = make_dataset()- session = MagicMock()+ session = self.sessionwith app.test_request_context("/datasets"):with (patch.object(DatasetService, "get_datasets", return_value=([dataset], 1)),@@ -222,7 +245,7 @@ class TestDatasetList:patch.object(DatasetService, "get_datasets_by_ids", return_value=(datasets, 2)) as by_ids_mock,patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),):- resp, status = method(api, MagicMock(), "tenant-1", current_user)+ resp, status = method(api, self.session, "tenant-1", current_user)by_ids_mock.assert_called_once()assert status == 200assert resp["total"] == 2@@ -251,7 +274,7 @@ class TestDatasetList:return_value=permissions,) as get_permissions,):- resp, status = method(api, MagicMock(), "tenant-1", current_user)+ resp, status = method(api, self.session, "tenant-1", current_user)get_permissions.assert_called_once_with("tenant-1", current_user.id, session=ANY)assert status == 200assert resp["data"][0]["permission_keys"] == ["dataset.acl.readonly", "dataset.acl.edit"]@@ -281,7 +304,7 @@ class TestDatasetList:),patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),):- method(api, MagicMock(), "tenant-1", current_user)+ method(api, self.session, "tenant-1", current_user)assert get_datasets.call_args.kwargs["accessible_dataset_ids"] == []assert get_datasets.call_args.kwargs["include_own_datasets"] is False@@ -308,7 +331,7 @@ class TestDatasetList:),patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),):- method(api, MagicMock(), "tenant-1", current_user)+ method(api, self.session, "tenant-1", current_user)assert get_datasets.call_args.kwargs["accessible_dataset_ids"] is Nonedef test_get_restricted_whitelist_overrides_default_read_permission(@@ -374,7 +397,7 @@ class TestDatasetList:),patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),):- method(api, MagicMock(), "tenant-1", current_user)+ method(api, self.session, "tenant-1", current_user)assert get_datasets.call_args.kwargs["accessible_dataset_ids"] == ["dataset-whitelist-only",]@@ -399,9 +422,9 @@ class TestDatasetList:),patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),):- method(api, MagicMock(), "tenant-1", current_user)+ method(api, self.session, "tenant-1", current_user)session = get_datasets_by_ids.call_args.kwargs["session"]- assert isinstance(session, MagicMock)+ assert session is self.sessionassert get_datasets_by_ids.call_args.args == (["dataset-1"], "tenant-1")assert get_datasets_by_ids.call_args.kwargs == {"user": current_user,@@ -420,7 +443,7 @@ class TestDatasetList:patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),):- resp, status = method(api, MagicMock(), "tenant-1", current_user)+ resp, status = method(api, self.session, "tenant-1", current_user)assert status == 200def test_get_allows_legacy_weighted_score_without_weight_type(self, app: Flask):@@ -453,7 +476,7 @@ class TestDatasetList:patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),):- resp, status = method(api, MagicMock(), "tenant-1", current_user)+ resp, status = method(api, self.session, "tenant-1", current_user)assert status == 200assert resp["data"][0]["retrieval_model_dict"]["weights"]["weight_type"] is None@@ -467,7 +490,7 @@ class TestDatasetList:patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),patch.object(ProviderManager, "get_configurations", return_value=MagicMock(get_models=lambda **_: [])),):- resp, status = method(api, MagicMock(), "tenant-1", current_user)+ resp, status = method(api, self.session, "tenant-1", current_user)assert status == 200retrieval_model = resp["data"][0]["retrieval_model_dict"]assert retrieval_model["search_method"] == "semantic_search"@@ -491,7 +514,7 @@ class TestDatasetList:patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),patch.object(ProviderManager, "get_configurations", return_value=config),):- resp, status = method(api, MagicMock(), "tenant-1", current_user)+ resp, status = method(api, self.session, "tenant-1", current_user)assert resp["data"][0]["embedding_available"] is Falsedef test_partial_members_permission(self, app: Flask):@@ -499,8 +522,9 @@ class TestDatasetList:method = unwrap(api.get)current_user = self._mock_user()datasets = [make_dataset(permission="partial_members")]- session = MagicMock()- session.execute.return_value.all.return_value = [("ds-1", "u1")]+ session = self.session+ session.add(DatasetPermission(dataset_id="ds-1", account_id="u1", tenant_id="tenant-1"))+ session.flush()with app.test_request_context("/datasets"):with (patch.object(DatasetService, "get_datasets", return_value=(datasets, 1)),@@ -510,7 +534,7 @@ class TestDatasetList:assert resp["data"][0]["partial_member_list"] == ["u1"]-class TestDatasetListApiPost:+class TestDatasetListApiPost(_UsesSQLiteSession):def test_post_success(self, app: Flask):api = DatasetListApi()method = unwrap(api.post)@@ -522,7 +546,7 @@ class TestDatasetListApiPost:patch.object(type(console_ns), "payload", payload),patch.object(DatasetService, "create_empty_dataset", return_value=dataset),):- _, status = method(api, DatasetCreatePayload(**payload), MagicMock(), "tenant-1", user)+ _, status = method(api, DatasetCreatePayload(**payload), self.session, "tenant-1", user)assert status == 201def test_post_forbidden(self, app: Flask):@@ -532,7 +556,7 @@ class TestDatasetListApiPost:user = make_account(TenantAccountRole.NORMAL)with app.test_request_context("/datasets", json=payload), patch.object(type(console_ns), "payload", payload):with pytest.raises(Forbidden):- method(api, DatasetCreatePayload(**payload), MagicMock(), "tenant-1", user)+ method(api, DatasetCreatePayload(**payload), self.session, "tenant-1", user)def test_post_duplicate_name(self, app: Flask):api = DatasetListApi()@@ -547,14 +571,14 @@ class TestDatasetListApiPost:),):with pytest.raises(DatasetNameDuplicateError):- method(api, DatasetCreatePayload(**payload), MagicMock(), "tenant-1", user)+ method(api, DatasetCreatePayload(**payload), self.session, "tenant-1", user)def test_post_invalid_payload_missing_name(self, app: Flask):api = DatasetListApi()method = unwrap(api.post)with app.test_request_context("/datasets", json={}), patch.object(type(console_ns), "payload", {}):with pytest.raises(ValueError):- method(api, DatasetCreatePayload(), MagicMock(), "tenant-1", make_account())+ method(api, DatasetCreatePayload(), self.session, "tenant-1", make_account())def test_post_invalid_indexing_technique(self, app: Flask):api = DatasetListApi()@@ -562,7 +586,7 @@ class TestDatasetListApiPost:payload = {"name": "bad", "indexing_technique": "invalid-tech"}with app.test_request_context("/datasets", json=payload), patch.object(type(console_ns), "payload", payload):with pytest.raises(ValueError, match="Invalid indexing technique"):- method(api, DatasetCreatePayload(**payload), MagicMock(), "tenant-1", make_account())+ method(api, DatasetCreatePayload(**payload), self.session, "tenant-1", make_account())def test_post_invalid_provider(self, app: Flask):api = DatasetListApi()@@ -570,10 +594,10 @@ class TestDatasetListApiPost:payload = {"name": "bad", "provider": "unknown"}with app.test_request_context("/datasets", json=payload), patch.object(type(console_ns), "payload", payload):with pytest.raises(ValueError, match="Invalid provider"):- method(api, DatasetCreatePayload(**payload), MagicMock(), "tenant-1", make_account())+ method(api, DatasetCreatePayload(**payload), self.session, "tenant-1", make_account())-class TestDatasetApiGet:+class TestDatasetApiGet(_UsesSQLiteSession):def test_get_success_basic(self, app: Flask):api = DatasetApi()method = unwrap(api.get)@@ -588,7 +612,7 @@ class TestDatasetApiGet:patch("controllers.console.datasets.datasets.create_plugin_provider_manager") as provider_manager_mock,):provider_manager_mock.return_value.get_configurations.return_value.get_models.return_value = []- data, status = method(api, MagicMock(), tenant_id, user, dataset_id)+ data, status = method(api, self.session, tenant_id, user, dataset_id)assert status == 200assert data["embedding_available"] is True@@ -597,7 +621,7 @@ class TestDatasetApiGet:api = DatasetApi()method = unwrap(api.get)dataset_id = "123e4567-e89b-12d3-a456-426614174000"- user = MagicMock(id="account-1")+ user = make_account()tenant_id = "tenant-1"dataset = make_dataset(id=dataset_id)with (@@ -619,7 +643,7 @@ class TestDatasetApiGet:patch("controllers.console.datasets.datasets.create_plugin_provider_manager") as provider_manager_mock,):provider_manager_mock.return_value.get_configurations.return_value.get_models.return_value = []- data, status = method(api, MagicMock(), tenant_id, user, dataset_id)+ data, status = method(api, self.session, tenant_id, user, dataset_id)get_permissions.assert_called_once_with(tenant_id, user.id, dataset_id=dataset_id, session=ANY)assert status == 200assert data["permission_keys"] == ["dataset.acl.readonly", "dataset.acl.edit"]@@ -636,7 +660,7 @@ class TestDatasetApiGet:patch("controllers.console.datasets.datasets.create_plugin_provider_manager") as provider_manager_mock,):provider_manager_mock.return_value.get_configurations.return_value.get_models.return_value = []- data, status = method(api, MagicMock(), "tenant", make_account(), dataset_id)+ data, status = method(api, self.session, "tenant", make_account(), dataset_id)assert status == 200assert data["external_retrieval_model"] == {"top_k": 2, "score_threshold": 0.0, "score_threshold_enabled": None}@@ -649,7 +673,7 @@ class TestDatasetApiGet:patch.object(DatasetService, "get_dataset", return_value=None),):with pytest.raises(NotFound, match="Dataset not found"):- method(api, MagicMock(), "tenant", make_account(), dataset_id)+ method(api, self.session, "tenant", make_account(), dataset_id)def test_get_permission_denied(self, app: Flask):api = DatasetApi()@@ -666,7 +690,7 @@ class TestDatasetApiGet:),):with pytest.raises(Forbidden, match="no access"):- method(api, MagicMock(), "tenant", make_account(), dataset_id)+ method(api, self.session, "tenant", make_account(), dataset_id)def test_get_high_quality_embedding_unavailable(self, app: Flask):api = DatasetApi()@@ -687,7 +711,7 @@ class TestDatasetApiGet:patch("controllers.console.datasets.datasets.create_plugin_provider_manager") as provider_manager_mock,):provider_manager_mock.return_value.get_configurations.return_value.get_models.return_value = []- data, _ = method(api, MagicMock(), tenant_id, user, dataset_id)+ data, _ = method(api, self.session, tenant_id, user, dataset_id)assert data["embedding_available"] is Falsedef test_get_partial_members_permission(self, app: Flask):@@ -704,11 +728,11 @@ class TestDatasetApiGet:patch("controllers.console.datasets.datasets.create_plugin_provider_manager") as provider_manager_mock,):provider_manager_mock.return_value.get_configurations.return_value.get_models.return_value = []- data, _ = method(api, MagicMock(), "tenant", make_account(), dataset_id)+ data, _ = method(api, self.session, "tenant", make_account(), dataset_id)assert data["partial_member_list"] == partial_members-class TestDatasetApiPatch:+class TestDatasetApiPatch(_UsesSQLiteSession):def test_patch_success_basic(self, app: Flask):api = DatasetApi()method = unwrap(api.patch)@@ -725,7 +749,7 @@ class TestDatasetApiPatch:patch.object(DatasetService, "update_dataset", return_value=dataset),patch.object(DatasetPermissionService, "get_dataset_partial_member_list", return_value=[]),):- result, status = method(api, DatasetUpdatePayload(), MagicMock(), tenant_id, user, dataset_id)+ result, status = method(api, DatasetUpdatePayload(), self.session, tenant_id, user, dataset_id)assert status == 200assert result["partial_member_list"] == []@@ -737,7 +761,7 @@ class TestDatasetApiPatch:patch.object(DatasetService, "get_dataset", return_value=None),):with pytest.raises(NotFound, match="Dataset not found"):- method(api, DatasetUpdatePayload(), MagicMock(), "tenant-1", make_account(), "missing")+ method(api, DatasetUpdatePayload(), self.session, "tenant-1", make_account(), "missing")def test_patch_permission_denied(self, app: Flask):api = DatasetApi()@@ -752,7 +776,7 @@ class TestDatasetApiPatch:patch.object(DatasetPermissionService, "check_permission", side_effect=Forbidden("no permission")),):with pytest.raises(Forbidden):- method(api, DatasetUpdatePayload(), MagicMock(), "tenant", make_account(), dataset_id)+ method(api, DatasetUpdatePayload(), self.session, "tenant", make_account(), dataset_id)def test_patch_partial_members_update(self, app: Flask):api = DatasetApi()@@ -769,7 +793,7 @@ class TestDatasetApiPatch:patch.object(DatasetPermissionService, "update_partial_member_list", return_value=None),patch.object(DatasetPermissionService, "get_dataset_partial_member_list", return_value=["u1", "u2"]),):- result, _ = method(api, DatasetUpdatePayload(), MagicMock(), "tenant", make_account(), dataset_id)+ result, _ = method(api, DatasetUpdatePayload(), self.session, "tenant", make_account(), dataset_id)assert result["partial_member_list"] == ["u1", "u2"]def test_patch_clear_partial_members(self, app: Flask):@@ -787,11 +811,11 @@ class TestDatasetApiPatch:patch.object(DatasetPermissionService, "clear_partial_member_list", return_value=None),patch.object(DatasetPermissionService, "get_dataset_partial_member_list", return_value=[]),):- result, _ = method(api, DatasetUpdatePayload(), MagicMock(), "tenant", make_account(), dataset_id)+ result, _ = method(api, DatasetUpdatePayload(), self.session, "tenant", make_account(), dataset_id)assert result["partial_member_list"] == []-class TestDatasetApiDelete:+class TestDatasetApiDelete(_UsesSQLiteSession):def test_delete_success(self, app: Flask):api = DatasetApi()method = unwrap(api.delete)@@ -802,7 +826,7 @@ class TestDatasetApiDelete:patch.object(DatasetService, "delete_dataset", return_value=True),patch.object(DatasetPermissionService, "clear_partial_member_list", return_value=None),):- result, status = method(api, MagicMock(), user, dataset_id)+ result, status = method(api, self.session, user, dataset_id)assert status == 204assert result == ""@@ -813,7 +837,7 @@ class TestDatasetApiDelete:user = make_account(TenantAccountRole.NORMAL)with app.test_request_context(f"/datasets/{dataset_id}"):with pytest.raises(Forbidden):- method(api, MagicMock(), user, dataset_id)+ method(api, self.session, user, dataset_id)def test_delete_dataset_not_found(self, app: Flask):api = DatasetApi()@@ -825,7 +849,7 @@ class TestDatasetApiDelete:patch.object(DatasetService, "delete_dataset", return_value=False),):with pytest.raises(NotFound, match="Dataset not found"):- method(api, MagicMock(), user, dataset_id)+ method(api, self.session, user, dataset_id)def test_delete_dataset_in_use(self, app: Flask):api = DatasetApi()@@ -837,10 +861,10 @@ class TestDatasetApiDelete:patch.object(DatasetService, "delete_dataset", side_effect=services.errors.dataset.DatasetInUseError()),):with pytest.raises(DatasetInUseError):- method(api, MagicMock(), user, dataset_id)+ method(api, self.session, user, dataset_id)-class TestDatasetUseCheckApi:+class TestDatasetUseCheckApi(_UsesSQLiteSession):@pytest.mark.parametrize("is_using", [True, False])def test_get_use_check(self, app: Flask, is_using: bool):api = DatasetUseCheckApi()@@ -848,7 +872,7 @@ class TestDatasetUseCheckApi:dataset_id = "dataset-id"dataset = make_dataset(id=dataset_id)current_user = make_account()- session = MagicMock()+ session = self.sessionwith (app.test_request_context(f"/datasets/{dataset_id}/use-check"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,@@ -867,7 +891,7 @@ class TestDatasetUseCheckApi:api = DatasetUseCheckApi()method = unwrap(api.get)dataset = make_dataset(id="dataset-id")- session = MagicMock()+ session = self.sessionwith (app.test_request_context("/datasets/dataset-id/use-check"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),@@ -884,11 +908,11 @@ class TestDatasetUseCheckApi:"api_cls",[DatasetUseCheckApi, DatasetIndexingStatusApi, DatasetErrorDocs, DatasetAutoDisableLogApi],)-def test_dataset_scoped_read_permission_denied(app: Flask, api_cls):+def test_dataset_scoped_read_permission_denied(app: Flask, api_cls, sqlite_session: Session):api = api_cls()method = unwrap(api.get)dataset = make_dataset(id="dataset-1")- session = MagicMock()+ session = sqlite_sessionwith (app.test_request_context("/"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),@@ -902,7 +926,7 @@ def test_dataset_scoped_read_permission_denied(app: Flask, api_cls):method(api, session, "tenant-1", make_account(), "dataset-1")-class TestDatasetQueryApi:+class TestDatasetQueryApi(_UsesSQLiteSession):def _query_record(self, index: int = 1) -> DatasetQuery:query = DatasetQuery(dataset_id="dataset-id",@@ -929,7 +953,7 @@ class TestDatasetQueryApi:patch.object(DatasetService, "check_dataset_permission", return_value=None),patch.object(DatasetService, "get_dataset_queries", return_value=(queries, 2)),):- response, status = method(api, MagicMock(), current_user, dataset_id)+ response, status = method(api, self.session, current_user, dataset_id)assert status == 200assert response["total"] == 2assert response["page"] == 1@@ -952,24 +976,30 @@ class TestDatasetQueryApi:dataset = make_dataset(id="dataset-id")query = self._query_record()query.content = json.dumps([{"content_type": "image_query", "content": "file-1"}])- upload_file = SimpleNamespace(- id="file-1",+ upload_file = UploadFile(+ tenant_id="tenant-1",+ storage_type=StorageType.LOCAL,+ key="image.png",name="image.png",size=10,extension="png",mime_type="image/png",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="account-1",+ created_at=datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC),+ used=False,)- session = MagicMock()- session.scalar.return_value = upload_file+ upload_file.id = "file-1"+ session = self.session+ session.add(upload_file)+ session.flush()with (app.test_request_context("/datasets/queries"),patch.object(DatasetService, "get_dataset", return_value=dataset),patch.object(DatasetService, "check_dataset_permission", return_value=None),patch.object(DatasetService, "get_dataset_queries", return_value=([query], 1)),- patch("models.dataset.db") as db_mock,patch("models.dataset.sign_upload_file_preview_url", return_value="signed-url"),):- db_mock.session.scalar.return_value = upload_fileresponse, status = method(api, session, make_account(), "dataset-id")assert status == 200@@ -987,8 +1017,7 @@ class TestDatasetQueryApi:},}]- session.scalar.assert_called_once()- db_mock.session.scalar.assert_not_called()+ assert session.get(UploadFile, "file-1") is upload_filedef test_get_queries_dataset_not_found(self, app: Flask):api = DatasetQueryApi()@@ -1000,7 +1029,7 @@ class TestDatasetQueryApi:patch.object(DatasetService, "get_dataset", return_value=None),):with pytest.raises(NotFound, match="Dataset not found"):- method(api, MagicMock(), current_user, dataset_id)+ method(api, self.session, current_user, dataset_id)def test_get_queries_permission_denied(self, app: Flask):api = DatasetQueryApi()@@ -1018,7 +1047,7 @@ class TestDatasetQueryApi:),):with pytest.raises(Forbidden):- method(api, MagicMock(), current_user, dataset_id)+ method(api, self.session, current_user, dataset_id)def test_get_queries_pagination_has_more(self, app: Flask):api = DatasetQueryApi()@@ -1033,13 +1062,13 @@ class TestDatasetQueryApi:patch.object(DatasetService, "check_dataset_permission", return_value=None),patch.object(DatasetService, "get_dataset_queries", return_value=(queries, 40)),):- response, status = method(api, MagicMock(), current_user, dataset_id)+ response, status = method(api, self.session, current_user, dataset_id)assert status == 200assert response["has_more"] is Trueassert len(response["data"]) == 20-class TestDatasetIndexingEstimateApi:+class TestDatasetIndexingEstimateApi(_UsesSQLiteSession):def _upload_file(self, *, tenant_id: str = "tenant-1", file_id: str = "file-1") -> UploadFile:upload_file = UploadFile(tenant_id=tenant_id,@@ -1072,8 +1101,9 @@ class TestDatasetIndexingEstimateApi:method = unwrap(api.post)payload = self._base_payload()mock_file = self._upload_file()- session = MagicMock()- session.scalars.return_value.all.return_value = [mock_file]+ session = self.session+ session.add(mock_file)+ session.flush()mock_response = IndexingEstimate(total_segments=100, preview=[])@@ -1102,8 +1132,7 @@ class TestDatasetIndexingEstimateApi:api = DatasetIndexingEstimateApi()method = unwrap(api.post)payload = self._base_payload()- session = MagicMock()- session.scalars.return_value.all.return_value = None+ session = self.sessionwith (app.test_request_context("/"),patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),@@ -1122,8 +1151,9 @@ class TestDatasetIndexingEstimateApi:method = unwrap(api.post)mock_file = self._upload_file()payload = self._base_payload()- session = MagicMock()- session.scalars.return_value.all.return_value = [mock_file]+ session = self.session+ session.add(mock_file)+ session.flush()with (app.test_request_context("/"),patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),@@ -1146,8 +1176,9 @@ class TestDatasetIndexingEstimateApi:method = unwrap(api.post)mock_file = self._upload_file()payload = self._base_payload()- session = MagicMock()- session.scalars.return_value.all.return_value = [mock_file]+ session = self.session+ session.add(mock_file)+ session.flush()with (app.test_request_context("/"),patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),@@ -1170,8 +1201,9 @@ class TestDatasetIndexingEstimateApi:method = unwrap(api.post)mock_file = self._upload_file()payload = self._base_payload()- session = MagicMock()- session.scalars.return_value.all.return_value = [mock_file]+ session = self.session+ session.add(mock_file)+ session.flush()with (app.test_request_context("/"),patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),@@ -1189,16 +1221,16 @@ class TestDatasetIndexingEstimateApi:)-class TestDatasetRelatedAppListApi:+class TestDatasetRelatedAppListApi(_UsesSQLiteSession):def test_get_success(self, app: Flask):api = DatasetRelatedAppListApi()method = unwrap(api.get)dataset = make_dataset(id="dataset-1")app1 = make_related_app(id="app-1", name="App 1")app2 = make_related_app(id="app-2", name="App 2")- join1 = MagicMock(app_id="app-1")- join2 = MagicMock(app_id="app-2")- session = MagicMock()+ join1 = AppDatasetJoin(app_id="app-1", dataset_id="dataset-1")+ join2 = AppDatasetJoin(app_id="app-2", dataset_id="dataset-1")+ session = self.sessionwith (app.test_request_context("/"),patch("controllers.console.datasets.datasets.DatasetService.get_dataset", return_value=dataset),@@ -1251,7 +1283,7 @@ class TestDatasetRelatedAppListApi:patch("controllers.console.datasets.datasets.DatasetService.get_dataset", return_value=None),):with pytest.raises(NotFound):- method(api, MagicMock(), make_account(), "dataset-1")+ method(api, self.session, make_account(), "dataset-1")def test_get_permission_denied(self, app: Flask):api = DatasetRelatedAppListApi()@@ -1266,16 +1298,16 @@ class TestDatasetRelatedAppListApi:),):with pytest.raises(Forbidden):- method(api, MagicMock(), make_account(), "dataset-1")+ method(api, self.session, make_account(), "dataset-1")def test_get_filters_none_apps(self, app: Flask):api = DatasetRelatedAppListApi()method = unwrap(api.get)dataset = make_dataset(id="dataset-1")app1 = make_related_app()- join1 = MagicMock(app_id="app-1")- join2 = MagicMock(app_id="app-2")- session = MagicMock()+ join1 = AppDatasetJoin(app_id="app-1", dataset_id="dataset-1")+ join2 = AppDatasetJoin(app_id="app-2", dataset_id="dataset-1")+ session = self.sessionwith (app.test_request_context("/"),patch("controllers.console.datasets.datasets.DatasetService.get_dataset", return_value=dataset),@@ -1303,26 +1335,17 @@ class TestDatasetRelatedAppListApi:]-class TestDatasetIndexingStatusApi:+class TestDatasetIndexingStatusApi(_UsesSQLiteSession):def test_get_success_with_documents(self, app: Flask):api = DatasetIndexingStatusApi()method = unwrap(api.get)dataset = make_dataset(id="dataset-1")current_user = make_account()- document = MagicMock()- document.id = "doc-1"- document.indexing_status = "completed"- document.processing_started_at = None- document.parsing_completed_at = None- document.cleaning_completed_at = None- document.splitting_completed_at = None- document.completed_at = None- document.paused_at = None- document.error = None- document.stopped_at = None- session = MagicMock()- session.scalars.return_value.all.return_value = [document]- session.scalar.return_value = 3+ document = make_document_status()+ session = self.session+ session.add(document)+ session.add_all([make_document_segment(position=position, completed=True) for position in range(1, 4)])+ session.flush()with (app.test_request_context("/"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,@@ -1337,16 +1360,13 @@ class TestDatasetIndexingStatusApi:assert item["total_segments"] == 3get_dataset.assert_called_once_with("dataset-1", "tenant-1", session=session)check_permission.assert_called_once_with(dataset, current_user, session)- assert {"dataset-1", "tenant-1"} <= set(session.scalars.call_args.args[0].compile().params.values())- for segment_count_call in session.scalar.call_args_list:- assert {"dataset-1", "tenant-1", "doc-1"} <= set(segment_count_call.args[0].compile().params.values())+ assert session.get(Document, "doc-1") is documentdef test_get_success_no_documents(self, app: Flask):api = DatasetIndexingStatusApi()method = unwrap(api.get)dataset = make_dataset(id="dataset-1")- session = MagicMock()- session.scalars.return_value.all.return_value = []+ session = self.sessionwith (app.test_request_context("/"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),@@ -1360,20 +1380,11 @@ class TestDatasetIndexingStatusApi:api = DatasetIndexingStatusApi()method = unwrap(api.get)dataset = make_dataset(id="dataset-1")- document = MagicMock()- document.id = "doc-1"- document.indexing_status = "indexing"- document.processing_started_at = None- document.parsing_completed_at = None- document.cleaning_completed_at = None- document.splitting_completed_at = None- document.completed_at = None- document.paused_at = None- document.error = None- document.stopped_at = None- session = MagicMock()- session.scalars.return_value.all.return_value = [document]- session.scalar.side_effect = [2, 5]+ document = make_document_status(indexing_status=IndexingStatus.INDEXING)+ session = self.session+ session.add(document)+ session.add_all([make_document_segment(position=position, completed=position <= 2) for position in range(1, 6)])+ session.flush()with (app.test_request_context("/"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),@@ -1386,7 +1397,7 @@ class TestDatasetIndexingStatusApi:assert item["total_segments"] == 5-class TestDatasetApiKeyApi:+class TestDatasetApiKeyApi(_UsesSQLiteSession):def test_get_api_keys_success(self, app: Flask):api = DatasetApiKeyApi()method = unwrap(api.get)@@ -1404,8 +1415,11 @@ class TestDatasetApiKeyApi:last_used_at=None,created_at=None,)- session = MagicMock()- session.scalars.return_value.all.return_value = [mock_key_1, mock_key_2]+ session = self.session+ mock_key_1.tenant_id = "tenant-1"+ mock_key_2.tenant_id = "tenant-1"+ session.add_all([mock_key_1, mock_key_2])+ session.flush()with app.test_request_context("/"):response = method(api, session, "tenant-1")assert "data" in response@@ -1418,30 +1432,31 @@ class TestDatasetApiKeyApi:def test_post_create_api_key_success(self, app: Flask):api = DatasetApiKeyApi()method = unwrap(api.post)- mock_token = MagicMock()- mock_token.id = "new-key-id"- mock_token.last_used_at = None- mock_token.created_at = datetime.datetime(2024, 1, 1, 0, 0, 0, tzinfo=datetime.UTC)- mock_api_token_cls = MagicMock()- mock_api_token_cls.return_value = mock_token- mock_api_token_cls.generate_api_key.return_value = "dataset-abc123"- session = MagicMock()- session.scalar.return_value = 3- with app.test_request_context("/"), patch("controllers.console.datasets.datasets.ApiToken", mock_api_token_cls):+ session = self.session+ with (+ app.test_request_context("/"),+ patch.object(ApiToken, "generate_api_key", return_value="dataset-abc123") as generate_api_key,+ ):response, status = method(api, session, "tenant-1")assert status == 200assert isinstance(response, dict)- assert response["id"] == "new-key-id"assert response["token"] == "dataset-abc123"assert response["type"] == "dataset"assert response["created_at"] is not None- mock_api_token_cls.generate_api_key.assert_called_once_with("dataset-", 24, session=session)+ generate_api_key.assert_called_once_with("dataset-", 24, session=session)+ assert session.get(ApiToken, response["id"]).token == "dataset-abc123"def test_post_exceed_max_keys(self, app: Flask):api = DatasetApiKeyApi()method = unwrap(api.post)- session = MagicMock()- session.scalar.return_value = 10+ session = self.session+ session.add_all(+ [+ ApiToken(id=f"key-{index}", tenant_id="tenant-1", type="dataset", token=f"ds-{index}")+ for index in range(10)+ ]+ )+ session.flush()with app.test_request_context("/"):with pytest.raises(BadRequest) as exc_info:method(api, session, "tenant-1")@@ -1452,36 +1467,42 @@ class TestDatasetApiKeyApi:}-class TestDatasetApiDeleteApi:+class TestDatasetApiDeleteApi(_UsesSQLiteSession):def test_delete_success(self, app: Flask):api = DatasetApiDeleteApi()method = unwrap(api.delete)- mock_key = MagicMock()- session = MagicMock()- session.scalar.return_value = mock_key- with app.test_request_context("/"):+ session = self.session+ key = ApiToken(id="api-key-id", tenant_id="tenant-1", type="dataset", token="dataset-secret")+ session.add(key)+ session.flush()+ with (+ app.test_request_context("/"),+ patch("controllers.console.datasets.datasets.ApiTokenCache.delete") as delete_cache,+ ):response, status = method(api, session, "tenant-1", "api-key-id")assert status == 204assert response == ""+ delete_cache.assert_called_once()+ session.flush()+ assert session.get(ApiToken, "api-key-id") is Nonedef test_delete_key_not_found(self, app: Flask):api = DatasetApiDeleteApi()method = unwrap(api.delete)- session = MagicMock()- session.scalar.return_value = None+ session = self.sessionwith app.test_request_context("/"):with pytest.raises(NotFound):method(api, session, "tenant-1", "api-key-id")-class TestDatasetEnableApiApi:+class TestDatasetEnableApiApi(_UsesSQLiteSession):@pytest.mark.parametrize(("status_value", "enabled"), [("enable", True), ("disable", False)])def test_update_api_status(self, app: Flask, status_value: str, enabled: bool):api = DatasetEnableApiApi()method = unwrap(api.post)dataset = make_dataset(id="dataset-1")current_user = make_account()- session = MagicMock()+ session = self.sessionwith (app.test_request_context("/"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,@@ -1499,7 +1520,7 @@ class TestDatasetEnableApiApi:api = DatasetEnableApiApi()method = unwrap(api.post)dataset = make_dataset(id="dataset-1")- session = MagicMock()+ session = self.sessionwith (app.test_request_context("/"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset),@@ -1582,7 +1603,7 @@ class TestDatasetRetrievalSettingApi:]-class TestDatasetRetrievalSettingMockApi:+class TestDatasetRetrievalSettingMockApi(_UsesSQLiteSession):def test_get_success(self, app: Flask):api = DatasetRetrievalSettingMockApi()method = unwrap(api.get)@@ -1597,14 +1618,14 @@ class TestDatasetRetrievalSettingMockApi:assert response["retrieval_method"] == ["semantic"]-class TestDatasetErrorDocs:+class TestDatasetErrorDocs(_UsesSQLiteSession):def test_get_success(self, app: Flask):api = DatasetErrorDocs()method = unwrap(api.get)dataset = make_dataset(id="dataset-1")error_doc = make_document_status(id="error-doc", indexing_status=IndexingStatus.ERROR, error="failed")current_user = make_account()- session = MagicMock()+ session = self.sessionwith (app.test_request_context("/"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,@@ -1624,7 +1645,7 @@ class TestDatasetErrorDocs:def test_get_dataset_not_found(self, app: Flask):api = DatasetErrorDocs()method = unwrap(api.get)- session = MagicMock()+ session = self.sessionwith (app.test_request_context("/"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=None) as get_dataset,@@ -1634,7 +1655,7 @@ class TestDatasetErrorDocs:get_dataset.assert_called_once_with("dataset-1", "tenant-1", session=session)-class TestDatasetPermissionUserListApi:+class TestDatasetPermissionUserListApi(_UsesSQLiteSession):def test_get_success(self, app: Flask):api = DatasetPermissionUserListApi()method = unwrap(api.get)@@ -1649,7 +1670,7 @@ class TestDatasetPermissionUserListApi:return_value=users,),):- response, status = method(api, MagicMock(), make_account(), "dataset-1")+ response, status = method(api, self.session, make_account(), "dataset-1")assert status == 200assert response["data"] == users@@ -1666,17 +1687,17 @@ class TestDatasetPermissionUserListApi:),):with pytest.raises(Forbidden):- method(api, MagicMock(), make_account(), "dataset-1")+ method(api, self.session, make_account(), "dataset-1")-class TestDatasetAutoDisableLogApi:+class TestDatasetAutoDisableLogApi(_UsesSQLiteSession):def test_get_success(self, app: Flask):api = DatasetAutoDisableLogApi()method = unwrap(api.get)dataset = make_dataset(id="dataset-1")logs = {"document_ids": ["doc-1"], "count": 1}current_user = make_account()- session = MagicMock()+ session = self.sessionwith (app.test_request_context("/"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=dataset) as get_dataset,@@ -1693,7 +1714,7 @@ class TestDatasetAutoDisableLogApi:def test_get_dataset_not_found(self, app: Flask):api = DatasetAutoDisableLogApi()method = unwrap(api.get)- session = MagicMock()+ session = self.sessionwith (app.test_request_context("/"),patch.object(DatasetService, "get_dataset_for_tenant", return_value=None) as get_dataset,