Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: migrate document service sessions and ORM models to SQLite (#40536)
Asuka Minatoc2c187648408
diff --git a/api/tests/unit_tests/services/test_dataset_service_document.py b/api/tests/unit_tests/services/test_dataset_service_document.pyindex 7093053d1cb..1763cfab7b9 100644--- a/api/tests/unit_tests/services/test_dataset_service_document.py+++ b/api/tests/unit_tests/services/test_dataset_service_document.py@@ -1,5 +1,15 @@"""Unit tests for DocumentService behaviors in dataset_service."""+from datetime import datetime++from sqlalchemy import event, select+from sqlalchemy.orm import Session++from models.account import Tenant+from models.dataset import Dataset, DatasetCollectionBinding, DocumentSegment+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus+from models.model import UploadFile+from models.source import DataSourceOauthBindingfrom services.dataset_ref_service import DatasetRefServicefrom .dataset_service_test_helpers import (@@ -8,7 +18,6 @@ from .dataset_service_test_helpers import (CloudPlan,DatasetProcessRule,DatasetService,- DatasetServiceUnitDataFactory,DataSource,Document,DocumentIndexingError,@@ -18,7 +27,6 @@ from .dataset_service_test_helpers import (IndexStructureType,InfoList,KnowledgeConfig,- MagicMock,NotFound,NotionIcon,NotionInfo,@@ -32,18 +40,117 @@ from .dataset_service_test_helpers import (Segmentation,SimpleNamespace,WebsiteInfo,- _make_dataset,- _make_document,_make_features,_make_lock_context,_make_upload_knowledge_config,- create_autospec,json,patch,pytest,)+def _account(*, account_id: str = "user-1", tenant_id: str = "tenant-1") -> Account:+ account = Account(name="User", email=f"{account_id}@example.com")+ account.id = account_id+ tenant = Tenant(name="Tenant")+ tenant.id = tenant_id+ account._current_tenant = tenant+ return account+++def _dataset_row(+ *,+ dataset_id: str = "dataset-1",+ tenant_id: str = "tenant-1",+ built_in_field_enabled: bool = False,+ data_source_type: str | None = None,+ indexing_technique: str | None = "economy",+) -> Dataset:+ return Dataset(+ id=dataset_id,+ tenant_id=tenant_id,+ name="Dataset",+ description="",+ provider="vendor",+ created_by="user-1",+ maintainer="user-1",+ built_in_field_enabled=built_in_field_enabled,+ chunk_structure=IndexStructureType.PARAGRAPH_INDEX,+ data_source_type=data_source_type,+ indexing_technique=indexing_technique,+ )+++def _document_row(+ *,+ document_id: str = "document-1",+ dataset_id: str = "dataset-1",+ tenant_id: str = "tenant-1",+ name: str = "Document",+ indexing_status: str = IndexingStatus.COMPLETED,+ data_source_type: str = DataSourceType.UPLOAD_FILE,+ data_source_info: str = "{}",+ enabled: bool = True,+ archived: bool = False,+ is_paused: bool = False,+) -> Document:+ return Document(+ id=document_id,+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ position=1,+ data_source_type=data_source_type,+ data_source_info=data_source_info,+ batch="batch-1",+ name=name,+ created_from=DocumentCreatedFrom.API,+ created_by="user-1",+ created_at=datetime(2026, 1, 1),+ updated_at=datetime(2026, 1, 2),+ indexing_status=indexing_status,+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ word_count=10,+ enabled=enabled,+ archived=archived,+ is_paused=is_paused,+ completed_at=datetime(2026, 1, 2) if indexing_status == IndexingStatus.COMPLETED else None,+ )+++def _upload_file(*, file_id: str, tenant_id: str = "tenant-1", name: str = "upload.txt") -> UploadFile:+ upload_file = UploadFile(+ tenant_id=tenant_id,+ storage_type="opendal",+ key=f"key-{file_id}",+ name=name,+ size=1,+ extension="txt",+ mime_type="text/plain",+ created_by_role="account",+ created_by="user-1",+ created_at=datetime(2026, 1, 1),+ used=False,+ )+ upload_file.id = file_id+ return upload_file+++def _process_rule(+ *, dataset_id: str = "dataset-1", rule_id: str = "rule-1", mode: str = "automatic"+) -> DatasetProcessRule:+ rules = (+ json.dumps(DatasetProcessRule.AUTOMATIC_RULES)+ if mode == "automatic"+ else Rule(+ pre_processing_rules=[PreProcessingRule(id="remove_extra_spaces", enabled=True)],+ segmentation=Segmentation(separator="\n", max_tokens=100),+ ).model_dump_json()+ )+ process_rule = DatasetProcessRule(dataset_id=dataset_id, mode=mode, rules=rules, created_by="user-1")+ process_rule.id = rule_id+ return process_rule++class _RetryFlagLock:def __init__(self, store: "_RetryFlagStore", key: str):self.store = store@@ -102,183 +209,212 @@ class TestDocumentServiceDisplayStatus:assert DocumentService.build_display_status_filters("missing") == ()def test_apply_display_status_filter_returns_original_query_for_unknown_status(self):- query = MagicMock()+ query = select(Document)result = DocumentService.apply_display_status_filter(query, "missing")assert result is query- query.where.assert_not_called()def test_apply_display_status_filter_applies_where_for_known_status(self):- query = MagicMock()- filtered_query = MagicMock()- query.where.return_value = filtered_query+ query = select(Document)result = DocumentService.apply_display_status_filter(query, "enabled")- assert result is filtered_query- query.where.assert_called_once()+ assert result is not query+ assert "documents.enabled" in str(result)class TestDocumentServiceRetrieval:- def test_get_document_by_id_uses_provided_session(self):- session = MagicMock()- expected_document = DatasetServiceUnitDataFactory.create_document_mock(document_id="document-1")- session.get.return_value = expected_document+ def test_get_document_by_id_uses_provided_session(self, sqlite_session: Session):+ document = _document_row()+ sqlite_session.add(document)+ sqlite_session.commit()++ assert DocumentService.get_document_by_id(document.id, session=sqlite_session) is document+ assert DocumentService.get_document_by_id("missing", session=sqlite_session) is None++ def test_get_document_by_ids_enforces_dataset_owner_and_state(self, sqlite_session: Session):+ dataset = _dataset_row()+ expected = _document_row(document_id="expected")+ sqlite_session.add_all(+ [+ dataset,+ expected,+ _document_row(document_id="disabled", enabled=False),+ _document_row(document_id="archived", archived=True),+ _document_row(document_id="waiting", indexing_status=IndexingStatus.WAITING),+ _document_row(document_id="other-dataset", dataset_id="dataset-2"),+ _document_row(document_id="other-tenant", tenant_id="tenant-2"),+ ]+ )+ sqlite_session.commit()- result = DocumentService.get_document_by_id("document-1", session=session)+ documents = DocumentService.get_document_by_ids(+ DatasetRefService.create_dataset_ref(dataset),+ ["expected", "disabled", "archived", "waiting", "other-dataset", "other-tenant"],+ sqlite_session,+ )- assert result is expected_document- session.get.assert_called_once_with(Document, "document-1")+ assert [document.id for document in documents] == [expected.id]class TestDocumentServiceMutations:"""Unit tests for DocumentService mutation and orchestration helpers."""- @pytest.fixture- def rename_account_context(self):- class FakeAccount:- pass-- current_user = FakeAccount()- current_user.id = "user-123"- current_user.current_tenant_id = "tenant-123"-- with (- patch("services.dataset_service.Account", FakeAccount),- patch("services.dataset_service.current_user", current_user),- ):- yield current_user-@pytest.mark.parametrize(("archived", "expected"), [(True, True), (False, False)])def test_check_archived_returns_boolean_status(self, archived, expected):- document = DatasetServiceUnitDataFactory.create_document_mock(archived=archived)+ document = _document_row(archived=archived)assert DocumentService.check_archived(document) is expected- def test_delete_documents_limits_query_and_cleanup_to_dataset_ref(self):- session = MagicMock()- dataset = _make_dataset(- dataset_id="dataset-1",- tenant_id="tenant-1",- doc_form=IndexStructureType.PARAGRAPH_INDEX,+ def test_delete_documents_limits_query_and_cleanup_to_dataset_ref(self, sqlite_session: Session):+ dataset = _dataset_row()+ document = _document_row(+ document_id="doc-1",+ data_source_info=json.dumps({"upload_file_id": "file-1"}),)- document = _make_document(document_id="doc-1", dataset_id=dataset.id, tenant_id=dataset.tenant_id)-- with (- patch("services.dataset_service.batch_clean_document_task") as clean_task,- ):- session.scalars.return_value.all.return_value = [document]+ other_dataset = _document_row(document_id="other-dataset", dataset_id="dataset-2")+ other_tenant = _document_row(document_id="other-tenant", tenant_id="tenant-2")+ sqlite_session.add_all([dataset, document, other_dataset, other_tenant])+ sqlite_session.commit()+ with patch("services.dataset_service.batch_clean_document_task") as clean_task:dataset_ref = DatasetRefService.create_dataset_ref(dataset)DocumentService.delete_documents(dataset_ref,- ["doc-1", "other-doc"],- dataset.doc_form,- session,+ [document.id, other_dataset.id, other_tenant.id],+ IndexStructureType.PARAGRAPH_INDEX,+ sqlite_session,)- stmt = session.scalars.call_args.args[0]- compiled = stmt.compile()- statement = str(compiled)- assert "documents.id IN" in statement- assert "documents.tenant_id" in statement- assert "documents.dataset_id" in statement- assert ["doc-1", "other-doc"] in compiled.params.values()- assert dataset.tenant_id in compiled.params.values()- assert dataset.id in compiled.params.values()- session.delete.assert_called_once_with(document)- session.commit.assert_called_once()- clean_task.delay.assert_called_once_with(["doc-1"], dataset.id, dataset.doc_form, [])-- def test_rename_document_raises_when_dataset_is_missing(self, rename_account_context):- session = MagicMock()-- with patch.object(DatasetService, "get_dataset", return_value=None):- with pytest.raises(ValueError, match="Dataset not found"):- DocumentService.rename_document("dataset-1", "doc-1", "New Name", session)+ assert sqlite_session.get(Document, document.id) is None+ assert sqlite_session.get(Document, other_dataset.id) is other_dataset+ assert sqlite_session.get(Document, other_tenant.id) is other_tenant+ clean_task.delay.assert_called_once_with(+ [document.id], dataset.id, IndexStructureType.PARAGRAPH_INDEX, ["file-1"]+ )- def test_rename_document_raises_when_document_is_missing(self, rename_account_context):- dataset = DatasetServiceUnitDataFactory.create_dataset_mock()- session = MagicMock()+ def test_delete_documents_with_empty_ids_does_not_commit(self, sqlite_session: Session):+ commits = 0- with (- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DocumentService, "get_document", return_value=None),- ):- with pytest.raises(ValueError, match="Document not found"):- DocumentService.rename_document(dataset.id, "doc-1", "New Name", session)+ def count_commit(_session):+ nonlocal commits+ commits += 1- def test_rename_document_rejects_cross_tenant_access(self, rename_account_context):- dataset = DatasetServiceUnitDataFactory.create_dataset_mock()- document = DatasetServiceUnitDataFactory.create_document_mock(tenant_id="tenant-other")- session = MagicMock()+ event.listen(sqlite_session, "after_commit", count_commit)+ DocumentService.delete_documents(+ DatasetRefService.create_dataset_ref(_dataset_row()), [], IndexStructureType.PARAGRAPH_INDEX, sqlite_session+ )+ event.remove(sqlite_session, "after_commit", count_commit)+ assert commits == 0- with (- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DocumentService, "get_document", return_value=document),- ):+ def test_rename_document_raises_when_dataset_is_missing(self, sqlite_session: Session):+ with patch("services.dataset_service.current_user", _account()):+ with pytest.raises(ValueError, match="Dataset not found"):+ DocumentService.rename_document("dataset-1", "doc-1", "New Name", sqlite_session)++ def test_rename_document_raises_when_document_is_missing(self, sqlite_session: Session):+ dataset = _dataset_row()+ sqlite_session.add(dataset)+ sqlite_session.commit()+ with patch("services.dataset_service.current_user", _account()):+ with pytest.raises(ValueError, match="Document not found"):+ DocumentService.rename_document(dataset.id, "doc-1", "New Name", sqlite_session)++ def test_rename_document_rejects_cross_tenant_access(self, sqlite_session: Session):+ dataset = _dataset_row()+ document = _document_row(tenant_id="tenant-other")+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()+ with patch("services.dataset_service.current_user", _account()):with pytest.raises(ValueError, match="No permission"):- DocumentService.rename_document(dataset.id, document.id, "New Name", session)+ DocumentService.rename_document(dataset.id, document.id, "New Name", sqlite_session)- def test_rename_document_updates_document_metadata_and_upload_file_name(self, rename_account_context):- session = MagicMock()- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(- built_in_field_enabled=True,- tenant_id="tenant-1",+ def test_rename_document_updates_document_metadata_and_upload_file_name(self, sqlite_session: Session):+ dataset = _dataset_row(built_in_field_enabled=True)+ document = _document_row(+ data_source_info=json.dumps({"upload_file_id": "file-1"}),)- document = DatasetServiceUnitDataFactory.create_document_mock(- tenant_id="tenant-1",- doc_metadata={"title": "Old"},- data_source_info_dict={"upload_file_id": "file-1"},+ document.doc_metadata = {BuiltInField.document_name: "Old"}+ upload_file = UploadFile(+ tenant_id=dataset.tenant_id,+ storage_type="opendal",+ key="key",+ name="old.txt",+ size=1,+ extension="txt",+ mime_type="text/plain",+ created_by_role="account",+ created_by="user-1",+ created_at=datetime(2026, 1, 1),+ used=False,)- rename_account_context.current_tenant_id = "tenant-1"+ upload_file.id = "file-1"+ sqlite_session.add_all([dataset, document, upload_file])+ sqlite_session.commit()+ commits = 0- with (- patch.object(DatasetService, "get_dataset", return_value=dataset),- patch.object(DocumentService, "get_document", return_value=document),- ):- result = DocumentService.rename_document(dataset.id, document.id, "New Name", session)+ def count_commit(_session):+ nonlocal commits+ commits += 1++ event.listen(sqlite_session, "after_commit", count_commit)+ with patch("services.dataset_service.current_user", _account()):+ result = DocumentService.rename_document(dataset.id, document.id, "New Name", sqlite_session)+ event.remove(sqlite_session, "after_commit", count_commit)assert result is documentassert document.name == "New Name"assert document.doc_metadata[BuiltInField.document_name] == "New Name"- session.add.assert_called_once_with(document)- session.execute.assert_called()- session.flush.assert_called_once()-- def test_recover_document_raises_when_document_is_not_paused(self):- document = DatasetServiceUnitDataFactory.create_document_mock(is_paused=False)- session = MagicMock()+ assert sqlite_session.get(UploadFile, upload_file.id).name == "New Name"+ assert commits == 0+ def test_recover_document_raises_when_document_is_not_paused(self, unbound_session: Session):+ document = _document_row(is_paused=False)with pytest.raises(DocumentIndexingError):- DocumentService.recover_document(document, session)-- def test_retry_document_raises_when_retry_flag_is_already_set(self, rename_account_context):- document = DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-1")- session = MagicMock()-- with patch("services.dataset_service.redis_client") as mock_redis:- mock_redis.lock.return_value.acquire.return_value = False+ DocumentService.recover_document(document, unbound_session)+ def test_recover_document_persists_and_dispatches(self, sqlite_session: Session):+ document = _document_row(is_paused=True)+ sqlite_session.add(document)+ sqlite_session.commit()+ with (+ patch("services.dataset_service.redis_client") as redis,+ patch("services.dataset_service.recover_document_indexing_task") as task,+ ):+ DocumentService.recover_document(document, sqlite_session)++ sqlite_session.expire_all()+ recovered = sqlite_session.get(Document, document.id)+ assert recovered is not None+ assert recovered.is_paused is False+ redis.delete.assert_called_once_with(f"document_{document.id}_is_paused")+ task.delay.assert_called_once_with(document.dataset_id, document.id)++ def test_retry_document_raises_when_retry_flag_is_already_set(self, sqlite_session: Session):+ document = _document_row(indexing_status=IndexingStatus.ERROR)+ sqlite_session.add(document)+ sqlite_session.commit()+ retry_flags = _RetryFlagStore({f"document_{document.id}_is_retried": "other-request"})+ with (+ patch("services.dataset_service.current_user", _account()),+ patch("services.dataset_service.redis_client", retry_flags),+ ):with pytest.raises(ValueError, match="being retried"):- DocumentService.retry_document("dataset-1", [document], session)+ DocumentService.retry_document("dataset-1", [document], sqlite_session)def test_retry_document_leaves_batch_unchanged_when_later_document_is_already_being_retried(- self, rename_account_context+ self, sqlite_session: Session):- first_document = DatasetServiceUnitDataFactory.create_document_mock(- document_id="doc-1", indexing_status="error"- )- second_document = DatasetServiceUnitDataFactory.create_document_mock(- document_id="doc-2", indexing_status="error"- )+ first_document = _document_row(document_id="doc-1", indexing_status=IndexingStatus.ERROR)+ second_document = _document_row(document_id="doc-2", indexing_status=IndexingStatus.ERROR)+ sqlite_session.add_all([first_document, second_document])+ sqlite_session.commit()first_retry_key = "document_doc-1_is_retried"second_retry_key = "document_doc-2_is_retried"retry_flags = _RetryFlagStore({second_retry_key: "other-request"})- session = MagicMock()-with (+ patch("services.dataset_service.current_user", _account()),patch("services.dataset_service.redis_client", retry_flags),patch("services.dataset_service.retry_document_indexing_task") as retry_task,):@@ -286,16 +422,16 @@ class TestDocumentServiceMutations:DocumentService.retry_document("dataset-1",[first_document, second_document],- session,+ sqlite_session,)- assert first_document.indexing_status == "error"- assert second_document.indexing_status == "error"+ assert first_document.indexing_status == IndexingStatus.ERROR+ assert second_document.indexing_status == IndexingStatus.ERRORassert first_retry_key not in retry_flags.valuesassert retry_flags.values[second_retry_key] == "other-request"retry_task.delay.assert_not_called()- def test_retry_document_does_not_release_a_retry_flag_reacquired_by_another_request(self, rename_account_context):+ def test_retry_document_does_not_release_a_retry_flag_reacquired_by_another_request(self, sqlite_session: Session):first_retry_key = "document_doc-1_is_retried"second_retry_key = "document_doc-2_is_retried"retry_flags = _RetryFlagStore(@@ -303,65 +439,93 @@ class TestDocumentServiceMutations:replacement_on_conflict=(first_retry_key, "new-owner"),)documents = [- DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-1", indexing_status="error"),- DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-2", indexing_status="error"),+ _document_row(document_id="doc-1", indexing_status=IndexingStatus.ERROR),+ _document_row(document_id="doc-2", indexing_status=IndexingStatus.ERROR),]+ sqlite_session.add_all(documents)+ sqlite_session.commit()- with patch("services.dataset_service.redis_client", retry_flags):+ with (+ patch("services.dataset_service.current_user", _account()),+ patch("services.dataset_service.redis_client", retry_flags),+ ):with pytest.raises(ValueError, match="being retried"):- DocumentService.retry_document("dataset-1", documents, MagicMock())+ DocumentService.retry_document("dataset-1", documents, sqlite_session)assert retry_flags.values[first_retry_key] == "new-owner"assert retry_flags.values[second_retry_key] == "other-request"- def test_retry_document_releases_flags_when_status_commit_fails(self, rename_account_context):+ def test_retry_document_releases_flags_when_status_commit_fails(self, sqlite_session: Session):retry_flags = _RetryFlagStore()- document = DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-1", indexing_status="error")- session = MagicMock()- session.commit.side_effect = RuntimeError("database unavailable")+ document = _document_row(indexing_status=IndexingStatus.ERROR)+ sqlite_session.add(document)+ sqlite_session.commit()++ def fail_commit(_session):+ raise RuntimeError("database unavailable")+ event.listen(sqlite_session, "before_commit", fail_commit)with (+ patch("services.dataset_service.current_user", _account()),patch("services.dataset_service.redis_client", retry_flags),patch("services.dataset_service.retry_document_indexing_task") as retry_task,):with pytest.raises(RuntimeError, match="database unavailable"):- DocumentService.retry_document("dataset-1", [document], session)+ DocumentService.retry_document("dataset-1", [document], sqlite_session)+ event.remove(sqlite_session, "before_commit", fail_commit)assert retry_flags.values == {}- session.rollback.assert_called_once_with()retry_task.delay.assert_not_called()- def test_sync_website_document_raises_when_sync_flag_exists(self):- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")- document = DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-1", dataset_id=dataset.id)- session = MagicMock()+ def test_retry_document_persists_status_and_dispatches(self, sqlite_session: Session):+ documents = [+ _document_row(document_id="doc-1", indexing_status=IndexingStatus.ERROR),+ _document_row(document_id="doc-2", indexing_status=IndexingStatus.PAUSED),+ ]+ sqlite_session.add_all(documents)+ sqlite_session.commit()+ retry_flags = _RetryFlagStore()+ with (+ patch("services.dataset_service.current_user", _account()),+ patch("services.dataset_service.redis_client", retry_flags),+ patch("services.dataset_service.retry_document_indexing_task") as task,+ ):+ DocumentService.retry_document("dataset-1", documents, sqlite_session)++ sqlite_session.expire_all()+ statuses = sqlite_session.scalars(select(Document.indexing_status).order_by(Document.id)).all()+ assert statuses == [IndexingStatus.WAITING, IndexingStatus.WAITING]+ task.delay.assert_called_once_with("dataset-1", ["doc-1", "doc-2"], "user-1")+ def test_sync_website_document_raises_when_sync_flag_exists(self, sqlite_session: Session):+ dataset = _dataset_row()+ document = _document_row()with patch("services.dataset_service.redis_client") as mock_redis:mock_redis.get.return_value = "1"with pytest.raises(ValueError, match="being synced"):- DocumentService.sync_website_document(dataset, document, session)+ DocumentService.sync_website_document(dataset, document, sqlite_session)- def test_sync_website_document_rejects_document_outside_dataset(self):- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")- document = DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-1", dataset_id="dataset-2")+ def test_sync_website_document_rejects_document_outside_dataset(self, sqlite_session: Session):+ dataset = _dataset_row()+ document = _document_row(dataset_id="dataset-2")with (pytest.raises(ValueError, match="Document not found"),patch("services.dataset_service.redis_client") as mock_redis,):- DocumentService.sync_website_document(dataset, document, MagicMock())+ DocumentService.sync_website_document(dataset, document, sqlite_session)mock_redis.get.assert_not_called()- def test_sync_website_document_updates_status_sets_cache_and_dispatches_task(self):- session = MagicMock()- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")- document = DatasetServiceUnitDataFactory.create_document_mock(- document_id="doc-1",- dataset_id=dataset.id,- data_source_info_dict={"mode": "crawl"},+ def test_sync_website_document_updates_status_sets_cache_and_dispatches_task(self, sqlite_session: Session):+ dataset = _dataset_row()+ document = _document_row(+ data_source_type=DataSourceType.WEBSITE_CRAWL,+ data_source_info=json.dumps({"mode": "crawl"}),)+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()with (patch("services.dataset_service.redis_client") as mock_redis,@@ -369,13 +533,14 @@ class TestDocumentServiceMutations:):mock_redis.get.return_value = None- DocumentService.sync_website_document(dataset, document, session)+ DocumentService.sync_website_document(dataset, document, sqlite_session)- assert document.indexing_status == "waiting"- assert '"mode": "scrape"' in document.data_source_info- session.add.assert_called_once_with(document)- session.commit.assert_called_once()- mock_redis.setex.assert_called_once_with("document_doc-1_is_sync", 600, 1)+ sqlite_session.expire_all()+ synced = sqlite_session.get(Document, document.id)+ assert synced is not None+ assert synced.indexing_status == IndexingStatus.WAITING+ assert synced.data_source_info_dict["mode"] == "scrape"+ mock_redis.setex.assert_called_once_with(f"document_{document.id}_is_sync", 600, 1)sync_task.delay.assert_called_once_with(dataset.id, document.id)@@ -384,17 +549,14 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:@pytest.fixturedef account_context(self):- account = create_autospec(Account, instance=True)- account.id = "user-1"- account.current_tenant_id = "tenant-1"+ account = _account()with patch("services.dataset_service.current_user", account):yield accountdef test_save_document_without_dataset_id_creates_high_quality_dataset_with_default_retrieval_model(- self, account_context+ self, account_context, sqlite_session: Session):- session = MagicMock()knowledge_config = KnowledgeConfig(indexing_technique="high_quality",data_source=DataSource(@@ -408,24 +570,21 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:summary_index_setting={"enable": True},is_multimodal=True,)- created_dataset = SimpleNamespace(- id="dataset-1",- tenant_id="tenant-1",- name="",- description=None,+ binding = DatasetCollectionBinding(+ provider_name="provider",+ model_name="embedding-model",+ type="dataset",+ collection_name="collection",)- first_document = SimpleNamespace(name="VeryLongDocumentNameForDataset.txt")+ binding.id = "binding-1"+ first_document = _document_row(name="VeryLongDocumentNameForDataset.txt")with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",- return_value=SimpleNamespace(id="binding-1"),+ return_value=binding,),- patch(- "services.dataset_service.Dataset",- side_effect=lambda **kwargs: created_dataset.__dict__.update(kwargs) or created_dataset,- ) as dataset_cls,patch.object(DocumentService, "save_document_with_dataset_id", return_value=([first_document], "batch-1")) as save_document,@@ -434,33 +593,32 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:tenant_id="tenant-1",knowledge_config=knowledge_config,account=account_context,- session=session,+ session=sqlite_session,)- assert dataset is created_datasetassert documents == [first_document]assert batch == "batch-1"- assert created_dataset.collection_binding_id == "binding-1"- assert created_dataset.retrieval_model["search_method"] == RetrievalMethod.SEMANTIC_SEARCH- assert created_dataset.retrieval_model["top_k"] == 4- assert created_dataset.summary_index_setting == {"enable": True}- assert created_dataset.is_multimodal is True- assert created_dataset.name == first_document.name[:18] + "..."+ assert dataset.collection_binding_id == "binding-1"+ assert dataset.retrieval_model["search_method"] == RetrievalMethod.SEMANTIC_SEARCH+ assert dataset.retrieval_model["top_k"] == 4+ assert dataset.summary_index_setting == {"enable": True}+ assert dataset.is_multimodal is True+ assert dataset.name == first_document.name[:18] + "..."assert (- created_dataset.description+ dataset.description== "useful for when you want to answer queries about the VeryLongDocumentNameForDataset.txt")- dataset_cls.assert_called_once()+ assert sqlite_session.get(Dataset, dataset.id) is datasetsave_document.assert_called_once_with(- created_dataset,+ dataset,knowledge_config,account_context,- session=session,+ session=sqlite_session,)- assert session.flush.call_count == 2- def test_save_document_without_dataset_id_uses_provided_retrieval_model(self, account_context):- session = MagicMock()+ def test_save_document_without_dataset_id_uses_provided_retrieval_model(+ self, account_context, sqlite_session: Session+ ):retrieval_model = RetrievalModel(search_method=RetrievalMethod.SEMANTIC_SEARCH,reranking_enable=True,@@ -482,31 +640,30 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:),retrieval_model=retrieval_model,)- created_dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1", name="", description=None)+ first_document = _document_row(name="Doc")with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),- patch(- "services.dataset_service.Dataset",- side_effect=lambda **kwargs: created_dataset.__dict__.update(kwargs) or created_dataset,- ),patch.object(DocumentService,"save_document_with_dataset_id",- return_value=([SimpleNamespace(name="Doc")], "batch-1"),+ return_value=([first_document], "batch-1"),),):- DocumentService.save_document_without_dataset_id(+ dataset, _, _ = DocumentService.save_document_without_dataset_id("tenant-1",knowledge_config,account_context,- session,+ sqlite_session,)- assert created_dataset.retrieval_model == retrieval_model.model_dump()- assert created_dataset.collection_binding_id is None+ assert dataset.retrieval_model == retrieval_model.model_dump()+ assert dataset.collection_binding_id is None+ assert sqlite_session.get(Dataset, dataset.id) is dataset- def test_save_document_without_dataset_id_rejects_sandbox_batch_upload(self, account_context):+ def test_save_document_without_dataset_id_rejects_sandbox_batch_upload(+ self, account_context, unbound_session: Session+ ):knowledge_config = KnowledgeConfig(indexing_technique="economy",data_source=DataSource(@@ -524,9 +681,10 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:),patch.object(DocumentService, "check_documents_upload_quota") as check_quota,):- session = MagicMock()with pytest.raises(ValueError, match="does not support batch upload"):- DocumentService.save_document_without_dataset_id("tenant-1", knowledge_config, account_context, session)+ DocumentService.save_document_without_dataset_id(+ "tenant-1", knowledge_config, account_context, unbound_session+ )check_quota.assert_not_called()@@ -536,15 +694,15 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:@pytest.fixturedef account_context(self):- account = create_autospec(Account, instance=True)- account.id = "user-1"- account.current_tenant_id = "tenant-1"+ account = _account()with patch("services.dataset_service.current_user", account):yield account- def test_update_document_with_dataset_id_raises_when_document_is_missing(self, account_context):- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")+ def test_update_document_with_dataset_id_raises_when_document_is_missing(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row()document_data = KnowledgeConfig(original_document_id="doc-1",indexing_technique="economy",@@ -555,25 +713,24 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:)),)- session = MagicMock()-- with (- patch.object(DocumentService, "get_document", return_value=None),- patch.object(DatasetService, "check_dataset_model_setting") as check_model_setting,- ):+ with patch.object(DatasetService, "check_dataset_model_setting") as check_model_setting:with pytest.raises(NotFound, match="Document not found"):DocumentService.update_document_with_dataset_id(dataset,document_data,account_context,- session=session,+ session=sqlite_session,)check_model_setting.assert_called_once_with(dataset)- def test_update_document_with_dataset_id_rejects_non_available_documents(self, account_context):- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")- document = SimpleNamespace(display_status="indexing")+ def test_update_document_with_dataset_id_rejects_non_available_documents(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row()+ document = _document_row(document_id="doc-1", indexing_status=IndexingStatus.INDEXING)+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()document_data = KnowledgeConfig(original_document_id="doc-1",indexing_technique="economy",@@ -584,25 +741,46 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:)),)- session = MagicMock()-- with (- patch.object(DocumentService, "get_document", return_value=document),- patch.object(DatasetService, "check_dataset_model_setting"),- ):+ with patch.object(DatasetService, "check_dataset_model_setting"):with pytest.raises(ValueError, match="Document is not available"):DocumentService.update_document_with_dataset_id(dataset,document_data,account_context,- session=session,+ session=sqlite_session,)- def test_update_document_with_dataset_id_upload_file_process_rule_and_name_override(self, account_context):- session = MagicMock()- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")- document = _make_document()- document.dataset_process_rule_id = "old-rule"+ def test_update_document_with_dataset_id_upload_file_process_rule_and_name_override(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row()+ document = _document_row(document_id="doc-1")+ upload_file = UploadFile(+ tenant_id=dataset.tenant_id,+ storage_type="opendal",+ key="key",+ name="upload.txt",+ size=1,+ extension="txt",+ mime_type="text/plain",+ created_by_role="account",+ created_by=account_context.id,+ created_at=datetime(2026, 1, 1),+ used=False,+ )+ upload_file.id = "file-1"+ segment = DocumentSegment(+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id=document.id,+ position=1,+ content="content",+ word_count=1,+ tokens=1,+ created_by=account_context.id,+ )+ sqlite_session.add_all([dataset, document, upload_file, segment])+ sqlite_session.commit()document_data = KnowledgeConfig(original_document_id="doc-1",indexing_technique="economy",@@ -622,26 +800,23 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:name="Renamed document",doc_form=IndexStructureType.QA_INDEX,)- created_process_rule = SimpleNamespace(id="rule-2")+ updated_at = datetime(2026, 2, 1)with (- patch.object(DocumentService, "get_document", return_value=document),patch.object(DatasetService, "check_dataset_model_setting"),- patch("services.dataset_service.DatasetProcessRule", return_value=created_process_rule),- patch("services.dataset_service.naive_utc_now", return_value="now"),+ patch("services.dataset_service.naive_utc_now", return_value=updated_at),patch("services.dataset_service.document_indexing_update_task") as update_task,):- session.scalar.return_value = SimpleNamespace(id="file-1", name="upload.txt")-result = DocumentService.update_document_with_dataset_id(dataset,document_data,account_context,- session=session,+ session=sqlite_session,)assert result is document- assert document.dataset_process_rule_id == "rule-2"+ assert document.dataset_process_rule_id is not None+ assert sqlite_session.get(DatasetProcessRule, document.dataset_process_rule_id) is not Noneassert document.data_source_type == "upload_file"assert document.data_source_info == '{"upload_file_id": "file-1"}'assert document.name == "Renamed document"@@ -651,17 +826,23 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:assert document.parsing_completed_at is Noneassert document.cleaning_completed_at is Noneassert document.splitting_completed_at is None- assert document.updated_at == "now"+ assert document.updated_at == updated_atassert document.created_from == "web"assert document.doc_form == IndexStructureType.QA_INDEX- assert session.commit.call_count == 3- session.execute.assert_called()+ sqlite_session.expire_all()+ persisted = sqlite_session.get(Document, document.id)+ assert persisted is not None+ assert persisted.name == "Renamed document"+ assert sqlite_session.get(DocumentSegment, segment.id).status == "re_segment"update_task.delay.assert_called_once_with(document.dataset_id, document.id)- def test_update_document_with_dataset_id_notion_import_requires_binding(self, account_context):- session = MagicMock()- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")- document = SimpleNamespace(display_status="available", id="doc-1", dataset_id="dataset-1")+ def test_update_document_with_dataset_id_notion_import_requires_binding(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row()+ document = _document_row(document_id="doc-1")+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()document_data = KnowledgeConfig(original_document_id="doc-1",indexing_technique="economy",@@ -679,24 +860,32 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:),)- with (- patch.object(DocumentService, "get_document", return_value=document),- patch.object(DatasetService, "check_dataset_model_setting"),- ):- session.scalar.return_value = None-+ with patch.object(DatasetService, "check_dataset_model_setting"):with pytest.raises(ValueError, match="Data source binding not found"):DocumentService.update_document_with_dataset_id(dataset,document_data,account_context,- session=session,+ session=sqlite_session,)- def test_update_document_with_dataset_id_website_crawl_updates_segments_and_dispatches_task(self, account_context):- session = MagicMock()- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")- document = _make_document()+ def test_update_document_with_dataset_id_website_crawl_updates_segments_and_dispatches_task(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row()+ document = _document_row(document_id="doc-1")+ segment = DocumentSegment(+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ document_id=document.id,+ position=1,+ content="content",+ word_count=1,+ tokens=1,+ created_by=account_context.id,+ )+ sqlite_session.add_all([dataset, document, segment])+ sqlite_session.commit()document_data = KnowledgeConfig(original_document_id="doc-1",indexing_technique="economy",@@ -715,16 +904,15 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:)with (- patch.object(DocumentService, "get_document", return_value=document),patch.object(DatasetService, "check_dataset_model_setting"),- patch("services.dataset_service.naive_utc_now", return_value="now"),+ patch("services.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),patch("services.dataset_service.document_indexing_update_task") as update_task,):result = DocumentService.update_document_with_dataset_id(dataset,document_data,account_context,- session=session,+ session=sqlite_session,)assert result is document@@ -735,7 +923,8 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:)assert document.name == ""assert document.doc_form == IndexStructureType.PARENT_CHILD_INDEX- session.execute.assert_called()+ sqlite_session.expire_all()+ assert sqlite_session.get(DocumentSegment, segment.id).status == "re_segment"update_task.delay.assert_called_once_with("dataset-1", "doc-1")@@ -882,9 +1071,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:@pytest.fixturedef account_context(self):- account = create_autospec(Account, instance=True)- account.id = "user-1"- account.current_tenant_id = "tenant-1"+ account = _account()with (patch("services.dataset_service.current_user", account),@@ -892,22 +1079,25 @@ class TestDocumentServiceSaveDocumentWithDatasetId:):yield account- def test_save_document_with_dataset_id_requires_file_info_for_upload_source(self, account_context):- dataset = _make_dataset()+ def test_save_document_with_dataset_id_requires_file_info_for_upload_source(+ self, account_context, unbound_session: Session+ ):+ dataset = _dataset_row()knowledge_config = _make_upload_knowledge_config(file_ids=None)with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=True)):- session = MagicMock()with pytest.raises(ValueError, match="File source info is required"):DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=unbound_session,)- def test_save_document_with_dataset_id_blocks_batch_upload_for_sandbox_plan(self, account_context):- dataset = _make_dataset()+ def test_save_document_with_dataset_id_blocks_batch_upload_for_sandbox_plan(+ self, account_context, unbound_session: Session+ ):+ dataset = _dataset_row()knowledge_config = _make_upload_knowledge_config(file_ids=["file-1", "file-2"])with (@@ -917,19 +1107,18 @@ class TestDocumentServiceSaveDocumentWithDatasetId:),patch.object(DocumentService, "check_documents_upload_quota") as check_quota,):- session = MagicMock()with pytest.raises(ValueError, match="does not support batch upload"):DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=unbound_session,)check_quota.assert_not_called()- def test_save_document_with_dataset_id_enforces_batch_upload_limit(self, account_context):- dataset = _make_dataset()+ def test_save_document_with_dataset_id_enforces_batch_upload_limit(self, account_context, unbound_session: Session):+ dataset = _dataset_row()knowledge_config = _make_upload_knowledge_config(file_ids=["file-1", "file-2"])with (@@ -937,21 +1126,23 @@ class TestDocumentServiceSaveDocumentWithDatasetId:patch("services.dataset_service.dify_config.BATCH_UPLOAD_LIMIT", 1),patch.object(DocumentService, "check_documents_upload_quota") as check_quota,):- session = MagicMock()with pytest.raises(ValueError, match="batch upload limit of 1"):DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=unbound_session,)check_quota.assert_not_called()- def test_save_document_with_dataset_id_updates_existing_document_and_data_source_type(self, account_context):- dataset = _make_dataset(data_source_type=None)+ def test_save_document_with_dataset_id_updates_existing_document_and_data_source_type(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row(data_source_type=None)knowledge_config = _make_upload_knowledge_config(original_document_id="doc-1", file_ids=["file-1"])- updated_document = _make_document(document_id="doc-1", batch="batch-existing")+ updated_document = _document_row(document_id="doc-1")+ updated_document.batch = "batch-existing"with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),@@ -959,55 +1150,57 @@ class TestDocumentServiceSaveDocumentWithDatasetId:DocumentService, "update_document_with_dataset_id", return_value=updated_document) as update_document,):- session = MagicMock()documents, batch = DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=sqlite_session,)assert dataset.data_source_type == "upload_file"assert documents == [updated_document]assert batch == "batch-existing"- update_document.assert_called_once_with(dataset, knowledge_config, account_context, session=session)+ update_document.assert_called_once_with(dataset, knowledge_config, account_context, session=sqlite_session)- def test_save_document_with_dataset_id_requires_data_source_for_new_documents(self, account_context):- dataset = _make_dataset()+ def test_save_document_with_dataset_id_requires_data_source_for_new_documents(+ self, account_context, unbound_session: Session+ ):+ dataset = _dataset_row()knowledge_config = _make_upload_knowledge_config(data_source=None)with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)):- session = MagicMock()with pytest.raises(ValueError, match="Data source is required when creating new documents"):DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=unbound_session,)- def test_save_document_with_dataset_id_requires_existing_process_rule_for_custom_mode(self, account_context):- dataset = _make_dataset()+ def test_save_document_with_dataset_id_requires_existing_process_rule_for_custom_mode(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row()+ sqlite_session.add(dataset)+ sqlite_session.commit()knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"],process_rule=ProcessRule(mode="custom"),)with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)):- session = MagicMock()- session.scalar.return_value = Nonewith pytest.raises(ValueError, match="No process rule found"):DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=sqlite_session,)- session.scalar.assert_called_once()-- def test_save_document_with_dataset_id_rejects_invalid_indexing_technique(self, account_context):- dataset = _make_dataset(indexing_technique=None)+ def test_save_document_with_dataset_id_rejects_invalid_indexing_technique(+ self, account_context, unbound_session: Session+ ):+ dataset = _dataset_row(indexing_technique=None)knowledge_config = SimpleNamespace(doc_form=IndexStructureType.PARAGRAPH_INDEX,original_document_id=None,@@ -1016,17 +1209,18 @@ class TestDocumentServiceSaveDocumentWithDatasetId:)with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)):- session = MagicMock()with pytest.raises(ValueError, match="Indexing technique is invalid"):DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=unbound_session,)- def test_save_document_with_dataset_id_returns_empty_for_invalid_process_rule_mode(self, account_context):- dataset = _make_dataset()+ def test_save_document_with_dataset_id_returns_empty_for_invalid_process_rule_mode(+ self, account_context, unbound_session: Session+ ):+ dataset = _dataset_row()knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"])knowledge_config.process_rule = SimpleNamespace(mode="unsupported-mode", rules=None)@@ -1035,77 +1229,61 @@ class TestDocumentServiceSaveDocumentWithDatasetId:dataset,knowledge_config,account_context,- session=MagicMock(),+ session=unbound_session,)assert documents == []assert batch == ""- def test_save_document_with_dataset_id_upload_file_creates_and_reindexes_documents(self, account_context):- session = MagicMock()- dataset = _make_dataset()- dataset_process_rule = SimpleNamespace(id="rule-1")+ def test_save_document_with_dataset_id_upload_file_creates_and_reindexes_documents(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row(data_source_type=DataSourceType.UPLOAD_FILE)+ dataset_process_rule = _process_rule()knowledge_config = _make_upload_knowledge_config(file_ids=["file-1", "file-2"])- duplicate_document = _make_document(document_id="doc-duplicate", name="existing.txt")- created_document = _make_document(document_id="doc-created", name="new.txt")- upload_file_a = SimpleNamespace(id="file-1", name="existing.txt")- upload_file_b = SimpleNamespace(id="file-2", name="new.txt")+ duplicate_document = _document_row(document_id="doc-duplicate", name="existing.txt")+ upload_file_a = _upload_file(file_id="file-1", name="existing.txt")+ upload_file_b = _upload_file(file_id="file-2", name="new.txt")+ sqlite_session.add_all([dataset, dataset_process_rule, duplicate_document, upload_file_a, upload_file_b])+ sqlite_session.commit()with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.redis_client") as mock_redis,- patch.object(DocumentService, "get_documents_position", return_value=4),- patch.object(DocumentService, "build_document", return_value=created_document) as build_document,patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,patch("services.dataset_service.DuplicateDocumentIndexingTaskProxy") as duplicate_proxy_cls,- patch("services.dataset_service.naive_utc_now", return_value="now"),+ patch("services.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),patch("services.dataset_service.time.strftime", return_value="20260101010101"),patch("services.dataset_service.secrets.randbelow", return_value=23),):mock_redis.lock.return_value = _make_lock_context()- session.scalars.return_value.all.side_effect = [- [upload_file_a, upload_file_b],- [duplicate_document],- ]-documents, batch = DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,dataset_process_rule=dataset_process_rule,- session=session,+ session=sqlite_session,)- assert documents == [duplicate_document, created_document]+ assert [document.name for document in documents] == ["existing.txt", "new.txt"]assert batch == "20260101010101100023"assert duplicate_document.dataset_process_rule_id == "rule-1"- assert duplicate_document.updated_at == "now"+ assert duplicate_document.updated_at == datetime(2026, 2, 1)assert duplicate_document.batch == batch- assert duplicate_document.indexing_status == "waiting"- build_document.assert_called_once_with(- dataset,- "rule-1",- "upload_file",- IndexStructureType.PARAGRAPH_INDEX,- "English",- {"upload_file_id": "file-2"},- "web",- 4,- account_context,- "new.txt",- batch,- )- document_proxy_cls.assert_called_once_with(dataset.tenant_id, dataset.id, ["doc-created"])+ assert duplicate_document.indexing_status == IndexingStatus.WAITING+ created_document = next(document for document in documents if document.name == "new.txt")+ sqlite_session.expire_all()+ assert sqlite_session.get(Document, created_document.id) is not None+ document_proxy_cls.assert_called_once_with(dataset.tenant_id, dataset.id, [created_document.id])document_proxy_cls.return_value.delay.assert_called_once()duplicate_proxy_cls.assert_called_once_with(dataset.tenant_id, dataset.id, ["doc-duplicate"])duplicate_proxy_cls.return_value.delay.assert_called_once()def test_save_document_with_dataset_id_notion_import_truncates_names_and_cleans_removed_pages(- self, account_context+ self, account_context, sqlite_session: Session):- session = MagicMock()- dataset = _make_dataset()- dataset_process_rule = SimpleNamespace(id="rule-1")+ dataset = _dataset_row(data_source_type=DataSourceType.NOTION_IMPORT)+ dataset_process_rule = _process_rule()notion_page_name = "a" * 300knowledge_config = KnowledgeConfig(indexing_technique="economy",@@ -1132,42 +1310,50 @@ class TestDocumentServiceSaveDocumentWithDatasetId:doc_form=IndexStructureType.PARAGRAPH_INDEX,doc_language="English",)- existing_keep = _make_document(document_id="doc-keep")+ existing_keep = _document_row(+ document_id="doc-keep",+ data_source_type=DataSourceType.NOTION_IMPORT,+ )existing_keep.data_source_info = json.dumps({"notion_page_id": "page-keep"})- existing_remove = _make_document(document_id="doc-remove")+ existing_remove = _document_row(+ document_id="doc-remove",+ data_source_type=DataSourceType.NOTION_IMPORT,+ )existing_remove.data_source_info = json.dumps({"notion_page_id": "page-remove"})- created_document = _make_document(document_id="doc-new")+ sqlite_session.add_all([dataset, dataset_process_rule, existing_keep, existing_remove])+ sqlite_session.commit()with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.redis_client") as mock_redis,- patch.object(DocumentService, "get_documents_position", return_value=1),- patch.object(DocumentService, "build_document", return_value=created_document) as build_document,patch("services.dataset_service.clean_notion_document_task") as clean_task,patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,patch("services.dataset_service.uuid.uuid4", return_value="doc-new"),):mock_redis.lock.return_value = _make_lock_context()- session.scalars.return_value.all.return_value = [existing_keep, existing_remove]-documents, _ = DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,dataset_process_rule=dataset_process_rule,- session=session,+ session=sqlite_session,)+ created_document = next(document for document in documents if document.id == "doc-new")assert created_document in documents- assert len(build_document.call_args.args[9]) == 255+ assert len(created_document.name) == 255+ assert sqlite_session.get(Document, created_document.id) is created_documentclean_task.delay.assert_called_once_with(["doc-remove"], dataset.id)document_proxy_cls.assert_called_once_with(dataset.tenant_id, dataset.id, ["doc-new"])document_proxy_cls.return_value.delay.assert_called_once()- def test_save_document_with_dataset_id_website_crawl_truncates_long_urls(self, account_context):- session = MagicMock()- dataset = _make_dataset()- dataset_process_rule = SimpleNamespace(id="rule-1")+ def test_save_document_with_dataset_id_website_crawl_truncates_long_urls(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row(data_source_type=DataSourceType.WEBSITE_CRAWL)+ dataset_process_rule = _process_rule()+ sqlite_session.add_all([dataset, dataset_process_rule])+ sqlite_session.commit()long_url = "https://example.com/" + ("a" * 260)short_url = "https://example.com/short"knowledge_config = KnowledgeConfig(@@ -1186,18 +1372,9 @@ class TestDocumentServiceSaveDocumentWithDatasetId:doc_form=IndexStructureType.PARAGRAPH_INDEX,doc_language="English",)- first_document = _make_document(document_id="doc-1")- second_document = _make_document(document_id="doc-2")-with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.redis_client") as mock_redis,- patch.object(DocumentService, "get_documents_position", return_value=2),- patch.object(- DocumentService,- "build_document",- side_effect=[first_document, second_document],- ) as build_document,patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,):mock_redis.lock.return_value = _make_lock_context()@@ -1207,13 +1384,14 @@ class TestDocumentServiceSaveDocumentWithDatasetId:knowledge_config,account_context,dataset_process_rule=dataset_process_rule,- session=session,+ session=sqlite_session,)- assert documents == [first_document, second_document]- assert build_document.call_args_list[0].args[9] == long_url[:200] + "..."- assert build_document.call_args_list[1].args[9] == short_url- document_proxy_cls.assert_called_once_with(dataset.tenant_id, dataset.id, ["doc-1", "doc-2"])+ assert [document.name for document in documents] == [long_url[:200] + "...", short_url]+ assert sqlite_session.scalars(select(Document).order_by(Document.position)).all() == documents+ document_proxy_cls.assert_called_once_with(+ dataset.tenant_id, dataset.id, [document.id for document in documents]+ )document_proxy_cls.return_value.delay.assert_called_once()@@ -1221,16 +1399,16 @@ class TestDocumentServiceBatchUpdateStatus:"""Unit tests for batch_update_document_status orchestration and helper branches."""def test_prepare_disable_update_requires_completed_document(self):- document = _make_document(indexing_status="waiting")+ document = _document_row(indexing_status=IndexingStatus.WAITING)document.completed_at = Nonewith pytest.raises(DocumentIndexingError, match="is not completed"):- DocumentService._prepare_disable_update(document, user=SimpleNamespace(id="user-1"), now="now")+ DocumentService._prepare_disable_update(document, user=_account(), now=datetime(2026, 2, 1))def test_prepare_archive_update_sets_async_task_for_enabled_document(self):- document = _make_document(enabled=True, archived=False)+ document = _document_row(enabled=True, archived=False)- result = DocumentService._prepare_archive_update(document, user=SimpleNamespace(id="user-1"), now="now")+ result = DocumentService._prepare_archive_update(document, user=_account(), now=datetime(2026, 2, 1))assert result is not Noneassert result["updates"]["archived"] is True@@ -1238,59 +1416,63 @@ class TestDocumentServiceBatchUpdateStatus:assert result["async_task"]["args"] == [document.id]def test_prepare_unarchive_update_sets_async_task_for_enabled_document(self):- document = _make_document(enabled=True, archived=True)+ document = _document_row(enabled=True, archived=True)- result = DocumentService._prepare_unarchive_update(document, now="now")+ result = DocumentService._prepare_unarchive_update(document, now=datetime(2026, 2, 1))assert result is not Noneassert result["updates"]["archived"] is Falseassert result["set_cache"] is Trueassert result["async_task"]["args"] == [document.id]- def test_batch_update_document_status_rejects_indexing_documents(self):- session = MagicMock()- dataset = _make_dataset()- document = _make_document(name="Busy document")+ def test_batch_update_document_status_rejects_indexing_documents(self, sqlite_session: Session):+ dataset = _dataset_row()+ document = _document_row(name="Busy document")+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()- with (- patch.object(DocumentService, "get_document", return_value=document),- patch("services.dataset_service.redis_client") as mock_redis,- ):+ with patch("services.dataset_service.redis_client") as mock_redis:mock_redis.get.return_value = "1"with pytest.raises(DocumentIndexingError, match="Busy document is being indexed"):DocumentService.batch_update_document_status(- dataset, [document.id], "archive", SimpleNamespace(id="user-1"), session+ dataset, [document.id], "archive", _account(), sqlite_session)- session.flush.assert_not_called()+ sqlite_session.refresh(document)+ assert document.archived is False++ def test_batch_update_document_status_rolls_back_when_commit_fails(self, sqlite_session: Session):+ dataset = _dataset_row()+ document = _document_row(enabled=False)+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()- def test_batch_update_document_status_rolls_back_when_commit_fails(self):- session = MagicMock()- dataset = _make_dataset()- document = _make_document(enabled=False)+ def fail_commit(_session):+ raise RuntimeError("commit failed")+ event.listen(sqlite_session, "before_commit", fail_commit)with (- patch.object(DocumentService, "get_document", return_value=document),patch("services.dataset_service.redis_client") as mock_redis,):mock_redis.get.return_value = None- session.commit.side_effect = RuntimeError("commit failed")with pytest.raises(RuntimeError, match="commit failed"):DocumentService.batch_update_document_status(- dataset, [document.id], "enable", SimpleNamespace(id="user-1"), session+ dataset, [document.id], "enable", _account(), sqlite_session)+ event.remove(sqlite_session, "before_commit", fail_commit)- session.rollback.assert_called_once()+ sqlite_session.refresh(document)+ assert document.enabled is False- def test_batch_update_document_status_raises_async_task_error_after_commit(self):- session = MagicMock()- dataset = _make_dataset()- document = _make_document(enabled=False)+ def test_batch_update_document_status_raises_async_task_error_after_commit(self, sqlite_session: Session):+ dataset = _dataset_row()+ document = _document_row(enabled=False)+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()with (- patch.object(DocumentService, "get_document", return_value=document),patch("services.dataset_service.redis_client") as mock_redis,patch("services.dataset_service.add_document_to_index_task") as add_task,):@@ -1299,10 +1481,11 @@ class TestDocumentServiceBatchUpdateStatus:with pytest.raises(RuntimeError, match="task failed"):DocumentService.batch_update_document_status(- dataset, [document.id], "enable", SimpleNamespace(id="user-1"), session+ dataset, [document.id], "enable", _account(), sqlite_session)- session.commit.assert_called_once()+ sqlite_session.refresh(document)+ assert document.enabled is Truemock_redis.setex.assert_called_once_with(f"document_{document.id}_indexing", 600, 1)@@ -1311,25 +1494,36 @@ class TestDocumentServiceTenantAndUpdateEdges:@pytest.fixturedef account_context(self):- account = create_autospec(Account, instance=True)- account.id = "user-1"- account.current_tenant_id = "tenant-1"+ account = _account()with patch("services.dataset_service.current_user", account):yield account- def test_get_tenant_documents_count_returns_query_count(self, account_context):- session = MagicMock()- session.scalar.return_value = 12+ def test_get_tenant_documents_count_scopes_state_and_tenant(self, account_context, sqlite_session: Session):+ sqlite_session.add_all(+ [+ _document_row(document_id="one"),+ _document_row(document_id="two"),+ _document_row(document_id="disabled", enabled=False),+ _document_row(document_id="archived", archived=True),+ _document_row(document_id="unfinished", indexing_status=IndexingStatus.WAITING),+ _document_row(document_id="foreign", tenant_id="tenant-2"),+ ]+ )+ sqlite_session.commit()- result = DocumentService.get_tenant_documents_count(session)+ result = DocumentService.get_tenant_documents_count(sqlite_session)- assert result == 12+ assert result == 2- def test_update_document_with_dataset_id_uses_automatic_process_rule_payload(self, account_context):- session = MagicMock()- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")- document = _make_document()+ def test_update_document_with_dataset_id_uses_automatic_process_rule_payload(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row()+ document = _document_row(document_id="doc-1")+ upload_file = _upload_file(file_id="file-1")+ sqlite_session.add_all([dataset, document, upload_file])+ sqlite_session.commit()document_data = KnowledgeConfig(original_document_id="doc-1",indexing_technique="economy",@@ -1348,62 +1542,56 @@ class TestDocumentServiceTenantAndUpdateEdges:),doc_form=IndexStructureType.PARAGRAPH_INDEX,)- created_process_rule = SimpleNamespace(id="rule-2")+ updated_at = datetime(2026, 2, 1)with (- patch.object(DocumentService, "get_document", return_value=document),- patch("services.dataset_service.DatasetProcessRule") as process_rule_cls,patch.object(DatasetService, "check_dataset_model_setting"),- patch("services.dataset_service.naive_utc_now", return_value="now"),+ patch("services.dataset_service.naive_utc_now", return_value=updated_at),patch("services.dataset_service.document_indexing_update_task") as update_task,):- process_rule_cls.AUTOMATIC_RULES = DatasetProcessRule.AUTOMATIC_RULES- process_rule_cls.return_value = created_process_rule- session.scalar.return_value = SimpleNamespace(id="file-1", name="upload.txt")-result = DocumentService.update_document_with_dataset_id(dataset,document_data,account_context,- session=session,+ session=sqlite_session,)assert result is document- assert document.dataset_process_rule_id == "rule-2"+ assert document.dataset_process_rule_id is not Noneassert document.name == "upload.txt"- assert process_rule_cls.call_args.kwargs == {- "dataset_id": "dataset-1",- "mode": "automatic",- "rules": json.dumps(DatasetProcessRule.AUTOMATIC_RULES),- "created_by": "user-1",- }- assert session.commit.call_count == 3+ process_rule = sqlite_session.get(DatasetProcessRule, document.dataset_process_rule_id)+ assert process_rule is not None+ assert process_rule.mode == "automatic"+ assert process_rule.rules == json.dumps(DatasetProcessRule.AUTOMATIC_RULES)update_task.delay.assert_called_once_with("dataset-1", "doc-1")- def test_update_document_with_dataset_id_requires_upload_file_info(self, account_context):- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")+ def test_update_document_with_dataset_id_requires_upload_file_info(self, account_context, sqlite_session: Session):+ dataset = _dataset_row()+ document = _document_row(document_id="doc-1")+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()document_data = KnowledgeConfig(original_document_id="doc-1",indexing_technique="economy",data_source=DataSource(info_list=InfoList(data_source_type="upload_file")),)- with (- patch.object(DocumentService, "get_document", return_value=_make_document()),- patch.object(DatasetService, "check_dataset_model_setting"),- ):- session = MagicMock()+ with patch.object(DatasetService, "check_dataset_model_setting"):with pytest.raises(ValueError, match="No file info list found"):DocumentService.update_document_with_dataset_id(dataset,document_data,account_context,- session=session,+ session=sqlite_session,)- def test_update_document_with_dataset_id_raises_when_upload_file_is_missing(self, account_context):- session = MagicMock()- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")+ def test_update_document_with_dataset_id_raises_when_upload_file_is_missing(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row()+ document = _document_row(document_id="doc-1")+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()document_data = KnowledgeConfig(original_document_id="doc-1",indexing_technique="economy",@@ -1415,45 +1603,49 @@ class TestDocumentServiceTenantAndUpdateEdges:),)- with (- patch.object(DocumentService, "get_document", return_value=_make_document()),- patch.object(DatasetService, "check_dataset_model_setting"),- ):- session.scalar.return_value = None-+ with patch.object(DatasetService, "check_dataset_model_setting"):with pytest.raises(FileNotExistsError):DocumentService.update_document_with_dataset_id(dataset,document_data,account_context,- session=session,+ session=sqlite_session,)- def test_update_document_with_dataset_id_requires_notion_info_list(self, account_context):- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")+ def test_update_document_with_dataset_id_requires_notion_info_list(self, account_context, sqlite_session: Session):+ dataset = _dataset_row()+ document = _document_row(document_id="doc-1")+ sqlite_session.add_all([dataset, document])+ sqlite_session.commit()document_data = KnowledgeConfig(original_document_id="doc-1",indexing_technique="economy",data_source=DataSource(info_list=InfoList(data_source_type="notion_import")),)- with (- patch.object(DocumentService, "get_document", return_value=_make_document()),- patch.object(DatasetService, "check_dataset_model_setting"),- ):- session = MagicMock()+ with patch.object(DatasetService, "check_dataset_model_setting"):with pytest.raises(ValueError, match="No notion info list found"):DocumentService.update_document_with_dataset_id(dataset,document_data,account_context,- session=session,+ session=sqlite_session,)- def test_update_document_with_dataset_id_notion_import_updates_page_info(self, account_context):- session = MagicMock()- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")- document = _make_document()+ def test_update_document_with_dataset_id_notion_import_updates_page_info(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row()+ document = _document_row(document_id="doc-1")+ binding = DataSourceOauthBinding(+ tenant_id=dataset.tenant_id,+ access_token="token",+ provider="notion",+ source_info={"workspace_id": '"workspace-1"'},+ disabled=False,+ )+ sqlite_session.add_all([dataset, document, binding])+ sqlite_session.commit()document_data = KnowledgeConfig(original_document_id="doc-1",indexing_technique="economy",@@ -1476,18 +1668,15 @@ class TestDocumentServiceTenantAndUpdateEdges:)with (- patch.object(DocumentService, "get_document", return_value=document),patch.object(DatasetService, "check_dataset_model_setting"),- patch("services.dataset_service.naive_utc_now", return_value="now"),+ patch("services.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),patch("services.dataset_service.document_indexing_update_task") as update_task,):- session.scalar.return_value = SimpleNamespace(id="binding-1")-result = DocumentService.update_document_with_dataset_id(dataset,document_data,account_context,- session=session,+ session=sqlite_session,)assert result is document@@ -1502,6 +1691,7 @@ class TestDocumentServiceTenantAndUpdateEdges:"type": "database",})+ sqlite_session.refresh(document)update_task.delay.assert_called_once_with("dataset-1", "doc-1")@@ -1510,15 +1700,14 @@ class TestDocumentServiceSaveWithoutDatasetBilling:@pytest.fixturedef account_context(self):- account = create_autospec(Account, instance=True)- account.id = "user-1"- account.current_tenant_id = "tenant-1"+ account = _account()with patch("services.dataset_service.current_user", account):yield account- def test_save_document_without_dataset_id_counts_notion_pages_for_quota(self, account_context):- session = MagicMock()+ def test_save_document_without_dataset_id_counts_notion_pages_for_quota(+ self, account_context, sqlite_session: Session+ ):knowledge_config = KnowledgeConfig(indexing_technique="economy",data_source=DataSource(@@ -1542,33 +1731,32 @@ class TestDocumentServiceSaveWithoutDatasetBilling:)),)- created_dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1", name="", description=None)features = _make_features(enabled=True)+ document = _document_row(name="Doc")with (patch("services.dataset_service.FeatureService.get_features", return_value=features),patch("services.dataset_service.dify_config.BATCH_UPLOAD_LIMIT", "10"),patch.object(DocumentService, "check_documents_upload_quota") as check_quota,- patch(- "services.dataset_service.Dataset",- side_effect=lambda **kwargs: created_dataset.__dict__.update(kwargs) or created_dataset,- ),patch.object(DocumentService,"save_document_with_dataset_id",- return_value=([SimpleNamespace(name="Doc")], "batch-1"),+ return_value=([document], "batch-1"),),):- DocumentService.save_document_without_dataset_id(+ dataset, _, _ = DocumentService.save_document_without_dataset_id("tenant-1",knowledge_config,account_context,- session,+ sqlite_session,)check_quota.assert_called_once_with(3, features)+ assert sqlite_session.get(Dataset, dataset.id) is dataset- def test_save_document_without_dataset_id_enforces_batch_limit_for_website_urls(self, account_context):+ def test_save_document_without_dataset_id_enforces_batch_limit_for_website_urls(+ self, account_context, unbound_session: Session+ ):knowledge_config = KnowledgeConfig(indexing_technique="economy",data_source=DataSource(@@ -1589,9 +1777,10 @@ class TestDocumentServiceSaveWithoutDatasetBilling:patch("services.dataset_service.dify_config.BATCH_UPLOAD_LIMIT", "1"),patch.object(DocumentService, "check_documents_upload_quota") as check_quota,):- session = MagicMock()with pytest.raises(ValueError, match="batch upload limit of 1"):- DocumentService.save_document_without_dataset_id("tenant-1", knowledge_config, account_context, session)+ DocumentService.save_document_without_dataset_id(+ "tenant-1", knowledge_config, account_context, unbound_session+ )check_quota.assert_not_called()@@ -1727,9 +1916,7 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:@pytest.fixturedef account_context(self):- account = create_autospec(Account, instance=True)- account.id = "user-1"- account.current_tenant_id = "tenant-1"+ account = _account()with (patch("services.dataset_service.current_user", account),@@ -1738,21 +1925,29 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:yield accountdef test_save_document_with_dataset_id_initializes_high_quality_dataset_from_default_embedding_model(- self, account_context+ self, account_context, sqlite_session: Session):- dataset = _make_dataset(data_source_type=None, indexing_technique=None)+ dataset = _dataset_row(data_source_type=None, indexing_technique=None)knowledge_config = _make_upload_knowledge_config(original_document_id="doc-1", file_ids=["file-1"])knowledge_config.indexing_technique = "high_quality"knowledge_config.embedding_model = Noneknowledge_config.embedding_model_provider = None- updated_document = _make_document(batch="batch-existing")+ updated_document = _document_row(document_id="doc-1")+ updated_document.batch = "batch-existing"+ binding = DatasetCollectionBinding(+ provider_name="default-provider",+ model_name="default-embedding",+ type="dataset",+ collection_name="collection",+ )+ binding.id = "binding-1"with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.ModelManager") as model_manager_cls,patch("services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",- return_value=SimpleNamespace(id="binding-1"),+ return_value=binding,) as get_binding,patch.object(DocumentService, "update_document_with_dataset_id", return_value=updated_document),):@@ -1761,12 +1956,11 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:provider="default-provider",)- session = MagicMock()documents, batch = DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=sqlite_session,)assert documents == [updated_document]@@ -1783,10 +1977,12 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:"top_k": 4,"score_threshold_enabled": False,}- get_binding.assert_called_once_with("default-provider", "default-embedding", session)+ get_binding.assert_called_once_with("default-provider", "default-embedding", sqlite_session)- def test_save_document_with_dataset_id_uses_explicit_embedding_and_retrieval_model(self, account_context):- dataset = _make_dataset(indexing_technique=None)+ def test_save_document_with_dataset_id_uses_explicit_embedding_and_retrieval_model(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row(indexing_technique=None)knowledge_config = _make_upload_knowledge_config(original_document_id="doc-1", file_ids=["file-1"])knowledge_config.indexing_technique = "high_quality"knowledge_config.embedding_model = "explicit-model"@@ -1802,28 +1998,38 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:score_threshold_enabled=True,score_threshold=0.3,)+ binding = DatasetCollectionBinding(+ provider_name="explicit-provider",+ model_name="explicit-model",+ type="dataset",+ collection_name="collection",+ )+ binding.id = "binding-2"+ updated_document = _document_row(document_id="doc-1")with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.ModelManager") as model_manager_cls,patch("services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",- return_value=SimpleNamespace(id="binding-2"),+ return_value=binding,) as get_binding,- patch.object(DocumentService, "update_document_with_dataset_id", return_value=_make_document()),+ patch.object(DocumentService, "update_document_with_dataset_id", return_value=updated_document),):- session = MagicMock()- DocumentService.save_document_with_dataset_id(dataset, knowledge_config, account_context, session=session)+ DocumentService.save_document_with_dataset_id(+ dataset, knowledge_config, account_context, session=sqlite_session+ )model_manager_cls.for_tenant.return_value.get_default_model_instance.assert_not_called()- get_binding.assert_called_once_with("explicit-provider", "explicit-model", session)+ get_binding.assert_called_once_with("explicit-provider", "explicit-model", sqlite_session)assert dataset.embedding_model == "explicit-model"assert dataset.embedding_model_provider == "explicit-provider"assert dataset.retrieval_model == knowledge_config.retrieval_model.model_dump()- def test_save_document_with_dataset_id_creates_custom_process_rule_for_new_upload_document(self, account_context):- session = MagicMock()- dataset = _make_dataset()+ def test_save_document_with_dataset_id_creates_custom_process_rule_for_new_upload_document(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row(data_source_type=DataSourceType.UPLOAD_FILE)knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"],process_rule=ProcessRule(@@ -1834,148 +2040,126 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:),),)- created_process_rule = SimpleNamespace(id="rule-custom")- created_document = _make_document(document_id="doc-created", name="file.txt")+ upload_file = _upload_file(file_id="file-1", name="file.txt")+ sqlite_session.add_all([dataset, upload_file])+ sqlite_session.commit()with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.DatasetProcessRule") as process_rule_cls,- patch.object(DocumentService, "get_documents_position", return_value=3),- patch.object(DocumentService, "build_document", return_value=created_document),patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,patch("services.dataset_service.time.strftime", return_value="20260101010101"),patch("services.dataset_service.secrets.randbelow", return_value=23),):mock_redis.lock.return_value = _make_lock_context()- process_rule_cls.return_value = created_process_rule- session.scalars.return_value.all.side_effect = [[SimpleNamespace(id="file-1", name="file.txt")], []]-documents, batch = DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=sqlite_session,)- assert documents == [created_document]+ assert len(documents) == 1+ created_document = documents[0]+ assert created_document.name == "file.txt"assert batch == "20260101010101100023"- assert process_rule_cls.call_args.kwargs == {- "dataset_id": "dataset-1",- "mode": "custom",- "rules": knowledge_config.process_rule.rules.model_dump_json(),- "created_by": "user-1",- }- document_proxy_cls.assert_called_once_with("tenant-1", "dataset-1", ["doc-created"])+ created_rule = sqlite_session.get(DatasetProcessRule, created_document.dataset_process_rule_id)+ assert created_rule is not None+ assert created_rule.mode == "custom"+ assert created_rule.rules == knowledge_config.process_rule.rules.model_dump_json()+ document_proxy_cls.assert_called_once_with("tenant-1", "dataset-1", [created_document.id])document_proxy_cls.return_value.delay.assert_called_once()def test_save_document_with_dataset_id_creates_automatic_process_rule_for_new_upload_document(- self, account_context+ self, account_context, sqlite_session: Session):- session = MagicMock()- dataset = _make_dataset()+ dataset = _dataset_row(data_source_type=DataSourceType.UPLOAD_FILE)knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"],process_rule=ProcessRule(mode="automatic"),)- created_process_rule = SimpleNamespace(id="rule-auto")- created_document = _make_document(document_id="doc-created", name="file.txt")+ sqlite_session.add_all([dataset, _upload_file(file_id="file-1", name="file.txt")])+ sqlite_session.commit()with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.DatasetProcessRule") as process_rule_cls,- patch.object(DocumentService, "get_documents_position", return_value=1),- patch.object(DocumentService, "build_document", return_value=created_document),patch("services.dataset_service.DocumentIndexingTaskProxy"),patch("services.dataset_service.time.strftime", return_value="20260101010101"),patch("services.dataset_service.secrets.randbelow", return_value=23),):mock_redis.lock.return_value = _make_lock_context()- process_rule_cls.AUTOMATIC_RULES = DatasetProcessRule.AUTOMATIC_RULES- process_rule_cls.return_value = created_process_rule- session.scalars.return_value.all.side_effect = [[SimpleNamespace(id="file-1", name="file.txt")], []]-- DocumentService.save_document_with_dataset_id(+ documents, _ = DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=sqlite_session,)- assert process_rule_cls.call_args.kwargs == {- "dataset_id": "dataset-1",- "mode": "automatic",- "rules": json.dumps(DatasetProcessRule.AUTOMATIC_RULES),- "created_by": "user-1",- }- assert session.flush.call_count >= 2+ created_rule = sqlite_session.get(DatasetProcessRule, documents[0].dataset_process_rule_id)+ assert created_rule is not None+ assert created_rule.mode == "automatic"+ assert created_rule.rules == json.dumps(DatasetProcessRule.AUTOMATIC_RULES)+ assert sqlite_session.get(Document, documents[0].id) is documents[0]def test_save_document_with_dataset_id_creates_fallback_automatic_process_rule_when_latest_is_missing(- self, account_context+ self, account_context, sqlite_session: Session):- session = MagicMock()- dataset = _make_dataset()+ dataset = _dataset_row(data_source_type=DataSourceType.UPLOAD_FILE)knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"], process_rule=None)- created_process_rule = SimpleNamespace(id="rule-fallback")- created_document = _make_document(document_id="doc-created", name="file.txt")+ sqlite_session.add_all([dataset, _upload_file(file_id="file-1", name="file.txt")])+ sqlite_session.commit()with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.redis_client") as mock_redis,- patch("services.dataset_service.DatasetProcessRule") as process_rule_cls,- patch.object(DocumentService, "get_documents_position", return_value=1),- patch.object(DocumentService, "build_document", return_value=created_document),patch("services.dataset_service.DocumentIndexingTaskProxy"),patch("services.dataset_service.time.strftime", return_value="20260101010101"),patch("services.dataset_service.secrets.randbelow", return_value=23),):mock_redis.lock.return_value = _make_lock_context()- process_rule_cls.AUTOMATIC_RULES = DatasetProcessRule.AUTOMATIC_RULES- process_rule_cls.return_value = created_process_rule- session.scalar.return_value = None- session.scalars.return_value.all.side_effect = [[SimpleNamespace(id="file-1", name="file.txt")], []]-- DocumentService.save_document_with_dataset_id(+ documents, _ = DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=sqlite_session,)- session.scalar.assert_called_once()- assert process_rule_cls.call_args.kwargs == {- "dataset_id": "dataset-1",- "mode": "automatic",- "rules": json.dumps(DatasetProcessRule.AUTOMATIC_RULES),- "created_by": "user-1",- }+ created_rule = sqlite_session.get(DatasetProcessRule, documents[0].dataset_process_rule_id)+ assert created_rule is not None+ assert created_rule.mode == "automatic"+ assert created_rule.rules == json.dumps(DatasetProcessRule.AUTOMATIC_RULES)- def test_save_document_with_dataset_id_raises_when_upload_file_lookup_is_incomplete(self, account_context):- session = MagicMock()- dataset = _make_dataset()+ def test_save_document_with_dataset_id_raises_when_upload_file_lookup_is_incomplete(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row(data_source_type=DataSourceType.UPLOAD_FILE)knowledge_config = _make_upload_knowledge_config(file_ids=["file-1", "file-2"])+ sqlite_session.add_all([dataset, _upload_file(file_id="file-1", name="file.txt")])+ sqlite_session.commit()with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.redis_client") as mock_redis,- patch.object(DocumentService, "get_documents_position", return_value=1),patch("services.dataset_service.time.strftime", return_value="20260101010101"),patch("services.dataset_service.secrets.randbelow", return_value=23),):mock_redis.lock.return_value = _make_lock_context()- session.scalars.return_value.all.return_value = [SimpleNamespace(id="file-1", name="file.txt")]-with pytest.raises(FileNotExistsError, match="One or more files not found"):DocumentService.save_document_with_dataset_id(dataset,knowledge_config,account_context,- session=session,+ session=sqlite_session,)- def test_save_document_with_dataset_id_requires_notion_info_list_for_notion_import(self, account_context):- dataset = _make_dataset()+ def test_save_document_with_dataset_id_requires_notion_info_list_for_notion_import(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row(data_source_type=DataSourceType.NOTION_IMPORT)+ process_rule = _process_rule()+ sqlite_session.add_all([dataset, process_rule])+ sqlite_session.commit()knowledge_config = KnowledgeConfig(indexing_technique="economy",data_source=DataSource(info_list=InfoList(data_source_type="notion_import")),@@ -1986,7 +2170,6 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.redis_client") as mock_redis,- patch.object(DocumentService, "get_documents_position", return_value=1),):mock_redis.lock.return_value = _make_lock_context()with pytest.raises(ValueError, match="No notion info list found"):@@ -1994,12 +2177,17 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:dataset,knowledge_config,account_context,- dataset_process_rule=SimpleNamespace(id="rule-1"),- session=MagicMock(),+ dataset_process_rule=process_rule,+ session=sqlite_session,)- def test_save_document_with_dataset_id_requires_website_info_list_for_website_crawl(self, account_context):- dataset = _make_dataset()+ def test_save_document_with_dataset_id_requires_website_info_list_for_website_crawl(+ self, account_context, sqlite_session: Session+ ):+ dataset = _dataset_row(data_source_type=DataSourceType.WEBSITE_CRAWL)+ process_rule = _process_rule()+ sqlite_session.add_all([dataset, process_rule])+ sqlite_session.commit()knowledge_config = KnowledgeConfig(indexing_technique="economy",data_source=DataSource(info_list=InfoList(data_source_type="website_crawl")),@@ -2010,7 +2198,6 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:with (patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),patch("services.dataset_service.redis_client") as mock_redis,- patch.object(DocumentService, "get_documents_position", return_value=1),):mock_redis.lock.return_value = _make_lock_context()with pytest.raises(ValueError, match="No website info list found"):@@ -2018,6 +2205,6 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:dataset,knowledge_config,account_context,- dataset_process_rule=SimpleNamespace(id="rule-1"),- session=MagicMock(),+ dataset_process_rule=process_rule,+ session=sqlite_session,)