Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: migrate document service sessions and ORM models to SQLite (#40536)

Asuka Minatoc2c187648408
diff --git a/api/tests/unit_tests/services/test_dataset_service_document.py b/api/tests/unit_tests/services/test_dataset_service_document.py
index 7093053d1cb..1763cfab7b9 100644
--- a/api/tests/unit_tests/services/test_dataset_service_document.py
+++ b/api/tests/unit_tests/services/test_dataset_service_document.py
@@ -1,5 +1,15 @@
"""Unit tests for DocumentService behaviors in dataset_service."""
+from datetime import datetime
+
+from sqlalchemy import event, select
+from sqlalchemy.orm import Session
+
+from models.account import Tenant
+from models.dataset import Dataset, DatasetCollectionBinding, DocumentSegment
+from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus
+from models.model import UploadFile
+from models.source import DataSourceOauthBinding
from services.dataset_ref_service import DatasetRefService
from .dataset_service_test_helpers import (
@@ -8,7 +18,6 @@ from .dataset_service_test_helpers import (
CloudPlan,
DatasetProcessRule,
DatasetService,
- DatasetServiceUnitDataFactory,
DataSource,
Document,
DocumentIndexingError,
@@ -18,7 +27,6 @@ from .dataset_service_test_helpers import (
IndexStructureType,
InfoList,
KnowledgeConfig,
- MagicMock,
NotFound,
NotionIcon,
NotionInfo,
@@ -32,18 +40,117 @@ from .dataset_service_test_helpers import (
Segmentation,
SimpleNamespace,
WebsiteInfo,
- _make_dataset,
- _make_document,
_make_features,
_make_lock_context,
_make_upload_knowledge_config,
- create_autospec,
json,
patch,
pytest,
)
+def _account(*, account_id: str = "user-1", tenant_id: str = "tenant-1") -> Account:
+ account = Account(name="User", email=f"{account_id}@example.com")
+ account.id = account_id
+ tenant = Tenant(name="Tenant")
+ tenant.id = tenant_id
+ account._current_tenant = tenant
+ return account
+
+
+def _dataset_row(
+ *,
+ dataset_id: str = "dataset-1",
+ tenant_id: str = "tenant-1",
+ built_in_field_enabled: bool = False,
+ data_source_type: str | None = None,
+ indexing_technique: str | None = "economy",
+) -> Dataset:
+ return Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ name="Dataset",
+ description="",
+ provider="vendor",
+ created_by="user-1",
+ maintainer="user-1",
+ built_in_field_enabled=built_in_field_enabled,
+ chunk_structure=IndexStructureType.PARAGRAPH_INDEX,
+ data_source_type=data_source_type,
+ indexing_technique=indexing_technique,
+ )
+
+
+def _document_row(
+ *,
+ document_id: str = "document-1",
+ dataset_id: str = "dataset-1",
+ tenant_id: str = "tenant-1",
+ name: str = "Document",
+ indexing_status: str = IndexingStatus.COMPLETED,
+ data_source_type: str = DataSourceType.UPLOAD_FILE,
+ data_source_info: str = "{}",
+ enabled: bool = True,
+ archived: bool = False,
+ is_paused: bool = False,
+) -> Document:
+ return Document(
+ id=document_id,
+ tenant_id=tenant_id,
+ dataset_id=dataset_id,
+ position=1,
+ data_source_type=data_source_type,
+ data_source_info=data_source_info,
+ batch="batch-1",
+ name=name,
+ created_from=DocumentCreatedFrom.API,
+ created_by="user-1",
+ created_at=datetime(2026, 1, 1),
+ updated_at=datetime(2026, 1, 2),
+ indexing_status=indexing_status,
+ doc_form=IndexStructureType.PARAGRAPH_INDEX,
+ word_count=10,
+ enabled=enabled,
+ archived=archived,
+ is_paused=is_paused,
+ completed_at=datetime(2026, 1, 2) if indexing_status == IndexingStatus.COMPLETED else None,
+ )
+
+
+def _upload_file(*, file_id: str, tenant_id: str = "tenant-1", name: str = "upload.txt") -> UploadFile:
+ upload_file = UploadFile(
+ tenant_id=tenant_id,
+ storage_type="opendal",
+ key=f"key-{file_id}",
+ name=name,
+ size=1,
+ extension="txt",
+ mime_type="text/plain",
+ created_by_role="account",
+ created_by="user-1",
+ created_at=datetime(2026, 1, 1),
+ used=False,
+ )
+ upload_file.id = file_id
+ return upload_file
+
+
+def _process_rule(
+ *, dataset_id: str = "dataset-1", rule_id: str = "rule-1", mode: str = "automatic"
+) -> DatasetProcessRule:
+ rules = (
+ json.dumps(DatasetProcessRule.AUTOMATIC_RULES)
+ if mode == "automatic"
+ else Rule(
+ pre_processing_rules=[PreProcessingRule(id="remove_extra_spaces", enabled=True)],
+ segmentation=Segmentation(separator="\n", max_tokens=100),
+ ).model_dump_json()
+ )
+ process_rule = DatasetProcessRule(dataset_id=dataset_id, mode=mode, rules=rules, created_by="user-1")
+ process_rule.id = rule_id
+ return process_rule
+
+
class _RetryFlagLock:
def __init__(self, store: "_RetryFlagStore", key: str):
self.store = store
@@ -102,183 +209,212 @@ class TestDocumentServiceDisplayStatus:
assert DocumentService.build_display_status_filters("missing") == ()
def test_apply_display_status_filter_returns_original_query_for_unknown_status(self):
- query = MagicMock()
+ query = select(Document)
result = DocumentService.apply_display_status_filter(query, "missing")
assert result is query
- query.where.assert_not_called()
def test_apply_display_status_filter_applies_where_for_known_status(self):
- query = MagicMock()
- filtered_query = MagicMock()
- query.where.return_value = filtered_query
+ query = select(Document)
result = DocumentService.apply_display_status_filter(query, "enabled")
- assert result is filtered_query
- query.where.assert_called_once()
+ assert result is not query
+ assert "documents.enabled" in str(result)
class TestDocumentServiceRetrieval:
- def test_get_document_by_id_uses_provided_session(self):
- session = MagicMock()
- expected_document = DatasetServiceUnitDataFactory.create_document_mock(document_id="document-1")
- session.get.return_value = expected_document
+ def test_get_document_by_id_uses_provided_session(self, sqlite_session: Session):
+ document = _document_row()
+ sqlite_session.add(document)
+ sqlite_session.commit()
+
+ assert DocumentService.get_document_by_id(document.id, session=sqlite_session) is document
+ assert DocumentService.get_document_by_id("missing", session=sqlite_session) is None
+
+ def test_get_document_by_ids_enforces_dataset_owner_and_state(self, sqlite_session: Session):
+ dataset = _dataset_row()
+ expected = _document_row(document_id="expected")
+ sqlite_session.add_all(
+ [
+ dataset,
+ expected,
+ _document_row(document_id="disabled", enabled=False),
+ _document_row(document_id="archived", archived=True),
+ _document_row(document_id="waiting", indexing_status=IndexingStatus.WAITING),
+ _document_row(document_id="other-dataset", dataset_id="dataset-2"),
+ _document_row(document_id="other-tenant", tenant_id="tenant-2"),
+ ]
+ )
+ sqlite_session.commit()
- result = DocumentService.get_document_by_id("document-1", session=session)
+ documents = DocumentService.get_document_by_ids(
+ DatasetRefService.create_dataset_ref(dataset),
+ ["expected", "disabled", "archived", "waiting", "other-dataset", "other-tenant"],
+ sqlite_session,
+ )
- assert result is expected_document
- session.get.assert_called_once_with(Document, "document-1")
+ assert [document.id for document in documents] == [expected.id]
class TestDocumentServiceMutations:
"""Unit tests for DocumentService mutation and orchestration helpers."""
- @pytest.fixture
- def rename_account_context(self):
- class FakeAccount:
- pass
-
- current_user = FakeAccount()
- current_user.id = "user-123"
- current_user.current_tenant_id = "tenant-123"
-
- with (
- patch("services.dataset_service.Account", FakeAccount),
- patch("services.dataset_service.current_user", current_user),
- ):
- yield current_user
-
@pytest.mark.parametrize(("archived", "expected"), [(True, True), (False, False)])
def test_check_archived_returns_boolean_status(self, archived, expected):
- document = DatasetServiceUnitDataFactory.create_document_mock(archived=archived)
+ document = _document_row(archived=archived)
assert DocumentService.check_archived(document) is expected
- def test_delete_documents_limits_query_and_cleanup_to_dataset_ref(self):
- session = MagicMock()
- dataset = _make_dataset(
- dataset_id="dataset-1",
- tenant_id="tenant-1",
- doc_form=IndexStructureType.PARAGRAPH_INDEX,
+ def test_delete_documents_limits_query_and_cleanup_to_dataset_ref(self, sqlite_session: Session):
+ dataset = _dataset_row()
+ document = _document_row(
+ document_id="doc-1",
+ data_source_info=json.dumps({"upload_file_id": "file-1"}),
)
- document = _make_document(document_id="doc-1", dataset_id=dataset.id, tenant_id=dataset.tenant_id)
-
- with (
- patch("services.dataset_service.batch_clean_document_task") as clean_task,
- ):
- session.scalars.return_value.all.return_value = [document]
+ other_dataset = _document_row(document_id="other-dataset", dataset_id="dataset-2")
+ other_tenant = _document_row(document_id="other-tenant", tenant_id="tenant-2")
+ sqlite_session.add_all([dataset, document, other_dataset, other_tenant])
+ sqlite_session.commit()
+ with patch("services.dataset_service.batch_clean_document_task") as clean_task:
dataset_ref = DatasetRefService.create_dataset_ref(dataset)
DocumentService.delete_documents(
dataset_ref,
- ["doc-1", "other-doc"],
- dataset.doc_form,
- session,
+ [document.id, other_dataset.id, other_tenant.id],
+ IndexStructureType.PARAGRAPH_INDEX,
+ sqlite_session,
)
- stmt = session.scalars.call_args.args[0]
- compiled = stmt.compile()
- statement = str(compiled)
- assert "documents.id IN" in statement
- assert "documents.tenant_id" in statement
- assert "documents.dataset_id" in statement
- assert ["doc-1", "other-doc"] in compiled.params.values()
- assert dataset.tenant_id in compiled.params.values()
- assert dataset.id in compiled.params.values()
- session.delete.assert_called_once_with(document)
- session.commit.assert_called_once()
- clean_task.delay.assert_called_once_with(["doc-1"], dataset.id, dataset.doc_form, [])
-
- def test_rename_document_raises_when_dataset_is_missing(self, rename_account_context):
- session = MagicMock()
-
- with patch.object(DatasetService, "get_dataset", return_value=None):
- with pytest.raises(ValueError, match="Dataset not found"):
- DocumentService.rename_document("dataset-1", "doc-1", "New Name", session)
+ assert sqlite_session.get(Document, document.id) is None
+ assert sqlite_session.get(Document, other_dataset.id) is other_dataset
+ assert sqlite_session.get(Document, other_tenant.id) is other_tenant
+ clean_task.delay.assert_called_once_with(
+ [document.id], dataset.id, IndexStructureType.PARAGRAPH_INDEX, ["file-1"]
+ )
- def test_rename_document_raises_when_document_is_missing(self, rename_account_context):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock()
- session = MagicMock()
+ def test_delete_documents_with_empty_ids_does_not_commit(self, sqlite_session: Session):
+ commits = 0
- with (
- patch.object(DatasetService, "get_dataset", return_value=dataset),
- patch.object(DocumentService, "get_document", return_value=None),
- ):
- with pytest.raises(ValueError, match="Document not found"):
- DocumentService.rename_document(dataset.id, "doc-1", "New Name", session)
+ def count_commit(_session):
+ nonlocal commits
+ commits += 1
- def test_rename_document_rejects_cross_tenant_access(self, rename_account_context):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock()
- document = DatasetServiceUnitDataFactory.create_document_mock(tenant_id="tenant-other")
- session = MagicMock()
+ event.listen(sqlite_session, "after_commit", count_commit)
+ DocumentService.delete_documents(
+ DatasetRefService.create_dataset_ref(_dataset_row()), [], IndexStructureType.PARAGRAPH_INDEX, sqlite_session
+ )
+ event.remove(sqlite_session, "after_commit", count_commit)
+ assert commits == 0
- with (
- patch.object(DatasetService, "get_dataset", return_value=dataset),
- patch.object(DocumentService, "get_document", return_value=document),
- ):
+ def test_rename_document_raises_when_dataset_is_missing(self, sqlite_session: Session):
+ with patch("services.dataset_service.current_user", _account()):
+ with pytest.raises(ValueError, match="Dataset not found"):
+ DocumentService.rename_document("dataset-1", "doc-1", "New Name", sqlite_session)
+
+ def test_rename_document_raises_when_document_is_missing(self, sqlite_session: Session):
+ dataset = _dataset_row()
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
+ with patch("services.dataset_service.current_user", _account()):
+ with pytest.raises(ValueError, match="Document not found"):
+ DocumentService.rename_document(dataset.id, "doc-1", "New Name", sqlite_session)
+
+ def test_rename_document_rejects_cross_tenant_access(self, sqlite_session: Session):
+ dataset = _dataset_row()
+ document = _document_row(tenant_id="tenant-other")
+ sqlite_session.add_all([dataset, document])
+ sqlite_session.commit()
+ with patch("services.dataset_service.current_user", _account()):
with pytest.raises(ValueError, match="No permission"):
- DocumentService.rename_document(dataset.id, document.id, "New Name", session)
+ DocumentService.rename_document(dataset.id, document.id, "New Name", sqlite_session)
- def test_rename_document_updates_document_metadata_and_upload_file_name(self, rename_account_context):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(
- built_in_field_enabled=True,
- tenant_id="tenant-1",
+ def test_rename_document_updates_document_metadata_and_upload_file_name(self, sqlite_session: Session):
+ dataset = _dataset_row(built_in_field_enabled=True)
+ document = _document_row(
+ data_source_info=json.dumps({"upload_file_id": "file-1"}),
)
- document = DatasetServiceUnitDataFactory.create_document_mock(
- tenant_id="tenant-1",
- doc_metadata={"title": "Old"},
- data_source_info_dict={"upload_file_id": "file-1"},
+ document.doc_metadata = {BuiltInField.document_name: "Old"}
+ upload_file = UploadFile(
+ tenant_id=dataset.tenant_id,
+ storage_type="opendal",
+ key="key",
+ name="old.txt",
+ size=1,
+ extension="txt",
+ mime_type="text/plain",
+ created_by_role="account",
+ created_by="user-1",
+ created_at=datetime(2026, 1, 1),
+ used=False,
)
- rename_account_context.current_tenant_id = "tenant-1"
+ upload_file.id = "file-1"
+ sqlite_session.add_all([dataset, document, upload_file])
+ sqlite_session.commit()
+ commits = 0
- with (
- patch.object(DatasetService, "get_dataset", return_value=dataset),
- patch.object(DocumentService, "get_document", return_value=document),
- ):
- result = DocumentService.rename_document(dataset.id, document.id, "New Name", session)
+ def count_commit(_session):
+ nonlocal commits
+ commits += 1
+
+ event.listen(sqlite_session, "after_commit", count_commit)
+ with patch("services.dataset_service.current_user", _account()):
+ result = DocumentService.rename_document(dataset.id, document.id, "New Name", sqlite_session)
+ event.remove(sqlite_session, "after_commit", count_commit)
assert result is document
assert document.name == "New Name"
assert document.doc_metadata[BuiltInField.document_name] == "New Name"
- session.add.assert_called_once_with(document)
- session.execute.assert_called()
- session.flush.assert_called_once()
-
- def test_recover_document_raises_when_document_is_not_paused(self):
- document = DatasetServiceUnitDataFactory.create_document_mock(is_paused=False)
- session = MagicMock()
+ assert sqlite_session.get(UploadFile, upload_file.id).name == "New Name"
+ assert commits == 0
+ def test_recover_document_raises_when_document_is_not_paused(self, unbound_session: Session):
+ document = _document_row(is_paused=False)
with pytest.raises(DocumentIndexingError):
- DocumentService.recover_document(document, session)
-
- def test_retry_document_raises_when_retry_flag_is_already_set(self, rename_account_context):
- document = DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-1")
- session = MagicMock()
-
- with patch("services.dataset_service.redis_client") as mock_redis:
- mock_redis.lock.return_value.acquire.return_value = False
+ DocumentService.recover_document(document, unbound_session)
+ def test_recover_document_persists_and_dispatches(self, sqlite_session: Session):
+ document = _document_row(is_paused=True)
+ sqlite_session.add(document)
+ sqlite_session.commit()
+ with (
+ patch("services.dataset_service.redis_client") as redis,
+ patch("services.dataset_service.recover_document_indexing_task") as task,
+ ):
+ DocumentService.recover_document(document, sqlite_session)
+
+ sqlite_session.expire_all()
+ recovered = sqlite_session.get(Document, document.id)
+ assert recovered is not None
+ assert recovered.is_paused is False
+ redis.delete.assert_called_once_with(f"document_{document.id}_is_paused")
+ task.delay.assert_called_once_with(document.dataset_id, document.id)
+
+ def test_retry_document_raises_when_retry_flag_is_already_set(self, sqlite_session: Session):
+ document = _document_row(indexing_status=IndexingStatus.ERROR)
+ sqlite_session.add(document)
+ sqlite_session.commit()
+ retry_flags = _RetryFlagStore({f"document_{document.id}_is_retried": "other-request"})
+ with (
+ patch("services.dataset_service.current_user", _account()),
+ patch("services.dataset_service.redis_client", retry_flags),
+ ):
with pytest.raises(ValueError, match="being retried"):
- DocumentService.retry_document("dataset-1", [document], session)
+ DocumentService.retry_document("dataset-1", [document], sqlite_session)
def test_retry_document_leaves_batch_unchanged_when_later_document_is_already_being_retried(
- self, rename_account_context
+ self, sqlite_session: Session
):
- first_document = DatasetServiceUnitDataFactory.create_document_mock(
- document_id="doc-1", indexing_status="error"
- )
- second_document = DatasetServiceUnitDataFactory.create_document_mock(
- document_id="doc-2", indexing_status="error"
- )
+ first_document = _document_row(document_id="doc-1", indexing_status=IndexingStatus.ERROR)
+ second_document = _document_row(document_id="doc-2", indexing_status=IndexingStatus.ERROR)
+ sqlite_session.add_all([first_document, second_document])
+ sqlite_session.commit()
first_retry_key = "document_doc-1_is_retried"
second_retry_key = "document_doc-2_is_retried"
retry_flags = _RetryFlagStore({second_retry_key: "other-request"})
- session = MagicMock()
-
with (
+ patch("services.dataset_service.current_user", _account()),
patch("services.dataset_service.redis_client", retry_flags),
patch("services.dataset_service.retry_document_indexing_task") as retry_task,
):
@@ -286,16 +422,16 @@ class TestDocumentServiceMutations:
DocumentService.retry_document(
"dataset-1",
[first_document, second_document],
- session,
+ sqlite_session,
)
- assert first_document.indexing_status == "error"
- assert second_document.indexing_status == "error"
+ assert first_document.indexing_status == IndexingStatus.ERROR
+ assert second_document.indexing_status == IndexingStatus.ERROR
assert first_retry_key not in retry_flags.values
assert retry_flags.values[second_retry_key] == "other-request"
retry_task.delay.assert_not_called()
- def test_retry_document_does_not_release_a_retry_flag_reacquired_by_another_request(self, rename_account_context):
+ def test_retry_document_does_not_release_a_retry_flag_reacquired_by_another_request(self, sqlite_session: Session):
first_retry_key = "document_doc-1_is_retried"
second_retry_key = "document_doc-2_is_retried"
retry_flags = _RetryFlagStore(
@@ -303,65 +439,93 @@ class TestDocumentServiceMutations:
replacement_on_conflict=(first_retry_key, "new-owner"),
)
documents = [
- DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-1", indexing_status="error"),
- DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-2", indexing_status="error"),
+ _document_row(document_id="doc-1", indexing_status=IndexingStatus.ERROR),
+ _document_row(document_id="doc-2", indexing_status=IndexingStatus.ERROR),
]
+ sqlite_session.add_all(documents)
+ sqlite_session.commit()
- with patch("services.dataset_service.redis_client", retry_flags):
+ with (
+ patch("services.dataset_service.current_user", _account()),
+ patch("services.dataset_service.redis_client", retry_flags),
+ ):
with pytest.raises(ValueError, match="being retried"):
- DocumentService.retry_document("dataset-1", documents, MagicMock())
+ DocumentService.retry_document("dataset-1", documents, sqlite_session)
assert retry_flags.values[first_retry_key] == "new-owner"
assert retry_flags.values[second_retry_key] == "other-request"
- def test_retry_document_releases_flags_when_status_commit_fails(self, rename_account_context):
+ def test_retry_document_releases_flags_when_status_commit_fails(self, sqlite_session: Session):
retry_flags = _RetryFlagStore()
- document = DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-1", indexing_status="error")
- session = MagicMock()
- session.commit.side_effect = RuntimeError("database unavailable")
+ document = _document_row(indexing_status=IndexingStatus.ERROR)
+ sqlite_session.add(document)
+ sqlite_session.commit()
+
+ def fail_commit(_session):
+ raise RuntimeError("database unavailable")
+ event.listen(sqlite_session, "before_commit", fail_commit)
with (
+ patch("services.dataset_service.current_user", _account()),
patch("services.dataset_service.redis_client", retry_flags),
patch("services.dataset_service.retry_document_indexing_task") as retry_task,
):
with pytest.raises(RuntimeError, match="database unavailable"):
- DocumentService.retry_document("dataset-1", [document], session)
+ DocumentService.retry_document("dataset-1", [document], sqlite_session)
+ event.remove(sqlite_session, "before_commit", fail_commit)
assert retry_flags.values == {}
- session.rollback.assert_called_once_with()
retry_task.delay.assert_not_called()
- def test_sync_website_document_raises_when_sync_flag_exists(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- document = DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-1", dataset_id=dataset.id)
- session = MagicMock()
+ def test_retry_document_persists_status_and_dispatches(self, sqlite_session: Session):
+ documents = [
+ _document_row(document_id="doc-1", indexing_status=IndexingStatus.ERROR),
+ _document_row(document_id="doc-2", indexing_status=IndexingStatus.PAUSED),
+ ]
+ sqlite_session.add_all(documents)
+ sqlite_session.commit()
+ retry_flags = _RetryFlagStore()
+ with (
+ patch("services.dataset_service.current_user", _account()),
+ patch("services.dataset_service.redis_client", retry_flags),
+ patch("services.dataset_service.retry_document_indexing_task") as task,
+ ):
+ DocumentService.retry_document("dataset-1", documents, sqlite_session)
+
+ sqlite_session.expire_all()
+ statuses = sqlite_session.scalars(select(Document.indexing_status).order_by(Document.id)).all()
+ assert statuses == [IndexingStatus.WAITING, IndexingStatus.WAITING]
+ task.delay.assert_called_once_with("dataset-1", ["doc-1", "doc-2"], "user-1")
+ def test_sync_website_document_raises_when_sync_flag_exists(self, sqlite_session: Session):
+ dataset = _dataset_row()
+ document = _document_row()
with patch("services.dataset_service.redis_client") as mock_redis:
mock_redis.get.return_value = "1"
with pytest.raises(ValueError, match="being synced"):
- DocumentService.sync_website_document(dataset, document, session)
+ DocumentService.sync_website_document(dataset, document, sqlite_session)
- def test_sync_website_document_rejects_document_outside_dataset(self):
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- document = DatasetServiceUnitDataFactory.create_document_mock(document_id="doc-1", dataset_id="dataset-2")
+ def test_sync_website_document_rejects_document_outside_dataset(self, sqlite_session: Session):
+ dataset = _dataset_row()
+ document = _document_row(dataset_id="dataset-2")
with (
pytest.raises(ValueError, match="Document not found"),
patch("services.dataset_service.redis_client") as mock_redis,
):
- DocumentService.sync_website_document(dataset, document, MagicMock())
+ DocumentService.sync_website_document(dataset, document, sqlite_session)
mock_redis.get.assert_not_called()
- def test_sync_website_document_updates_status_sets_cache_and_dispatches_task(self):
- session = MagicMock()
- dataset = DatasetServiceUnitDataFactory.create_dataset_mock(dataset_id="dataset-1")
- document = DatasetServiceUnitDataFactory.create_document_mock(
- document_id="doc-1",
- dataset_id=dataset.id,
- data_source_info_dict={"mode": "crawl"},
+ def test_sync_website_document_updates_status_sets_cache_and_dispatches_task(self, sqlite_session: Session):
+ dataset = _dataset_row()
+ document = _document_row(
+ data_source_type=DataSourceType.WEBSITE_CRAWL,
+ data_source_info=json.dumps({"mode": "crawl"}),
)
+ sqlite_session.add_all([dataset, document])
+ sqlite_session.commit()
with (
patch("services.dataset_service.redis_client") as mock_redis,
@@ -369,13 +533,14 @@ class TestDocumentServiceMutations:
):
mock_redis.get.return_value = None
- DocumentService.sync_website_document(dataset, document, session)
+ DocumentService.sync_website_document(dataset, document, sqlite_session)
- assert document.indexing_status == "waiting"
- assert '"mode": "scrape"' in document.data_source_info
- session.add.assert_called_once_with(document)
- session.commit.assert_called_once()
- mock_redis.setex.assert_called_once_with("document_doc-1_is_sync", 600, 1)
+ sqlite_session.expire_all()
+ synced = sqlite_session.get(Document, document.id)
+ assert synced is not None
+ assert synced.indexing_status == IndexingStatus.WAITING
+ assert synced.data_source_info_dict["mode"] == "scrape"
+ mock_redis.setex.assert_called_once_with(f"document_{document.id}_is_sync", 600, 1)
sync_task.delay.assert_called_once_with(dataset.id, document.id)
@@ -384,17 +549,14 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:
@pytest.fixture
def account_context(self):
- account = create_autospec(Account, instance=True)
- account.id = "user-1"
- account.current_tenant_id = "tenant-1"
+ account = _account()
with patch("services.dataset_service.current_user", account):
yield account
def test_save_document_without_dataset_id_creates_high_quality_dataset_with_default_retrieval_model(
- self, account_context
+ self, account_context, sqlite_session: Session
):
- session = MagicMock()
knowledge_config = KnowledgeConfig(
indexing_technique="high_quality",
data_source=DataSource(
@@ -408,24 +570,21 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:
summary_index_setting={"enable": True},
is_multimodal=True,
)
- created_dataset = SimpleNamespace(
- id="dataset-1",
- tenant_id="tenant-1",
- name="",
- description=None,
+ binding = DatasetCollectionBinding(
+ provider_name="provider",
+ model_name="embedding-model",
+ type="dataset",
+ collection_name="collection",
)
- first_document = SimpleNamespace(name="VeryLongDocumentNameForDataset.txt")
+ binding.id = "binding-1"
+ first_document = _document_row(name="VeryLongDocumentNameForDataset.txt")
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch(
"services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",
- return_value=SimpleNamespace(id="binding-1"),
+ return_value=binding,
),
- patch(
- "services.dataset_service.Dataset",
- side_effect=lambda **kwargs: created_dataset.__dict__.update(kwargs) or created_dataset,
- ) as dataset_cls,
patch.object(
DocumentService, "save_document_with_dataset_id", return_value=([first_document], "batch-1")
) as save_document,
@@ -434,33 +593,32 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:
tenant_id="tenant-1",
knowledge_config=knowledge_config,
account=account_context,
- session=session,
+ session=sqlite_session,
)
- assert dataset is created_dataset
assert documents == [first_document]
assert batch == "batch-1"
- assert created_dataset.collection_binding_id == "binding-1"
- assert created_dataset.retrieval_model["search_method"] == RetrievalMethod.SEMANTIC_SEARCH
- assert created_dataset.retrieval_model["top_k"] == 4
- assert created_dataset.summary_index_setting == {"enable": True}
- assert created_dataset.is_multimodal is True
- assert created_dataset.name == first_document.name[:18] + "..."
+ assert dataset.collection_binding_id == "binding-1"
+ assert dataset.retrieval_model["search_method"] == RetrievalMethod.SEMANTIC_SEARCH
+ assert dataset.retrieval_model["top_k"] == 4
+ assert dataset.summary_index_setting == {"enable": True}
+ assert dataset.is_multimodal is True
+ assert dataset.name == first_document.name[:18] + "..."
assert (
- created_dataset.description
+ dataset.description
== "useful for when you want to answer queries about the VeryLongDocumentNameForDataset.txt"
)
- dataset_cls.assert_called_once()
+ assert sqlite_session.get(Dataset, dataset.id) is dataset
save_document.assert_called_once_with(
- created_dataset,
+ dataset,
knowledge_config,
account_context,
- session=session,
+ session=sqlite_session,
)
- assert session.flush.call_count == 2
- def test_save_document_without_dataset_id_uses_provided_retrieval_model(self, account_context):
- session = MagicMock()
+ def test_save_document_without_dataset_id_uses_provided_retrieval_model(
+ self, account_context, sqlite_session: Session
+ ):
retrieval_model = RetrievalModel(
search_method=RetrievalMethod.SEMANTIC_SEARCH,
reranking_enable=True,
@@ -482,31 +640,30 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:
),
retrieval_model=retrieval_model,
)
- created_dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1", name="", description=None)
+ first_document = _document_row(name="Doc")
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
- patch(
- "services.dataset_service.Dataset",
- side_effect=lambda **kwargs: created_dataset.__dict__.update(kwargs) or created_dataset,
- ),
patch.object(
DocumentService,
"save_document_with_dataset_id",
- return_value=([SimpleNamespace(name="Doc")], "batch-1"),
+ return_value=([first_document], "batch-1"),
),
):
- DocumentService.save_document_without_dataset_id(
+ dataset, _, _ = DocumentService.save_document_without_dataset_id(
"tenant-1",
knowledge_config,
account_context,
- session,
+ sqlite_session,
)
- assert created_dataset.retrieval_model == retrieval_model.model_dump()
- assert created_dataset.collection_binding_id is None
+ assert dataset.retrieval_model == retrieval_model.model_dump()
+ assert dataset.collection_binding_id is None
+ assert sqlite_session.get(Dataset, dataset.id) is dataset
- def test_save_document_without_dataset_id_rejects_sandbox_batch_upload(self, account_context):
+ def test_save_document_without_dataset_id_rejects_sandbox_batch_upload(
+ self, account_context, unbound_session: Session
+ ):
knowledge_config = KnowledgeConfig(
indexing_technique="economy",
data_source=DataSource(
@@ -524,9 +681,10 @@ class TestDocumentServiceSaveDocumentWithoutDatasetId:
),
patch.object(DocumentService, "check_documents_upload_quota") as check_quota,
):
- session = MagicMock()
with pytest.raises(ValueError, match="does not support batch upload"):
- DocumentService.save_document_without_dataset_id("tenant-1", knowledge_config, account_context, session)
+ DocumentService.save_document_without_dataset_id(
+ "tenant-1", knowledge_config, account_context, unbound_session
+ )
check_quota.assert_not_called()
@@ -536,15 +694,15 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:
@pytest.fixture
def account_context(self):
- account = create_autospec(Account, instance=True)
- account.id = "user-1"
- account.current_tenant_id = "tenant-1"
+ account = _account()
with patch("services.dataset_service.current_user", account):
yield account
- def test_update_document_with_dataset_id_raises_when_document_is_missing(self, account_context):
- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")
+ def test_update_document_with_dataset_id_raises_when_document_is_missing(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row()
document_data = KnowledgeConfig(
original_document_id="doc-1",
indexing_technique="economy",
@@ -555,25 +713,24 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:
)
),
)
- session = MagicMock()
-
- with (
- patch.object(DocumentService, "get_document", return_value=None),
- patch.object(DatasetService, "check_dataset_model_setting") as check_model_setting,
- ):
+ with patch.object(DatasetService, "check_dataset_model_setting") as check_model_setting:
with pytest.raises(NotFound, match="Document not found"):
DocumentService.update_document_with_dataset_id(
dataset,
document_data,
account_context,
- session=session,
+ session=sqlite_session,
)
check_model_setting.assert_called_once_with(dataset)
- def test_update_document_with_dataset_id_rejects_non_available_documents(self, account_context):
- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")
- document = SimpleNamespace(display_status="indexing")
+ def test_update_document_with_dataset_id_rejects_non_available_documents(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row()
+ document = _document_row(document_id="doc-1", indexing_status=IndexingStatus.INDEXING)
+ sqlite_session.add_all([dataset, document])
+ sqlite_session.commit()
document_data = KnowledgeConfig(
original_document_id="doc-1",
indexing_technique="economy",
@@ -584,25 +741,46 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:
)
),
)
- session = MagicMock()
-
- with (
- patch.object(DocumentService, "get_document", return_value=document),
- patch.object(DatasetService, "check_dataset_model_setting"),
- ):
+ with patch.object(DatasetService, "check_dataset_model_setting"):
with pytest.raises(ValueError, match="Document is not available"):
DocumentService.update_document_with_dataset_id(
dataset,
document_data,
account_context,
- session=session,
+ session=sqlite_session,
)
- def test_update_document_with_dataset_id_upload_file_process_rule_and_name_override(self, account_context):
- session = MagicMock()
- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")
- document = _make_document()
- document.dataset_process_rule_id = "old-rule"
+ def test_update_document_with_dataset_id_upload_file_process_rule_and_name_override(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row()
+ document = _document_row(document_id="doc-1")
+ upload_file = UploadFile(
+ tenant_id=dataset.tenant_id,
+ storage_type="opendal",
+ key="key",
+ name="upload.txt",
+ size=1,
+ extension="txt",
+ mime_type="text/plain",
+ created_by_role="account",
+ created_by=account_context.id,
+ created_at=datetime(2026, 1, 1),
+ used=False,
+ )
+ upload_file.id = "file-1"
+ segment = DocumentSegment(
+ tenant_id=dataset.tenant_id,
+ dataset_id=dataset.id,
+ document_id=document.id,
+ position=1,
+ content="content",
+ word_count=1,
+ tokens=1,
+ created_by=account_context.id,
+ )
+ sqlite_session.add_all([dataset, document, upload_file, segment])
+ sqlite_session.commit()
document_data = KnowledgeConfig(
original_document_id="doc-1",
indexing_technique="economy",
@@ -622,26 +800,23 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:
name="Renamed document",
doc_form=IndexStructureType.QA_INDEX,
)
- created_process_rule = SimpleNamespace(id="rule-2")
+ updated_at = datetime(2026, 2, 1)
with (
- patch.object(DocumentService, "get_document", return_value=document),
patch.object(DatasetService, "check_dataset_model_setting"),
- patch("services.dataset_service.DatasetProcessRule", return_value=created_process_rule),
- patch("services.dataset_service.naive_utc_now", return_value="now"),
+ patch("services.dataset_service.naive_utc_now", return_value=updated_at),
patch("services.dataset_service.document_indexing_update_task") as update_task,
):
- session.scalar.return_value = SimpleNamespace(id="file-1", name="upload.txt")
-
result = DocumentService.update_document_with_dataset_id(
dataset,
document_data,
account_context,
- session=session,
+ session=sqlite_session,
)
assert result is document
- assert document.dataset_process_rule_id == "rule-2"
+ assert document.dataset_process_rule_id is not None
+ assert sqlite_session.get(DatasetProcessRule, document.dataset_process_rule_id) is not None
assert document.data_source_type == "upload_file"
assert document.data_source_info == '{"upload_file_id": "file-1"}'
assert document.name == "Renamed document"
@@ -651,17 +826,23 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:
assert document.parsing_completed_at is None
assert document.cleaning_completed_at is None
assert document.splitting_completed_at is None
- assert document.updated_at == "now"
+ assert document.updated_at == updated_at
assert document.created_from == "web"
assert document.doc_form == IndexStructureType.QA_INDEX
- assert session.commit.call_count == 3
- session.execute.assert_called()
+ sqlite_session.expire_all()
+ persisted = sqlite_session.get(Document, document.id)
+ assert persisted is not None
+ assert persisted.name == "Renamed document"
+ assert sqlite_session.get(DocumentSegment, segment.id).status == "re_segment"
update_task.delay.assert_called_once_with(document.dataset_id, document.id)
- def test_update_document_with_dataset_id_notion_import_requires_binding(self, account_context):
- session = MagicMock()
- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")
- document = SimpleNamespace(display_status="available", id="doc-1", dataset_id="dataset-1")
+ def test_update_document_with_dataset_id_notion_import_requires_binding(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row()
+ document = _document_row(document_id="doc-1")
+ sqlite_session.add_all([dataset, document])
+ sqlite_session.commit()
document_data = KnowledgeConfig(
original_document_id="doc-1",
indexing_technique="economy",
@@ -679,24 +860,32 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:
),
)
- with (
- patch.object(DocumentService, "get_document", return_value=document),
- patch.object(DatasetService, "check_dataset_model_setting"),
- ):
- session.scalar.return_value = None
-
+ with patch.object(DatasetService, "check_dataset_model_setting"):
with pytest.raises(ValueError, match="Data source binding not found"):
DocumentService.update_document_with_dataset_id(
dataset,
document_data,
account_context,
- session=session,
+ session=sqlite_session,
)
- def test_update_document_with_dataset_id_website_crawl_updates_segments_and_dispatches_task(self, account_context):
- session = MagicMock()
- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")
- document = _make_document()
+ def test_update_document_with_dataset_id_website_crawl_updates_segments_and_dispatches_task(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row()
+ document = _document_row(document_id="doc-1")
+ segment = DocumentSegment(
+ tenant_id=dataset.tenant_id,
+ dataset_id=dataset.id,
+ document_id=document.id,
+ position=1,
+ content="content",
+ word_count=1,
+ tokens=1,
+ created_by=account_context.id,
+ )
+ sqlite_session.add_all([dataset, document, segment])
+ sqlite_session.commit()
document_data = KnowledgeConfig(
original_document_id="doc-1",
indexing_technique="economy",
@@ -715,16 +904,15 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:
)
with (
- patch.object(DocumentService, "get_document", return_value=document),
patch.object(DatasetService, "check_dataset_model_setting"),
- patch("services.dataset_service.naive_utc_now", return_value="now"),
+ patch("services.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),
patch("services.dataset_service.document_indexing_update_task") as update_task,
):
result = DocumentService.update_document_with_dataset_id(
dataset,
document_data,
account_context,
- session=session,
+ session=sqlite_session,
)
assert result is document
@@ -735,7 +923,8 @@ class TestDocumentServiceUpdateDocumentWithDatasetId:
)
assert document.name == ""
assert document.doc_form == IndexStructureType.PARENT_CHILD_INDEX
- session.execute.assert_called()
+ sqlite_session.expire_all()
+ assert sqlite_session.get(DocumentSegment, segment.id).status == "re_segment"
update_task.delay.assert_called_once_with("dataset-1", "doc-1")
@@ -882,9 +1071,7 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
@pytest.fixture
def account_context(self):
- account = create_autospec(Account, instance=True)
- account.id = "user-1"
- account.current_tenant_id = "tenant-1"
+ account = _account()
with (
patch("services.dataset_service.current_user", account),
@@ -892,22 +1079,25 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
):
yield account
- def test_save_document_with_dataset_id_requires_file_info_for_upload_source(self, account_context):
- dataset = _make_dataset()
+ def test_save_document_with_dataset_id_requires_file_info_for_upload_source(
+ self, account_context, unbound_session: Session
+ ):
+ dataset = _dataset_row()
knowledge_config = _make_upload_knowledge_config(file_ids=None)
with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=True)):
- session = MagicMock()
with pytest.raises(ValueError, match="File source info is required"):
DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=unbound_session,
)
- def test_save_document_with_dataset_id_blocks_batch_upload_for_sandbox_plan(self, account_context):
- dataset = _make_dataset()
+ def test_save_document_with_dataset_id_blocks_batch_upload_for_sandbox_plan(
+ self, account_context, unbound_session: Session
+ ):
+ dataset = _dataset_row()
knowledge_config = _make_upload_knowledge_config(file_ids=["file-1", "file-2"])
with (
@@ -917,19 +1107,18 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
),
patch.object(DocumentService, "check_documents_upload_quota") as check_quota,
):
- session = MagicMock()
with pytest.raises(ValueError, match="does not support batch upload"):
DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=unbound_session,
)
check_quota.assert_not_called()
- def test_save_document_with_dataset_id_enforces_batch_upload_limit(self, account_context):
- dataset = _make_dataset()
+ def test_save_document_with_dataset_id_enforces_batch_upload_limit(self, account_context, unbound_session: Session):
+ dataset = _dataset_row()
knowledge_config = _make_upload_knowledge_config(file_ids=["file-1", "file-2"])
with (
@@ -937,21 +1126,23 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
patch("services.dataset_service.dify_config.BATCH_UPLOAD_LIMIT", 1),
patch.object(DocumentService, "check_documents_upload_quota") as check_quota,
):
- session = MagicMock()
with pytest.raises(ValueError, match="batch upload limit of 1"):
DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=unbound_session,
)
check_quota.assert_not_called()
- def test_save_document_with_dataset_id_updates_existing_document_and_data_source_type(self, account_context):
- dataset = _make_dataset(data_source_type=None)
+ def test_save_document_with_dataset_id_updates_existing_document_and_data_source_type(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row(data_source_type=None)
knowledge_config = _make_upload_knowledge_config(original_document_id="doc-1", file_ids=["file-1"])
- updated_document = _make_document(document_id="doc-1", batch="batch-existing")
+ updated_document = _document_row(document_id="doc-1")
+ updated_document.batch = "batch-existing"
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
@@ -959,55 +1150,57 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
DocumentService, "update_document_with_dataset_id", return_value=updated_document
) as update_document,
):
- session = MagicMock()
documents, batch = DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=sqlite_session,
)
assert dataset.data_source_type == "upload_file"
assert documents == [updated_document]
assert batch == "batch-existing"
- update_document.assert_called_once_with(dataset, knowledge_config, account_context, session=session)
+ update_document.assert_called_once_with(dataset, knowledge_config, account_context, session=sqlite_session)
- def test_save_document_with_dataset_id_requires_data_source_for_new_documents(self, account_context):
- dataset = _make_dataset()
+ def test_save_document_with_dataset_id_requires_data_source_for_new_documents(
+ self, account_context, unbound_session: Session
+ ):
+ dataset = _dataset_row()
knowledge_config = _make_upload_knowledge_config(data_source=None)
with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)):
- session = MagicMock()
with pytest.raises(ValueError, match="Data source is required when creating new documents"):
DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=unbound_session,
)
- def test_save_document_with_dataset_id_requires_existing_process_rule_for_custom_mode(self, account_context):
- dataset = _make_dataset()
+ def test_save_document_with_dataset_id_requires_existing_process_rule_for_custom_mode(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row()
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
knowledge_config = _make_upload_knowledge_config(
file_ids=["file-1"],
process_rule=ProcessRule(mode="custom"),
)
with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)):
- session = MagicMock()
- session.scalar.return_value = None
with pytest.raises(ValueError, match="No process rule found"):
DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=sqlite_session,
)
- session.scalar.assert_called_once()
-
- def test_save_document_with_dataset_id_rejects_invalid_indexing_technique(self, account_context):
- dataset = _make_dataset(indexing_technique=None)
+ def test_save_document_with_dataset_id_rejects_invalid_indexing_technique(
+ self, account_context, unbound_session: Session
+ ):
+ dataset = _dataset_row(indexing_technique=None)
knowledge_config = SimpleNamespace(
doc_form=IndexStructureType.PARAGRAPH_INDEX,
original_document_id=None,
@@ -1016,17 +1209,18 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
)
with patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)):
- session = MagicMock()
with pytest.raises(ValueError, match="Indexing technique is invalid"):
DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=unbound_session,
)
- def test_save_document_with_dataset_id_returns_empty_for_invalid_process_rule_mode(self, account_context):
- dataset = _make_dataset()
+ def test_save_document_with_dataset_id_returns_empty_for_invalid_process_rule_mode(
+ self, account_context, unbound_session: Session
+ ):
+ dataset = _dataset_row()
knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"])
knowledge_config.process_rule = SimpleNamespace(mode="unsupported-mode", rules=None)
@@ -1035,77 +1229,61 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
dataset,
knowledge_config,
account_context,
- session=MagicMock(),
+ session=unbound_session,
)
assert documents == []
assert batch == ""
- def test_save_document_with_dataset_id_upload_file_creates_and_reindexes_documents(self, account_context):
- session = MagicMock()
- dataset = _make_dataset()
- dataset_process_rule = SimpleNamespace(id="rule-1")
+ def test_save_document_with_dataset_id_upload_file_creates_and_reindexes_documents(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row(data_source_type=DataSourceType.UPLOAD_FILE)
+ dataset_process_rule = _process_rule()
knowledge_config = _make_upload_knowledge_config(file_ids=["file-1", "file-2"])
- duplicate_document = _make_document(document_id="doc-duplicate", name="existing.txt")
- created_document = _make_document(document_id="doc-created", name="new.txt")
- upload_file_a = SimpleNamespace(id="file-1", name="existing.txt")
- upload_file_b = SimpleNamespace(id="file-2", name="new.txt")
+ duplicate_document = _document_row(document_id="doc-duplicate", name="existing.txt")
+ upload_file_a = _upload_file(file_id="file-1", name="existing.txt")
+ upload_file_b = _upload_file(file_id="file-2", name="new.txt")
+ sqlite_session.add_all([dataset, dataset_process_rule, duplicate_document, upload_file_a, upload_file_b])
+ sqlite_session.commit()
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.redis_client") as mock_redis,
- patch.object(DocumentService, "get_documents_position", return_value=4),
- patch.object(DocumentService, "build_document", return_value=created_document) as build_document,
patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,
patch("services.dataset_service.DuplicateDocumentIndexingTaskProxy") as duplicate_proxy_cls,
- patch("services.dataset_service.naive_utc_now", return_value="now"),
+ patch("services.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),
patch("services.dataset_service.time.strftime", return_value="20260101010101"),
patch("services.dataset_service.secrets.randbelow", return_value=23),
):
mock_redis.lock.return_value = _make_lock_context()
- session.scalars.return_value.all.side_effect = [
- [upload_file_a, upload_file_b],
- [duplicate_document],
- ]
-
documents, batch = DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
dataset_process_rule=dataset_process_rule,
- session=session,
+ session=sqlite_session,
)
- assert documents == [duplicate_document, created_document]
+ assert [document.name for document in documents] == ["existing.txt", "new.txt"]
assert batch == "20260101010101100023"
assert duplicate_document.dataset_process_rule_id == "rule-1"
- assert duplicate_document.updated_at == "now"
+ assert duplicate_document.updated_at == datetime(2026, 2, 1)
assert duplicate_document.batch == batch
- assert duplicate_document.indexing_status == "waiting"
- build_document.assert_called_once_with(
- dataset,
- "rule-1",
- "upload_file",
- IndexStructureType.PARAGRAPH_INDEX,
- "English",
- {"upload_file_id": "file-2"},
- "web",
- 4,
- account_context,
- "new.txt",
- batch,
- )
- document_proxy_cls.assert_called_once_with(dataset.tenant_id, dataset.id, ["doc-created"])
+ assert duplicate_document.indexing_status == IndexingStatus.WAITING
+ created_document = next(document for document in documents if document.name == "new.txt")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(Document, created_document.id) is not None
+ document_proxy_cls.assert_called_once_with(dataset.tenant_id, dataset.id, [created_document.id])
document_proxy_cls.return_value.delay.assert_called_once()
duplicate_proxy_cls.assert_called_once_with(dataset.tenant_id, dataset.id, ["doc-duplicate"])
duplicate_proxy_cls.return_value.delay.assert_called_once()
def test_save_document_with_dataset_id_notion_import_truncates_names_and_cleans_removed_pages(
- self, account_context
+ self, account_context, sqlite_session: Session
):
- session = MagicMock()
- dataset = _make_dataset()
- dataset_process_rule = SimpleNamespace(id="rule-1")
+ dataset = _dataset_row(data_source_type=DataSourceType.NOTION_IMPORT)
+ dataset_process_rule = _process_rule()
notion_page_name = "a" * 300
knowledge_config = KnowledgeConfig(
indexing_technique="economy",
@@ -1132,42 +1310,50 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
doc_form=IndexStructureType.PARAGRAPH_INDEX,
doc_language="English",
)
- existing_keep = _make_document(document_id="doc-keep")
+ existing_keep = _document_row(
+ document_id="doc-keep",
+ data_source_type=DataSourceType.NOTION_IMPORT,
+ )
existing_keep.data_source_info = json.dumps({"notion_page_id": "page-keep"})
- existing_remove = _make_document(document_id="doc-remove")
+ existing_remove = _document_row(
+ document_id="doc-remove",
+ data_source_type=DataSourceType.NOTION_IMPORT,
+ )
existing_remove.data_source_info = json.dumps({"notion_page_id": "page-remove"})
- created_document = _make_document(document_id="doc-new")
+ sqlite_session.add_all([dataset, dataset_process_rule, existing_keep, existing_remove])
+ sqlite_session.commit()
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.redis_client") as mock_redis,
- patch.object(DocumentService, "get_documents_position", return_value=1),
- patch.object(DocumentService, "build_document", return_value=created_document) as build_document,
patch("services.dataset_service.clean_notion_document_task") as clean_task,
patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,
patch("services.dataset_service.uuid.uuid4", return_value="doc-new"),
):
mock_redis.lock.return_value = _make_lock_context()
- session.scalars.return_value.all.return_value = [existing_keep, existing_remove]
-
documents, _ = DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
dataset_process_rule=dataset_process_rule,
- session=session,
+ session=sqlite_session,
)
+ created_document = next(document for document in documents if document.id == "doc-new")
assert created_document in documents
- assert len(build_document.call_args.args[9]) == 255
+ assert len(created_document.name) == 255
+ assert sqlite_session.get(Document, created_document.id) is created_document
clean_task.delay.assert_called_once_with(["doc-remove"], dataset.id)
document_proxy_cls.assert_called_once_with(dataset.tenant_id, dataset.id, ["doc-new"])
document_proxy_cls.return_value.delay.assert_called_once()
- def test_save_document_with_dataset_id_website_crawl_truncates_long_urls(self, account_context):
- session = MagicMock()
- dataset = _make_dataset()
- dataset_process_rule = SimpleNamespace(id="rule-1")
+ def test_save_document_with_dataset_id_website_crawl_truncates_long_urls(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row(data_source_type=DataSourceType.WEBSITE_CRAWL)
+ dataset_process_rule = _process_rule()
+ sqlite_session.add_all([dataset, dataset_process_rule])
+ sqlite_session.commit()
long_url = "https://example.com/" + ("a" * 260)
short_url = "https://example.com/short"
knowledge_config = KnowledgeConfig(
@@ -1186,18 +1372,9 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
doc_form=IndexStructureType.PARAGRAPH_INDEX,
doc_language="English",
)
- first_document = _make_document(document_id="doc-1")
- second_document = _make_document(document_id="doc-2")
-
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.redis_client") as mock_redis,
- patch.object(DocumentService, "get_documents_position", return_value=2),
- patch.object(
- DocumentService,
- "build_document",
- side_effect=[first_document, second_document],
- ) as build_document,
patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,
):
mock_redis.lock.return_value = _make_lock_context()
@@ -1207,13 +1384,14 @@ class TestDocumentServiceSaveDocumentWithDatasetId:
knowledge_config,
account_context,
dataset_process_rule=dataset_process_rule,
- session=session,
+ session=sqlite_session,
)
- assert documents == [first_document, second_document]
- assert build_document.call_args_list[0].args[9] == long_url[:200] + "..."
- assert build_document.call_args_list[1].args[9] == short_url
- document_proxy_cls.assert_called_once_with(dataset.tenant_id, dataset.id, ["doc-1", "doc-2"])
+ assert [document.name for document in documents] == [long_url[:200] + "...", short_url]
+ assert sqlite_session.scalars(select(Document).order_by(Document.position)).all() == documents
+ document_proxy_cls.assert_called_once_with(
+ dataset.tenant_id, dataset.id, [document.id for document in documents]
+ )
document_proxy_cls.return_value.delay.assert_called_once()
@@ -1221,16 +1399,16 @@ class TestDocumentServiceBatchUpdateStatus:
"""Unit tests for batch_update_document_status orchestration and helper branches."""
def test_prepare_disable_update_requires_completed_document(self):
- document = _make_document(indexing_status="waiting")
+ document = _document_row(indexing_status=IndexingStatus.WAITING)
document.completed_at = None
with pytest.raises(DocumentIndexingError, match="is not completed"):
- DocumentService._prepare_disable_update(document, user=SimpleNamespace(id="user-1"), now="now")
+ DocumentService._prepare_disable_update(document, user=_account(), now=datetime(2026, 2, 1))
def test_prepare_archive_update_sets_async_task_for_enabled_document(self):
- document = _make_document(enabled=True, archived=False)
+ document = _document_row(enabled=True, archived=False)
- result = DocumentService._prepare_archive_update(document, user=SimpleNamespace(id="user-1"), now="now")
+ result = DocumentService._prepare_archive_update(document, user=_account(), now=datetime(2026, 2, 1))
assert result is not None
assert result["updates"]["archived"] is True
@@ -1238,59 +1416,63 @@ class TestDocumentServiceBatchUpdateStatus:
assert result["async_task"]["args"] == [document.id]
def test_prepare_unarchive_update_sets_async_task_for_enabled_document(self):
- document = _make_document(enabled=True, archived=True)
+ document = _document_row(enabled=True, archived=True)
- result = DocumentService._prepare_unarchive_update(document, now="now")
+ result = DocumentService._prepare_unarchive_update(document, now=datetime(2026, 2, 1))
assert result is not None
assert result["updates"]["archived"] is False
assert result["set_cache"] is True
assert result["async_task"]["args"] == [document.id]
- def test_batch_update_document_status_rejects_indexing_documents(self):
- session = MagicMock()
- dataset = _make_dataset()
- document = _make_document(name="Busy document")
+ def test_batch_update_document_status_rejects_indexing_documents(self, sqlite_session: Session):
+ dataset = _dataset_row()
+ document = _document_row(name="Busy document")
+ sqlite_session.add_all([dataset, document])
+ sqlite_session.commit()
- with (
- patch.object(DocumentService, "get_document", return_value=document),
- patch("services.dataset_service.redis_client") as mock_redis,
- ):
+ with patch("services.dataset_service.redis_client") as mock_redis:
mock_redis.get.return_value = "1"
with pytest.raises(DocumentIndexingError, match="Busy document is being indexed"):
DocumentService.batch_update_document_status(
- dataset, [document.id], "archive", SimpleNamespace(id="user-1"), session
+ dataset, [document.id], "archive", _account(), sqlite_session
)
- session.flush.assert_not_called()
+ sqlite_session.refresh(document)
+ assert document.archived is False
+
+ def test_batch_update_document_status_rolls_back_when_commit_fails(self, sqlite_session: Session):
+ dataset = _dataset_row()
+ document = _document_row(enabled=False)
+ sqlite_session.add_all([dataset, document])
+ sqlite_session.commit()
- def test_batch_update_document_status_rolls_back_when_commit_fails(self):
- session = MagicMock()
- dataset = _make_dataset()
- document = _make_document(enabled=False)
+ def fail_commit(_session):
+ raise RuntimeError("commit failed")
+ event.listen(sqlite_session, "before_commit", fail_commit)
with (
- patch.object(DocumentService, "get_document", return_value=document),
patch("services.dataset_service.redis_client") as mock_redis,
):
mock_redis.get.return_value = None
- session.commit.side_effect = RuntimeError("commit failed")
with pytest.raises(RuntimeError, match="commit failed"):
DocumentService.batch_update_document_status(
- dataset, [document.id], "enable", SimpleNamespace(id="user-1"), session
+ dataset, [document.id], "enable", _account(), sqlite_session
)
+ event.remove(sqlite_session, "before_commit", fail_commit)
- session.rollback.assert_called_once()
+ sqlite_session.refresh(document)
+ assert document.enabled is False
- def test_batch_update_document_status_raises_async_task_error_after_commit(self):
- session = MagicMock()
- dataset = _make_dataset()
- document = _make_document(enabled=False)
+ def test_batch_update_document_status_raises_async_task_error_after_commit(self, sqlite_session: Session):
+ dataset = _dataset_row()
+ document = _document_row(enabled=False)
+ sqlite_session.add_all([dataset, document])
+ sqlite_session.commit()
with (
- patch.object(DocumentService, "get_document", return_value=document),
patch("services.dataset_service.redis_client") as mock_redis,
patch("services.dataset_service.add_document_to_index_task") as add_task,
):
@@ -1299,10 +1481,11 @@ class TestDocumentServiceBatchUpdateStatus:
with pytest.raises(RuntimeError, match="task failed"):
DocumentService.batch_update_document_status(
- dataset, [document.id], "enable", SimpleNamespace(id="user-1"), session
+ dataset, [document.id], "enable", _account(), sqlite_session
)
- session.commit.assert_called_once()
+ sqlite_session.refresh(document)
+ assert document.enabled is True
mock_redis.setex.assert_called_once_with(f"document_{document.id}_indexing", 600, 1)
@@ -1311,25 +1494,36 @@ class TestDocumentServiceTenantAndUpdateEdges:
@pytest.fixture
def account_context(self):
- account = create_autospec(Account, instance=True)
- account.id = "user-1"
- account.current_tenant_id = "tenant-1"
+ account = _account()
with patch("services.dataset_service.current_user", account):
yield account
- def test_get_tenant_documents_count_returns_query_count(self, account_context):
- session = MagicMock()
- session.scalar.return_value = 12
+ def test_get_tenant_documents_count_scopes_state_and_tenant(self, account_context, sqlite_session: Session):
+ sqlite_session.add_all(
+ [
+ _document_row(document_id="one"),
+ _document_row(document_id="two"),
+ _document_row(document_id="disabled", enabled=False),
+ _document_row(document_id="archived", archived=True),
+ _document_row(document_id="unfinished", indexing_status=IndexingStatus.WAITING),
+ _document_row(document_id="foreign", tenant_id="tenant-2"),
+ ]
+ )
+ sqlite_session.commit()
- result = DocumentService.get_tenant_documents_count(session)
+ result = DocumentService.get_tenant_documents_count(sqlite_session)
- assert result == 12
+ assert result == 2
- def test_update_document_with_dataset_id_uses_automatic_process_rule_payload(self, account_context):
- session = MagicMock()
- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")
- document = _make_document()
+ def test_update_document_with_dataset_id_uses_automatic_process_rule_payload(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row()
+ document = _document_row(document_id="doc-1")
+ upload_file = _upload_file(file_id="file-1")
+ sqlite_session.add_all([dataset, document, upload_file])
+ sqlite_session.commit()
document_data = KnowledgeConfig(
original_document_id="doc-1",
indexing_technique="economy",
@@ -1348,62 +1542,56 @@ class TestDocumentServiceTenantAndUpdateEdges:
),
doc_form=IndexStructureType.PARAGRAPH_INDEX,
)
- created_process_rule = SimpleNamespace(id="rule-2")
+ updated_at = datetime(2026, 2, 1)
with (
- patch.object(DocumentService, "get_document", return_value=document),
- patch("services.dataset_service.DatasetProcessRule") as process_rule_cls,
patch.object(DatasetService, "check_dataset_model_setting"),
- patch("services.dataset_service.naive_utc_now", return_value="now"),
+ patch("services.dataset_service.naive_utc_now", return_value=updated_at),
patch("services.dataset_service.document_indexing_update_task") as update_task,
):
- process_rule_cls.AUTOMATIC_RULES = DatasetProcessRule.AUTOMATIC_RULES
- process_rule_cls.return_value = created_process_rule
- session.scalar.return_value = SimpleNamespace(id="file-1", name="upload.txt")
-
result = DocumentService.update_document_with_dataset_id(
dataset,
document_data,
account_context,
- session=session,
+ session=sqlite_session,
)
assert result is document
- assert document.dataset_process_rule_id == "rule-2"
+ assert document.dataset_process_rule_id is not None
assert document.name == "upload.txt"
- assert process_rule_cls.call_args.kwargs == {
- "dataset_id": "dataset-1",
- "mode": "automatic",
- "rules": json.dumps(DatasetProcessRule.AUTOMATIC_RULES),
- "created_by": "user-1",
- }
- assert session.commit.call_count == 3
+ process_rule = sqlite_session.get(DatasetProcessRule, document.dataset_process_rule_id)
+ assert process_rule is not None
+ assert process_rule.mode == "automatic"
+ assert process_rule.rules == json.dumps(DatasetProcessRule.AUTOMATIC_RULES)
update_task.delay.assert_called_once_with("dataset-1", "doc-1")
- def test_update_document_with_dataset_id_requires_upload_file_info(self, account_context):
- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")
+ def test_update_document_with_dataset_id_requires_upload_file_info(self, account_context, sqlite_session: Session):
+ dataset = _dataset_row()
+ document = _document_row(document_id="doc-1")
+ sqlite_session.add_all([dataset, document])
+ sqlite_session.commit()
document_data = KnowledgeConfig(
original_document_id="doc-1",
indexing_technique="economy",
data_source=DataSource(info_list=InfoList(data_source_type="upload_file")),
)
- with (
- patch.object(DocumentService, "get_document", return_value=_make_document()),
- patch.object(DatasetService, "check_dataset_model_setting"),
- ):
- session = MagicMock()
+ with patch.object(DatasetService, "check_dataset_model_setting"):
with pytest.raises(ValueError, match="No file info list found"):
DocumentService.update_document_with_dataset_id(
dataset,
document_data,
account_context,
- session=session,
+ session=sqlite_session,
)
- def test_update_document_with_dataset_id_raises_when_upload_file_is_missing(self, account_context):
- session = MagicMock()
- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")
+ def test_update_document_with_dataset_id_raises_when_upload_file_is_missing(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row()
+ document = _document_row(document_id="doc-1")
+ sqlite_session.add_all([dataset, document])
+ sqlite_session.commit()
document_data = KnowledgeConfig(
original_document_id="doc-1",
indexing_technique="economy",
@@ -1415,45 +1603,49 @@ class TestDocumentServiceTenantAndUpdateEdges:
),
)
- with (
- patch.object(DocumentService, "get_document", return_value=_make_document()),
- patch.object(DatasetService, "check_dataset_model_setting"),
- ):
- session.scalar.return_value = None
-
+ with patch.object(DatasetService, "check_dataset_model_setting"):
with pytest.raises(FileNotExistsError):
DocumentService.update_document_with_dataset_id(
dataset,
document_data,
account_context,
- session=session,
+ session=sqlite_session,
)
- def test_update_document_with_dataset_id_requires_notion_info_list(self, account_context):
- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")
+ def test_update_document_with_dataset_id_requires_notion_info_list(self, account_context, sqlite_session: Session):
+ dataset = _dataset_row()
+ document = _document_row(document_id="doc-1")
+ sqlite_session.add_all([dataset, document])
+ sqlite_session.commit()
document_data = KnowledgeConfig(
original_document_id="doc-1",
indexing_technique="economy",
data_source=DataSource(info_list=InfoList(data_source_type="notion_import")),
)
- with (
- patch.object(DocumentService, "get_document", return_value=_make_document()),
- patch.object(DatasetService, "check_dataset_model_setting"),
- ):
- session = MagicMock()
+ with patch.object(DatasetService, "check_dataset_model_setting"):
with pytest.raises(ValueError, match="No notion info list found"):
DocumentService.update_document_with_dataset_id(
dataset,
document_data,
account_context,
- session=session,
+ session=sqlite_session,
)
- def test_update_document_with_dataset_id_notion_import_updates_page_info(self, account_context):
- session = MagicMock()
- dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1")
- document = _make_document()
+ def test_update_document_with_dataset_id_notion_import_updates_page_info(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row()
+ document = _document_row(document_id="doc-1")
+ binding = DataSourceOauthBinding(
+ tenant_id=dataset.tenant_id,
+ access_token="token",
+ provider="notion",
+ source_info={"workspace_id": '"workspace-1"'},
+ disabled=False,
+ )
+ sqlite_session.add_all([dataset, document, binding])
+ sqlite_session.commit()
document_data = KnowledgeConfig(
original_document_id="doc-1",
indexing_technique="economy",
@@ -1476,18 +1668,15 @@ class TestDocumentServiceTenantAndUpdateEdges:
)
with (
- patch.object(DocumentService, "get_document", return_value=document),
patch.object(DatasetService, "check_dataset_model_setting"),
- patch("services.dataset_service.naive_utc_now", return_value="now"),
+ patch("services.dataset_service.naive_utc_now", return_value=datetime(2026, 2, 1)),
patch("services.dataset_service.document_indexing_update_task") as update_task,
):
- session.scalar.return_value = SimpleNamespace(id="binding-1")
-
result = DocumentService.update_document_with_dataset_id(
dataset,
document_data,
account_context,
- session=session,
+ session=sqlite_session,
)
assert result is document
@@ -1502,6 +1691,7 @@ class TestDocumentServiceTenantAndUpdateEdges:
"type": "database",
}
)
+ sqlite_session.refresh(document)
update_task.delay.assert_called_once_with("dataset-1", "doc-1")
@@ -1510,15 +1700,14 @@ class TestDocumentServiceSaveWithoutDatasetBilling:
@pytest.fixture
def account_context(self):
- account = create_autospec(Account, instance=True)
- account.id = "user-1"
- account.current_tenant_id = "tenant-1"
+ account = _account()
with patch("services.dataset_service.current_user", account):
yield account
- def test_save_document_without_dataset_id_counts_notion_pages_for_quota(self, account_context):
- session = MagicMock()
+ def test_save_document_without_dataset_id_counts_notion_pages_for_quota(
+ self, account_context, sqlite_session: Session
+ ):
knowledge_config = KnowledgeConfig(
indexing_technique="economy",
data_source=DataSource(
@@ -1542,33 +1731,32 @@ class TestDocumentServiceSaveWithoutDatasetBilling:
)
),
)
- created_dataset = SimpleNamespace(id="dataset-1", tenant_id="tenant-1", name="", description=None)
features = _make_features(enabled=True)
+ document = _document_row(name="Doc")
with (
patch("services.dataset_service.FeatureService.get_features", return_value=features),
patch("services.dataset_service.dify_config.BATCH_UPLOAD_LIMIT", "10"),
patch.object(DocumentService, "check_documents_upload_quota") as check_quota,
- patch(
- "services.dataset_service.Dataset",
- side_effect=lambda **kwargs: created_dataset.__dict__.update(kwargs) or created_dataset,
- ),
patch.object(
DocumentService,
"save_document_with_dataset_id",
- return_value=([SimpleNamespace(name="Doc")], "batch-1"),
+ return_value=([document], "batch-1"),
),
):
- DocumentService.save_document_without_dataset_id(
+ dataset, _, _ = DocumentService.save_document_without_dataset_id(
"tenant-1",
knowledge_config,
account_context,
- session,
+ sqlite_session,
)
check_quota.assert_called_once_with(3, features)
+ assert sqlite_session.get(Dataset, dataset.id) is dataset
- def test_save_document_without_dataset_id_enforces_batch_limit_for_website_urls(self, account_context):
+ def test_save_document_without_dataset_id_enforces_batch_limit_for_website_urls(
+ self, account_context, unbound_session: Session
+ ):
knowledge_config = KnowledgeConfig(
indexing_technique="economy",
data_source=DataSource(
@@ -1589,9 +1777,10 @@ class TestDocumentServiceSaveWithoutDatasetBilling:
patch("services.dataset_service.dify_config.BATCH_UPLOAD_LIMIT", "1"),
patch.object(DocumentService, "check_documents_upload_quota") as check_quota,
):
- session = MagicMock()
with pytest.raises(ValueError, match="batch upload limit of 1"):
- DocumentService.save_document_without_dataset_id("tenant-1", knowledge_config, account_context, session)
+ DocumentService.save_document_without_dataset_id(
+ "tenant-1", knowledge_config, account_context, unbound_session
+ )
check_quota.assert_not_called()
@@ -1727,9 +1916,7 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
@pytest.fixture
def account_context(self):
- account = create_autospec(Account, instance=True)
- account.id = "user-1"
- account.current_tenant_id = "tenant-1"
+ account = _account()
with (
patch("services.dataset_service.current_user", account),
@@ -1738,21 +1925,29 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
yield account
def test_save_document_with_dataset_id_initializes_high_quality_dataset_from_default_embedding_model(
- self, account_context
+ self, account_context, sqlite_session: Session
):
- dataset = _make_dataset(data_source_type=None, indexing_technique=None)
+ dataset = _dataset_row(data_source_type=None, indexing_technique=None)
knowledge_config = _make_upload_knowledge_config(original_document_id="doc-1", file_ids=["file-1"])
knowledge_config.indexing_technique = "high_quality"
knowledge_config.embedding_model = None
knowledge_config.embedding_model_provider = None
- updated_document = _make_document(batch="batch-existing")
+ updated_document = _document_row(document_id="doc-1")
+ updated_document.batch = "batch-existing"
+ binding = DatasetCollectionBinding(
+ provider_name="default-provider",
+ model_name="default-embedding",
+ type="dataset",
+ collection_name="collection",
+ )
+ binding.id = "binding-1"
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.ModelManager") as model_manager_cls,
patch(
"services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",
- return_value=SimpleNamespace(id="binding-1"),
+ return_value=binding,
) as get_binding,
patch.object(DocumentService, "update_document_with_dataset_id", return_value=updated_document),
):
@@ -1761,12 +1956,11 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
provider="default-provider",
)
- session = MagicMock()
documents, batch = DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=sqlite_session,
)
assert documents == [updated_document]
@@ -1783,10 +1977,12 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
"top_k": 4,
"score_threshold_enabled": False,
}
- get_binding.assert_called_once_with("default-provider", "default-embedding", session)
+ get_binding.assert_called_once_with("default-provider", "default-embedding", sqlite_session)
- def test_save_document_with_dataset_id_uses_explicit_embedding_and_retrieval_model(self, account_context):
- dataset = _make_dataset(indexing_technique=None)
+ def test_save_document_with_dataset_id_uses_explicit_embedding_and_retrieval_model(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row(indexing_technique=None)
knowledge_config = _make_upload_knowledge_config(original_document_id="doc-1", file_ids=["file-1"])
knowledge_config.indexing_technique = "high_quality"
knowledge_config.embedding_model = "explicit-model"
@@ -1802,28 +1998,38 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
score_threshold_enabled=True,
score_threshold=0.3,
)
+ binding = DatasetCollectionBinding(
+ provider_name="explicit-provider",
+ model_name="explicit-model",
+ type="dataset",
+ collection_name="collection",
+ )
+ binding.id = "binding-2"
+ updated_document = _document_row(document_id="doc-1")
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.ModelManager") as model_manager_cls,
patch(
"services.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding",
- return_value=SimpleNamespace(id="binding-2"),
+ return_value=binding,
) as get_binding,
- patch.object(DocumentService, "update_document_with_dataset_id", return_value=_make_document()),
+ patch.object(DocumentService, "update_document_with_dataset_id", return_value=updated_document),
):
- session = MagicMock()
- DocumentService.save_document_with_dataset_id(dataset, knowledge_config, account_context, session=session)
+ DocumentService.save_document_with_dataset_id(
+ dataset, knowledge_config, account_context, session=sqlite_session
+ )
model_manager_cls.for_tenant.return_value.get_default_model_instance.assert_not_called()
- get_binding.assert_called_once_with("explicit-provider", "explicit-model", session)
+ get_binding.assert_called_once_with("explicit-provider", "explicit-model", sqlite_session)
assert dataset.embedding_model == "explicit-model"
assert dataset.embedding_model_provider == "explicit-provider"
assert dataset.retrieval_model == knowledge_config.retrieval_model.model_dump()
- def test_save_document_with_dataset_id_creates_custom_process_rule_for_new_upload_document(self, account_context):
- session = MagicMock()
- dataset = _make_dataset()
+ def test_save_document_with_dataset_id_creates_custom_process_rule_for_new_upload_document(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row(data_source_type=DataSourceType.UPLOAD_FILE)
knowledge_config = _make_upload_knowledge_config(
file_ids=["file-1"],
process_rule=ProcessRule(
@@ -1834,148 +2040,126 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
),
),
)
- created_process_rule = SimpleNamespace(id="rule-custom")
- created_document = _make_document(document_id="doc-created", name="file.txt")
+ upload_file = _upload_file(file_id="file-1", name="file.txt")
+ sqlite_session.add_all([dataset, upload_file])
+ sqlite_session.commit()
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.redis_client") as mock_redis,
- patch("services.dataset_service.DatasetProcessRule") as process_rule_cls,
- patch.object(DocumentService, "get_documents_position", return_value=3),
- patch.object(DocumentService, "build_document", return_value=created_document),
patch("services.dataset_service.DocumentIndexingTaskProxy") as document_proxy_cls,
patch("services.dataset_service.time.strftime", return_value="20260101010101"),
patch("services.dataset_service.secrets.randbelow", return_value=23),
):
mock_redis.lock.return_value = _make_lock_context()
- process_rule_cls.return_value = created_process_rule
- session.scalars.return_value.all.side_effect = [[SimpleNamespace(id="file-1", name="file.txt")], []]
-
documents, batch = DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=sqlite_session,
)
- assert documents == [created_document]
+ assert len(documents) == 1
+ created_document = documents[0]
+ assert created_document.name == "file.txt"
assert batch == "20260101010101100023"
- assert process_rule_cls.call_args.kwargs == {
- "dataset_id": "dataset-1",
- "mode": "custom",
- "rules": knowledge_config.process_rule.rules.model_dump_json(),
- "created_by": "user-1",
- }
- document_proxy_cls.assert_called_once_with("tenant-1", "dataset-1", ["doc-created"])
+ created_rule = sqlite_session.get(DatasetProcessRule, created_document.dataset_process_rule_id)
+ assert created_rule is not None
+ assert created_rule.mode == "custom"
+ assert created_rule.rules == knowledge_config.process_rule.rules.model_dump_json()
+ document_proxy_cls.assert_called_once_with("tenant-1", "dataset-1", [created_document.id])
document_proxy_cls.return_value.delay.assert_called_once()
def test_save_document_with_dataset_id_creates_automatic_process_rule_for_new_upload_document(
- self, account_context
+ self, account_context, sqlite_session: Session
):
- session = MagicMock()
- dataset = _make_dataset()
+ dataset = _dataset_row(data_source_type=DataSourceType.UPLOAD_FILE)
knowledge_config = _make_upload_knowledge_config(
file_ids=["file-1"],
process_rule=ProcessRule(mode="automatic"),
)
- created_process_rule = SimpleNamespace(id="rule-auto")
- created_document = _make_document(document_id="doc-created", name="file.txt")
+ sqlite_session.add_all([dataset, _upload_file(file_id="file-1", name="file.txt")])
+ sqlite_session.commit()
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.redis_client") as mock_redis,
- patch("services.dataset_service.DatasetProcessRule") as process_rule_cls,
- patch.object(DocumentService, "get_documents_position", return_value=1),
- patch.object(DocumentService, "build_document", return_value=created_document),
patch("services.dataset_service.DocumentIndexingTaskProxy"),
patch("services.dataset_service.time.strftime", return_value="20260101010101"),
patch("services.dataset_service.secrets.randbelow", return_value=23),
):
mock_redis.lock.return_value = _make_lock_context()
- process_rule_cls.AUTOMATIC_RULES = DatasetProcessRule.AUTOMATIC_RULES
- process_rule_cls.return_value = created_process_rule
- session.scalars.return_value.all.side_effect = [[SimpleNamespace(id="file-1", name="file.txt")], []]
-
- DocumentService.save_document_with_dataset_id(
+ documents, _ = DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=sqlite_session,
)
- assert process_rule_cls.call_args.kwargs == {
- "dataset_id": "dataset-1",
- "mode": "automatic",
- "rules": json.dumps(DatasetProcessRule.AUTOMATIC_RULES),
- "created_by": "user-1",
- }
- assert session.flush.call_count >= 2
+ created_rule = sqlite_session.get(DatasetProcessRule, documents[0].dataset_process_rule_id)
+ assert created_rule is not None
+ assert created_rule.mode == "automatic"
+ assert created_rule.rules == json.dumps(DatasetProcessRule.AUTOMATIC_RULES)
+ assert sqlite_session.get(Document, documents[0].id) is documents[0]
def test_save_document_with_dataset_id_creates_fallback_automatic_process_rule_when_latest_is_missing(
- self, account_context
+ self, account_context, sqlite_session: Session
):
- session = MagicMock()
- dataset = _make_dataset()
+ dataset = _dataset_row(data_source_type=DataSourceType.UPLOAD_FILE)
knowledge_config = _make_upload_knowledge_config(file_ids=["file-1"], process_rule=None)
- created_process_rule = SimpleNamespace(id="rule-fallback")
- created_document = _make_document(document_id="doc-created", name="file.txt")
+ sqlite_session.add_all([dataset, _upload_file(file_id="file-1", name="file.txt")])
+ sqlite_session.commit()
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.redis_client") as mock_redis,
- patch("services.dataset_service.DatasetProcessRule") as process_rule_cls,
- patch.object(DocumentService, "get_documents_position", return_value=1),
- patch.object(DocumentService, "build_document", return_value=created_document),
patch("services.dataset_service.DocumentIndexingTaskProxy"),
patch("services.dataset_service.time.strftime", return_value="20260101010101"),
patch("services.dataset_service.secrets.randbelow", return_value=23),
):
mock_redis.lock.return_value = _make_lock_context()
- process_rule_cls.AUTOMATIC_RULES = DatasetProcessRule.AUTOMATIC_RULES
- process_rule_cls.return_value = created_process_rule
- session.scalar.return_value = None
- session.scalars.return_value.all.side_effect = [[SimpleNamespace(id="file-1", name="file.txt")], []]
-
- DocumentService.save_document_with_dataset_id(
+ documents, _ = DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=sqlite_session,
)
- session.scalar.assert_called_once()
- assert process_rule_cls.call_args.kwargs == {
- "dataset_id": "dataset-1",
- "mode": "automatic",
- "rules": json.dumps(DatasetProcessRule.AUTOMATIC_RULES),
- "created_by": "user-1",
- }
+ created_rule = sqlite_session.get(DatasetProcessRule, documents[0].dataset_process_rule_id)
+ assert created_rule is not None
+ assert created_rule.mode == "automatic"
+ assert created_rule.rules == json.dumps(DatasetProcessRule.AUTOMATIC_RULES)
- def test_save_document_with_dataset_id_raises_when_upload_file_lookup_is_incomplete(self, account_context):
- session = MagicMock()
- dataset = _make_dataset()
+ def test_save_document_with_dataset_id_raises_when_upload_file_lookup_is_incomplete(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row(data_source_type=DataSourceType.UPLOAD_FILE)
knowledge_config = _make_upload_knowledge_config(file_ids=["file-1", "file-2"])
+ sqlite_session.add_all([dataset, _upload_file(file_id="file-1", name="file.txt")])
+ sqlite_session.commit()
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.redis_client") as mock_redis,
- patch.object(DocumentService, "get_documents_position", return_value=1),
patch("services.dataset_service.time.strftime", return_value="20260101010101"),
patch("services.dataset_service.secrets.randbelow", return_value=23),
):
mock_redis.lock.return_value = _make_lock_context()
- session.scalars.return_value.all.return_value = [SimpleNamespace(id="file-1", name="file.txt")]
-
with pytest.raises(FileNotExistsError, match="One or more files not found"):
DocumentService.save_document_with_dataset_id(
dataset,
knowledge_config,
account_context,
- session=session,
+ session=sqlite_session,
)
- def test_save_document_with_dataset_id_requires_notion_info_list_for_notion_import(self, account_context):
- dataset = _make_dataset()
+ def test_save_document_with_dataset_id_requires_notion_info_list_for_notion_import(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row(data_source_type=DataSourceType.NOTION_IMPORT)
+ process_rule = _process_rule()
+ sqlite_session.add_all([dataset, process_rule])
+ sqlite_session.commit()
knowledge_config = KnowledgeConfig(
indexing_technique="economy",
data_source=DataSource(info_list=InfoList(data_source_type="notion_import")),
@@ -1986,7 +2170,6 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.redis_client") as mock_redis,
- patch.object(DocumentService, "get_documents_position", return_value=1),
):
mock_redis.lock.return_value = _make_lock_context()
with pytest.raises(ValueError, match="No notion info list found"):
@@ -1994,12 +2177,17 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
dataset,
knowledge_config,
account_context,
- dataset_process_rule=SimpleNamespace(id="rule-1"),
- session=MagicMock(),
+ dataset_process_rule=process_rule,
+ session=sqlite_session,
)
- def test_save_document_with_dataset_id_requires_website_info_list_for_website_crawl(self, account_context):
- dataset = _make_dataset()
+ def test_save_document_with_dataset_id_requires_website_info_list_for_website_crawl(
+ self, account_context, sqlite_session: Session
+ ):
+ dataset = _dataset_row(data_source_type=DataSourceType.WEBSITE_CRAWL)
+ process_rule = _process_rule()
+ sqlite_session.add_all([dataset, process_rule])
+ sqlite_session.commit()
knowledge_config = KnowledgeConfig(
indexing_technique="economy",
data_source=DataSource(info_list=InfoList(data_source_type="website_crawl")),
@@ -2010,7 +2198,6 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
with (
patch("services.dataset_service.FeatureService.get_features", return_value=_make_features(enabled=False)),
patch("services.dataset_service.redis_client") as mock_redis,
- patch.object(DocumentService, "get_documents_position", return_value=1),
):
mock_redis.lock.return_value = _make_lock_context()
with pytest.raises(ValueError, match="No website info list found"):
@@ -2018,6 +2205,6 @@ class TestDocumentServiceSaveDocumentAdditionalBranches:
dataset,
knowledge_config,
account_context,
- dataset_process_rule=SimpleNamespace(id="rule-1"),
- session=MagicMock(),
+ dataset_process_rule=process_rule,
+ session=sqlite_session,
)