Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: migrate dataset download sessions and ORM models to SQLite (#40630)

Asuka Minato797835226260
diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py
index 4149a97eaba..88dab19cd99 100644
--- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py
+++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document_download.py
@@ -8,18 +8,25 @@ upload-file documents, and rejects unsupported or missing file cases.
from __future__ import annotations
import importlib
+import json
import sys
from collections import UserDict
+from datetime import UTC, datetime
from inspect import unwrap
from io import BytesIO
-from types import SimpleNamespace
-from unittest.mock import MagicMock
from zipfile import ZipFile
import pytest
from flask import Flask
+from sqlalchemy.orm import Session
from werkzeug.exceptions import NotFound
+from extensions.storage.storage_type import StorageType
+from models.account import Account, TenantAccountRole
+from models.dataset import Dataset, Document
+from models.enums import CreatorUserRole
+from models.model import UploadFile
+
@pytest.fixture
def app() -> Flask:
@@ -67,31 +74,57 @@ def datasets_document_module(monkeypatch: pytest.MonkeyPatch):
return importlib.import_module(module_name)
-def _mock_user(*, is_dataset_editor: bool = True) -> SimpleNamespace:
- """Build a minimal user object compatible with dataset permission checks."""
- return SimpleNamespace(is_dataset_editor=is_dataset_editor, id="user-123")
+def _account(*, is_dataset_editor: bool = True) -> Account:
+ """Build a real account with the role used by dataset permission checks."""
+ account = Account(name="Dataset User", email="[email protected]")
+ account.id = "user-123"
+ account.role = TenantAccountRole.EDITOR if is_dataset_editor else TenantAccountRole.NORMAL
+ return account
-def _mock_document(
+def _document(
*,
document_id: str,
tenant_id: str,
data_source_type: str,
upload_file_id: str | None,
-) -> SimpleNamespace:
- """Build a minimal document object used by the controller."""
- data_source_info_dict: dict[str, object] | None = None
- if upload_file_id is not None:
- data_source_info_dict = {"upload_file_id": upload_file_id}
- else:
- data_source_info_dict = {}
-
- return SimpleNamespace(
+) -> Document:
+ """Build a real document entity used by the controller."""
+ document = Document(
id=document_id,
tenant_id=tenant_id,
+ dataset_id="ds-1",
+ position=1,
data_source_type=data_source_type,
- data_source_info_dict=data_source_info_dict,
+ data_source_info=json.dumps({"upload_file_id": upload_file_id}) if upload_file_id is not None else "{}",
+ batch="batch-1",
+ name="document.txt",
+ created_from="web",
+ created_by="user-123",
+ )
+ return document
+
+
+def _dataset() -> Dataset:
+ return Dataset(id="ds-1", tenant_id="tenant-123", name="Dataset", created_by="user-123")
+
+
+def _upload_file(*, file_id: str, name: str = "document.txt", key: str = "key") -> UploadFile:
+ upload_file = UploadFile(
+ tenant_id="tenant-123",
+ storage_type=StorageType.LOCAL,
+ key=key,
+ name=name,
+ size=1,
+ extension="txt",
+ mime_type="text/plain",
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by="user-123",
+ created_at=datetime.now(UTC),
+ used=False,
)
+ upload_file.id = file_id
+ return upload_file
def _wire_common_success_mocks(
@@ -111,12 +144,12 @@ def _wire_common_success_mocks(
monkeypatch.setattr(
module.DatasetService,
"get_dataset_for_tenant",
- lambda *_args, **_kwargs: SimpleNamespace(id="ds-1", tenant_id="tenant-123"),
+ lambda *_args, **_kwargs: _dataset(),
)
monkeypatch.setattr(module.DatasetService, "check_dataset_permission", lambda *_args, **_kwargs: None)
# Return a document that will be validated inside DocumentResource.get_document.
- document = _mock_document(
+ document = _document(
document_id="doc-1",
tenant_id=document_tenant_id,
data_source_type=data_source_type,
@@ -131,7 +164,7 @@ def _wire_common_success_mocks(
# Mock UploadFile lookup via FileService batch helper.
upload_files_by_id: dict[str, object] = {}
if upload_file_exists and upload_file_id is not None:
- upload_files_by_id[upload_file_id] = SimpleNamespace(id=upload_file_id)
+ upload_files_by_id[upload_file_id] = _upload_file(file_id=upload_file_id)
monkeypatch.setattr(module.FileService, "get_upload_files_by_ids", lambda *_args, **_kwargs: upload_files_by_id)
# Mock signing helper so the returned URL is deterministic.
@@ -154,25 +187,23 @@ def _mock_send_file(obj, **kwargs): # type: ignore[no-untyped-def]
def test_batch_download_zip_returns_send_file(
- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch
+ app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
) -> None:
"""Ensure batch ZIP download returns a zip attachment via `send_file`."""
- monkeypatch.setattr(
- datasets_document_module.DatasetService, "get_dataset", lambda *_args, **_kwargs: SimpleNamespace(id="ds-1")
- )
+ monkeypatch.setattr(datasets_document_module.DatasetService, "get_dataset", lambda *_args, **_kwargs: _dataset())
monkeypatch.setattr(
datasets_document_module.DatasetService, "check_dataset_permission", lambda *_args, **_kwargs: None
)
# Two upload-file documents, each referencing an UploadFile.
- doc1 = _mock_document(
+ doc1 = _document(
document_id="11111111-1111-1111-1111-111111111111",
tenant_id="tenant-123",
data_source_type="upload_file",
upload_file_id="file-1",
)
- doc2 = _mock_document(
+ doc2 = _document(
document_id="22222222-2222-2222-2222-222222222222",
tenant_id="tenant-123",
data_source_type="upload_file",
@@ -187,8 +218,8 @@ def test_batch_download_zip_returns_send_file(
datasets_document_module.FileService,
"get_upload_files_by_ids",
lambda *_args, **_kwargs: {
- "file-1": SimpleNamespace(id="file-1", name="a.txt", key="k1"),
- "file-2": SimpleNamespace(id="file-2", name="b.txt", key="k2"),
+ "file-1": _upload_file(file_id="file-1", name="a.txt", key="k1"),
+ "file-2": _upload_file(file_id="file-2", name="b.txt", key="k2"),
},
)
@@ -208,7 +239,7 @@ def test_batch_download_zip_returns_send_file(
):
api = datasets_document_module.DocumentBatchDownloadZipApi()
method = unwrap(api.post)
- result = method(api, MagicMock(), "tenant-123", _mock_user(), dataset_id="ds-1")
+ result = method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1")
# Assert: we returned via send_file with correct mime type and attachment.
assert result["_send_file_kwargs"]["mimetype"] == "application/zip"
@@ -221,25 +252,23 @@ def test_batch_download_zip_returns_send_file(
def test_batch_download_zip_response_is_openable_zip(
- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch
+ app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
) -> None:
"""Ensure the real Flask `send_file` response body is a valid ZIP that can be opened."""
# Arrange: same controller mocks as the lightweight send_file test, but we keep the real `send_file`.
- monkeypatch.setattr(
- datasets_document_module.DatasetService, "get_dataset", lambda *_args, **_kwargs: SimpleNamespace(id="ds-1")
- )
+ monkeypatch.setattr(datasets_document_module.DatasetService, "get_dataset", lambda *_args, **_kwargs: _dataset())
monkeypatch.setattr(
datasets_document_module.DatasetService, "check_dataset_permission", lambda *_args, **_kwargs: None
)
- doc1 = _mock_document(
+ doc1 = _document(
document_id="33333333-3333-3333-3333-333333333333",
tenant_id="tenant-123",
data_source_type="upload_file",
upload_file_id="file-1",
)
- doc2 = _mock_document(
+ doc2 = _document(
document_id="44444444-4444-4444-4444-444444444444",
tenant_id="tenant-123",
data_source_type="upload_file",
@@ -254,8 +283,8 @@ def test_batch_download_zip_response_is_openable_zip(
datasets_document_module.FileService,
"get_upload_files_by_ids",
lambda *_args, **_kwargs: {
- "file-1": SimpleNamespace(id="file-1", name="a.txt", key="k1"),
- "file-2": SimpleNamespace(id="file-2", name="b.txt", key="k2"),
+ "file-1": _upload_file(file_id="file-1", name="a.txt", key="k1"),
+ "file-2": _upload_file(file_id="file-2", name="b.txt", key="k2"),
},
)
@@ -274,7 +303,7 @@ def test_batch_download_zip_response_is_openable_zip(
):
api = datasets_document_module.DocumentBatchDownloadZipApi()
method = unwrap(api.post)
- response = method(api, MagicMock(), "tenant-123", _mock_user(), dataset_id="ds-1")
+ response = method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1")
# Assert: response body is a valid ZIP and contains the expected entries.
response.direct_passthrough = False
@@ -288,18 +317,16 @@ def test_batch_download_zip_response_is_openable_zip(
def test_batch_download_zip_rejects_non_upload_file_document(
- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch
+ app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
) -> None:
"""Ensure batch ZIP download rejects non upload-file documents."""
- monkeypatch.setattr(
- datasets_document_module.DatasetService, "get_dataset", lambda *_args, **_kwargs: SimpleNamespace(id="ds-1")
- )
+ monkeypatch.setattr(datasets_document_module.DatasetService, "get_dataset", lambda *_args, **_kwargs: _dataset())
monkeypatch.setattr(
datasets_document_module.DatasetService, "check_dataset_permission", lambda *_args, **_kwargs: None
)
- doc = _mock_document(
+ doc = _document(
document_id="55555555-5555-5555-5555-555555555555",
tenant_id="tenant-123",
data_source_type="website_crawl",
@@ -319,11 +346,11 @@ def test_batch_download_zip_rejects_non_upload_file_document(
api = datasets_document_module.DocumentBatchDownloadZipApi()
method = unwrap(api.post)
with pytest.raises(NotFound):
- method(api, MagicMock(), "tenant-123", _mock_user(), dataset_id="ds-1")
+ method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1")
def test_document_download_returns_url_for_upload_file_document(
- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch
+ app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
) -> None:
"""Ensure upload-file documents return a `{url}` JSON payload."""
@@ -341,13 +368,13 @@ def test_document_download_returns_url_for_upload_file_document(
with app.test_request_context("/datasets/ds-1/documents/doc-1/download", method="GET"):
api = datasets_document_module.DocumentDownloadApi()
method = unwrap(api.get)
- result = method(api, MagicMock(), "tenant-123", _mock_user(), dataset_id="ds-1", document_id="doc-1")
+ result = method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1", document_id="doc-1")
assert result == {"url": "https://example.com/signed"}
def test_document_download_rejects_non_upload_file_document(
- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch
+ app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
) -> None:
"""Ensure non-upload documents raise 404 (no file to download)."""
@@ -365,11 +392,11 @@ def test_document_download_rejects_non_upload_file_document(
api = datasets_document_module.DocumentDownloadApi()
method = unwrap(api.get)
with pytest.raises(NotFound):
- method(api, MagicMock(), "tenant-123", _mock_user(), dataset_id="ds-1", document_id="doc-1")
+ method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1", document_id="doc-1")
def test_document_download_rejects_missing_upload_file_id(
- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch
+ app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
) -> None:
"""Ensure missing `upload_file_id` raises 404."""
@@ -387,11 +414,11 @@ def test_document_download_rejects_missing_upload_file_id(
api = datasets_document_module.DocumentDownloadApi()
method = unwrap(api.get)
with pytest.raises(NotFound):
- method(api, MagicMock(), "tenant-123", _mock_user(), dataset_id="ds-1", document_id="doc-1")
+ method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1", document_id="doc-1")
def test_document_download_rejects_when_upload_file_record_missing(
- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch
+ app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
) -> None:
"""Ensure missing UploadFile row raises 404."""
@@ -409,11 +436,11 @@ def test_document_download_rejects_when_upload_file_record_missing(
api = datasets_document_module.DocumentDownloadApi()
method = unwrap(api.get)
with pytest.raises(NotFound):
- method(api, MagicMock(), "tenant-123", _mock_user(), dataset_id="ds-1", document_id="doc-1")
+ method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1", document_id="doc-1")
def test_document_download_rejects_document_owner_mismatch(
- app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch
+ app: Flask, datasets_document_module, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
) -> None:
"""Ensure an owner mismatch is rejected by the shared document resolver."""
@@ -431,4 +458,4 @@ def test_document_download_rejects_document_owner_mismatch(
api = datasets_document_module.DocumentDownloadApi()
method = unwrap(api.get)
with pytest.raises(NotFound):
- method(api, MagicMock(), "tenant-123", _mock_user(), dataset_id="ds-1", document_id="doc-1")
+ method(api, sqlite_session, "tenant-123", _account(), dataset_id="ds-1", document_id="doc-1")