Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: migrate RAG retrieval sessions and ORM models to SQLite (#40512)

Asuka Minato31dfa21f3601
diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py
index 15854a44c47..35a70103623 100644
--- a/api/core/rag/datasource/retrieval_service.py
+++ b/api/core/rag/datasource/retrieval_service.py
@@ -918,11 +918,18 @@ class RetrievalService:
def get_segment_attachment_info(
cls, dataset_id: str, tenant_id: str, attachment_id: str, session: Session
) -> SegmentAttachmentResult | None:
- upload_file = session.scalar(select(UploadFile).where(UploadFile.id == attachment_id).limit(1))
+ """Resolve an attachment only through its tenant and dataset ownership chain."""
+ upload_file = session.scalar(
+ select(UploadFile).where(UploadFile.id == attachment_id, UploadFile.tenant_id == tenant_id).limit(1)
+ )
if upload_file:
attachment_binding = session.scalar(
select(SegmentAttachmentBinding)
- .where(SegmentAttachmentBinding.attachment_id == upload_file.id)
+ .where(
+ SegmentAttachmentBinding.attachment_id == upload_file.id,
+ SegmentAttachmentBinding.tenant_id == tenant_id,
+ SegmentAttachmentBinding.dataset_id == dataset_id,
+ )
.limit(1)
)
if attachment_binding:
diff --git a/api/tests/unit_tests/core/datasource/test_datasource_manager.py b/api/tests/unit_tests/core/datasource/test_datasource_manager.py
index 5308a4c1922..5e38166a5cd 100644
--- a/api/tests/unit_tests/core/datasource/test_datasource_manager.py
+++ b/api/tests/unit_tests/core/datasource/test_datasource_manager.py
@@ -1,9 +1,9 @@
import types
-from collections.abc import Generator, Iterator
+from collections.abc import Generator
+from datetime import UTC, datetime
import pytest
from pytest_mock import MockerFixture
-from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session, sessionmaker
from contexts.wrapper import RecyclableContextVar
@@ -12,21 +12,29 @@ from core.datasource.datasource_manager import DatasourceManager
from core.datasource.entities.datasource_entities import DatasourceMessage, DatasourceProviderType
from core.datasource.errors import DatasourceProviderNotFoundError
from core.workflow.file_reference import parse_file_reference
+from extensions.storage.storage_type import StorageType
from graphon.enums import WorkflowNodeExecutionStatus
from graphon.file import File, FileTransferMethod, FileType
from graphon.node_events import StreamChunkEvent, StreamCompletedEvent
-from models.base import TypeBase
+from models.enums import CreatorUserRole
+from models.model import UploadFile
from models.tools import ToolFile
@pytest.fixture
-def tool_file_session(sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]:
- """Bind datasource-owned lookups to a SQLite ToolFile table."""
- TypeBase.metadata.create_all(sqlite_engine, tables=[TypeBase.metadata.tables[ToolFile.__tablename__]])
- session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False)
- monkeypatch.setattr(datasource_manager_module.session_factory, "create_session", session_maker)
- with session_maker() as session:
- yield session
+def datasource_session(
+ sqlite_session: Session,
+ sqlite_session_factory: sessionmaker[Session],
+ monkeypatch: pytest.MonkeyPatch,
+) -> Session:
+ """Bind datasource-owned lookups to the shared SQLite test database."""
+ monkeypatch.setattr(datasource_manager_module.session_factory, "create_session", sqlite_session_factory)
+ return sqlite_session
+
+
+def tool_file_session(datasource_session: Session) -> Session:
+ return datasource_session
def _persist_tool_file(session: Session, *, file_id: str, tenant_id: str) -> ToolFile:
@@ -45,6 +53,27 @@ def _persist_tool_file(session: Session, *, file_id: str, tenant_id: str) -> Too
return tool_file
+def _persist_upload_file(session: Session, *, file_id: str, tenant_id: str) -> UploadFile:
+ upload_file = UploadFile(
+ tenant_id=tenant_id,
+ storage_type=StorageType.LOCAL,
+ key="files/file.txt",
+ name="f",
+ size=1,
+ extension="txt",
+ mime_type="text/plain",
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by="user-1",
+ created_at=datetime.now(UTC),
+ used=True,
+ source_url="http://x",
+ )
+ upload_file.id = file_id
+ session.add(upload_file)
+ session.commit()
+ return upload_file
+
+
def _gen_messages_text_only(text: str) -> Generator[DatasourceMessage, None, None]:
yield DatasourceMessage(
type=DatasourceMessage.MessageType.TEXT,
@@ -631,52 +660,18 @@ def test_stream_node_events_skips_file_build_for_non_online_types(mocker: Mocker
assert events[-1].node_run_result.outputs["file"] is None
-def test_get_upload_file_by_id_builds_file(mocker: MockerFixture):
- # fake UploadFile row
- fake_row = types.SimpleNamespace(
- id="fid",
- name="f",
- extension="txt",
- mime_type="text/plain",
- size=1,
- key="k",
- source_url="http://x",
- )
-
- class _S:
- def __init__(self, row):
- self._row = row
-
- def __enter__(self):
- return self
-
- def __exit__(self, *exc):
- return False
-
- def scalar(self, *_args, **_kwargs):
- return self._row
-
- mocker.patch("core.datasource.datasource_manager.session_factory.create_session", return_value=_S(fake_row))
+def test_get_upload_file_by_id_builds_file(datasource_session: Session):
+ _persist_upload_file(datasource_session, file_id="fid", tenant_id="t1")
f = DatasourceManager.get_upload_file_by_id(file_id="fid", tenant_id="t1")
assert f.related_id == "fid"
assert f.extension == ".txt"
assert parse_file_reference(f.reference).storage_key is None
- assert f.storage_key == "k"
-
-
-def test_get_upload_file_by_id_raises_when_missing(mocker: MockerFixture):
- class _S:
- def __enter__(self):
- return self
-
- def __exit__(self, *exc):
- return False
+ assert f.storage_key == "files/file.txt"
- def scalar(self, *_args, **_kwargs):
- return None
- mocker.patch("core.datasource.datasource_manager.session_factory.create_session", return_value=_S())
+def test_get_upload_file_by_id_raises_when_missing(datasource_session: Session):
+ _persist_upload_file(datasource_session, file_id="fid", tenant_id="other-tenant")
with pytest.raises(ValueError, match="UploadFile not found for file_id=fid, tenant_id=t1"):
DatasourceManager.get_upload_file_by_id(file_id="fid", tenant_id="t1")
diff --git a/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py b/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py
index 589d29d8240..4c78edf6615 100644
--- a/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py
+++ b/api/tests/unit_tests/core/rag/datasource/test_datasource_retrieval.py
@@ -1,9 +1,13 @@
-from types import SimpleNamespace
+from datetime import UTC, datetime
from typing import Any
-from unittest.mock import MagicMock, Mock, call, patch
+from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
from uuid import uuid4
import pytest
+from pytest_mock import MockerFixture
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import Session
from core.rag.datasource import retrieval_service as retrieval_service_module
from core.rag.datasource.retrieval_service import RetrievalService
@@ -13,7 +17,11 @@ from core.rag.index_processor.constant.query_type import QueryType
from core.rag.models.document import Document
from core.rag.rerank.rerank_type import RerankMode
from core.rag.retrieval.retrieval_methods import RetrievalMethod
-from models.dataset import Dataset
+from extensions.storage.storage_type import StorageType
+from models.dataset import ChildChunk, Dataset, DocumentSegment, DocumentSegmentSummary, SegmentAttachmentBinding
+from models.dataset import Document as DatasetDocument
+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, SegmentStatus, SummaryStatus
+from models.model import UploadFile
def create_mock_document(
@@ -93,42 +101,74 @@ class _ImmediateExecutor:
return future
-class _FakeExecuteScalarResult:
- def __init__(self, data: list) -> None:
- self._data = data
-
- def all(self) -> list:
- return self._data
-
-
-class _FakeExecuteResult:
- def __init__(self, data: list) -> None:
- self._data = data
-
- def scalars(self) -> _FakeExecuteScalarResult:
- return _FakeExecuteScalarResult(self._data)
-
-
-class _FakeScalarsResult:
- def __init__(self, data: list) -> None:
- self._data = data
-
- def all(self) -> list:
- return self._data
+def _dataset(*, dataset_id: str = "dataset-id", tenant_id: str = "tenant-id") -> Dataset:
+ return Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ name="Dataset",
+ description="",
+ created_by="user-id",
+ is_multimodal=False,
+ chunk_structure=IndexStructureType.PARENT_CHILD_INDEX,
+ )
-class _FakeSession:
- def __init__(self, execute_payloads: list[list], scalars_payloads: list[list]) -> None:
- self._execute_payloads = list(execute_payloads)
- self._scalars_payloads = list(scalars_payloads)
+def _dataset_document(*, document_id: str, doc_form: str) -> DatasetDocument:
+ return DatasetDocument(
+ id=document_id,
+ tenant_id="tenant-id",
+ dataset_id="dataset-id",
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch="batch",
+ name=document_id,
+ created_from=DocumentCreatedFrom.API,
+ created_by="user-id",
+ doc_form=doc_form,
+ )
- def execute(self, stmt):
- data = self._execute_payloads.pop(0) if self._execute_payloads else []
- return _FakeExecuteResult(data)
- def scalars(self, stmt):
- data = self._scalars_payloads.pop(0) if self._scalars_payloads else []
- return _FakeScalarsResult(data)
+def _segment(*, segment_id: str, document_id: str, index_node_id: str) -> DocumentSegment:
+ segment = DocumentSegment(
+ tenant_id="tenant-id",
+ dataset_id="dataset-id",
+ document_id=document_id,
+ position=1,
+ content=f"content-{segment_id}",
+ word_count=1,
+ tokens=1,
+ created_by="user-id",
+ index_node_id=index_node_id,
+ status=SegmentStatus.COMPLETED,
+ )
+ segment.id = segment_id
+ return segment
+
+
+def _upload_file(
+ *,
+ file_id: str,
+ tenant_id: str = "tenant-id",
+ name: str | None = None,
+ extension: str = "png",
+ mime_type: str | None = None,
+ size: int = 42,
+) -> UploadFile:
+ upload = UploadFile(
+ tenant_id=tenant_id,
+ storage_type=StorageType.LOCAL,
+ key=f"files/{file_id}",
+ name=name or file_id,
+ size=size,
+ extension=extension,
+ mime_type=mime_type or f"image/{extension}",
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by="user-id",
+ created_at=datetime.now(UTC),
+ used=True,
+ )
+ upload.id = file_id
+ return upload
class _SimpleRetrievalChildChunk:
@@ -158,13 +198,7 @@ class _SimpleRetrievalSegment:
class TestRetrievalServiceInternals:
@pytest.fixture
def internal_dataset(self) -> Dataset:
- dataset = Dataset(
- id="dataset-id",
- tenant_id="tenant-id",
- is_multimodal=False,
- chunk_structure=IndexStructureType.PARENT_CHILD_INDEX,
- )
- return dataset
+ return _dataset()
@pytest.fixture
def internal_flask_app(self):
@@ -174,15 +208,11 @@ class TestRetrievalServiceInternals:
return app
@pytest.fixture
- def vector_session(self, monkeypatch: pytest.MonkeyPatch) -> MagicMock:
- session = MagicMock()
- session_context = MagicMock()
- session_context.__enter__.return_value = session
- session_class = MagicMock(return_value=session_context)
- session.context = session_context
- monkeypatch.setattr(retrieval_service_module, "Session", session_class)
- monkeypatch.setattr(retrieval_service_module, "db", SimpleNamespace(engine=Mock()))
- return session
+ def vector_engine(self, mocker: MockerFixture, sqlite_engine: Engine) -> Engine:
+ mocker.patch.object(
+ type(retrieval_service_module.db), "engine", new_callable=PropertyMock, return_value=sqlite_engine
+ )
+ return sqlite_engine
def test_retrieve_with_attachment_ids_only(self, monkeypatch: pytest.MonkeyPatch, internal_dataset):
with (
@@ -229,15 +259,15 @@ class TestRetrievalServiceInternals:
@patch("core.rag.datasource.retrieval_service.ExternalDatasetService.fetch_external_knowledge_retrieval")
@patch("core.rag.datasource.retrieval_service.MetadataFilteringCondition.model_validate")
- def test_external_retrieve_with_metadata_conditions(self, mock_validate, mock_fetch):
+ def test_external_retrieve_with_metadata_conditions(self, mock_validate, mock_fetch, sqlite_session: Session):
mock_validate.return_value = "validated-condition"
expected_documents = [create_mock_document("external-doc", "external-1", 0.8, provider="external")]
mock_fetch.return_value = expected_documents
- session = MagicMock()
- session.scalar.return_value = SimpleNamespace(tenant_id="tenant-1")
+ sqlite_session.add(_dataset(dataset_id="dataset-1", tenant_id="tenant-1"))
+ sqlite_session.commit()
results = RetrievalService.external_retrieve(
- session=session,
+ session=sqlite_session,
dataset_id="dataset-1",
query="test query",
external_retrieval_model={"top_k": 3},
@@ -252,61 +282,54 @@ class TestRetrievalServiceInternals:
query="test query",
external_retrieval_parameters={"top_k": 3},
metadata_condition="validated-condition",
- session=session,
+ session=sqlite_session,
)
- def test_external_retrieve_returns_empty_when_dataset_not_found(self):
- session = MagicMock()
- session.scalar.return_value = None
-
- results = RetrievalService.external_retrieve(session=session, dataset_id="missing", query="q")
+ def test_external_retrieve_returns_empty_when_dataset_not_found(self, sqlite_session: Session):
+ results = RetrievalService.external_retrieve(session=sqlite_session, dataset_id="missing", query="q")
assert results == []
- @patch("core.rag.datasource.retrieval_service.Session")
- def test_get_dataset_queries_by_id(self, mock_session_class):
- expected_dataset = Dataset()
- mock_session = Mock()
- mock_session.scalar.return_value = expected_dataset
- mock_session_class.return_value.__enter__.return_value = mock_session
+ def test_get_dataset_queries_by_id(self, mocker: MockerFixture, sqlite_session: Session, sqlite_engine: Engine):
+ expected_dataset = _dataset(dataset_id="dataset-123")
+ sqlite_session.add(expected_dataset)
+ sqlite_session.commit()
+ mocker.patch.object(
+ type(retrieval_service_module.db), "engine", new_callable=PropertyMock, return_value=sqlite_engine
+ )
- with patch.object(retrieval_service_module, "db", SimpleNamespace(engine=Mock())):
- result = RetrievalService._get_dataset("dataset-123")
+ result = RetrievalService._get_dataset("dataset-123")
- assert result == expected_dataset
- mock_session.scalar.assert_called_once()
+ assert result is not None
+ assert result.id == expected_dataset.id
+ assert result.tenant_id == expected_dataset.tenant_id
@patch("core.rag.datasource.retrieval_service.Keyword")
@patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset")
- def test_keyword_search_success(self, mock_get_dataset, mock_keyword_class, internal_dataset, internal_flask_app):
+ def test_keyword_search_success(
+ self, mock_get_dataset, mock_keyword_class, internal_dataset, internal_flask_app, vector_engine
+ ):
mock_get_dataset.return_value = internal_dataset
keyword_instance = Mock()
keyword_instance.search.return_value = [create_mock_document("keyword-content", "kw-1", 0.91)]
mock_keyword_class.return_value = keyword_instance
all_documents: list[Document] = []
exceptions: list[str] = []
- session = MagicMock()
- engine = Mock()
-
- with (
- patch.object(retrieval_service_module, "db", SimpleNamespace(engine=engine)),
- patch("core.rag.datasource.retrieval_service.Session") as session_class,
- ):
- session_class.return_value.__enter__.return_value = session
- RetrievalService.keyword_search(
- flask_app=internal_flask_app,
- dataset_id=internal_dataset.id,
- query='query "with quotes"',
- top_k=5,
- all_documents=all_documents,
- exceptions=exceptions,
- )
+ RetrievalService.keyword_search(
+ flask_app=internal_flask_app,
+ dataset_id=internal_dataset.id,
+ query='query "with quotes"',
+ top_k=5,
+ all_documents=all_documents,
+ exceptions=exceptions,
+ )
assert len(all_documents) == 1
assert exceptions == []
keyword_instance.search.assert_called_once()
- assert keyword_instance.search.call_args.kwargs["session"] is session
- session_class.assert_called_once_with(engine)
+ used_session = keyword_instance.search.call_args.kwargs["session"]
+ assert isinstance(used_session, Session)
+ assert used_session.bind is vector_engine
@patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset")
def test_keyword_search_appends_exception_when_dataset_missing(self, mock_get_dataset, internal_flask_app):
@@ -329,7 +352,7 @@ class TestRetrievalServiceInternals:
@patch("core.rag.datasource.retrieval_service.Keyword")
@patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset")
def test_keyword_search_appends_exception_when_search_fails(
- self, mock_get_dataset, mock_keyword_class, internal_dataset, internal_flask_app
+ self, mock_get_dataset, mock_keyword_class, internal_dataset, internal_flask_app, vector_engine
):
mock_get_dataset.return_value = internal_dataset
keyword_instance = Mock()
@@ -337,30 +360,25 @@ class TestRetrievalServiceInternals:
mock_keyword_class.return_value = keyword_instance
all_documents: list[Document] = []
exceptions: list[str] = []
- session = MagicMock()
-
- with (
- patch.object(retrieval_service_module, "db", SimpleNamespace(engine=Mock())),
- patch("core.rag.datasource.retrieval_service.Session") as session_class,
- ):
- session_class.return_value.__enter__.return_value = session
- RetrievalService.keyword_search(
- flask_app=internal_flask_app,
- dataset_id=internal_dataset.id,
- query="query",
- top_k=2,
- all_documents=all_documents,
- exceptions=exceptions,
- )
+ RetrievalService.keyword_search(
+ flask_app=internal_flask_app,
+ dataset_id=internal_dataset.id,
+ query="query",
+ top_k=2,
+ all_documents=all_documents,
+ exceptions=exceptions,
+ )
assert all_documents == []
assert exceptions == ["keyword failed"]
- assert keyword_instance.search.call_args.kwargs["session"] is session
+ used_session = keyword_instance.search.call_args.kwargs["session"]
+ assert isinstance(used_session, Session)
+ assert used_session.bind is vector_engine
@patch("core.rag.datasource.retrieval_service.Vector")
@patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset")
def test_embedding_search_text_without_reranking(
- self, mock_get_dataset, mock_vector_class, internal_dataset, internal_flask_app, vector_session
+ self, mock_get_dataset, mock_vector_class, internal_dataset, internal_flask_app, vector_engine
):
internal_dataset.is_multimodal = False
mock_get_dataset.return_value = internal_dataset
@@ -386,13 +404,15 @@ class TestRetrievalServiceInternals:
assert len(all_documents) == 1
assert exceptions == []
- mock_vector_class.assert_called_once_with(dataset=internal_dataset, session=vector_session)
+ used_session = mock_vector_class.call_args.kwargs["session"]
+ assert isinstance(used_session, Session)
+ assert used_session.bind is vector_engine
vector_instance.search_by_vector.assert_called_once()
@patch("core.rag.datasource.retrieval_service.Vector")
@patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset")
def test_embedding_search_image_non_multimodal_returns_early(
- self, mock_get_dataset, mock_vector_class, internal_dataset, internal_flask_app, vector_session
+ self, mock_get_dataset, mock_vector_class, internal_dataset, internal_flask_app, vector_engine
):
internal_dataset.is_multimodal = False
mock_get_dataset.return_value = internal_dataset
@@ -430,7 +450,7 @@ class TestRetrievalServiceInternals:
mock_model_manager_class,
internal_dataset,
internal_flask_app,
- vector_session,
+ vector_engine,
):
internal_dataset.is_multimodal = True
mock_get_dataset.return_value = internal_dataset
@@ -440,7 +460,6 @@ class TestRetrievalServiceInternals:
vector_instance = Mock()
def search_by_file(**_kwargs):
- assert vector_session.context.__exit__.call_count == 0
return original_docs
vector_instance.search_by_file.side_effect = search_by_file
@@ -491,7 +510,7 @@ class TestRetrievalServiceInternals:
mock_model_manager_class,
internal_dataset,
internal_flask_app,
- vector_session,
+ vector_engine,
):
internal_dataset.is_multimodal = True
mock_get_dataset.return_value = internal_dataset
@@ -543,7 +562,7 @@ class TestRetrievalServiceInternals:
mock_processor_class,
internal_dataset,
internal_flask_app,
- vector_session,
+ vector_engine,
):
internal_dataset.is_multimodal = False
mock_get_dataset.return_value = internal_dataset
@@ -584,7 +603,7 @@ class TestRetrievalServiceInternals:
@patch("core.rag.datasource.retrieval_service.Vector")
@patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset")
def test_embedding_search_appends_exception_when_vector_fails(
- self, mock_get_dataset, mock_vector_class, internal_dataset, internal_flask_app, vector_session
+ self, mock_get_dataset, mock_vector_class, internal_dataset, internal_flask_app, vector_engine
):
mock_get_dataset.return_value = internal_dataset
vector_instance = Mock()
@@ -612,7 +631,7 @@ class TestRetrievalServiceInternals:
@patch("core.rag.datasource.retrieval_service.Vector")
@patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset")
def test_full_text_index_search_without_reranking(
- self, mock_get_dataset, mock_vector_class, internal_dataset, internal_flask_app, vector_session
+ self, mock_get_dataset, mock_vector_class, internal_dataset, internal_flask_app, vector_engine
):
mock_get_dataset.return_value = internal_dataset
vector_instance = Mock()
@@ -647,7 +666,7 @@ class TestRetrievalServiceInternals:
mock_processor_class,
internal_dataset,
internal_flask_app,
- vector_session,
+ vector_engine,
):
mock_get_dataset.return_value = internal_dataset
original_docs = [create_mock_document("fulltext", "ft-1", 0.68)]
@@ -707,7 +726,7 @@ class TestRetrievalServiceInternals:
@patch("core.rag.datasource.retrieval_service.Vector")
@patch("core.rag.datasource.retrieval_service.RetrievalService._get_dataset")
def test_full_text_index_search_appends_exception_when_search_fails(
- self, mock_get_dataset, mock_vector_class, internal_dataset, internal_flask_app, vector_session
+ self, mock_get_dataset, mock_vector_class, internal_dataset, internal_flask_app, vector_engine
):
mock_get_dataset.return_value = internal_dataset
vector_instance = Mock()
@@ -731,27 +750,21 @@ class TestRetrievalServiceInternals:
assert all_documents == []
assert exceptions == ["fulltext failed"]
- def test_format_retrieval_documents_with_empty_input_returns_empty_list(self):
- assert RetrievalService.format_retrieval_documents(MagicMock(), []) == []
+ def test_format_retrieval_documents_with_empty_input_returns_empty_list(self, unbound_session: Session):
+ assert RetrievalService.format_retrieval_documents(unbound_session, []) == []
- def test_format_retrieval_documents_without_document_id_returns_empty_list(self):
+ def test_format_retrieval_documents_without_document_id_returns_empty_list(self, unbound_session: Session):
documents = [Document(page_content="content", metadata={"doc_id": "doc-1", "score": 0.4}, provider="dify")]
- assert RetrievalService.format_retrieval_documents(MagicMock(), documents) == []
+ assert RetrievalService.format_retrieval_documents(unbound_session, documents) == []
def test_format_retrieval_documents_with_parent_child_summary_and_attachments(
- self, monkeypatch: pytest.MonkeyPatch
+ self, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session
):
- dataset_doc_parent = SimpleNamespace(
- id="doc-parent",
- doc_form=IndexStructureType.PARENT_CHILD_INDEX,
- dataset_id="dataset-id",
- )
- dataset_doc_text = SimpleNamespace(id="doc-text", doc_form="paragraph", dataset_id="dataset-id")
- dataset_doc_parent_summary = SimpleNamespace(
- id="doc-parent-summary",
- doc_form=IndexStructureType.PARENT_CHILD_INDEX,
- dataset_id="dataset-id",
+ dataset_doc_parent = _dataset_document(document_id="doc-parent", doc_form=IndexStructureType.PARENT_CHILD_INDEX)
+ dataset_doc_text = _dataset_document(document_id="doc-text", doc_form=IndexStructureType.PARAGRAPH_INDEX)
+ dataset_doc_parent_summary = _dataset_document(
+ document_id="doc-parent-summary", doc_form=IndexStructureType.PARENT_CHILD_INDEX
)
monkeypatch.setattr(retrieval_service_module, "RetrievalChildChunk", _SimpleRetrievalChildChunk)
@@ -834,37 +847,55 @@ class TestRetrievalServiceInternals:
),
]
- child_chunk = SimpleNamespace(
- id="child-chunk-1",
+ child_chunk = ChildChunk(
+ tenant_id="tenant-id",
+ dataset_id="dataset-id",
+ document_id="doc-parent",
segment_id="segment-parent",
- index_node_id="child-node-1",
- content="child details",
position=2,
+ content="child details",
+ word_count=2,
+ created_by="user-id",
+ index_node_id="child-node-1",
)
- segment_parent = SimpleNamespace(id="segment-parent", document_id="doc-parent", index_node_id="parent-node")
- segment_text = SimpleNamespace(id="segment-text", document_id="doc-text", index_node_id="index-node-1")
- segment_summary = SimpleNamespace(id="segment-summary", document_id="doc-text", index_node_id="summary-node")
- segment_parent_summary = SimpleNamespace(
- id="segment-parent-summary",
+ child_chunk.id = "child-chunk-1"
+ segment_parent = _segment(segment_id="segment-parent", document_id="doc-parent", index_node_id="parent-node")
+ segment_text = _segment(segment_id="segment-text", document_id="doc-text", index_node_id="index-node-1")
+ segment_summary = _segment(segment_id="segment-summary", document_id="doc-text", index_node_id="summary-node")
+ segment_parent_summary = _segment(
+ segment_id="segment-parent-summary",
document_id="doc-parent-summary",
index_node_id="summary-parent-node",
)
-
- fake_session = _FakeSession(
- execute_payloads=[
- [child_chunk],
- [segment_text],
- [segment_parent, segment_text],
- [segment_summary, segment_parent_summary],
- ],
- scalars_payloads=[
- [dataset_doc_parent, dataset_doc_text, dataset_doc_parent_summary],
- [
- SimpleNamespace(chunk_id="segment-summary", summary_content="summary for text"),
- SimpleNamespace(chunk_id="segment-parent-summary", summary_content="summary for parent"),
- ],
- ],
+ text_summary = DocumentSegmentSummary(
+ dataset_id="dataset-id",
+ document_id="doc-text",
+ chunk_id="segment-summary",
+ summary_content="summary for text",
+ status=SummaryStatus.COMPLETED,
+ )
+ parent_summary = DocumentSegmentSummary(
+ dataset_id="dataset-id",
+ document_id="doc-parent-summary",
+ chunk_id="segment-parent-summary",
+ summary_content="summary for parent",
+ status=SummaryStatus.COMPLETED,
+ )
+ sqlite_session.add_all(
+ [
+ dataset_doc_parent,
+ dataset_doc_text,
+ dataset_doc_parent_summary,
+ child_chunk,
+ segment_parent,
+ segment_text,
+ segment_summary,
+ segment_parent_summary,
+ text_summary,
+ parent_summary,
+ ]
)
+ sqlite_session.commit()
monkeypatch.setattr(
RetrievalService,
"get_segment_attachment_infos",
@@ -896,7 +927,7 @@ class TestRetrievalServiceInternals:
],
)
- result = RetrievalService.format_retrieval_documents(fake_session, input_documents)
+ result = RetrievalService.format_retrieval_documents(sqlite_session, input_documents)
assert len(result) == 4
result_by_segment_id = {item.segment.id: item for item in result}
@@ -910,16 +941,17 @@ class TestRetrievalServiceInternals:
assert result_by_segment_id["segment-parent-summary"].summary == "summary for parent"
assert result_by_segment_id["segment-parent-summary"].child_chunks == []
- def test_format_retrieval_documents_rolls_back_and_raises_when_db_fails(self):
- session = MagicMock()
- session.scalars.side_effect = RuntimeError("db error")
-
+ def test_format_retrieval_documents_rolls_back_and_raises_when_db_fails(
+ self, mocker: MockerFixture, sqlite_session: Session
+ ):
documents = [Document(page_content="content", metadata={"document_id": "doc-1"}, provider="dify")]
+ sqlite_session.connection().invalidate()
+ rollback = mocker.spy(sqlite_session, "rollback")
- with pytest.raises(RuntimeError, match="db error"):
- RetrievalService.format_retrieval_documents(session, documents)
+ with pytest.raises(SQLAlchemyError):
+ RetrievalService.format_retrieval_documents(sqlite_session, documents)
- session.rollback.assert_called_once()
+ rollback.assert_called_once()
def test_retrieve_internal_returns_early_without_query_or_attachment(self, internal_dataset, internal_flask_app):
all_documents: list[Document] = []
@@ -991,7 +1023,7 @@ class TestRetrievalServiceInternals:
)
def test_retrieve_internal_hybrid_weighted_attachment_flow(
- self, monkeypatch: pytest.MonkeyPatch, internal_dataset, internal_flask_app, vector_session
+ self, monkeypatch: pytest.MonkeyPatch, internal_dataset, internal_flask_app, vector_engine
):
executor = _ImmediateExecutor()
monkeypatch.setattr(retrieval_service_module, "ThreadPoolExecutor", lambda *args, **kwargs: executor)
@@ -1069,19 +1101,19 @@ class TestRetrievalServiceInternals:
processor_instance.invoke.assert_called_once()
@patch("core.rag.datasource.retrieval_service.sign_upload_file_preview_url", return_value="signed://file")
- def test_get_segment_attachment_info_success(self, mock_sign):
- upload_file = SimpleNamespace(
- id="upload-1",
- name="file-name",
- extension="png",
- mime_type="image/png",
- size=42,
+ def test_get_segment_attachment_info_success(self, mock_sign, sqlite_session: Session):
+ upload_file = _upload_file(file_id="upload-1", name="file-name")
+ binding = SegmentAttachmentBinding(
+ tenant_id="tenant-id",
+ dataset_id="dataset-id",
+ document_id="document-1",
+ segment_id="segment-1",
+ attachment_id="upload-1",
)
- binding = SimpleNamespace(segment_id="segment-1", attachment_id="upload-1")
- session = Mock()
- session.scalar.side_effect = [upload_file, binding]
+ sqlite_session.add_all([upload_file, binding])
+ sqlite_session.commit()
- result = RetrievalService.get_segment_attachment_info("dataset-id", "tenant-id", "upload-1", session)
+ result = RetrievalService.get_segment_attachment_info("dataset-id", "tenant-id", "upload-1", sqlite_session)
assert result == {
"attachment_info": {
@@ -1096,84 +1128,79 @@ class TestRetrievalServiceInternals:
}
mock_sign.assert_called_once_with("upload-1", "png")
- def test_get_segment_attachment_info_returns_none_when_binding_missing(self):
- upload_file = SimpleNamespace(
- id="upload-1",
- name="file-name",
- extension="png",
- mime_type="image/png",
- size=42,
- )
- session = Mock()
- session.scalar.side_effect = [upload_file, None]
+ def test_get_segment_attachment_info_returns_none_when_binding_missing(self, sqlite_session: Session):
+ sqlite_session.add(_upload_file(file_id="upload-1", name="file-name"))
+ sqlite_session.commit()
- result = RetrievalService.get_segment_attachment_info("dataset-id", "tenant-id", "upload-1", session)
+ result = RetrievalService.get_segment_attachment_info("dataset-id", "tenant-id", "upload-1", sqlite_session)
assert result is None
- def test_get_segment_attachment_info_returns_none_when_upload_file_missing(self):
- session = Mock()
- session.scalar.return_value = None
+ @pytest.mark.parametrize(
+ ("upload_tenant_id", "binding_tenant_id", "binding_dataset_id"),
+ [
+ pytest.param("other-tenant", "other-tenant", "dataset-id", id="foreign-upload"),
+ pytest.param("tenant-id", "other-tenant", "dataset-id", id="foreign-binding-tenant"),
+ pytest.param("tenant-id", "tenant-id", "other-dataset", id="foreign-binding-dataset"),
+ ],
+ )
+ def test_get_segment_attachment_info_rejects_foreign_owner_chain(
+ self,
+ sqlite_session: Session,
+ upload_tenant_id: str,
+ binding_tenant_id: str,
+ binding_dataset_id: str,
+ ):
+ upload_file = _upload_file(file_id="upload-1", tenant_id=upload_tenant_id)
+ binding = SegmentAttachmentBinding(
+ tenant_id=binding_tenant_id,
+ dataset_id=binding_dataset_id,
+ document_id="document-1",
+ segment_id="segment-1",
+ attachment_id="upload-1",
+ )
+ sqlite_session.add_all([upload_file, binding])
+ sqlite_session.commit()
- result = RetrievalService.get_segment_attachment_info("dataset-id", "tenant-id", "upload-1", session)
+ result = RetrievalService.get_segment_attachment_info("dataset-id", "tenant-id", "upload-1", sqlite_session)
assert result is None
- def test_get_segment_attachment_infos_returns_empty_when_upload_files_missing(self):
- scalars_result = Mock()
- scalars_result.all.return_value = []
- session = Mock()
- session.scalars.return_value = scalars_result
+ def test_get_segment_attachment_info_returns_none_when_upload_file_missing(self, sqlite_session: Session):
+ result = RetrievalService.get_segment_attachment_info("dataset-id", "tenant-id", "upload-1", sqlite_session)
+
+ assert result is None
- result = RetrievalService.get_segment_attachment_infos(["upload-1"], session)
+ def test_get_segment_attachment_infos_returns_empty_when_upload_files_missing(self, sqlite_session: Session):
+ result = RetrievalService.get_segment_attachment_infos(["upload-1"], sqlite_session)
assert result == []
- def test_get_segment_attachment_infos_returns_empty_when_bindings_missing(self):
- upload_file = SimpleNamespace(
- id="upload-1",
- name="file-name",
- extension="png",
- mime_type="image/png",
- size=42,
- )
- upload_scalars = Mock()
- upload_scalars.all.return_value = [upload_file]
- binding_scalars = Mock()
- binding_scalars.all.return_value = []
- session = Mock()
- session.scalars.side_effect = [upload_scalars, binding_scalars]
+ def test_get_segment_attachment_infos_returns_empty_when_bindings_missing(self, sqlite_session: Session):
+ sqlite_session.add(_upload_file(file_id="upload-1", name="file-name"))
+ sqlite_session.commit()
- result = RetrievalService.get_segment_attachment_infos(["upload-1"], session)
+ result = RetrievalService.get_segment_attachment_infos(["upload-1"], sqlite_session)
assert result == []
@patch("core.rag.datasource.retrieval_service.sign_upload_file_preview_url", return_value="signed://file")
- def test_get_segment_attachment_infos_success(self, mock_sign):
- upload_file_1 = SimpleNamespace(
- id="upload-1",
- name="file-1",
- extension="png",
- mime_type="image/png",
- size=42,
+ def test_get_segment_attachment_infos_success(self, mock_sign, sqlite_session: Session):
+ upload_file_1 = _upload_file(file_id="upload-1", name="file-1")
+ upload_file_2 = _upload_file(
+ file_id="upload-2", name="file-2", extension="jpg", mime_type="image/jpeg", size=99
)
- upload_file_2 = SimpleNamespace(
- id="upload-2",
- name="file-2",
- extension="jpg",
- mime_type="image/jpeg",
- size=99,
+ binding = SegmentAttachmentBinding(
+ tenant_id="tenant-id",
+ dataset_id="dataset-id",
+ document_id="document-1",
+ segment_id="segment-1",
+ attachment_id="upload-1",
)
- binding = SimpleNamespace(attachment_id="upload-1", segment_id="segment-1")
-
- upload_scalars = Mock()
- upload_scalars.all.return_value = [upload_file_1, upload_file_2]
- binding_scalars = Mock()
- binding_scalars.all.return_value = [binding]
- session = Mock()
- session.scalars.side_effect = [upload_scalars, binding_scalars]
+ sqlite_session.add_all([upload_file_1, upload_file_2, binding])
+ sqlite_session.commit()
- result = RetrievalService.get_segment_attachment_infos(["upload-1", "upload-2"], session)
+ result = RetrievalService.get_segment_attachment_infos(["upload-1", "upload-2"], sqlite_session)
assert result == [
{
diff --git a/api/tests/unit_tests/core/rag/datasource/test_retrieval_attachment_access.py b/api/tests/unit_tests/core/rag/datasource/test_retrieval_attachment_access.py
index aa6dd66975c..6ebfbf4c053 100644
--- a/api/tests/unit_tests/core/rag/datasource/test_retrieval_attachment_access.py
+++ b/api/tests/unit_tests/core/rag/datasource/test_retrieval_attachment_access.py
@@ -5,7 +5,6 @@ from uuid import uuid4
import pytest
from sqlalchemy import select
-from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session, sessionmaker
from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
@@ -104,8 +103,8 @@ def test_segment_attachment_lookup_grants_returned_upload_files_to_current_scope
@pytest.mark.parametrize("sqlite_session", [(Dataset, DatasetDocument, DocumentSegment)], indirect=True)
def test_knowledge_retrieval_grants_returned_segments_to_current_scope(
monkeypatch: pytest.MonkeyPatch,
- sqlite_engine: Engine,
sqlite_session: Session,
+ sqlite_session_factory: sessionmaker[Session],
) -> None:
tenant_id = str(uuid4())
dataset_id = str(uuid4())
@@ -153,8 +152,7 @@ def test_knowledge_retrieval_grants_returned_segments_to_current_scope(
lambda **kwargs: [RagDocument(page_content="segment content", provider="dify")],
)
monkeypatch.setattr(RetrievalService, "format_retrieval_documents", lambda _session, documents: [record])
- factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False)
- monkeypatch.setattr(dataset_retrieval_module.session_factory, "create_session", factory)
+ monkeypatch.setattr(dataset_retrieval_module.session_factory, "create_session", sqlite_session_factory)
scope = FileAccessScope(
tenant_id=tenant_id,
user_id=str(uuid4()),
diff --git a/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py b/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py
index dd78ae7fd14..bac97d01430 100644
--- a/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py
+++ b/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py
@@ -1,4 +1,5 @@
import base64
+import json
import sys
import types
from datetime import UTC, datetime
@@ -11,12 +12,29 @@ from sqlalchemy.orm import Session
from core.rag.models.document import Document
from extensions.storage.storage_type import StorageType
-from models.dataset import Whitelist
+from models.dataset import Dataset, Whitelist
from models.enums import CreatorUserRole
from models.model import UploadFile
from tests.unit_tests.config_override import apply_config_overrides
+def _dataset(**overrides) -> Dataset:
+ values = {
+ "id": "dataset-1",
+ "tenant_id": "tenant-1",
+ "name": "Dataset",
+ "description": "",
+ "created_by": "user-1",
+ "embedding_model_provider": "openai",
+ "embedding_model": "text-embedding-3-small",
+ }
+ index_struct_dict = overrides.pop("index_struct_dict", ...)
+ if index_struct_dict is not ...:
+ overrides["index_struct"] = json.dumps(index_struct_dict) if index_struct_dict is not None else None
+ values.update(overrides)
+ return Dataset(**values)
+
+
def _register_fake_factory_module(monkeypatch: pytest.MonkeyPatch, module_path: str, class_name: str):
fake_module = types.ModuleType(module_path)
fake_cls = type(class_name, (), {})
@@ -154,7 +172,7 @@ def test_get_vector_factory_entry_point_overrides_builtin(vector_factory_module,
def test_vector_init_uses_default_and_custom_attributes(vector_factory_module, unbound_session: Session):
- dataset = SimpleNamespace(id="dataset-1")
+ dataset = _dataset()
with patch.object(vector_factory_module.Vector, "_init_vector", return_value="processor") as init_vector:
default_vector = vector_factory_module.Vector(dataset, session=unbound_session)
@@ -193,11 +211,7 @@ def test_lazy_embeddings_defer_real_load_until_first_embed_call(vector_factory_m
for_tenant_mock = MagicMock(side_effect=AssertionError("ModelManager.for_tenant must not be called eagerly"))
monkeypatch.setattr(vector_factory_module.ModelManager, "for_tenant", for_tenant_mock)
- dataset = SimpleNamespace(
- tenant_id="tenant-1",
- embedding_model_provider="openai",
- embedding_model="text-embedding-3-small",
- )
+ dataset = _dataset()
proxy = vector_factory_module._LazyEmbeddings(dataset)
@@ -244,9 +258,7 @@ def test_init_vector_prefers_dataset_index_struct(
)
vector = vector_factory_module.Vector.__new__(vector_factory_module.Vector)
- vector._dataset = SimpleNamespace(
- index_struct_dict={"type": vector_factory_module.VectorType.UPSTASH}, tenant_id="tenant-1"
- )
+ vector._dataset = _dataset(index_struct_dict={"type": vector_factory_module.VectorType.UPSTASH})
vector._attributes = ["doc_id"]
vector._embeddings = "embeddings"
@@ -281,7 +293,7 @@ def test_init_vector_uses_whitelist_override(
)
vector = vector_factory_module.Vector.__new__(vector_factory_module.Vector)
- vector._dataset = SimpleNamespace(index_struct_dict=None, tenant_id=tenant_id)
+ vector._dataset = _dataset(tenant_id=tenant_id, index_struct_dict=None)
vector._attributes = ["doc_id"]
vector._embeddings = "embeddings"
@@ -297,7 +309,7 @@ def test_init_vector_raises_when_vector_store_missing(
apply_config_overrides(monkeypatch, VECTOR_STORE=None, VECTOR_STORE_WHITELIST_ENABLE=False)
vector = vector_factory_module.Vector.__new__(vector_factory_module.Vector)
- vector._dataset = SimpleNamespace(index_struct_dict=None, tenant_id="tenant-1")
+ vector._dataset = _dataset(index_struct_dict=None)
vector._attributes = []
vector._embeddings = "embeddings"
@@ -563,11 +575,7 @@ def test_get_embeddings_builds_cache_embedding(vector_factory_module, monkeypatc
monkeypatch.setattr(vector_factory_module, "CacheEmbedding", MagicMock(return_value="cached-embedding"))
vector = vector_factory_module.Vector.__new__(vector_factory_module.Vector)
- vector._dataset = SimpleNamespace(
- tenant_id="tenant-1",
- embedding_model_provider="openai",
- embedding_model="text-embedding-3-small",
- )
+ vector._dataset = _dataset()
result = vector._get_embeddings()
diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_methods.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_methods.py
index 6768651d6df..4bf883cba6d 100644
--- a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_methods.py
+++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval_methods.py
@@ -5,7 +5,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
-from sqlalchemy import Engine, select
+from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from core.rag.index_processor.constant.index_type import IndexTechniqueType
@@ -25,15 +25,12 @@ class RetrievalDatabase:
@pytest.fixture
-def retrieval_database(sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch) -> RetrievalDatabase:
+def retrieval_database(
+ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch
+) -> RetrievalDatabase:
"""Bind every retrieval-owned session to a disposable SQLite database."""
- Dataset.metadata.create_all(
- sqlite_engine,
- tables=[Dataset.__table__, DatasetDocument.__table__, DocumentSegment.__table__, RateLimitLog.__table__],
- )
- session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False)
- monkeypatch.setattr(retrieval_module.session_factory, "create_session", session_maker)
- return RetrievalDatabase(session_maker=session_maker)
+ monkeypatch.setattr(retrieval_module.session_factory, "create_session", sqlite_session_factory)
+ return RetrievalDatabase(session_maker=sqlite_session_factory)
def _persist_dataset(