Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: migrate RAG processor residuals to SQLite ORM models (#40608)

Asuka Minatoedb32fba9223
diff --git a/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.py b/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.py
index 40c14147f53..692a9b80972 100644
--- a/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.py
+++ b/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.py
@@ -17,8 +17,9 @@ from extensions.storage.storage_type import StorageType
from graphon.model_runtime.entities.llm_entities import LLMResult, LLMUsage
from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, ImagePromptMessageContent
from graphon.model_runtime.entities.model_entities import ModelFeature, ModelType
-from models.dataset import DocumentSegment, SegmentAttachmentBinding
-from models.enums import CreatorUserRole
+from models.dataset import Dataset, DocumentCreatedFrom, DocumentSegment, SegmentAttachmentBinding
+from models.dataset import Document as DatasetDocument
+from models.enums import CreatorUserRole, DataSourceType
from models.model import UploadFile
@@ -36,19 +37,34 @@ class TestParagraphIndexProcessor:
return ParagraphIndexProcessor()
@pytest.fixture
- def dataset(self) -> Mock:
- dataset = Mock()
- dataset.id = "dataset-1"
- dataset.tenant_id = "tenant-1"
- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
- dataset.is_multimodal = True
+ def dataset(self) -> Dataset:
+ dataset = Dataset(
+ id="dataset-1",
+ tenant_id="tenant-1",
+ name="Dataset",
+ created_by="user-1",
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ is_multimodal=True,
+ )
+ self.session.add(dataset)
+ self.session.flush()
return dataset
@pytest.fixture
- def dataset_document(self) -> Mock:
- document = Mock()
- document.id = "doc-1"
- document.created_by = "user-1"
+ def dataset_document(self, dataset: Dataset) -> DatasetDocument:
+ document = DatasetDocument(
+ id="doc-1",
+ tenant_id=dataset.tenant_id,
+ dataset_id=dataset.id,
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch="batch-1",
+ name="Document",
+ created_from=DocumentCreatedFrom.API,
+ created_by="user-1",
+ )
+ self.session.add(document)
+ self.session.flush()
return document
@pytest.fixture
@@ -209,7 +225,7 @@ class TestParagraphIndexProcessor:
assert mock_validate.call_count == 1
def test_load_creates_vector_and_multimodal_when_high_quality(
- self, processor: ParagraphIndexProcessor, dataset: Mock
+ self, processor: ParagraphIndexProcessor, dataset: Dataset
) -> None:
docs = [Document(page_content="chunk", metadata={})]
multimodal_docs = [AttachmentDocument(page_content="image", metadata={})]
@@ -227,7 +243,7 @@ class TestParagraphIndexProcessor:
mock_keyword_cls.assert_not_called()
def test_load_uses_keyword_add_texts_with_keywords_when_economy(
- self, processor: ParagraphIndexProcessor, dataset: Mock
+ self, processor: ParagraphIndexProcessor, dataset: Dataset
) -> None:
dataset.indexing_technique = IndexTechniqueType.ECONOMY
docs = [Document(page_content="chunk", metadata={})]
@@ -240,7 +256,7 @@ class TestParagraphIndexProcessor:
mock_keyword_cls.return_value.add_texts.assert_called_once_with(docs, session, keywords_list=keywords_list)
def test_load_uses_keyword_add_texts_without_keywords_when_economy(
- self, processor: ParagraphIndexProcessor, dataset: Mock
+ self, processor: ParagraphIndexProcessor, dataset: Dataset
) -> None:
dataset.indexing_technique = IndexTechniqueType.ECONOMY
docs = [Document(page_content="chunk", metadata={})]
@@ -251,7 +267,7 @@ class TestParagraphIndexProcessor:
mock_keyword_cls.return_value.add_texts.assert_called_once_with(docs, session)
- def test_clean_deletes_summaries_and_vector(self, processor: ParagraphIndexProcessor, dataset: Mock) -> None:
+ def test_clean_deletes_summaries_and_vector(self, processor: ParagraphIndexProcessor, dataset: Dataset) -> None:
session = self.session
segment = DocumentSegment(
tenant_id=dataset.tenant_id,
@@ -280,7 +296,7 @@ class TestParagraphIndexProcessor:
vector.delete_by_ids.assert_called_once_with(["node-1"])
def test_clean_economy_deletes_summaries_and_keywords(
- self, processor: ParagraphIndexProcessor, dataset: Mock
+ self, processor: ParagraphIndexProcessor, dataset: Dataset
) -> None:
dataset.indexing_technique = IndexTechniqueType.ECONOMY
session = self.session
@@ -296,7 +312,7 @@ class TestParagraphIndexProcessor:
mock_summary.assert_called_once_with(dataset, None, session=session)
mock_keyword_cls.return_value.delete.assert_called_once()
- def test_clean_deletes_keywords_by_ids(self, processor: ParagraphIndexProcessor, dataset: Mock) -> None:
+ def test_clean_deletes_keywords_by_ids(self, processor: ParagraphIndexProcessor, dataset: Dataset) -> None:
dataset.indexing_technique = IndexTechniqueType.ECONOMY
session = self.session
with patch("core.rag.index_processor.processor.paragraph_index_processor.Keyword") as mock_keyword_cls:
@@ -305,7 +321,7 @@ class TestParagraphIndexProcessor:
mock_keyword_cls.return_value.delete_by_ids.assert_called_once_with(["node-2"], session)
def test_index_list_chunks_high_quality(
- self, processor: ParagraphIndexProcessor, dataset: Mock, dataset_document: Mock
+ self, processor: ParagraphIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument
) -> None:
session = self.session
phase_events: list[str] = []
@@ -346,7 +362,7 @@ class TestParagraphIndexProcessor:
mock_vector_cls.return_value.create_multimodal.assert_called_once()
def test_index_list_chunks_economy(
- self, processor: ParagraphIndexProcessor, dataset: Mock, dataset_document: Mock
+ self, processor: ParagraphIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument
) -> None:
dataset.indexing_technique = IndexTechniqueType.ECONOMY
session = self.session
@@ -376,7 +392,7 @@ class TestParagraphIndexProcessor:
mock_keyword_cls.return_value.add_texts.assert_called_once()
def test_index_multimodal_structure_handles_files_and_account_lookup(
- self, processor: ParagraphIndexProcessor, dataset: Mock, dataset_document: Mock
+ self, processor: ParagraphIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument
) -> None:
chunk_with_files = SimpleNamespace(
content="content-1",
@@ -421,7 +437,7 @@ class TestParagraphIndexProcessor:
assert account_session is not session
def test_index_multimodal_structure_requires_valid_account(
- self, processor: ParagraphIndexProcessor, dataset: Mock, dataset_document: Mock
+ self, processor: ParagraphIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument
) -> None:
structure = SimpleNamespace(general_chunks=[SimpleNamespace(content="content", files=None)])
session = self.session
diff --git a/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.py b/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.py
index 1a7e54e35c1..cb3dc7b23da 100644
--- a/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.py
+++ b/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.py
@@ -1,6 +1,6 @@
from contextlib import nullcontext
from types import SimpleNamespace
-from unittest.mock import MagicMock, Mock, patch
+from unittest.mock import Mock, patch
import pytest
from sqlalchemy import event
@@ -11,7 +11,9 @@ from core.rag.entities import ParentMode, Rule, Segmentation
from core.rag.index_processor.constant.index_type import IndexTechniqueType
from core.rag.index_processor.processor.parent_child_index_processor import ParentChildIndexProcessor
from core.rag.models.document import AttachmentDocument, ChildDocument, Document
-from models.dataset import ChildChunk, DatasetProcessRule, DocumentSegment
+from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentCreatedFrom, DocumentSegment
+from models.dataset import Document as DatasetDocument
+from models.enums import DataSourceType
class TestParentChildIndexProcessor:
@@ -28,20 +30,34 @@ class TestParentChildIndexProcessor:
return ParentChildIndexProcessor()
@pytest.fixture
- def dataset(self) -> Mock:
- dataset = Mock()
- dataset.id = "dataset-1"
- dataset.tenant_id = "tenant-1"
- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY
- dataset.is_multimodal = True
+ def dataset(self) -> Dataset:
+ dataset = Dataset(
+ id="dataset-1",
+ tenant_id="tenant-1",
+ name="Dataset",
+ created_by="user-1",
+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,
+ is_multimodal=True,
+ )
+ self.session.add(dataset)
+ self.session.flush()
return dataset
@pytest.fixture
- def dataset_document(self) -> Mock:
- document = Mock()
- document.id = "doc-1"
- document.created_by = "user-1"
- document.dataset_process_rule_id = None
+ def dataset_document(self, dataset: Dataset) -> DatasetDocument:
+ document = DatasetDocument(
+ id="doc-1",
+ tenant_id=dataset.tenant_id,
+ dataset_id=dataset.id,
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch="batch-1",
+ name="Document",
+ created_from=DocumentCreatedFrom.API,
+ created_by="user-1",
+ )
+ self.session.add(document)
+ self.session.flush()
return document
def _segmentation(self) -> SimpleNamespace:
@@ -206,7 +222,7 @@ class TestParentChildIndexProcessor:
assert len(result[0].children or []) == 2
assert result[0].attachments is not None
- def test_load_creates_vectors_for_child_docs(self, processor: ParentChildIndexProcessor, dataset: Mock) -> None:
+ def test_load_creates_vectors_for_child_docs(self, processor: ParentChildIndexProcessor, dataset: Dataset) -> None:
parent_doc = Document(
page_content="parent",
metadata={},
@@ -229,7 +245,7 @@ class TestParentChildIndexProcessor:
assert all(isinstance(doc, Document) for doc in formatted_docs)
vector.create_multimodal.assert_called_once_with(multimodal_docs)
- def test_clean_with_precomputed_child_ids(self, processor: ParentChildIndexProcessor, dataset: Mock) -> None:
+ def test_clean_with_precomputed_child_ids(self, processor: ParentChildIndexProcessor, dataset: Dataset) -> None:
session = self.session
with (
@@ -248,7 +264,7 @@ class TestParentChildIndexProcessor:
assert session.query(ChildChunk).count() == 0
def test_clean_queries_child_ids_when_not_precomputed(
- self, processor: ParentChildIndexProcessor, dataset: Mock
+ self, processor: ParentChildIndexProcessor, dataset: Dataset
) -> None:
session = self.session
parent = DocumentSegment(
@@ -290,7 +306,7 @@ class TestParentChildIndexProcessor:
vector.delete_by_ids.assert_called_once_with(["child-1", "child-2"])
- def test_clean_dataset_wide_cleanup(self, processor: ParentChildIndexProcessor, dataset: Mock) -> None:
+ def test_clean_dataset_wide_cleanup(self, processor: ParentChildIndexProcessor, dataset: Dataset) -> None:
session = self.session
with (
@@ -302,7 +318,9 @@ class TestParentChildIndexProcessor:
vector.delete.assert_called_once()
assert session.query(ChildChunk).count() == 0
- def test_clean_deletes_summaries_when_requested(self, processor: ParentChildIndexProcessor, dataset: Mock) -> None:
+ def test_clean_deletes_summaries_when_requested(
+ self, processor: ParentChildIndexProcessor, dataset: Dataset
+ ) -> None:
session = self.session
segment = DocumentSegment(
tenant_id=dataset.tenant_id,
@@ -329,7 +347,7 @@ class TestParentChildIndexProcessor:
mock_summary.assert_called_once_with(dataset, [segment.id], session=session)
def test_clean_deletes_all_summaries_when_node_ids_missing(
- self, processor: ParentChildIndexProcessor, dataset: Mock
+ self, processor: ParentChildIndexProcessor, dataset: Dataset
) -> None:
with (
patch(
@@ -372,7 +390,7 @@ class TestParentChildIndexProcessor:
assert child_docs[0].metadata["doc_hash"] == "hash"
def test_index_creates_process_rule_segments_and_vectors(
- self, processor: ParentChildIndexProcessor, dataset: Mock, dataset_document: Mock
+ self, processor: ParentChildIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument
) -> None:
parent_childs = SimpleNamespace(
parent_mode=ParentMode.PARAGRAPH,
@@ -427,7 +445,7 @@ class TestParentChildIndexProcessor:
mock_vector_cls.return_value.create_multimodal.assert_called_once()
def test_index_uses_content_files_when_files_missing(
- self, processor: ParentChildIndexProcessor, dataset: Mock, dataset_document: Mock
+ self, processor: ParentChildIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument
) -> None:
parent_childs = SimpleNamespace(
parent_mode=ParentMode.PARAGRAPH,
@@ -470,7 +488,7 @@ class TestParentChildIndexProcessor:
assert account_session is not session
def test_index_raises_when_account_missing(
- self, processor: ParentChildIndexProcessor, dataset: Mock, dataset_document: Mock
+ self, processor: ParentChildIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument
) -> None:
parent_childs = SimpleNamespace(
parent_mode=ParentMode.PARAGRAPH,
@@ -492,7 +510,7 @@ class TestParentChildIndexProcessor:
),
):
with pytest.raises(ValueError, match="Invalid account"):
- processor.index(dataset, dataset_document, {"parent_child_chunks": []}, MagicMock())
+ processor.index(dataset, dataset_document, {"parent_child_chunks": []}, self.session)
def test_format_preview_returns_parent_child_structure(self, processor: ParentChildIndexProcessor) -> None:
parent_childs = SimpleNamespace(
diff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_task_proxy.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_task_proxy.py
index e9f5bcab1d2..ac6d4d5fd01 100644
--- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_task_proxy.py
+++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_task_proxy.py
@@ -1,3 +1,4 @@
+from datetime import datetime
from types import SimpleNamespace
from unittest.mock import Mock
@@ -5,6 +6,9 @@ import pytest
from pytest_mock import MockerFixture
from enums import CloudPlan
+from extensions.storage.storage_type import StorageType
+from models.enums import CreatorUserRole
+from models.model import UploadFile
from services.rag_pipeline.rag_pipeline_task_proxy import RagPipelineTaskProxy
@@ -146,7 +150,20 @@ def test_send_to_tenant_queue_sets_waiting_time_and_calls_delay(mocker: MockerFi
def test_upload_invoke_entities_returns_file_id(mocker: MockerFixture, proxy) -> None:
- upload_file = SimpleNamespace(id="uploaded-file-1")
+ upload_file = UploadFile(
+ tenant_id="tenant-1",
+ storage_type=StorageType.LOCAL,
+ key="rag-pipeline.json",
+ name="rag-pipeline.json",
+ size=1,
+ extension="json",
+ mime_type="application/json",
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by="user-1",
+ created_at=datetime(2025, 1, 1),
+ used=True,
+ )
+ upload_file.id = "uploaded-file-1"
file_service_cls = mocker.patch("services.rag_pipeline.rag_pipeline_task_proxy.FileService")
file_service_cls.return_value.upload_text.return_value = upload_file
mocker.patch("services.rag_pipeline.rag_pipeline_task_proxy.db", SimpleNamespace(engine="fake-engine"))
diff --git a/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py b/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py
index cb72de5b53e..f89fb17f6e0 100644
--- a/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py
+++ b/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py
@@ -1,5 +1,6 @@
import json
import logging
+from datetime import datetime
from unittest.mock import MagicMock, Mock, patch
import pytest
@@ -7,6 +8,9 @@ import pytest
from core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity
from core.rag.pipeline.queue import TenantIsolatedTaskQueue
from enums import CloudPlan
+from extensions.storage.storage_type import StorageType
+from models.enums import CreatorUserRole
+from models.model import UploadFile
from services.rag_pipeline.rag_pipeline_task_proxy import RagPipelineTaskProxy
@@ -66,9 +70,21 @@ class RagPipelineTaskProxyTestDataFactory:
return RagPipelineTaskProxy(dataset_tenant_id, user_id, rag_pipeline_invoke_entities)
@staticmethod
- def create_mock_upload_file(file_id: str = "file-123") -> Mock:
- """Create mock upload file."""
- upload_file = Mock()
+ def create_upload_file(file_id: str = "file-123") -> UploadFile:
+ """Create a transient upload file returned by the mocked storage service."""
+ upload_file = UploadFile(
+ tenant_id="tenant-123",
+ storage_type=StorageType.LOCAL,
+ key="rag-pipeline.json",
+ name="rag-pipeline.json",
+ size=1,
+ extension="json",
+ mime_type="application/json",
+ created_by_role=CreatorUserRole.ACCOUNT,
+ created_by="user-456",
+ created_at=datetime(2025, 1, 1),
+ used=True,
+ )
upload_file.id = file_id
return upload_file
@@ -155,8 +171,8 @@ class TestRagPipelineTaskProxy:
proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
mock_file_service = Mock()
mock_file_service_class.return_value = mock_file_service
- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
- mock_file_service.upload_text.return_value = mock_upload_file
+ upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")
+ mock_file_service.upload_text.return_value = upload_file
# Act
result = proxy._upload_invoke_entities()
@@ -193,8 +209,8 @@ class TestRagPipelineTaskProxy:
proxy = RagPipelineTaskProxy("tenant-123", "user-456", entities)
mock_file_service = Mock()
mock_file_service_class.return_value = mock_file_service
- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-456")
- mock_file_service.upload_text.return_value = mock_upload_file
+ upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-456")
+ mock_file_service.upload_text.return_value = upload_file
# Act
result = proxy._upload_invoke_entities()
@@ -331,7 +347,7 @@ class TestRagPipelineTaskProxy:
mock_file_service = Mock()
mock_file_service_class.return_value = mock_file_service
- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")
mock_file_service.upload_text.return_value = mock_upload_file
# Act
@@ -357,7 +373,7 @@ class TestRagPipelineTaskProxy:
mock_file_service = Mock()
mock_file_service_class.return_value = mock_file_service
- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")
mock_file_service.upload_text.return_value = mock_upload_file
# Act
@@ -381,7 +397,7 @@ class TestRagPipelineTaskProxy:
mock_file_service = Mock()
mock_file_service_class.return_value = mock_file_service
- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")
mock_file_service.upload_text.return_value = mock_upload_file
# Act
@@ -399,9 +415,8 @@ class TestRagPipelineTaskProxy:
mock_file_service = Mock()
mock_file_service_class.return_value = mock_file_service
- mock_upload_file = Mock()
- mock_upload_file.id = "" # Empty file ID
- mock_file_service.upload_text.return_value = mock_upload_file
+ upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("")
+ mock_file_service.upload_text.return_value = upload_file
# Act & Assert
with pytest.raises(ValueError, match="upload_file_id is empty"):
@@ -422,7 +437,7 @@ class TestRagPipelineTaskProxy:
mock_file_service = Mock()
mock_file_service_class.return_value = mock_file_service
- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")
mock_file_service.upload_text.return_value = mock_upload_file
# Act
@@ -446,7 +461,7 @@ class TestRagPipelineTaskProxy:
mock_file_service = Mock()
mock_file_service_class.return_value = mock_file_service
- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")
mock_file_service.upload_text.return_value = mock_upload_file
# Act
@@ -472,7 +487,7 @@ class TestRagPipelineTaskProxy:
mock_file_service = Mock()
mock_file_service_class.return_value = mock_file_service
- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")
mock_file_service.upload_text.return_value = mock_upload_file
# Act