Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: migrate RAG processor residuals to SQLite ORM models (#40608)
Asuka Minatoedb32fba9223
diff --git a/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.py b/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.pyindex 40c14147f53..692a9b80972 100644--- a/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.py+++ b/api/tests/unit_tests/core/rag/indexing/processor/test_paragraph_index_processor.py@@ -17,8 +17,9 @@ from extensions.storage.storage_type import StorageTypefrom graphon.model_runtime.entities.llm_entities import LLMResult, LLMUsagefrom graphon.model_runtime.entities.message_entities import AssistantPromptMessage, ImagePromptMessageContentfrom graphon.model_runtime.entities.model_entities import ModelFeature, ModelType-from models.dataset import DocumentSegment, SegmentAttachmentBinding-from models.enums import CreatorUserRole+from models.dataset import Dataset, DocumentCreatedFrom, DocumentSegment, SegmentAttachmentBinding+from models.dataset import Document as DatasetDocument+from models.enums import CreatorUserRole, DataSourceTypefrom models.model import UploadFile@@ -36,19 +37,34 @@ class TestParagraphIndexProcessor:return ParagraphIndexProcessor()@pytest.fixture- def dataset(self) -> Mock:- dataset = Mock()- dataset.id = "dataset-1"- dataset.tenant_id = "tenant-1"- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY- dataset.is_multimodal = True+ def dataset(self) -> Dataset:+ dataset = Dataset(+ id="dataset-1",+ tenant_id="tenant-1",+ name="Dataset",+ created_by="user-1",+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ is_multimodal=True,+ )+ self.session.add(dataset)+ self.session.flush()return dataset@pytest.fixture- def dataset_document(self) -> Mock:- document = Mock()- document.id = "doc-1"- document.created_by = "user-1"+ def dataset_document(self, dataset: Dataset) -> DatasetDocument:+ document = DatasetDocument(+ id="doc-1",+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch-1",+ name="Document",+ created_from=DocumentCreatedFrom.API,+ created_by="user-1",+ )+ self.session.add(document)+ self.session.flush()return document@pytest.fixture@@ -209,7 +225,7 @@ class TestParagraphIndexProcessor:assert mock_validate.call_count == 1def test_load_creates_vector_and_multimodal_when_high_quality(- self, processor: ParagraphIndexProcessor, dataset: Mock+ self, processor: ParagraphIndexProcessor, dataset: Dataset) -> None:docs = [Document(page_content="chunk", metadata={})]multimodal_docs = [AttachmentDocument(page_content="image", metadata={})]@@ -227,7 +243,7 @@ class TestParagraphIndexProcessor:mock_keyword_cls.assert_not_called()def test_load_uses_keyword_add_texts_with_keywords_when_economy(- self, processor: ParagraphIndexProcessor, dataset: Mock+ self, processor: ParagraphIndexProcessor, dataset: Dataset) -> None:dataset.indexing_technique = IndexTechniqueType.ECONOMYdocs = [Document(page_content="chunk", metadata={})]@@ -240,7 +256,7 @@ class TestParagraphIndexProcessor:mock_keyword_cls.return_value.add_texts.assert_called_once_with(docs, session, keywords_list=keywords_list)def test_load_uses_keyword_add_texts_without_keywords_when_economy(- self, processor: ParagraphIndexProcessor, dataset: Mock+ self, processor: ParagraphIndexProcessor, dataset: Dataset) -> None:dataset.indexing_technique = IndexTechniqueType.ECONOMYdocs = [Document(page_content="chunk", metadata={})]@@ -251,7 +267,7 @@ class TestParagraphIndexProcessor:mock_keyword_cls.return_value.add_texts.assert_called_once_with(docs, session)- def test_clean_deletes_summaries_and_vector(self, processor: ParagraphIndexProcessor, dataset: Mock) -> None:+ def test_clean_deletes_summaries_and_vector(self, processor: ParagraphIndexProcessor, dataset: Dataset) -> None:session = self.sessionsegment = DocumentSegment(tenant_id=dataset.tenant_id,@@ -280,7 +296,7 @@ class TestParagraphIndexProcessor:vector.delete_by_ids.assert_called_once_with(["node-1"])def test_clean_economy_deletes_summaries_and_keywords(- self, processor: ParagraphIndexProcessor, dataset: Mock+ self, processor: ParagraphIndexProcessor, dataset: Dataset) -> None:dataset.indexing_technique = IndexTechniqueType.ECONOMYsession = self.session@@ -296,7 +312,7 @@ class TestParagraphIndexProcessor:mock_summary.assert_called_once_with(dataset, None, session=session)mock_keyword_cls.return_value.delete.assert_called_once()- def test_clean_deletes_keywords_by_ids(self, processor: ParagraphIndexProcessor, dataset: Mock) -> None:+ def test_clean_deletes_keywords_by_ids(self, processor: ParagraphIndexProcessor, dataset: Dataset) -> None:dataset.indexing_technique = IndexTechniqueType.ECONOMYsession = self.sessionwith patch("core.rag.index_processor.processor.paragraph_index_processor.Keyword") as mock_keyword_cls:@@ -305,7 +321,7 @@ class TestParagraphIndexProcessor:mock_keyword_cls.return_value.delete_by_ids.assert_called_once_with(["node-2"], session)def test_index_list_chunks_high_quality(- self, processor: ParagraphIndexProcessor, dataset: Mock, dataset_document: Mock+ self, processor: ParagraphIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument) -> None:session = self.sessionphase_events: list[str] = []@@ -346,7 +362,7 @@ class TestParagraphIndexProcessor:mock_vector_cls.return_value.create_multimodal.assert_called_once()def test_index_list_chunks_economy(- self, processor: ParagraphIndexProcessor, dataset: Mock, dataset_document: Mock+ self, processor: ParagraphIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument) -> None:dataset.indexing_technique = IndexTechniqueType.ECONOMYsession = self.session@@ -376,7 +392,7 @@ class TestParagraphIndexProcessor:mock_keyword_cls.return_value.add_texts.assert_called_once()def test_index_multimodal_structure_handles_files_and_account_lookup(- self, processor: ParagraphIndexProcessor, dataset: Mock, dataset_document: Mock+ self, processor: ParagraphIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument) -> None:chunk_with_files = SimpleNamespace(content="content-1",@@ -421,7 +437,7 @@ class TestParagraphIndexProcessor:assert account_session is not sessiondef test_index_multimodal_structure_requires_valid_account(- self, processor: ParagraphIndexProcessor, dataset: Mock, dataset_document: Mock+ self, processor: ParagraphIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument) -> None:structure = SimpleNamespace(general_chunks=[SimpleNamespace(content="content", files=None)])session = self.sessiondiff --git a/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.py b/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.pyindex 1a7e54e35c1..cb3dc7b23da 100644--- a/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.py+++ b/api/tests/unit_tests/core/rag/indexing/processor/test_parent_child_index_processor.py@@ -1,6 +1,6 @@from contextlib import nullcontextfrom types import SimpleNamespace-from unittest.mock import MagicMock, Mock, patch+from unittest.mock import Mock, patchimport pytestfrom sqlalchemy import event@@ -11,7 +11,9 @@ from core.rag.entities import ParentMode, Rule, Segmentationfrom core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom core.rag.index_processor.processor.parent_child_index_processor import ParentChildIndexProcessorfrom core.rag.models.document import AttachmentDocument, ChildDocument, Document-from models.dataset import ChildChunk, DatasetProcessRule, DocumentSegment+from models.dataset import ChildChunk, Dataset, DatasetProcessRule, DocumentCreatedFrom, DocumentSegment+from models.dataset import Document as DatasetDocument+from models.enums import DataSourceTypeclass TestParentChildIndexProcessor:@@ -28,20 +30,34 @@ class TestParentChildIndexProcessor:return ParentChildIndexProcessor()@pytest.fixture- def dataset(self) -> Mock:- dataset = Mock()- dataset.id = "dataset-1"- dataset.tenant_id = "tenant-1"- dataset.indexing_technique = IndexTechniqueType.HIGH_QUALITY- dataset.is_multimodal = True+ def dataset(self) -> Dataset:+ dataset = Dataset(+ id="dataset-1",+ tenant_id="tenant-1",+ name="Dataset",+ created_by="user-1",+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ is_multimodal=True,+ )+ self.session.add(dataset)+ self.session.flush()return dataset@pytest.fixture- def dataset_document(self) -> Mock:- document = Mock()- document.id = "doc-1"- document.created_by = "user-1"- document.dataset_process_rule_id = None+ def dataset_document(self, dataset: Dataset) -> DatasetDocument:+ document = DatasetDocument(+ id="doc-1",+ tenant_id=dataset.tenant_id,+ dataset_id=dataset.id,+ position=1,+ data_source_type=DataSourceType.UPLOAD_FILE,+ batch="batch-1",+ name="Document",+ created_from=DocumentCreatedFrom.API,+ created_by="user-1",+ )+ self.session.add(document)+ self.session.flush()return documentdef _segmentation(self) -> SimpleNamespace:@@ -206,7 +222,7 @@ class TestParentChildIndexProcessor:assert len(result[0].children or []) == 2assert result[0].attachments is not None- def test_load_creates_vectors_for_child_docs(self, processor: ParentChildIndexProcessor, dataset: Mock) -> None:+ def test_load_creates_vectors_for_child_docs(self, processor: ParentChildIndexProcessor, dataset: Dataset) -> None:parent_doc = Document(page_content="parent",metadata={},@@ -229,7 +245,7 @@ class TestParentChildIndexProcessor:assert all(isinstance(doc, Document) for doc in formatted_docs)vector.create_multimodal.assert_called_once_with(multimodal_docs)- def test_clean_with_precomputed_child_ids(self, processor: ParentChildIndexProcessor, dataset: Mock) -> None:+ def test_clean_with_precomputed_child_ids(self, processor: ParentChildIndexProcessor, dataset: Dataset) -> None:session = self.sessionwith (@@ -248,7 +264,7 @@ class TestParentChildIndexProcessor:assert session.query(ChildChunk).count() == 0def test_clean_queries_child_ids_when_not_precomputed(- self, processor: ParentChildIndexProcessor, dataset: Mock+ self, processor: ParentChildIndexProcessor, dataset: Dataset) -> None:session = self.sessionparent = DocumentSegment(@@ -290,7 +306,7 @@ class TestParentChildIndexProcessor:vector.delete_by_ids.assert_called_once_with(["child-1", "child-2"])- def test_clean_dataset_wide_cleanup(self, processor: ParentChildIndexProcessor, dataset: Mock) -> None:+ def test_clean_dataset_wide_cleanup(self, processor: ParentChildIndexProcessor, dataset: Dataset) -> None:session = self.sessionwith (@@ -302,7 +318,9 @@ class TestParentChildIndexProcessor:vector.delete.assert_called_once()assert session.query(ChildChunk).count() == 0- def test_clean_deletes_summaries_when_requested(self, processor: ParentChildIndexProcessor, dataset: Mock) -> None:+ def test_clean_deletes_summaries_when_requested(+ self, processor: ParentChildIndexProcessor, dataset: Dataset+ ) -> None:session = self.sessionsegment = DocumentSegment(tenant_id=dataset.tenant_id,@@ -329,7 +347,7 @@ class TestParentChildIndexProcessor:mock_summary.assert_called_once_with(dataset, [segment.id], session=session)def test_clean_deletes_all_summaries_when_node_ids_missing(- self, processor: ParentChildIndexProcessor, dataset: Mock+ self, processor: ParentChildIndexProcessor, dataset: Dataset) -> None:with (patch(@@ -372,7 +390,7 @@ class TestParentChildIndexProcessor:assert child_docs[0].metadata["doc_hash"] == "hash"def test_index_creates_process_rule_segments_and_vectors(- self, processor: ParentChildIndexProcessor, dataset: Mock, dataset_document: Mock+ self, processor: ParentChildIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument) -> None:parent_childs = SimpleNamespace(parent_mode=ParentMode.PARAGRAPH,@@ -427,7 +445,7 @@ class TestParentChildIndexProcessor:mock_vector_cls.return_value.create_multimodal.assert_called_once()def test_index_uses_content_files_when_files_missing(- self, processor: ParentChildIndexProcessor, dataset: Mock, dataset_document: Mock+ self, processor: ParentChildIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument) -> None:parent_childs = SimpleNamespace(parent_mode=ParentMode.PARAGRAPH,@@ -470,7 +488,7 @@ class TestParentChildIndexProcessor:assert account_session is not sessiondef test_index_raises_when_account_missing(- self, processor: ParentChildIndexProcessor, dataset: Mock, dataset_document: Mock+ self, processor: ParentChildIndexProcessor, dataset: Dataset, dataset_document: DatasetDocument) -> None:parent_childs = SimpleNamespace(parent_mode=ParentMode.PARAGRAPH,@@ -492,7 +510,7 @@ class TestParentChildIndexProcessor:),):with pytest.raises(ValueError, match="Invalid account"):- processor.index(dataset, dataset_document, {"parent_child_chunks": []}, MagicMock())+ processor.index(dataset, dataset_document, {"parent_child_chunks": []}, self.session)def test_format_preview_returns_parent_child_structure(self, processor: ParentChildIndexProcessor) -> None:parent_childs = SimpleNamespace(diff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_task_proxy.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_task_proxy.pyindex e9f5bcab1d2..ac6d4d5fd01 100644--- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_task_proxy.py+++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_task_proxy.py@@ -1,3 +1,4 @@+from datetime import datetimefrom types import SimpleNamespacefrom unittest.mock import Mock@@ -5,6 +6,9 @@ import pytestfrom pytest_mock import MockerFixturefrom enums import CloudPlan+from extensions.storage.storage_type import StorageType+from models.enums import CreatorUserRole+from models.model import UploadFilefrom services.rag_pipeline.rag_pipeline_task_proxy import RagPipelineTaskProxy@@ -146,7 +150,20 @@ def test_send_to_tenant_queue_sets_waiting_time_and_calls_delay(mocker: MockerFidef test_upload_invoke_entities_returns_file_id(mocker: MockerFixture, proxy) -> None:- upload_file = SimpleNamespace(id="uploaded-file-1")+ upload_file = UploadFile(+ tenant_id="tenant-1",+ storage_type=StorageType.LOCAL,+ key="rag-pipeline.json",+ name="rag-pipeline.json",+ size=1,+ extension="json",+ mime_type="application/json",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="user-1",+ created_at=datetime(2025, 1, 1),+ used=True,+ )+ upload_file.id = "uploaded-file-1"file_service_cls = mocker.patch("services.rag_pipeline.rag_pipeline_task_proxy.FileService")file_service_cls.return_value.upload_text.return_value = upload_filemocker.patch("services.rag_pipeline.rag_pipeline_task_proxy.db", SimpleNamespace(engine="fake-engine"))diff --git a/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py b/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.pyindex cb72de5b53e..f89fb17f6e0 100644--- a/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py+++ b/api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py@@ -1,5 +1,6 @@import jsonimport logging+from datetime import datetimefrom unittest.mock import MagicMock, Mock, patchimport pytest@@ -7,6 +8,9 @@ import pytestfrom core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntityfrom core.rag.pipeline.queue import TenantIsolatedTaskQueuefrom enums import CloudPlan+from extensions.storage.storage_type import StorageType+from models.enums import CreatorUserRole+from models.model import UploadFilefrom services.rag_pipeline.rag_pipeline_task_proxy import RagPipelineTaskProxy@@ -66,9 +70,21 @@ class RagPipelineTaskProxyTestDataFactory:return RagPipelineTaskProxy(dataset_tenant_id, user_id, rag_pipeline_invoke_entities)@staticmethod- def create_mock_upload_file(file_id: str = "file-123") -> Mock:- """Create mock upload file."""- upload_file = Mock()+ def create_upload_file(file_id: str = "file-123") -> UploadFile:+ """Create a transient upload file returned by the mocked storage service."""+ upload_file = UploadFile(+ tenant_id="tenant-123",+ storage_type=StorageType.LOCAL,+ key="rag-pipeline.json",+ name="rag-pipeline.json",+ size=1,+ extension="json",+ mime_type="application/json",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by="user-456",+ created_at=datetime(2025, 1, 1),+ used=True,+ )upload_file.id = file_idreturn upload_file@@ -155,8 +171,8 @@ class TestRagPipelineTaskProxy:proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()mock_file_service = Mock()mock_file_service_class.return_value = mock_file_service- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")- mock_file_service.upload_text.return_value = mock_upload_file+ upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")+ mock_file_service.upload_text.return_value = upload_file# Actresult = proxy._upload_invoke_entities()@@ -193,8 +209,8 @@ class TestRagPipelineTaskProxy:proxy = RagPipelineTaskProxy("tenant-123", "user-456", entities)mock_file_service = Mock()mock_file_service_class.return_value = mock_file_service- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-456")- mock_file_service.upload_text.return_value = mock_upload_file+ upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-456")+ mock_file_service.upload_text.return_value = upload_file# Actresult = proxy._upload_invoke_entities()@@ -331,7 +347,7 @@ class TestRagPipelineTaskProxy:mock_file_service = Mock()mock_file_service_class.return_value = mock_file_service- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")mock_file_service.upload_text.return_value = mock_upload_file# Act@@ -357,7 +373,7 @@ class TestRagPipelineTaskProxy:mock_file_service = Mock()mock_file_service_class.return_value = mock_file_service- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")mock_file_service.upload_text.return_value = mock_upload_file# Act@@ -381,7 +397,7 @@ class TestRagPipelineTaskProxy:mock_file_service = Mock()mock_file_service_class.return_value = mock_file_service- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")mock_file_service.upload_text.return_value = mock_upload_file# Act@@ -399,9 +415,8 @@ class TestRagPipelineTaskProxy:mock_file_service = Mock()mock_file_service_class.return_value = mock_file_service- mock_upload_file = Mock()- mock_upload_file.id = "" # Empty file ID- mock_file_service.upload_text.return_value = mock_upload_file+ upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("")+ mock_file_service.upload_text.return_value = upload_file# Act & Assertwith pytest.raises(ValueError, match="upload_file_id is empty"):@@ -422,7 +437,7 @@ class TestRagPipelineTaskProxy:mock_file_service = Mock()mock_file_service_class.return_value = mock_file_service- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")mock_file_service.upload_text.return_value = mock_upload_file# Act@@ -446,7 +461,7 @@ class TestRagPipelineTaskProxy:mock_file_service = Mock()mock_file_service_class.return_value = mock_file_service- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")mock_file_service.upload_text.return_value = mock_upload_file# Act@@ -472,7 +487,7 @@ class TestRagPipelineTaskProxy:mock_file_service = Mock()mock_file_service_class.return_value = mock_file_service- mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")+ mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_upload_file("file-123")mock_file_service.upload_text.return_value = mock_upload_file# Act