Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: use SQLite sessions in clean dataset (#39051)
Asuka Minatoe6ede05fe9fb
diff --git a/api/tests/unit_tests/tasks/test_clean_dataset_task.py b/api/tests/unit_tests/tasks/test_clean_dataset_task.pyindex 4a2dcc44145..3ac74ec1f4f 100644--- a/api/tests/unit_tests/tasks/test_clean_dataset_task.py+++ b/api/tests/unit_tests/tasks/test_clean_dataset_task.py@@ -11,13 +11,33 @@ This module tests the dataset cleanup task functionality including:- Segment attachment cleanup"""+import jsonimport uuid+from collections.abc import Iterator+from datetime import UTC, datetimefrom unittest.mock import MagicMock, patchimport pytest+from sqlalchemy import Engine, event+from sqlalchemy.orm import Session, sessionmakerfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType-from models.enums import DataSourceType+from extensions.storage.storage_type import StorageType+from models.base import TypeBase+from models.dataset import (+ AppDatasetJoin,+ DatasetMetadata,+ DatasetMetadataBinding,+ DatasetProcessRule,+ DatasetQuery,+ Document,+ DocumentSegment,+ Pipeline,+ SegmentAttachmentBinding,+)+from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus+from models.model import UploadFile+from models.workflow import Workflow, WorkflowTypefrom tasks.clean_dataset_task import clean_dataset_task# ============================================================================@@ -26,54 +46,54 @@ from tasks.clean_dataset_task import clean_dataset_task@pytest.fixture-def tenant_id():+def tenant_id() -> str:"""Generate a unique tenant ID for testing."""return str(uuid.uuid4())@pytest.fixture-def dataset_id():+def dataset_id() -> str:"""Generate a unique dataset ID for testing."""return str(uuid.uuid4())@pytest.fixture-def collection_binding_id():+def collection_binding_id() -> str:"""Generate a unique collection binding ID for testing."""return str(uuid.uuid4())@pytest.fixture-def pipeline_id():+def pipeline_id() -> str:"""Generate a unique pipeline ID for testing."""return str(uuid.uuid4())@pytest.fixture-def mock_db_session():- """Mock database session via session_factory.create_session()."""- with patch("tasks.clean_dataset_task.session_factory", autospec=True) as mock_sf:- mock_session = MagicMock()- # context manager for create_session()- cm = MagicMock()- cm.__enter__.return_value = mock_session- cm.__exit__.return_value = None- mock_sf.create_session.return_value = cm-- # Setup scalars for select queries- mock_session.scalars.return_value.all.return_value = []-- # Setup execute for JOIN queries- mock_session.execute.return_value.all.return_value = []-- # Yield an object with a `.session` attribute to keep tests unchanged- wrapper = MagicMock()- wrapper.session = mock_session- yield wrapper+def orm_session_maker(+ sqlite_engine: Engine,+ sqlite_session_factory: sessionmaker[Session],+) -> sessionmaker[Session]:+ """Create the cleanup tables and return the suite's real SQLite session factory."""+ models = (+ Document,+ DocumentSegment,+ SegmentAttachmentBinding,+ UploadFile,+ DatasetProcessRule,+ DatasetQuery,+ AppDatasetJoin,+ DatasetMetadata,+ DatasetMetadataBinding,+ Pipeline,+ Workflow,+ )+ TypeBase.metadata.create_all(sqlite_engine, tables=[model.__table__ for model in models])+ return sqlite_session_factory@pytest.fixture-def mock_storage():+def mock_storage() -> Iterator[MagicMock]:"""Mock storage client."""with patch("tasks.clean_dataset_task.storage", autospec=True) as mock_storage:mock_storage.delete.return_value = None@@ -81,7 +101,7 @@ def mock_storage():@pytest.fixture-def mock_index_processor_factory():+def mock_index_processor_factory() -> Iterator[dict[str, MagicMock]]:"""Mock IndexProcessorFactory."""with patch("tasks.clean_dataset_task.IndexProcessorFactory", autospec=True) as mock_factory:mock_processor = MagicMock()@@ -98,42 +118,104 @@ def mock_index_processor_factory():@pytest.fixture-def mock_get_image_upload_file_ids():+def mock_get_image_upload_file_ids() -> Iterator[MagicMock]:"""Mock get_image_upload_file_ids function."""with patch("tasks.clean_dataset_task.get_image_upload_file_ids", autospec=True) as mock_func:mock_func.return_value = []yield mock_func-def mock_document():- """Create a mock Document object."""- doc = MagicMock()- doc.id = str(uuid.uuid4())- doc.tenant_id = str(uuid.uuid4())- doc.dataset_id = str(uuid.uuid4())- doc.data_source_type = DataSourceType.UPLOAD_FILE- doc.data_source_info = '{"upload_file_id": "test-file-id"}'- doc.data_source_info_dict = {"upload_file_id": "test-file-id"}- return doc---def mock_segment():- """Create a mock DocumentSegment object."""- segment = MagicMock()- segment.id = str(uuid.uuid4())- segment.content = "Test segment content"- return segment---def mock_upload_file():- """Create a mock UploadFile object."""- upload_file = MagicMock()- upload_file.id = str(uuid.uuid4())- upload_file.key = f"test_files/{uuid.uuid4()}.txt"- return upload_file+def _run_clean_dataset(+ *,+ dataset_id: str,+ tenant_id: str,+ collection_binding_id: str,+ pipeline_id: str | None = None,+) -> None:+ clean_dataset_task(+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ indexing_technique=IndexTechniqueType.HIGH_QUALITY,+ index_struct='{"type": "paragraph"}',+ collection_binding_id=collection_binding_id,+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ pipeline_id=pipeline_id,+ )+++def _persist_document(session_maker: sessionmaker[Session], *, dataset_id: str, tenant_id: str) -> Document:+ document = Document(+ id=str(uuid.uuid4()),+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ position=1,+ data_source_type=DataSourceType.LOCAL_FILE,+ batch="batch",+ name="Document",+ created_from=DocumentCreatedFrom.API,+ created_by=str(uuid.uuid4()),+ indexing_status=IndexingStatus.COMPLETED,+ doc_form=IndexStructureType.PARAGRAPH_INDEX,+ )+ with session_maker.begin() as session:+ session.add(document)+ return document+++def _persist_pipeline_and_workflow(+ session_maker: sessionmaker[Session],+ *,+ pipeline_id: str,+ tenant_id: str,+) -> Workflow:+ pipeline = Pipeline(tenant_id=tenant_id, name="Pipeline", description="Pipeline")+ pipeline.id = pipeline_id+ workflow = Workflow.new(+ tenant_id=tenant_id,+ app_id=pipeline_id,+ type=WorkflowType.RAG_PIPELINE.value,+ version="v1",+ graph=json.dumps({"nodes": [], "edges": []}),+ features="{}",+ created_by=str(uuid.uuid4()),+ environment_variables=[],+ conversation_variables=[],+ rag_pipeline_variables=[],+ )+ with session_maker.begin() as session:+ session.add_all([pipeline, workflow])+ return workflow+++def _persist_attachment(+ session_maker: sessionmaker[Session],+ *,+ dataset_id: str,+ tenant_id: str,+) -> tuple[SegmentAttachmentBinding, UploadFile]:+ attachment_file = UploadFile(+ tenant_id=tenant_id,+ storage_type=StorageType.LOCAL,+ key=f"attachments/{uuid.uuid4()}.pdf",+ name="attachment.pdf",+ size=10,+ extension="pdf",+ mime_type="application/pdf",+ created_by_role=CreatorUserRole.ACCOUNT,+ created_by=str(uuid.uuid4()),+ created_at=datetime.now(UTC),+ used=False,+ )+ binding = SegmentAttachmentBinding(+ tenant_id=tenant_id,+ dataset_id=dataset_id,+ document_id=str(uuid.uuid4()),+ segment_id=str(uuid.uuid4()),+ attachment_id=attachment_file.id,+ )+ with session_maker.begin() as session:+ session.add_all([attachment_file, binding])+ return binding, attachment_file# ============================================================================@@ -154,38 +236,50 @@ class TestErrorHandling:dataset_id: str,tenant_id: str,collection_binding_id: str,- mock_db_session,- mock_storage,- mock_index_processor_factory,- mock_get_image_upload_file_ids,+ orm_session_maker: sessionmaker[Session],+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,):"""Test that session is closed even if rollback fails.Scenario:- Database commit fails- - Rollback also fails- - Session should still be closed+ - Rollback completes, then its event hook raises+ - Session cleanup should still make the factory reusableExpected behavior:- - Session.close() is called regardless of rollback failure+ - The database rollback preserves the pending row+ - The task session closes and a new session remains usable"""- # Arrange- mock_db_session.session.commit.side_effect = Exception("Commit failed")- mock_db_session.session.rollback.side_effect = Exception("Rollback failed")+ # Arrange: persist a row whose attempted deletion must be rolled back.+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)++ def fail_commit(_session: Session) -> None:+ raise RuntimeError("Commit failed")++ def fail_rollback(_session: Session) -> None:+ raise RuntimeError("Rollback failed")++ event.listen(orm_session_maker.class_, "before_commit", fail_commit)+ event.listen(orm_session_maker.class_, "after_rollback", fail_rollback)# Act- clean_dataset_task(- dataset_id=dataset_id,- tenant_id=tenant_id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- index_struct='{"type": "paragraph"}',- collection_binding_id=collection_binding_id,- doc_form=IndexStructureType.PARAGRAPH_INDEX,- )+ try:+ _run_clean_dataset(+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ collection_binding_id=collection_binding_id,+ )+ finally:+ event.remove(orm_session_maker.class_, "before_commit", fail_commit)+ event.remove(orm_session_maker.class_, "after_rollback", fail_rollback)- # Assert- mock_db_session.session.close.assert_called_once()+ # Assert: rollback happened before its hook failed, and the closed task+ # session did not prevent a new real session from reading the row.+ with orm_session_maker() as session:+ assert session.get(Document, document.id) is not None# ============================================================================@@ -201,11 +295,11 @@ class TestPipelineAndWorkflowDeletion:dataset_id: str,tenant_id: str,collection_binding_id: str,- pipeline_id,- mock_db_session,- mock_storage,- mock_index_processor_factory,- mock_get_image_upload_file_ids,+ pipeline_id: str,+ orm_session_maker: sessionmaker[Session],+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,):"""Test that pipeline and workflow are deleted when pipeline_id is provided.@@ -214,30 +308,35 @@ class TestPipelineAndWorkflowDeletion:- Pipeline record is deleted- Related workflow record is deleted"""+ workflow = _persist_pipeline_and_workflow(+ orm_session_maker,+ pipeline_id=pipeline_id,+ tenant_id=tenant_id,+ )+# Act- clean_dataset_task(+ _run_clean_dataset(dataset_id=dataset_id,tenant_id=tenant_id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- index_struct='{"type": "paragraph"}',collection_binding_id=collection_binding_id,- doc_form=IndexStructureType.PARAGRAPH_INDEX,pipeline_id=pipeline_id,)- # Assert - verify execute was called for delete operations- # 1 attachment JOIN query + 5 base deletes + 2 pipeline/workflow deletes = 8- assert mock_db_session.session.execute.call_count >= 8+ # Assert+ with orm_session_maker() as session:+ assert session.get(Pipeline, pipeline_id) is None+ assert session.get(Workflow, workflow.id) is Nonedef test_clean_dataset_task_without_pipeline_id(self,dataset_id: str,tenant_id: str,collection_binding_id: str,- mock_db_session,- mock_storage,- mock_index_processor_factory,- mock_get_image_upload_file_ids,+ pipeline_id: str,+ orm_session_maker: sessionmaker[Session],+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,):"""Test that pipeline/workflow deletion is skipped when pipeline_id is None.@@ -245,20 +344,24 @@ class TestPipelineAndWorkflowDeletion:Expected behavior:- Pipeline and workflow deletion queries are not executed"""+ workflow = _persist_pipeline_and_workflow(+ orm_session_maker,+ pipeline_id=pipeline_id,+ tenant_id=tenant_id,+ )+# Act- clean_dataset_task(+ _run_clean_dataset(dataset_id=dataset_id,tenant_id=tenant_id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- index_struct='{"type": "paragraph"}',collection_binding_id=collection_binding_id,- doc_form=IndexStructureType.PARAGRAPH_INDEX,pipeline_id=None,)- # Assert - verify execute was called for delete operations- # 1 attachment JOIN query + 5 base deletes = 6- assert mock_db_session.session.execute.call_count == 6+ # Assert+ with orm_session_maker() as session:+ assert session.get(Pipeline, pipeline_id) is not None+ assert session.get(Workflow, workflow.id) is not None# ============================================================================@@ -274,10 +377,10 @@ class TestSegmentAttachmentCleanup:dataset_id: str,tenant_id: str,collection_binding_id: str,- mock_db_session,- mock_storage,- mock_index_processor_factory,- mock_get_image_upload_file_ids,+ orm_session_maker: sessionmaker[Session],+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,):"""Test that segment attachments are cleaned up properly.@@ -292,42 +395,34 @@ class TestSegmentAttachmentCleanup:- Binding records are deleted from database"""# Arrange- mock_binding = MagicMock()- mock_binding.attachment_id = str(uuid.uuid4())-- mock_attachment_file = MagicMock()- mock_attachment_file.id = mock_binding.attachment_id- mock_attachment_file.key = f"attachments/{uuid.uuid4()}.pdf"-- # Setup execute to return attachment with binding- mock_db_session.session.execute.return_value.all.return_value = [(mock_binding, mock_attachment_file)]+ binding, attachment_file = _persist_attachment(+ orm_session_maker,+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ )# Act- clean_dataset_task(+ _run_clean_dataset(dataset_id=dataset_id,tenant_id=tenant_id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- index_struct='{"type": "paragraph"}',collection_binding_id=collection_binding_id,- doc_form=IndexStructureType.PARAGRAPH_INDEX,)# Assert- mock_storage.delete.assert_called_with(mock_attachment_file.key)- # Attachment file and binding are deleted in batch; verify DELETEs were issued- execute_sqls = [" ".join(str(c[0][0]).split()) for c in mock_db_session.session.execute.call_args_list]- assert any("DELETE FROM upload_files" in sql for sql in execute_sqls)- assert any("DELETE FROM segment_attachment_bindings" in sql for sql in execute_sqls)+ mock_storage.delete.assert_called_once_with(attachment_file.key)+ with orm_session_maker() as session:+ assert session.get(UploadFile, attachment_file.id) is None+ assert session.get(SegmentAttachmentBinding, binding.id) is Nonedef test_clean_dataset_task_attachment_storage_failure(self,dataset_id: str,tenant_id: str,collection_binding_id: str,- mock_db_session,- mock_storage,- mock_index_processor_factory,- mock_get_image_upload_file_ids,+ orm_session_maker: sessionmaker[Session],+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,):"""Test that cleanup continues even if attachment storage deletion fails.@@ -337,32 +432,25 @@ class TestSegmentAttachmentCleanup:- Attachment file and binding are still deleted from database"""# Arrange- mock_binding = MagicMock()- mock_binding.attachment_id = str(uuid.uuid4())-- mock_attachment_file = MagicMock()- mock_attachment_file.id = mock_binding.attachment_id- mock_attachment_file.key = f"attachments/{uuid.uuid4()}.pdf"-- mock_db_session.session.execute.return_value.all.return_value = [(mock_binding, mock_attachment_file)]+ binding, attachment_file = _persist_attachment(+ orm_session_maker,+ dataset_id=dataset_id,+ tenant_id=tenant_id,+ )mock_storage.delete.side_effect = Exception("Storage error")# Act- clean_dataset_task(+ _run_clean_dataset(dataset_id=dataset_id,tenant_id=tenant_id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- index_struct='{"type": "paragraph"}',collection_binding_id=collection_binding_id,- doc_form=IndexStructureType.PARAGRAPH_INDEX,)# Assert - storage delete was attempted- mock_storage.delete.assert_called_once()- # Records are deleted in batch; verify DELETEs were issued- execute_sqls = [" ".join(str(c[0][0]).split()) for c in mock_db_session.session.execute.call_args_list]- assert any("DELETE FROM upload_files" in sql for sql in execute_sqls)- assert any("DELETE FROM segment_attachment_bindings" in sql for sql in execute_sqls)+ mock_storage.delete.assert_called_once_with(attachment_file.key)+ with orm_session_maker() as session:+ assert session.get(UploadFile, attachment_file.id) is None+ assert session.get(SegmentAttachmentBinding, binding.id) is None# ============================================================================@@ -373,34 +461,35 @@ class TestSegmentAttachmentCleanup:class TestEdgeCases:"""Test edge cases and boundary conditions."""- def test_clean_dataset_task_session_always_closed(+ def test_clean_dataset_task_commits_cleanup_and_factory_remains_usable(self,dataset_id: str,tenant_id: str,collection_binding_id: str,- mock_db_session,- mock_storage,- mock_index_processor_factory,- mock_get_image_upload_file_ids,+ orm_session_maker: sessionmaker[Session],+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,):"""- Test that database session is always closed regardless of success or failure.+ Test that cleanup commits and the task-owned session releases its resources.Expected behavior:- - Session.close() is called in finally block+ - The document deletion is committed+ - A subsequent real session can use the same factory"""+ document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)+# Act- clean_dataset_task(+ _run_clean_dataset(dataset_id=dataset_id,tenant_id=tenant_id,- indexing_technique=IndexTechniqueType.HIGH_QUALITY,- index_struct='{"type": "paragraph"}',collection_binding_id=collection_binding_id,- doc_form=IndexStructureType.PARAGRAPH_INDEX,)# Assert- mock_db_session.session.close.assert_called_once()+ with orm_session_maker() as session:+ assert session.get(Document, document.id) is None# ============================================================================@@ -416,10 +505,10 @@ class TestIndexProcessorParameters:dataset_id: str,tenant_id: str,collection_binding_id: str,- mock_db_session,- mock_storage,- mock_index_processor_factory,- mock_get_image_upload_file_ids,+ orm_session_maker: sessionmaker[Session],+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,):"""Test that correct parameters are passed to IndexProcessor.clean().@@ -460,7 +549,9 @@ class TestIndexProcessorParameters:assert call_args[0][1] is None# Verify keyword arguments- assert call_args[1]["session"] is mock_db_session.session+ cleanup_session = call_args[1]["session"]+ assert isinstance(cleanup_session, Session)+ assert cleanup_session.get_bind() is orm_session_maker.kw["bind"]assert call_args[1]["with_keywords"] is Trueassert call_args[1]["delete_child_chunks"] is Trueschedule_refresh.assert_called_once_with(tenant_id)@@ -470,11 +561,11 @@ class TestIndexProcessorParameters:dataset_id: str,tenant_id: str,collection_binding_id: str,- mock_db_session,- mock_storage,- mock_index_processor_factory,- mock_get_image_upload_file_ids,- ):+ orm_session_maker: sessionmaker[Session],+ mock_storage: MagicMock,+ mock_index_processor_factory: dict[str, MagicMock],+ mock_get_image_upload_file_ids: MagicMock,+ ) -> None:mock_index_processor_factory["processor"].clean.side_effect = RuntimeError("vector cleanup failed")with patch("tasks.clean_dataset_task.schedule_billing_vector_space_refresh") as schedule_refresh: