Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test(api): bring dataset cleanup and summary adapter tests under strict pyrefly (#45698)
yetuge8b41f2e4db49
diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.tomlindex 345cbdad1a0..b1f30e11d7d 100644--- a/api/tests/unit_tests/pyrefly.toml+++ b/api/tests/unit_tests/pyrefly.toml@@ -303,7 +303,6 @@ project-excludes = ["services/test_snippet_dsl_service.py","services/test_snippet_generate_service.py","services/test_snippet_service.py",- "services/knowledge/summaries/test_adapters.py","services/trigger/test_trigger_provider_service.py","services/trigger/test_webhook_service.py","services/test_workflow_collaboration_service.py",@@ -313,10 +312,8 @@ project-excludes = ["services/test_workflow_draft_variable_service.py","services/workflow/test_workflow_converter.py","services/test_workflow_event_snapshot_service.py",- "tasks/test_clean_dataset_task.py","tasks/test_document_indexing_task.py","tasks/test_document_indexing_update_task.py",- "tasks/test_enable_segments_to_index_task.py","tasks/test_human_input_timeout_tasks.py","tasks/app_generate/test_workflow_execute_task.py","../e2e_tests/contracts/test_pytest_dify.py",diff --git a/api/tests/unit_tests/services/knowledge/summaries/test_adapters.py b/api/tests/unit_tests/services/knowledge/summaries/test_adapters.pyindex c60a9ee7bf0..768097030c3 100644--- a/api/tests/unit_tests/services/knowledge/summaries/test_adapters.py+++ b/api/tests/unit_tests/services/knowledge/summaries/test_adapters.py@@ -196,7 +196,7 @@ class TestGenerateAndCreate:usage = _usage()generate = _install_summary_generator(monkeypatch, usage=usage)- def call_model(**kwargs):+ def call_model(**kwargs: object) -> tuple[str, MagicMock]:assert not sqlite_session.in_transaction()return "generated summary", usage@@ -304,7 +304,7 @@ class TestVectorizeSummary:summary = _persist_summary(sqlite_session, dataset, document, segment)vector = _install_vector_dependencies(monkeypatch)- def write_vector(*args, **kwargs):+ def write_vector(*args: object, **kwargs: object) -> None:assert not sqlite_session.in_transaction()if vector.add_texts.call_count == 1:raise ConnectionError("connection timeout")@@ -436,12 +436,12 @@ class TestBatchAndGeneration:sqlite_session.commit()sessions = []- def create_session():+ def create_session() -> Session:session = sqlite_session_factory()sessions.append(session)return session- def generate(**kwargs):+ def generate(**kwargs: object) -> list[object]:assert sessionsassert all(not session.in_transaction() for session in sessions)return []diff --git a/api/tests/unit_tests/tasks/test_clean_dataset_task.py b/api/tests/unit_tests/tasks/test_clean_dataset_task.pyindex 36c50aa219d..aa007285139 100644--- a/api/tests/unit_tests/tasks/test_clean_dataset_task.py+++ b/api/tests/unit_tests/tasks/test_clean_dataset_task.py@@ -28,14 +28,9 @@ from sqlalchemy.orm import Session, sessionmakerfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom extensions.storage.storage_type import StorageType-from models.base import TypeBasefrom models.dataset import (- AppDatasetJoin,ChildChunk,DatasetMetadata,- DatasetMetadataBinding,- DatasetProcessRule,- DatasetQuery,Document,DocumentSegment,Pipeline,@@ -77,26 +72,8 @@ def pipeline_id() -> str:@pytest.fixture-def orm_session_maker(- sqlite_engine: Engine,- sqlite_session_factory: sessionmaker[Session],-) -> sessionmaker[Session]:- """Create the cleanup tables and return the suite's real SQLite session factory."""- models = (- Document,- DocumentSegment,- ChildChunk,- SegmentAttachmentBinding,- UploadFile,- DatasetProcessRule,- DatasetQuery,- AppDatasetJoin,- DatasetMetadata,- DatasetMetadataBinding,- Pipeline,- Workflow,- )- TypeBase.metadata.create_all(sqlite_engine, tables=[model.__table__ for model in models])+def orm_session_maker(sqlite_session_factory: sessionmaker[Session]) -> sessionmaker[Session]:+ """Return the suite's real SQLite session factory over the shared full schema."""return sqlite_session_factory@@ -129,7 +106,7 @@ def mock_index_processor_factory() -> Iterator[dict[str, MagicMock]]:def mock_get_image_upload_file_ids() -> Iterator[MagicMock]:"""Mock get_image_upload_file_ids function."""with patch("tasks.clean_dataset_task.get_image_upload_file_ids", autospec=True) as mock_func:- mock_func.return_value = []+ mock_func.return_value = list[str]()yield mock_func@@ -338,7 +315,7 @@ class TestErrorHandling:mock_storage: MagicMock,mock_index_processor_factory: dict[str, MagicMock],mock_get_image_upload_file_ids: MagicMock,- ):+ ) -> None:"""Test that session is closed even if rollback fails.@@ -400,7 +377,7 @@ class TestPipelineAndWorkflowDeletion:mock_storage: MagicMock,mock_index_processor_factory: dict[str, MagicMock],mock_get_image_upload_file_ids: MagicMock,- ):+ ) -> None:"""Test that pipeline and workflow are deleted when pipeline_id is provided.@@ -437,7 +414,7 @@ class TestPipelineAndWorkflowDeletion:mock_storage: MagicMock,mock_index_processor_factory: dict[str, MagicMock],mock_get_image_upload_file_ids: MagicMock,- ):+ ) -> None:"""Test that pipeline/workflow deletion is skipped when pipeline_id is None.@@ -490,7 +467,7 @@ class TestSegmentAttachmentCleanup:)binding, attachment_file = _persist_attachment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)segment.content = f""- document.data_source_type = "upload_file"+ document.data_source_type = DataSourceType.UPLOAD_FILEdocument.data_source_info = json.dumps({"upload_file_id": attachment_file.id})binding.document_id = document.idbinding.segment_id = segment.id@@ -557,7 +534,7 @@ class TestSegmentAttachmentCleanup:mock_storage: MagicMock,mock_index_processor_factory: dict[str, MagicMock],mock_get_image_upload_file_ids: MagicMock,- ):+ ) -> None:"""Test that segment attachments are cleaned up properly.@@ -599,7 +576,7 @@ class TestSegmentAttachmentCleanup:mock_storage: MagicMock,mock_index_processor_factory: dict[str, MagicMock],mock_get_image_upload_file_ids: MagicMock,- ):+ ) -> None:"""Test that cleanup continues even if attachment storage deletion fails.@@ -646,7 +623,7 @@ class TestEdgeCases:mock_storage: MagicMock,mock_index_processor_factory: dict[str, MagicMock],mock_get_image_upload_file_ids: MagicMock,- ):+ ) -> None:"""Test that cleanup commits and the task-owned session releases its resources.@@ -769,7 +746,7 @@ class TestIndexProcessorParameters:mock_storage: MagicMock,mock_index_processor_factory: dict[str, MagicMock],mock_get_image_upload_file_ids: MagicMock,- ):+ ) -> None:"""Test that correct parameters are passed to IndexProcessor.clean().@@ -990,7 +967,7 @@ class TestStepIsolation:]with orm_session_maker.begin() as session:for document, source_file in zip(documents, source_files, strict=True):- document.data_source_type = "upload_file"+ document.data_source_type = DataSourceType.UPLOAD_FILEdocument.data_source_info = json.dumps({"upload_file_id": source_file.id})session.add_all([document, source_file])@@ -1063,7 +1040,7 @@ class TestStepIsolation:mock_storage: MagicMock,mock_index_processor_factory: dict[str, MagicMock],mock_get_image_upload_file_ids: MagicMock,- ):+ ) -> None:"""A single commit at the end is what strands data when any step fails."""document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)_persist_segment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id)@@ -1356,7 +1333,7 @@ class TestBatchedDeletes:mock_storage: MagicMock,mock_index_processor_factory: dict[str, MagicMock],mock_get_image_upload_file_ids: MagicMock,- ):+ ) -> None:"""With a batch size smaller than the row count the loop must still drain thetable, rather than stopping after the first batch."""documents = [_persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id) for _ in range(5)]@@ -1448,13 +1425,19 @@ def test_dataset_cleanup_keeps_a_leased_legacy_source_file(@pytest.mark.parametrize("shared_tenant", [None, "same", "other"])-def test_crawl_cache_cleanup_preserves_shared_jobs(orm_session_maker, tenant_id, dataset_id, shared_tenant):+def test_crawl_cache_cleanup_preserves_shared_jobs(+ orm_session_maker: sessionmaker[Session],+ tenant_id: str,+ dataset_id: str,+ shared_tenant: str | None,+) -> None:from services import dataset_storage_cleanup_service as cleanupdocument = _persist_document(orm_session_maker, tenant_id=tenant_id, dataset_id=dataset_id)with orm_session_maker.begin() as session:row = session.get(Document, document.id)- row.data_source_type = "website_crawl"+ assert row is not None+ row.data_source_type = DataSourceType.WEBSITE_CRAWLrow.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})if shared_tenant:other = _persist_document(@@ -1464,7 +1447,8 @@ def test_crawl_cache_cleanup_preserves_shared_jobs(orm_session_maker, tenant_id,)with orm_session_maker.begin() as session:row = session.get(Document, other.id)- row.data_source_type = "website_crawl"+ assert row is not None+ row.data_source_type = DataSourceType.WEBSITE_CRAWLrow.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})with patch.object(cleanup, "storage", autospec=True) as mock:cleanup.cleanup_dataset_crawl_cache(tenant_id, dataset_id)@@ -1475,13 +1459,19 @@ def test_crawl_cache_cleanup_preserves_shared_jobs(orm_session_maker, tenant_id,@pytest.mark.parametrize("error", [FileNotFoundError(), PermissionError("denied")])-def test_crawl_cache_failure_preserves_document(orm_session_maker, tenant_id, dataset_id, error):+def test_crawl_cache_failure_preserves_document(+ orm_session_maker: sessionmaker[Session],+ tenant_id: str,+ dataset_id: str,+ error: Exception,+) -> None:from services import dataset_storage_cleanup_service as cleanupdocument = _persist_document(orm_session_maker, tenant_id=tenant_id, dataset_id=dataset_id)with orm_session_maker.begin() as session:row = session.get(Document, document.id)- row.data_source_type = "website_crawl"+ assert row is not None+ row.data_source_type = DataSourceType.WEBSITE_CRAWLrow.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})with patch.object(cleanup, "storage", autospec=True) as mock:mock.delete.side_effect = error@@ -1497,7 +1487,7 @@ def test_crawl_cache_failure_preserves_document(orm_session_maker, tenant_id, da@pytest.mark.parametrize("info", ["broken", "[]", '{"provider":"watercrawl","job_id":"job"}', '{"provider":"firecrawl","job_id":"../job"}'])-def test_crawl_cache_ignores_invalid_or_other_provider_info(info):+def test_crawl_cache_ignores_invalid_or_other_provider_info(info: str) -> None:from services.dataset_storage_cleanup_service import _crawl_cache_keyassert _crawl_cache_key(info) is None