Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test(api): bring dataset cleanup and summary adapter tests under strict pyrefly (#45698)

yetuge8b41f2e4db49
diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.toml
index 345cbdad1a0..b1f30e11d7d 100644
--- a/api/tests/unit_tests/pyrefly.toml
+++ b/api/tests/unit_tests/pyrefly.toml
@@ -303,7 +303,6 @@ project-excludes = [
"services/test_snippet_dsl_service.py",
"services/test_snippet_generate_service.py",
"services/test_snippet_service.py",
- "services/knowledge/summaries/test_adapters.py",
"services/trigger/test_trigger_provider_service.py",
"services/trigger/test_webhook_service.py",
"services/test_workflow_collaboration_service.py",
@@ -313,10 +312,8 @@ project-excludes = [
"services/test_workflow_draft_variable_service.py",
"services/workflow/test_workflow_converter.py",
"services/test_workflow_event_snapshot_service.py",
- "tasks/test_clean_dataset_task.py",
"tasks/test_document_indexing_task.py",
"tasks/test_document_indexing_update_task.py",
- "tasks/test_enable_segments_to_index_task.py",
"tasks/test_human_input_timeout_tasks.py",
"tasks/app_generate/test_workflow_execute_task.py",
"../e2e_tests/contracts/test_pytest_dify.py",
diff --git a/api/tests/unit_tests/services/knowledge/summaries/test_adapters.py b/api/tests/unit_tests/services/knowledge/summaries/test_adapters.py
index c60a9ee7bf0..768097030c3 100644
--- a/api/tests/unit_tests/services/knowledge/summaries/test_adapters.py
+++ b/api/tests/unit_tests/services/knowledge/summaries/test_adapters.py
@@ -196,7 +196,7 @@ class TestGenerateAndCreate:
usage = _usage()
generate = _install_summary_generator(monkeypatch, usage=usage)
- def call_model(**kwargs):
+ def call_model(**kwargs: object) -> tuple[str, MagicMock]:
assert not sqlite_session.in_transaction()
return "generated summary", usage
@@ -304,7 +304,7 @@ class TestVectorizeSummary:
summary = _persist_summary(sqlite_session, dataset, document, segment)
vector = _install_vector_dependencies(monkeypatch)
- def write_vector(*args, **kwargs):
+ def write_vector(*args: object, **kwargs: object) -> None:
assert not sqlite_session.in_transaction()
if vector.add_texts.call_count == 1:
raise ConnectionError("connection timeout")
@@ -436,12 +436,12 @@ class TestBatchAndGeneration:
sqlite_session.commit()
sessions = []
- def create_session():
+ def create_session() -> Session:
session = sqlite_session_factory()
sessions.append(session)
return session
- def generate(**kwargs):
+ def generate(**kwargs: object) -> list[object]:
assert sessions
assert all(not session.in_transaction() for session in sessions)
return []
diff --git a/api/tests/unit_tests/tasks/test_clean_dataset_task.py b/api/tests/unit_tests/tasks/test_clean_dataset_task.py
index 36c50aa219d..aa007285139 100644
--- a/api/tests/unit_tests/tasks/test_clean_dataset_task.py
+++ b/api/tests/unit_tests/tasks/test_clean_dataset_task.py
@@ -28,14 +28,9 @@ from sqlalchemy.orm import Session, sessionmaker
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from extensions.storage.storage_type import StorageType
-from models.base import TypeBase
from models.dataset import (
- AppDatasetJoin,
ChildChunk,
DatasetMetadata,
- DatasetMetadataBinding,
- DatasetProcessRule,
- DatasetQuery,
Document,
DocumentSegment,
Pipeline,
@@ -77,26 +72,8 @@ def pipeline_id() -> str:
@pytest.fixture
-def orm_session_maker(
- sqlite_engine: Engine,
- sqlite_session_factory: sessionmaker[Session],
-) -> sessionmaker[Session]:
- """Create the cleanup tables and return the suite's real SQLite session factory."""
- models = (
- Document,
- DocumentSegment,
- ChildChunk,
- SegmentAttachmentBinding,
- UploadFile,
- DatasetProcessRule,
- DatasetQuery,
- AppDatasetJoin,
- DatasetMetadata,
- DatasetMetadataBinding,
- Pipeline,
- Workflow,
- )
- TypeBase.metadata.create_all(sqlite_engine, tables=[model.__table__ for model in models])
+def orm_session_maker(sqlite_session_factory: sessionmaker[Session]) -> sessionmaker[Session]:
+ """Return the suite's real SQLite session factory over the shared full schema."""
return sqlite_session_factory
@@ -129,7 +106,7 @@ def mock_index_processor_factory() -> Iterator[dict[str, MagicMock]]:
def mock_get_image_upload_file_ids() -> Iterator[MagicMock]:
"""Mock get_image_upload_file_ids function."""
with patch("tasks.clean_dataset_task.get_image_upload_file_ids", autospec=True) as mock_func:
- mock_func.return_value = []
+ mock_func.return_value = list[str]()
yield mock_func
@@ -338,7 +315,7 @@ class TestErrorHandling:
mock_storage: MagicMock,
mock_index_processor_factory: dict[str, MagicMock],
mock_get_image_upload_file_ids: MagicMock,
- ):
+ ) -> None:
"""
Test that session is closed even if rollback fails.
@@ -400,7 +377,7 @@ class TestPipelineAndWorkflowDeletion:
mock_storage: MagicMock,
mock_index_processor_factory: dict[str, MagicMock],
mock_get_image_upload_file_ids: MagicMock,
- ):
+ ) -> None:
"""
Test that pipeline and workflow are deleted when pipeline_id is provided.
@@ -437,7 +414,7 @@ class TestPipelineAndWorkflowDeletion:
mock_storage: MagicMock,
mock_index_processor_factory: dict[str, MagicMock],
mock_get_image_upload_file_ids: MagicMock,
- ):
+ ) -> None:
"""
Test that pipeline/workflow deletion is skipped when pipeline_id is None.
@@ -490,7 +467,7 @@ class TestSegmentAttachmentCleanup:
)
binding, attachment_file = _persist_attachment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
segment.content = f"![image](https://example.com/files/{attachment_file.id}/file-preview)"
- document.data_source_type = "upload_file"
+ document.data_source_type = DataSourceType.UPLOAD_FILE
document.data_source_info = json.dumps({"upload_file_id": attachment_file.id})
binding.document_id = document.id
binding.segment_id = segment.id
@@ -557,7 +534,7 @@ class TestSegmentAttachmentCleanup:
mock_storage: MagicMock,
mock_index_processor_factory: dict[str, MagicMock],
mock_get_image_upload_file_ids: MagicMock,
- ):
+ ) -> None:
"""
Test that segment attachments are cleaned up properly.
@@ -599,7 +576,7 @@ class TestSegmentAttachmentCleanup:
mock_storage: MagicMock,
mock_index_processor_factory: dict[str, MagicMock],
mock_get_image_upload_file_ids: MagicMock,
- ):
+ ) -> None:
"""
Test that cleanup continues even if attachment storage deletion fails.
@@ -646,7 +623,7 @@ class TestEdgeCases:
mock_storage: MagicMock,
mock_index_processor_factory: dict[str, MagicMock],
mock_get_image_upload_file_ids: MagicMock,
- ):
+ ) -> None:
"""
Test that cleanup commits and the task-owned session releases its resources.
@@ -769,7 +746,7 @@ class TestIndexProcessorParameters:
mock_storage: MagicMock,
mock_index_processor_factory: dict[str, MagicMock],
mock_get_image_upload_file_ids: MagicMock,
- ):
+ ) -> None:
"""
Test that correct parameters are passed to IndexProcessor.clean().
@@ -990,7 +967,7 @@ class TestStepIsolation:
]
with orm_session_maker.begin() as session:
for document, source_file in zip(documents, source_files, strict=True):
- document.data_source_type = "upload_file"
+ document.data_source_type = DataSourceType.UPLOAD_FILE
document.data_source_info = json.dumps({"upload_file_id": source_file.id})
session.add_all([document, source_file])
@@ -1063,7 +1040,7 @@ class TestStepIsolation:
mock_storage: MagicMock,
mock_index_processor_factory: dict[str, MagicMock],
mock_get_image_upload_file_ids: MagicMock,
- ):
+ ) -> None:
"""A single commit at the end is what strands data when any step fails."""
document = _persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id)
_persist_segment(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id, document_id=document.id)
@@ -1356,7 +1333,7 @@ class TestBatchedDeletes:
mock_storage: MagicMock,
mock_index_processor_factory: dict[str, MagicMock],
mock_get_image_upload_file_ids: MagicMock,
- ):
+ ) -> None:
"""With a batch size smaller than the row count the loop must still drain the
table, rather than stopping after the first batch."""
documents = [_persist_document(orm_session_maker, dataset_id=dataset_id, tenant_id=tenant_id) for _ in range(5)]
@@ -1448,13 +1425,19 @@ def test_dataset_cleanup_keeps_a_leased_legacy_source_file(
@pytest.mark.parametrize("shared_tenant", [None, "same", "other"])
-def test_crawl_cache_cleanup_preserves_shared_jobs(orm_session_maker, tenant_id, dataset_id, shared_tenant):
+def test_crawl_cache_cleanup_preserves_shared_jobs(
+ orm_session_maker: sessionmaker[Session],
+ tenant_id: str,
+ dataset_id: str,
+ shared_tenant: str | None,
+) -> None:
from services import dataset_storage_cleanup_service as cleanup
document = _persist_document(orm_session_maker, tenant_id=tenant_id, dataset_id=dataset_id)
with orm_session_maker.begin() as session:
row = session.get(Document, document.id)
- row.data_source_type = "website_crawl"
+ assert row is not None
+ row.data_source_type = DataSourceType.WEBSITE_CRAWL
row.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})
if shared_tenant:
other = _persist_document(
@@ -1464,7 +1447,8 @@ def test_crawl_cache_cleanup_preserves_shared_jobs(orm_session_maker, tenant_id,
)
with orm_session_maker.begin() as session:
row = session.get(Document, other.id)
- row.data_source_type = "website_crawl"
+ assert row is not None
+ row.data_source_type = DataSourceType.WEBSITE_CRAWL
row.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})
with patch.object(cleanup, "storage", autospec=True) as mock:
cleanup.cleanup_dataset_crawl_cache(tenant_id, dataset_id)
@@ -1475,13 +1459,19 @@ def test_crawl_cache_cleanup_preserves_shared_jobs(orm_session_maker, tenant_id,
@pytest.mark.parametrize("error", [FileNotFoundError(), PermissionError("denied")])
-def test_crawl_cache_failure_preserves_document(orm_session_maker, tenant_id, dataset_id, error):
+def test_crawl_cache_failure_preserves_document(
+ orm_session_maker: sessionmaker[Session],
+ tenant_id: str,
+ dataset_id: str,
+ error: Exception,
+) -> None:
from services import dataset_storage_cleanup_service as cleanup
document = _persist_document(orm_session_maker, tenant_id=tenant_id, dataset_id=dataset_id)
with orm_session_maker.begin() as session:
row = session.get(Document, document.id)
- row.data_source_type = "website_crawl"
+ assert row is not None
+ row.data_source_type = DataSourceType.WEBSITE_CRAWL
row.data_source_info = json.dumps({"provider": "firecrawl", "job_id": "job-42"})
with patch.object(cleanup, "storage", autospec=True) as mock:
mock.delete.side_effect = error
@@ -1497,7 +1487,7 @@ def test_crawl_cache_failure_preserves_document(orm_session_maker, tenant_id, da
@pytest.mark.parametrize(
"info", ["broken", "[]", '{"provider":"watercrawl","job_id":"job"}', '{"provider":"firecrawl","job_id":"../job"}']
)
-def test_crawl_cache_ignores_invalid_or_other_provider_info(info):
+def test_crawl_cache_ignores_invalid_or_other_provider_info(info: str) -> None:
from services.dataset_storage_cleanup_service import _crawl_cache_key
assert _crawl_cache_key(info) is None