Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor(api): enable strict typing for text and PDF extractor tests (#43635)

Eddy ZHANGf3ecedab5618
diff --git a/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py
index 5fab1844cbd..2f1b5287d74 100644
--- a/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py
+++ b/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py
@@ -1,3 +1,4 @@
+import io
from dataclasses import dataclass
from unittest.mock import MagicMock, patch
from uuid import uuid4
@@ -66,12 +67,12 @@ def test_extract_images_formats(
expected_mime: str,
expected_ext: str,
inject_session: bool,
-):
+) -> None:
# Mock page and image objects
mock_page = MagicMock()
mock_image_obj = MagicMock()
- def mock_extract(buf, fb_format=None):
+ def mock_extract(buf: io.BytesIO, fb_format: str | None = None) -> None:
buf.write(image_bytes)
mock_image_obj.extract.side_effect = mock_extract
@@ -118,8 +119,10 @@ def test_extract_images_formats(
)
@pytest.mark.parametrize("sqlite_session", [(UploadFile,)], indirect=True)
def test_extract_images_get_objects_scenarios(
- mock_dependencies: _Dependencies, get_objects_side_effect, get_objects_return_value
-):
+ mock_dependencies: _Dependencies,
+ get_objects_side_effect: Exception | None,
+ get_objects_return_value: list[object] | None,
+) -> None:
mock_page = MagicMock()
if get_objects_side_effect:
mock_page.get_objects.side_effect = get_objects_side_effect
@@ -136,7 +139,7 @@ def test_extract_images_get_objects_scenarios(
@pytest.mark.parametrize("sqlite_session", [(UploadFile,)], indirect=True)
-def test_extract_calls_extract_images(mock_dependencies: _Dependencies, monkeypatch: pytest.MonkeyPatch):
+def test_extract_calls_extract_images(mock_dependencies: _Dependencies, monkeypatch: pytest.MonkeyPatch) -> None:
# Mock pypdfium2
mock_pdf_doc = MagicMock()
mock_page = MagicMock()
@@ -166,7 +169,7 @@ def test_extract_calls_extract_images(mock_dependencies: _Dependencies, monkeypa
@pytest.mark.parametrize("sqlite_session", [(UploadFile,)], indirect=True)
-def test_extract_images_failures(mock_dependencies: _Dependencies):
+def test_extract_images_failures(mock_dependencies: _Dependencies) -> None:
# Mock page and image objects
mock_page = MagicMock()
mock_image_obj_fail = MagicMock()
@@ -178,7 +181,7 @@ def test_extract_images_failures(mock_dependencies: _Dependencies):
# Second image is OK (JPEG)
jpeg_bytes = b"\xff\xd8\xff some image data"
- def mock_extract(buf, fb_format=None):
+ def mock_extract(buf: io.BytesIO, fb_format: str | None = None) -> None:
buf.write(jpeg_bytes)
mock_image_obj_ok.extract.side_effect = mock_extract
@@ -198,7 +201,7 @@ def test_extract_images_failures(mock_dependencies: _Dependencies):
assert mock_dependencies.storage.saves == [(upload_file.key, jpeg_bytes)]
-def test_extract_images_skipped_without_tenant_context():
+def test_extract_images_skipped_without_tenant_context() -> None:
"""PDFs loaded from a URL have no tenant/user context; image extraction must be skipped."""
mock_page = MagicMock()
diff --git a/api/tests/unit_tests/core/rag/extractor/test_text_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_text_extractor.py
index 92edf3f4b8f..8fe0fa069af 100644
--- a/api/tests/unit_tests/core/rag/extractor/test_text_extractor.py
+++ b/api/tests/unit_tests/core/rag/extractor/test_text_extractor.py
@@ -8,7 +8,7 @@ from core.rag.extractor.text_extractor import TextExtractor
class TestTextExtractor:
- def test_extract_autodetect_preserves_full_file_beyond_sample(self, tmp_path: Path):
+ def test_extract_autodetect_preserves_full_file_beyond_sample(self, tmp_path: Path) -> None:
file_path = tmp_path / "large.txt"
content = "Document text\n" * 50_000 + "End of document"
file_path.write_text(content, encoding="utf-16")
@@ -20,7 +20,7 @@ class TestTextExtractor:
assert docs[0].page_content == content
assert docs[0].metadata == {"source": str(file_path)}
- def test_extract_success(self, tmp_path: Path):
+ def test_extract_success(self, tmp_path: Path) -> None:
file_path = tmp_path / "data.txt"
file_path.write_text("hello world", encoding="utf-8")
@@ -31,12 +31,12 @@ class TestTextExtractor:
assert docs[0].page_content == "hello world"
assert docs[0].metadata == {"source": str(file_path)}
- def test_extract_autodetect_success_after_decode_error(self, monkeypatch: pytest.MonkeyPatch):
+ def test_extract_autodetect_success_after_decode_error(self, monkeypatch: pytest.MonkeyPatch) -> None:
extractor = TextExtractor("dummy.txt", autodetect_encoding=True)
calls = []
- def fake_read_text(self, encoding=None):
+ def fake_read_text(self: Path, encoding: str | None = None) -> str:
calls.append(encoding)
if encoding is None:
raise UnicodeDecodeError("utf-8", b"x", 0, 1, "decode")
@@ -56,10 +56,10 @@ class TestTextExtractor:
assert docs[0].page_content == "decoded text"
assert calls == [None, "bad", "utf-8"]
- def test_extract_autodetect_all_fail_raises_runtime_error(self, monkeypatch: pytest.MonkeyPatch):
+ def test_extract_autodetect_all_fail_raises_runtime_error(self, monkeypatch: pytest.MonkeyPatch) -> None:
extractor = TextExtractor("dummy.txt", autodetect_encoding=True)
- def always_decode_error(self, encoding=None):
+ def always_decode_error(self: Path, encoding: str | None = None) -> str:
raise UnicodeDecodeError("utf-8", b"x", 0, 1, "decode")
monkeypatch.setattr(Path, "read_text", always_decode_error, raising=True)
@@ -68,10 +68,12 @@ class TestTextExtractor:
with pytest.raises(RuntimeError, match="all detected encodings failed"):
extractor.extract()
- def test_extract_decode_error_without_autodetect_raises_runtime_error(self, monkeypatch: pytest.MonkeyPatch):
+ def test_extract_decode_error_without_autodetect_raises_runtime_error(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
extractor = TextExtractor("dummy.txt", autodetect_encoding=False)
- def always_decode_error(self, encoding=None):
+ def always_decode_error(self: Path, encoding: str | None = None) -> str:
raise UnicodeDecodeError("utf-8", b"x", 0, 1, "decode")
monkeypatch.setattr(Path, "read_text", always_decode_error, raising=True)
@@ -79,10 +81,10 @@ class TestTextExtractor:
with pytest.raises(RuntimeError, match="specified encoding failed"):
extractor.extract()
- def test_extract_wraps_non_decode_exceptions(self, monkeypatch: pytest.MonkeyPatch):
+ def test_extract_wraps_non_decode_exceptions(self, monkeypatch: pytest.MonkeyPatch) -> None:
extractor = TextExtractor("dummy.txt")
- def raise_other(self, encoding=None):
+ def raise_other(self: Path, encoding: str | None = None) -> str:
raise OSError("io error")
monkeypatch.setattr(Path, "read_text", raise_other, raising=True)
diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.toml
index 52d9b303870..381f2f78e42 100644
--- a/api/tests/unit_tests/pyrefly.toml
+++ b/api/tests/unit_tests/pyrefly.toml
@@ -449,8 +449,6 @@ project-excludes = [
"core/rag/extractor/test_jina_reader_extractor.py",
"core/rag/extractor/test_markdown_extractor.py",
"core/rag/extractor/test_notion_extractor.py",
- "core/rag/extractor/test_pdf_extractor.py",
- "core/rag/extractor/test_text_extractor.py",
"core/rag/extractor/test_word_extractor.py",
"core/rag/extractor/unstructured/test_unstructured_extractors.py",
"core/rag/extractor/watercrawl/test_watercrawl.py",