Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(api): enable strict typing for datasource file upload tests (#43653)
Eddy ZHANG0e78aa5829be
diff --git a/api/tests/unit_tests/core/datasource/test_file_upload.py b/api/tests/unit_tests/core/datasource/test_file_upload.pyindex 4319cc3e60f..18c9b6c3924 100644--- a/api/tests/unit_tests/core/datasource/test_file_upload.py+++ b/api/tests/unit_tests/core/datasource/test_file_upload.py@@ -155,7 +155,7 @@ class TestFileTypeValidation:("tiff", False),],)- def test_image_extension_in_constants(self, extension, expected_in_set):+ def test_image_extension_in_constants(self, extension: str, expected_in_set: bool) -> None:"""Test that image extensions are correctly defined in constants."""# Actresult = extension in IMAGE_EXTENSIONS or extension.lower() in IMAGE_EXTENSIONS@@ -167,7 +167,7 @@ class TestFileTypeValidation:"extension",["mp4", "mov", "mpeg", "webm", "MP4", "MOV"],)- def test_video_extension_in_constants(self, extension):+ def test_video_extension_in_constants(self, extension: str) -> None:"""Test that video extensions are correctly defined in constants."""# Act & Assertassert extension in VIDEO_EXTENSIONS or extension.lower() in VIDEO_EXTENSIONS@@ -176,7 +176,7 @@ class TestFileTypeValidation:"extension",["mp3", "m4a", "wav", "amr", "mpga", "MP3", "WAV"],)- def test_audio_extension_in_constants(self, extension):+ def test_audio_extension_in_constants(self, extension: str) -> None:"""Test that audio extensions are correctly defined in constants."""# Act & Assertassert extension in AUDIO_EXTENSIONS or extension.lower() in AUDIO_EXTENSIONS@@ -185,12 +185,12 @@ class TestFileTypeValidation:"extension",["txt", "pdf", "docx", "xlsx", "csv", "md", "html", "TXT", "PDF"],)- def test_document_extension_in_constants(self, extension):+ def test_document_extension_in_constants(self, extension: str) -> None:"""Test that document extensions are correctly defined in constants."""# Act & Assertassert extension in DOCUMENT_EXTENSIONS or extension.lower() in DOCUMENT_EXTENSIONS- def test_dataset_source_document_validation(self):+ def test_dataset_source_document_validation(self) -> None:"""Test dataset source document type validation logic."""# Arrangevalid_extensions = ["pdf", "txt", "docx"]@@ -216,7 +216,7 @@ class TestFileSizeLimiting:- Edge cases (exactly at limit)"""- def test_is_file_size_within_limit_image(self):+ def test_is_file_size_within_limit_image(self) -> None:"""Test file size validation logic for images.This test validates the size limit checking algorithm for image files.@@ -264,7 +264,7 @@ class TestFileSizeLimiting:assert check_size(image_ext, size_exceeds_limit) is False # Should reject files over limitassert check_size(image_ext, size_at_limit) is True # Should accept files exactly at limit- def test_is_file_size_within_limit_video(self):+ def test_is_file_size_within_limit_video(self) -> None:"""Test file size validation logic for videos."""# Arrangevideo_ext = "mp4"@@ -289,7 +289,7 @@ class TestFileSizeLimiting:assert check_size(video_ext, size_exceeds_limit) is Falseassert check_size(video_ext, size_at_limit) is True- def test_is_file_size_within_limit_audio(self):+ def test_is_file_size_within_limit_audio(self) -> None:"""Test file size validation logic for audio files."""# Arrangeaudio_ext = "mp3"@@ -314,7 +314,7 @@ class TestFileSizeLimiting:assert check_size(audio_ext, size_exceeds_limit) is Falseassert check_size(audio_ext, size_at_limit) is True- def test_is_file_size_within_limit_general(self):+ def test_is_file_size_within_limit_general(self) -> None:"""Test file size validation logic for general files."""# Arrangegeneral_ext = "pdf"@@ -353,7 +353,7 @@ class TestVirusScanningIntegration:- Future: Scan service unavailability"""- def test_no_virus_scanning_currently_implemented(self):+ def test_no_virus_scanning_currently_implemented(self) -> None:"""Test that no virus scanning is currently implemented."""# This test documents that virus scanning is not yet implemented# When virus scanning is added, this test should be updated@@ -392,7 +392,7 @@ class TestStoragePathGeneration:- Extension preservation"""- def test_storage_path_format(self):+ def test_storage_path_format(self) -> None:"""Test that storage path follows correct format."""# Arrangetenant_id = str(uuid.uuid4())@@ -407,7 +407,7 @@ class TestStoragePathGeneration:assert tenant_id in file_keyassert file_key.endswith(f".{extension}")- def test_storage_path_uniqueness(self):+ def test_storage_path_uniqueness(self) -> None:"""Test that UUID generation ensures unique paths."""# Arrange & Actuuid1 = str(uuid.uuid4())@@ -416,7 +416,7 @@ class TestStoragePathGeneration:# Assertassert uuid1 != uuid2- def test_storage_path_includes_tenant_id(self):+ def test_storage_path_includes_tenant_id(self) -> None:"""Test that storage path includes tenant ID."""# Arrangetenant_id = str(uuid.uuid4())@@ -438,7 +438,7 @@ class TestStoragePathGeneration:("test.DOCX", "docx"),],)- def test_extension_extraction_and_lowercasing(self, filename, expected_ext):+ def test_extension_extraction_and_lowercasing(self, filename: str, expected_ext: str) -> None:"""Test that file extension is correctly extracted and lowercased."""# Actextension = os.path.splitext(filename)[1].lstrip(".").lower()@@ -457,7 +457,7 @@ class TestDuplicateDetection:- Same content with different names"""- def test_file_hash_generation(self):+ def test_file_hash_generation(self) -> None:"""Test that file hash is generated correctly using SHA3-256.File hashing is critical for duplicate detection. The system uses SHA3-256@@ -486,7 +486,7 @@ class TestDuplicateDetection:# Verify hash contains only valid hexadecimal charactersassert all(c in "0123456789abcdef" for c in actual_hash)- def test_identical_content_same_hash(self):+ def test_identical_content_same_hash(self) -> None:"""Test that identical content produces same hash."""# Arrangecontent = b"identical content"@@ -498,7 +498,7 @@ class TestDuplicateDetection:# Assertassert hash1 == hash2- def test_different_content_different_hash(self):+ def test_different_content_different_hash(self) -> None:"""Test that different content produces different hash."""# Arrangecontent1 = b"content one"@@ -511,7 +511,7 @@ class TestDuplicateDetection:# Assertassert hash1 != hash2- def test_hash_consistency(self):+ def test_hash_consistency(self) -> None:"""Test that hash generation is consistent across multiple calls."""# Arrangecontent = b"consistent content"@@ -536,7 +536,7 @@ class TestInvalidFilenameHandling:"invalid_char",["/", "\\"],)- def test_filename_contains_invalid_characters(self, invalid_char):+ def test_filename_contains_invalid_characters(self, invalid_char: str) -> None:"""Test detection of invalid characters in filename.Security-critical test that validates rejection of path separators.@@ -558,7 +558,7 @@ class TestInvalidFilenameHandling:# Assert - Should detect the invalid characterassert has_invalid_char is True- def test_valid_filename_no_invalid_characters(self):+ def test_valid_filename_no_invalid_characters(self) -> None:"""Test that valid filenames pass validation."""# Arrangefilename = "valid_file-name_123.txt"@@ -571,7 +571,7 @@ class TestInvalidFilenameHandling:assert has_invalid_char is False@pytest.mark.parametrize("safe_char", [":", "*", "?", '"', "<", ">", "|"])- def test_filename_allows_safe_metadata_characters(self, safe_char):+ def test_filename_allows_safe_metadata_characters(self, safe_char: str) -> None:"""Test that non-separator punctuation remains allowed in filenames."""filename = f"candidate{safe_char}resume.txt"invalid_chars = ["/", "\\"]@@ -580,7 +580,7 @@ class TestInvalidFilenameHandling:assert has_invalid_char is False- def test_extremely_long_filename_truncation(self):+ def test_extremely_long_filename_truncation(self) -> None:"""Test handling of extremely long filenames."""# Arrangelong_name = "a" * 250@@ -598,7 +598,7 @@ class TestInvalidFilenameHandling:assert len(truncated_filename) <= max_length + len(extension) + 1assert truncated_filename.endswith(".txt")- def test_path_traversal_detection(self):+ def test_path_traversal_detection(self) -> None:"""Test that path traversal attempts are detected."""# Arrangemalicious_filenames = [@@ -635,7 +635,7 @@ class TestBlacklistedExtensions:("BAT", {"exe", "bat", "sh"}, True),],)- def test_blacklist_extension_checking(self, extension, blacklist, should_block):+ def test_blacklist_extension_checking(self, extension: str, blacklist: set[str], should_block: bool) -> None:"""Test blacklist extension checking logic."""# Actis_blocked = extension.lower() in blacklist@@ -643,7 +643,7 @@ class TestBlacklistedExtensions:# Assertassert is_blocked == should_block- def test_empty_blacklist_allows_all(self):+ def test_empty_blacklist_allows_all(self) -> None:"""Test that empty blacklist allows all extensions."""# Arrangeextensions = ["exe", "bat", "txt", "pdf", "dll"]@@ -653,7 +653,7 @@ class TestBlacklistedExtensions:for ext in extensions:assert ext.lower() not in blacklist- def test_blacklist_configuration(self):+ def test_blacklist_configuration(self) -> None:"""Test that blacklist configuration is accessible."""# Actblacklist = dify_config.UPLOAD_FILE_EXTENSION_BLACKLIST@@ -672,17 +672,17 @@ class TestUserRoleHandling:- Correct creator role values"""- def test_account_user_role_value(self):+ def test_account_user_role_value(self) -> None:"""Test Account user role enum value."""# Act & Assertassert CreatorUserRole.ACCOUNT.value == "account"- def test_end_user_role_value(self):+ def test_end_user_role_value(self) -> None:"""Test EndUser role enum value."""# Act & Assertassert CreatorUserRole.END_USER.value == "end_user"- def test_creator_role_detection_account(self):+ def test_creator_role_detection_account(self) -> None:"""Test creator role detection for Account user."""# Arrangefrom models import Account@@ -696,7 +696,7 @@ class TestUserRoleHandling:# Assertassert role == CreatorUserRole.ACCOUNT- def test_creator_role_detection_end_user(self):+ def test_creator_role_detection_end_user(self) -> None:"""Test creator role detection for EndUser."""# Arrangefrom models import Account, EndUser@@ -726,7 +726,7 @@ class TestSourceUrlGeneration:- Automatic URL generation logic"""- def test_source_url_format(self):+ def test_source_url_format(self) -> None:"""Test that source URL follows expected format."""# Arrangefile_id = str(uuid.uuid4())@@ -739,7 +739,7 @@ class TestSourceUrlGeneration:assert source_url.startswith("https://")assert file_id in source_url- def test_custom_source_url_preservation(self):+ def test_custom_source_url_preservation(self) -> None:"""Test that custom source URL is used when provided."""# Arrangecustom_url = "https://custom.example.com/file/abc"@@ -751,7 +751,7 @@ class TestSourceUrlGeneration:# Assertassert final_url == custom_url- def test_automatic_source_url_generation(self):+ def test_automatic_source_url_generation(self) -> None:"""Test automatic source URL generation when not provided."""# Arrangecustom_url = ""@@ -775,7 +775,7 @@ class TestFileUploadIntegration:- Error inheritance"""- def test_file_too_large_error_exists(self):+ def test_file_too_large_error_exists(self) -> None:"""Test that FileTooLargeError is defined and properly structured."""# Actfrom services.errors.file import FileTooLargeError@@ -786,7 +786,7 @@ class TestFileUploadIntegration:error = FileTooLargeError()assert error is not None- def test_unsupported_file_type_error_exists(self):+ def test_unsupported_file_type_error_exists(self) -> None:"""Test that UnsupportedFileTypeError is defined and properly structured."""# Actfrom services.errors.file import UnsupportedFileTypeError@@ -797,7 +797,7 @@ class TestFileUploadIntegration:error = UnsupportedFileTypeError()assert error is not None- def test_blocked_file_extension_error_exists(self):+ def test_blocked_file_extension_error_exists(self) -> None:"""Test that BlockedFileExtensionError is defined and properly structured."""# Actfrom services.errors.file import BlockedFileExtensionError@@ -808,7 +808,7 @@ class TestFileUploadIntegration:error = BlockedFileExtensionError()assert error is not None- def test_file_not_exists_error_exists(self):+ def test_file_not_exists_error_exists(self) -> None:"""Test that FileNotExistsError is defined and properly structured."""# Actfrom services.errors.file import FileNotExistsError@@ -841,7 +841,7 @@ class TestFileExtensionNormalization:("mixed.CaSe.PnG", "png"),],)- def test_extension_extraction_and_normalization(self, filename, expected_extension):+ def test_extension_extraction_and_normalization(self, filename: str, expected_extension: str) -> None:"""Test that file extensions are correctly extracted and normalized to lowercase.This mimics the logic in FileService.upload_file where:@@ -853,7 +853,7 @@ class TestFileExtensionNormalization:# Assert - Verify correct extraction and normalizationassert extension == expected_extension- def test_filename_without_extension(self):+ def test_filename_without_extension(self) -> None:"""Test handling of filenames without extensions."""# Arrangefilename = "README"@@ -864,7 +864,7 @@ class TestFileExtensionNormalization:# Assert - Should return empty stringassert extension == ""- def test_hidden_file_with_extension(self):+ def test_hidden_file_with_extension(self) -> None:"""Test handling of hidden files (starting with dot) with extensions."""# Arrangefilename = ".gitignore"@@ -875,7 +875,7 @@ class TestFileExtensionNormalization:# Assert - Should return empty string (no extension after the dot)assert extension == ""- def test_hidden_file_with_actual_extension(self):+ def test_hidden_file_with_actual_extension(self) -> None:"""Test handling of hidden files with actual extensions."""# Arrangefilename = ".config.json"@@ -897,7 +897,7 @@ class TestFilenameValidation:- Empty filename detection"""- def test_empty_filename_detection(self):+ def test_empty_filename_detection(self) -> None:"""Test detection of empty filenames."""# Arrangeempty_filenames = ["", " ", " ", "\t", "\n"]@@ -906,7 +906,7 @@ class TestFilenameValidation:for filename in empty_filenames:assert filename.strip() == ""- def test_filename_with_spaces(self):+ def test_filename_with_spaces(self) -> None:"""Test that filenames with spaces are handled correctly."""# Arrangefilename = "my document with spaces.pdf"@@ -918,7 +918,7 @@ class TestFilenameValidation:# Assert - Spaces are allowedassert has_invalid is False- def test_filename_with_unicode_characters(self):+ def test_filename_with_unicode_characters(self) -> None:"""Test that filenames with unicode characters are handled."""# Arrangeunicode_filenames = [@@ -934,7 +934,7 @@ class TestFilenameValidation:has_invalid = any(c in filename for c in invalid_chars)assert has_invalid is False- def test_filename_length_boundary_cases(self):+ def test_filename_length_boundary_cases(self) -> None:"""Test filename length at various boundary conditions."""# Arrangemax_length = 200@@ -992,7 +992,7 @@ class TestMimeTypeHandling:("html", "text/"),],)- def test_mime_type_category_mapping(self, extension, expected_mime_prefix):+ def test_mime_type_category_mapping(self, extension: str, expected_mime_prefix: str) -> None:"""Test that file extensions map to appropriate MIME type categories.This validates the general category of MIME types expected for different@@ -1019,7 +1019,7 @@ class TestMimeTypeHandling:# Assert - Verify MIME type starts with expected prefixassert mime_type.startswith(expected_mime_prefix)- def test_unknown_extension_fallback_mime_type(self):+ def test_unknown_extension_fallback_mime_type(self) -> None:"""Test that unknown extensions fall back to generic MIME type."""# Arrangeunknown_extensions = ["xyz", "unknown", "custom"]@@ -1041,7 +1041,7 @@ class TestStorageKeyGeneration:- Collision prevention"""- def test_storage_key_components(self):+ def test_storage_key_components(self) -> None:"""Test that storage keys contain all required components.Storage keys should follow the format:@@ -1067,7 +1067,7 @@ class TestStorageKeyGeneration:assert parts[0] == "upload_files"assert parts[1] == tenant_id- def test_uuid_collision_probability(self):+ def test_uuid_collision_probability(self) -> None:"""Test UUID generation for collision resistance.UUIDs should be unique across multiple generations to prevent@@ -1082,7 +1082,7 @@ class TestStorageKeyGeneration:# Assert - All should be uniqueassert len(generated_uuids) == len(set(generated_uuids))- def test_storage_key_path_safety(self):+ def test_storage_key_path_safety(self) -> None:"""Test that generated storage keys don't contain path traversal sequences."""# Arrangetenant_id = str(uuid.uuid4())@@ -1108,7 +1108,7 @@ class TestFileHashingConsistency:- Binary content handling"""- def test_hash_algorithm_sha3_256(self):+ def test_hash_algorithm_sha3_256(self) -> None:"""Test that SHA3-256 algorithm produces expected hash length."""# Arrangecontent = b"test content"@@ -1120,7 +1120,7 @@ class TestFileHashingConsistency:assert len(file_hash) == 64assert all(c in "0123456789abcdef" for c in file_hash)- def test_hash_deterministic_behavior(self):+ def test_hash_deterministic_behavior(self) -> None:"""Test that hashing the same content always produces the same hash.This is critical for duplicate detection functionality.@@ -1136,7 +1136,7 @@ class TestFileHashingConsistency:# Assert - All hashes should be identicalassert hash1 == hash2 == hash3- def test_hash_sensitivity_to_content_changes(self):+ def test_hash_sensitivity_to_content_changes(self) -> None:"""Test that even small changes in content produce different hashes."""# Arrangecontent1 = b"original content"@@ -1153,7 +1153,7 @@ class TestFileHashingConsistency:assert hash1 != hash3assert hash2 != hash3- def test_hash_binary_content_handling(self):+ def test_hash_binary_content_handling(self) -> None:"""Test that binary content is properly hashed."""# Arrange - Create binary content with various byte valuesbinary_content = bytes(range(256)) # All possible byte values@@ -1165,7 +1165,7 @@ class TestFileHashingConsistency:assert len(file_hash) == 64assert file_hash is not None- def test_hash_empty_content(self):+ def test_hash_empty_content(self) -> None:"""Test hashing of empty content."""# Arrangeempty_content = b""@@ -1190,7 +1190,7 @@ class TestConfigurationValidation:- Configuration accessibility"""- def test_upload_size_limits_are_positive(self):+ def test_upload_size_limits_are_positive(self) -> None:"""Test that all upload size limits are positive values."""# Act & Assert - All size limits should be positiveassert dify_config.UPLOAD_FILE_SIZE_LIMIT > 0@@ -1198,7 +1198,7 @@ class TestConfigurationValidation:assert dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT > 0assert dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT > 0- def test_upload_size_limits_reasonable_values(self):+ def test_upload_size_limits_reasonable_values(self) -> None:"""Test that upload size limits are within reasonable ranges.This prevents misconfiguration that could cause issues.@@ -1212,7 +1212,7 @@ class TestConfigurationValidation:assert min_size <= dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT <= max_sizeassert min_size <= dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT <= max_size- def test_video_size_limit_larger_than_image(self):+ def test_video_size_limit_larger_than_image(self) -> None:"""Test that video size limit is typically larger than image limit.This reflects the expected configuration where videos are larger files.@@ -1220,7 +1220,7 @@ class TestConfigurationValidation:# Assert - Video limit should generally be >= image limitassert dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT >= dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT- def test_blacklist_is_set_type(self):+ def test_blacklist_is_set_type(self) -> None:"""Test that file extension blacklist is a set for efficient lookup."""# Actblacklist = dify_config.UPLOAD_FILE_EXTENSION_BLACKLIST@@ -1228,7 +1228,7 @@ class TestConfigurationValidation:# Assert - Should be a set for O(1) lookupassert isinstance(blacklist, set)- def test_blacklist_extensions_are_lowercase(self):+ def test_blacklist_extensions_are_lowercase(self) -> None:"""Test that all blacklisted extensions are stored in lowercase.This ensures case-insensitive comparison works correctly.@@ -1251,7 +1251,7 @@ class TestFileConstants:- Proper categorization"""- def test_image_extensions_set_properties(self):+ def test_image_extensions_set_properties(self) -> None:"""Test that IMAGE_EXTENSIONS frozenset has expected properties."""# Assert - Should be immutableassert isinstance(IMAGE_EXTENSIONS, frozenset)@@ -1262,7 +1262,7 @@ class TestFileConstants:for ext in common_images:assert ext in IMAGE_EXTENSIONS or ext.upper() in IMAGE_EXTENSIONS- def test_video_extensions_set_properties(self):+ def test_video_extensions_set_properties(self) -> None:"""Test that VIDEO_EXTENSIONS frozenset has expected properties."""# Assert - Should be immutableassert isinstance(VIDEO_EXTENSIONS, frozenset)@@ -1273,7 +1273,7 @@ class TestFileConstants:for ext in common_videos:assert ext in VIDEO_EXTENSIONS or ext.upper() in VIDEO_EXTENSIONS- def test_audio_extensions_set_properties(self):+ def test_audio_extensions_set_properties(self) -> None:"""Test that AUDIO_EXTENSIONS frozenset has expected properties."""# Assert - Should be immutableassert isinstance(AUDIO_EXTENSIONS, frozenset)@@ -1284,7 +1284,7 @@ class TestFileConstants:for ext in common_audio:assert ext in AUDIO_EXTENSIONS or ext.upper() in AUDIO_EXTENSIONS- def test_document_extensions_set_properties(self):+ def test_document_extensions_set_properties(self) -> None:"""Test that DOCUMENT_EXTENSIONS frozenset has expected properties."""# Assert - Should be immutableassert isinstance(DOCUMENT_EXTENSIONS, frozenset)@@ -1295,7 +1295,7 @@ class TestFileConstants:for ext in common_docs:assert ext in DOCUMENT_EXTENSIONS or ext.upper() in DOCUMENT_EXTENSIONS- def test_no_extension_overlap_between_categories(self):+ def test_no_extension_overlap_between_categories(self) -> None:"""Test that extensions don't appear in multiple incompatible categories.While some overlap might be intentional, major categories should be distinct.diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.tomlindex abd297d7910..0c2c3c32351 100644--- a/api/tests/unit_tests/pyrefly.toml+++ b/api/tests/unit_tests/pyrefly.toml@@ -342,7 +342,6 @@ project-excludes = ["core/datasource/test_datasource_file_manager.py","core/datasource/test_datasource_manager.py","core/datasource/test_errors.py",- "core/datasource/test_file_upload.py","core/datasource/test_notion_provider.py","core/datasource/test_website_crawl.py","core/datasource/utils/test_message_transformer.py",