Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor(api): enable strict typing for remaining datasource tests (#44048)

Eddy ZHANG50f0051005b3
diff --git a/api/tests/unit_tests/.ruff.toml b/api/tests/unit_tests/.ruff.toml
index 30a367112d4..da7ddcb1ae3 100644
--- a/api/tests/unit_tests/.ruff.toml
+++ b/api/tests/unit_tests/.ruff.toml
@@ -129,8 +129,6 @@ extend-select = ["ANN401", "ARG"]
"core/base/test_app_generator_tts_publisher.py" = ["ARG002"]
"core/callback_handler/test_agent_tool_callback_handler.py" = ["ARG002"]
"core/callback_handler/test_workflow_tool_callback_handler.py" = ["ARG002"]
-"core/datasource/__base/test_datasource_provider.py" = ["ARG002"]
-"core/datasource/test_website_crawl.py" = ["ARG002"]
"core/entities/test_entities_mcp_provider.py" = ["ARG001"]
"core/entities/test_entities_provider_configuration.py" = ["ANN401", "ARG001", "ARG005", "TID251"]
"core/external_data_tool/api/test_api.py" = ["ARG001"]
diff --git a/api/tests/unit_tests/core/datasource/__base/test_datasource_provider.py b/api/tests/unit_tests/core/datasource/__base/test_datasource_provider.py
index 9059f1e1815..4592b3763f2 100644
--- a/api/tests/unit_tests/core/datasource/__base/test_datasource_provider.py
+++ b/api/tests/unit_tests/core/datasource/__base/test_datasource_provider.py
@@ -19,7 +19,7 @@ class ConcreteDatasourcePluginProviderController(DatasourcePluginProviderControl
"""
def get_datasource(self, datasource_name: str) -> DatasourcePlugin:
- raise AssertionError("Credential validation must not fetch a datasource")
+ raise AssertionError(f"Credential validation must not fetch datasource {datasource_name!r}")
class TestDatasourcePluginProviderController:
diff --git a/api/tests/unit_tests/core/datasource/online_document/test_online_document_plugin.py b/api/tests/unit_tests/core/datasource/online_document/test_online_document_plugin.py
index acd2c1b258e..30232b437c0 100644
--- a/api/tests/unit_tests/core/datasource/online_document/test_online_document_plugin.py
+++ b/api/tests/unit_tests/core/datasource/online_document/test_online_document_plugin.py
@@ -12,7 +12,7 @@ from tests.unit_tests.core.datasource.factories import datasource_entity
class TestOnlineDocumentDatasourcePlugin:
- def test_init(self):
+ def test_init(self) -> None:
# Arrange
entity = datasource_entity("test_name")
runtime = DatasourceRuntime(tenant_id="test_tenant")
@@ -36,7 +36,7 @@ class TestOnlineDocumentDatasourcePlugin:
assert plugin.icon == icon
assert plugin.plugin_unique_identifier == plugin_unique_identifier
- def test_get_online_document_pages(self):
+ def test_get_online_document_pages(self) -> None:
# Arrange
entity = datasource_entity("test_name")
entity.identity.provider = "test_provider"
@@ -86,7 +86,7 @@ class TestOnlineDocumentDatasourcePlugin:
provider_type=provider_type,
)
- def test_get_online_document_page_content(self):
+ def test_get_online_document_page_content(self) -> None:
# Arrange
entity = datasource_entity("test_name")
entity.identity.provider = "test_provider"
@@ -137,7 +137,7 @@ class TestOnlineDocumentDatasourcePlugin:
provider_type=provider_type,
)
- def test_datasource_provider_type(self):
+ def test_datasource_provider_type(self) -> None:
# Arrange
entity = datasource_entity("test_name")
runtime = DatasourceRuntime(tenant_id="test_tenant")
diff --git a/api/tests/unit_tests/core/datasource/online_drive/test_online_drive_plugin.py b/api/tests/unit_tests/core/datasource/online_drive/test_online_drive_plugin.py
index 2aa23ec687a..cb11c706b64 100644
--- a/api/tests/unit_tests/core/datasource/online_drive/test_online_drive_plugin.py
+++ b/api/tests/unit_tests/core/datasource/online_drive/test_online_drive_plugin.py
@@ -13,7 +13,7 @@ from tests.unit_tests.core.datasource.factories import datasource_entity
class TestOnlineDriveDatasourcePlugin:
- def test_init(self):
+ def test_init(self) -> None:
# Arrange
entity = datasource_entity("test_name")
runtime = DatasourceRuntime(tenant_id="test_tenant")
@@ -37,7 +37,7 @@ class TestOnlineDriveDatasourcePlugin:
assert plugin.icon == icon
assert plugin.plugin_unique_identifier == plugin_unique_identifier
- def test_online_drive_browse_files(self):
+ def test_online_drive_browse_files(self) -> None:
# Arrange
entity = datasource_entity("test_name")
entity.identity.provider = "test_provider"
@@ -84,7 +84,7 @@ class TestOnlineDriveDatasourcePlugin:
provider_type=provider_type,
)
- def test_online_drive_download_file(self):
+ def test_online_drive_download_file(self) -> None:
# Arrange
entity = datasource_entity("test_name")
entity.identity.provider = "test_provider"
@@ -131,7 +131,7 @@ class TestOnlineDriveDatasourcePlugin:
provider_type=provider_type,
)
- def test_datasource_provider_type(self):
+ def test_datasource_provider_type(self) -> None:
# Arrange
entity = datasource_entity("test_name")
runtime = DatasourceRuntime(tenant_id="test_tenant")
diff --git a/api/tests/unit_tests/core/datasource/test_website_crawl.py b/api/tests/unit_tests/core/datasource/test_website_crawl.py
index fc2ff7b1e22..ed99d489673 100644
--- a/api/tests/unit_tests/core/datasource/test_website_crawl.py
+++ b/api/tests/unit_tests/core/datasource/test_website_crawl.py
@@ -98,7 +98,7 @@ def crawl_request(crawl_options: CrawlOptions) -> CrawlRequest:
class TestCrawlOptions:
"""Test suite for CrawlOptions data class."""
- def test_crawl_options_defaults(self):
+ def test_crawl_options_defaults(self) -> None:
"""Test that CrawlOptions has correct default values."""
options = CrawlOptions()
@@ -111,7 +111,7 @@ class TestCrawlOptions:
assert options.max_depth is None
assert options.use_sitemap is True
- def test_get_include_paths_with_values(self, crawl_options: CrawlOptions):
+ def test_get_include_paths_with_values(self, crawl_options: CrawlOptions) -> None:
"""Test parsing include paths from comma-separated string."""
paths = crawl_options.get_include_paths()
@@ -119,14 +119,14 @@ class TestCrawlOptions:
assert "/blog/*" in paths
assert "/docs/*" in paths
- def test_get_include_paths_empty(self):
+ def test_get_include_paths_empty(self) -> None:
"""Test that empty includes returns empty list."""
options = CrawlOptions(includes=None)
paths = options.get_include_paths()
assert paths == []
- def test_get_exclude_paths_with_values(self, crawl_options: CrawlOptions):
+ def test_get_exclude_paths_with_values(self, crawl_options: CrawlOptions) -> None:
"""Test parsing exclude paths from comma-separated string."""
paths = crawl_options.get_exclude_paths()
@@ -134,14 +134,14 @@ class TestCrawlOptions:
assert "/admin/*" in paths
assert "/private/*" in paths
- def test_get_exclude_paths_empty(self):
+ def test_get_exclude_paths_empty(self) -> None:
"""Test that empty excludes returns empty list."""
options = CrawlOptions(excludes=None)
paths = options.get_exclude_paths()
assert paths == []
- def test_max_depth_limiting(self):
+ def test_max_depth_limiting(self) -> None:
"""Test that max_depth can be set to limit crawl depth."""
options = CrawlOptions(max_depth=5, crawl_sub_pages=True)
@@ -157,7 +157,7 @@ class TestCrawlOptions:
class TestWebsiteCrawlDatasourcePlugin:
"""Test suite for WebsiteCrawlDatasourcePlugin."""
- def test_plugin_initialization(self, mock_datasource_entity: DatasourceEntity):
+ def test_plugin_initialization(self, mock_datasource_entity: DatasourceEntity) -> None:
"""Test that plugin initializes correctly with required parameters."""
from core.datasource.__base.datasource_runtime import DatasourceRuntime
@@ -175,7 +175,7 @@ class TestWebsiteCrawlDatasourcePlugin:
assert plugin.entity == mock_datasource_entity
assert plugin.datasource_provider_type() == DatasourceProviderType.WEBSITE_CRAWL
- def test_get_website_crawl(self, mock_datasource_entity: DatasourceEntity, mocker: MockerFixture):
+ def test_get_website_crawl(self, mock_datasource_entity: DatasourceEntity, mocker: MockerFixture) -> None:
"""Test that get_website_crawl calls PluginDatasourceManager correctly."""
from core.datasource.__base.datasource_runtime import DatasourceRuntime
@@ -219,7 +219,7 @@ class TestWebsiteCrawlDatasourcePlugin:
class TestWebsiteCrawlDatasourcePluginProviderController:
"""Test suite for WebsiteCrawlDatasourcePluginProviderController."""
- def test_provider_controller_initialization(self, mock_provider_entity: DatasourceProviderEntityWithPlugin):
+ def test_provider_controller_initialization(self, mock_provider_entity: DatasourceProviderEntityWithPlugin) -> None:
"""Test provider controller initialization."""
controller = WebsiteCrawlDatasourcePluginProviderController(
entity=mock_provider_entity,
@@ -232,7 +232,7 @@ class TestWebsiteCrawlDatasourcePluginProviderController:
assert controller.plugin_unique_identifier == "test_unique_id"
assert controller.provider_type == DatasourceProviderType.WEBSITE_CRAWL
- def test_get_datasource_success(self, mock_provider_entity: DatasourceProviderEntityWithPlugin):
+ def test_get_datasource_success(self, mock_provider_entity: DatasourceProviderEntityWithPlugin) -> None:
"""Test retrieving a datasource by name."""
controller = WebsiteCrawlDatasourcePluginProviderController(
entity=mock_provider_entity,
@@ -247,7 +247,7 @@ class TestWebsiteCrawlDatasourcePluginProviderController:
assert datasource.tenant_id == "test_tenant"
assert datasource.plugin_unique_identifier == "test_unique_id"
- def test_get_datasource_not_found(self, mock_provider_entity: DatasourceProviderEntityWithPlugin):
+ def test_get_datasource_not_found(self, mock_provider_entity: DatasourceProviderEntityWithPlugin) -> None:
"""Test that ValueError is raised when datasource is not found."""
controller = WebsiteCrawlDatasourcePluginProviderController(
entity=mock_provider_entity,
@@ -268,7 +268,7 @@ class TestWebsiteCrawlDatasourcePluginProviderController:
class TestWaterCrawlProvider:
"""Test suite for WaterCrawl provider crawling functionality."""
- def test_crawl_url_basic(self, mocker: MockerFixture):
+ def test_crawl_url_basic(self, mocker: MockerFixture) -> None:
"""Test basic URL crawling without sub-pages."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -286,7 +286,7 @@ class TestWaterCrawlProvider:
assert spider_options["max_depth"] == 1
assert spider_options["page_limit"] == 1
- def test_crawl_url_with_sub_pages(self, mocker: MockerFixture):
+ def test_crawl_url_with_sub_pages(self, mocker: MockerFixture) -> None:
"""Test URL crawling with sub-pages enabled."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -305,7 +305,7 @@ class TestWaterCrawlProvider:
assert spider_options["max_depth"] == 3
assert spider_options["page_limit"] == 50
- def test_crawl_url_max_depth_limiting(self, mocker: MockerFixture):
+ def test_crawl_url_max_depth_limiting(self, mocker: MockerFixture) -> None:
"""Test that max_depth properly limits crawl depth."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -321,7 +321,7 @@ class TestWaterCrawlProvider:
spider_options = call_args.kwargs["spider_options"]
assert spider_options["max_depth"] == 2
- def test_crawl_url_with_include_exclude_paths(self, mocker: MockerFixture):
+ def test_crawl_url_with_include_exclude_paths(self, mocker: MockerFixture) -> None:
"""Test URL crawling with include and exclude path filters."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -349,7 +349,7 @@ class TestWaterCrawlProvider:
assert "/admin/*" in spider_options["exclude_paths"]
assert "/private/*" in spider_options["exclude_paths"]
- def test_crawl_url_content_extraction_options(self, mocker: MockerFixture):
+ def test_crawl_url_content_extraction_options(self, mocker: MockerFixture) -> None:
"""Test that content extraction options are properly configured."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -367,7 +367,7 @@ class TestWaterCrawlProvider:
assert page_options["wait_time"] == 2000
assert page_options["include_html"] is False
- def test_crawl_url_minimum_wait_time(self, mocker: MockerFixture):
+ def test_crawl_url_minimum_wait_time(self, mocker: MockerFixture) -> None:
"""Test that wait_time has a minimum value of 1000ms."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -392,7 +392,7 @@ class TestWaterCrawlProvider:
class TestCrawlStatus:
"""Test suite for crawl status checking and result retrieval."""
- def test_get_crawl_status_active(self, mocker: MockerFixture):
+ def test_get_crawl_status_active(self, mocker: MockerFixture) -> None:
"""Test getting status of an active crawl job."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -413,7 +413,7 @@ class TestCrawlStatus:
assert status["current"] == 5
assert status["data"] == []
- def test_get_crawl_status_completed(self, mocker: MockerFixture):
+ def test_get_crawl_status_completed(self, mocker: MockerFixture) -> None:
"""Test getting status of a completed crawl job with results."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -447,7 +447,7 @@ class TestCrawlStatus:
assert len(status["data"]) == 1
assert status["time_consuming"] == 15.5
- def test_get_crawl_url_data(self, mocker: MockerFixture):
+ def test_get_crawl_url_data(self, mocker: MockerFixture) -> None:
"""Test retrieving specific URL data from crawl results."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -472,11 +472,11 @@ class TestCrawlStatus:
assert data["title"] == "Target"
assert data["markdown"] == "# Target Page"
- def test_get_crawl_url_data_not_found(self, mocker: MockerFixture):
+ def test_get_crawl_url_data_not_found(self, mocker: MockerFixture) -> None:
"""Test that None is returned when URL is not in results."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
- mock_instance.get_crawl_request_results.return_value = {"results": [], "next": None}
+ mock_instance.get_crawl_request_results.return_value = {"results": list[object](), "next": None}
provider = WaterCrawlProvider(api_key="test_key")
data = provider.get_crawl_url_data("test-job-789", "https://example.com/nonexistent")
@@ -494,7 +494,9 @@ class TestWebsiteService:
@patch("services.data_source.website_service.current_user")
@patch("services.data_source.website_service.DatasourceProviderService")
- def test_crawl_url_firecrawl(self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture):
+ def test_crawl_url_firecrawl(
+ self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture
+ ) -> None:
"""Test crawling with Firecrawl provider."""
# Setup mocks
mock_current_user.current_tenant_id = "test_tenant"
@@ -529,7 +531,9 @@ class TestWebsiteService:
@patch("services.data_source.website_service.current_user")
@patch("services.data_source.website_service.DatasourceProviderService")
- def test_crawl_url_watercrawl(self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture):
+ def test_crawl_url_watercrawl(
+ self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture
+ ) -> None:
"""Test crawling with WaterCrawl provider."""
# Setup mocks
mock_current_user.current_tenant_id = "test_tenant"
@@ -561,7 +565,9 @@ class TestWebsiteService:
@patch("services.data_source.website_service.current_user")
@patch("services.data_source.website_service.DatasourceProviderService")
- def test_crawl_url_jinareader(self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture):
+ def test_crawl_url_jinareader(
+ self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture
+ ) -> None:
"""Test crawling with JinaReader provider."""
# Setup mocks
mock_current_user.current_tenant_id = "test_tenant"
@@ -592,28 +598,28 @@ class TestWebsiteService:
assert result["status"] == "active"
assert result["job_id"] == "task-789"
- def test_document_create_args_validate_success(self):
+ def test_document_create_args_validate_success(self) -> None:
"""Test validation of valid document creation arguments."""
args = {"provider": "watercrawl", "url": "https://example.com", "options": {"limit": 10}}
# Should not raise any exception
WebsiteService.document_create_args_validate(args)
- def test_document_create_args_validate_missing_provider(self):
+ def test_document_create_args_validate_missing_provider(self) -> None:
"""Test validation fails when provider is missing."""
args = {"url": "https://example.com", "options": {"limit": 10}}
with pytest.raises(ValueError, match="Provider is required"):
WebsiteService.document_create_args_validate(args)
- def test_document_create_args_validate_missing_url(self):
+ def test_document_create_args_validate_missing_url(self) -> None:
"""Test validation fails when URL is missing."""
args = {"provider": "watercrawl", "options": {"limit": 10}}
with pytest.raises(ValueError, match="URL is required"):
WebsiteService.document_create_args_validate(args)
- def test_document_create_args_validate_missing_options(self):
+ def test_document_create_args_validate_missing_options(self) -> None:
"""Test validation fails when options are missing."""
args = {"provider": "watercrawl", "url": "https://example.com"}
@@ -629,7 +635,7 @@ class TestWebsiteService:
class TestLinkFollowingLogic:
"""Test suite for link following and navigation logic."""
- def test_link_following_with_includes(self, mocker: MockerFixture):
+ def test_link_following_with_includes(self, mocker: MockerFixture) -> None:
"""Test that only links matching include patterns are followed."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -646,7 +652,7 @@ class TestLinkFollowingLogic:
assert "/blog/*" in spider_options["include_paths"]
assert "/news/*" in spider_options["include_paths"]
- def test_link_following_with_excludes(self, mocker: MockerFixture):
+ def test_link_following_with_excludes(self, mocker: MockerFixture) -> None:
"""Test that links matching exclude patterns are not followed."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -663,7 +669,7 @@ class TestLinkFollowingLogic:
assert "/login/*" in spider_options["exclude_paths"]
assert "/logout/*" in spider_options["exclude_paths"]
- def test_link_following_respects_max_depth(self, mocker: MockerFixture):
+ def test_link_following_respects_max_depth(self, mocker: MockerFixture) -> None:
"""Test that link following stops at specified max depth."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -679,7 +685,7 @@ class TestLinkFollowingLogic:
spider_options = call_args.kwargs["spider_options"]
assert spider_options["max_depth"] == 1
- def test_link_following_page_limit(self, mocker: MockerFixture):
+ def test_link_following_page_limit(self, mocker: MockerFixture) -> None:
"""Test that link following respects page limit."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -710,7 +716,7 @@ class TestRobotsTxtRespect:
the service layer properly configures providers to respect robots.txt.
"""
- def test_watercrawl_provider_respects_robots_txt(self, mocker: MockerFixture):
+ def test_watercrawl_provider_respects_robots_txt(self, mocker: MockerFixture) -> None:
"""
Test that WaterCrawl provider is configured to respect robots.txt.
@@ -726,7 +732,7 @@ class TestRobotsTxtRespect:
assert provider.client is not None
mock_client.assert_called_once_with("test_key", "https://app.watercrawl.dev/")
- def test_firecrawl_provider_respects_robots_txt(self, mocker: MockerFixture):
+ def test_firecrawl_provider_respects_robots_txt(self) -> None:
"""
Test that Firecrawl provider respects robots.txt.
@@ -741,7 +747,7 @@ class TestRobotsTxtRespect:
assert app.api_key == "test_key"
assert app.base_url == "https://api.firecrawl.dev"
- def test_crawl_respects_domain_restrictions(self, mocker: MockerFixture):
+ def test_crawl_respects_domain_restrictions(self, mocker: MockerFixture) -> None:
"""
Test that crawl operations respect domain restrictions.
@@ -771,7 +777,7 @@ class TestRobotsTxtRespect:
class TestContentExtraction:
"""Test suite for content extraction from crawled pages."""
- def test_structure_data_with_metadata(self, mocker: MockerFixture):
+ def test_structure_data_with_metadata(self, mocker: MockerFixture) -> None:
"""Test that content is properly structured with metadata."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
@@ -796,7 +802,7 @@ class TestContentExtraction:
assert structured["source_url"] == "https://example.com/page"
assert structured["markdown"] == "# Page Title\n\nPage content here."
- def test_structure_data_fallback_title(self, mocker: MockerFixture):
+ def test_structure_data_fallback_title(self, mocker: MockerFixture) -> None:
"""Test that fallback title is used when og:title is not available."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
@@ -811,7 +817,7 @@ class TestContentExtraction:
assert structured["title"] == "Fallback Title"
- def test_structure_data_invalid_result(self, mocker: MockerFixture):
+ def test_structure_data_invalid_result(self, mocker: MockerFixture) -> None:
"""Test that ValueError is raised for invalid result objects."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
@@ -823,7 +829,7 @@ class TestContentExtraction:
with pytest.raises(ValueError, match="Invalid result object"):
provider._structure_data(result_object)
- def test_scrape_url_content_extraction(self, mocker: MockerFixture):
+ def test_scrape_url_content_extraction(self, mocker: MockerFixture) -> None:
"""Test content extraction from single URL scraping."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -843,7 +849,7 @@ class TestContentExtraction:
assert result["markdown"] == "# Main Content"
assert result["source_url"] == "https://example.com"
- def test_only_main_content_extraction(self, mocker: MockerFixture):
+ def test_only_main_content_extraction(self, mocker: MockerFixture) -> None:
"""Test that only_main_content option filters out non-content elements."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -871,7 +877,7 @@ class TestErrorHandling:
@patch("services.data_source.website_service.current_user")
@patch("services.data_source.website_service.DatasourceProviderService")
- def test_invalid_provider_error(self, mock_provider_service: Mock, mock_current_user: Mock):
+ def test_invalid_provider_error(self, mock_provider_service: Mock, mock_current_user: Mock) -> None:
"""Test that invalid provider raises ValueError."""
from services.data_source.website_service import WebsiteCrawlApiRequest
@@ -893,7 +899,7 @@ class TestErrorHandling:
)
).crawl_url(api_request)
- def test_missing_api_key_error(self, mocker: MockerFixture):
+ def test_missing_api_key_error(self, mocker: MockerFixture) -> None:
"""Test that missing API key is handled properly at the httpx client level."""
# Mock the client to avoid actual httpx initialization
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
@@ -905,7 +911,7 @@ class TestErrorHandling:
# Verify the client was initialized with the API key
mock_client.assert_called_once_with("test_key", None)
- def test_crawl_status_for_nonexistent_job(self, mocker: MockerFixture):
+ def test_crawl_status_for_nonexistent_job(self, mocker: MockerFixture) -> None:
"""Test handling of status check for non-existent job."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -930,7 +936,7 @@ class TestErrorHandling:
class TestCrawlWorkflow:
"""Integration-style tests for complete crawl workflows."""
- def test_complete_crawl_workflow(self, mocker: MockerFixture):
+ def test_complete_crawl_workflow(self, mocker: MockerFixture) -> None:
"""Test a complete crawl workflow from start to finish."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -989,7 +995,7 @@ class TestCrawlWorkflow:
assert data is not None
assert data["title"] == "Page 1"
- def test_single_page_scrape_workflow(self, mocker: MockerFixture):
+ def test_single_page_scrape_workflow(self, mocker: MockerFixture) -> None:
"""Test workflow for scraping a single page without crawling."""
mock_client = mocker.patch("core.rag.extractor.watercrawl.provider.WaterCrawlAPIClient")
mock_instance = mock_client.return_value
@@ -1006,7 +1012,9 @@ class TestCrawlWorkflow:
assert result["title"] == "Single Page"
assert result["description"] == "A single page"
- assert "Single Page" in result["markdown"]
+ markdown = result["markdown"]
+ assert markdown is not None
+ assert "Single Page" in markdown
assert result["source_url"] == "https://example.com/single-page"
@@ -1027,7 +1035,7 @@ class TestAdvancedCrawlScenarios:
- Timeout handling
"""
- def test_pagination_in_crawl_results(self, mocker: MockerFixture):
+ def test_pagination_in_crawl_results(self, mocker: MockerFixture) -> None:
"""
Test that pagination is properly handled when retrieving crawl results.
@@ -1071,7 +1079,7 @@ class TestAdvancedCrawlScenarios:
assert results[0]["title"] == "Page 1"
assert results[149]["title"] == "Page 150"
- def test_large_scale_crawl_configuration(self, mocker: MockerFixture):
+ def test_large_scale_crawl_configuration(self, mocker: MockerFixture) -> None:
"""
Test configuration for large-scale crawls with high page limits.
@@ -1104,7 +1112,7 @@ class TestAdvancedCrawlScenarios:
assert spider_options["page_limit"] == 1000
assert spider_options["max_depth"] == 5
- def test_crawl_with_custom_wait_time(self, mocker: MockerFixture):
+ def test_crawl_with_custom_wait_time(self, mocker: MockerFixture) -> None:
"""
Test that custom wait times are properly applied to page loads.
@@ -1127,7 +1135,7 @@ class TestAdvancedCrawlScenarios:
# Verify wait time is set correctly
assert page_options["wait_time"] == 3000
- def test_crawl_status_progress_tracking(self, mocker: MockerFixture):
+ def test_crawl_status_progress_tracking(self, mocker: MockerFixture) -> None:
"""
Test that crawl progress is accurately tracked and reported.
@@ -1157,7 +1165,7 @@ class TestAdvancedCrawlScenarios:
progress_percentage = (status["current"] / status["total"]) * 100
assert progress_percentage == 60.0
- def test_crawl_with_sitemap_usage(self, mocker: MockerFixture):
+ def test_crawl_with_sitemap_usage(self, mocker: MockerFixture) -> None:
"""
Test that sitemap.xml is utilized when use_sitemap is enabled.
@@ -1179,7 +1187,7 @@ class TestAdvancedCrawlScenarios:
call_args = mock_instance.create_crawl_request.call_args
assert call_args is not None
- def test_empty_crawl_results(self, mocker: MockerFixture):
+ def test_empty_crawl_results(self, mocker: MockerFixture) -> None:
"""
Test handling of crawls that return no results.
@@ -1195,7 +1203,7 @@ class TestAdvancedCrawlScenarios:
"options": {"spider_options": {"page_limit": 10}},
"duration": "00:00:05.000000",
}
- mock_instance.get_crawl_request_results.return_value = {"results": [], "next": None}
+ mock_instance.get_crawl_request_results.return_value = {"results": list[object](), "next": None}
provider = WaterCrawlProvider(api_key="test_key")
status = provider.get_crawl_status("empty-job")
@@ -1206,7 +1214,7 @@ class TestAdvancedCrawlScenarios:
assert status["total"] == 10
assert len(status["data"]) == 0
- def test_crawl_with_multiple_include_patterns(self, mocker: MockerFixture):
+ def test_crawl_with_multiple_include_patterns(self, mocker: MockerFixture) -> None:
"""
Test crawling with multiple include patterns for fine-grained control.
@@ -1237,7 +1245,7 @@ class TestAdvancedCrawlScenarios:
assert "/articles/*" in spider_options["include_paths"]
assert "/docs/*" in spider_options["include_paths"]
- def test_crawl_duration_calculation(self, mocker: MockerFixture):
+ def test_crawl_duration_calculation(self, mocker: MockerFixture) -> None:
"""
Test accurate calculation of crawl duration from time strings.
@@ -1262,7 +1270,7 @@ class TestAdvancedCrawlScenarios:
"options": {"spider_options": {"page_limit": 10}},
"duration": duration_str,
}
- mock_instance.get_crawl_request_results.return_value = {"results": [], "next": None}
+ mock_instance.get_crawl_request_results.return_value = {"results": list[object](), "next": None}
provider = WaterCrawlProvider(api_key="test_key")
status = provider.get_crawl_status("duration-test")
@@ -1288,7 +1296,7 @@ class TestProviderSpecificFeatures:
@patch("services.data_source.website_service.DatasourceProviderService")
def test_firecrawl_with_prompt_parameter(
self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture
- ):
+ ) -> None:
"""
Test Firecrawl's prompt parameter for AI-guided extraction.
@@ -1342,7 +1350,7 @@ class TestProviderSpecificFeatures:
@patch("services.data_source.website_service.DatasourceProviderService")
def test_jinareader_single_page_mode(
self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture
- ):
+ ) -> None:
"""
Test JinaReader's single-page scraping mode.
@@ -1387,7 +1395,7 @@ class TestProviderSpecificFeatures:
@patch("services.data_source.website_service.DatasourceProviderService")
def test_watercrawl_with_tag_filtering(
self, mock_provider_service: Mock, mock_current_user: Mock, mocker: MockerFixture
- ):
+ ) -> None:
"""
Test WaterCrawl's HTML tag filtering capabilities.
@@ -1428,7 +1436,7 @@ class TestProviderSpecificFeatures:
assert result["status"] == "active"
assert result["job_id"] == "tag-filter-job"
- def test_firecrawl_base_url_configuration(self, mocker: MockerFixture):
+ def test_firecrawl_base_url_configuration(self) -> None:
"""
Test that Firecrawl can be configured with custom base URLs.
@@ -1444,7 +1452,7 @@ class TestProviderSpecificFeatures:
assert app.base_url == custom_base_url
assert app.api_key == "test_key"
- def test_watercrawl_base_url_default(self, mocker: MockerFixture):
+ def test_watercrawl_base_url_default(self, mocker: MockerFixture) -> None:
"""
Test WaterCrawl's default base URL configuration.
@@ -1473,7 +1481,7 @@ class TestDataStructureValidation:
and transformed into the expected format.
"""
- def test_crawl_request_to_api_request_conversion(self):
+ def test_crawl_request_to_api_request_conversion(self) -> None:
"""
Test conversion from API request to internal CrawlRequest format.
@@ -1513,7 +1521,7 @@ class TestDataStructureValidation:
assert crawl_request.options.max_depth == 3
assert crawl_request.options.use_sitemap is True
- def test_crawl_options_path_parsing(self):
+ def test_crawl_options_path_parsing(self) -> None:
"""
Test that include/exclude paths are correctly parsed from strings.
@@ -1537,7 +1545,7 @@ class TestDataStructureValidation:
assert "/private/*" in exclude_paths
assert "/test/*" in exclude_paths
- def test_crawl_options_with_whitespace(self):
+ def test_crawl_options_with_whitespace(self) -> None:
"""
Test that whitespace in path strings is handled correctly.
@@ -1555,7 +1563,7 @@ class TestDataStructureValidation:
assert len(include_paths) == 3
assert len(exclude_paths) == 2
- def test_website_crawl_message_structure(self):
+ def test_website_crawl_message_structure(self) -> None:
"""
Test the structure of WebsiteCrawlMessage entity.
@@ -1575,7 +1583,7 @@ class TestDataStructureValidation:
assert message.result.completed == 10
assert isinstance(message.result.web_info_list, list)
- def test_datasource_identity_structure(self):
+ def test_datasource_identity_structure(self) -> None:
"""
Test that DatasourceIdentity contains all required fields.
@@ -1613,7 +1621,7 @@ class TestEdgeCasesAndBoundaries:
and exceptional scenarios.
"""
- def test_crawl_with_zero_limit(self, mocker: MockerFixture):
+ def test_crawl_with_zero_limit(self, mocker: MockerFixture) -> None:
"""
Test behavior when limit is set to zero.
@@ -1633,7 +1641,7 @@ class TestEdgeCasesAndBoundaries:
# Verify crawl was created (implementation may handle this differently)
assert result["status"] == "active"
- def test_crawl_with_very_large_limit(self, mocker: MockerFixture):
+ def test_crawl_with_very_large_limit(self, mocker: MockerFixture) -> None:
"""
Test crawl configuration with extremely large page limits.
@@ -1656,7 +1664,7 @@ class TestEdgeCasesAndBoundaries:
spider_options = call_args.kwargs["spider_options"]
assert spider_options["page_limit"] == 10000
- def test_crawl_with_empty_url(self):
+ def test_crawl_with_empty_url(self) -> None:
"""
Test that empty URLs are rejected with appropriate error.
@@ -1669,7 +1677,7 @@ class TestEdgeCasesAndBoundaries:
with pytest.raises(ValueError, match="URL is required"):
WebsiteCrawlApiRequest.from_args({"provider": "watercrawl", "url": "", "options": {"limit": 10}})
- def test_crawl_with_special_characters_in_paths(self, mocker: MockerFixture):
+ def test_crawl_with_special_characters_in_paths(self, mocker: MockerFixture) -> None:
"""
Test handling of special characters in include/exclude paths.
@@ -1697,7 +1705,7 @@ class TestEdgeCasesAndBoundaries:
assert "/blog/[0-9]+/*" in spider_options["include_paths"]
assert "/category/(tech|science)/*" in spider_options["include_paths"]
- def test_crawl_status_with_null_duration(self, mocker: MockerFixture):
+ def test_crawl_status_with_null_duration(self, mocker: MockerFixture) -> None:
"""
Test handling of null/missing duration in crawl status.
@@ -1719,7 +1727,7 @@ class TestEdgeCasesAndBoundaries:
# Verify null duration is handled (should default to 0)
assert status["time_consuming"] == 0
- def test_structure_data_with_missing_metadata_fields(self, mocker: MockerFixture):
+ def test_structure_data_with_missing_metadata_fields(self, mocker: MockerFixture) -> None:
"""
Test content extraction when metadata fields are missing.
@@ -1735,7 +1743,7 @@ class TestEdgeCasesAndBoundaries:
"url": "https://example.com/minimal",
"result": {
"markdown": "# Minimal Content",
- "metadata": {}, # Empty metadata
+ "metadata": dict[str, object](), # Empty metadata
},
}
@@ -1747,7 +1755,7 @@ class TestEdgeCasesAndBoundaries:
assert structured["source_url"] == "https://example.com/minimal"
assert structured["markdown"] == "# Minimal Content"
- def test_get_results_with_empty_pages(self, mocker: MockerFixture):
+ def test_get_results_with_empty_pages(self, mocker: MockerFixture) -> None:
"""
Test pagination handling when some pages return empty results.
@@ -1768,7 +1776,7 @@ class TestEdgeCasesAndBoundaries:
],
"next": "page2",
},
- {"results": [], "next": None}, # Empty page breaks the loop
+ {"results": list[object](), "next": None}, # Empty page breaks the loop
]
provider = WaterCrawlProvider(api_key="test_key")
diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.toml
index a7a5f66f721..e524fc4f9bd 100644
--- a/api/tests/unit_tests/pyrefly.toml
+++ b/api/tests/unit_tests/pyrefly.toml
@@ -257,9 +257,6 @@ project-excludes = [
"core/callback_handler/test_index_tool_callback_handler.py",
"core/callback_handler/test_workflow_tool_callback_handler.py",
"core/datasource/entities/test_datasource_entities.py",
- "core/datasource/online_document/test_online_document_plugin.py",
- "core/datasource/online_drive/test_online_drive_plugin.py",
- "core/datasource/test_website_crawl.py",
"core/entities/test_entities_mcp_provider.py",
"core/entities/test_entities_provider_configuration.py",
"core/extension/test_api_based_extension_requestor.py",