Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: convert timeout from milliseconds to seconds in Python SDK (#1894)
devin-ai-integration[bot]21103b5a5815
diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex 567f6e253..86a32a1fd 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -151,7 +151,7 @@ class ScrapeOptions(pydantic.BaseModel):excludeTags: Optional[List[str]] = NoneonlyMainContent: Optional[bool] = NonewaitFor: Optional[int] = None- timeout: Optional[int] = None+ timeout: Optional[int] = 30000location: Optional[LocationConfig] = Nonemobile: Optional[bool] = NoneskipTlsVerification: Optional[bool] = None@@ -305,7 +305,7 @@ class MapParams(pydantic.BaseModel):includeSubdomains: Optional[bool] = NonesitemapOnly: Optional[bool] = Nonelimit: Optional[int] = None- timeout: Optional[int] = None+ timeout: Optional[int] = 30000useIndex: Optional[bool] = Noneclass MapResponse(pydantic.BaseModel):@@ -469,7 +469,7 @@ class FirecrawlApp:exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,wait_for: Optional[int] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,location: Optional[LocationConfig] = None,mobile: Optional[bool] = None,skip_tls_verification: Optional[bool] = None,@@ -593,7 +593,7 @@ class FirecrawlApp:f'{self.api_url}/v1/scrape',headers=_headers,json=scrape_params,- timeout=(timeout + 5000 if timeout else None)+ timeout=(timeout / 1000.0 + 5 if timeout is not None else None))if response.status_code == 200:@@ -620,7 +620,7 @@ class FirecrawlApp:lang: Optional[str] = None,country: Optional[str] = None,location: Optional[str] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,scrape_options: Optional[ScrapeOptions] = None,**kwargs) -> SearchResponse:"""@@ -1189,7 +1189,7 @@ class FirecrawlApp:include_subdomains: Optional[bool] = None,sitemap_only: Optional[bool] = None,limit: Optional[int] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,use_index: Optional[bool] = None,**kwargs) -> MapResponse:"""@@ -1280,7 +1280,7 @@ class FirecrawlApp:exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,wait_for: Optional[int] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,location: Optional[LocationConfig] = None,mobile: Optional[bool] = None,skip_tls_verification: Optional[bool] = None,@@ -1421,7 +1421,7 @@ class FirecrawlApp:exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,wait_for: Optional[int] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,location: Optional[LocationConfig] = None,mobile: Optional[bool] = None,skip_tls_verification: Optional[bool] = None,@@ -1561,7 +1561,7 @@ class FirecrawlApp:exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,wait_for: Optional[int] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,location: Optional[LocationConfig] = None,mobile: Optional[bool] = None,skip_tls_verification: Optional[bool] = None,@@ -2204,7 +2204,7 @@ class FirecrawlApp:requests.RequestException: If the request fails after the specified retries."""for attempt in range(retries):- response = requests.post(url, headers=headers, json=data, timeout=((data["timeout"] + 5000) if "timeout" in data else None))+ response = requests.post(url, headers=headers, json=data, timeout=((data["timeout"] / 1000.0 + 5) if "timeout" in data and data["timeout"] is not None else None))if response.status_code == 502:time.sleep(backoff_factor * (2 ** attempt))else:@@ -2975,7 +2975,7 @@ class AsyncFirecrawlApp(FirecrawlApp):exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,wait_for: Optional[int] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,location: Optional[LocationConfig] = None,mobile: Optional[bool] = None,skip_tls_verification: Optional[bool] = None,@@ -3109,7 +3109,7 @@ class AsyncFirecrawlApp(FirecrawlApp):exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,wait_for: Optional[int] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,location: Optional[LocationConfig] = None,mobile: Optional[bool] = None,skip_tls_verification: Optional[bool] = None,@@ -3247,7 +3247,7 @@ class AsyncFirecrawlApp(FirecrawlApp):exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,wait_for: Optional[int] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,location: Optional[LocationConfig] = None,mobile: Optional[bool] = None,skip_tls_verification: Optional[bool] = None,@@ -3732,7 +3732,7 @@ class AsyncFirecrawlApp(FirecrawlApp):include_subdomains: Optional[bool] = None,sitemap_only: Optional[bool] = None,limit: Optional[int] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,params: Optional[MapParams] = None) -> MapResponse:"""Asynchronously map and discover links from a URL.@@ -4467,7 +4467,7 @@ class AsyncFirecrawlApp(FirecrawlApp):lang: Optional[str] = None,country: Optional[str] = None,location: Optional[str] = None,- timeout: Optional[int] = None,+ timeout: Optional[int] = 30000,scrape_options: Optional[ScrapeOptions] = None,params: Optional[Union[Dict[str, Any], SearchParams]] = None,**kwargs) -> SearchResponse:diff --git a/apps/python-sdk/tests/test_timeout_conversion.py b/apps/python-sdk/tests/test_timeout_conversion.pynew file mode 100644index 000000000..3c09c34bd--- /dev/null+++ b/apps/python-sdk/tests/test_timeout_conversion.py@@ -0,0 +1,117 @@+import unittest+from unittest.mock import patch, MagicMock+import os+from firecrawl import FirecrawlApp+++class TestTimeoutConversion(unittest.TestCase):++ @patch('requests.post')+ def test_scrape_url_timeout_conversion(self, mock_post):+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ 'success': True,+ 'data': {+ 'markdown': 'Test content'+ }+ }+ mock_post.return_value = mock_response++ app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))+ app.scrape_url('https://example.com', timeout=60000)++ args, kwargs = mock_post.call_args+ self.assertEqual(kwargs['timeout'], 65.0)++ @patch('requests.post')+ def test_scrape_url_default_timeout(self, mock_post):+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ 'success': True,+ 'data': {+ 'markdown': 'Test content'+ }+ }+ mock_post.return_value = mock_response++ app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))+ app.scrape_url('https://example.com')++ args, kwargs = mock_post.call_args+ self.assertEqual(kwargs['timeout'], 35.0)++ @patch('requests.post')+ def test_post_request_timeout_conversion(self, mock_post):+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_post.return_value = mock_response++ app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))++ data = {'timeout': 30000}+ headers = {'Content-Type': 'application/json'}++ app._post_request('https://example.com/api', data, headers)++ args, kwargs = mock_post.call_args+ self.assertEqual(kwargs['timeout'], 35.0)++ @patch('requests.post')+ def test_post_request_default_timeout(self, mock_post):+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_post.return_value = mock_response++ app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))++ data = {'timeout': 30000, 'url': 'https://example.com'}+ headers = {'Content-Type': 'application/json'}++ app._post_request('https://example.com/api', data, headers)++ args, kwargs = mock_post.call_args+ self.assertEqual(kwargs['timeout'], 35.0)++ @patch('requests.post')+ def test_timeout_edge_cases(self, mock_post):+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ 'success': True,+ 'data': {+ 'markdown': 'Test content'+ }+ }+ mock_post.return_value = mock_response++ app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))++ app.scrape_url('https://example.com', timeout=1000)+ args, kwargs = mock_post.call_args+ self.assertEqual(kwargs['timeout'], 6.0)++ app.scrape_url('https://example.com', timeout=0)+ args, kwargs = mock_post.call_args+ self.assertEqual(kwargs['timeout'], 5.0)++ @patch('requests.post')+ def test_post_request_no_timeout_key(self, mock_post):+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_post.return_value = mock_response++ app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))++ data = {'url': 'https://example.com'}+ headers = {'Content-Type': 'application/json'}++ app._post_request('https://example.com/api', data, headers)++ args, kwargs = mock_post.call_args+ self.assertIsNone(kwargs['timeout'])+++if __name__ == '__main__':+ unittest.main()