Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Fix search validation of custom date ranges and "qdr:h" (#1993)
Micah Stairsac278ccc1b30
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 5c4e14037..7f0ecc4cf 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "3.1.0"+__version__ = "3.1.1"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")@@ -84,4 +84,4 @@ __all__ = ['V1JsonConfig','V1ScrapeOptions','V1ChangeTrackingOptions',-]\ No newline at end of file+]diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.pyindex d41c50409..fd58c9f58 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py@@ -11,7 +11,7 @@ class TestSearchValidation:request = SearchRequest(query="")with pytest.raises(ValueError, match="Query cannot be empty"):_validate_search_request(request)-+request = SearchRequest(query=" ")with pytest.raises(ValueError, match="Query cannot be empty"):_validate_search_request(request)@@ -22,12 +22,12 @@ class TestSearchValidation:request = SearchRequest(query="test", limit=0)with pytest.raises(ValueError, match="Limit must be positive"):_validate_search_request(request)-+# Negative limitrequest = SearchRequest(query="test", limit=-1)with pytest.raises(ValueError, match="Limit must be positive"):_validate_search_request(request)-+# Too high limitrequest = SearchRequest(query="test", limit=101)with pytest.raises(ValueError, match="Limit cannot exceed 100"):@@ -39,12 +39,12 @@ class TestSearchValidation:request = SearchRequest(query="test", timeout=0)with pytest.raises(ValueError, match="Timeout must be positive"):_validate_search_request(request)-+# Negative timeoutrequest = SearchRequest(query="test", timeout=-1000)with pytest.raises(ValueError, match="Timeout must be positive"):_validate_search_request(request)-+# Too high timeoutrequest = SearchRequest(query="test", timeout=300001)with pytest.raises(ValueError, match="Timeout cannot exceed 300000ms"):@@ -56,12 +56,12 @@ class TestSearchValidation:request = SearchRequest(query="test", sources=["invalid_source"])with pytest.raises(ValueError, match="Invalid source type"):_validate_search_request(request)-+# Invalid object sourcerequest = SearchRequest(query="test", sources=[Source(type="invalid_source")])with pytest.raises(ValueError, match="Invalid source type"):_validate_search_request(request)-+# Mixed valid/invalid sourcesrequest = SearchRequest(query="test", sources=["web", "invalid_source"])with pytest.raises(ValueError, match="Invalid source type"):@@ -73,7 +73,7 @@ class TestSearchValidation:request = SearchRequest(query="test", location="")with pytest.raises(ValueError, match="Location must be a non-empty string"):_validate_search_request(request)-+# Whitespace locationrequest = SearchRequest(query="test", location=" ")with pytest.raises(ValueError, match="Location must be a non-empty string"):@@ -82,19 +82,49 @@ class TestSearchValidation:def test_validate_invalid_tbs(self):"""Test validation of invalid tbs values."""invalid_tbs_values = ["invalid", "qdr:x", "yesterday", "last_week"]-+for invalid_tbs in invalid_tbs_values:request = SearchRequest(query="test", tbs=invalid_tbs)with pytest.raises(ValueError, match="Invalid tbs value"):_validate_search_request(request)+ def test_validate_custom_date_ranges(self):+ """Test validation of custom date range formats."""+ valid_custom_ranges = [+ "cdr:1,cd_min:1/1/2024,cd_max:12/31/2024",+ "cdr:1,cd_min:12/1/2024,cd_max:12/31/2024",+ "cdr:1,cd_min:2/28/2023,cd_max:3/1/2023",+ "cdr:1,cd_min:10/15/2023,cd_max:11/15/2023"+ ]++ for valid_range in valid_custom_ranges:+ request = SearchRequest(query="test", tbs=valid_range)+ validated = _validate_search_request(request)+ assert validated == request++ def test_validate_invalid_custom_date_ranges(self):+ """Test validation of invalid custom date range formats."""+ # Invalid custom date ranges+ invalid_custom_ranges = [+ "cdr:1,cd_min:2/28/2023", # Missing cd_max+ "cdr:1,cd_max:2/28/2023", # Missing cd_min+ "cdr:2,cd_min:1/1/2024,cd_max:12/31/2024", # Wrong cdr value+ "cdr:cd_min:1/1/2024,cd_max:12/31/2024", # Missing :1+ "custom:1,cd_min:1/1/2024,cd_max:12/31/2024" # Wrong prefix+ ]++ for invalid_range in invalid_custom_ranges:+ request = SearchRequest(query="test", tbs=invalid_range)+ with pytest.raises(ValueError, match="Invalid"):+ _validate_search_request(request)+def test_validate_valid_requests(self):"""Test that valid requests pass validation."""# Minimal valid requestrequest = SearchRequest(query="test")validated = _validate_search_request(request)assert validated == request-+# Request with all optional parametersrequest = SearchRequest(query="test query",@@ -107,7 +137,7 @@ class TestSearchValidation:)validated = _validate_search_request(request)assert validated == request-+# Request with object sourcesrequest = SearchRequest(query="test",@@ -122,17 +152,17 @@ class TestSearchValidation:request = SearchRequest(query="test", limit=100)validated = _validate_search_request(request)assert validated == request-+# Maximum valid timeoutrequest = SearchRequest(query="test", timeout=300000)validated = _validate_search_request(request)assert validated == request-+# Minimum valid limitrequest = SearchRequest(query="test", limit=1)validated = _validate_search_request(request)assert validated == request-+# Minimum valid timeoutrequest = SearchRequest(query="test", timeout=1)validated = _validate_search_request(request)@@ -191,16 +221,16 @@ class TestSearchRequestModel:data1 = request1.model_dump(by_alias=True)assert "ignore_invalid_urls" in data1 # No alias, uses snake_caseassert data1["ignore_invalid_urls"] is None-+# Test with explicit False valuerequest2 = SearchRequest(query="test",ignore_invalid_urls=False,scrape_options=ScrapeOptions(formats=["markdown"]))-+# Check that aliases are used in model_dump with by_alias=Truedata2 = request2.model_dump(by_alias=True)assert "ignore_invalid_urls" in data2 # No alias, uses snake_caseassert "scrape_options" in data2 # No alias, uses snake_case- assert data2["ignore_invalid_urls"] is False\ No newline at end of file+ assert data2["ignore_invalid_urls"] is Falsediff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.pyindex d8e05b118..b91683a5a 100644--- a/apps/python-sdk/firecrawl/v2/methods/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/search.py@@ -2,6 +2,7 @@Search functionality for Firecrawl v2 API."""+import refrom typing import Optional, Dict, Any, Unionfrom ..types import SearchRequest, SearchData, SearchResult, Documentfrom ..utils.normalize import normalize_document_input@@ -119,11 +120,18 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:# Validate tbs (time-based search, if provided)if request.tbs is not None:valid_tbs_values = {- "qdr:d", "qdr:w", "qdr:m", "qdr:y", # Google time filters+ "qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y", # Google time filters"d", "w", "m", "y" # Short forms}- if request.tbs not in valid_tbs_values:- raise ValueError(f"Invalid tbs value: {request.tbs}. Valid values: {valid_tbs_values}")++ if request.tbs in valid_tbs_values:+ pass # Valid predefined value+ elif request.tbs.startswith("cdr:"):+ custom_date_pattern = r"^cdr:1,cd_min:\d{1,2}/\d{1,2}/\d{4},cd_max:\d{1,2}/\d{1,2}/\d{4}$"+ if not re.match(custom_date_pattern, request.tbs):+ raise ValueError(f"Invalid custom date range format: {request.tbs}. Expected format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")+ else:+ raise ValueError(f"Invalid tbs value: {request.tbs}. Valid values: {valid_tbs_values} or custom date range format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")# Validate scrape_options (if provided)if request.scrape_options is not None:@@ -166,4 +174,4 @@ def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:data["scrapeOptions"] = scrape_datadata.pop("scrape_options", None)- return data\ No newline at end of file+ return data