Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Fix search validation of custom date ranges and "qdr:h" (#1993)

Micah Stairsac278ccc1b30
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 5c4e14037..7f0ecc4cf 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "3.1.0"
+__version__ = "3.1.1"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
@@ -84,4 +84,4 @@ __all__ = [
'V1JsonConfig',
'V1ScrapeOptions',
'V1ChangeTrackingOptions',
-]
\ No newline at end of file
+]
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py
index d41c50409..fd58c9f58 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py
@@ -11,7 +11,7 @@ class TestSearchValidation:
request = SearchRequest(query="")
with pytest.raises(ValueError, match="Query cannot be empty"):
_validate_search_request(request)
-
+
request = SearchRequest(query=" ")
with pytest.raises(ValueError, match="Query cannot be empty"):
_validate_search_request(request)
@@ -22,12 +22,12 @@ class TestSearchValidation:
request = SearchRequest(query="test", limit=0)
with pytest.raises(ValueError, match="Limit must be positive"):
_validate_search_request(request)
-
+
# Negative limit
request = SearchRequest(query="test", limit=-1)
with pytest.raises(ValueError, match="Limit must be positive"):
_validate_search_request(request)
-
+
# Too high limit
request = SearchRequest(query="test", limit=101)
with pytest.raises(ValueError, match="Limit cannot exceed 100"):
@@ -39,12 +39,12 @@ class TestSearchValidation:
request = SearchRequest(query="test", timeout=0)
with pytest.raises(ValueError, match="Timeout must be positive"):
_validate_search_request(request)
-
+
# Negative timeout
request = SearchRequest(query="test", timeout=-1000)
with pytest.raises(ValueError, match="Timeout must be positive"):
_validate_search_request(request)
-
+
# Too high timeout
request = SearchRequest(query="test", timeout=300001)
with pytest.raises(ValueError, match="Timeout cannot exceed 300000ms"):
@@ -56,12 +56,12 @@ class TestSearchValidation:
request = SearchRequest(query="test", sources=["invalid_source"])
with pytest.raises(ValueError, match="Invalid source type"):
_validate_search_request(request)
-
+
# Invalid object source
request = SearchRequest(query="test", sources=[Source(type="invalid_source")])
with pytest.raises(ValueError, match="Invalid source type"):
_validate_search_request(request)
-
+
# Mixed valid/invalid sources
request = SearchRequest(query="test", sources=["web", "invalid_source"])
with pytest.raises(ValueError, match="Invalid source type"):
@@ -73,7 +73,7 @@ class TestSearchValidation:
request = SearchRequest(query="test", location="")
with pytest.raises(ValueError, match="Location must be a non-empty string"):
_validate_search_request(request)
-
+
# Whitespace location
request = SearchRequest(query="test", location=" ")
with pytest.raises(ValueError, match="Location must be a non-empty string"):
@@ -82,19 +82,49 @@ class TestSearchValidation:
def test_validate_invalid_tbs(self):
"""Test validation of invalid tbs values."""
invalid_tbs_values = ["invalid", "qdr:x", "yesterday", "last_week"]
-
+
for invalid_tbs in invalid_tbs_values:
request = SearchRequest(query="test", tbs=invalid_tbs)
with pytest.raises(ValueError, match="Invalid tbs value"):
_validate_search_request(request)
+ def test_validate_custom_date_ranges(self):
+ """Test validation of custom date range formats."""
+ valid_custom_ranges = [
+ "cdr:1,cd_min:1/1/2024,cd_max:12/31/2024",
+ "cdr:1,cd_min:12/1/2024,cd_max:12/31/2024",
+ "cdr:1,cd_min:2/28/2023,cd_max:3/1/2023",
+ "cdr:1,cd_min:10/15/2023,cd_max:11/15/2023"
+ ]
+
+ for valid_range in valid_custom_ranges:
+ request = SearchRequest(query="test", tbs=valid_range)
+ validated = _validate_search_request(request)
+ assert validated == request
+
+ def test_validate_invalid_custom_date_ranges(self):
+ """Test validation of invalid custom date range formats."""
+ # Invalid custom date ranges
+ invalid_custom_ranges = [
+ "cdr:1,cd_min:2/28/2023", # Missing cd_max
+ "cdr:1,cd_max:2/28/2023", # Missing cd_min
+ "cdr:2,cd_min:1/1/2024,cd_max:12/31/2024", # Wrong cdr value
+ "cdr:cd_min:1/1/2024,cd_max:12/31/2024", # Missing :1
+ "custom:1,cd_min:1/1/2024,cd_max:12/31/2024" # Wrong prefix
+ ]
+
+ for invalid_range in invalid_custom_ranges:
+ request = SearchRequest(query="test", tbs=invalid_range)
+ with pytest.raises(ValueError, match="Invalid"):
+ _validate_search_request(request)
+
def test_validate_valid_requests(self):
"""Test that valid requests pass validation."""
# Minimal valid request
request = SearchRequest(query="test")
validated = _validate_search_request(request)
assert validated == request
-
+
# Request with all optional parameters
request = SearchRequest(
query="test query",
@@ -107,7 +137,7 @@ class TestSearchValidation:
)
validated = _validate_search_request(request)
assert validated == request
-
+
# Request with object sources
request = SearchRequest(
query="test",
@@ -122,17 +152,17 @@ class TestSearchValidation:
request = SearchRequest(query="test", limit=100)
validated = _validate_search_request(request)
assert validated == request
-
+
# Maximum valid timeout
request = SearchRequest(query="test", timeout=300000)
validated = _validate_search_request(request)
assert validated == request
-
+
# Minimum valid limit
request = SearchRequest(query="test", limit=1)
validated = _validate_search_request(request)
assert validated == request
-
+
# Minimum valid timeout
request = SearchRequest(query="test", timeout=1)
validated = _validate_search_request(request)
@@ -191,16 +221,16 @@ class TestSearchRequestModel:
data1 = request1.model_dump(by_alias=True)
assert "ignore_invalid_urls" in data1 # No alias, uses snake_case
assert data1["ignore_invalid_urls"] is None
-
+
# Test with explicit False value
request2 = SearchRequest(
query="test",
ignore_invalid_urls=False,
scrape_options=ScrapeOptions(formats=["markdown"])
)
-
+
# Check that aliases are used in model_dump with by_alias=True
data2 = request2.model_dump(by_alias=True)
assert "ignore_invalid_urls" in data2 # No alias, uses snake_case
assert "scrape_options" in data2 # No alias, uses snake_case
- assert data2["ignore_invalid_urls"] is False
\ No newline at end of file
+ assert data2["ignore_invalid_urls"] is False
diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.py
index d8e05b118..b91683a5a 100644
--- a/apps/python-sdk/firecrawl/v2/methods/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/search.py
@@ -2,6 +2,7 @@
Search functionality for Firecrawl v2 API.
"""
+import re
from typing import Optional, Dict, Any, Union
from ..types import SearchRequest, SearchData, SearchResult, Document
from ..utils.normalize import normalize_document_input
@@ -119,11 +120,18 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:
# Validate tbs (time-based search, if provided)
if request.tbs is not None:
valid_tbs_values = {
- "qdr:d", "qdr:w", "qdr:m", "qdr:y", # Google time filters
+ "qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y", # Google time filters
"d", "w", "m", "y" # Short forms
}
- if request.tbs not in valid_tbs_values:
- raise ValueError(f"Invalid tbs value: {request.tbs}. Valid values: {valid_tbs_values}")
+
+ if request.tbs in valid_tbs_values:
+ pass # Valid predefined value
+ elif request.tbs.startswith("cdr:"):
+ custom_date_pattern = r"^cdr:1,cd_min:\d{1,2}/\d{1,2}/\d{4},cd_max:\d{1,2}/\d{1,2}/\d{4}$"
+ if not re.match(custom_date_pattern, request.tbs):
+ raise ValueError(f"Invalid custom date range format: {request.tbs}. Expected format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")
+ else:
+ raise ValueError(f"Invalid tbs value: {request.tbs}. Valid values: {valid_tbs_values} or custom date range format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")
# Validate scrape_options (if provided)
if request.scrape_options is not None:
@@ -166,4 +174,4 @@ def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
data["scrapeOptions"] = scrape_data
data.pop("scrape_options", None)
- return data
\ No newline at end of file
+ return data