Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Python sdk fix/batch validate limit (#2399)

Rafael Millercdd1e897c1f6
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index dfea7024c..c5e2be6df 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.10.1"
+__version__ = "4.10.2"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/conftest.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/conftest.py
new file mode 100644
index 000000000..6f1fe5dfa
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/conftest.py
@@ -0,0 +1,62 @@
+"""
+Pytest configuration for async E2E tests.
+Ensures environment variables are loaded before any test runs.
+"""
+import os
+import pytest
+from dotenv import load_dotenv, find_dotenv
+
+# Load environment IMMEDIATELY at module import time (before pytest collects tests)
+# This ensures env vars are loaded before the first test runs
+_env_loaded = False
+
+def _ensure_env_loaded():
+ """Ensure environment is loaded exactly once, synchronously."""
+ global _env_loaded
+ if not _env_loaded:
+ # Find and load .env from multiple possible locations
+ env_file = find_dotenv(usecwd=True)
+ if env_file:
+ load_dotenv(env_file, override=True)
+ else:
+ # Try loading from current directory
+ load_dotenv(override=True)
+ _env_loaded = True
+
+# Load env immediately when this module is imported
+_ensure_env_loaded()
+
+
[email protected](scope="session", autouse=True)
+def load_environment():
+ """Ensure environment variables are loaded before running any tests."""
+ # Double-check env is loaded (should already be done at import time)
+ _ensure_env_loaded()
+
+ # Validate required environment variables
+ required_vars = ["API_KEY", "API_URL"]
+ missing_vars = [var for var in required_vars if not os.getenv(var)]
+
+ if missing_vars:
+ pytest.skip(f"Skipping E2E tests: Missing required environment variables: {', '.join(missing_vars)}")
+
+ yield
+
+
[email protected](scope="function")
+def api_key():
+ """Provide API key for tests."""
+ key = os.getenv("API_KEY")
+ if not key:
+ pytest.skip("API_KEY not set")
+ return key
+
+
[email protected](scope="function")
+def api_url():
+ """Provide API URL for tests."""
+ url = os.getenv("API_URL")
+ if not url:
+ pytest.skip("API_URL not set")
+ return url
+
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py
index e0611543d..52fd04c4f 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py
@@ -1,22 +1,11 @@
-import os
import asyncio
import pytest
-from dotenv import load_dotenv
from firecrawl import AsyncFirecrawl
-load_dotenv()
-
-if not os.getenv("API_KEY"):
- raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
- raise ValueError("API_URL is not set")
-
-
@pytest.mark.asyncio
-async def test_async_batch_start_and_status():
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+async def test_async_batch_start_and_status(api_key, api_url):
+ client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
start = await client.start_batch_scrape([
"https://docs.firecrawl.dev",
"https://firecrawl.dev",
@@ -33,8 +22,8 @@ async def test_async_batch_start_and_status():
@pytest.mark.asyncio
-async def test_async_batch_wait_minimal():
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+async def test_async_batch_wait_minimal(api_key, api_url):
+ client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
job = await client.batch_scrape([
"https://docs.firecrawl.dev",
"https://firecrawl.dev",
@@ -43,8 +32,8 @@ async def test_async_batch_wait_minimal():
@pytest.mark.asyncio
-async def test_async_batch_wait_with_all_params():
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+async def test_async_batch_wait_with_all_params(api_key, api_url):
+ client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
job = await client.batch_scrape(
[
@@ -69,8 +58,8 @@ async def test_async_batch_wait_with_all_params():
@pytest.mark.asyncio
-async def test_async_cancel_batch():
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
+async def test_async_cancel_batch(api_key, api_url):
+ client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
start = await client.start_batch_scrape([
"https://docs.firecrawl.dev",
"https://firecrawl.dev",
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py
index da119b2ac..2544445af 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py
@@ -275,4 +275,4 @@ class TestCrawlE2E:
assert params_data is not None
assert params_data.limit is not None or params_data.include_paths is not None or params_data.max_discovery_depth is not None
assert 'blog/.*' in params_data.include_paths
- assert 'documentation/.*' in params_data.include_paths
\ No newline at end of file
+ assert 'docs/.*' in params_data.include_paths
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
index 72fb60961..958cd78db 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py
@@ -4,13 +4,15 @@ from ...utils.http_client_async import AsyncHttpClient
from ...utils.validation import prepare_scrape_options
from ...utils.error_handler import handle_response_error
from ...utils.normalize import normalize_document_input
+from ...methods.batch import validate_batch_urls
import time
-
def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwargs) -> Dict[str, Any]:
if not urls:
raise ValueError("URLs list cannot be empty")
- payload: Dict[str, Any] = {"urls": [u.strip() for u in urls]}
+
+ validated_urls = validate_batch_urls([u.strip() if isinstance(u, str) else u for u in urls])
+ payload: Dict[str, Any] = {"urls": validated_urls}
if options:
opts = prepare_scrape_options(options)
if opts:
diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.py
index 591a8068b..53d2ea44d 100644
--- a/apps/python-sdk/firecrawl/v2/methods/batch.py
+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py
@@ -345,9 +345,6 @@ def validate_batch_urls(urls: List[str]) -> List[str]:
if not urls:
raise ValueError("URLs list cannot be empty")
- if len(urls) > 1000: # Assuming API limit
- raise ValueError("Too many URLs (maximum 1000)")
-
validated_urls = []
for url in urls:
if not url or not isinstance(url, str):