Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Python sdk fix/batch validate limit (#2399)
Rafael Millercdd1e897c1f6
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex dfea7024c..c5e2be6df 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.10.1"+__version__ = "4.10.2"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/conftest.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/conftest.pynew file mode 100644index 000000000..6f1fe5dfa--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/conftest.py@@ -0,0 +1,62 @@+"""+Pytest configuration for async E2E tests.+Ensures environment variables are loaded before any test runs.+"""+import os+import pytest+from dotenv import load_dotenv, find_dotenv++# Load environment IMMEDIATELY at module import time (before pytest collects tests)+# This ensures env vars are loaded before the first test runs+_env_loaded = False++def _ensure_env_loaded():+ """Ensure environment is loaded exactly once, synchronously."""+ global _env_loaded+ if not _env_loaded:+ # Find and load .env from multiple possible locations+ env_file = find_dotenv(usecwd=True)+ if env_file:+ load_dotenv(env_file, override=True)+ else:+ # Try loading from current directory+ load_dotenv(override=True)+ _env_loaded = True++# Load env immediately when this module is imported+_ensure_env_loaded()++[email protected](scope="session", autouse=True)+def load_environment():+ """Ensure environment variables are loaded before running any tests."""+ # Double-check env is loaded (should already be done at import time)+ _ensure_env_loaded()++ # Validate required environment variables+ required_vars = ["API_KEY", "API_URL"]+ missing_vars = [var for var in required_vars if not os.getenv(var)]++ if missing_vars:+ pytest.skip(f"Skipping E2E tests: Missing required environment variables: {', '.join(missing_vars)}")++ yield++[email protected](scope="function")+def api_key():+ """Provide API key for tests."""+ key = os.getenv("API_KEY")+ if not key:+ pytest.skip("API_KEY not set")+ return key++[email protected](scope="function")+def api_url():+ """Provide API URL for tests."""+ url = os.getenv("API_URL")+ if not url:+ pytest.skip("API_URL not set")+ return url+diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.pyindex e0611543d..52fd04c4f 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_batch_scrape.py@@ -1,22 +1,11 @@-import osimport asyncioimport pytest-from dotenv import load_dotenvfrom firecrawl import AsyncFirecrawl-load_dotenv()--if not os.getenv("API_KEY"):- raise ValueError("API_KEY is not set")--if not os.getenv("API_URL"):- raise ValueError("API_URL is not set")--@pytest.mark.asyncio-async def test_async_batch_start_and_status():- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))+async def test_async_batch_start_and_status(api_key, api_url):+ client = AsyncFirecrawl(api_key=api_key, api_url=api_url)start = await client.start_batch_scrape(["https://docs.firecrawl.dev","https://firecrawl.dev",@@ -33,8 +22,8 @@ async def test_async_batch_start_and_status():@pytest.mark.asyncio-async def test_async_batch_wait_minimal():- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))+async def test_async_batch_wait_minimal(api_key, api_url):+ client = AsyncFirecrawl(api_key=api_key, api_url=api_url)job = await client.batch_scrape(["https://docs.firecrawl.dev","https://firecrawl.dev",@@ -43,8 +32,8 @@ async def test_async_batch_wait_minimal():@pytest.mark.asyncio-async def test_async_batch_wait_with_all_params():- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))+async def test_async_batch_wait_with_all_params(api_key, api_url):+ client = AsyncFirecrawl(api_key=api_key, api_url=api_url)json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}job = await client.batch_scrape([@@ -69,8 +58,8 @@ async def test_async_batch_wait_with_all_params():@pytest.mark.asyncio-async def test_async_cancel_batch():- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))+async def test_async_cancel_batch(api_key, api_url):+ client = AsyncFirecrawl(api_key=api_key, api_url=api_url)start = await client.start_batch_scrape(["https://docs.firecrawl.dev","https://firecrawl.dev",diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.pyindex da119b2ac..2544445af 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_crawl.py@@ -275,4 +275,4 @@ class TestCrawlE2E:assert params_data is not Noneassert params_data.limit is not None or params_data.include_paths is not None or params_data.max_discovery_depth is not Noneassert 'blog/.*' in params_data.include_paths- assert 'documentation/.*' in params_data.include_paths\ No newline at end of file+ assert 'docs/.*' in params_data.include_paths\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py b/apps/python-sdk/firecrawl/v2/methods/aio/batch.pyindex 72fb60961..958cd78db 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/batch.py@@ -4,13 +4,15 @@ from ...utils.http_client_async import AsyncHttpClientfrom ...utils.validation import prepare_scrape_optionsfrom ...utils.error_handler import handle_response_errorfrom ...utils.normalize import normalize_document_input+from ...methods.batch import validate_batch_urlsimport time-def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwargs) -> Dict[str, Any]:if not urls:raise ValueError("URLs list cannot be empty")- payload: Dict[str, Any] = {"urls": [u.strip() for u in urls]}++ validated_urls = validate_batch_urls([u.strip() if isinstance(u, str) else u for u in urls])+ payload: Dict[str, Any] = {"urls": validated_urls}if options:opts = prepare_scrape_options(options)if opts:diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.pyindex 591a8068b..53d2ea44d 100644--- a/apps/python-sdk/firecrawl/v2/methods/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py@@ -345,9 +345,6 @@ def validate_batch_urls(urls: List[str]) -> List[str]:if not urls:raise ValueError("URLs list cannot be empty")- if len(urls) > 1000: # Assuming API limit- raise ValueError("Too many URLs (maximum 1000)")-validated_urls = []for url in urls:if not url or not isinstance(url, str):