Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick; (#1726)

Nicolas17ff8be67b26
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 4bdc40a02..26b05e327 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "1.28.0",
+ "version": "1.29.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts
index b75f39708..3bebe6424 100644
--- a/apps/js-sdk/firecrawl/src/index.ts
+++ b/apps/js-sdk/firecrawl/src/index.ts
@@ -229,6 +229,7 @@ export interface CrawlParams {
* If not provided, the crawler may use the robots.txt crawl delay if available.
*/
delay?: number;
+ allowSubdomains?: boolean;
maxConcurrency?: number;
}
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 33a7f33c0..404027b1c 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -13,7 +13,7 @@ import os
from .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa
-__version__ = "2.11.0"
+__version__ = "2.12.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py
index 9d4ec3588..8a39abf55 100644
--- a/apps/python-sdk/firecrawl/firecrawl.py
+++ b/apps/python-sdk/firecrawl/firecrawl.py
@@ -273,6 +273,7 @@ class CrawlParams(pydantic.BaseModel):
regexOnFullURL: Optional[bool] = None
delay: Optional[int] = None # Delay in seconds between scrapes
maxConcurrency: Optional[int] = None
+ allowSubdomains: Optional[bool] = None
class CrawlResponse(pydantic.BaseModel):
"""Response from crawling operations."""
@@ -708,6 +709,7 @@ class FirecrawlApp:
ignore_query_parameters: Optional[bool] = None,
regex_on_full_url: Optional[bool] = None,
delay: Optional[int] = None,
+ allow_subdomains: Optional[bool] = None,
max_concurrency: Optional[int] = None,
poll_interval: Optional[int] = 2,
idempotency_key: Optional[str] = None,
@@ -733,6 +735,7 @@ class FirecrawlApp:
ignore_query_parameters (Optional[bool]): Ignore URL parameters
regex_on_full_url (Optional[bool]): Apply regex to full URLs
delay (Optional[int]): Delay in seconds between scrapes
+ allow_subdomains (Optional[bool]): Follow subdomains
max_concurrency (Optional[int]): Maximum number of concurrent scrapes
poll_interval (Optional[int]): Seconds between status checks (default: 2)
idempotency_key (Optional[str]): Unique key to prevent duplicate requests
@@ -783,6 +786,8 @@ class FirecrawlApp:
crawl_params['regexOnFullURL'] = regex_on_full_url
if delay is not None:
crawl_params['delay'] = delay
+ if allow_subdomains is not None:
+ crawl_params['allowSubdomains'] = allow_subdomains
if max_concurrency is not None:
crawl_params['maxConcurrency'] = max_concurrency
@@ -827,6 +832,8 @@ class FirecrawlApp:
ignore_query_parameters: Optional[bool] = None,
regex_on_full_url: Optional[bool] = None,
delay: Optional[int] = None,
+ allow_subdomains: Optional[bool] = None,
+ max_concurrency: Optional[int] = None,
idempotency_key: Optional[str] = None,
**kwargs
) -> CrawlResponse:
@@ -850,6 +857,7 @@ class FirecrawlApp:
ignore_query_parameters (Optional[bool]): Ignore URL parameters
regex_on_full_url (Optional[bool]): Apply regex to full URLs
delay (Optional[int]): Delay in seconds between scrapes
+ allow_subdomains (Optional[bool]): Follow subdomains
max_concurrency (Optional[int]): Maximum number of concurrent scrapes
idempotency_key (Optional[str]): Unique key to prevent duplicate requests
**kwargs: Additional parameters to pass to the API
@@ -900,6 +908,8 @@ class FirecrawlApp:
crawl_params['regexOnFullURL'] = regex_on_full_url
if delay is not None:
crawl_params['delay'] = delay
+ if allow_subdomains is not None:
+ crawl_params['allowSubdomains'] = allow_subdomains
if max_concurrency is not None:
crawl_params['maxConcurrency'] = max_concurrency
@@ -1080,6 +1090,7 @@ class FirecrawlApp:
ignore_query_parameters: Optional[bool] = None,
regex_on_full_url: Optional[bool] = None,
delay: Optional[int] = None,
+ allow_subdomains: Optional[bool] = None,
max_concurrency: Optional[int] = None,
idempotency_key: Optional[str] = None,
**kwargs
@@ -1104,6 +1115,7 @@ class FirecrawlApp:
ignore_query_parameters (Optional[bool]): Ignore URL parameters
regex_on_full_url (Optional[bool]): Apply regex to full URLs
delay (Optional[int]): Delay in seconds between scrapes
+ allow_subdomains (Optional[bool]): Follow subdomains
max_concurrency (Optional[int]): Maximum number of concurrent scrapes
idempotency_key (Optional[str]): Unique key to prevent duplicate requests
**kwargs: Additional parameters to pass to the API
@@ -1130,6 +1142,7 @@ class FirecrawlApp:
ignore_query_parameters=ignore_query_parameters,
regex_on_full_url=regex_on_full_url,
delay=delay,
+ allow_subdomains=allow_subdomains,
max_concurrency=max_concurrency,
idempotency_key=idempotency_key,
**kwargs
@@ -3325,6 +3338,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
ignore_query_parameters: Optional[bool] = None,
regex_on_full_url: Optional[bool] = None,
delay: Optional[int] = None,
+ allow_subdomains: Optional[bool] = None,
poll_interval: Optional[int] = 2,
idempotency_key: Optional[str] = None,
**kwargs
@@ -3349,6 +3363,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
ignore_query_parameters (Optional[bool]): Ignore URL parameters
regex_on_full_url (Optional[bool]): Apply regex to full URLs
delay (Optional[int]): Delay in seconds between scrapes
+ allow_subdomains (Optional[bool]): Follow subdomains
poll_interval (Optional[int]): Seconds between status checks (default: 2)
idempotency_key (Optional[str]): Unique key to prevent duplicate requests
**kwargs: Additional parameters to pass to the API
@@ -3398,6 +3413,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
crawl_params['regexOnFullURL'] = regex_on_full_url
if delay is not None:
crawl_params['delay'] = delay
+ if allow_subdomains is not None:
+ crawl_params['allowSubdomains'] = allow_subdomains
# Add any additional kwargs
crawl_params.update(kwargs)
@@ -3441,6 +3458,7 @@ class AsyncFirecrawlApp(FirecrawlApp):
ignore_query_parameters: Optional[bool] = None,
regex_on_full_url: Optional[bool] = None,
delay: Optional[int] = None,
+ allow_subdomains: Optional[bool] = None,
poll_interval: Optional[int] = 2,
idempotency_key: Optional[str] = None,
**kwargs
@@ -3510,6 +3528,8 @@ class AsyncFirecrawlApp(FirecrawlApp):
crawl_params['regexOnFullURL'] = regex_on_full_url
if delay is not None:
crawl_params['delay'] = delay
+ if allow_subdomains is not None:
+ crawl_params['allowSubdomains'] = allow_subdomains
# Add any additional kwargs
crawl_params.update(kwargs)