Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick; (#1726)
Nicolas17ff8be67b26
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 4bdc40a02..26b05e327 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "1.28.0",+ "version": "1.29.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex b75f39708..3bebe6424 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -229,6 +229,7 @@ export interface CrawlParams {* If not provided, the crawler may use the robots.txt crawl delay if available.*/delay?: number;+ allowSubdomains?: boolean;maxConcurrency?: number;}diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 33a7f33c0..404027b1c 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -13,7 +13,7 @@ import osfrom .firecrawl import FirecrawlApp, AsyncFirecrawlApp, JsonConfig, ScrapeOptions, ChangeTrackingOptions # noqa-__version__ = "2.11.0"+__version__ = "2.12.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex 9d4ec3588..8a39abf55 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -273,6 +273,7 @@ class CrawlParams(pydantic.BaseModel):regexOnFullURL: Optional[bool] = Nonedelay: Optional[int] = None # Delay in seconds between scrapesmaxConcurrency: Optional[int] = None+ allowSubdomains: Optional[bool] = Noneclass CrawlResponse(pydantic.BaseModel):"""Response from crawling operations."""@@ -708,6 +709,7 @@ class FirecrawlApp:ignore_query_parameters: Optional[bool] = None,regex_on_full_url: Optional[bool] = None,delay: Optional[int] = None,+ allow_subdomains: Optional[bool] = None,max_concurrency: Optional[int] = None,poll_interval: Optional[int] = 2,idempotency_key: Optional[str] = None,@@ -733,6 +735,7 @@ class FirecrawlApp:ignore_query_parameters (Optional[bool]): Ignore URL parametersregex_on_full_url (Optional[bool]): Apply regex to full URLsdelay (Optional[int]): Delay in seconds between scrapes+ allow_subdomains (Optional[bool]): Follow subdomainsmax_concurrency (Optional[int]): Maximum number of concurrent scrapespoll_interval (Optional[int]): Seconds between status checks (default: 2)idempotency_key (Optional[str]): Unique key to prevent duplicate requests@@ -783,6 +786,8 @@ class FirecrawlApp:crawl_params['regexOnFullURL'] = regex_on_full_urlif delay is not None:crawl_params['delay'] = delay+ if allow_subdomains is not None:+ crawl_params['allowSubdomains'] = allow_subdomainsif max_concurrency is not None:crawl_params['maxConcurrency'] = max_concurrency@@ -827,6 +832,8 @@ class FirecrawlApp:ignore_query_parameters: Optional[bool] = None,regex_on_full_url: Optional[bool] = None,delay: Optional[int] = None,+ allow_subdomains: Optional[bool] = None,+ max_concurrency: Optional[int] = None,idempotency_key: Optional[str] = None,**kwargs) -> CrawlResponse:@@ -850,6 +857,7 @@ class FirecrawlApp:ignore_query_parameters (Optional[bool]): Ignore URL parametersregex_on_full_url (Optional[bool]): Apply regex to full URLsdelay (Optional[int]): Delay in seconds between scrapes+ allow_subdomains (Optional[bool]): Follow subdomainsmax_concurrency (Optional[int]): Maximum number of concurrent scrapesidempotency_key (Optional[str]): Unique key to prevent duplicate requests**kwargs: Additional parameters to pass to the API@@ -900,6 +908,8 @@ class FirecrawlApp:crawl_params['regexOnFullURL'] = regex_on_full_urlif delay is not None:crawl_params['delay'] = delay+ if allow_subdomains is not None:+ crawl_params['allowSubdomains'] = allow_subdomainsif max_concurrency is not None:crawl_params['maxConcurrency'] = max_concurrency@@ -1080,6 +1090,7 @@ class FirecrawlApp:ignore_query_parameters: Optional[bool] = None,regex_on_full_url: Optional[bool] = None,delay: Optional[int] = None,+ allow_subdomains: Optional[bool] = None,max_concurrency: Optional[int] = None,idempotency_key: Optional[str] = None,**kwargs@@ -1104,6 +1115,7 @@ class FirecrawlApp:ignore_query_parameters (Optional[bool]): Ignore URL parametersregex_on_full_url (Optional[bool]): Apply regex to full URLsdelay (Optional[int]): Delay in seconds between scrapes+ allow_subdomains (Optional[bool]): Follow subdomainsmax_concurrency (Optional[int]): Maximum number of concurrent scrapesidempotency_key (Optional[str]): Unique key to prevent duplicate requests**kwargs: Additional parameters to pass to the API@@ -1130,6 +1142,7 @@ class FirecrawlApp:ignore_query_parameters=ignore_query_parameters,regex_on_full_url=regex_on_full_url,delay=delay,+ allow_subdomains=allow_subdomains,max_concurrency=max_concurrency,idempotency_key=idempotency_key,**kwargs@@ -3325,6 +3338,7 @@ class AsyncFirecrawlApp(FirecrawlApp):ignore_query_parameters: Optional[bool] = None,regex_on_full_url: Optional[bool] = None,delay: Optional[int] = None,+ allow_subdomains: Optional[bool] = None,poll_interval: Optional[int] = 2,idempotency_key: Optional[str] = None,**kwargs@@ -3349,6 +3363,7 @@ class AsyncFirecrawlApp(FirecrawlApp):ignore_query_parameters (Optional[bool]): Ignore URL parametersregex_on_full_url (Optional[bool]): Apply regex to full URLsdelay (Optional[int]): Delay in seconds between scrapes+ allow_subdomains (Optional[bool]): Follow subdomainspoll_interval (Optional[int]): Seconds between status checks (default: 2)idempotency_key (Optional[str]): Unique key to prevent duplicate requests**kwargs: Additional parameters to pass to the API@@ -3398,6 +3413,8 @@ class AsyncFirecrawlApp(FirecrawlApp):crawl_params['regexOnFullURL'] = regex_on_full_urlif delay is not None:crawl_params['delay'] = delay+ if allow_subdomains is not None:+ crawl_params['allowSubdomains'] = allow_subdomains# Add any additional kwargscrawl_params.update(kwargs)@@ -3441,6 +3458,7 @@ class AsyncFirecrawlApp(FirecrawlApp):ignore_query_parameters: Optional[bool] = None,regex_on_full_url: Optional[bool] = None,delay: Optional[int] = None,+ allow_subdomains: Optional[bool] = None,poll_interval: Optional[int] = 2,idempotency_key: Optional[str] = None,**kwargs@@ -3510,6 +3528,8 @@ class AsyncFirecrawlApp(FirecrawlApp):crawl_params['regexOnFullURL'] = regex_on_full_urlif delay is not None:crawl_params['delay'] = delay+ if allow_subdomains is not None:+ crawl_params['allowSubdomains'] = allow_subdomains# Add any additional kwargscrawl_params.update(kwargs)