Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add delay parameter to crawl options in all SDKs (#1514)
devin-ai-integration[bot]0512ad6bce0b
diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex ed090e10f..ccb64521f 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -215,6 +215,11 @@ export interface CrawlParams {deduplicateSimilarURLs?: boolean;ignoreQueryParameters?: boolean;regexOnFullURL?: boolean;+ /**+ * Delay in seconds between scrapes. This helps respect website rate limits.+ * If not provided, the crawler may use the robots.txt crawl delay if available.+ */+ delay?: number;}/**diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex 0942fe45e..d51c3208a 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -259,6 +259,7 @@ class CrawlParams(pydantic.BaseModel):deduplicateSimilarURLs: Optional[bool] = NoneignoreQueryParameters: Optional[bool] = NoneregexOnFullURL: Optional[bool] = None+ delay: Optional[int] = None # Delay in seconds between scrapesclass CrawlResponse(pydantic.BaseModel):"""Response from crawling operations."""@@ -681,6 +682,7 @@ class FirecrawlApp:deduplicate_similar_urls: Optional[bool] = None,ignore_query_parameters: Optional[bool] = None,regex_on_full_url: Optional[bool] = None,+ delay: Optional[int] = None,poll_interval: Optional[int] = 2,idempotency_key: Optional[str] = None,**kwargs@@ -703,6 +705,7 @@ class FirecrawlApp:deduplicate_similar_urls (Optional[bool]): Remove similar URLsignore_query_parameters (Optional[bool]): Ignore URL parametersregex_on_full_url (Optional[bool]): Apply regex to full URLs+ delay (Optional[int]): Delay in seconds between scrapespoll_interval (Optional[int]): Seconds between status checks (default: 2)idempotency_key (Optional[str]): Unique key to prevent duplicate requests**kwargs: Additional parameters to pass to the API@@ -748,6 +751,8 @@ class FirecrawlApp:crawl_params['ignoreQueryParameters'] = ignore_query_parametersif regex_on_full_url is not None:crawl_params['regexOnFullURL'] = regex_on_full_url+ if delay is not None:+ crawl_params['delay'] = delay# Add any additional kwargscrawl_params.update(kwargs)@@ -788,6 +793,7 @@ class FirecrawlApp:deduplicate_similar_urls: Optional[bool] = None,ignore_query_parameters: Optional[bool] = None,regex_on_full_url: Optional[bool] = None,+ delay: Optional[int] = None,idempotency_key: Optional[str] = None,**kwargs) -> CrawlResponse:@@ -854,6 +860,8 @@ class FirecrawlApp:crawl_params['ignoreQueryParameters'] = ignore_query_parametersif regex_on_full_url is not None:crawl_params['regexOnFullURL'] = regex_on_full_url+ if delay is not None:+ crawl_params['delay'] = delay# Add any additional kwargscrawl_params.update(kwargs)@@ -3240,6 +3248,7 @@ class AsyncFirecrawlApp(FirecrawlApp):deduplicate_similar_urls: Optional[bool] = None,ignore_query_parameters: Optional[bool] = None,regex_on_full_url: Optional[bool] = None,+ delay: Optional[int] = None,poll_interval: Optional[int] = 2,idempotency_key: Optional[str] = None,**kwargs@@ -3262,6 +3271,7 @@ class AsyncFirecrawlApp(FirecrawlApp):deduplicate_similar_urls (Optional[bool]): Remove similar URLsignore_query_parameters (Optional[bool]): Ignore URL parametersregex_on_full_url (Optional[bool]): Apply regex to full URLs+ delay (Optional[int]): Delay in seconds between scrapespoll_interval (Optional[int]): Seconds between status checks (default: 2)idempotency_key (Optional[str]): Unique key to prevent duplicate requests**kwargs: Additional parameters to pass to the API@@ -3307,6 +3317,8 @@ class AsyncFirecrawlApp(FirecrawlApp):crawl_params['ignoreQueryParameters'] = ignore_query_parametersif regex_on_full_url is not None:crawl_params['regexOnFullURL'] = regex_on_full_url+ if delay is not None:+ crawl_params['delay'] = delay# Add any additional kwargscrawl_params.update(kwargs)@@ -3348,6 +3360,7 @@ class AsyncFirecrawlApp(FirecrawlApp):deduplicate_similar_urls: Optional[bool] = None,ignore_query_parameters: Optional[bool] = None,regex_on_full_url: Optional[bool] = None,+ delay: Optional[int] = None,poll_interval: Optional[int] = 2,idempotency_key: Optional[str] = None,**kwargs@@ -3412,6 +3425,8 @@ class AsyncFirecrawlApp(FirecrawlApp):crawl_params['ignoreQueryParameters'] = ignore_query_parametersif regex_on_full_url is not None:crawl_params['regexOnFullURL'] = regex_on_full_url+ if delay is not None:+ crawl_params['delay'] = delay# Add any additional kwargscrawl_params.update(kwargs)diff --git a/apps/rust-sdk/src/crawl.rs b/apps/rust-sdk/src/crawl.rsindex a5f30f40b..b6522c452 100644--- a/apps/rust-sdk/src/crawl.rs+++ b/apps/rust-sdk/src/crawl.rs@@ -138,6 +138,8 @@ pub struct CrawlOptions {#[serde(skip)]pub idempotency_key: Option<String>,+ pub delay: Option<u32>,+/// When using `FirecrawlApp::crawl_url`, this is how often the status of the job should be checked, in milliseconds. (default: `2000`)#[serde(skip)]pub poll_interval: Option<u64>,