Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sdk): add ignoreRobotsTxt and robotsUserAgent to JS, Python, and Java SDKs (#3375)
firecrawl-spring[bot]fcf6b89aaee0
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex 7a01c94b8..726299377 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.2.0"+version = "1.2.1"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/CrawlOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/CrawlOptions.javaindex 68bcb6ca7..a69390c0e 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/CrawlOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/CrawlOptions.java@@ -21,6 +21,8 @@ public class CrawlOptions {private Boolean crawlEntireDomain;private Boolean allowExternalLinks;private Boolean allowSubdomains;+ private Boolean ignoreRobotsTxt;+ private String robotsUserAgent;private Integer delay;private Integer maxConcurrency;private Object webhook;@@ -42,6 +44,8 @@ public class CrawlOptions {public Boolean getCrawlEntireDomain() { return crawlEntireDomain; }public Boolean getAllowExternalLinks() { return allowExternalLinks; }public Boolean getAllowSubdomains() { return allowSubdomains; }+ public Boolean getIgnoreRobotsTxt() { return ignoreRobotsTxt; }+ public String getRobotsUserAgent() { return robotsUserAgent; }public Integer getDelay() { return delay; }public Integer getMaxConcurrency() { return maxConcurrency; }public Object getWebhook() { return webhook; }@@ -64,6 +68,8 @@ public class CrawlOptions {private Boolean crawlEntireDomain;private Boolean allowExternalLinks;private Boolean allowSubdomains;+ private Boolean ignoreRobotsTxt;+ private String robotsUserAgent;private Integer delay;private Integer maxConcurrency;private Object webhook;@@ -107,6 +113,12 @@ public class CrawlOptions {/** Follow subdomains. */public Builder allowSubdomains(Boolean allowSubdomains) { this.allowSubdomains = allowSubdomains; return this; }+ /** Ignore the website's robots.txt rules. Enterprise only. */+ public Builder ignoreRobotsTxt(Boolean ignoreRobotsTxt) { this.ignoreRobotsTxt = ignoreRobotsTxt; return this; }++ /** Custom User-Agent string for robots.txt evaluation. Enterprise only. */+ public Builder robotsUserAgent(String robotsUserAgent) { this.robotsUserAgent = robotsUserAgent; return this; }+/** Delay in milliseconds between requests. */public Builder delay(Integer delay) { this.delay = delay; return this; }@@ -141,6 +153,8 @@ public class CrawlOptions {o.crawlEntireDomain = this.crawlEntireDomain;o.allowExternalLinks = this.allowExternalLinks;o.allowSubdomains = this.allowSubdomains;+ o.ignoreRobotsTxt = this.ignoreRobotsTxt;+ o.robotsUserAgent = this.robotsUserAgent;o.delay = this.delay;o.maxConcurrency = this.maxConcurrency;o.webhook = this.webhook;diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex fe2c696cd..4ca17ed50 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.18.2",+ "version": "4.18.3","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex d8dcd88b1..58a329791 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -482,6 +482,8 @@ export interface CrawlOptions {crawlEntireDomain?: boolean;allowExternalLinks?: boolean;allowSubdomains?: boolean;+ ignoreRobotsTxt?: boolean;+ robotsUserAgent?: string | null;delay?: number | null;maxConcurrency?: number | null;webhook?: string | WebhookConfig | null;diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 8da0cc9f9..98765238e 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.22.1"+__version__ = "4.22.2"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.pyindex 2f679bcab..68bda449f 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py@@ -49,6 +49,8 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:"crawl_entire_domain": "crawlEntireDomain","allow_external_links": "allowExternalLinks","allow_subdomains": "allowSubdomains",+ "ignore_robots_txt": "ignoreRobotsTxt",+ "robots_user_agent": "robotsUserAgent","delay": "delay","max_concurrency": "maxConcurrency","regex_on_full_url": "regexOnFullURL",@@ -335,6 +337,8 @@ async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequ"crawlEntireDomain": "crawl_entire_domain","allowExternalLinks": "allow_external_links","allowSubdomains": "allow_subdomains",+ "ignoreRobotsTxt": "ignore_robots_txt",+ "robotsUserAgent": "robots_user_agent","maxConcurrency": "max_concurrency","scrapeOptions": "scrape_options","zeroDataRetention": "zero_data_retention",diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex b05dfd75d..9f1a8b742 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -89,6 +89,7 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:"allow_external_links": "allowExternalLinks","allow_subdomains": "allowSubdomains","ignore_robots_txt": "ignoreRobotsTxt",+ "robots_user_agent": "robotsUserAgent","delay": "delay","max_concurrency": "maxConcurrency","regex_on_full_url": "regexOnFullURL",@@ -512,6 +513,8 @@ def crawl_params_preview(client: HttpClient, request: CrawlParamsRequest) -> Cra"crawlEntireDomain": "crawl_entire_domain","allowExternalLinks": "allow_external_links","allowSubdomains": "allow_subdomains",+ "ignoreRobotsTxt": "ignore_robots_txt",+ "robotsUserAgent": "robots_user_agent","maxConcurrency": "max_concurrency","scrapeOptions": "scrape_options","zeroDataRetention": "zero_data_retention"diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex ac4aec5cd..37a3763a8 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -599,6 +599,7 @@ class CrawlRequest(BaseModel):allow_external_links: bool = Falseallow_subdomains: bool = Falseignore_robots_txt: bool = False+ robots_user_agent: Optional[str] = Nonedelay: Optional[int] = Nonemax_concurrency: Optional[int] = Nonewebhook: Optional[Union[str, WebhookConfig]] = None@@ -694,6 +695,7 @@ class CrawlParamsData(BaseModel):allow_external_links: bool = Falseallow_subdomains: bool = Falseignore_robots_txt: bool = False+ robots_user_agent: Optional[str] = Nonedelay: Optional[int] = Nonemax_concurrency: Optional[int] = Nonewebhook: Optional[Union[str, WebhookConfig]] = None