Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(sdk): add ignoreRobotsTxt and robotsUserAgent to JS, Python, and Java SDKs (#3375)

firecrawl-spring[bot]fcf6b89aaee0
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index 7a01c94b8..726299377 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.2.0"
+version = "1.2.1"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/CrawlOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/CrawlOptions.java
index 68bcb6ca7..a69390c0e 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/CrawlOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/CrawlOptions.java
@@ -21,6 +21,8 @@ public class CrawlOptions {
private Boolean crawlEntireDomain;
private Boolean allowExternalLinks;
private Boolean allowSubdomains;
+ private Boolean ignoreRobotsTxt;
+ private String robotsUserAgent;
private Integer delay;
private Integer maxConcurrency;
private Object webhook;
@@ -42,6 +44,8 @@ public class CrawlOptions {
public Boolean getCrawlEntireDomain() { return crawlEntireDomain; }
public Boolean getAllowExternalLinks() { return allowExternalLinks; }
public Boolean getAllowSubdomains() { return allowSubdomains; }
+ public Boolean getIgnoreRobotsTxt() { return ignoreRobotsTxt; }
+ public String getRobotsUserAgent() { return robotsUserAgent; }
public Integer getDelay() { return delay; }
public Integer getMaxConcurrency() { return maxConcurrency; }
public Object getWebhook() { return webhook; }
@@ -64,6 +68,8 @@ public class CrawlOptions {
private Boolean crawlEntireDomain;
private Boolean allowExternalLinks;
private Boolean allowSubdomains;
+ private Boolean ignoreRobotsTxt;
+ private String robotsUserAgent;
private Integer delay;
private Integer maxConcurrency;
private Object webhook;
@@ -107,6 +113,12 @@ public class CrawlOptions {
/** Follow subdomains. */
public Builder allowSubdomains(Boolean allowSubdomains) { this.allowSubdomains = allowSubdomains; return this; }
+ /** Ignore the website's robots.txt rules. Enterprise only. */
+ public Builder ignoreRobotsTxt(Boolean ignoreRobotsTxt) { this.ignoreRobotsTxt = ignoreRobotsTxt; return this; }
+
+ /** Custom User-Agent string for robots.txt evaluation. Enterprise only. */
+ public Builder robotsUserAgent(String robotsUserAgent) { this.robotsUserAgent = robotsUserAgent; return this; }
+
/** Delay in milliseconds between requests. */
public Builder delay(Integer delay) { this.delay = delay; return this; }
@@ -141,6 +153,8 @@ public class CrawlOptions {
o.crawlEntireDomain = this.crawlEntireDomain;
o.allowExternalLinks = this.allowExternalLinks;
o.allowSubdomains = this.allowSubdomains;
+ o.ignoreRobotsTxt = this.ignoreRobotsTxt;
+ o.robotsUserAgent = this.robotsUserAgent;
o.delay = this.delay;
o.maxConcurrency = this.maxConcurrency;
o.webhook = this.webhook;
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index fe2c696cd..4ca17ed50 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.18.2",
+ "version": "4.18.3",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index d8dcd88b1..58a329791 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -482,6 +482,8 @@ export interface CrawlOptions {
crawlEntireDomain?: boolean;
allowExternalLinks?: boolean;
allowSubdomains?: boolean;
+ ignoreRobotsTxt?: boolean;
+ robotsUserAgent?: string | null;
delay?: number | null;
maxConcurrency?: number | null;
webhook?: string | WebhookConfig | null;
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 8da0cc9f9..98765238e 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.22.1"
+__version__ = "4.22.2"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
index 2f679bcab..68bda449f 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
@@ -49,6 +49,8 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:
"crawl_entire_domain": "crawlEntireDomain",
"allow_external_links": "allowExternalLinks",
"allow_subdomains": "allowSubdomains",
+ "ignore_robots_txt": "ignoreRobotsTxt",
+ "robots_user_agent": "robotsUserAgent",
"delay": "delay",
"max_concurrency": "maxConcurrency",
"regex_on_full_url": "regexOnFullURL",
@@ -335,6 +337,8 @@ async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequ
"crawlEntireDomain": "crawl_entire_domain",
"allowExternalLinks": "allow_external_links",
"allowSubdomains": "allow_subdomains",
+ "ignoreRobotsTxt": "ignore_robots_txt",
+ "robotsUserAgent": "robots_user_agent",
"maxConcurrency": "max_concurrency",
"scrapeOptions": "scrape_options",
"zeroDataRetention": "zero_data_retention",
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
index b05dfd75d..9f1a8b742 100644
--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -89,6 +89,7 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:
"allow_external_links": "allowExternalLinks",
"allow_subdomains": "allowSubdomains",
"ignore_robots_txt": "ignoreRobotsTxt",
+ "robots_user_agent": "robotsUserAgent",
"delay": "delay",
"max_concurrency": "maxConcurrency",
"regex_on_full_url": "regexOnFullURL",
@@ -512,6 +513,8 @@ def crawl_params_preview(client: HttpClient, request: CrawlParamsRequest) -> Cra
"crawlEntireDomain": "crawl_entire_domain",
"allowExternalLinks": "allow_external_links",
"allowSubdomains": "allow_subdomains",
+ "ignoreRobotsTxt": "ignore_robots_txt",
+ "robotsUserAgent": "robots_user_agent",
"maxConcurrency": "max_concurrency",
"scrapeOptions": "scrape_options",
"zeroDataRetention": "zero_data_retention"
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index ac4aec5cd..37a3763a8 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -599,6 +599,7 @@ class CrawlRequest(BaseModel):
allow_external_links: bool = False
allow_subdomains: bool = False
ignore_robots_txt: bool = False
+ robots_user_agent: Optional[str] = None
delay: Optional[int] = None
max_concurrency: Optional[int] = None
webhook: Optional[Union[str, WebhookConfig]] = None
@@ -694,6 +695,7 @@ class CrawlParamsData(BaseModel):
allow_external_links: bool = False
allow_subdomains: bool = False
ignore_robots_txt: bool = False
+ robots_user_agent: Optional[str] = None
delay: Optional[int] = None
max_concurrency: Optional[int] = None
webhook: Optional[Union[str, WebhookConfig]] = None