Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(crawl): add robotsUserAgent parameter for custom robots.txt checks (#3325)

firecrawl-spring[bot]ef10cd3846a0
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 2d4e439da..15de2f77e 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -529,6 +529,31 @@ describe("Crawl tests", () => {
10 * scrapeTimeout,
);
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "accepts robotsUserAgent parameter",
+ async () => {
+ const robotsIdentity = await idmux({
+ name: "crawl/robotsUserAgent",
+ credits: 10000,
+ flags: { ignoreRobots: "allowed" },
+ });
+
+ const results = await crawl(
+ {
+ url: base,
+ limit: 3,
+ robotsUserAgent: "MyCustomBot",
+ },
+ robotsIdentity,
+ );
+
+ expect(results.success).toBe(true);
+ expect(results.status).toBe("completed");
+ expect(results.completed).toBeGreaterThan(0);
+ },
+ 10 * scrapeTimeout,
+ );
+
concurrentIf(TEST_PRODUCTION || HAS_PROXY)(
"shows warning when crawl results ≤ 1 and URL is not base domain",
async () => {
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 67bf7eb10..a2aea48b2 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -29,7 +29,10 @@ export async function crawlController(
const preNormalizedBody = req.body;
req.body = crawlRequestSchema.parse(req.body);
- const permissions = checkPermissions(req.body, req.acuc?.flags);
+ const permissions = checkPermissions(
+ { ...req.body, crawlerOptions: req.body },
+ req.acuc?.flags,
+ );
if (permissions.error) {
return res.status(403).json({
success: false,
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 4f59e94c3..91de703ed 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -31,7 +31,10 @@ export async function crawlController(
const preNormalizedBody = req.body;
req.body = crawlRequestSchema.parse(req.body);
- const permissions = checkPermissions(req.body, req.acuc?.flags);
+ const permissions = checkPermissions(
+ { ...req.body, crawlerOptions: req.body },
+ req.acuc?.flags,
+ );
if (permissions.error) {
return res.status(403).json({
success: false,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index d9fbd46bb..caf8bfe23 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -910,6 +910,7 @@ export const crawlerOptions = z.strictObject({
allowExternalLinks: z.boolean().prefault(false),
allowSubdomains: z.boolean().prefault(false),
ignoreRobotsTxt: z.boolean().prefault(false),
+ robotsUserAgent: z.string().optional(),
sitemap: z.enum(["skip", "include", "only"]).prefault("include"),
deduplicateSimilarURLs: z.boolean().prefault(true),
ignoreQueryParameters: z.boolean().prefault(false),
@@ -1343,6 +1344,7 @@ export function toV0CrawlerOptions(x: CrawlerOptions) {
allowExternalContentLinks: x.allowExternalLinks,
allowSubdomains: x.allowSubdomains,
ignoreRobotsTxt: x.ignoreRobotsTxt,
+ robotsUserAgent: x.robotsUserAgent,
ignoreSitemap: x.sitemap === "skip",
sitemapOnly: x.sitemap === "only",
deduplicateSimilarURLs: x.deduplicateSimilarURLs,
@@ -1363,6 +1365,7 @@ export function toV2CrawlerOptions(x: any): CrawlerOptions {
allowExternalLinks: x.allowExternalContentLinks,
allowSubdomains: x.allowSubdomains,
ignoreRobotsTxt: x.ignoreRobotsTxt,
+ robotsUserAgent: x.robotsUserAgent,
sitemap: x.sitemapOnly ? "only" : x.ignoreSitemap ? "skip" : "include",
deduplicateSimilarURLs: x.deduplicateSimilarURLs,
ignoreQueryParameters: x.ignoreQueryParameters,
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 6ba51b950..5699c6657 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -566,6 +566,7 @@ export function crawlToCrawler(
(getScrapeZDR(teamFlags) === "forced" || sc.zeroDataRetention) ?? false,
location: sc.scrapeOptions?.location,
headers: sc.scrapeOptions?.headers,
+ robotsUserAgent: sc.crawlerOptions?.robotsUserAgent,
});
if (sc.robots !== undefined) {
diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.ts
index 8bca74fb1..e64956dba 100644
--- a/apps/api/src/lib/permissions.ts
+++ b/apps/api/src/lib/permissions.ts
@@ -1,5 +1,5 @@
import { TeamFlags } from "../controllers/v2/types";
-import { getScrapeZDR } from "./zdr-helpers";
+import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";
type LocationOptions = { country?: string };
@@ -9,6 +9,10 @@ interface APIRequest {
scrapeOptions?: {
location?: LocationOptions;
};
+ crawlerOptions?: {
+ ignoreRobotsTxt?: boolean;
+ robotsUserAgent?: string;
+ };
}
const SUPPORT_EMAIL = "[email protected]";
@@ -19,12 +23,37 @@ export function checkPermissions(
): { error?: string } {
// zdr perms — scrapeZDR must be 'allowed' or 'forced' for request-scoped ZDR
const scrapeMode = getScrapeZDR(flags);
- if (request.zeroDataRetention && scrapeMode !== "allowed" && scrapeMode !== "forced") {
+ if (
+ request.zeroDataRetention &&
+ scrapeMode !== "allowed" &&
+ scrapeMode !== "forced"
+ ) {
return {
error: `Zero Data Retention (ZDR) is not enabled for your team. Contact ${SUPPORT_EMAIL} to enable this feature.`,
};
}
+ // robots perms — ignoreRobots must be 'allowed' or 'forced'
+ const robotsMode = getIgnoreRobots(flags);
+ if (
+ request.crawlerOptions?.ignoreRobotsTxt &&
+ robotsMode !== "allowed" &&
+ robotsMode !== "forced"
+ ) {
+ return {
+ error: `The ignoreRobotsTxt parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,
+ };
+ }
+ if (
+ request.crawlerOptions?.robotsUserAgent &&
+ robotsMode !== "allowed" &&
+ robotsMode !== "forced"
+ ) {
+ return {
+ error: `The robotsUserAgent parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,
+ };
+ }
+
// ip whitelist perms
const needsWhitelist =
request.location?.country === "us-whitelist" ||
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 9e5e1f6d2..1b27a8c0a 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -24,10 +24,14 @@ export async function fetchRobotsTxt(
url,
zeroDataRetention,
location,
+ headers,
+ skipCache,
}: {
url: string;
zeroDataRetention: boolean;
location?: ScrapeOptions["location"];
+ headers?: Record<string, string>;
+ skipCache?: boolean;
},
scrapeId: string,
logger: Logger,
@@ -39,7 +43,7 @@ export async function fetchRobotsTxt(
const shouldPrioritizeFireEngine = location && useFireEngine;
const forceEngine: Engine[] = [
- ...(useIndex ? ["index" as const] : []),
+ ...(useIndex && !skipCache ? ["index" as const] : []),
...(shouldPrioritizeFireEngine
? [
"fire-engine;tlsclient" as const,
@@ -68,8 +72,9 @@ export async function fetchRobotsTxt(
scrapeOptions.parse({
formats: ["rawHtml"],
timeout: 8000,
- maxAge: ROBOTS_MAX_AGE,
+ ...(skipCache ? { maxAge: 0 } : { maxAge: ROBOTS_MAX_AGE }),
...(location ? { location } : {}),
+ ...(headers ? { headers } : {}),
}),
{
forceEngine,
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index a0e4b2069..661f1eeaf 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -73,6 +73,7 @@ export class WebCrawler {
private zeroDataRetention: boolean;
private location?: ScrapeOptions["location"];
private headers?: Record<string, string>;
+ private robotsUserAgent?: string;
constructor({
jobId,
@@ -94,6 +95,7 @@ export class WebCrawler {
zeroDataRetention,
location,
headers,
+ robotsUserAgent,
}: {
jobId: string;
initialUrl: string;
@@ -114,6 +116,7 @@ export class WebCrawler {
zeroDataRetention?: boolean;
location?: ScrapeOptions["location"];
headers?: Record<string, string>;
+ robotsUserAgent?: string;
}) {
this.jobId = jobId;
this.initialUrl = initialUrl;
@@ -143,6 +146,7 @@ export class WebCrawler {
this.currentDiscoveryDepth = currentDiscoveryDepth ?? 0;
this.location = location;
this.headers = headers;
+ this.robotsUserAgent = robotsUserAgent;
}
public setBaseUrl(newBase: string): void {
@@ -398,12 +402,10 @@ export class WebCrawler {
}
}
- const isAllowed =
- this.ignoreRobotsTxt || skipRobots
- ? true
- : ((this.robots.isAllowed(link, "FireCrawlAgent") ||
- this.robots.isAllowed(link, "FirecrawlAgent")) ??
- true);
+ const isAllowed = this.isRobotsAllowed(
+ link,
+ this.ignoreRobotsTxt || skipRobots,
+ );
// Check if the link is disallowed by robots.txt
if (!isAllowed) {
this.logger.debug(`Link disallowed by robots.txt: ${link}`, {
@@ -458,6 +460,15 @@ export class WebCrawler {
url: this.robotsTxtUrl,
zeroDataRetention: this.zeroDataRetention,
location: this.location,
+ ...(this.robotsUserAgent
+ ? {
+ headers: {
+ ...this.headers,
+ "User-Agent": this.robotsUserAgent,
+ },
+ skipCache: true,
+ }
+ : {}),
},
this.jobId,
this.logger,
@@ -488,9 +499,10 @@ export class WebCrawler {
const checker = createRobotsChecker(this.initialUrl, txt);
this.robots = checker.robots;
this.robotsTxtUrl = checker.robotsTxtUrl;
- const delay =
- this.robots.getCrawlDelay("FireCrawlAgent") ||
- this.robots.getCrawlDelay("FirecrawlAgent");
+ const delay = this.robotsUserAgent
+ ? this.robots.getCrawlDelay(this.robotsUserAgent)
+ : this.robots.getCrawlDelay("FireCrawlAgent") ||
+ this.robots.getCrawlDelay("FirecrawlAgent");
this.robotsCrawlDelay = delay !== undefined ? delay : null;
const sitemaps = this.robots.getSitemaps();
@@ -743,7 +755,13 @@ export class WebCrawler {
url: string,
ignoreRobotsTxt: boolean = false,
): boolean {
- return ignoreRobotsTxt ? true : isUrlAllowedByRobots(url, this.robots);
+ return ignoreRobotsTxt
+ ? true
+ : isUrlAllowedByRobots(
+ url,
+ this.robots,
+ this.robotsUserAgent ? [this.robotsUserAgent] : undefined,
+ );
}
public isFile(url: string): boolean {