Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawl): add robotsUserAgent parameter for custom robots.txt checks (#3325)
firecrawl-spring[bot]ef10cd3846a0
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 2d4e439da..15de2f77e 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -529,6 +529,31 @@ describe("Crawl tests", () => {10 * scrapeTimeout,);+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "accepts robotsUserAgent parameter",+ async () => {+ const robotsIdentity = await idmux({+ name: "crawl/robotsUserAgent",+ credits: 10000,+ flags: { ignoreRobots: "allowed" },+ });++ const results = await crawl(+ {+ url: base,+ limit: 3,+ robotsUserAgent: "MyCustomBot",+ },+ robotsIdentity,+ );++ expect(results.success).toBe(true);+ expect(results.status).toBe("completed");+ expect(results.completed).toBeGreaterThan(0);+ },+ 10 * scrapeTimeout,+ );+concurrentIf(TEST_PRODUCTION || HAS_PROXY)("shows warning when crawl results ≤ 1 and URL is not base domain",async () => {diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 67bf7eb10..a2aea48b2 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -29,7 +29,10 @@ export async function crawlController(const preNormalizedBody = req.body;req.body = crawlRequestSchema.parse(req.body);- const permissions = checkPermissions(req.body, req.acuc?.flags);+ const permissions = checkPermissions(+ { ...req.body, crawlerOptions: req.body },+ req.acuc?.flags,+ );if (permissions.error) {return res.status(403).json({success: false,diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 4f59e94c3..91de703ed 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -31,7 +31,10 @@ export async function crawlController(const preNormalizedBody = req.body;req.body = crawlRequestSchema.parse(req.body);- const permissions = checkPermissions(req.body, req.acuc?.flags);+ const permissions = checkPermissions(+ { ...req.body, crawlerOptions: req.body },+ req.acuc?.flags,+ );if (permissions.error) {return res.status(403).json({success: false,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex d9fbd46bb..caf8bfe23 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -910,6 +910,7 @@ export const crawlerOptions = z.strictObject({allowExternalLinks: z.boolean().prefault(false),allowSubdomains: z.boolean().prefault(false),ignoreRobotsTxt: z.boolean().prefault(false),+ robotsUserAgent: z.string().optional(),sitemap: z.enum(["skip", "include", "only"]).prefault("include"),deduplicateSimilarURLs: z.boolean().prefault(true),ignoreQueryParameters: z.boolean().prefault(false),@@ -1343,6 +1344,7 @@ export function toV0CrawlerOptions(x: CrawlerOptions) {allowExternalContentLinks: x.allowExternalLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,+ robotsUserAgent: x.robotsUserAgent,ignoreSitemap: x.sitemap === "skip",sitemapOnly: x.sitemap === "only",deduplicateSimilarURLs: x.deduplicateSimilarURLs,@@ -1363,6 +1365,7 @@ export function toV2CrawlerOptions(x: any): CrawlerOptions {allowExternalLinks: x.allowExternalContentLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,+ robotsUserAgent: x.robotsUserAgent,sitemap: x.sitemapOnly ? "only" : x.ignoreSitemap ? "skip" : "include",deduplicateSimilarURLs: x.deduplicateSimilarURLs,ignoreQueryParameters: x.ignoreQueryParameters,diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 6ba51b950..5699c6657 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -566,6 +566,7 @@ export function crawlToCrawler((getScrapeZDR(teamFlags) === "forced" || sc.zeroDataRetention) ?? false,location: sc.scrapeOptions?.location,headers: sc.scrapeOptions?.headers,+ robotsUserAgent: sc.crawlerOptions?.robotsUserAgent,});if (sc.robots !== undefined) {diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.tsindex 8bca74fb1..e64956dba 100644--- a/apps/api/src/lib/permissions.ts+++ b/apps/api/src/lib/permissions.ts@@ -1,5 +1,5 @@import { TeamFlags } from "../controllers/v2/types";-import { getScrapeZDR } from "./zdr-helpers";+import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";type LocationOptions = { country?: string };@@ -9,6 +9,10 @@ interface APIRequest {scrapeOptions?: {location?: LocationOptions;};+ crawlerOptions?: {+ ignoreRobotsTxt?: boolean;+ robotsUserAgent?: string;+ };}const SUPPORT_EMAIL = "[email protected]";@@ -19,12 +23,37 @@ export function checkPermissions(): { error?: string } {// zdr perms — scrapeZDR must be 'allowed' or 'forced' for request-scoped ZDRconst scrapeMode = getScrapeZDR(flags);- if (request.zeroDataRetention && scrapeMode !== "allowed" && scrapeMode !== "forced") {+ if (+ request.zeroDataRetention &&+ scrapeMode !== "allowed" &&+ scrapeMode !== "forced"+ ) {return {error: `Zero Data Retention (ZDR) is not enabled for your team. Contact ${SUPPORT_EMAIL} to enable this feature.`,};}+ // robots perms — ignoreRobots must be 'allowed' or 'forced'+ const robotsMode = getIgnoreRobots(flags);+ if (+ request.crawlerOptions?.ignoreRobotsTxt &&+ robotsMode !== "allowed" &&+ robotsMode !== "forced"+ ) {+ return {+ error: `The ignoreRobotsTxt parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,+ };+ }+ if (+ request.crawlerOptions?.robotsUserAgent &&+ robotsMode !== "allowed" &&+ robotsMode !== "forced"+ ) {+ return {+ error: `The robotsUserAgent parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,+ };+ }+// ip whitelist permsconst needsWhitelist =request.location?.country === "us-whitelist" ||diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 9e5e1f6d2..1b27a8c0a 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -24,10 +24,14 @@ export async function fetchRobotsTxt(url,zeroDataRetention,location,+ headers,+ skipCache,}: {url: string;zeroDataRetention: boolean;location?: ScrapeOptions["location"];+ headers?: Record<string, string>;+ skipCache?: boolean;},scrapeId: string,logger: Logger,@@ -39,7 +43,7 @@ export async function fetchRobotsTxt(const shouldPrioritizeFireEngine = location && useFireEngine;const forceEngine: Engine[] = [- ...(useIndex ? ["index" as const] : []),+ ...(useIndex && !skipCache ? ["index" as const] : []),...(shouldPrioritizeFireEngine? ["fire-engine;tlsclient" as const,@@ -68,8 +72,9 @@ export async function fetchRobotsTxt(scrapeOptions.parse({formats: ["rawHtml"],timeout: 8000,- maxAge: ROBOTS_MAX_AGE,+ ...(skipCache ? { maxAge: 0 } : { maxAge: ROBOTS_MAX_AGE }),...(location ? { location } : {}),+ ...(headers ? { headers } : {}),}),{forceEngine,diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex a0e4b2069..661f1eeaf 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -73,6 +73,7 @@ export class WebCrawler {private zeroDataRetention: boolean;private location?: ScrapeOptions["location"];private headers?: Record<string, string>;+ private robotsUserAgent?: string;constructor({jobId,@@ -94,6 +95,7 @@ export class WebCrawler {zeroDataRetention,location,headers,+ robotsUserAgent,}: {jobId: string;initialUrl: string;@@ -114,6 +116,7 @@ export class WebCrawler {zeroDataRetention?: boolean;location?: ScrapeOptions["location"];headers?: Record<string, string>;+ robotsUserAgent?: string;}) {this.jobId = jobId;this.initialUrl = initialUrl;@@ -143,6 +146,7 @@ export class WebCrawler {this.currentDiscoveryDepth = currentDiscoveryDepth ?? 0;this.location = location;this.headers = headers;+ this.robotsUserAgent = robotsUserAgent;}public setBaseUrl(newBase: string): void {@@ -398,12 +402,10 @@ export class WebCrawler {}}- const isAllowed =- this.ignoreRobotsTxt || skipRobots- ? true- : ((this.robots.isAllowed(link, "FireCrawlAgent") ||- this.robots.isAllowed(link, "FirecrawlAgent")) ??- true);+ const isAllowed = this.isRobotsAllowed(+ link,+ this.ignoreRobotsTxt || skipRobots,+ );// Check if the link is disallowed by robots.txtif (!isAllowed) {this.logger.debug(`Link disallowed by robots.txt: ${link}`, {@@ -458,6 +460,15 @@ export class WebCrawler {url: this.robotsTxtUrl,zeroDataRetention: this.zeroDataRetention,location: this.location,+ ...(this.robotsUserAgent+ ? {+ headers: {+ ...this.headers,+ "User-Agent": this.robotsUserAgent,+ },+ skipCache: true,+ }+ : {}),},this.jobId,this.logger,@@ -488,9 +499,10 @@ export class WebCrawler {const checker = createRobotsChecker(this.initialUrl, txt);this.robots = checker.robots;this.robotsTxtUrl = checker.robotsTxtUrl;- const delay =- this.robots.getCrawlDelay("FireCrawlAgent") ||- this.robots.getCrawlDelay("FirecrawlAgent");+ const delay = this.robotsUserAgent+ ? this.robots.getCrawlDelay(this.robotsUserAgent)+ : this.robots.getCrawlDelay("FireCrawlAgent") ||+ this.robots.getCrawlDelay("FirecrawlAgent");this.robotsCrawlDelay = delay !== undefined ? delay : null;const sitemaps = this.robots.getSitemaps();@@ -743,7 +755,13 @@ export class WebCrawler {url: string,ignoreRobotsTxt: boolean = false,): boolean {- return ignoreRobotsTxt ? true : isUrlAllowedByRobots(url, this.robots);+ return ignoreRobotsTxt+ ? true+ : isUrlAllowedByRobots(+ url,+ this.robots,+ this.robotsUserAgent ? [this.robotsUserAgent] : undefined,+ );}public isFile(url: string): boolean {