Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): migrate ignoreRobots org flag to disabled/allowed/forced pattern (#3328)
firecrawl-spring[bot]518e86e0260e
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex e336ffb6d..637a94b1e 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1281,7 +1281,7 @@ export type AuthCreditUsageChunk = {};export type TeamFlags = {- ignoreRobots?: boolean;+ ignoreRobots?: boolean | "disabled" | "allowed" | "forced";unblockedDomains?: string[];forceZDR?: boolean;allowZDR?: boolean;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 2860ff440..baa285b38 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1299,7 +1299,7 @@ type Account = {};export type TeamFlags = {- ignoreRobots?: boolean;+ ignoreRobots?: boolean | "disabled" | "allowed" | "forced";unblockedDomains?: string[];forceZDR?: boolean;allowZDR?: boolean;diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 0093b92c1..6ba51b950 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -6,7 +6,7 @@ import { logger as _logger } from "./logger";import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils";import type { Logger } from "winston";import { withSpan, setSpanAttributes } from "./otel-tracer";-import { getScrapeZDR } from "./zdr-helpers";+import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";export type StoredCrawl = {originUrl?: string;@@ -556,11 +556,14 @@ export function crawlToCrawler(sc.crawlerOptions?.allowExternalContentLinks ?? false,allowSubdomains: sc.crawlerOptions?.allowSubdomains ?? false,ignoreRobotsTxt:- teamFlags?.ignoreRobots ?? sc.crawlerOptions?.ignoreRobotsTxt ?? false,+ getIgnoreRobots(teamFlags) === "forced" ||+ (getIgnoreRobots(teamFlags) === "allowed" &&+ (sc.crawlerOptions?.ignoreRobotsTxt ?? false)),regexOnFullURL: sc.crawlerOptions?.regexOnFullURL ?? false,maxDiscoveryDepth: sc.crawlerOptions?.maxDiscoveryDepth,currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,- zeroDataRetention: (getScrapeZDR(teamFlags) === "forced" || sc.zeroDataRetention) ?? false,+ zeroDataRetention:+ (getScrapeZDR(teamFlags) === "forced" || sc.zeroDataRetention) ?? false,location: sc.scrapeOptions?.location,headers: sc.scrapeOptions?.headers,});diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.tsindex 1033a0266..1187609f7 100644--- a/apps/api/src/lib/zdr-helpers.ts+++ b/apps/api/src/lib/zdr-helpers.ts@@ -1,6 +1,8 @@import type { TeamFlags } from "../controllers/v2/types";-type ZDRMode = "disabled" | "allowed" | "forced";+type OrgFlagMode = "disabled" | "allowed" | "forced";++type ZDRMode = OrgFlagMode;/*** Resolves the effective ZDR mode for scrape endpoints from team flags.@@ -27,3 +29,16 @@ export function getSearchZDR(flags: TeamFlags | undefined): ZDRMode {if (flags?.searchZDR === "allowed" || flags?.allowZDR) return "allowed";return "disabled";}++/**+ * Resolves the effective ignoreRobots mode from team flags.+ *+ * Handles both the new enum format ("disabled"/"allowed"/"forced") and+ * the legacy boolean format for backward compatibility during migration.+ */+export function getIgnoreRobots(flags: TeamFlags | undefined): OrgFlagMode {+ if (flags?.ignoreRobots === "forced" || flags?.ignoreRobots === true)+ return "forced";+ if (flags?.ignoreRobots === "allowed") return "allowed";+ return "disabled";+}