Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): migrate ignoreRobots org flag to disabled/allowed/forced pattern (#3328)

firecrawl-spring[bot]518e86e0260e
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index e336ffb6d..637a94b1e 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1281,7 +1281,7 @@ export type AuthCreditUsageChunk = {
};
export type TeamFlags = {
- ignoreRobots?: boolean;
+ ignoreRobots?: boolean | "disabled" | "allowed" | "forced";
unblockedDomains?: string[];
forceZDR?: boolean;
allowZDR?: boolean;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 2860ff440..baa285b38 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1299,7 +1299,7 @@ type Account = {
};
export type TeamFlags = {
- ignoreRobots?: boolean;
+ ignoreRobots?: boolean | "disabled" | "allowed" | "forced";
unblockedDomains?: string[];
forceZDR?: boolean;
allowZDR?: boolean;
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 0093b92c1..6ba51b950 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -6,7 +6,7 @@ import { logger as _logger } from "./logger";
import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils";
import type { Logger } from "winston";
import { withSpan, setSpanAttributes } from "./otel-tracer";
-import { getScrapeZDR } from "./zdr-helpers";
+import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";
export type StoredCrawl = {
originUrl?: string;
@@ -556,11 +556,14 @@ export function crawlToCrawler(
sc.crawlerOptions?.allowExternalContentLinks ?? false,
allowSubdomains: sc.crawlerOptions?.allowSubdomains ?? false,
ignoreRobotsTxt:
- teamFlags?.ignoreRobots ?? sc.crawlerOptions?.ignoreRobotsTxt ?? false,
+ getIgnoreRobots(teamFlags) === "forced" ||
+ (getIgnoreRobots(teamFlags) === "allowed" &&
+ (sc.crawlerOptions?.ignoreRobotsTxt ?? false)),
regexOnFullURL: sc.crawlerOptions?.regexOnFullURL ?? false,
maxDiscoveryDepth: sc.crawlerOptions?.maxDiscoveryDepth,
currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,
- zeroDataRetention: (getScrapeZDR(teamFlags) === "forced" || sc.zeroDataRetention) ?? false,
+ zeroDataRetention:
+ (getScrapeZDR(teamFlags) === "forced" || sc.zeroDataRetention) ?? false,
location: sc.scrapeOptions?.location,
headers: sc.scrapeOptions?.headers,
});
diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.ts
index 1033a0266..1187609f7 100644
--- a/apps/api/src/lib/zdr-helpers.ts
+++ b/apps/api/src/lib/zdr-helpers.ts
@@ -1,6 +1,8 @@
import type { TeamFlags } from "../controllers/v2/types";
-type ZDRMode = "disabled" | "allowed" | "forced";
+type OrgFlagMode = "disabled" | "allowed" | "forced";
+
+type ZDRMode = OrgFlagMode;
/**
* Resolves the effective ZDR mode for scrape endpoints from team flags.
@@ -27,3 +29,16 @@ export function getSearchZDR(flags: TeamFlags | undefined): ZDRMode {
if (flags?.searchZDR === "allowed" || flags?.allowZDR) return "allowed";
return "disabled";
}
+
+/**
+ * Resolves the effective ignoreRobots mode from team flags.
+ *
+ * Handles both the new enum format ("disabled"/"allowed"/"forced") and
+ * the legacy boolean format for backward compatibility during migration.
+ */
+export function getIgnoreRobots(flags: TeamFlags | undefined): OrgFlagMode {
+ if (flags?.ignoreRobots === "forced" || flags?.ignoreRobots === true)
+ return "forced";
+ if (flags?.ignoreRobots === "allowed") return "allowed";
+ return "disabled";
+}