Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add customRobotsAgent flag separate from ignoreRobots (#3377)
firecrawl-spring[bot]b5f8ac3e5121
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 287b28e9e..88a8228a8 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -534,7 +534,7 @@ describe("Crawl tests", () => {const robotsIdentity = await idmux({name: "crawl/robotsUserAgent",credits: 10000,- flags: { ignoreRobots: "allowed" },+ flags: { customRobotsAgent: "allowed" },});const results = await crawl(diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex cf645b702..792bf3d10 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1282,6 +1282,7 @@ export type AuthCreditUsageChunk = {export type TeamFlags = {ignoreRobots?: "disabled" | "allowed" | "forced";+ customRobotsAgent?: "disabled" | "allowed";unblockedDomains?: string[];forceZDR?: boolean;allowZDR?: boolean;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex caf8bfe23..a2bc2cf1c 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1301,6 +1301,7 @@ type Account = {export type TeamFlags = {ignoreRobots?: "disabled" | "allowed" | "forced";+ customRobotsAgent?: "disabled" | "allowed";unblockedDomains?: string[];forceZDR?: boolean;allowZDR?: boolean;diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.tsindex e64956dba..4f7478072 100644--- a/apps/api/src/lib/permissions.ts+++ b/apps/api/src/lib/permissions.ts@@ -1,5 +1,9 @@import { TeamFlags } from "../controllers/v2/types";-import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";+import {+ getScrapeZDR,+ getIgnoreRobots,+ getCustomRobotsAgent,+} from "./zdr-helpers";type LocationOptions = { country?: string };@@ -44,10 +48,11 @@ export function checkPermissions(error: `The ignoreRobotsTxt parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,};}+ // customRobotsAgent perms — separate flag for robotsUserAgent+ const customAgentMode = getCustomRobotsAgent(flags);if (request.crawlerOptions?.robotsUserAgent &&- robotsMode !== "allowed" &&- robotsMode !== "forced"+ customAgentMode !== "allowed") {return {error: `The robotsUserAgent parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.tsindex 9636cd616..abd42ee4f 100644--- a/apps/api/src/lib/zdr-helpers.ts+++ b/apps/api/src/lib/zdr-helpers.ts@@ -38,3 +38,14 @@ export function getIgnoreRobots(flags: TeamFlags | undefined): OrgFlagMode {if (flags?.ignoreRobots === "allowed") return "allowed";return "disabled";}++/**+ * Resolves the effective customRobotsAgent mode from team flags.+ * Only supports "disabled" (default) and "allowed" — no "forced" mode.+ */+export function getCustomRobotsAgent(+ flags: TeamFlags | undefined,+): "disabled" | "allowed" {+ if (flags?.customRobotsAgent === "allowed") return "allowed";+ return "disabled";+}