Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add customRobotsAgent flag separate from ignoreRobots (#3377)

firecrawl-spring[bot]b5f8ac3e5121
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 287b28e9e..88a8228a8 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -534,7 +534,7 @@ describe("Crawl tests", () => {
const robotsIdentity = await idmux({
name: "crawl/robotsUserAgent",
credits: 10000,
- flags: { ignoreRobots: "allowed" },
+ flags: { customRobotsAgent: "allowed" },
});
const results = await crawl(
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index cf645b702..792bf3d10 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1282,6 +1282,7 @@ export type AuthCreditUsageChunk = {
export type TeamFlags = {
ignoreRobots?: "disabled" | "allowed" | "forced";
+ customRobotsAgent?: "disabled" | "allowed";
unblockedDomains?: string[];
forceZDR?: boolean;
allowZDR?: boolean;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index caf8bfe23..a2bc2cf1c 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1301,6 +1301,7 @@ type Account = {
export type TeamFlags = {
ignoreRobots?: "disabled" | "allowed" | "forced";
+ customRobotsAgent?: "disabled" | "allowed";
unblockedDomains?: string[];
forceZDR?: boolean;
allowZDR?: boolean;
diff --git a/apps/api/src/lib/permissions.ts b/apps/api/src/lib/permissions.ts
index e64956dba..4f7478072 100644
--- a/apps/api/src/lib/permissions.ts
+++ b/apps/api/src/lib/permissions.ts
@@ -1,5 +1,9 @@
import { TeamFlags } from "../controllers/v2/types";
-import { getScrapeZDR, getIgnoreRobots } from "./zdr-helpers";
+import {
+ getScrapeZDR,
+ getIgnoreRobots,
+ getCustomRobotsAgent,
+} from "./zdr-helpers";
type LocationOptions = { country?: string };
@@ -44,10 +48,11 @@ export function checkPermissions(
error: `The ignoreRobotsTxt parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,
};
}
+ // customRobotsAgent perms — separate flag for robotsUserAgent
+ const customAgentMode = getCustomRobotsAgent(flags);
if (
request.crawlerOptions?.robotsUserAgent &&
- robotsMode !== "allowed" &&
- robotsMode !== "forced"
+ customAgentMode !== "allowed"
) {
return {
error: `The robotsUserAgent parameter is an enterprise feature. Contact ${SUPPORT_EMAIL} to explore whether it can be enabled for your team.`,
diff --git a/apps/api/src/lib/zdr-helpers.ts b/apps/api/src/lib/zdr-helpers.ts
index 9636cd616..abd42ee4f 100644
--- a/apps/api/src/lib/zdr-helpers.ts
+++ b/apps/api/src/lib/zdr-helpers.ts
@@ -38,3 +38,14 @@ export function getIgnoreRobots(flags: TeamFlags | undefined): OrgFlagMode {
if (flags?.ignoreRobots === "allowed") return "allowed";
return "disabled";
}
+
+/**
+ * Resolves the effective customRobotsAgent mode from team flags.
+ * Only supports "disabled" (default) and "allowed" — no "forced" mode.
+ */
+export function getCustomRobotsAgent(
+ flags: TeamFlags | undefined,
+): "disabled" | "allowed" {
+ if (flags?.customRobotsAgent === "allowed") return "allowed";
+ return "disabled";
+}