Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(agent): add safe search option to /v2/agent (#5456)
Rakshith Ramprakash85e494e48406
diff --git a/apps/api/openapi.json b/apps/api/openapi.jsonindex 6da32eb5c..352d0d1ab 100644--- a/apps/api/openapi.json+++ b/apps/api/openapi.json@@ -2789,6 +2789,10 @@"type": "string","enum": ["low", "medium", "high"],"description": "Reasoning budget used for the agent run (only present for runs that set effort)"+ },+ "safe": {+ "type": "boolean",+ "description": "Whether the run's web searches filtered explicit content (only present for runs that set `safe: true`)"}}}@@ -2951,6 +2955,10 @@}}},+ "safe": {+ "type": "boolean",+ "description": "When `true`, every web search the agent makes during the run filters explicit content, the same filter as the search endpoint's `safe` option. Pages the agent scrapes or opens in a browser are not filtered. Omit to keep the default behavior, which does not apply the filter. Omitted on a follow-up turn keeps the thread's setting; `false` turns it off."+ },"exchange": {"type": "object","description": "Let the agent call your team's [Alexandria](https://docs.firecrawl.dev/features/alexandria) data providers during the run. Without this object the run uses the web only (a follow-up turn inherits the previous turn's settings). See [Use your connected Alexandria tools](https://docs.firecrawl.dev/features/agent#use-your-connected-alexandria-tools).",@@ -3213,6 +3221,10 @@"enum": ["low", "medium", "high"],"description": "Reasoning budget used for the agent run (only present for runs that set effort)"},+ "safe": {+ "type": "boolean",+ "description": "Whether the run's web searches filtered explicit content (only present for runs that set `safe: true`)"+ },"error": {"type": "string","description": "Error message (only present when status is failed)"diff --git a/apps/api/src/controllers/v2/__tests__/agent-list.test.ts b/apps/api/src/controllers/v2/__tests__/agent-list.test.tsindex 9e44ca779..d61a0fc56 100644--- a/apps/api/src/controllers/v2/__tests__/agent-list.test.ts+++ b/apps/api/src/controllers/v2/__tests__/agent-list.test.ts@@ -107,6 +107,16 @@ describe("agentListController options", () => {});});+ it("lists safe only for a run that set it", async () => {+ mockAgentRow(JSON.stringify({ prompt: "p", model: "spark-2", safe: true }));+ expect(await listOptions()).toMatchObject({ safe: true });++ mockAgentRow(+ JSON.stringify({ prompt: "p", model: "spark-2", safe: false }),+ );+ expect(await listOptions()).not.toHaveProperty("safe");+ });+it("reads a quoted threadTurn as a number", async () => {mockAgentRow(JSON.stringify({ prompt: "p", threadId: "thread-1", threadTurn: "3" }),@@ -199,6 +209,29 @@ describe("agentListController zero data retention rows", () => {});});+ it("keeps safe on a ZDR row", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue({ ok: true, json: async () => [] }),+ );+ mockAgentRow(+ JSON.stringify({+ prompt: "secret prompt",+ model: "spark-2",+ safe: true,+ zeroDataRetention: true,+ }),+ );++ expect(await listOptions()).toEqual({+ prompt: "",+ model: "spark-2",+ effort: "medium",+ safe: true,+ zeroDataRetention: true,+ });+ });+it("drops content from a recent ZDR run not yet in ClickHouse", async () => {vi.stubGlobal("fetch",diff --git a/apps/api/src/controllers/v2/__tests__/agent-safe-search.test.ts b/apps/api/src/controllers/v2/__tests__/agent-safe-search.test.tsnew file mode 100644index 000000000..b57d38582--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/agent-safe-search.test.ts@@ -0,0 +1,103 @@+vi.mock("../../../lib/crawl-regex", async () => {+ const { z } = await import("zod");+ return {+ addPathRegexIssues: vi.fn(),+ pathPatternsSchema: z.array(z.string()),+ };+});+import { agentController } from "../agent";+import { agentRequestSchema } from "../types";+vi.mock("../../../lib/logger", () => {+ const logger = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ child: vi.fn(),+ };+ logger.child.mockReturnValue(logger);+ return { logger };+});+vi.mock("../../../services/logging/log_job", () => ({ logRequest: vi.fn() }));+vi.mock("../../../lib/external-request-id", () => ({+ externalRequestId: () => undefined,+}));+vi.mock("../../../config", () => ({+ config: {+ EXTRACT_V3_BETA_URL: "https://agent.example",+ USE_DB_AUTHENTICATION: false,+ AGENT_INTEROP_SECRET: "test",+ },+}));+vi.mock("../../../db/rpc", () => ({ agentConsumeFreeRequestIfLeft: vi.fn() }));+vi.mock("../../../lib/threat-protection/request", () => ({+ resolveThreatProtection: vi.fn().mockResolvedValue({}),+ checkUrlsAgainstThreatPolicy: vi.fn(),+}));+vi.mock("../../../lib/scrape-billing", () => ({+ calculateThreatScanCredits: vi.fn(),+}));+vi.mock("../../../services/billing/credit_billing", () => ({+ billTeam: vi.fn(),+}));+vi.mock("../../../lib/siem-logging", () => ({+ emitRejectedScrapeActivityEvents: vi.fn(),+}));+vi.mock("../agent-thread", () => ({+ fetchAgentThread: vi.fn(),+ threadErrorFor: vi.fn(),+}));++describe("agentRequestSchema safe", () => {+ const base = { prompt: "Find the pricing page" };++ it.each([true, false])("accepts safe: %s", safe => {+ expect(agentRequestSchema.parse({ ...base, safe }).safe).toBe(safe);+ });++ it("leaves safe undefined when omitted", () => {+ expect(agentRequestSchema.parse(base).safe).toBeUndefined();+ });++ it.each(["true", 1, "active", null])("rejects safe: %j", safe => {+ expect(() => agentRequestSchema.parse({ ...base, safe })).toThrow();+ });+});++describe("agentController safe passthrough", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue({ status: 200, json: async () => ({}) }),+ );+ });+ afterEach(() => vi.unstubAllGlobals());++ const run = async (body: Record<string, unknown>) => {+ const req = {+ body,+ auth: { team_id: "team-test" },+ acuc: { flags: {}, api_key: "test-key" },+ };+ const res = { status: vi.fn().mockReturnThis(), json: vi.fn() };+ await agentController(req as any, res as any);+ expect(res.status).toHaveBeenCalledWith(200);+ return JSON.parse(vi.mocked(fetch).mock.calls[0][1]!.body as string);+ };++ it.each([true, false])(+ "forwards safe: %s to the agent service",+ async safe => {+ const body = await run({ prompt: "Find the pricing page", safe });++ expect(body.safe).toBe(safe);+ },+ );++ it("sends no safe key when the caller omits it", async () => {+ const body = await run({ prompt: "Find the pricing page" });++ expect(body).not.toHaveProperty("safe");+ });+});diff --git a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts b/apps/api/src/controllers/v2/__tests__/agent-status.test.tsindex 68214d009..b1c60ca09 100644--- a/apps/api/src/controllers/v2/__tests__/agent-status.test.ts+++ b/apps/api/src/controllers/v2/__tests__/agent-status.test.ts@@ -119,6 +119,47 @@ describe("agentStatusController", () => {expect(body.effort).toBeUndefined();});+ it("returns safe for a run that filtered its searches", async () => {+ (getAgentJobAccess as Mock).mockResolvedValue({+ teamId: "team-123",+ expiresAtMs: Date.now() + 60_000,+ });+ (getExtractV3AgentStatus as Mock).mockResolvedValue({+ id: "job-123",+ success: true,+ status: "processing",+ model: "spark-2",+ safe: true,+ });++ const res = buildRes();+ await agentStatusController(baseReq, res);++ expect(res.status).toHaveBeenCalledWith(200);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({ status: "processing", safe: true }),+ );+ });++ it("leaves safe undefined when the run did not set it", async () => {+ (getAgentJobAccess as Mock).mockResolvedValue({+ teamId: "team-123",+ expiresAtMs: Date.now() + 60_000,+ });+ (getExtractV3AgentStatus as Mock).mockResolvedValue({+ id: "job-123",+ success: true,+ status: "success",+ model: "spark-2",+ data: { result: "ok" },+ });++ const res = buildRes();+ await agentStatusController(baseReq, res);++ expect((res.json as Mock).mock.calls[0][0].safe).toBeUndefined();+ });+it("returns terminal state and result metadata from extract-v3", async () => {(getAgentJobAccess as Mock).mockResolvedValue({teamId: "team-123",diff --git a/apps/api/src/controllers/v2/agent-list.ts b/apps/api/src/controllers/v2/agent-list.tsindex fa08e332e..f01dc1be6 100644--- a/apps/api/src/controllers/v2/agent-list.ts+++ b/apps/api/src/controllers/v2/agent-list.ts@@ -60,11 +60,16 @@ function withoutZdrContent(options: any): any {model: options.model,modelPreset: options.modelPreset,effort: options.effort,+ safe: options.safe,zeroDataRetention: true,}: options;}+function safeOption(options: any): { safe?: true } {+ return options?.safe === true ? { safe: true } : {};+}+function zdrOptions(options: any): { zeroDataRetention?: true } {return isZdrOptions(options) ? { zeroDataRetention: true } : {};}@@ -340,6 +345,7 @@ export async function agentListController(effort:dbOptions.effort ??(dbOptions.model === "spark-2" ? "medium" : undefined),+ ...safeOption(dbOptions),...threadOptions(dbOptions),...zdrOptions(dbOptions),}@@ -354,6 +360,7 @@ export async function agentListController((recentOptions.modelPreset === "spark-2"? "medium": undefined),+ ...safeOption(recentOptions),...threadOptions(recentOptions),...zdrOptions(recentOptions),}diff --git a/apps/api/src/controllers/v2/agent-status.ts b/apps/api/src/controllers/v2/agent-status.tsindex 77fab6716..58494d655 100644--- a/apps/api/src/controllers/v2/agent-status.ts+++ b/apps/api/src/controllers/v2/agent-status.ts@@ -133,6 +133,7 @@ export async function agentStatusController(: {}),model,effort,+ safe: agent.safe,threadId: thread.threadId,threadTurn: thread.threadTurn,mode: thread.mode,diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex 3515d4b58..0b7a59498 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -316,6 +316,7 @@ export async function agentController(mode: req.body.mode,exchange: req.body.exchange,location: req.body.location,+ safe: req.body.safe,...(zeroDataRetention ? { zeroDataRetention: true } : {}),...(zeroDataRetentionMinutes !== null? { zeroDataRetentionMinutes }diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 7f969acb9..dc8d05c39 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1212,6 +1212,7 @@ export const agentRequestSchema = zmode: z.enum(["extract", "chat"]).optional(),exchange: agentExchangeSchema.optional(),location: countryLocationSchema.optional(),+ safe: z.boolean().optional(),zeroDataRetention: z.boolean().optional(),})// spark-1 is retired and spark-2 is the default. The spark-1 preset names@@ -1731,6 +1732,7 @@ export type AgentListResponse =// type release.model: "spark-1-pro" | "spark-1-mini" | "spark-2" | (string & {});effort?: "low" | "medium" | "high";+ safe?: boolean;threadId?: string;threadTurn?: number;zeroDataRetention?: boolean;@@ -1891,6 +1893,7 @@ export type AgentStatusResponse =stopReason?: "credit_limit_reached";model?: "spark-1-pro" | "spark-1-mini" | "spark-2";effort?: "low" | "medium" | "high";+ safe?: boolean;expiresAt: string;creditsUsed?: number;threadId?: string;@@ -1910,6 +1913,7 @@ type AgentThreadRun = {urls?: string[];schema?: unknown;effort?: "low" | "medium" | "high";+ safe?: boolean;status:| "processing"| "succeeded"diff --git a/apps/api/src/lib/extract-v3-status.ts b/apps/api/src/lib/extract-v3-status.tsindex b0cb9e79d..9a892c265 100644--- a/apps/api/src/lib/extract-v3-status.ts+++ b/apps/api/src/lib/extract-v3-status.ts@@ -13,6 +13,7 @@ type ExtractV3AgentStatus = {error?: string;model?: "spark-1-pro" | "spark-1-mini" | "spark-2";effort?: "low" | "medium" | "high";+ safe?: boolean;threadId?: string;threadTurn?: number;mode?: "extract" | "chat";