Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): make agent hints opt-in
Rakshith Ramprakash0769644a0d3d
diff --git a/apps/api/AGENT_HINTS.md b/apps/api/AGENT_HINTS.mdindex 9059d5024..a55f26bb8 100644--- a/apps/api/AGENT_HINTS.md+++ b/apps/api/AGENT_HINTS.md@@ -2,10 +2,10 @@The v2 `POST /search`, `/scrape`, `/parse`, and `/map` routes may return an optional top-level `agent_hints: string[]`. Each response contains at most two hints: one cross-endpoint suggestion and a feedback request when the receiver supports the completed operation. Existing data, errors, warning fields, and HTTP statuses remain unchanged.-Clients can suppress all hints for one request with:+Hints are disabled by default. Trusted agent adapters such as the Firecrawl MCP server and CLI can enable them for one request with:```http-X-Firecrawl-Agent-Hints: false+X-Firecrawl-Agent-Hints: true```The header applies to the business request. No request-body schema changes are required. SDKs and other adapters should retain the top-level field when unwrapping `data` and should preserve it on error results. The strings recommend conditional next steps; receiving one does not execute another request or indicate user authorization.@@ -14,7 +14,7 @@ The header applies to the business request. No request-body schema changes are rThe accompanying JavaScript and Python SDK changes preserve `agent_hints` alongside normal flattened Document, SearchData, and MapData results and on typed errors. Python covers synchronous and asynchronous clients. Markdown and HTML retain their existing content.-Raw HTTP callers can opt out with the header above. This change does not add a client-wide SDK hint option; applications can omit the metadata when passing results to a model. The SDK release must precede upgrading pinned CLI/MCP dependencies for their keyed SDK paths to retain the field.+Raw HTTP and ordinary SDK callers receive no hints unless their adapter explicitly sets the header above. This change does not add a client-wide SDK hint option. The SDK release must precede upgrading pinned CLI/MCP dependencies for their keyed SDK paths to retain the field.## Initial rulesdiff --git a/apps/api/src/__tests__/snips/v2/agent-hints.test.ts b/apps/api/src/__tests__/snips/v2/agent-hints.test.tsindex ec71ef0a0..a6c810cab 100644--- a/apps/api/src/__tests__/snips/v2/agent-hints.test.ts+++ b/apps/api/src/__tests__/snips/v2/agent-hints.test.ts@@ -19,6 +19,7 @@ describeIf(TEST_PRODUCTION)("Agent hints", () => {const response = await request(TEST_API_URL).post("/v2/scrape").set("Authorization", `Bearer ${identity.apiKey}`)+ .set("X-Firecrawl-Agent-Hints", "true").send({ url: TEST_SUITE_WEBSITE, timeout: scrapeTimeout });expect(response.statusCode).toBe(200);expect(response.body.success).toBe(true);@@ -37,12 +38,11 @@ describeIf(TEST_PRODUCTION)("Agent hints", () => {);it(- "honors the opt-out header on a completed map",+ "leaves hints off by default on a completed map",async () => {const response = await request(TEST_API_URL).post("/v2/map").set("Authorization", `Bearer ${identity.apiKey}`)- .set("X-Firecrawl-Agent-Hints", "false").send({ url: TEST_SUITE_WEBSITE, limit: 1, timeout: scrapeTimeout });expect(response.statusCode).toBe(200);expect(response.body).not.toHaveProperty("agent_hints");@@ -54,6 +54,7 @@ describeIf(TEST_PRODUCTION)("Agent hints", () => {const response = await request(TEST_API_URL).post("/v2/scrape").set("Authorization", `Bearer ${identity.apiKey}`)+ .set("X-Firecrawl-Agent-Hints", "true").send({ url: "not-a-url" });expect(response.statusCode).toBe(400);expect(response.body.success).toBe(false);diff --git a/apps/api/src/middlewares/__tests__/agent-hints.test.ts b/apps/api/src/middlewares/__tests__/agent-hints.test.tsindex c6176f618..a45e66a02 100644--- a/apps/api/src/middlewares/__tests__/agent-hints.test.ts+++ b/apps/api/src/middlewares/__tests__/agent-hints.test.ts@@ -30,15 +30,25 @@ function appFor(}describe("agent hint response middleware", () => {- it("adds top-level metadata without changing data or warning", async () => {+ it("leaves the original envelope unchanged by default", async () => {const body = { success: true, data: { web: [] }, warning: "existing" };const response = await request(appFor({ body })).post("/").send({});expect(response.statusCode).toBe(200);+ expect(response.body).toEqual(body);+ });++ it("adds top-level metadata only when explicitly enabled", async () => {+ const body = { success: true, data: { web: [] }, warning: "existing" };+ const response = await request(appFor({ body }))+ .post("/")+ .set("X-Firecrawl-Agent-Hints", " TRUE ")+ .send({});+ expect(response.statusCode).toBe(200);expect(response.body).toMatchObject(body);expect(response.body.agent_hints).toHaveLength(1);});- it("opt-out header preserves the original envelope", async () => {+ it("values other than true preserve the original envelope", async () => {const body = {success: true,data: { web: [{ url: "https://example.com" }] },@@ -56,14 +66,20 @@ describe("agent hint response middleware", () => {{ zdr: true },{ feedback: false },])("does not invent accepted feedback for %j", async settings => {- const response = await request(appFor(settings)).post("/").send({});+ const response = await request(appFor(settings))+ .post("/")+ .set("X-Firecrawl-Agent-Hints", "true")+ .send({});expect(response.body).not.toHaveProperty("agent_hints");});it("omits feedback when database authentication is unavailable", async () => {config.USE_DB_AUTHENTICATION = false;try {- const response = await request(appFor()).post("/").send({});+ const response = await request(appFor())+ .post("/")+ .set("X-Firecrawl-Agent-Hints", "true")+ .send({});expect(response.body).not.toHaveProperty("agent_hints");} finally {config.USE_DB_AUTHENTICATION = true;@@ -79,6 +95,7 @@ describe("agent hint response middleware", () => {};const response = await request(appFor({ body, status: 400 })).post("/")+ .set("X-Firecrawl-Agent-Hints", "true").send({});expect(response.statusCode).toBe(400);expect(response.body).toEqual(body);diff --git a/apps/api/src/middlewares/agent-hints.ts b/apps/api/src/middlewares/agent-hints.tsindex 692e6d67d..f43898d8d 100644--- a/apps/api/src/middlewares/agent-hints.ts+++ b/apps/api/src/middlewares/agent-hints.ts@@ -18,7 +18,7 @@ export function agentHintsMiddleware(endpoint: AgentHintEndpoint,): RequestHandler {return (req, res, next) => {- if (req.get("X-Firecrawl-Agent-Hints")?.trim().toLowerCase() === "false")+ if (req.get("X-Firecrawl-Agent-Hints")?.trim().toLowerCase() !== "true")return next();const json = res.json;res.json = function (body) {