Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): make agent hints opt-in

Rakshith Ramprakash0769644a0d3d
diff --git a/apps/api/AGENT_HINTS.md b/apps/api/AGENT_HINTS.md
index 9059d5024..a55f26bb8 100644
--- a/apps/api/AGENT_HINTS.md
+++ b/apps/api/AGENT_HINTS.md
@@ -2,10 +2,10 @@
The v2 `POST /search`, `/scrape`, `/parse`, and `/map` routes may return an optional top-level `agent_hints: string[]`. Each response contains at most two hints: one cross-endpoint suggestion and a feedback request when the receiver supports the completed operation. Existing data, errors, warning fields, and HTTP statuses remain unchanged.
-Clients can suppress all hints for one request with:
+Hints are disabled by default. Trusted agent adapters such as the Firecrawl MCP server and CLI can enable them for one request with:
```http
-X-Firecrawl-Agent-Hints: false
+X-Firecrawl-Agent-Hints: true
```
The header applies to the business request. No request-body schema changes are required. SDKs and other adapters should retain the top-level field when unwrapping `data` and should preserve it on error results. The strings recommend conditional next steps; receiving one does not execute another request or indicate user authorization.
@@ -14,7 +14,7 @@ The header applies to the business request. No request-body schema changes are r
The accompanying JavaScript and Python SDK changes preserve `agent_hints` alongside normal flattened Document, SearchData, and MapData results and on typed errors. Python covers synchronous and asynchronous clients. Markdown and HTML retain their existing content.
-Raw HTTP callers can opt out with the header above. This change does not add a client-wide SDK hint option; applications can omit the metadata when passing results to a model. The SDK release must precede upgrading pinned CLI/MCP dependencies for their keyed SDK paths to retain the field.
+Raw HTTP and ordinary SDK callers receive no hints unless their adapter explicitly sets the header above. This change does not add a client-wide SDK hint option. The SDK release must precede upgrading pinned CLI/MCP dependencies for their keyed SDK paths to retain the field.
## Initial rules
diff --git a/apps/api/src/__tests__/snips/v2/agent-hints.test.ts b/apps/api/src/__tests__/snips/v2/agent-hints.test.ts
index ec71ef0a0..a6c810cab 100644
--- a/apps/api/src/__tests__/snips/v2/agent-hints.test.ts
+++ b/apps/api/src/__tests__/snips/v2/agent-hints.test.ts
@@ -19,6 +19,7 @@ describeIf(TEST_PRODUCTION)("Agent hints", () => {
const response = await request(TEST_API_URL)
.post("/v2/scrape")
.set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("X-Firecrawl-Agent-Hints", "true")
.send({ url: TEST_SUITE_WEBSITE, timeout: scrapeTimeout });
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
@@ -37,12 +38,11 @@ describeIf(TEST_PRODUCTION)("Agent hints", () => {
);
it(
- "honors the opt-out header on a completed map",
+ "leaves hints off by default on a completed map",
async () => {
const response = await request(TEST_API_URL)
.post("/v2/map")
.set("Authorization", `Bearer ${identity.apiKey}`)
- .set("X-Firecrawl-Agent-Hints", "false")
.send({ url: TEST_SUITE_WEBSITE, limit: 1, timeout: scrapeTimeout });
expect(response.statusCode).toBe(200);
expect(response.body).not.toHaveProperty("agent_hints");
@@ -54,6 +54,7 @@ describeIf(TEST_PRODUCTION)("Agent hints", () => {
const response = await request(TEST_API_URL)
.post("/v2/scrape")
.set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("X-Firecrawl-Agent-Hints", "true")
.send({ url: "not-a-url" });
expect(response.statusCode).toBe(400);
expect(response.body.success).toBe(false);
diff --git a/apps/api/src/middlewares/__tests__/agent-hints.test.ts b/apps/api/src/middlewares/__tests__/agent-hints.test.ts
index c6176f618..a45e66a02 100644
--- a/apps/api/src/middlewares/__tests__/agent-hints.test.ts
+++ b/apps/api/src/middlewares/__tests__/agent-hints.test.ts
@@ -30,15 +30,25 @@ function appFor(
}
describe("agent hint response middleware", () => {
- it("adds top-level metadata without changing data or warning", async () => {
+ it("leaves the original envelope unchanged by default", async () => {
const body = { success: true, data: { web: [] }, warning: "existing" };
const response = await request(appFor({ body })).post("/").send({});
expect(response.statusCode).toBe(200);
+ expect(response.body).toEqual(body);
+ });
+
+ it("adds top-level metadata only when explicitly enabled", async () => {
+ const body = { success: true, data: { web: [] }, warning: "existing" };
+ const response = await request(appFor({ body }))
+ .post("/")
+ .set("X-Firecrawl-Agent-Hints", " TRUE ")
+ .send({});
+ expect(response.statusCode).toBe(200);
expect(response.body).toMatchObject(body);
expect(response.body.agent_hints).toHaveLength(1);
});
- it("opt-out header preserves the original envelope", async () => {
+ it("values other than true preserve the original envelope", async () => {
const body = {
success: true,
data: { web: [{ url: "https://example.com" }] },
@@ -56,14 +66,20 @@ describe("agent hint response middleware", () => {
{ zdr: true },
{ feedback: false },
])("does not invent accepted feedback for %j", async settings => {
- const response = await request(appFor(settings)).post("/").send({});
+ const response = await request(appFor(settings))
+ .post("/")
+ .set("X-Firecrawl-Agent-Hints", "true")
+ .send({});
expect(response.body).not.toHaveProperty("agent_hints");
});
it("omits feedback when database authentication is unavailable", async () => {
config.USE_DB_AUTHENTICATION = false;
try {
- const response = await request(appFor()).post("/").send({});
+ const response = await request(appFor())
+ .post("/")
+ .set("X-Firecrawl-Agent-Hints", "true")
+ .send({});
expect(response.body).not.toHaveProperty("agent_hints");
} finally {
config.USE_DB_AUTHENTICATION = true;
@@ -79,6 +95,7 @@ describe("agent hint response middleware", () => {
};
const response = await request(appFor({ body, status: 400 }))
.post("/")
+ .set("X-Firecrawl-Agent-Hints", "true")
.send({});
expect(response.statusCode).toBe(400);
expect(response.body).toEqual(body);
diff --git a/apps/api/src/middlewares/agent-hints.ts b/apps/api/src/middlewares/agent-hints.ts
index 692e6d67d..f43898d8d 100644
--- a/apps/api/src/middlewares/agent-hints.ts
+++ b/apps/api/src/middlewares/agent-hints.ts
@@ -18,7 +18,7 @@ export function agentHintsMiddleware(
endpoint: AgentHintEndpoint,
): RequestHandler {
return (req, res, next) => {
- if (req.get("X-Firecrawl-Agent-Hints")?.trim().toLowerCase() === "false")
+ if (req.get("X-Firecrawl-Agent-Hints")?.trim().toLowerCase() !== "true")
return next();
const json = res.json;
res.json = function (body) {