Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): let a valid X-Integration header override a body integration of cli (#5260)
Erik Engervall469f9974dd0d
diff --git a/apps/api/src/__tests__/snips/v2/integration-header.test.ts b/apps/api/src/__tests__/snips/v2/integration-header.test.tsindex c7384e54f..3e127bdcf 100644--- a/apps/api/src/__tests__/snips/v2/integration-header.test.ts+++ b/apps/api/src/__tests__/snips/v2/integration-header.test.ts@@ -79,6 +79,43 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("X-Integration header", () => {scrapeTimeout + 60000,);+ concurrentIf(CAN_READ_LOG)(+ "lets a valid header override a body integration of cli",+ async () => {+ const url = createTestIdUrl();+ const res = await scrapeWithHeader(+ "/v2/scrape",+ { url, integration: "cli" },+ identity,+ "hermes",+ );++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ expect(await loggedIntegration(identity, url)).toBe("hermes");+ },+ scrapeTimeout + 60000,+ );++ it.concurrent(+ "keeps a body integration of cli when the header is invalid",+ async () => {+ const url = createTestIdUrl();+ const res = await scrapeWithHeader(+ "/v2/scrape",+ { url, integration: "cli" },+ identity,+ "not a valid tag!",+ );++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ if (!CAN_READ_LOG) return;+ expect(await loggedIntegration(identity, url)).toBe("cli");+ },+ scrapeTimeout + 60000,+ );+it.concurrent("ignores an invalid header and still succeeds",async () => {diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 8e109140a..9b352f34a 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -52,7 +52,7 @@ import {type TeamLimits,} from "../services/autumn/autumn.service";import { ReplyError } from "ioredis";-import { integrationFromHeader } from "../utils/integration";+import { resolveIntegration } from "../utils/integration";const ACUC_TTL_SECONDS = 600;// A chunk built on the fail-open limits is cached only briefly, so a@@ -627,7 +627,7 @@ async function handleKeylessAuth(}const origin = req.body?.origin;- const integration = req.body?.integration ?? integrationFromHeader(req);+ const integration = resolveIntegration(req.body?.integration, req);// No origin/surface gate: any request without an API key may use the free// tier on the allowlisted endpoints (the API itself is free). origin and// integration are still recorded below for abuse monitoring, and pick thediff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex f91344477..1b52931bf 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -26,10 +26,7 @@ import type { RequestWithAuth } from "../v1/types";import { checkCreditsMiddleware, wrap } from "../../routes/shared";import { SEARCH_CREDITS_FEATURE_ID } from "../../services/autumn/autumn.service";import { deprecationMiddleware } from "../../lib/deprecations";-import {- integrationFromHeader,- integrationSchema,-} from "../../utils/integration";+import { integrationSchema, resolveIntegration } from "../../utils/integration";import { requestOrigin } from "../../lib/request-origin";const SEARCH_CREDITS_PER_TEN_RESULTS = 2;@@ -362,7 +359,7 @@ function createResearchController(external_request_id: externalRequestId(authedReq),team_id: authedReq.auth.team_id,origin: requestOrigin(params, req),- integration: params.integration ?? integrationFromHeader(req) ?? null,+ integration: resolveIntegration(params.integration, req) ?? null,target_hint: targetHint,zeroDataRetention,api_key_id: authedReq.acuc?.api_key_id ?? null,diff --git a/apps/api/src/lib/keyless-signup-link.test.ts b/apps/api/src/lib/keyless-signup-link.test.tsindex 24a4dbfbc..37e60e291 100644--- a/apps/api/src/lib/keyless-signup-link.test.ts+++ b/apps/api/src/lib/keyless-signup-link.test.ts@@ -227,6 +227,16 @@ describe("keylessSignupSurface", () => {// v1 schemas prefault a missing origin to "api"; the header still counts.[{ body: { origin: "api" }, headers: { "x-origin": "cli" } }, "cli"],[{ body: { origin: "api" }, headers: { "x-origin": "mcp-x@1" } }, "mcp"],+ // A valid X-Integration header replaces a missing or cli body integration.+ [{ body: {}, headers: { "x-integration": "cli" } }, "cli"],+ [+ { body: { integration: "cli" }, headers: { "x-integration": "hermes" } },+ "api",+ ],+ [+ { body: { integration: "cli" }, headers: { "x-integration": "a b!" } },+ "cli",+ ],// An explicit non-default body origin still wins over the header.[{ body: { origin: "[email protected]" }, headers: { "x-origin": "cli" } },diff --git a/apps/api/src/lib/keyless-signup-link.ts b/apps/api/src/lib/keyless-signup-link.tsindex 1913c87fe..f3136527d 100644--- a/apps/api/src/lib/keyless-signup-link.ts+++ b/apps/api/src/lib/keyless-signup-link.ts@@ -2,6 +2,7 @@ import { createCipheriv } from "node:crypto";import { isIPv4 } from "node:net";import { config } from "../config";import { logger } from "./logger";+import { resolveIntegration } from "../utils/integration";// Keyless prompts link to signup at https://firecrawl.dev/k/<token>. The token// is the prompt itself, encrypted: nothing is stored per identity, and the web@@ -337,9 +338,7 @@ export function keylessSignupSurface(req: RequestLike): KeylessSignupSurface {bodyOrigin && bodyOrigin !== "api"? bodyOrigin: (headerOrigin ?? bodyOrigin);- const integration =- lowerString(body.integration) ??- lowerString(firstHeader(req, "x-integration"));+ const integration = resolveIntegration(lowerString(body.integration), req);if (config.KEYLESS_PROXY_SECRET &&firstHeader(req, "x-firecrawl-keyless-secret") ===diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex cc56a229c..21131e0fb 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -43,7 +43,7 @@ import {isAgentInteropSecretValid,} from "../lib/agent-interop";import { inferIntegration } from "../lib/infer-integration";-import { integrationFromHeader } from "../utils/integration";+import { resolveIntegration } from "../utils/integration";export function checkCreditsMiddleware(_minimum?: number,@@ -331,8 +331,8 @@ export function authMiddleware(}/**- * Fills a missing body `integration` from a valid X-Integration header, then- * from inference. The body value always wins; an invalid header is ignored.+ * Sets the body `integration` via resolveIntegration, then infers a still+ * missing one. An invalid header is ignored.*/export function inferIntegrationMiddleware(req: Request,@@ -340,18 +340,13 @@ export function inferIntegrationMiddleware(next: NextFunction,) {const body = req.body;- if (- !body ||- typeof body !== "object" ||- Array.isArray(body) ||- body.integration !== undefined- ) {+ if (!body || typeof body !== "object" || Array.isArray(body)) {return next();}- const header = integrationFromHeader(req);- if (header) {- body.integration = header;+ const resolved = resolveIntegration(body.integration, req);+ if (resolved !== undefined) {+ body.integration = resolved;} else if (config.INFER_INTEGRATION_ENABLED) {const inferred = inferIntegration({origin: req.get("origin"),diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 526e1191c..b0698fcd8 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -206,12 +206,14 @@ v2Router.post("/search/:jobId/feedback",authMiddleware(RateLimiterMode.Account),validateJobIdParam,+ inferIntegrationMiddleware,wrap(searchFeedbackController),);v2Router.post("/feedback",authMiddleware(RateLimiterMode.Account),+ inferIntegrationMiddleware,wrap(feedbackController),);@@ -269,6 +271,7 @@ v2Router.post("/scrape/:jobId/interact",authMiddleware(RateLimiterMode.BrowserExecute, { allowKeyless: true }),validateJobIdParam,+ inferIntegrationMiddleware,wrap(scrapeInteractController),);diff --git a/apps/api/src/utils/integration.test.ts b/apps/api/src/utils/integration.test.tsnew file mode 100644index 000000000..b38bcc145--- /dev/null+++ b/apps/api/src/utils/integration.test.ts@@ -0,0 +1,43 @@+import { describe, expect, it } from "vitest";+import { integrationFromHeader, resolveIntegration } from "./integration";++const withHeader = (value?: string) => ({+ headers: value === undefined ? {} : { "x-integration": value },+});++describe("integrationFromHeader", () => {+ it("normalizes a valid header", () => {+ expect(integrationFromHeader(withHeader(" Hermes "))).toBe("hermes");+ expect(integrationFromHeader(withHeader("_rtk"))).toBe("_rtk");+ });++ it("ignores a missing or invalid header", () => {+ expect(integrationFromHeader(withHeader())).toBeUndefined();+ expect(integrationFromHeader(withHeader(""))).toBeUndefined();+ expect(+ integrationFromHeader(withHeader("not a valid tag!")),+ ).toBeUndefined();+ });+});++describe("resolveIntegration", () => {+ it("lets a valid header replace a missing or cli value", () => {+ expect(resolveIntegration(undefined, withHeader("hermes"))).toBe("hermes");+ expect(resolveIntegration(null, withHeader("hermes"))).toBe("hermes");+ expect(resolveIntegration("cli", withHeader("_rtk"))).toBe("_rtk");+ });++ it("keeps any other explicit value over the header", () => {+ expect(resolveIntegration("n8n", withHeader("hermes"))).toBe("n8n");+ expect(resolveIntegration("_custom", withHeader("hermes"))).toBe("_custom");+ });++ it("keeps the explicit value when the header is missing or invalid", () => {+ expect(resolveIntegration("cli", withHeader())).toBe("cli");+ expect(resolveIntegration("cli", withHeader("not a valid tag!"))).toBe(+ "cli",+ );+ expect(resolveIntegration(undefined, withHeader("nope"))).toBeUndefined();+ expect(resolveIntegration(null, withHeader())).toBeNull();+ });+});diff --git a/apps/api/src/utils/integration.ts b/apps/api/src/utils/integration.tsindex 2a5d0438a..eb4cd276e 100644--- a/apps/api/src/utils/integration.ts+++ b/apps/api/src/utils/integration.ts@@ -46,3 +46,15 @@ export function integrationFromHeader(req: {? value: undefined;}++/**+ * The integration to attribute: a valid X-Integration header replaces a+ * missing or `cli` explicit value; any other explicit value wins.+ */+export function resolveIntegration<T>(+ explicit: T,+ req: { headers?: Record<string, string | string[] | undefined> },+): T | string {+ if (explicit != null && explicit !== "cli") return explicit;+ return integrationFromHeader(req) ?? explicit;+}