Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): accept X-Integration header as integration attribution (#5246)
Erik Engervallbafad0bebcc8
diff --git a/apps/api/src/__tests__/snips/v2/integration-header.test.ts b/apps/api/src/__tests__/snips/v2/integration-header.test.tsnew file mode 100644index 000000000..c7384e54f--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/integration-header.test.ts@@ -0,0 +1,100 @@+import { config } from "../../../config";+import {+ ALLOW_TEST_SUITE_WEBSITE,+ concurrentIf,+ createTestIdUrl,+ describeIf,+} from "../lib";+import request, { idmux, Identity, scrapeTimeout, TEST_API_URL } from "./lib";+import { HAS_JOB_LOG, waitForJobLogRow } from "../job-log";++const CAN_READ_LOG = !!config.USE_DB_AUTHENTICATION && HAS_JOB_LOG;++async function scrapeWithHeader(+ path: "/v1/scrape" | "/v2/scrape",+ body: Record<string, unknown>,+ identity: Identity,+ integrationHeader: string,+) {+ return await request(TEST_API_URL)+ .post(path)+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .set("User-Agent", "node")+ .set("X-Integration", integrationHeader)+ .send(body);+}++async function loggedIntegration(identity: Identity, url: string) {+ const row = await waitForJobLogRow<{ integration: string | null }>(+ "requests",+ "team_id = {teamId: UUID} AND kind = 'scrape' AND target_hint = {url: String}",+ { teamId: identity.teamId, url },+ );+ expect(row).not.toBeNull();+ return row!.integration;+}++describeIf(ALLOW_TEST_SUITE_WEBSITE)("X-Integration header", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "integration-header",+ concurrency: 100,+ credits: 1000000,+ });+ }, 10000);++ concurrentIf(CAN_READ_LOG).each(["/v1/scrape", "/v2/scrape"] as const)(+ "attributes %s to a header-only integration",+ async path => {+ const url = createTestIdUrl();+ const res = await scrapeWithHeader(path, { url }, identity, " N8N ");++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ expect(await loggedIntegration(identity, url)).toBe("n8n");+ },+ scrapeTimeout + 60000,+ );++ concurrentIf(CAN_READ_LOG)(+ "prefers the body integration over the header",+ async () => {+ const url = createTestIdUrl();+ const res = await scrapeWithHeader(+ "/v2/scrape",+ { url, integration: "_integration_header_test" },+ identity,+ "n8n",+ );++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ expect(await loggedIntegration(identity, url)).toBe(+ "_integration_header_test",+ );+ },+ scrapeTimeout + 60000,+ );++ it.concurrent(+ "ignores an invalid header and still succeeds",+ async () => {+ const url = createTestIdUrl();+ const res = await scrapeWithHeader(+ "/v2/scrape",+ { url },+ identity,+ "not-a-real-integration",+ );++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ if (!CAN_READ_LOG) return;+ expect(await loggedIntegration(identity, url)).toBeNull();+ },+ scrapeTimeout + 60000,+ );+});diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex e52bef2a7..8e109140a 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -52,6 +52,7 @@ import {type TeamLimits,} from "../services/autumn/autumn.service";import { ReplyError } from "ioredis";+import { integrationFromHeader } from "../utils/integration";const ACUC_TTL_SECONDS = 600;// A chunk built on the fail-open limits is cached only briefly, so a@@ -626,7 +627,7 @@ async function handleKeylessAuth(}const origin = req.body?.origin;- const integration = req.body?.integration;+ const integration = req.body?.integration ?? integrationFromHeader(req);// No origin/surface gate: any request without an API key may use the free// tier on the allowlisted endpoints (the API itself is free). origin and// integration are still recorded below for abuse monitoring, and pick the@@ -668,7 +669,7 @@ async function handleKeylessAuth(canonicalLog: "keyless/consume",ip,origin: req.body?.origin,- integration: req.body?.integration,+ integration,blocked: true,reason: "suspicious",...(signupRef ? { signupRef } : {}),diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex fb29390b6..8cc53948b 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -25,7 +25,10 @@ import type {import type { RequestWithAuth } from "../v1/types";import { wrap } from "../../routes/shared";import { deprecationMiddleware } from "../../lib/deprecations";-import { integrationSchema } from "../../utils/integration";+import {+ integrationFromHeader,+ integrationSchema,+} from "../../utils/integration";import { requestOrigin } from "../../lib/request-origin";const SEARCH_CREDITS_PER_TEN_RESULTS = 2;@@ -351,7 +354,7 @@ function createResearchController(external_request_id: externalRequestId(authedReq),team_id: authedReq.auth.team_id,origin: requestOrigin(params, req),- integration: params.integration ?? null,+ integration: params.integration ?? integrationFromHeader(req) ?? null,target_hint: targetHint,zeroDataRetention,api_key_id: authedReq.acuc?.api_key_id ?? null,diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex ced568870..a4a88619a 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -43,6 +43,7 @@ import {isAgentInteropSecretValid,} from "../lib/agent-interop";import { inferIntegration } from "../lib/infer-integration";+import { integrationFromHeader } from "../utils/integration";export function checkCreditsMiddleware(_minimum?: number,@@ -339,6 +340,10 @@ export function authMiddleware(};}+/**+ * Fills a missing body `integration` from a valid X-Integration header, then+ * from inference. The body value always wins; an invalid header is ignored.+ */export function inferIntegrationMiddleware(req: Request,_res: Response,@@ -346,12 +351,18 @@ export function inferIntegrationMiddleware() {const body = req.body;if (- config.INFER_INTEGRATION_ENABLED &&- body &&- typeof body === "object" &&- !Array.isArray(body) &&- body.integration === undefined+ !body ||+ typeof body !== "object" ||+ Array.isArray(body) ||+ body.integration !== undefined) {+ return next();+ }++ const header = integrationFromHeader(req);+ if (header) {+ body.integration = header;+ } else if (config.INFER_INTEGRATION_ENABLED) {const inferred = inferIntegration({origin: req.get("origin"),referer: req.get("referer"),diff --git a/apps/api/src/utils/integration.ts b/apps/api/src/utils/integration.tsindex 45c47738a..2a5d0438a 100644--- a/apps/api/src/utils/integration.ts+++ b/apps/api/src/utils/integration.ts@@ -34,3 +34,15 @@ export const integrationSchema = z.join(" | ")}`,},);++/** The X-Integration header as an integration value, or undefined when it is missing or invalid. */+export function integrationFromHeader(req: {+ headers?: Record<string, string | string[] | undefined>;+}): string | undefined {+ const header = req.headers?.["x-integration"];+ const value =+ typeof header === "string" ? header.trim().toLowerCase() : undefined;+ return value && integrationSchema.safeParse(value).success+ ? value+ : undefined;+}