Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(keyless): emit pseudonymous exhaustion cohorts (#4240)
Himadri Mishra1ce668cd0048
diff --git a/apps/api/src/__tests__/snips/v2/document-converter.test.ts b/apps/api/src/__tests__/snips/v2/document-converter.test.tsindex 1bab83848..d4ad87896 100644--- a/apps/api/src/__tests__/snips/v2/document-converter.test.ts+++ b/apps/api/src/__tests__/snips/v2/document-converter.test.ts@@ -6,7 +6,7 @@ describe("Document Converter tests", () => {const samplesDir = path.join(process.cwd(), "samples");const expectedMarkdownBase = (documentText: string) =>- `**Hello!**\n\n${documentText} file to test the Firecrawl Document Converter.\n\n*Italic*\n\n**Bold**\n\nUnderlined\n\n~~Strikethrough~~\n\n| | |\n| --- | --- |\n| Header 1 | Header 2 |\n| Value 1 | Value 2 |\n`;+ `**Hello!**\n\n${documentText} file to test the Firecrawl Document Converter.\n\n*Italic*\n\n**Bold**\n\nUnderlined\n\n~~Strikethrough~~\n\n| Header 1 | Header 2 |\n| --- | --- |\n| Value 1 | Value 2 |\n`;const sampleFiles = [{ file: "sample.docx", name: "DOCX" },@@ -38,7 +38,7 @@ describe("Document Converter tests", () => {});describe("XLSX document conversion", () => {- const expectedMarkdown = `## Sheet1\n\n| | |\n| --- | --- |\n| sample file | test |\n| | |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n| Macbook | 1200 |\n\n## Sheet2\n\n| | |\n| --- | --- |\n| other tab | |\n| | |\n| Name | Price |\n| ChatGPT | 20 |\n| Claude | 17 |\n| Perplexity | 20 |\n`;+ const expectedMarkdown = `## Sheet1\n\n| sample file | test |\n| --- | --- |\n| | |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n| Macbook | 1200 |\n\n## Sheet2\n\n| | |\n| --- | --- |\n| other tab | |\n| | |\n| Name | Price |\n| ChatGPT | 20 |\n| Claude | 17 |\n| Perplexity | 20 |\n`;it("should convert XLSX document and return expected markdown", () => {const filePath = path.join(samplesDir, "sample.xlsx");@@ -54,7 +54,7 @@ describe("Document Converter tests", () => {const csv = Buffer.from("Name,Price\niPhone,1000\niPad,800\n");const markdown = convertDocumentToMarkdown(new Uint8Array(csv), ".csv");expect(markdown).toBe(- "| | |\n| --- | --- |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n",+ "| Name | Price |\n| --- | --- |\n| iPhone | 1000 |\n| iPad | 800 |\n",);});});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex a7cf7fbdb..8998cfd6d 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -54,6 +54,10 @@ const configSchema = z.object({// forward the real client IP for keyless rate-limiting via the// `x-firecrawl-keyless-ip` header. Untrusted callers can't override their IP.KEYLESS_PROXY_SECRET: z.string().optional(),+ // Dedicated HMAC key for joining keyless quota-exhaustion events to the+ // existing privacy-controlled conversion pipeline. Never use the proxy or+ // credential secrets here: this value is only an analytics pseudonymizer.+ KEYLESS_CONVERSION_HMAC_SECRET: emptyStringAsUndefined(z.string().min(32)),// Dedicated signer/verifier secret for short-lived MCP delegated credentials.// Keep separate from KEYLESS_PROXY_SECRET because delegated credentials can// authorize billed requests for a managed OAuth connection.diff --git a/apps/api/src/controllers/__tests__/auth.test.ts b/apps/api/src/controllers/__tests__/auth.test.tsindex ed1f0ab31..1f9b80d85 100644--- a/apps/api/src/controllers/__tests__/auth.test.ts+++ b/apps/api/src/controllers/__tests__/auth.test.ts@@ -10,6 +10,12 @@ import {getAutumnRateLimiter,getRateLimiter,} from "../../services/rate-limiter";+import {+ consumeKeylessRequest,+ isKeylessConfigured,+ keylessConversionCohort,+} from "../../lib/keyless";+import { logger } from "../../lib/logger";import { db } from "../../db/connection";import { autumnService } from "../../services/autumn/autumn.service";@@ -50,6 +56,19 @@ vi.mock("../../services/rate-limiter", () => ({getAutumnRateLimiter: vi.fn(),}));+vi.mock("../../lib/keyless", async importOriginal => {+ const actual = await importOriginal<typeof import("../../lib/keyless")>();+ return {+ ...actual,+ consumeKeylessRequest: vi.fn(),+ isKeylessConfigured: vi.fn(),+ };+});++vi.mock("../../lib/spur", () => ({+ isKeylessIpSuspicious: vi.fn().mockResolvedValue(false),+}));+vi.mock("../../services/autumn/autumn.service", () => ({autumnService: {getRateLimitMultiplier: vi.fn(),@@ -63,12 +82,15 @@ vi.mock("../../services/agent-sponsor", () => ({describe("authenticateUser", () => {const originalUseDbAuth = config.USE_DB_AUTHENTICATION;const originalKeylessProxySecret = config.KEYLESS_PROXY_SECRET;+ const originalKeylessConversionHmacSecret =+ config.KEYLESS_CONVERSION_HMAC_SECRET;const originalMcpDelegatedCredentialSecret =config.MCP_DELEGATED_CREDENTIAL_SECRET;const originalIntrospectUrl = config.OAUTH_INTROSPECT_URL;const originalIntrospectSecret = config.OAUTH_INTROSPECT_SECRET;beforeEach(() => {+ vi.mocked(isKeylessConfigured).mockReturnValue(false);vi.mocked(autumnService.getRateLimitMultiplier).mockResolvedValue(1);vi.mocked(getAutumnRateLimiter).mockReturnValue({consume: vi.fn().mockResolvedValue(undefined),@@ -78,6 +100,7 @@ describe("authenticateUser", () => {afterEach(() => {config.USE_DB_AUTHENTICATION = originalUseDbAuth;config.KEYLESS_PROXY_SECRET = originalKeylessProxySecret;+ config.KEYLESS_CONVERSION_HMAC_SECRET = originalKeylessConversionHmacSecret;config.MCP_DELEGATED_CREDENTIAL_SECRET =originalMcpDelegatedCredentialSecret;config.OAUTH_INTROSPECT_URL = originalIntrospectUrl;@@ -129,6 +152,46 @@ describe("authenticateUser", () => {);});+ it("logs a conversion cohort for middleware keyless quota exhaustion", async () => {+ config.USE_DB_AUTHENTICATION = true;+ config.KEYLESS_CONVERSION_HMAC_SECRET = "a".repeat(32);+ vi.mocked(isKeylessConfigured).mockReturnValue(true);+ vi.mocked(consumeKeylessRequest).mockResolvedValue({+ ok: false,+ reason: "requests",+ requestsUsed: 10,+ creditsUsed: 2,+ retryAfterSeconds: 42,+ });+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);++ const auth = await authenticateUser(+ {+ headers: {},+ socket: { remoteAddress: "203.0.113.8" },+ },+ {},+ RateLimiterMode.Scrape,+ { allowKeyless: true },+ );++ expect(auth).toEqual(+ expect.objectContaining({+ success: false,+ status: 429,+ keylessReason: "requests",+ }),+ );+ expect(warn).toHaveBeenCalledWith(+ "Keyless request blocked",+ expect.objectContaining({+ event: "keyless_exhausted",+ reason: "requests",+ conversionCohort: keylessConversionCohort("203.0.113.8"),+ }),+ );+ });+it("writes normal API-key ACUC entries to the general-purpose cache", async () => {config.USE_DB_AUTHENTICATION = true;vi.mocked(getValue).mockResolvedValue(null);diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex cd789a197..0c1bb4aad 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -10,6 +10,7 @@ import {KEYLESS_FREE_TIER_LIMIT_MESSAGE,consumeKeylessRequest,isKeylessConfigured,+ keylessExhaustionTelemetry,isKeylessIpEligible,keylessTeamId,} from "../lib/keyless";@@ -518,6 +519,7 @@ async function handleKeylessAuth(event: "keyless_exhausted",reason: result.reason,retryAfterSeconds: result.retryAfterSeconds,+ ...keylessExhaustionTelemetry(ip),});return {success: false,diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex f4d242afa..4ecbc7036 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -42,6 +42,7 @@ import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forciimport responseTime from "response-time";import { shutdownWebhookQueue } from "./services/webhook";import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";+import { isKeylessConfigured } from "./lib/keyless";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -54,6 +55,12 @@ logger.info("Network info dump", {networkInterfaces: os.networkInterfaces(),});+if (isKeylessConfigured() && !config.KEYLESS_CONVERSION_HMAC_SECRET) {+ logger.warn(+ "Keyless conversion cohort logging is disabled: set KEYLESS_CONVERSION_HMAC_SECRET to enable privacy-safe quota-to-account measurement",+ );+}+// Install cacheable lookup for all other requestscacheableLookup.install(http.globalAgent);cacheableLookup.install(https.globalAgent);diff --git a/apps/api/src/lib/keyless.conversion.test.ts b/apps/api/src/lib/keyless.conversion.test.tsnew file mode 100644index 000000000..fa3386eb5--- /dev/null+++ b/apps/api/src/lib/keyless.conversion.test.ts@@ -0,0 +1,72 @@+import { afterEach, describe, expect, it, vi } from "vitest";++vi.mock("../services/rate-limiter", () => ({+ redisRateLimitClient: { ttl: vi.fn() },+}));+import { config } from "../config";+import { logger } from "./logger";+import {+ KEYLESS_CONVERSION_COHORT_VERSION,+ keylessConversionCohort,+ keylessExhaustionTelemetry,+ keylessLimitBody,+} from "./keyless";+import { redisRateLimitClient } from "../services/rate-limiter";++describe("keyless conversion cohort telemetry", () => {+ const originalSecret = config.KEYLESS_CONVERSION_HMAC_SECRET;++ afterEach(() => {+ config.KEYLESS_CONVERSION_HMAC_SECRET = originalSecret;+ vi.restoreAllMocks();+ });++ it("emits a deterministic, versioned HMAC cohort rather than the IP", () => {+ config.KEYLESS_CONVERSION_HMAC_SECRET = "a".repeat(32);++ const cohort = keylessConversionCohort("203.0.113.8");++ expect(cohort).toMatch(+ new RegExp(`^${KEYLESS_CONVERSION_COHORT_VERSION}:`),+ );+ expect(cohort).not.toContain("203.0.113.8");+ expect(keylessConversionCohort("203.0.113.8")).toBe(cohort);+ expect(keylessConversionCohort("203.0.113.9")).not.toBe(cohort);+ expect(keylessConversionCohort("::ffff:203.0.113.8")).toBe(cohort);+ expect(keylessConversionCohort("::FFFF:203.0.113.8")).toBe(cohort);+ expect(keylessExhaustionTelemetry("203.0.113.8")).toEqual({+ conversionCohort: cohort,+ });+ });++ it("does not emit a cohort when the dedicated analytics secret is unset", () => {+ config.KEYLESS_CONVERSION_HMAC_SECRET = undefined;++ expect(keylessConversionCohort("203.0.113.8")).toBeUndefined();+ expect(keylessExhaustionTelemetry("203.0.113.8")).toEqual({});+ });++ it("does not mint a cohort for a blank IP value", () => {+ config.KEYLESS_CONVERSION_HMAC_SECRET = "a".repeat(32);++ expect(keylessConversionCohort(" ")).toBeUndefined();+ expect(keylessExhaustionTelemetry(" ")).toEqual({});+ });++ it("adds the cohort to reservation-limit exhaustion telemetry", async () => {+ config.KEYLESS_CONVERSION_HMAC_SECRET = "b".repeat(32);+ vi.spyOn(redisRateLimitClient, "ttl").mockResolvedValue(42);+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);++ await keylessLimitBody("preview_keyless_203.0.113.8", "search");++ expect(warn).toHaveBeenCalledWith(+ "Keyless request blocked",+ expect.objectContaining({+ event: "keyless_exhausted",+ reason: "credits",+ conversionCohort: keylessConversionCohort("203.0.113.8"),+ }),+ );+ });+});diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex ef8281483..3ff9a53fb 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -1,3 +1,4 @@+import { createHmac } from "node:crypto";import { isIPv4 } from "node:net";import { v5 as uuidv5 } from "uuid";import { config } from "../config";@@ -36,6 +37,34 @@ export function isKeylessConfigured(): boolean {const DAY_SECONDS = 86400;+// This value is emitted only on quota exhaustion. It is a versioned, keyed+// pseudonym—not an IP address—so analytics can join the event to the existing+// privacy-controlled signup/OAuth matching pipeline without expanding raw-IP+// logging. Rotation intentionally creates a new cohort namespace.+export const KEYLESS_CONVERSION_COHORT_VERSION = "v1";++function normalizeKeylessIpv4(ip: string): string {+ const trimmed = ip.trim();+ const lower = trimmed.toLowerCase();+ return lower.startsWith("::ffff:") && isIPv4(trimmed.slice(7))+ ? trimmed.slice(7)+ : trimmed;+}++export function keylessConversionCohort(ip: string): string | undefined {+ const secret = config.KEYLESS_CONVERSION_HMAC_SECRET;+ const normalizedIp = normalizeKeylessIpv4(ip);+ if (!secret || !normalizedIp) return undefined;+ return `${KEYLESS_CONVERSION_COHORT_VERSION}:${createHmac("sha256", secret)+ .update(normalizedIp)+ .digest("base64url")}`;+}++export function keylessExhaustionTelemetry(ip: string): Record<string, string> {+ const conversionCohort = keylessConversionCohort(ip);+ return conversionCohort ? { conversionCohort } : {};+}+// Keyless teams reuse the `preview_` prefix so billing (autumn `isPreviewTeam`)// and GCS persistence are skipped automatically, with a dedicated infix so the// IP can be recovered when charging credits after a request completes.@@ -80,8 +109,7 @@ export function keylessTeamUuid(* is treated as IPv4.*/export function isKeylessIpEligible(ip: string): boolean {- const normalized = ip.startsWith("::ffff:") ? ip.slice("::ffff:".length) : ip;- return isIPv4(normalized);+ return isIPv4(normalizeKeylessIpv4(ip));}const requestsKey = (ip: string) => `keyless_requests:${ip}`;@@ -141,6 +169,7 @@ export async function keylessLimitBody(reason: "credits",mode,retryAfterSeconds,+ ...keylessExhaustionTelemetry(ip ?? ""),});return {success: false,