Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(keyless): emit pseudonymous exhaustion cohorts (#4240)

Himadri Mishra1ce668cd0048
diff --git a/apps/api/src/__tests__/snips/v2/document-converter.test.ts b/apps/api/src/__tests__/snips/v2/document-converter.test.ts
index 1bab83848..d4ad87896 100644
--- a/apps/api/src/__tests__/snips/v2/document-converter.test.ts
+++ b/apps/api/src/__tests__/snips/v2/document-converter.test.ts
@@ -6,7 +6,7 @@ describe("Document Converter tests", () => {
const samplesDir = path.join(process.cwd(), "samples");
const expectedMarkdownBase = (documentText: string) =>
- `**Hello!**\n\n${documentText} file to test the Firecrawl Document Converter.\n\n*Italic*\n\n**Bold**\n\nUnderlined\n\n~~Strikethrough~~\n\n| | |\n| --- | --- |\n| Header 1 | Header 2 |\n| Value 1 | Value 2 |\n`;
+ `**Hello!**\n\n${documentText} file to test the Firecrawl Document Converter.\n\n*Italic*\n\n**Bold**\n\nUnderlined\n\n~~Strikethrough~~\n\n| Header 1 | Header 2 |\n| --- | --- |\n| Value 1 | Value 2 |\n`;
const sampleFiles = [
{ file: "sample.docx", name: "DOCX" },
@@ -38,7 +38,7 @@ describe("Document Converter tests", () => {
});
describe("XLSX document conversion", () => {
- const expectedMarkdown = `## Sheet1\n\n| | |\n| --- | --- |\n| sample file | test |\n| | |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n| Macbook | 1200 |\n\n## Sheet2\n\n| | |\n| --- | --- |\n| other tab | |\n| | |\n| Name | Price |\n| ChatGPT | 20 |\n| Claude | 17 |\n| Perplexity | 20 |\n`;
+ const expectedMarkdown = `## Sheet1\n\n| sample file | test |\n| --- | --- |\n| | |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n| Macbook | 1200 |\n\n## Sheet2\n\n| | |\n| --- | --- |\n| other tab | |\n| | |\n| Name | Price |\n| ChatGPT | 20 |\n| Claude | 17 |\n| Perplexity | 20 |\n`;
it("should convert XLSX document and return expected markdown", () => {
const filePath = path.join(samplesDir, "sample.xlsx");
@@ -54,7 +54,7 @@ describe("Document Converter tests", () => {
const csv = Buffer.from("Name,Price\niPhone,1000\niPad,800\n");
const markdown = convertDocumentToMarkdown(new Uint8Array(csv), ".csv");
expect(markdown).toBe(
- "| | |\n| --- | --- |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n",
+ "| Name | Price |\n| --- | --- |\n| iPhone | 1000 |\n| iPad | 800 |\n",
);
});
});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index a7cf7fbdb..8998cfd6d 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -54,6 +54,10 @@ const configSchema = z.object({
// forward the real client IP for keyless rate-limiting via the
// `x-firecrawl-keyless-ip` header. Untrusted callers can't override their IP.
KEYLESS_PROXY_SECRET: z.string().optional(),
+ // Dedicated HMAC key for joining keyless quota-exhaustion events to the
+ // existing privacy-controlled conversion pipeline. Never use the proxy or
+ // credential secrets here: this value is only an analytics pseudonymizer.
+ KEYLESS_CONVERSION_HMAC_SECRET: emptyStringAsUndefined(z.string().min(32)),
// Dedicated signer/verifier secret for short-lived MCP delegated credentials.
// Keep separate from KEYLESS_PROXY_SECRET because delegated credentials can
// authorize billed requests for a managed OAuth connection.
diff --git a/apps/api/src/controllers/__tests__/auth.test.ts b/apps/api/src/controllers/__tests__/auth.test.ts
index ed1f0ab31..1f9b80d85 100644
--- a/apps/api/src/controllers/__tests__/auth.test.ts
+++ b/apps/api/src/controllers/__tests__/auth.test.ts
@@ -10,6 +10,12 @@ import {
getAutumnRateLimiter,
getRateLimiter,
} from "../../services/rate-limiter";
+import {
+ consumeKeylessRequest,
+ isKeylessConfigured,
+ keylessConversionCohort,
+} from "../../lib/keyless";
+import { logger } from "../../lib/logger";
import { db } from "../../db/connection";
import { autumnService } from "../../services/autumn/autumn.service";
@@ -50,6 +56,19 @@ vi.mock("../../services/rate-limiter", () => ({
getAutumnRateLimiter: vi.fn(),
}));
+vi.mock("../../lib/keyless", async importOriginal => {
+ const actual = await importOriginal<typeof import("../../lib/keyless")>();
+ return {
+ ...actual,
+ consumeKeylessRequest: vi.fn(),
+ isKeylessConfigured: vi.fn(),
+ };
+});
+
+vi.mock("../../lib/spur", () => ({
+ isKeylessIpSuspicious: vi.fn().mockResolvedValue(false),
+}));
+
vi.mock("../../services/autumn/autumn.service", () => ({
autumnService: {
getRateLimitMultiplier: vi.fn(),
@@ -63,12 +82,15 @@ vi.mock("../../services/agent-sponsor", () => ({
describe("authenticateUser", () => {
const originalUseDbAuth = config.USE_DB_AUTHENTICATION;
const originalKeylessProxySecret = config.KEYLESS_PROXY_SECRET;
+ const originalKeylessConversionHmacSecret =
+ config.KEYLESS_CONVERSION_HMAC_SECRET;
const originalMcpDelegatedCredentialSecret =
config.MCP_DELEGATED_CREDENTIAL_SECRET;
const originalIntrospectUrl = config.OAUTH_INTROSPECT_URL;
const originalIntrospectSecret = config.OAUTH_INTROSPECT_SECRET;
beforeEach(() => {
+ vi.mocked(isKeylessConfigured).mockReturnValue(false);
vi.mocked(autumnService.getRateLimitMultiplier).mockResolvedValue(1);
vi.mocked(getAutumnRateLimiter).mockReturnValue({
consume: vi.fn().mockResolvedValue(undefined),
@@ -78,6 +100,7 @@ describe("authenticateUser", () => {
afterEach(() => {
config.USE_DB_AUTHENTICATION = originalUseDbAuth;
config.KEYLESS_PROXY_SECRET = originalKeylessProxySecret;
+ config.KEYLESS_CONVERSION_HMAC_SECRET = originalKeylessConversionHmacSecret;
config.MCP_DELEGATED_CREDENTIAL_SECRET =
originalMcpDelegatedCredentialSecret;
config.OAUTH_INTROSPECT_URL = originalIntrospectUrl;
@@ -129,6 +152,46 @@ describe("authenticateUser", () => {
);
});
+ it("logs a conversion cohort for middleware keyless quota exhaustion", async () => {
+ config.USE_DB_AUTHENTICATION = true;
+ config.KEYLESS_CONVERSION_HMAC_SECRET = "a".repeat(32);
+ vi.mocked(isKeylessConfigured).mockReturnValue(true);
+ vi.mocked(consumeKeylessRequest).mockResolvedValue({
+ ok: false,
+ reason: "requests",
+ requestsUsed: 10,
+ creditsUsed: 2,
+ retryAfterSeconds: 42,
+ });
+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);
+
+ const auth = await authenticateUser(
+ {
+ headers: {},
+ socket: { remoteAddress: "203.0.113.8" },
+ },
+ {},
+ RateLimiterMode.Scrape,
+ { allowKeyless: true },
+ );
+
+ expect(auth).toEqual(
+ expect.objectContaining({
+ success: false,
+ status: 429,
+ keylessReason: "requests",
+ }),
+ );
+ expect(warn).toHaveBeenCalledWith(
+ "Keyless request blocked",
+ expect.objectContaining({
+ event: "keyless_exhausted",
+ reason: "requests",
+ conversionCohort: keylessConversionCohort("203.0.113.8"),
+ }),
+ );
+ });
+
it("writes normal API-key ACUC entries to the general-purpose cache", async () => {
config.USE_DB_AUTHENTICATION = true;
vi.mocked(getValue).mockResolvedValue(null);
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index cd789a197..0c1bb4aad 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -10,6 +10,7 @@ import {
KEYLESS_FREE_TIER_LIMIT_MESSAGE,
consumeKeylessRequest,
isKeylessConfigured,
+ keylessExhaustionTelemetry,
isKeylessIpEligible,
keylessTeamId,
} from "../lib/keyless";
@@ -518,6 +519,7 @@ async function handleKeylessAuth(
event: "keyless_exhausted",
reason: result.reason,
retryAfterSeconds: result.retryAfterSeconds,
+ ...keylessExhaustionTelemetry(ip),
});
return {
success: false,
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index f4d242afa..4ecbc7036 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -42,6 +42,7 @@ import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forci
import responseTime from "response-time";
import { shutdownWebhookQueue } from "./services/webhook";
import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";
+import { isKeylessConfigured } from "./lib/keyless";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
@@ -54,6 +55,12 @@ logger.info("Network info dump", {
networkInterfaces: os.networkInterfaces(),
});
+if (isKeylessConfigured() && !config.KEYLESS_CONVERSION_HMAC_SECRET) {
+ logger.warn(
+ "Keyless conversion cohort logging is disabled: set KEYLESS_CONVERSION_HMAC_SECRET to enable privacy-safe quota-to-account measurement",
+ );
+}
+
// Install cacheable lookup for all other requests
cacheableLookup.install(http.globalAgent);
cacheableLookup.install(https.globalAgent);
diff --git a/apps/api/src/lib/keyless.conversion.test.ts b/apps/api/src/lib/keyless.conversion.test.ts
new file mode 100644
index 000000000..fa3386eb5
--- /dev/null
+++ b/apps/api/src/lib/keyless.conversion.test.ts
@@ -0,0 +1,72 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../services/rate-limiter", () => ({
+ redisRateLimitClient: { ttl: vi.fn() },
+}));
+import { config } from "../config";
+import { logger } from "./logger";
+import {
+ KEYLESS_CONVERSION_COHORT_VERSION,
+ keylessConversionCohort,
+ keylessExhaustionTelemetry,
+ keylessLimitBody,
+} from "./keyless";
+import { redisRateLimitClient } from "../services/rate-limiter";
+
+describe("keyless conversion cohort telemetry", () => {
+ const originalSecret = config.KEYLESS_CONVERSION_HMAC_SECRET;
+
+ afterEach(() => {
+ config.KEYLESS_CONVERSION_HMAC_SECRET = originalSecret;
+ vi.restoreAllMocks();
+ });
+
+ it("emits a deterministic, versioned HMAC cohort rather than the IP", () => {
+ config.KEYLESS_CONVERSION_HMAC_SECRET = "a".repeat(32);
+
+ const cohort = keylessConversionCohort("203.0.113.8");
+
+ expect(cohort).toMatch(
+ new RegExp(`^${KEYLESS_CONVERSION_COHORT_VERSION}:`),
+ );
+ expect(cohort).not.toContain("203.0.113.8");
+ expect(keylessConversionCohort("203.0.113.8")).toBe(cohort);
+ expect(keylessConversionCohort("203.0.113.9")).not.toBe(cohort);
+ expect(keylessConversionCohort("::ffff:203.0.113.8")).toBe(cohort);
+ expect(keylessConversionCohort("::FFFF:203.0.113.8")).toBe(cohort);
+ expect(keylessExhaustionTelemetry("203.0.113.8")).toEqual({
+ conversionCohort: cohort,
+ });
+ });
+
+ it("does not emit a cohort when the dedicated analytics secret is unset", () => {
+ config.KEYLESS_CONVERSION_HMAC_SECRET = undefined;
+
+ expect(keylessConversionCohort("203.0.113.8")).toBeUndefined();
+ expect(keylessExhaustionTelemetry("203.0.113.8")).toEqual({});
+ });
+
+ it("does not mint a cohort for a blank IP value", () => {
+ config.KEYLESS_CONVERSION_HMAC_SECRET = "a".repeat(32);
+
+ expect(keylessConversionCohort(" ")).toBeUndefined();
+ expect(keylessExhaustionTelemetry(" ")).toEqual({});
+ });
+
+ it("adds the cohort to reservation-limit exhaustion telemetry", async () => {
+ config.KEYLESS_CONVERSION_HMAC_SECRET = "b".repeat(32);
+ vi.spyOn(redisRateLimitClient, "ttl").mockResolvedValue(42);
+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);
+
+ await keylessLimitBody("preview_keyless_203.0.113.8", "search");
+
+ expect(warn).toHaveBeenCalledWith(
+ "Keyless request blocked",
+ expect.objectContaining({
+ event: "keyless_exhausted",
+ reason: "credits",
+ conversionCohort: keylessConversionCohort("203.0.113.8"),
+ }),
+ );
+ });
+});
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
index ef8281483..3ff9a53fb 100644
--- a/apps/api/src/lib/keyless.ts
+++ b/apps/api/src/lib/keyless.ts
@@ -1,3 +1,4 @@
+import { createHmac } from "node:crypto";
import { isIPv4 } from "node:net";
import { v5 as uuidv5 } from "uuid";
import { config } from "../config";
@@ -36,6 +37,34 @@ export function isKeylessConfigured(): boolean {
const DAY_SECONDS = 86400;
+// This value is emitted only on quota exhaustion. It is a versioned, keyed
+// pseudonym—not an IP address—so analytics can join the event to the existing
+// privacy-controlled signup/OAuth matching pipeline without expanding raw-IP
+// logging. Rotation intentionally creates a new cohort namespace.
+export const KEYLESS_CONVERSION_COHORT_VERSION = "v1";
+
+function normalizeKeylessIpv4(ip: string): string {
+ const trimmed = ip.trim();
+ const lower = trimmed.toLowerCase();
+ return lower.startsWith("::ffff:") && isIPv4(trimmed.slice(7))
+ ? trimmed.slice(7)
+ : trimmed;
+}
+
+export function keylessConversionCohort(ip: string): string | undefined {
+ const secret = config.KEYLESS_CONVERSION_HMAC_SECRET;
+ const normalizedIp = normalizeKeylessIpv4(ip);
+ if (!secret || !normalizedIp) return undefined;
+ return `${KEYLESS_CONVERSION_COHORT_VERSION}:${createHmac("sha256", secret)
+ .update(normalizedIp)
+ .digest("base64url")}`;
+}
+
+export function keylessExhaustionTelemetry(ip: string): Record<string, string> {
+ const conversionCohort = keylessConversionCohort(ip);
+ return conversionCohort ? { conversionCohort } : {};
+}
+
// Keyless teams reuse the `preview_` prefix so billing (autumn `isPreviewTeam`)
// and GCS persistence are skipped automatically, with a dedicated infix so the
// IP can be recovered when charging credits after a request completes.
@@ -80,8 +109,7 @@ export function keylessTeamUuid(
* is treated as IPv4.
*/
export function isKeylessIpEligible(ip: string): boolean {
- const normalized = ip.startsWith("::ffff:") ? ip.slice("::ffff:".length) : ip;
- return isIPv4(normalized);
+ return isIPv4(normalizeKeylessIpv4(ip));
}
const requestsKey = (ip: string) => `keyless_requests:${ip}`;
@@ -141,6 +169,7 @@ export async function keylessLimitBody(
reason: "credits",
mode,
retryAfterSeconds,
+ ...keylessExhaustionTelemetry(ip ?? ""),
});
return {
success: false,