Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): carry the caller's external request id on every charge (#4735)
Gergő Móricz9062637a783e
diff --git a/apps/api/src/__tests__/controllers/fireclaw.test.ts b/apps/api/src/__tests__/controllers/fireclaw.test.tsindex 6471f8ea5..824d1cabd 100644--- a/apps/api/src/__tests__/controllers/fireclaw.test.ts+++ b/apps/api/src/__tests__/controllers/fireclaw.test.ts@@ -44,6 +44,7 @@ const getACUCTeamMock = getACUCTeam as MockedFunction<typeof getACUCTeam>;function buildReq(overrides: any = {}): any {return {body: { plays: 1 },+ headers: {},auth: { team_id: "team_test", org_id: "org_test" },// A real org: it is what gates the Autumn credit check, so the billable// path is the one under test here.@@ -114,7 +115,11 @@ describe("fireclawController credit gating (Autumn)", () => {"org_test",200,1,- expect.objectContaining({ endpoint: "fireclaw" }),+ // No External-Request-Id header: the charge carries none.+ expect.objectContaining({+ endpoint: "fireclaw",+ externalRequestId: null,+ }),);expect(res.statusCode).toBe(200);expect(res.body).toEqual(@@ -127,6 +132,28 @@ describe("fireclawController credit gating (Autumn)", () => {);});+ it("carries the caller's External-Request-Id on the charge", async () => {+ const req = buildReq({+ body: { plays: 1 },+ headers: { "external-request-id": "partner-op-42" },+ });+ const res = buildRes();+ checkCreditsMock.mockResolvedValue({ allowed: true, remaining: 5000 });++ await fireclawController(req, res);++ expect(billTeamMock).toHaveBeenCalledWith(+ "team_test",+ "org_test",+ 100,+ 1,+ expect.objectContaining({+ endpoint: "fireclaw",+ externalRequestId: "partner-op-42",+ }),+ );+ });+it("bills against the fallback ACUC's org when req.acuc is absent", async () => {const req = buildReq({ acuc: undefined });const res = buildRes();diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex bb189b483..eb9818ff3 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -302,7 +302,11 @@ export async function crawlController(req: Request, res: Response) {org_id: orgId,basePriority: 21,});- const billing = { endpoint: "crawl" as const, jobId: id };+ const billing = {+ endpoint: "crawl" as const,+ jobId: id,+ externalRequestId: externalRequestId(req),+ };const jobs = urls.map(url => {const uuid = uuidv7();return {@@ -359,7 +363,11 @@ export async function crawlController(req: Request, res: Response) {team_id,origin: req.body.origin ?? defaultOrigin,integration: req.body.integration,- billing: { endpoint: "crawl", jobId: id },+ billing: {+ endpoint: "crawl",+ jobId: id,+ externalRequestId: externalRequestId(req),+ },crawl_id: id,zeroDataRetention: false, // not supported on v0apiKeyId: chunk?.api_key_id ?? null,diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex f0be7166f..90319d219 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -109,7 +109,11 @@ async function scrapeHelper(internalOptions,origin: req.body.origin ?? defaultOrigin,integration: req.body.integration,- billing: { endpoint: "scrape", jobId },+ billing: {+ endpoint: "scrape",+ jobId,+ externalRequestId: externalRequestId(req),+ },startTime: Date.now(),zeroDataRetention: false, // not supported on v0apiKeyId,diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex c924b1d6a..506ce7f2e 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -95,7 +95,12 @@ async function searchHelper(org_id,searchCredits,api_key_id,- { endpoint: "search", jobId, chargeId: jobId },+ {+ endpoint: "search",+ jobId,+ chargeId: jobId,+ externalRequestId: externalRequestId(req),+ },logger,).catch(error => {logger.error(@@ -127,7 +132,11 @@ async function searchHelper(org_id,basePriority: 20,});- const billing = { endpoint: "search" as const, jobId };+ const billing = {+ endpoint: "search" as const,+ jobId,+ externalRequestId: externalRequestId(req),+ };// filter out social media linksdiff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 8911b5d79..0b584ce6f 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -235,6 +235,7 @@ export async function batchScrapeController(req.acuc?.api_key_id ?? null,{endpoint: "batch_scrape",+ externalRequestId: externalRequestId(req),jobId: id,// Appends reuse the batch id but each append's threat scans are a// fresh charge — a shared key would underbill them. Appends stay@@ -406,7 +407,11 @@ export async function batchScrapeController(});}logger.debug("Using job priority " + jobPriority, { jobPriority });- const billing = { endpoint: "batch_scrape" as const, jobId: id };+ const billing = {+ endpoint: "batch_scrape" as const,+ jobId: id,+ externalRequestId: externalRequestId(req),+ };const jobs = urls.map(x => ({jobId: uuidv7(),diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex b109a1fb7..74fe3eea4 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -125,7 +125,7 @@ export async function crawlController(// No chargeId: a fresh crawl id is minted per request and the// rejected crawl is never persisted or queued, so there is no// stable per-charge identity that could dedupe a retry.- { endpoint: "crawl" },+ { endpoint: "crawl", externalRequestId: externalRequestId(req) },).catch(error => {_logger.error(`Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,@@ -333,7 +333,11 @@ export async function crawlController(internalOptions: sc.internalOptions,origin: req.body.origin,integration: req.body.integration,- billing: { endpoint: "crawl", jobId: id },+ billing: {+ endpoint: "crawl",+ jobId: id,+ externalRequestId: externalRequestId(req),+ },crawl_id: id,webhook: req.body.webhook,v1: true,diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 0c0668cbf..5250c1940 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -196,6 +196,7 @@ export async function extractController(req.acuc?.api_key_id ?? null,{endpoint: "extract",+ externalRequestId: externalRequestId(req),// Suffixed: the extract's MAIN charge (fire-0) uses the bare// extractId — a shared key would collapse the two into one charge.chargeId: `${extractId}:threat`,diff --git a/apps/api/src/controllers/v1/fireclaw.ts b/apps/api/src/controllers/v1/fireclaw.tsindex 9fadaccd3..969042b39 100644--- a/apps/api/src/controllers/v1/fireclaw.ts+++ b/apps/api/src/controllers/v1/fireclaw.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { externalRequestId } from "../../lib/external-request-id";import { RequestWithAuth } from "./types";import { billTeam } from "../../services/billing/credit_billing";import { getACUCTeam } from "../auth";@@ -81,7 +82,7 @@ export async function fireclawController(// No chargeId: fireclaw has no per-charge identity to key on — a// server-minted UUID here would be equivalent to firebill's own// per-request key, so this stays keyless until fireclaw carries one.- { endpoint: "fireclaw" },+ { endpoint: "fireclaw", externalRequestId: externalRequestId(req) },);} catch (error) {logger.error(`Fireclaw billing failed for team ${req.auth.team_id}`, {diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex d526dcfa6..c093aec49 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -562,6 +562,7 @@ export async function mapController(req.acuc?.api_key_id ?? null,{endpoint: "map",+ externalRequestId: externalRequestId(req),jobId: mapId,chargeId: mapId,},diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 7b4864ed6..5aa14a33d 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -306,7 +306,11 @@ async function scrapeControllerInner(skipNuq: true,origin,integration: req.body.integration,- billing: { endpoint: "scrape", jobId },+ billing: {+ endpoint: "scrape",+ jobId,+ externalRequestId: externalRequestId(req),+ },startTime: controllerStartTime,zeroDataRetention: zeroDataRetention ?? false,apiKeyId: req.acuc?.api_key_id ?? null,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 66b0c81b6..ed079cd2c 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -336,7 +336,12 @@ export async function searchController(req.acuc?.org_id ?? null,result.searchCredits,req.acuc?.api_key_id ?? null,- { endpoint: "search", jobId, chargeId: jobId },+ {+ endpoint: "search",+ jobId,+ chargeId: jobId,+ externalRequestId: externalRequestId(req),+ },).catch(error => {logger.error(`Failed to bill team ${req.auth.team_id} for ${result.searchCredits} credits: ${error}`,diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 3aef7f543..b3da3c831 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -389,7 +389,11 @@ export async function crawlController(internalOptions: sc.internalOptions,origin: req.body.origin,integration: req.body.integration,- billing: { endpoint: "crawl", jobId: id },+ billing: {+ endpoint: "crawl",+ jobId: id,+ externalRequestId: externalRequestId(req),+ },crawl_id: id,webhook: req.body.webhook,v1: true,diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex 37189c0ce..af716dd90 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -136,6 +136,7 @@ export async function extractController(req.acuc?.api_key_id ?? null,{endpoint: "extract",+ externalRequestId: externalRequestId(req),jobId: extractId,// Suffixed: the extract's MAIN charge (fire-0) uses the bare// extractId — a shared key would collapse the two into one charge.diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 7e05d3d22..363ed14e2 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -148,6 +148,7 @@ export async function mapController(// Suffixed so this early-return path can never collide with the main// map charge below, even if both ever billed the same mapId.chargeId: `${mapId}:avgrab`,+ externalRequestId: externalRequestId(req),},).catch(error => {logger.error(@@ -288,6 +289,7 @@ export async function mapController(req.acuc?.api_key_id ?? null,{endpoint: "map",+ externalRequestId: externalRequestId(req),jobId: mapId,chargeId: mapId,},diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 709df99a9..c70846f03 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -364,9 +364,12 @@ export async function parseController(const zeroDataRetention =getScrapeZDR(req.acuc?.flags) === "forced" ||(req.body.zeroDataRetention ?? false);- const billing: BillingMetadata = req.body.__agentInterop- ? { endpoint: "agent" as const, jobId }- : { endpoint: "parse" as const, jobId };+ const billing: BillingMetadata = {+ ...(req.body.__agentInterop+ ? { endpoint: "agent" as const, jobId }+ : { endpoint: "parse" as const, jobId }),+ externalRequestId: externalRequestId(req),+ };if (req.body.__agentInterop &&diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 16b346f5a..11a1b7365 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -567,6 +567,9 @@ export async function scrapeStopInteractiveBrowserController(req.acuc?.api_key_id ?? null,{endpoint: "interact",+ // Not set from this request: the stop request is not the one that+ // created the session, so its header would misattribute the charge.+ // firebill finds the creating request's id by the session id.jobId: session.id,chargeId: `${session.id}:scrape-browser`,},diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 0c3aa1eb5..83af538d8 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -217,9 +217,12 @@ export async function scrapeController(success: false,error: "Provider discovery does not support zero data retention.",});- const billing: BillingMetadata = req.body.__agentInterop- ? { endpoint: "agent" as const, jobId }- : { endpoint: "scrape" as const, jobId };+ const billing: BillingMetadata = {+ ...(req.body.__agentInterop+ ? { endpoint: "agent" as const, jobId }+ : { endpoint: "scrape" as const, jobId }),+ externalRequestId: externalRequestId(req),+ };if (req.body.__agentInterop &&diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 54bb66e5c..656d6e6c3 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -281,9 +281,12 @@ async function searchControllerInner(const shouldBill = req.body.__agentInterop?.shouldBill ?? true;const agentRequestId = req.body.__agentInterop?.requestId ?? null;- const billing: BillingMetadata = req.body.__agentInterop- ? { endpoint: "agent" as const, jobId }- : { endpoint: "search" as const, jobId };+ const billing: BillingMetadata = {+ ...(req.body.__agentInterop+ ? { endpoint: "agent" as const, jobId }+ : { endpoint: "search" as const, jobId }),+ externalRequestId: externalRequestId(req),+ };logger = logger.child({version: "v2",diff --git a/apps/api/src/services/autumn/__tests__/firebill.test.ts b/apps/api/src/services/autumn/__tests__/firebill.test.tsindex 1a3085a30..e97975e46 100644--- a/apps/api/src/services/autumn/__tests__/firebill.test.ts+++ b/apps/api/src/services/autumn/__tests__/firebill.test.ts@@ -145,6 +145,24 @@ const failureCauses = async () =>);describe("firebillTrack", () => {+ it("sends the caller's external request id with the charge, and no field without one", async () => {+ const fetchMock = vi.fn().mockImplementation(async () => ok());+ vi.stubGlobal("fetch", fetchMock);++ await firebillTrack({ ...params, externalRequestId: "partner-op-42" });+ expect(+ JSON.parse(fetchMock.mock.calls[0][1].body).external_request_id,+ ).toBe("partner-op-42");++ await firebillTrack(params);+ expect(+ Object.hasOwn(+ JSON.parse(fetchMock.mock.calls[1][1].body),+ "external_request_id",+ ),+ ).toBe(false);+ });+it("retries a refusal and reports success if the retry lands", async () => {const fetchMock = vi.fn()diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex 1ad5109b2..0bf1da9be 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -312,6 +312,7 @@ export class AutumnService {value,properties,idempotencyKey,+ externalRequestId,}: TrackParams,routed: boolean,): Promise<boolean> {@@ -328,6 +329,7 @@ export class AutumnService {value,properties,idempotencyKey,+ externalRequestId,});}@@ -772,6 +774,7 @@ export class AutumnService {properties,featureId = CREDITS_FEATURE_ID,idempotencyKey,+ externalRequestId,orgId,}: TrackCreditsParams): Promise<boolean> {if (!autumnClient) return false;@@ -790,6 +793,7 @@ export class AutumnService {value,properties,idempotencyKey,+ externalRequestId,},routed,);@@ -1025,6 +1029,7 @@ export class AutumnService {properties,featureId = CREDITS_FEATURE_ID,idempotencyKey,+ externalRequestId,orgId,}: TrackCreditsParams): Promise<void> {if (!autumnClient) return;@@ -1043,6 +1048,7 @@ export class AutumnService {value: -value,properties: { ...properties, source: "autumn_refund" },idempotencyKey,+ externalRequestId,},routed,);diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.tsindex 5c1580a68..2446aaa20 100644--- a/apps/api/src/services/autumn/firebill.ts+++ b/apps/api/src/services/autumn/firebill.ts@@ -398,6 +398,7 @@ async function firebillAttempt(value,properties,idempotencyKey,+ externalRequestId,}: TrackParams,): Promise<AttemptResult> {const url = firebillUrl(path);@@ -422,6 +423,12 @@ async function firebillAttempt(// than charged twice. Omitted → firebill mints a per-request UUID,// which dedupes only its own retries....(idempotencyKey ? { idempotency_key: idempotencyKey } : {}),+ // The partner's own operation id. With it on the charge, firebill+ // reports the operation without looking the request up in+ // ClickHouse, which lands seconds after the charge.+ ...(externalRequestId+ ? { external_request_id: externalRequestId }+ : {}),}),signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),});diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.tsindex c6c19642c..8b5946302 100644--- a/apps/api/src/services/autumn/types.ts+++ b/apps/api/src/services/autumn/types.ts@@ -25,6 +25,12 @@ export type TrackParams = {featureId: string;value: number;properties?: Record<string, unknown>;+ /**+ * The partner's own operation id (the caller's `External-Request-Id`).+ * firebill route only: sent as `external_request_id`, so firebill reports+ * the operation under it without a lookup. Opaque, forwarded verbatim.+ */+ externalRequestId?: string | null;/*** Stable per-charge identity, honored on the firebill route only (the* direct Autumn SDK does not expose its Idempotency-Key header). When set,@@ -114,6 +120,8 @@ export type TrackCreditsParams = {featureId?: string;/** See TrackParams.idempotencyKey. */idempotencyKey?: string;+ /** See TrackParams.externalRequestId. */+ externalRequestId?: string | null;/*** The team's org — the Autumn customer this usage bills against. The service* never looks it up: callers hold it already (a request's ACUC, a job'sdiff --git a/apps/api/src/services/billing/__tests__/credit_billing.test.ts b/apps/api/src/services/billing/__tests__/credit_billing.test.tsindex 5e5b730dd..22f1cb155 100644--- a/apps/api/src/services/billing/__tests__/credit_billing.test.ts+++ b/apps/api/src/services/billing/__tests__/credit_billing.test.ts@@ -61,6 +61,27 @@ beforeEach(() => {});describe("billTeam", () => {+ it("carries the caller's external request id on the charge, and omits it without one", async () => {+ await billTeam("team-1", "org-1", 3, 123, {+ endpoint: "scrape",+ jobId: "job-9",+ chargeId: "job-9",+ externalRequestId: "partner-op-42",+ });+ expect(trackCredits).toHaveBeenLastCalledWith(+ expect.objectContaining({ externalRequestId: "partner-op-42" }),+ );++ await billTeam("team-1", "org-1", 3, 123, {+ endpoint: "scrape",+ jobId: "job-9",+ externalRequestId: null,+ });+ expect(trackCredits).toHaveBeenLastCalledWith(+ expect.objectContaining({ externalRequestId: undefined }),+ );+ });+it("derives firebill idempotency keys from chargeId, and omits them without one", async () => {await billTeam("team-1", "org-1", 3, 123, {endpoint: "search",@@ -108,10 +129,14 @@ describe("billTeam", () => {endpoint: "map",jobId: "map-1",chargeId: "map-1",+ externalRequestId: "partner-op-42",});expect(refundCredits).toHaveBeenCalledWith(- expect.objectContaining({ idempotencyKey: "fc:refund:map:map-1" }),+ expect.objectContaining({+ idempotencyKey: "fc:refund:map:map-1",+ externalRequestId: "partner-op-42",+ }),);});diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex aacbc8030..a8d54da0a 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -52,6 +52,7 @@ export async function billTeam(idempotencyKey: billing.chargeId? `fc:track:${billing.endpoint}:${billing.chargeId}`: undefined,+ externalRequestId: billing.externalRequestId ?? undefined,});} else if (team_id !== "preview" && !team_id.startsWith("preview_")) {// Preview teams are never tracked anyway; a real team arriving without@@ -95,6 +96,7 @@ export async function billTeam(idempotencyKey: billing.chargeId? `fc:refund:${billing.endpoint}:${billing.chargeId}`: undefined,+ externalRequestId: billing.externalRequestId ?? undefined,});}}diff --git a/apps/api/src/services/billing/types.ts b/apps/api/src/services/billing/types.tsindex 4062dd183..2b078a1fe 100644--- a/apps/api/src/services/billing/types.ts+++ b/apps/api/src/services/billing/types.ts@@ -31,6 +31,14 @@ export type BillingMetadata = {* per-request UUID, which dedupes only firebill's own retries.*/chargeId?: string;+ /**+ * The caller's own id for the operation, from its `External-Request-Id`+ * header — the value `logRequest` records. Carried on the charge so+ * firebill can report the operation to the partner without looking the+ * request up in ClickHouse, which lands a few seconds after the charge.+ * Unset when the caller sent no header or the charge has no request.+ */+ externalRequestId?: string | null;};export function resolveBillingMetadata({diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 9817cf47c..1d6392bc1 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -218,6 +218,9 @@ async function billScrapeJob(// requeue, which re-runs the job under the same id: with this key,// the re-run dedupes instead of double-billing (firebill route).idempotencyKey: `fc:track:${billing.endpoint}:${job.id}`,+ // The caller's own operation id, carried on the charge so+ // firebill reports it without a request lookup.+ externalRequestId: billing.externalRequestId ?? undefined,}): false;// On the firebill route the ledger enqueue must be idempotent by the@@ -321,6 +324,7 @@ async function billScrapeJob(// Distinct from the track key: a refund is its own charge event// (same key would 409 as a duplicate of the track and be dropped).idempotencyKey: `fc:refund:${billing.endpoint}:${job.id}`,+ externalRequestId: billing.externalRequestId ?? undefined,});}}