Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Give the scrape billing path stable idempotency keys on the firebill route
Rakshith Ramprakash7201092dd15c
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.tsindex bf0598cd9..7c5bc469b 100644--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts@@ -715,6 +715,48 @@ describe("firebill routing", () => {expect(String(url)).toBe("http://proxy.test/firebill/v1/track");});+ it("sends a caller-supplied idempotency key to firebill, and omits the field otherwise", async () => {+ state.configRef = firebillConfig();+ const svc = makeService();++ await svc.trackCredits({+ teamId: "team-1",+ value: 5,+ properties: { source: "billScrapeJob", endpoint: "scrape" },+ idempotencyKey: "fc:track:scrape:job-123",+ });+ expect(JSON.parse(mockFetch.mock.calls[0]![1].body).idempotency_key).toBe(+ "fc:track:scrape:job-123",+ );++ await svc.trackCredits({+ teamId: "team-1",+ value: 5,+ properties: { source: "billTeam", endpoint: "search" },+ });+ expect(+ "idempotency_key" in JSON.parse(mockFetch.mock.calls[1]![1].body),+ ).toBe(false);+ });++ it("gives a refund its own key, distinct from the track's", async () => {+ state.configRef = firebillConfig();+ const svc = makeService();++ await svc.refundCredits({+ teamId: "team-1",+ value: 5,+ properties: { source: "billScrapeJob", endpoint: "scrape" },+ idempotencyKey: "fc:refund:scrape:job-123",+ });++ const [url, init] = mockFetch.mock.calls[0]!;+ expect(String(url)).toContain("/v1/refund");+ expect(JSON.parse(init.body).idempotency_key).toBe(+ "fc:refund:scrape:job-123",+ );+ });+it("routes trackCredits for an allowlisted org to firebill, not Autumn", async () => {state.configRef = firebillConfig();const svc = makeService();diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex 605f80b07..462f68e70 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -215,6 +215,7 @@ export class AutumnService {featureId,value,properties,+ idempotencyKey,}: TrackParams): Promise<boolean> {// Gradual firebill rollout: allowlisted orgs record usage via firebill (a// durable store that forwards to Autumn) instead of calling Autumn@@ -229,6 +230,7 @@ export class AutumnService {featureId,value,properties,+ idempotencyKey,});}@@ -516,6 +518,7 @@ export class AutumnService {value,properties,featureId = CREDITS_FEATURE_ID,+ idempotencyKey,}: TrackCreditsParams): Promise<boolean> {if (!autumnClient) return false;if (this.isPreviewTeam(teamId)) return false;@@ -528,6 +531,7 @@ export class AutumnService {featureId,value,properties,+ idempotencyKey,});} catch (error) {logger.error(@@ -693,6 +697,7 @@ export class AutumnService {value,properties,featureId = CREDITS_FEATURE_ID,+ idempotencyKey,}: TrackCreditsParams): Promise<void> {if (!autumnClient) return;if (this.isPreviewTeam(teamId)) return;@@ -705,6 +710,7 @@ export class AutumnService {featureId,value: -value,properties: { ...properties, source: "autumn_refund" },+ idempotencyKey,});} catch (error) {logger.error(diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.tsindex 044e19cdb..12714a17b 100644--- a/apps/api/src/services/autumn/firebill.ts+++ b/apps/api/src/services/autumn/firebill.ts@@ -53,6 +53,7 @@ export async function firebillTrack({featureId,value,properties,+ idempotencyKey,}: TrackParams): Promise<boolean> {const path = value < 0 ? "/v1/refund" : "/v1/track";// Plain concatenation rather than new URL(path, base): a leading-slash path@@ -74,6 +75,11 @@ export async function firebillTrack({feature_id: featureId,value: Math.abs(value),properties,+ // Stable per-charge key: firebill makes it the intent row's primary+ // key, so a caller retry (or a requeued job re-billing the same work)+ // is answered from the existing row instead of charged again. Omitted+ // β firebill mints a per-request UUID (dedupes only its own retries).+ ...(idempotencyKey ? { idempotency_key: idempotencyKey } : {}),}),signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),});diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.tsindex e1d4c272f..564c6b086 100644--- a/apps/api/src/services/autumn/types.ts+++ b/apps/api/src/services/autumn/types.ts@@ -23,6 +23,14 @@ export type TrackParams = {featureId: string;value: number;properties?: Record<string, unknown>;+ /**+ * Stable per-charge identity, honored on the firebill route only (the+ * direct Autumn SDK does not expose its Idempotency-Key header). When set,+ * a caller retry β or a requeued job re-billing the same work β dedupes+ * instead of double-billing. Must be unique per CHARGE, never a shared id+ * like a crawl id (every page shares it: collision = underbilling).+ */+ idempotencyKey?: string;};export type EnsureOrgProvisionedParams = {@@ -71,6 +79,8 @@ export type TrackCreditsParams = {value: number;properties?: Record<string, unknown>;featureId?: string;+ /** See TrackParams.idempotencyKey. */+ idempotencyKey?: string;};export type CreateEntityResult =diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex c57744744..3bc55ecf6 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -176,6 +176,12 @@ async function billScrapeJob(value: creditsToBeBilled,properties: autumnProperties,featureId,+ // The worker job id is the one identity that is unique per charge+ // (a crawl id is shared by every page β keying on it would collapse+ // a crawl's pages into one billed event) AND survives a stall+ // requeue, which re-runs the job under the same id: with this key,+ // the re-run dedupes instead of double-billing (firebill route).+ idempotencyKey: `fc:track:${billing.endpoint}:${job.id}`,});const billingJobId = uuidv7();logger.debug(@@ -225,6 +231,9 @@ async function billScrapeJob(value: creditsToBeBilled,properties: autumnProperties,featureId,+ // Distinct from the track key: a refund is its own charge event+ // (same key would 409 as a duplicate of the track and be dropped).+ idempotencyKey: `fc:refund:${billing.endpoint}:${job.id}`,});}// The billing operation never reached the queue, so no debit will