Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Give the scrape billing path stable idempotency keys on the firebill route

Rakshith Ramprakash7201092dd15c
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index bf0598cd9..7c5bc469b 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -715,6 +715,48 @@ describe("firebill routing", () => {
expect(String(url)).toBe("http://proxy.test/firebill/v1/track");
});
+ it("sends a caller-supplied idempotency key to firebill, and omits the field otherwise", async () => {
+ state.configRef = firebillConfig();
+ const svc = makeService();
+
+ await svc.trackCredits({
+ teamId: "team-1",
+ value: 5,
+ properties: { source: "billScrapeJob", endpoint: "scrape" },
+ idempotencyKey: "fc:track:scrape:job-123",
+ });
+ expect(JSON.parse(mockFetch.mock.calls[0]![1].body).idempotency_key).toBe(
+ "fc:track:scrape:job-123",
+ );
+
+ await svc.trackCredits({
+ teamId: "team-1",
+ value: 5,
+ properties: { source: "billTeam", endpoint: "search" },
+ });
+ expect(
+ "idempotency_key" in JSON.parse(mockFetch.mock.calls[1]![1].body),
+ ).toBe(false);
+ });
+
+ it("gives a refund its own key, distinct from the track's", async () => {
+ state.configRef = firebillConfig();
+ const svc = makeService();
+
+ await svc.refundCredits({
+ teamId: "team-1",
+ value: 5,
+ properties: { source: "billScrapeJob", endpoint: "scrape" },
+ idempotencyKey: "fc:refund:scrape:job-123",
+ });
+
+ const [url, init] = mockFetch.mock.calls[0]!;
+ expect(String(url)).toContain("/v1/refund");
+ expect(JSON.parse(init.body).idempotency_key).toBe(
+ "fc:refund:scrape:job-123",
+ );
+ });
+
it("routes trackCredits for an allowlisted org to firebill, not Autumn", async () => {
state.configRef = firebillConfig();
const svc = makeService();
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index 605f80b07..462f68e70 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -215,6 +215,7 @@ export class AutumnService {
featureId,
value,
properties,
+ idempotencyKey,
}: TrackParams): Promise<boolean> {
// Gradual firebill rollout: allowlisted orgs record usage via firebill (a
// durable store that forwards to Autumn) instead of calling Autumn
@@ -229,6 +230,7 @@ export class AutumnService {
featureId,
value,
properties,
+ idempotencyKey,
});
}
@@ -516,6 +518,7 @@ export class AutumnService {
value,
properties,
featureId = CREDITS_FEATURE_ID,
+ idempotencyKey,
}: TrackCreditsParams): Promise<boolean> {
if (!autumnClient) return false;
if (this.isPreviewTeam(teamId)) return false;
@@ -528,6 +531,7 @@ export class AutumnService {
featureId,
value,
properties,
+ idempotencyKey,
});
} catch (error) {
logger.error(
@@ -693,6 +697,7 @@ export class AutumnService {
value,
properties,
featureId = CREDITS_FEATURE_ID,
+ idempotencyKey,
}: TrackCreditsParams): Promise<void> {
if (!autumnClient) return;
if (this.isPreviewTeam(teamId)) return;
@@ -705,6 +710,7 @@ export class AutumnService {
featureId,
value: -value,
properties: { ...properties, source: "autumn_refund" },
+ idempotencyKey,
});
} catch (error) {
logger.error(
diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.ts
index 044e19cdb..12714a17b 100644
--- a/apps/api/src/services/autumn/firebill.ts
+++ b/apps/api/src/services/autumn/firebill.ts
@@ -53,6 +53,7 @@ export async function firebillTrack({
featureId,
value,
properties,
+ idempotencyKey,
}: TrackParams): Promise<boolean> {
const path = value < 0 ? "/v1/refund" : "/v1/track";
// Plain concatenation rather than new URL(path, base): a leading-slash path
@@ -74,6 +75,11 @@ export async function firebillTrack({
feature_id: featureId,
value: Math.abs(value),
properties,
+ // Stable per-charge key: firebill makes it the intent row's primary
+ // key, so a caller retry (or a requeued job re-billing the same work)
+ // is answered from the existing row instead of charged again. Omitted
+ // β†’ firebill mints a per-request UUID (dedupes only its own retries).
+ ...(idempotencyKey ? { idempotency_key: idempotencyKey } : {}),
}),
signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),
});
diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.ts
index e1d4c272f..564c6b086 100644
--- a/apps/api/src/services/autumn/types.ts
+++ b/apps/api/src/services/autumn/types.ts
@@ -23,6 +23,14 @@ export type TrackParams = {
featureId: string;
value: number;
properties?: Record<string, unknown>;
+ /**
+ * Stable per-charge identity, honored on the firebill route only (the
+ * direct Autumn SDK does not expose its Idempotency-Key header). When set,
+ * a caller retry β€” or a requeued job re-billing the same work β€” dedupes
+ * instead of double-billing. Must be unique per CHARGE, never a shared id
+ * like a crawl id (every page shares it: collision = underbilling).
+ */
+ idempotencyKey?: string;
};
export type EnsureOrgProvisionedParams = {
@@ -71,6 +79,8 @@ export type TrackCreditsParams = {
value: number;
properties?: Record<string, unknown>;
featureId?: string;
+ /** See TrackParams.idempotencyKey. */
+ idempotencyKey?: string;
};
export type CreateEntityResult =
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index c57744744..3bc55ecf6 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -176,6 +176,12 @@ async function billScrapeJob(
value: creditsToBeBilled,
properties: autumnProperties,
featureId,
+ // The worker job id is the one identity that is unique per charge
+ // (a crawl id is shared by every page β€” keying on it would collapse
+ // a crawl's pages into one billed event) AND survives a stall
+ // requeue, which re-runs the job under the same id: with this key,
+ // the re-run dedupes instead of double-billing (firebill route).
+ idempotencyKey: `fc:track:${billing.endpoint}:${job.id}`,
});
const billingJobId = uuidv7();
logger.debug(
@@ -225,6 +231,9 @@ async function billScrapeJob(
value: creditsToBeBilled,
properties: autumnProperties,
featureId,
+ // Distinct from the track key: a refund is its own charge event
+ // (same key would 409 as a duplicate of the track and be dropped).
+ idempotencyKey: `fc:refund:${billing.endpoint}:${job.id}`,
});
}
// The billing operation never reached the queue, so no debit will