Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Handle a firebill refusal, and ramp by percentage instead of by allowlist (#4341)

Rakshith Ramprakashf259e9ca86b0
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index fdb5d2b37..22a7fe05f 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -405,6 +405,12 @@ const configSchema = z.object({
FIREBILL_URL: emptyStringAsUndefined(z.string().url()),
FIREBILL_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),
FIREBILL_ORG_IDS: delimitedList(",").optional(),
+ // Sticky percentage ramp, on top of the allowlist above. The bucket is a
+ // hash of the org id, so an org that is in at 5 is still in at 30 β€” a ramp
+ // only ever adds, and never reshuffles who is on which path mid-rollout.
+ // 0 (the default) is also the kill switch: the allowlist still routes, and
+ // nothing else does.
+ FIREBILL_ROLLOUT_PERCENT: z.coerce.number().min(0).max(100).default(0),
// Miscellaneous
IDMUX_URL: z.string().optional(),
diff --git a/apps/api/src/lib/__tests__/rollout.test.ts b/apps/api/src/lib/__tests__/rollout.test.ts
new file mode 100644
index 000000000..c6d120433
--- /dev/null
+++ b/apps/api/src/lib/__tests__/rollout.test.ts
@@ -0,0 +1,40 @@
+import { describe, expect, it } from "vitest";
+import { sampled } from "../rollout";
+
+const orgs = Array.from({ length: 2000 }, (_, i) => `org-${i}`);
+
+describe("sampled", () => {
+ it("is off at 0 and on at 100", () => {
+ expect(orgs.some(o => sampled(o, 0))).toBe(false);
+ expect(orgs.every(o => sampled(o, 100))).toBe(true);
+ });
+
+ it("only ever adds as the percentage rises", () => {
+ // The property the whole ramp rests on: nobody is moved back off firebill
+ // by a later step, so each stage is a superset of the one before it.
+ const stages = [5, 10, 30, 60, 80, 100];
+ for (const org of orgs) {
+ let wasIn = false;
+ for (const percent of stages) {
+ const isIn = sampled(org, percent);
+ expect(!wasIn || isIn).toBe(true);
+ wasIn = isIn;
+ }
+ }
+ });
+
+ it("gives the same answer every call", () => {
+ for (const org of orgs.slice(0, 50)) {
+ const first = sampled(org, 37);
+ for (let i = 0; i < 5; i++) expect(sampled(org, 37)).toBe(first);
+ }
+ });
+
+ it("lands near the requested share", () => {
+ for (const percent of [5, 30, 60]) {
+ const hits = orgs.filter(o => sampled(o, percent)).length;
+ const share = (hits / orgs.length) * 100;
+ expect(Math.abs(share - percent)).toBeLessThan(5);
+ }
+ });
+});
diff --git a/apps/api/src/lib/rollout.ts b/apps/api/src/lib/rollout.ts
new file mode 100644
index 000000000..d604bb58e
--- /dev/null
+++ b/apps/api/src/lib/rollout.ts
@@ -0,0 +1,23 @@
+/**
+ * Deterministic percentage sampling for gradual rollouts.
+ *
+ * The bucket is a pure function of the cohort key, so a cohort that is in at
+ * 5% is still in at 30% β€” a ramp only ever adds, never reshuffles. That is the
+ * property `Math.random() < percent` does not have, and the reason to prefer
+ * this wherever a cohort should stay on one side of a rollout: billing routes,
+ * anything a support question could be asked about, anything you want to
+ * compare before and after.
+ *
+ * FNV-1a over the key, mapped onto [0, 100).
+ */
+export function sampled(cohortKey: string, percent: number): boolean {
+ if (percent <= 0) return false;
+ if (percent >= 100) return true;
+
+ let hash = 2166136261;
+ for (let i = 0; i < cohortKey.length; i++) {
+ hash ^= cohortKey.charCodeAt(i);
+ hash = Math.imul(hash, 16777619);
+ }
+ return ((hash >>> 0) / 0x1_0000_0000) * 100 < percent;
+}
diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts
index 1dd158c84..80f3bced1 100644
--- a/apps/api/src/search/highlights.ts
+++ b/apps/api/src/search/highlights.ts
@@ -9,6 +9,7 @@ import {
import type { HighlightFailureReason } from "./highlight-model";
import { config } from "../config";
import { logger as rootLogger } from "../lib/logger";
+import { sampled } from "../lib/rollout";
// How far back into the index we're willing to reach for highlight source text.
const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
@@ -28,18 +29,6 @@ export function highlightsEnvReady(): boolean {
);
}
-function sampled(cohortKey: string, percent: number): boolean {
- if (percent <= 0) return false;
- if (percent >= 100) return true;
-
- let hash = 2166136261;
- for (let i = 0; i < cohortKey.length; i++) {
- hash ^= cohortKey.charCodeAt(i);
- hash = Math.imul(hash, 16777619);
- }
- return ((hash >>> 0) / 0x1_0000_0000) * 100 < percent;
-}
-
export function searchHighlightsMode(options: {
requested?: boolean;
origin?: string | null;
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index ea5b15aa1..c260330fc 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -715,7 +715,7 @@ describe("firebill routing", () => {
expect(String(url)).toBe("http://proxy.test/firebill/v1/track");
});
- it("sends a caller-supplied idempotency key to firebill, and omits the field otherwise", async () => {
+ it("sends a caller-supplied idempotency key to firebill, and mints one otherwise", async () => {
state.configRef = firebillConfig();
const svc = makeService();
@@ -734,9 +734,14 @@ describe("firebill routing", () => {
value: 5,
properties: { source: "billTeam", endpoint: "search" },
});
- expect(
- "idempotency_key" in JSON.parse(mockFetch.mock.calls[1]![1].body),
- ).toBe(false);
+ // A key is always sent, even when the caller has none: firebillTrack retries
+ // once, and both attempts have to be the same event or an accepted-but-lost
+ // first attempt would be billed twice. firebill would have minted an
+ // equivalent UUID itself; minting it here is what makes it stable across
+ // the retry.
+ const minted = JSON.parse(mockFetch.mock.calls[1]![1].body).idempotency_key;
+ expect(minted).toEqual(expect.any(String));
+ expect(minted).not.toBe("fc:track:scrape:job-123");
});
it("a track/refund pair for one charge carries two distinct keys", async () => {
@@ -794,6 +799,9 @@ describe("firebill routing", () => {
feature_id: "CREDITS",
value: 42,
properties: { source: "test", endpoint: "extract" },
+ // Minted when the caller has none, so the retry inside firebillTrack is
+ // the same event rather than a second charge.
+ idempotency_key: expect.any(String),
});
});
diff --git a/apps/api/src/services/autumn/__tests__/firebill.test.ts b/apps/api/src/services/autumn/__tests__/firebill.test.ts
new file mode 100644
index 000000000..e6081886a
--- /dev/null
+++ b/apps/api/src/services/autumn/__tests__/firebill.test.ts
@@ -0,0 +1,168 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { firebillTrack, shouldRouteToFirebill } from "../firebill";
+import { firebillRetryTotal, firebillTrackTotal } from "../metrics";
+
+const { configState } = vi.hoisted(() => ({
+ configState: {
+ FIREBILL_URL: "https://firebill.test",
+ FIREBILL_SECRET: "secret",
+ FIREBILL_ORG_IDS: ["allow-listed-org"] as string[] | undefined,
+ FIREBILL_ROLLOUT_PERCENT: 0,
+ },
+}));
+
+vi.mock("../../../config", () => ({ config: configState }));
+vi.mock("../../../lib/logger", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+// Built fresh per call: a Response body can only be read once, so a shared
+// instance would make the second attempt look like a transport failure.
+const ok = () =>
+ new Response(JSON.stringify({ success: true }), { status: 200 });
+const refused = () =>
+ new Response(JSON.stringify({ success: false }), { status: 200 });
+
+const params = {
+ customerId: "org-1",
+ entityId: "team-1",
+ featureId: "CREDITS",
+ value: 3,
+ properties: {},
+ idempotencyKey: "fc:track:scrape:job-1",
+};
+
+beforeEach(() => {
+ configState.FIREBILL_URL = "https://firebill.test";
+ configState.FIREBILL_SECRET = "secret";
+ configState.FIREBILL_ORG_IDS = ["allow-listed-org"];
+ configState.FIREBILL_ROLLOUT_PERCENT = 0;
+ firebillTrackTotal.reset();
+ firebillRetryTotal.reset();
+});
+afterEach(() => vi.unstubAllGlobals());
+
+describe("shouldRouteToFirebill", () => {
+ it("routes the allowlist even at 0 percent", () => {
+ expect(shouldRouteToFirebill("allow-listed-org")).toBe(true);
+ });
+
+ it("routes nobody else at 0 percent β€” the kill switch", () => {
+ const others = Array.from({ length: 200 }, (_, i) => `other-${i}`);
+ expect(others.some(shouldRouteToFirebill)).toBe(false);
+ });
+
+ it("routes roughly the configured share once ramped", () => {
+ const orgs = Array.from({ length: 1000 }, (_, i) => `ramp-${i}`);
+ configState.FIREBILL_ROLLOUT_PERCENT = 30;
+ const share =
+ (orgs.filter(shouldRouteToFirebill).length / orgs.length) * 100;
+ expect(Math.abs(share - 30)).toBeLessThan(6);
+ });
+
+ it("stays off entirely when firebill is not configured", () => {
+ configState.FIREBILL_ROLLOUT_PERCENT = 100;
+ configState.FIREBILL_SECRET = "";
+ expect(shouldRouteToFirebill("allow-listed-org")).toBe(false);
+ });
+});
+
+describe("firebillTrack", () => {
+ it("retries a refusal and reports success if the retry lands", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockImplementationOnce(async () => refused())
+ .mockImplementationOnce(async () => ok());
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(firebillTrack(params)).resolves.toBe(true);
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ // Same key on both attempts, which is what makes the retry safe.
+ const keys = fetchMock.mock.calls.map(
+ c => JSON.parse(c[1].body).idempotency_key,
+ );
+ expect(keys).toEqual([params.idempotencyKey, params.idempotencyKey]);
+ });
+
+ it("gives up after the attempt limit and reports false", async () => {
+ const fetchMock = vi.fn().mockImplementation(async () => refused());
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(firebillTrack(params)).resolves.toBe(false);
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("reuses one key across attempts even when the caller supplies none", async () => {
+ // Without this the retry is a second event: firebill mints its own key per
+ // request, so an accepted-but-lost first attempt would be billed twice.
+ const fetchMock = vi
+ .fn()
+ .mockRejectedValueOnce(new Error("timeout"))
+ .mockImplementationOnce(async () => ok());
+ vi.stubGlobal("fetch", fetchMock);
+
+ await firebillTrack({ ...params, idempotencyKey: undefined });
+
+ const keys = fetchMock.mock.calls.map(
+ c => JSON.parse(c[1].body).idempotency_key,
+ );
+ expect(keys[0]).toBeTruthy();
+ expect(keys[0]).toBe(keys[1]);
+ });
+
+ it("mints a different key for each keyless call", async () => {
+ const fetchMock = vi.fn().mockImplementation(async () => ok());
+ vi.stubGlobal("fetch", fetchMock);
+
+ await firebillTrack({ ...params, idempotencyKey: undefined });
+ await firebillTrack({ ...params, idempotencyKey: undefined });
+
+ const [first, second] = fetchMock.mock.calls.map(
+ c => JSON.parse(c[1].body).idempotency_key,
+ );
+ expect(first).not.toBe(second);
+ });
+
+ it("separates an explicit refusal from an ambiguous transport failure", async () => {
+ const outcomes = async () =>
+ Object.fromEntries(
+ (await firebillTrackTotal.get()).values.map(v => [
+ v.labels.outcome,
+ v.value,
+ ]),
+ );
+
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockImplementation(async () => refused()),
+ );
+ await firebillTrack(params);
+ expect(await outcomes()).toEqual({ refused: 1 });
+
+ firebillTrackTotal.reset();
+ vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("timeout")));
+ await firebillTrack(params);
+ // A timeout is not proof the usage was lost β€” firebill may have taken it.
+ expect(await outcomes()).toEqual({ ambiguous: 1 });
+ });
+
+ it("retries a thrown error too", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockRejectedValueOnce(new Error("connection refused"))
+ .mockImplementationOnce(async () => ok());
+ vi.stubGlobal("fetch", fetchMock);
+
+ await expect(firebillTrack(params)).resolves.toBe(true);
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ });
+
+ it("sends a refund to /v1/refund with the value made positive", async () => {
+ const fetchMock = vi.fn().mockImplementation(async () => ok());
+ vi.stubGlobal("fetch", fetchMock);
+
+ await firebillTrack({ ...params, value: -7 });
+ expect(fetchMock.mock.calls[0][0]).toBe("https://firebill.test/v1/refund");
+ expect(JSON.parse(fetchMock.mock.calls[0][1].body).value).toBe(7);
+ });
+});
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index 4e751bc70..a26b7efcc 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -10,6 +10,7 @@ import {
firebillTrack,
shouldRouteToFirebill,
} from "./firebill";
+import { billingRouteTotal } from "./metrics";
import type {
CreateEntityParams,
CreateEntityResult,
@@ -238,13 +239,12 @@ export class AutumnService {
properties,
idempotencyKey,
}: TrackParams): Promise<boolean> {
- // Gradual firebill rollout: allowlisted orgs record usage via firebill (a
- // durable store that forwards to Autumn) instead of calling Autumn
- // directly. If the firebill call fails we must NOT fall back to Autumn β€”
- // firebill may have durably recorded the event and will deliver it later,
- // so a direct-Autumn fallback could double-bill the customer. firebillTrack
- // returns false on failure, exactly like an Autumn track failure.
+ // Gradual rollout: allowlisted orgs, and those in the sticky
+ // FIREBILL_ROLLOUT_PERCENT bucket, bill through firebill. No fallback to
+ // Autumn on failure β€” firebill may already own the event, and the SDK sends
+ // no idempotency key, so the pair could not be deduped.
if (shouldRouteToFirebill(customerId)) {
+ billingRouteTotal.labels("firebill").inc();
return await firebillTrack({
customerId,
entityId,
@@ -255,6 +255,8 @@ export class AutumnService {
});
}
+ billingRouteTotal.labels("direct").inc();
+
if (!autumnClient) return false;
try {
diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.ts
index 79ef008da..18dadeeed 100644
--- a/apps/api/src/services/autumn/firebill.ts
+++ b/apps/api/src/services/autumn/firebill.ts
@@ -1,6 +1,9 @@
+import { randomUUID } from "crypto";
import { config } from "../../config";
import { logger } from "../../lib/logger";
+import { sampled } from "../../lib/rollout";
import type { TrackParams } from "./types";
+import { firebillRetryTotal, firebillTrackTotal } from "./metrics";
/**
* Outcome of a firebill `/v1/lock` call, mirroring firebill's three-state
@@ -18,6 +21,13 @@ type FirebillLockResult =
// direct Autumn client.
const FIREBILL_TIMEOUT_MS = 5000;
+// Safe to retry because the idempotency key is stable across attempts: if the
+// first attempt did land (ambiguous confirm timeout), Autumn dedupes the second.
+// Small because a caller may be waiting, and a firebill refusing events usually
+// cannot reach the broker β€” which more attempts will not fix.
+const FIREBILL_ATTEMPTS = 2;
+const FIREBILL_RETRY_DELAY_MS = 150;
+
// FIREBILL_ORG_IDS is decoded once at startup by the config schema; the Set is
// built lazily on first use and cached, keyed on the decoded array reference.
let allowlistCache: { source: string[] | undefined; ids: Set<string> } | null =
@@ -37,13 +47,15 @@ function firebillOrgIds(): Set<string> {
}
/**
- * Whether usage tracking for this org should be routed through firebill
- * instead of directly to Autumn. Requires firebill to be fully configured AND
- * the org to be on the rollout allowlist.
+ * Whether this org's usage goes through firebill rather than straight to Autumn.
+ * Needs firebill configured, then either the allowlist or the sticky percentage.
*/
export function shouldRouteToFirebill(orgId: string): boolean {
if (!config.FIREBILL_URL || !config.FIREBILL_SECRET) return false;
- return firebillOrgIds().has(orgId);
+ // Always-on set: test orgs stay routed even at 0 percent.
+ if (firebillOrgIds().has(orgId)) return true;
+ // Sticky by org, so a ramp only ever adds and 0 is the kill switch.
+ return sampled(orgId, config.FIREBILL_ROLLOUT_PERCENT);
}
// Plain concatenation rather than new URL(path, base): a leading-slash path
@@ -53,26 +65,88 @@ function firebillUrl(path: string): string {
}
/**
- * Sends a usage event to firebill, which records it durably in Postgres and
- * forwards it to Autumn (replaying failed deliveries instead of losing them).
+ * Sends a usage event to firebill, which publishes it to a durable quorum queue
+ * and answers once the broker has confirmed it, then forwards it to Autumn from
+ * a consumer (retrying failed deliveries instead of losing them).
*
* A negative value is a refund (refundCredits negates before calling track);
* firebill's /v1/refund endpoint expects the POSITIVE amount and negates it
* itself, so the absolute value is sent either way.
*
* Returns true when firebill accepted the event, mirroring the boolean
- * contract of the direct Autumn track path. A `false` here means "not billed
- * yet" β€” firebill keeps retrying delivery on its side.
+ * contract of the direct Autumn track path. A `false` means firebill never
+ * took it: nothing is retrying, and the usage goes unbilled.
*/
-export async function firebillTrack({
- customerId,
- entityId,
- featureId,
- value,
- properties,
- idempotencyKey,
-}: TrackParams): Promise<boolean> {
- const path = value < 0 ? "/v1/refund" : "/v1/track";
+export async function firebillTrack(params: TrackParams): Promise<boolean> {
+ const operation = params.value < 0 ? "refund" : "track";
+ const path = `/v1/${operation}`;
+ // Both attempts must be the same event. Without a caller key firebill mints
+ // one per request, so a retry after an accepted-but-lost first attempt would
+ // be a second charge; minting here instead keeps one identity per call while
+ // separate calls stay distinct, exactly as before.
+ const attempted = {
+ ...params,
+ idempotencyKey: params.idempotencyKey ?? randomUUID(),
+ };
+
+ for (let attempt = 1; attempt < FIREBILL_ATTEMPTS; attempt++) {
+ const result = await firebillAttempt(path, attempted);
+ if (result.ok) {
+ firebillTrackTotal.labels(operation, "accepted").inc();
+ return true;
+ }
+ firebillRetryTotal.labels(result.reason).inc();
+ await new Promise(resolve => setTimeout(resolve, FIREBILL_RETRY_DELAY_MS));
+ }
+
+ const last = await firebillAttempt(path, attempted);
+ if (last.ok) {
+ firebillTrackTotal.labels(operation, "accepted").inc();
+ return true;
+ }
+
+ // `refused` only for an explicit `success: false`, which is firebill saying it
+ // did not take the event. A transport failure is `ambiguous`: firebill may
+ // have accepted it and be delivering it right now, so this is not proof of
+ // lost usage. Either way the caller is told false and must not assume a
+ // charge landed. A counter rather than a throw: billing must not fail the
+ // customer's request, and a log alone is too quiet to alert on.
+ const outcome = last.reason === "not_success" ? "refused" : "ambiguous";
+ logger.error(
+ outcome === "refused"
+ ? "firebill refused a usage event; it will not be billed"
+ : "firebill did not answer; the event may or may not have been accepted",
+ {
+ customerId: params.customerId,
+ entityId: params.entityId,
+ featureId: params.featureId,
+ value: params.value,
+ idempotencyKey: attempted.idempotencyKey,
+ callerSuppliedKey: params.idempotencyKey !== undefined,
+ path,
+ attempts: FIREBILL_ATTEMPTS,
+ reason: last.reason,
+ },
+ );
+ firebillTrackTotal.labels(operation, outcome).inc();
+ return false;
+}
+
+type AttemptResult =
+ | { ok: true }
+ | { ok: false; reason: "not_ok" | "not_success" | "exception" };
+
+async function firebillAttempt(
+ path: string,
+ {
+ customerId,
+ entityId,
+ featureId,
+ value,
+ properties,
+ idempotencyKey,
+ }: TrackParams,
+): Promise<AttemptResult> {
const url = firebillUrl(path);
try {
const response = await fetch(url, {
@@ -90,17 +164,17 @@ export async function firebillTrack({
feature_id: featureId,
value: Math.abs(value),
properties,
- // Stable per-charge key: firebill makes it the intent row's primary
- // key, so a caller retry (or a requeued job re-billing the same work)
- // is answered from the existing row instead of charged again. Omitted
- // β†’ firebill mints a per-request UUID (dedupes only its own retries).
+ // firebill carries this to Autumn as the Idempotency-Key on every
+ // attempt, so a requeued job re-billing the same work is deduped rather
+ // than charged twice. Omitted β†’ firebill mints a per-request UUID,
+ // which dedupes only its own retries.
...(idempotencyKey ? { idempotency_key: idempotencyKey } : {}),
}),
signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),
});
if (!response.ok) {
- logger.error("firebill track failed β€” non-OK response", {
+ logger.warn("firebill track attempt failed β€” non-OK response", {
customerId,
entityId,
featureId,
@@ -108,19 +182,19 @@ export async function firebillTrack({
path,
status: response.status,
});
- return false;
+ return { ok: false, reason: "not_ok" };
}
const body = (await response.json()) as { success?: boolean };
if (body.success !== true) {
- logger.error("firebill track did not succeed", {
+ logger.warn("firebill track attempt did not succeed", {
customerId,
entityId,
featureId,
value,
path,
});
- return false;
+ return { ok: false, reason: "not_success" };
}
logger.info("firebill track succeeded", {
@@ -130,14 +204,12 @@ export async function firebillTrack({
value,
path,
});
- return true;
+ return { ok: true };
} catch (error) {
- // DO NOT fall back to calling Autumn directly here: firebill may have
- // durably recorded the event before this call failed and will deliver it
- // to Autumn later, so a direct-Autumn fallback could double-bill the
- // customer. A firebill failure (timeout, 5xx, connection refused) is
- // treated exactly like an Autumn track failure: log and return false.
- logger.error("firebill track failed β€” firebill may be unavailable", {
+ // DO NOT fall back to Autumn directly: firebill may have accepted the event
+ // before this failed, and the Autumn SDK sends no idempotency key, so the
+ // pair could not be deduped and the customer would be billed twice.
+ logger.warn("firebill track attempt failed β€” firebill may be unavailable", {
customerId,
entityId,
featureId,
@@ -145,7 +217,7 @@ export async function firebillTrack({
path,
error,
});
- return false;
+ return { ok: false, reason: "exception" };
}
}
diff --git a/apps/api/src/services/autumn/metrics.ts b/apps/api/src/services/autumn/metrics.ts
new file mode 100644
index 000000000..abff7e25a
--- /dev/null
+++ b/apps/api/src/services/autumn/metrics.ts
@@ -0,0 +1,34 @@
+import { Counter } from "prom-client";
+
+/**
+ * Which path a usage event took. The ratio of `firebill` to `direct` is the
+ * live read on the rollout percentage β€” during a ramp it is the first thing to
+ * look at, because a routing bug shows up here long before it shows up in a
+ * balance.
+ */
+export const billingRouteTotal = new Counter({
+ name: "firecrawl_billing_route_total",
+ help: "Usage events by the path they took to Autumn",
+ labelNames: ["route"] as const, // firebill | direct
+});
+
+/**
+ * What firebill said. `refused` is an explicit `success: false` β€” firebill does
+ * not own the event and nobody else does, so usage is being dropped.
+ * `ambiguous` is a transport failure: firebill may have accepted it before the
+ * answer was lost. Both mean the caller was told false; only `refused` is proof
+ * the usage is gone. Alert on the pair.
+ */
+export const firebillTrackTotal = new Counter({
+ name: "firecrawl_firebill_track_total",
+ help: "Outcomes of usage events sent to firebill",
+ // operation: track|refund outcome: accepted|refused|ambiguous
+ labelNames: ["operation", "outcome"] as const,
+});
+
+/** Retries of a firebill call that answered `false` or threw. */
+export const firebillRetryTotal = new Counter({
+ name: "firecrawl_firebill_retry_total",
+ help: "Retried firebill calls, by why the previous attempt failed",
+ labelNames: ["reason"] as const, // not_ok | not_success | exception
+});
diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts
index 8083aa90d..0f4d60aaf 100644
--- a/apps/api/src/services/billing/credit_billing.ts
+++ b/apps/api/src/services/billing/credit_billing.ts
@@ -53,9 +53,9 @@ export async function billTeam(
if (await autumnService.isRoutedThroughFirebill(team_id)) {
// No compensating refund on the firebill route: the tracked charge
// is durable and correct, and a refund here poisons a retried
- // request β€” its track would dedupe against the intent row (no new
- // charge) while the ledger enqueue succeeds, leaving Autumn
- // net-zero for billed work.
+ // request β€” its track would be deduped by Autumn against the same
+ // idempotency key (no new charge) while the ledger enqueue succeeds,
+ // leaving Autumn net-zero for billed work.
logger?.warn(
"billing enqueue failed on the firebill route; charge stands",
{ team_id, credits, billing },