Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Handle a firebill refusal, and ramp by percentage instead of by allowlist (#4341)
Rakshith Ramprakashf259e9ca86b0
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex fdb5d2b37..22a7fe05f 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -405,6 +405,12 @@ const configSchema = z.object({FIREBILL_URL: emptyStringAsUndefined(z.string().url()),FIREBILL_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),FIREBILL_ORG_IDS: delimitedList(",").optional(),+ // Sticky percentage ramp, on top of the allowlist above. The bucket is a+ // hash of the org id, so an org that is in at 5 is still in at 30 β a ramp+ // only ever adds, and never reshuffles who is on which path mid-rollout.+ // 0 (the default) is also the kill switch: the allowlist still routes, and+ // nothing else does.+ FIREBILL_ROLLOUT_PERCENT: z.coerce.number().min(0).max(100).default(0),// MiscellaneousIDMUX_URL: z.string().optional(),diff --git a/apps/api/src/lib/__tests__/rollout.test.ts b/apps/api/src/lib/__tests__/rollout.test.tsnew file mode 100644index 000000000..c6d120433--- /dev/null+++ b/apps/api/src/lib/__tests__/rollout.test.ts@@ -0,0 +1,40 @@+import { describe, expect, it } from "vitest";+import { sampled } from "../rollout";++const orgs = Array.from({ length: 2000 }, (_, i) => `org-${i}`);++describe("sampled", () => {+ it("is off at 0 and on at 100", () => {+ expect(orgs.some(o => sampled(o, 0))).toBe(false);+ expect(orgs.every(o => sampled(o, 100))).toBe(true);+ });++ it("only ever adds as the percentage rises", () => {+ // The property the whole ramp rests on: nobody is moved back off firebill+ // by a later step, so each stage is a superset of the one before it.+ const stages = [5, 10, 30, 60, 80, 100];+ for (const org of orgs) {+ let wasIn = false;+ for (const percent of stages) {+ const isIn = sampled(org, percent);+ expect(!wasIn || isIn).toBe(true);+ wasIn = isIn;+ }+ }+ });++ it("gives the same answer every call", () => {+ for (const org of orgs.slice(0, 50)) {+ const first = sampled(org, 37);+ for (let i = 0; i < 5; i++) expect(sampled(org, 37)).toBe(first);+ }+ });++ it("lands near the requested share", () => {+ for (const percent of [5, 30, 60]) {+ const hits = orgs.filter(o => sampled(o, percent)).length;+ const share = (hits / orgs.length) * 100;+ expect(Math.abs(share - percent)).toBeLessThan(5);+ }+ });+});diff --git a/apps/api/src/lib/rollout.ts b/apps/api/src/lib/rollout.tsnew file mode 100644index 000000000..d604bb58e--- /dev/null+++ b/apps/api/src/lib/rollout.ts@@ -0,0 +1,23 @@+/**+ * Deterministic percentage sampling for gradual rollouts.+ *+ * The bucket is a pure function of the cohort key, so a cohort that is in at+ * 5% is still in at 30% β a ramp only ever adds, never reshuffles. That is the+ * property `Math.random() < percent` does not have, and the reason to prefer+ * this wherever a cohort should stay on one side of a rollout: billing routes,+ * anything a support question could be asked about, anything you want to+ * compare before and after.+ *+ * FNV-1a over the key, mapped onto [0, 100).+ */+export function sampled(cohortKey: string, percent: number): boolean {+ if (percent <= 0) return false;+ if (percent >= 100) return true;++ let hash = 2166136261;+ for (let i = 0; i < cohortKey.length; i++) {+ hash ^= cohortKey.charCodeAt(i);+ hash = Math.imul(hash, 16777619);+ }+ return ((hash >>> 0) / 0x1_0000_0000) * 100 < percent;+}diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.tsindex 1dd158c84..80f3bced1 100644--- a/apps/api/src/search/highlights.ts+++ b/apps/api/src/search/highlights.ts@@ -9,6 +9,7 @@ import {import type { HighlightFailureReason } from "./highlight-model";import { config } from "../config";import { logger as rootLogger } from "../lib/logger";+import { sampled } from "../lib/rollout";// How far back into the index we're willing to reach for highlight source text.const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days@@ -28,18 +29,6 @@ export function highlightsEnvReady(): boolean {);}-function sampled(cohortKey: string, percent: number): boolean {- if (percent <= 0) return false;- if (percent >= 100) return true;-- let hash = 2166136261;- for (let i = 0; i < cohortKey.length; i++) {- hash ^= cohortKey.charCodeAt(i);- hash = Math.imul(hash, 16777619);- }- return ((hash >>> 0) / 0x1_0000_0000) * 100 < percent;-}-export function searchHighlightsMode(options: {requested?: boolean;origin?: string | null;diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.tsindex ea5b15aa1..c260330fc 100644--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts@@ -715,7 +715,7 @@ describe("firebill routing", () => {expect(String(url)).toBe("http://proxy.test/firebill/v1/track");});- it("sends a caller-supplied idempotency key to firebill, and omits the field otherwise", async () => {+ it("sends a caller-supplied idempotency key to firebill, and mints one otherwise", async () => {state.configRef = firebillConfig();const svc = makeService();@@ -734,9 +734,14 @@ describe("firebill routing", () => {value: 5,properties: { source: "billTeam", endpoint: "search" },});- expect(- "idempotency_key" in JSON.parse(mockFetch.mock.calls[1]![1].body),- ).toBe(false);+ // A key is always sent, even when the caller has none: firebillTrack retries+ // once, and both attempts have to be the same event or an accepted-but-lost+ // first attempt would be billed twice. firebill would have minted an+ // equivalent UUID itself; minting it here is what makes it stable across+ // the retry.+ const minted = JSON.parse(mockFetch.mock.calls[1]![1].body).idempotency_key;+ expect(minted).toEqual(expect.any(String));+ expect(minted).not.toBe("fc:track:scrape:job-123");});it("a track/refund pair for one charge carries two distinct keys", async () => {@@ -794,6 +799,9 @@ describe("firebill routing", () => {feature_id: "CREDITS",value: 42,properties: { source: "test", endpoint: "extract" },+ // Minted when the caller has none, so the retry inside firebillTrack is+ // the same event rather than a second charge.+ idempotency_key: expect.any(String),});});diff --git a/apps/api/src/services/autumn/__tests__/firebill.test.ts b/apps/api/src/services/autumn/__tests__/firebill.test.tsnew file mode 100644index 000000000..e6081886a--- /dev/null+++ b/apps/api/src/services/autumn/__tests__/firebill.test.ts@@ -0,0 +1,168 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { firebillTrack, shouldRouteToFirebill } from "../firebill";+import { firebillRetryTotal, firebillTrackTotal } from "../metrics";++const { configState } = vi.hoisted(() => ({+ configState: {+ FIREBILL_URL: "https://firebill.test",+ FIREBILL_SECRET: "secret",+ FIREBILL_ORG_IDS: ["allow-listed-org"] as string[] | undefined,+ FIREBILL_ROLLOUT_PERCENT: 0,+ },+}));++vi.mock("../../../config", () => ({ config: configState }));+vi.mock("../../../lib/logger", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+}));++// Built fresh per call: a Response body can only be read once, so a shared+// instance would make the second attempt look like a transport failure.+const ok = () =>+ new Response(JSON.stringify({ success: true }), { status: 200 });+const refused = () =>+ new Response(JSON.stringify({ success: false }), { status: 200 });++const params = {+ customerId: "org-1",+ entityId: "team-1",+ featureId: "CREDITS",+ value: 3,+ properties: {},+ idempotencyKey: "fc:track:scrape:job-1",+};++beforeEach(() => {+ configState.FIREBILL_URL = "https://firebill.test";+ configState.FIREBILL_SECRET = "secret";+ configState.FIREBILL_ORG_IDS = ["allow-listed-org"];+ configState.FIREBILL_ROLLOUT_PERCENT = 0;+ firebillTrackTotal.reset();+ firebillRetryTotal.reset();+});+afterEach(() => vi.unstubAllGlobals());++describe("shouldRouteToFirebill", () => {+ it("routes the allowlist even at 0 percent", () => {+ expect(shouldRouteToFirebill("allow-listed-org")).toBe(true);+ });++ it("routes nobody else at 0 percent β the kill switch", () => {+ const others = Array.from({ length: 200 }, (_, i) => `other-${i}`);+ expect(others.some(shouldRouteToFirebill)).toBe(false);+ });++ it("routes roughly the configured share once ramped", () => {+ const orgs = Array.from({ length: 1000 }, (_, i) => `ramp-${i}`);+ configState.FIREBILL_ROLLOUT_PERCENT = 30;+ const share =+ (orgs.filter(shouldRouteToFirebill).length / orgs.length) * 100;+ expect(Math.abs(share - 30)).toBeLessThan(6);+ });++ it("stays off entirely when firebill is not configured", () => {+ configState.FIREBILL_ROLLOUT_PERCENT = 100;+ configState.FIREBILL_SECRET = "";+ expect(shouldRouteToFirebill("allow-listed-org")).toBe(false);+ });+});++describe("firebillTrack", () => {+ it("retries a refusal and reports success if the retry lands", async () => {+ const fetchMock = vi+ .fn()+ .mockImplementationOnce(async () => refused())+ .mockImplementationOnce(async () => ok());+ vi.stubGlobal("fetch", fetchMock);++ await expect(firebillTrack(params)).resolves.toBe(true);+ expect(fetchMock).toHaveBeenCalledTimes(2);+ // Same key on both attempts, which is what makes the retry safe.+ const keys = fetchMock.mock.calls.map(+ c => JSON.parse(c[1].body).idempotency_key,+ );+ expect(keys).toEqual([params.idempotencyKey, params.idempotencyKey]);+ });++ it("gives up after the attempt limit and reports false", async () => {+ const fetchMock = vi.fn().mockImplementation(async () => refused());+ vi.stubGlobal("fetch", fetchMock);++ await expect(firebillTrack(params)).resolves.toBe(false);+ expect(fetchMock).toHaveBeenCalledTimes(2);+ });++ it("reuses one key across attempts even when the caller supplies none", async () => {+ // Without this the retry is a second event: firebill mints its own key per+ // request, so an accepted-but-lost first attempt would be billed twice.+ const fetchMock = vi+ .fn()+ .mockRejectedValueOnce(new Error("timeout"))+ .mockImplementationOnce(async () => ok());+ vi.stubGlobal("fetch", fetchMock);++ await firebillTrack({ ...params, idempotencyKey: undefined });++ const keys = fetchMock.mock.calls.map(+ c => JSON.parse(c[1].body).idempotency_key,+ );+ expect(keys[0]).toBeTruthy();+ expect(keys[0]).toBe(keys[1]);+ });++ it("mints a different key for each keyless call", async () => {+ const fetchMock = vi.fn().mockImplementation(async () => ok());+ vi.stubGlobal("fetch", fetchMock);++ await firebillTrack({ ...params, idempotencyKey: undefined });+ await firebillTrack({ ...params, idempotencyKey: undefined });++ const [first, second] = fetchMock.mock.calls.map(+ c => JSON.parse(c[1].body).idempotency_key,+ );+ expect(first).not.toBe(second);+ });++ it("separates an explicit refusal from an ambiguous transport failure", async () => {+ const outcomes = async () =>+ Object.fromEntries(+ (await firebillTrackTotal.get()).values.map(v => [+ v.labels.outcome,+ v.value,+ ]),+ );++ vi.stubGlobal(+ "fetch",+ vi.fn().mockImplementation(async () => refused()),+ );+ await firebillTrack(params);+ expect(await outcomes()).toEqual({ refused: 1 });++ firebillTrackTotal.reset();+ vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("timeout")));+ await firebillTrack(params);+ // A timeout is not proof the usage was lost β firebill may have taken it.+ expect(await outcomes()).toEqual({ ambiguous: 1 });+ });++ it("retries a thrown error too", async () => {+ const fetchMock = vi+ .fn()+ .mockRejectedValueOnce(new Error("connection refused"))+ .mockImplementationOnce(async () => ok());+ vi.stubGlobal("fetch", fetchMock);++ await expect(firebillTrack(params)).resolves.toBe(true);+ expect(fetchMock).toHaveBeenCalledTimes(2);+ });++ it("sends a refund to /v1/refund with the value made positive", async () => {+ const fetchMock = vi.fn().mockImplementation(async () => ok());+ vi.stubGlobal("fetch", fetchMock);++ await firebillTrack({ ...params, value: -7 });+ expect(fetchMock.mock.calls[0][0]).toBe("https://firebill.test/v1/refund");+ expect(JSON.parse(fetchMock.mock.calls[0][1].body).value).toBe(7);+ });+});diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex 4e751bc70..a26b7efcc 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -10,6 +10,7 @@ import {firebillTrack,shouldRouteToFirebill,} from "./firebill";+import { billingRouteTotal } from "./metrics";import type {CreateEntityParams,CreateEntityResult,@@ -238,13 +239,12 @@ export class AutumnService {properties,idempotencyKey,}: TrackParams): Promise<boolean> {- // Gradual firebill rollout: allowlisted orgs record usage via firebill (a- // durable store that forwards to Autumn) instead of calling Autumn- // directly. If the firebill call fails we must NOT fall back to Autumn β- // firebill may have durably recorded the event and will deliver it later,- // so a direct-Autumn fallback could double-bill the customer. firebillTrack- // returns false on failure, exactly like an Autumn track failure.+ // Gradual rollout: allowlisted orgs, and those in the sticky+ // FIREBILL_ROLLOUT_PERCENT bucket, bill through firebill. No fallback to+ // Autumn on failure β firebill may already own the event, and the SDK sends+ // no idempotency key, so the pair could not be deduped.if (shouldRouteToFirebill(customerId)) {+ billingRouteTotal.labels("firebill").inc();return await firebillTrack({customerId,entityId,@@ -255,6 +255,8 @@ export class AutumnService {});}+ billingRouteTotal.labels("direct").inc();+if (!autumnClient) return false;try {diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.tsindex 79ef008da..18dadeeed 100644--- a/apps/api/src/services/autumn/firebill.ts+++ b/apps/api/src/services/autumn/firebill.ts@@ -1,6 +1,9 @@+import { randomUUID } from "crypto";import { config } from "../../config";import { logger } from "../../lib/logger";+import { sampled } from "../../lib/rollout";import type { TrackParams } from "./types";+import { firebillRetryTotal, firebillTrackTotal } from "./metrics";/*** Outcome of a firebill `/v1/lock` call, mirroring firebill's three-state@@ -18,6 +21,13 @@ type FirebillLockResult =// direct Autumn client.const FIREBILL_TIMEOUT_MS = 5000;+// Safe to retry because the idempotency key is stable across attempts: if the+// first attempt did land (ambiguous confirm timeout), Autumn dedupes the second.+// Small because a caller may be waiting, and a firebill refusing events usually+// cannot reach the broker β which more attempts will not fix.+const FIREBILL_ATTEMPTS = 2;+const FIREBILL_RETRY_DELAY_MS = 150;+// FIREBILL_ORG_IDS is decoded once at startup by the config schema; the Set is// built lazily on first use and cached, keyed on the decoded array reference.let allowlistCache: { source: string[] | undefined; ids: Set<string> } | null =@@ -37,13 +47,15 @@ function firebillOrgIds(): Set<string> {}/**- * Whether usage tracking for this org should be routed through firebill- * instead of directly to Autumn. Requires firebill to be fully configured AND- * the org to be on the rollout allowlist.+ * Whether this org's usage goes through firebill rather than straight to Autumn.+ * Needs firebill configured, then either the allowlist or the sticky percentage.*/export function shouldRouteToFirebill(orgId: string): boolean {if (!config.FIREBILL_URL || !config.FIREBILL_SECRET) return false;- return firebillOrgIds().has(orgId);+ // Always-on set: test orgs stay routed even at 0 percent.+ if (firebillOrgIds().has(orgId)) return true;+ // Sticky by org, so a ramp only ever adds and 0 is the kill switch.+ return sampled(orgId, config.FIREBILL_ROLLOUT_PERCENT);}// Plain concatenation rather than new URL(path, base): a leading-slash path@@ -53,26 +65,88 @@ function firebillUrl(path: string): string {}/**- * Sends a usage event to firebill, which records it durably in Postgres and- * forwards it to Autumn (replaying failed deliveries instead of losing them).+ * Sends a usage event to firebill, which publishes it to a durable quorum queue+ * and answers once the broker has confirmed it, then forwards it to Autumn from+ * a consumer (retrying failed deliveries instead of losing them).** A negative value is a refund (refundCredits negates before calling track);* firebill's /v1/refund endpoint expects the POSITIVE amount and negates it* itself, so the absolute value is sent either way.** Returns true when firebill accepted the event, mirroring the boolean- * contract of the direct Autumn track path. A `false` here means "not billed- * yet" β firebill keeps retrying delivery on its side.+ * contract of the direct Autumn track path. A `false` means firebill never+ * took it: nothing is retrying, and the usage goes unbilled.*/-export async function firebillTrack({- customerId,- entityId,- featureId,- value,- properties,- idempotencyKey,-}: TrackParams): Promise<boolean> {- const path = value < 0 ? "/v1/refund" : "/v1/track";+export async function firebillTrack(params: TrackParams): Promise<boolean> {+ const operation = params.value < 0 ? "refund" : "track";+ const path = `/v1/${operation}`;+ // Both attempts must be the same event. Without a caller key firebill mints+ // one per request, so a retry after an accepted-but-lost first attempt would+ // be a second charge; minting here instead keeps one identity per call while+ // separate calls stay distinct, exactly as before.+ const attempted = {+ ...params,+ idempotencyKey: params.idempotencyKey ?? randomUUID(),+ };++ for (let attempt = 1; attempt < FIREBILL_ATTEMPTS; attempt++) {+ const result = await firebillAttempt(path, attempted);+ if (result.ok) {+ firebillTrackTotal.labels(operation, "accepted").inc();+ return true;+ }+ firebillRetryTotal.labels(result.reason).inc();+ await new Promise(resolve => setTimeout(resolve, FIREBILL_RETRY_DELAY_MS));+ }++ const last = await firebillAttempt(path, attempted);+ if (last.ok) {+ firebillTrackTotal.labels(operation, "accepted").inc();+ return true;+ }++ // `refused` only for an explicit `success: false`, which is firebill saying it+ // did not take the event. A transport failure is `ambiguous`: firebill may+ // have accepted it and be delivering it right now, so this is not proof of+ // lost usage. Either way the caller is told false and must not assume a+ // charge landed. A counter rather than a throw: billing must not fail the+ // customer's request, and a log alone is too quiet to alert on.+ const outcome = last.reason === "not_success" ? "refused" : "ambiguous";+ logger.error(+ outcome === "refused"+ ? "firebill refused a usage event; it will not be billed"+ : "firebill did not answer; the event may or may not have been accepted",+ {+ customerId: params.customerId,+ entityId: params.entityId,+ featureId: params.featureId,+ value: params.value,+ idempotencyKey: attempted.idempotencyKey,+ callerSuppliedKey: params.idempotencyKey !== undefined,+ path,+ attempts: FIREBILL_ATTEMPTS,+ reason: last.reason,+ },+ );+ firebillTrackTotal.labels(operation, outcome).inc();+ return false;+}++type AttemptResult =+ | { ok: true }+ | { ok: false; reason: "not_ok" | "not_success" | "exception" };++async function firebillAttempt(+ path: string,+ {+ customerId,+ entityId,+ featureId,+ value,+ properties,+ idempotencyKey,+ }: TrackParams,+): Promise<AttemptResult> {const url = firebillUrl(path);try {const response = await fetch(url, {@@ -90,17 +164,17 @@ export async function firebillTrack({feature_id: featureId,value: Math.abs(value),properties,- // Stable per-charge key: firebill makes it the intent row's primary- // key, so a caller retry (or a requeued job re-billing the same work)- // is answered from the existing row instead of charged again. Omitted- // β firebill mints a per-request UUID (dedupes only its own retries).+ // firebill carries this to Autumn as the Idempotency-Key on every+ // attempt, so a requeued job re-billing the same work is deduped rather+ // than charged twice. Omitted β firebill mints a per-request UUID,+ // which dedupes only its own retries....(idempotencyKey ? { idempotency_key: idempotencyKey } : {}),}),signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),});if (!response.ok) {- logger.error("firebill track failed β non-OK response", {+ logger.warn("firebill track attempt failed β non-OK response", {customerId,entityId,featureId,@@ -108,19 +182,19 @@ export async function firebillTrack({path,status: response.status,});- return false;+ return { ok: false, reason: "not_ok" };}const body = (await response.json()) as { success?: boolean };if (body.success !== true) {- logger.error("firebill track did not succeed", {+ logger.warn("firebill track attempt did not succeed", {customerId,entityId,featureId,value,path,});- return false;+ return { ok: false, reason: "not_success" };}logger.info("firebill track succeeded", {@@ -130,14 +204,12 @@ export async function firebillTrack({value,path,});- return true;+ return { ok: true };} catch (error) {- // DO NOT fall back to calling Autumn directly here: firebill may have- // durably recorded the event before this call failed and will deliver it- // to Autumn later, so a direct-Autumn fallback could double-bill the- // customer. A firebill failure (timeout, 5xx, connection refused) is- // treated exactly like an Autumn track failure: log and return false.- logger.error("firebill track failed β firebill may be unavailable", {+ // DO NOT fall back to Autumn directly: firebill may have accepted the event+ // before this failed, and the Autumn SDK sends no idempotency key, so the+ // pair could not be deduped and the customer would be billed twice.+ logger.warn("firebill track attempt failed β firebill may be unavailable", {customerId,entityId,featureId,@@ -145,7 +217,7 @@ export async function firebillTrack({path,error,});- return false;+ return { ok: false, reason: "exception" };}}diff --git a/apps/api/src/services/autumn/metrics.ts b/apps/api/src/services/autumn/metrics.tsnew file mode 100644index 000000000..abff7e25a--- /dev/null+++ b/apps/api/src/services/autumn/metrics.ts@@ -0,0 +1,34 @@+import { Counter } from "prom-client";++/**+ * Which path a usage event took. The ratio of `firebill` to `direct` is the+ * live read on the rollout percentage β during a ramp it is the first thing to+ * look at, because a routing bug shows up here long before it shows up in a+ * balance.+ */+export const billingRouteTotal = new Counter({+ name: "firecrawl_billing_route_total",+ help: "Usage events by the path they took to Autumn",+ labelNames: ["route"] as const, // firebill | direct+});++/**+ * What firebill said. `refused` is an explicit `success: false` β firebill does+ * not own the event and nobody else does, so usage is being dropped.+ * `ambiguous` is a transport failure: firebill may have accepted it before the+ * answer was lost. Both mean the caller was told false; only `refused` is proof+ * the usage is gone. Alert on the pair.+ */+export const firebillTrackTotal = new Counter({+ name: "firecrawl_firebill_track_total",+ help: "Outcomes of usage events sent to firebill",+ // operation: track|refund outcome: accepted|refused|ambiguous+ labelNames: ["operation", "outcome"] as const,+});++/** Retries of a firebill call that answered `false` or threw. */+export const firebillRetryTotal = new Counter({+ name: "firecrawl_firebill_retry_total",+ help: "Retried firebill calls, by why the previous attempt failed",+ labelNames: ["reason"] as const, // not_ok | not_success | exception+});diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex 8083aa90d..0f4d60aaf 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -53,9 +53,9 @@ export async function billTeam(if (await autumnService.isRoutedThroughFirebill(team_id)) {// No compensating refund on the firebill route: the tracked charge// is durable and correct, and a refund here poisons a retried- // request β its track would dedupe against the intent row (no new- // charge) while the ledger enqueue succeeds, leaving Autumn- // net-zero for billed work.+ // request β its track would be deduped by Autumn against the same+ // idempotency key (no new charge) while the ledger enqueue succeeds,+ // leaving Autumn net-zero for billed work.logger?.warn("billing enqueue failed on the firebill route; charge stands",{ team_id, credits, billing },