Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(billing): read firebill's "I do not know", and stop blaming it for our own stalls (#4514)

Rakshith Ramprakashfb390af29f5d
diff --git a/apps/api/src/services/autumn/__tests__/firebill.test.ts b/apps/api/src/services/autumn/__tests__/firebill.test.ts
index 006b41201..c52442c44 100644
--- a/apps/api/src/services/autumn/__tests__/firebill.test.ts
+++ b/apps/api/src/services/autumn/__tests__/firebill.test.ts
@@ -4,8 +4,10 @@ import {
firebillTrack,
shouldRouteToFirebill,
} from "../firebill";
+import { logger } from "../../../lib/logger";
import {
firebillCheckTotal,
+ firebillFailureCauseTotal,
firebillRetryTotal,
firebillTrackTotal,
} from "../metrics";
@@ -30,6 +32,23 @@ const ok = () =>
new Response(JSON.stringify({ success: true }), { status: 200 });
const refused = () =>
new Response(JSON.stringify({ success: false }), { status: 200 });
+/** firebill's confirm timed out: it does not know whether it took the event. */
+const ambiguous = () =>
+ new Response(JSON.stringify({ success: false, ambiguous: true }), {
+ status: 504,
+ });
+
+/** What undici actually throws: the real error nested under a bare TypeError. */
+const wrapped = (inner: Error) =>
+ Object.assign(new TypeError("fetch failed"), { cause: inner });
+
+const abortError = () =>
+ Object.assign(new Error("The operation was aborted due to timeout"), {
+ name: "TimeoutError",
+ });
+
+const connectionError = (code: string) =>
+ Object.assign(new Error(`connect ${code} 10.0.0.1:8080`), { code });
const params = {
customerId: "org-1",
@@ -48,6 +67,10 @@ beforeEach(() => {
firebillTrackTotal.reset();
firebillRetryTotal.reset();
firebillCheckTotal.reset();
+ firebillFailureCauseTotal.reset();
+ vi.mocked(logger.info).mockClear();
+ vi.mocked(logger.warn).mockClear();
+ vi.mocked(logger.error).mockClear();
});
afterEach(() => vi.unstubAllGlobals());
@@ -90,6 +113,35 @@ describe("shouldRouteToFirebill", () => {
});
});
+/**
+ * The outcome counter alone β€” the one the alerts read, which must keep exactly
+ * the labels it always had.
+ */
+const trackOutcomes = async () =>
+ Object.fromEntries(
+ (await firebillTrackTotal.get()).values.map(v => [
+ v.labels.outcome,
+ v.value,
+ ]),
+ );
+
+const checkOutcomes = async () =>
+ Object.fromEntries(
+ (await firebillCheckTotal.get()).values.map(v => [
+ v.labels.outcome,
+ v.value,
+ ]),
+ );
+
+/** The separate cause counter, as `operation/cause`. */
+const failureCauses = async () =>
+ Object.fromEntries(
+ (await firebillFailureCauseTotal.get()).values.map(v => [
+ `${v.labels.operation}/${v.labels.cause}`,
+ v.value,
+ ]),
+ );
+
describe("firebillTrack", () => {
it("retries a refusal and reports success if the retry lands", async () => {
const fetchMock = vi
@@ -147,26 +199,252 @@ describe("firebillTrack", () => {
});
it("separates an explicit refusal from an ambiguous transport failure", async () => {
- const outcomes = async () =>
- Object.fromEntries(
- (await firebillTrackTotal.get()).values.map(v => [
- v.labels.outcome,
- v.value,
- ]),
- );
-
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => refused()),
);
await firebillTrack(params);
- expect(await outcomes()).toEqual({ refused: 1 });
+ expect(await trackOutcomes()).toEqual({ refused: 1 });
+ expect(await failureCauses()).toEqual({ "track/refused": 1 });
firebillTrackTotal.reset();
+ firebillFailureCauseTotal.reset();
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("timeout")));
await firebillTrack(params);
// A timeout is not proof the usage was lost β€” firebill may have taken it.
- expect(await outcomes()).toEqual({ ambiguous: 1 });
+ expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
+ expect(await failureCauses()).toEqual({ "track/connection": 1 });
+ });
+
+ // -----------------------------------------------------------------------
+ // The alerts' own series
+ // -----------------------------------------------------------------------
+
+ it("leaves the alerted counters with exactly the labels they had", async () => {
+ // `increase()` evaluates per series, so a `cause` label here would turn
+ // `increase(firecrawl_firebill_track_total{outcome="refused"}[15m]) > 0`
+ // into one threshold per cause. The detail lives beside it instead.
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockImplementation(async () => refused()),
+ );
+ await firebillTrack(params);
+
+ const [series] = (await firebillTrackTotal.get()).values;
+ expect(Object.keys(series.labels).sort()).toEqual(["operation", "outcome"]);
+
+ const [cause] = (await firebillFailureCauseTotal.get()).values;
+ expect(Object.keys(cause.labels).sort()).toEqual(["cause", "operation"]);
+ expect(cause.labels).toMatchObject({
+ operation: "track",
+ cause: "refused",
+ });
+ });
+
+ it("counts a refund's cause under its own operation", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockImplementation(async () => refused()),
+ );
+ await firebillTrack({ ...params, value: -7 });
+ expect(await failureCauses()).toEqual({ "refund/refused": 1 });
+ });
+
+ // -----------------------------------------------------------------------
+ // firebill saying "I do not know"
+ //
+ // A publish whose broker confirm timed out may have been taken anyway. Over
+ // eight days ~2,100 of these were answered `{"success": false}` β€” the same
+ // body a refusal sends β€” and every one of those events had in fact settled,
+ // while this client logged each as usage that would never be billed.
+ // -----------------------------------------------------------------------
+
+ it.each([
+ ["a 504 carrying the field", () => ambiguous()],
+ ["a 504 with no body at all", () => new Response(null, { status: 504 })],
+ [
+ "a 200 that says so by field",
+ () =>
+ new Response(JSON.stringify({ success: false, ambiguous: true }), {
+ status: 200,
+ }),
+ ],
+ ])("reads %s as ambiguous, never as refused", async (_label, respond) => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockImplementation(async () => respond()),
+ );
+
+ await expect(firebillTrack(params)).resolves.toBe(false);
+ expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
+ expect(await failureCauses()).toEqual({ "track/ambiguous": 1 });
+ });
+
+ // **An answer we cannot read is not proof of anything.** firebill may have
+ // taken the event and said so in a shape we did not understand, so calling it
+ // `refused` would log billed usage as lost β€” the same class of untruth this
+ // PR exists to remove, one layer down.
+ it.each([
+ ["an empty 200", () => new Response("", { status: 200 })],
+ [
+ "a 200 that is not JSON",
+ () => new Response("<html>502</html>", { status: 200 }),
+ ],
+ [
+ "a 200 with no success field",
+ () => new Response(JSON.stringify({ queued: true }), { status: 200 }),
+ ],
+ [
+ "a 200 whose success is not a boolean",
+ () => new Response(JSON.stringify({ success: "yes" }), { status: 200 }),
+ ],
+ ])("reads %s as unusable, not as a refusal", async (_label, respond) => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockImplementation(async () => respond()),
+ );
+
+ await expect(firebillTrack(params)).resolves.toBe(false);
+ expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
+ expect(await failureCauses()).toEqual({ "track/unusable": 1 });
+ expect(vi.mocked(logger.error).mock.calls.map(c => c[0])).not.toContain(
+ "firebill refused a usage event; it will not be billed",
+ );
+ });
+
+ it("still reads success:false as a refusal", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockImplementation(async () => refused()),
+ );
+
+ await expect(firebillTrack(params)).resolves.toBe(false);
+ expect(await trackOutcomes()).toEqual({ refused: 1 });
+ });
+
+ // A body that never finished arriving is the network failing, not firebill
+ // answering β€” it has to reach the transport classifier, not be swallowed as
+ // an unusable body.
+ it("treats a body that dies mid-read as a transport failure", async () => {
+ const dying = () => {
+ const response = new Response(JSON.stringify({ success: true }), {
+ status: 200,
+ });
+ vi.spyOn(response, "json").mockRejectedValue(
+ Object.assign(new TypeError("terminated"), {
+ cause: Object.assign(new Error("aborted"), {
+ code: "UND_ERR_BODY_TIMEOUT",
+ }),
+ }),
+ );
+ return response;
+ };
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockImplementation(async () => dying()),
+ );
+
+ await expect(firebillTrack(params)).resolves.toBe(false);
+ expect(await failureCauses()).toEqual({ "track/timeout": 1 });
+ expect(vi.mocked(logger.warn).mock.calls.map(c => c[0])).toContain(
+ "firebill track request did not complete β€” client-side timeout or connection error; firebill did not answer",
+ );
+ });
+
+ it("logs an ambiguous answer as unknown rather than as unbilled usage", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockImplementation(async () => ambiguous()),
+ );
+
+ await firebillTrack(params);
+
+ const messages = vi.mocked(logger.error).mock.calls.map(c => c[0]);
+ expect(messages).toContain(
+ "firebill did not answer; the event may or may not have been accepted",
+ );
+ expect(messages).not.toContain(
+ "firebill refused a usage event; it will not be billed",
+ );
+ });
+
+ // -----------------------------------------------------------------------
+ // Whose failure was it
+ //
+ // ~1,800 warnings per 30h said "firebill may be unavailable" against a
+ // firebill whose server-side p99 was 73ms. They were our own 5s deadline.
+ // -----------------------------------------------------------------------
+
+ it.each([
+ ["our own deadline firing", () => abortError(), "timeout"],
+ ["an undici-wrapped deadline", () => wrapped(abortError()), "timeout"],
+ [
+ "a refused connection",
+ () => wrapped(connectionError("ECONNREFUSED")),
+ "connection",
+ ],
+ [
+ "a reset connection",
+ () => wrapped(connectionError("ECONNRESET")),
+ "connection",
+ ],
+ [
+ "a DNS failure",
+ () => wrapped(connectionError("ENOTFOUND")),
+ "connection",
+ ],
+ ])("labels %s as a client-side %s", async (_label, error, cause) => {
+ vi.stubGlobal("fetch", vi.fn().mockRejectedValue(error()));
+
+ await expect(firebillTrack(params)).resolves.toBe(false);
+ expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
+ expect(await failureCauses()).toEqual({ [`track/${cause}`]: 1 });
+ });
+
+ it("stops blaming firebill for a request that never reached it", async () => {
+ vi.stubGlobal("fetch", vi.fn().mockRejectedValue(wrapped(abortError())));
+
+ await firebillTrack(params);
+
+ // The logger's own overloads make `calls` a one-element tuple to TS; every
+ // call site here passes (message, fields).
+ const warnings = vi.mocked(logger.warn).mock.calls as unknown as [
+ string,
+ Record<string, unknown>,
+ ][];
+ const messages = warnings.map(c => c[0]);
+ expect(messages).toContain(
+ "firebill track request did not complete β€” client-side timeout or connection error; firebill did not answer",
+ );
+ expect(messages.join(" ")).not.toContain("firebill may be unavailable");
+ // The error's own identity, so the class of stall is greppable.
+ // The innermost error's identity, not the `TypeError: fetch failed` undici
+ // wraps it in β€” so the class of stall is greppable.
+ expect(warnings[0][1]).toMatchObject({
+ cause: "timeout",
+ errorName: "TimeoutError",
+ timeoutMs: 5000,
+ });
+ });
+
+ it("names the status when firebill really did answer", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi
+ .fn()
+ .mockImplementation(async () => new Response("nope", { status: 502 })),
+ );
+
+ await expect(firebillTrack(params)).resolves.toBe(false);
+ expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
+ expect(await failureCauses()).toEqual({ "track/non_ok": 1 });
+ // A static message; the status is context, so log search can group these.
+ const [message, fields] = vi.mocked(logger.warn).mock
+ .calls[0] as unknown as [string, Record<string, unknown>];
+ expect(message).toBe(
+ "firebill track attempt failed β€” firebill answered a non-OK status",
+ );
+ expect(fields).toMatchObject({ status: 502 });
});
it("retries a thrown error too", async () => {
@@ -302,6 +580,82 @@ describe("firebillCheck", () => {
});
});
+ it.each([
+ ["our own deadline firing", () => wrapped(abortError()), "check/timeout"],
+ [
+ "a refused connection",
+ () => wrapped(connectionError("ECONNREFUSED")),
+ "check/connection",
+ ],
+ ])(
+ "blames the client, not firebill, for %s",
+ async (_label, error, expected) => {
+ vi.stubGlobal("fetch", vi.fn().mockRejectedValue(error()));
+
+ await expect(firebillCheck(checkParams)).resolves.toEqual({
+ status: "unavailable",
+ });
+ expect(await checkOutcomes()).toEqual({ unavailable: 1 });
+ expect(await failureCauses()).toEqual({ [expected]: 1 });
+ expect(vi.mocked(logger.error).mock.calls.map(c => c[0])).toContain(
+ "firebill check unavailable β€” the request did not complete (client-side timeout or connection error); firebill did not answer",
+ );
+ },
+ );
+
+ it("names the status in context when firebill answered one", async () => {
+ answer({ success: true, allowed: true }, 503);
+
+ await firebillCheck(checkParams);
+
+ expect(await checkOutcomes()).toEqual({ unavailable: 1 });
+ expect(await failureCauses()).toEqual({ "check/non_ok": 1 });
+ const [message, fields] = vi.mocked(logger.error).mock
+ .calls[0] as unknown as [string, Record<string, unknown>];
+ expect(message).toBe(
+ "firebill check unavailable β€” firebill answered a non-OK status",
+ );
+ expect(fields).toMatchObject({ status: 503 });
+ });
+
+ it("records no cause at all for a real answer", async () => {
+ answer({ success: true, allowed: true, remaining: 500 });
+ await firebillCheck(checkParams);
+ expect(await checkOutcomes()).toEqual({ allowed: 1 });
+ expect(await failureCauses()).toEqual({});
+ });
+
+ // `refused` is documented as an explicit `success: false`. An answer we could
+ // not read is not one, and labelling it so would have anyone slicing by cause
+ // counting unreadable answers as declines.
+ it.each([
+ ["a missing allowed", { success: true }, 200],
+ ["a non-boolean allowed", { success: true, allowed: "yes" }, 200],
+ ])("calls %s unusable rather than refused", async (_label, body, status) => {
+ answer(body, status);
+ await firebillCheck(checkParams);
+ expect(await failureCauses()).toEqual({ "check/unusable": 1 });
+ });
+
+ it("calls an unreadable body unusable rather than refused", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi
+ .fn()
+ .mockResolvedValue(new Response("<html>502</html>", { status: 200 })),
+ );
+ await expect(firebillCheck(checkParams)).resolves.toEqual({
+ status: "unavailable",
+ });
+ expect(await failureCauses()).toEqual({ "check/unusable": 1 });
+ });
+
+ it("keeps refused for an explicit success:false", async () => {
+ answer({ success: false });
+ await firebillCheck(checkParams);
+ expect(await failureCauses()).toEqual({ "check/refused": 1 });
+ });
+
it("does not retry β€” this is on the request path", async () => {
const fetchMock = vi.fn().mockRejectedValue(new Error("boom"));
vi.stubGlobal("fetch", fetchMock);
diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.ts
index 2b995aa8e..26506d754 100644
--- a/apps/api/src/services/autumn/firebill.ts
+++ b/apps/api/src/services/autumn/firebill.ts
@@ -5,6 +5,7 @@ import { sampled } from "../../lib/rollout";
import type { LockDeniedReason, TrackParams } from "./types";
import {
firebillCheckTotal,
+ firebillFailureCauseTotal,
firebillRetryTotal,
firebillTrackTotal,
} from "./metrics";
@@ -52,6 +53,142 @@ const FIREBILL_GATED_LOCK_TIMEOUT_MS = 10000;
const FIREBILL_ATTEMPTS = 2;
const FIREBILL_RETRY_DELAY_MS = 150;
+/**
+ * Why a firebill call did not produce a usable answer, at a cardinality safe to
+ * put on a counter. **`timeout` and `connection` are ours, not firebill's**:
+ * the request never completed, so firebill never answered and may never have
+ * been reached.
+ *
+ * - `non_ok` β€” firebill answered, with a status we cannot use.
+ * - `unusable` β€” firebill answered, and the answer is not one we can read: an
+ * empty or malformed body, or a shape missing the field we asked for. **Not a
+ * refusal**: an event may well have been accepted and said so in a body we
+ * could not parse.
+ * - `refused` β€” firebill answered `success: false`. On `/v1/track` that means
+ * it did not take the event; on `/v1/check` it means it declined to answer.
+ * - `ambiguous` β€” firebill answered "I do not know" (a 504, or `ambiguous:
+ * true`): the broker may hold the event already.
+ */
+type FirebillCause =
+ | "timeout"
+ | "connection"
+ | "non_ok"
+ | "unusable"
+ | "refused"
+ | "ambiguous";
+
+/** What a thrown fetch tells us, once it is unwrapped. */
+type TransportFailure = {
+ cause: "timeout" | "connection";
+ errorName?: string;
+ errorCode?: string;
+};
+
+const TIMEOUT_NAMES = new Set(["AbortError", "TimeoutError"]);
+const TIMEOUT_CODES = new Set([
+ "ABORT_ERR",
+ "UND_ERR_ABORTED",
+ "UND_ERR_CONNECT_TIMEOUT",
+ "UND_ERR_HEADERS_TIMEOUT",
+ "UND_ERR_BODY_TIMEOUT",
+]);
+
+/**
+ * Classify a thrown fetch: a deadline we imposed, or a socket/DNS failure.
+ *
+ * Walks `cause`, because undici reports the real error nested inside a bare
+ * `TypeError: fetch failed` β€” reading only the top level would call every
+ * ECONNREFUSED a generic exception.
+ *
+ * Anything that is not recognisably one of our deadlines is `connection`:
+ * ECONNREFUSED, ECONNRESET, EPIPE, ENOTFOUND, EAI_AGAIN and their kin all mean
+ * the request never got an answer, which is the distinction that matters.
+ */
+function classifyTransportError(error: unknown): TransportFailure {
+ let name: string | undefined;
+ let code: string | undefined;
+ let cause: "timeout" | "connection" = "connection";
+
+ // Innermost wins: the outer frame of a wrapped error is always
+ // `TypeError: fetch failed`, which identifies nothing.
+ for (let node: unknown = error, depth = 0; node && depth < 5; depth++) {
+ const candidate = node as { name?: string; code?: string; cause?: unknown };
+ if (typeof candidate.name === "string") name = candidate.name;
+ if (typeof candidate.code === "string") code = candidate.code;
+ if (
+ (candidate.name && TIMEOUT_NAMES.has(candidate.name)) ||
+ (candidate.code && TIMEOUT_CODES.has(candidate.code))
+ ) {
+ cause = "timeout";
+ break;
+ }
+ node = candidate.cause;
+ }
+
+ return {
+ cause,
+ ...(name ? { errorName: name } : {}),
+ ...(code ? { errorCode: code } : {}),
+ };
+}
+
+/**
+ * A body that arrived and would not parse β€” as distinct from one that never
+ * finished arriving, which is a transport failure and is thrown.
+ */
+const UNUSABLE_BODY = Symbol("unusable body");
+
+/** Whether a thrown error is, anywhere down its `cause` chain, a parse error. */
+function isSyntaxError(error: unknown): boolean {
+ for (let node: unknown = error, depth = 0; node && depth < 5; depth++) {
+ if (node instanceof SyntaxError) return true;
+ node = (node as { cause?: unknown }).cause;
+ }
+ return false;
+}
+
+/**
+ * Read a JSON body, separating "firebill sent something we cannot read" from
+ * "the body never arrived".
+ *
+ * **Only a parse error is caught.** A body that times out or loses its
+ * connection after the headers arrived throws here too, and swallowing it would
+ * report a transport failure as an answer firebill gave β€” the exact confusion
+ * this file is being changed to remove. Those are re-thrown so
+ * {@link classifyTransportError} sees them.
+ *
+ * Releases the socket either way: undici holds the connection until the body is
+ * consumed, so a firebill that is erroring would otherwise exhaust the pool and
+ * turn one failure into a run of them.
+ */
+async function readJson(
+ response: Response,
+): Promise<Record<string, unknown> | typeof UNUSABLE_BODY> {
+ try {
+ return (await response.json()) as Record<string, unknown>;
+ } catch (error) {
+ if (!isSyntaxError(error)) throw error;
+ response.body?.cancel().catch(() => {});
+ return UNUSABLE_BODY;
+ }
+}
+
+/**
+ * **firebill saying "I do not know".** A publish whose broker confirm timed out
+ * may have been taken anyway, so firebill answers `504` with
+ * `{"success": false, "ambiguous": true}` rather than the `{"success": false}`
+ * it sends when it is certain it took nothing.
+ *
+ * Read by field first and status second: the field is the contract, and the
+ * status is what an older firebill and every proxy in between would give us.
+ */
+function saysAmbiguous(
+ status: number,
+ body: Record<string, unknown> | typeof UNUSABLE_BODY,
+): boolean {
+ return (body !== UNUSABLE_BODY && body.ambiguous === true) || status === 504;
+}
+
// FIREBILL_ORG_IDS is decoded once at startup by the config schema; the Set is
// built lazily on first use and cached, keyed on the decoded array reference.
let allowlistCache: { source: string[] | undefined; ids: Set<string> } | null =
@@ -162,12 +299,18 @@ export async function firebillTrack(params: TrackParams): Promise<boolean> {
return true;
}
- // `refused` only for an explicit `success: false`, which is firebill saying it
- // did not take the event. A transport failure is `ambiguous`: firebill may
- // have accepted it and be delivering it right now, so this is not proof of
- // lost usage. Either way the caller is told false and must not assume a
- // charge landed. A counter rather than a throw: billing must not fail the
- // customer's request, and a log alone is too quiet to alert on.
+ // **`refused` only for an explicit `success: false`.** That is firebill
+ // saying it did not take the event, and it is the one case that is proof the
+ // usage is gone β€” which is what the alert on this label means. Everything
+ // else is `ambiguous`: firebill may have accepted it and be delivering it
+ // right now. A confirm timeout now says so itself (a 504, or `ambiguous:
+ // true`); before that it arrived as a plain `success: false` and was logged
+ // 2,100 times over eight days as usage that would never be billed, all of
+ // which had in fact settled.
+ //
+ // Either way the caller is told false and must not assume a charge landed. A
+ // counter rather than a throw: billing must not fail the customer's request,
+ // and a log alone is too quiet to alert on.
const outcome = last.reason === "not_success" ? "refused" : "ambiguous";
logger.error(
outcome === "refused"
@@ -183,15 +326,35 @@ export async function firebillTrack(params: TrackParams): Promise<boolean> {
path,
attempts: FIREBILL_ATTEMPTS,
reason: last.reason,
+ cause: last.cause,
+ ...(last.status !== undefined ? { status: last.status } : {}),
+ ...(last.errorName ? { errorName: last.errorName } : {}),
+ ...(last.errorCode ? { errorCode: last.errorCode } : {}),
},
);
firebillTrackTotal.labels(operation, outcome).inc();
+ // Beside the outcome, never inside it: the alerts read `outcome` with
+ // `increase()`, which evaluates per series, so a `cause` label on that counter
+ // would quietly turn one threshold into one threshold per cause.
+ firebillFailureCauseTotal.labels(operation, last.cause).inc();
return false;
}
type AttemptResult =
| { ok: true }
- | { ok: false; reason: "not_ok" | "not_success" | "exception" };
+ | {
+ ok: false;
+ /**
+ * Kept as-is for `firebillRetryTotal`, plus `ambiguous` for the answer
+ * firebill did not used to be able to give.
+ */
+ reason: "not_ok" | "not_success" | "ambiguous" | "exception";
+ /** Who failed. See {@link FirebillCause}. */
+ cause: FirebillCause;
+ status?: number;
+ errorName?: string;
+ errorCode?: string;
+ };
async function firebillAttempt(
path: string,
@@ -230,51 +393,85 @@ async function firebillAttempt(
signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),
});
+ const context = { customerId, entityId, featureId, value, path };
+
if (!response.ok) {
- logger.warn("firebill track attempt failed β€” non-OK response", {
- customerId,
- entityId,
- featureId,
- value,
- path,
+ const body = await readJson(response);
+ if (saysAmbiguous(response.status, body)) {
+ logger.warn("firebill did not know whether it took the event", {
+ ...context,
+ status: response.status,
+ });
+ return {
+ ok: false,
+ reason: "ambiguous",
+ cause: "ambiguous",
+ status: response.status,
+ };
+ }
+ logger.warn(
+ "firebill track attempt failed β€” firebill answered a non-OK status",
+ {
+ ...context,
+ status: response.status,
+ },
+ );
+ return {
+ ok: false,
+ reason: "not_ok",
+ cause: "non_ok",
status: response.status,
- });
- return { ok: false, reason: "not_ok" };
+ };
}
- const body = (await response.json()) as { success?: boolean };
- if (body.success !== true) {
- logger.warn("firebill track attempt did not succeed", {
- customerId,
- entityId,
- featureId,
- value,
- path,
+ const body = await readJson(response);
+ // Belt and braces: a 200 that still says it does not know is not a refusal.
+ if (saysAmbiguous(response.status, body)) {
+ logger.warn("firebill did not know whether it took the event", context);
+ return { ok: false, reason: "ambiguous", cause: "ambiguous" };
+ }
+ // **Only `success: false` is a refusal.** It is the one answer that is proof
+ // the event is gone, and it is what the `refused` alert means. A body we
+ // could not read, or one with no `success` in it, may well be firebill
+ // telling us it took the event in a shape we did not understand β€” so it is
+ // ambiguous, and the caller retries under the same key rather than logging
+ // usage as lost.
+ if (body !== UNUSABLE_BODY && body.success === false) {
+ logger.warn("firebill refused the event β€” it did not take it", context);
+ return { ok: false, reason: "not_success", cause: "refused" };
+ }
+ if (body === UNUSABLE_BODY || body.success !== true) {
+ logger.warn("firebill answered with a body we could not read", {
+ ...context,
+ status: response.status,
});
- return { ok: false, reason: "not_success" };
+ return { ok: false, reason: "ambiguous", cause: "unusable" };
}
- logger.info("firebill track succeeded", {
- customerId,
- entityId,
- featureId,
- value,
- path,
- });
+ logger.info("firebill track succeeded", context);
return { ok: true };
} catch (error) {
// DO NOT fall back to Autumn directly: firebill may have accepted the event
// before this failed, and the Autumn SDK sends no idempotency key, so the
// pair could not be deduped and the customer would be billed twice.
- logger.warn("firebill track attempt failed β€” firebill may be unavailable", {
- customerId,
- entityId,
- featureId,
- value,
- path,
- error,
- });
- return { ok: false, reason: "exception" };
+ const failure = classifyTransportError(error);
+ // **Not "firebill may be unavailable".** The request never completed, which
+ // says nothing about firebill: these are almost all our own 5s deadline
+ // firing against a service whose server-side p99 is 73ms.
+ logger.warn(
+ "firebill track request did not complete β€” client-side timeout or connection error; firebill did not answer",
+ {
+ customerId,
+ entityId,
+ featureId,
+ value,
+ path,
+ timeoutMs: FIREBILL_TIMEOUT_MS,
+ ...failure,
+ error,
+ },
+ );
+ return { ok: false, reason: "exception", ...failure };
}
}
@@ -325,8 +522,12 @@ export async function firebillCheck({
value: number;
properties?: Record<string, unknown>;
}): Promise<FirebillCheckResult> {
+ // `reason` is always a literal from the call sites below, so the rendered
+ // message stays a closed set that log search can group on; anything variable
+ // goes in `extra`.
const unavailable = (
reason: string,
+ cause: FirebillCause,
extra?: Record<string, unknown>,
): FirebillCheckResult => {
logger.error(`firebill check unavailable β€” ${reason}`, {
@@ -334,9 +535,11 @@ export async function firebillCheck({
entityId,
featureId,
value,
+ cause,
...extra,
});
firebillCheckTotal.labels("unavailable").inc();
+ firebillFailureCauseTotal.labels("check", cause).inc();
return { status: "unavailable" };
};
@@ -358,17 +561,28 @@ export async function firebillCheck({
});
if (!response.ok) {
- // Release the socket. Returning without reading or cancelling leaves the
- // body unconsumed, and undici keeps the connection pinned until it is β€”
- // so a firebill that is erroring would exhaust the pool and turn one
- // failure into a run of them. Matters most here: this path is the one
- // that fails open, so the leak would be silently widening the window in
- // which credit checks are skipped.
- response.body?.cancel().catch(() => {});
- return unavailable("non-OK response", { status: response.status });
+ // `readJson` releases the socket whatever the body turns out to be.
+ // Returning without reading or cancelling leaves it unconsumed, and
+ // undici keeps the connection pinned until it is β€” so a firebill that is
+ // erroring would exhaust the pool and turn one failure into a run of
+ // them. Matters most here: this path is the one that fails open, so the
+ // leak would be silently widening the window in which credit checks are
+ // skipped.
+ const errorBody = await readJson(response);
+ return unavailable(
+ "firebill answered a non-OK status",
+ saysAmbiguous(response.status, errorBody) ? "ambiguous" : "non_ok",
+ { status: response.status },
+ );
}
- const body = (await response.json()) as {
+ const answered = await readJson(response);
+ if (answered === UNUSABLE_BODY) {
+ return unavailable("answered with a body we could not read", "unusable", {
+ status: response.status,
+ });
+ }
+ const body = answered as {
success?: boolean;
allowed?: boolean;
remaining?: number;
@@ -376,13 +590,20 @@ export async function firebillCheck({
// `success: false` is firebill saying it does not know β€” an unanswered
// balance, or a gateway lookup that failed. Never a denial.
+ if (body.success === false) {
+ return unavailable("firebill could not answer", "refused");
+ }
+ // Anything else that is not `success: true` is an answer we cannot read
+ // rather than one firebill gave: `refused` is documented as an explicit
+ // `success: false`, and labelling a shape we did not understand with it
+ // would have anyone slicing by cause counting these as declines.
if (body.success !== true) {
- return unavailable("firebill could not answer");
+ return unavailable("answered without a usable `success`", "unusable");
}
// A missing or mis-shaped `allowed` is not something firebill sends today.
// Reading it as a denial would 402 a paying customer, so it fails open.
if (typeof body.allowed !== "boolean") {
- return unavailable("answered without a usable `allowed`");
+ return unavailable("answered without a usable `allowed`", "unusable");
}
// `remaining` clamps downstream limits, and the safe default inverts with
@@ -416,7 +637,15 @@ export async function firebillCheck({
}
return { status: "answered", allowed: body.allowed, remaining };
} catch (error) {
- return unavailable("request threw", { error });
+ const failure = classifyTransportError(error);
+ // **Not "firebill is unavailable".** The request never completed, which is
+ // a statement about this process's socket, DNS or deadline β€” not about
+ // firebill, which never got the chance to answer.
+ return unavailable(
+ "the request did not complete (client-side timeout or connection error); firebill did not answer",
+ failure.cause,
+ { timeoutMs: FIREBILL_TIMEOUT_MS, ...failure, error },
+ );
}
}
@@ -466,19 +695,31 @@ export async function firebillLock({
),
});
+ const context = { customerId, entityId, featureId, value, lockId };
+
if (!response.ok) {
- logger.error("firebill lock failed β€” non-OK response", {
- customerId,
- entityId,
- featureId,
- value,
- lockId,
+ response.body?.cancel().catch(() => {});
+ logger.error("firebill lock failed β€” firebill answered a non-OK status", {
+ ...context,
status: response.status,
});
+ firebillFailureCauseTotal.labels("lock", "non_ok").inc();
return { status: "unavailable" };
}
- const body = (await response.json()) as {
+ const answered = await readJson(response);
+ // An answer we could not parse is firebill answering, not the network
+ // failing β€” so it must not fall through to the transport catch below and be
+ // reported as a connection error.
+ if (answered === UNUSABLE_BODY) {
+ logger.error("firebill lock answered with a body we could not read", {
+ ...context,
+ status: response.status,
+ });
+ firebillFailureCauseTotal.labels("lock", "unusable").inc();
+ return { status: "unavailable" };
+ }
+ const body = answered as {
success?: boolean;
allowed?: boolean;
lock_id?: string;
@@ -487,13 +728,10 @@ export async function firebillLock({
};
if (body.success !== true) {
- logger.error("firebill lock did not succeed", {
- customerId,
- entityId,
- featureId,
- value,
- lockId,
- });
+ logger.error("firebill lock did not succeed", context);
+ firebillFailureCauseTotal
+ .labels("lock", body.success === false ? "refused" : "unusable")
+ .inc();
return { status: "unavailable" };
}
@@ -515,13 +753,11 @@ export async function firebillLock({
// treating it as a denial would hard-stop the check (skipped_no_credits),
// so it maps to unavailable β€” proceed unlocked β€” instead.
if (body.allowed !== true) {
- logger.error("firebill lock answered without a usable `allowed`", {
- customerId,
- entityId,
- featureId,
- value,
- lockId,
- });
+ logger.error(
+ "firebill lock answered without a usable `allowed`",
+ context,
+ );
+ firebillFailureCauseTotal.labels("lock", "unusable").inc();
return { status: "unavailable" };
}
@@ -545,14 +781,23 @@ export async function firebillLock({
...(operationToken ? { operationToken } : {}),
};
} catch (error) {
- logger.error("firebill lock failed β€” firebill may be unavailable", {
- customerId,
- entityId,
- featureId,
- value,
- lockId,
- error,
- });
+ const failure = classifyTransportError(error);
+ logger.error(
+ "firebill lock request did not complete β€” client-side timeout or connection error; firebill did not answer",
+ {
+ customerId,
+ entityId,
+ featureId,
+ value,
+ lockId,
+ timeoutMs: partnerJobToken
+ ? FIREBILL_GATED_LOCK_TIMEOUT_MS
+ : FIREBILL_TIMEOUT_MS,
+ ...failure,
+ error,
+ },
+ );
+ firebillFailureCauseTotal.labels("lock", failure.cause).inc();
return { status: "unavailable" };
}
}
@@ -630,39 +875,62 @@ export async function firebillFinalize({
signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),
});
+ const context = { lockId, action, overrideValue };
+
if (!response.ok) {
- logger.error("firebill finalize failed β€” non-OK response", {
- lockId,
- action,
- overrideValue,
+ const errorBody = await readJson(response);
+ // A settle firebill did not know it took is not a settle it refused. The
+ // finalize contract is a boolean either way β€” the schedule is what
+ // retries β€” but the log has to say which happened.
+ const ambiguous = saysAmbiguous(response.status, errorBody);
+ logger.error(
+ ambiguous
+ ? "firebill did not know whether it took the settle"
+ : "firebill finalize failed β€” firebill answered a non-OK status",
+ { ...context, status: response.status },
+ );
+ firebillFailureCauseTotal
+ .labels("finalize", ambiguous ? "ambiguous" : "non_ok")
+ .inc();
+ return false;
+ }
+
+ const answered = await readJson(response);
+ // Answered but unreadable, which is not the network failing β€” the transport
+ // catch below must not claim it was.
+ if (answered === UNUSABLE_BODY) {
+ logger.error("firebill finalize answered with a body we could not read", {
+ ...context,
status: response.status,
});
+ firebillFailureCauseTotal.labels("finalize", "unusable").inc();
return false;
}
-
- const body = (await response.json()) as { success?: boolean };
+ const body = answered as { success?: boolean };
if (body.success !== true) {
- logger.error("firebill finalize did not succeed", {
- lockId,
- action,
- overrideValue,
- });
+ logger.error("firebill finalize did not succeed", context);
+ firebillFailureCauseTotal
+ .labels("finalize", body.success === false ? "refused" : "unusable")
+ .inc();
return false;
}
- logger.info("firebill finalize succeeded", {
- lockId,
- action,
- overrideValue,
- });
+ logger.info("firebill finalize succeeded", context);
return true;
} catch (error) {
- logger.error("firebill finalize failed β€” firebill may be unavailable", {
- lockId,
- action,
- overrideValue,
- error,
- });
+ const failure = classifyTransportError(error);
+ logger.error(
+ "firebill finalize request did not complete β€” client-side timeout or connection error; firebill did not answer",
+ {
+ lockId,
+ action,
+ overrideValue,
+ timeoutMs: FIREBILL_TIMEOUT_MS,
+ ...failure,
+ error,
+ },
+ );
+ firebillFailureCauseTotal.labels("finalize", failure.cause).inc();
return false;
}
}
diff --git a/apps/api/src/services/autumn/metrics.ts b/apps/api/src/services/autumn/metrics.ts
index 16ecf49c7..5f696fdef 100644
--- a/apps/api/src/services/autumn/metrics.ts
+++ b/apps/api/src/services/autumn/metrics.ts
@@ -26,6 +26,39 @@ export const firebillTrackTotal = new Counter({
labelNames: ["operation", "outcome"] as const,
});
+/**
+ * **Who failed, when a firebill call did not produce a usable answer.**
+ *
+ * A separate series rather than a `cause` label on the counters above, and that
+ * is the whole point. `increase()` evaluates per series, so
+ * `increase(firecrawl_firebill_check_total{outcome="unavailable"}[15m]) > 5` β€”
+ * the live `FirebillCheckUnavailable` rule β€” would silently start comparing 5
+ * against each cause on its own: two causes sitting at 4 apiece would be 8
+ * unanswered credit checks that nobody is paged about. The counters those rules
+ * read stay byte-for-byte what they were; this one adds the detail beside them.
+ *
+ * - `timeout` / `connection` β€” the request never completed. **A client-side or
+ * transport failure, not firebill**: it never answered, and its own
+ * server-side p99 may be fine (73ms, while ~1,800 of these were logged per
+ * 30h against a 5s client deadline).
+ * - `non_ok` β€” firebill answered, with a status we cannot use.
+ * - `unusable` β€” firebill answered, and the answer is not one we can read.
+ * Never proof that anything was lost.
+ * - `refused` β€” firebill answered `success: false`: on a track it did not take
+ * the event, on a check it declined to answer.
+ * - `ambiguous` β€” firebill answered "I do not know" (a 504, or
+ * `ambiguous: true`).
+ *
+ * Only incremented on failure, so a healthy service produces no series at all.
+ */
+export const firebillFailureCauseTotal = new Counter({
+ name: "firecrawl_firebill_failure_cause_total",
+ help: "Why a firebill call did not produce a usable answer",
+ // operation: track|refund|check|lock|finalize
+ // cause: timeout|connection|non_ok|unusable|refused|ambiguous
+ labelNames: ["operation", "cause"] as const,
+});
+
/** Retries of a firebill call that answered `false` or threw. */
export const firebillRetryTotal = new Counter({
name: "firecrawl_firebill_retry_total",