Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(billing): read firebill's "I do not know", and stop blaming it for our own stalls (#4514)
Rakshith Ramprakashfb390af29f5d
diff --git a/apps/api/src/services/autumn/__tests__/firebill.test.ts b/apps/api/src/services/autumn/__tests__/firebill.test.tsindex 006b41201..c52442c44 100644--- a/apps/api/src/services/autumn/__tests__/firebill.test.ts+++ b/apps/api/src/services/autumn/__tests__/firebill.test.ts@@ -4,8 +4,10 @@ import {firebillTrack,shouldRouteToFirebill,} from "../firebill";+import { logger } from "../../../lib/logger";import {firebillCheckTotal,+ firebillFailureCauseTotal,firebillRetryTotal,firebillTrackTotal,} from "../metrics";@@ -30,6 +32,23 @@ const ok = () =>new Response(JSON.stringify({ success: true }), { status: 200 });const refused = () =>new Response(JSON.stringify({ success: false }), { status: 200 });+/** firebill's confirm timed out: it does not know whether it took the event. */+const ambiguous = () =>+ new Response(JSON.stringify({ success: false, ambiguous: true }), {+ status: 504,+ });++/** What undici actually throws: the real error nested under a bare TypeError. */+const wrapped = (inner: Error) =>+ Object.assign(new TypeError("fetch failed"), { cause: inner });++const abortError = () =>+ Object.assign(new Error("The operation was aborted due to timeout"), {+ name: "TimeoutError",+ });++const connectionError = (code: string) =>+ Object.assign(new Error(`connect ${code} 10.0.0.1:8080`), { code });const params = {customerId: "org-1",@@ -48,6 +67,10 @@ beforeEach(() => {firebillTrackTotal.reset();firebillRetryTotal.reset();firebillCheckTotal.reset();+ firebillFailureCauseTotal.reset();+ vi.mocked(logger.info).mockClear();+ vi.mocked(logger.warn).mockClear();+ vi.mocked(logger.error).mockClear();});afterEach(() => vi.unstubAllGlobals());@@ -90,6 +113,35 @@ describe("shouldRouteToFirebill", () => {});});+/**+ * The outcome counter alone β the one the alerts read, which must keep exactly+ * the labels it always had.+ */+const trackOutcomes = async () =>+ Object.fromEntries(+ (await firebillTrackTotal.get()).values.map(v => [+ v.labels.outcome,+ v.value,+ ]),+ );++const checkOutcomes = async () =>+ Object.fromEntries(+ (await firebillCheckTotal.get()).values.map(v => [+ v.labels.outcome,+ v.value,+ ]),+ );++/** The separate cause counter, as `operation/cause`. */+const failureCauses = async () =>+ Object.fromEntries(+ (await firebillFailureCauseTotal.get()).values.map(v => [+ `${v.labels.operation}/${v.labels.cause}`,+ v.value,+ ]),+ );+describe("firebillTrack", () => {it("retries a refusal and reports success if the retry lands", async () => {const fetchMock = vi@@ -147,26 +199,252 @@ describe("firebillTrack", () => {});it("separates an explicit refusal from an ambiguous transport failure", async () => {- const outcomes = async () =>- Object.fromEntries(- (await firebillTrackTotal.get()).values.map(v => [- v.labels.outcome,- v.value,- ]),- );-vi.stubGlobal("fetch",vi.fn().mockImplementation(async () => refused()),);await firebillTrack(params);- expect(await outcomes()).toEqual({ refused: 1 });+ expect(await trackOutcomes()).toEqual({ refused: 1 });+ expect(await failureCauses()).toEqual({ "track/refused": 1 });firebillTrackTotal.reset();+ firebillFailureCauseTotal.reset();vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("timeout")));await firebillTrack(params);// A timeout is not proof the usage was lost β firebill may have taken it.- expect(await outcomes()).toEqual({ ambiguous: 1 });+ expect(await trackOutcomes()).toEqual({ ambiguous: 1 });+ expect(await failureCauses()).toEqual({ "track/connection": 1 });+ });++ // -----------------------------------------------------------------------+ // The alerts' own series+ // -----------------------------------------------------------------------++ it("leaves the alerted counters with exactly the labels they had", async () => {+ // `increase()` evaluates per series, so a `cause` label here would turn+ // `increase(firecrawl_firebill_track_total{outcome="refused"}[15m]) > 0`+ // into one threshold per cause. The detail lives beside it instead.+ vi.stubGlobal(+ "fetch",+ vi.fn().mockImplementation(async () => refused()),+ );+ await firebillTrack(params);++ const [series] = (await firebillTrackTotal.get()).values;+ expect(Object.keys(series.labels).sort()).toEqual(["operation", "outcome"]);++ const [cause] = (await firebillFailureCauseTotal.get()).values;+ expect(Object.keys(cause.labels).sort()).toEqual(["cause", "operation"]);+ expect(cause.labels).toMatchObject({+ operation: "track",+ cause: "refused",+ });+ });++ it("counts a refund's cause under its own operation", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockImplementation(async () => refused()),+ );+ await firebillTrack({ ...params, value: -7 });+ expect(await failureCauses()).toEqual({ "refund/refused": 1 });+ });++ // -----------------------------------------------------------------------+ // firebill saying "I do not know"+ //+ // A publish whose broker confirm timed out may have been taken anyway. Over+ // eight days ~2,100 of these were answered `{"success": false}` β the same+ // body a refusal sends β and every one of those events had in fact settled,+ // while this client logged each as usage that would never be billed.+ // -----------------------------------------------------------------------++ it.each([+ ["a 504 carrying the field", () => ambiguous()],+ ["a 504 with no body at all", () => new Response(null, { status: 504 })],+ [+ "a 200 that says so by field",+ () =>+ new Response(JSON.stringify({ success: false, ambiguous: true }), {+ status: 200,+ }),+ ],+ ])("reads %s as ambiguous, never as refused", async (_label, respond) => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockImplementation(async () => respond()),+ );++ await expect(firebillTrack(params)).resolves.toBe(false);+ expect(await trackOutcomes()).toEqual({ ambiguous: 1 });+ expect(await failureCauses()).toEqual({ "track/ambiguous": 1 });+ });++ // **An answer we cannot read is not proof of anything.** firebill may have+ // taken the event and said so in a shape we did not understand, so calling it+ // `refused` would log billed usage as lost β the same class of untruth this+ // PR exists to remove, one layer down.+ it.each([+ ["an empty 200", () => new Response("", { status: 200 })],+ [+ "a 200 that is not JSON",+ () => new Response("<html>502</html>", { status: 200 }),+ ],+ [+ "a 200 with no success field",+ () => new Response(JSON.stringify({ queued: true }), { status: 200 }),+ ],+ [+ "a 200 whose success is not a boolean",+ () => new Response(JSON.stringify({ success: "yes" }), { status: 200 }),+ ],+ ])("reads %s as unusable, not as a refusal", async (_label, respond) => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockImplementation(async () => respond()),+ );++ await expect(firebillTrack(params)).resolves.toBe(false);+ expect(await trackOutcomes()).toEqual({ ambiguous: 1 });+ expect(await failureCauses()).toEqual({ "track/unusable": 1 });+ expect(vi.mocked(logger.error).mock.calls.map(c => c[0])).not.toContain(+ "firebill refused a usage event; it will not be billed",+ );+ });++ it("still reads success:false as a refusal", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockImplementation(async () => refused()),+ );++ await expect(firebillTrack(params)).resolves.toBe(false);+ expect(await trackOutcomes()).toEqual({ refused: 1 });+ });++ // A body that never finished arriving is the network failing, not firebill+ // answering β it has to reach the transport classifier, not be swallowed as+ // an unusable body.+ it("treats a body that dies mid-read as a transport failure", async () => {+ const dying = () => {+ const response = new Response(JSON.stringify({ success: true }), {+ status: 200,+ });+ vi.spyOn(response, "json").mockRejectedValue(+ Object.assign(new TypeError("terminated"), {+ cause: Object.assign(new Error("aborted"), {+ code: "UND_ERR_BODY_TIMEOUT",+ }),+ }),+ );+ return response;+ };+ vi.stubGlobal(+ "fetch",+ vi.fn().mockImplementation(async () => dying()),+ );++ await expect(firebillTrack(params)).resolves.toBe(false);+ expect(await failureCauses()).toEqual({ "track/timeout": 1 });+ expect(vi.mocked(logger.warn).mock.calls.map(c => c[0])).toContain(+ "firebill track request did not complete β client-side timeout or connection error; firebill did not answer",+ );+ });++ it("logs an ambiguous answer as unknown rather than as unbilled usage", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockImplementation(async () => ambiguous()),+ );++ await firebillTrack(params);++ const messages = vi.mocked(logger.error).mock.calls.map(c => c[0]);+ expect(messages).toContain(+ "firebill did not answer; the event may or may not have been accepted",+ );+ expect(messages).not.toContain(+ "firebill refused a usage event; it will not be billed",+ );+ });++ // -----------------------------------------------------------------------+ // Whose failure was it+ //+ // ~1,800 warnings per 30h said "firebill may be unavailable" against a+ // firebill whose server-side p99 was 73ms. They were our own 5s deadline.+ // -----------------------------------------------------------------------++ it.each([+ ["our own deadline firing", () => abortError(), "timeout"],+ ["an undici-wrapped deadline", () => wrapped(abortError()), "timeout"],+ [+ "a refused connection",+ () => wrapped(connectionError("ECONNREFUSED")),+ "connection",+ ],+ [+ "a reset connection",+ () => wrapped(connectionError("ECONNRESET")),+ "connection",+ ],+ [+ "a DNS failure",+ () => wrapped(connectionError("ENOTFOUND")),+ "connection",+ ],+ ])("labels %s as a client-side %s", async (_label, error, cause) => {+ vi.stubGlobal("fetch", vi.fn().mockRejectedValue(error()));++ await expect(firebillTrack(params)).resolves.toBe(false);+ expect(await trackOutcomes()).toEqual({ ambiguous: 1 });+ expect(await failureCauses()).toEqual({ [`track/${cause}`]: 1 });+ });++ it("stops blaming firebill for a request that never reached it", async () => {+ vi.stubGlobal("fetch", vi.fn().mockRejectedValue(wrapped(abortError())));++ await firebillTrack(params);++ // The logger's own overloads make `calls` a one-element tuple to TS; every+ // call site here passes (message, fields).+ const warnings = vi.mocked(logger.warn).mock.calls as unknown as [+ string,+ Record<string, unknown>,+ ][];+ const messages = warnings.map(c => c[0]);+ expect(messages).toContain(+ "firebill track request did not complete β client-side timeout or connection error; firebill did not answer",+ );+ expect(messages.join(" ")).not.toContain("firebill may be unavailable");+ // The error's own identity, so the class of stall is greppable.+ // The innermost error's identity, not the `TypeError: fetch failed` undici+ // wraps it in β so the class of stall is greppable.+ expect(warnings[0][1]).toMatchObject({+ cause: "timeout",+ errorName: "TimeoutError",+ timeoutMs: 5000,+ });+ });++ it("names the status when firebill really did answer", async () => {+ vi.stubGlobal(+ "fetch",+ vi+ .fn()+ .mockImplementation(async () => new Response("nope", { status: 502 })),+ );++ await expect(firebillTrack(params)).resolves.toBe(false);+ expect(await trackOutcomes()).toEqual({ ambiguous: 1 });+ expect(await failureCauses()).toEqual({ "track/non_ok": 1 });+ // A static message; the status is context, so log search can group these.+ const [message, fields] = vi.mocked(logger.warn).mock+ .calls[0] as unknown as [string, Record<string, unknown>];+ expect(message).toBe(+ "firebill track attempt failed β firebill answered a non-OK status",+ );+ expect(fields).toMatchObject({ status: 502 });});it("retries a thrown error too", async () => {@@ -302,6 +580,82 @@ describe("firebillCheck", () => {});});+ it.each([+ ["our own deadline firing", () => wrapped(abortError()), "check/timeout"],+ [+ "a refused connection",+ () => wrapped(connectionError("ECONNREFUSED")),+ "check/connection",+ ],+ ])(+ "blames the client, not firebill, for %s",+ async (_label, error, expected) => {+ vi.stubGlobal("fetch", vi.fn().mockRejectedValue(error()));++ await expect(firebillCheck(checkParams)).resolves.toEqual({+ status: "unavailable",+ });+ expect(await checkOutcomes()).toEqual({ unavailable: 1 });+ expect(await failureCauses()).toEqual({ [expected]: 1 });+ expect(vi.mocked(logger.error).mock.calls.map(c => c[0])).toContain(+ "firebill check unavailable β the request did not complete (client-side timeout or connection error); firebill did not answer",+ );+ },+ );++ it("names the status in context when firebill answered one", async () => {+ answer({ success: true, allowed: true }, 503);++ await firebillCheck(checkParams);++ expect(await checkOutcomes()).toEqual({ unavailable: 1 });+ expect(await failureCauses()).toEqual({ "check/non_ok": 1 });+ const [message, fields] = vi.mocked(logger.error).mock+ .calls[0] as unknown as [string, Record<string, unknown>];+ expect(message).toBe(+ "firebill check unavailable β firebill answered a non-OK status",+ );+ expect(fields).toMatchObject({ status: 503 });+ });++ it("records no cause at all for a real answer", async () => {+ answer({ success: true, allowed: true, remaining: 500 });+ await firebillCheck(checkParams);+ expect(await checkOutcomes()).toEqual({ allowed: 1 });+ expect(await failureCauses()).toEqual({});+ });++ // `refused` is documented as an explicit `success: false`. An answer we could+ // not read is not one, and labelling it so would have anyone slicing by cause+ // counting unreadable answers as declines.+ it.each([+ ["a missing allowed", { success: true }, 200],+ ["a non-boolean allowed", { success: true, allowed: "yes" }, 200],+ ])("calls %s unusable rather than refused", async (_label, body, status) => {+ answer(body, status);+ await firebillCheck(checkParams);+ expect(await failureCauses()).toEqual({ "check/unusable": 1 });+ });++ it("calls an unreadable body unusable rather than refused", async () => {+ vi.stubGlobal(+ "fetch",+ vi+ .fn()+ .mockResolvedValue(new Response("<html>502</html>", { status: 200 })),+ );+ await expect(firebillCheck(checkParams)).resolves.toEqual({+ status: "unavailable",+ });+ expect(await failureCauses()).toEqual({ "check/unusable": 1 });+ });++ it("keeps refused for an explicit success:false", async () => {+ answer({ success: false });+ await firebillCheck(checkParams);+ expect(await failureCauses()).toEqual({ "check/refused": 1 });+ });+it("does not retry β this is on the request path", async () => {const fetchMock = vi.fn().mockRejectedValue(new Error("boom"));vi.stubGlobal("fetch", fetchMock);diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.tsindex 2b995aa8e..26506d754 100644--- a/apps/api/src/services/autumn/firebill.ts+++ b/apps/api/src/services/autumn/firebill.ts@@ -5,6 +5,7 @@ import { sampled } from "../../lib/rollout";import type { LockDeniedReason, TrackParams } from "./types";import {firebillCheckTotal,+ firebillFailureCauseTotal,firebillRetryTotal,firebillTrackTotal,} from "./metrics";@@ -52,6 +53,142 @@ const FIREBILL_GATED_LOCK_TIMEOUT_MS = 10000;const FIREBILL_ATTEMPTS = 2;const FIREBILL_RETRY_DELAY_MS = 150;+/**+ * Why a firebill call did not produce a usable answer, at a cardinality safe to+ * put on a counter. **`timeout` and `connection` are ours, not firebill's**:+ * the request never completed, so firebill never answered and may never have+ * been reached.+ *+ * - `non_ok` β firebill answered, with a status we cannot use.+ * - `unusable` β firebill answered, and the answer is not one we can read: an+ * empty or malformed body, or a shape missing the field we asked for. **Not a+ * refusal**: an event may well have been accepted and said so in a body we+ * could not parse.+ * - `refused` β firebill answered `success: false`. On `/v1/track` that means+ * it did not take the event; on `/v1/check` it means it declined to answer.+ * - `ambiguous` β firebill answered "I do not know" (a 504, or `ambiguous:+ * true`): the broker may hold the event already.+ */+type FirebillCause =+ | "timeout"+ | "connection"+ | "non_ok"+ | "unusable"+ | "refused"+ | "ambiguous";++/** What a thrown fetch tells us, once it is unwrapped. */+type TransportFailure = {+ cause: "timeout" | "connection";+ errorName?: string;+ errorCode?: string;+};++const TIMEOUT_NAMES = new Set(["AbortError", "TimeoutError"]);+const TIMEOUT_CODES = new Set([+ "ABORT_ERR",+ "UND_ERR_ABORTED",+ "UND_ERR_CONNECT_TIMEOUT",+ "UND_ERR_HEADERS_TIMEOUT",+ "UND_ERR_BODY_TIMEOUT",+]);++/**+ * Classify a thrown fetch: a deadline we imposed, or a socket/DNS failure.+ *+ * Walks `cause`, because undici reports the real error nested inside a bare+ * `TypeError: fetch failed` β reading only the top level would call every+ * ECONNREFUSED a generic exception.+ *+ * Anything that is not recognisably one of our deadlines is `connection`:+ * ECONNREFUSED, ECONNRESET, EPIPE, ENOTFOUND, EAI_AGAIN and their kin all mean+ * the request never got an answer, which is the distinction that matters.+ */+function classifyTransportError(error: unknown): TransportFailure {+ let name: string | undefined;+ let code: string | undefined;+ let cause: "timeout" | "connection" = "connection";++ // Innermost wins: the outer frame of a wrapped error is always+ // `TypeError: fetch failed`, which identifies nothing.+ for (let node: unknown = error, depth = 0; node && depth < 5; depth++) {+ const candidate = node as { name?: string; code?: string; cause?: unknown };+ if (typeof candidate.name === "string") name = candidate.name;+ if (typeof candidate.code === "string") code = candidate.code;+ if (+ (candidate.name && TIMEOUT_NAMES.has(candidate.name)) ||+ (candidate.code && TIMEOUT_CODES.has(candidate.code))+ ) {+ cause = "timeout";+ break;+ }+ node = candidate.cause;+ }++ return {+ cause,+ ...(name ? { errorName: name } : {}),+ ...(code ? { errorCode: code } : {}),+ };+}++/**+ * A body that arrived and would not parse β as distinct from one that never+ * finished arriving, which is a transport failure and is thrown.+ */+const UNUSABLE_BODY = Symbol("unusable body");++/** Whether a thrown error is, anywhere down its `cause` chain, a parse error. */+function isSyntaxError(error: unknown): boolean {+ for (let node: unknown = error, depth = 0; node && depth < 5; depth++) {+ if (node instanceof SyntaxError) return true;+ node = (node as { cause?: unknown }).cause;+ }+ return false;+}++/**+ * Read a JSON body, separating "firebill sent something we cannot read" from+ * "the body never arrived".+ *+ * **Only a parse error is caught.** A body that times out or loses its+ * connection after the headers arrived throws here too, and swallowing it would+ * report a transport failure as an answer firebill gave β the exact confusion+ * this file is being changed to remove. Those are re-thrown so+ * {@link classifyTransportError} sees them.+ *+ * Releases the socket either way: undici holds the connection until the body is+ * consumed, so a firebill that is erroring would otherwise exhaust the pool and+ * turn one failure into a run of them.+ */+async function readJson(+ response: Response,+): Promise<Record<string, unknown> | typeof UNUSABLE_BODY> {+ try {+ return (await response.json()) as Record<string, unknown>;+ } catch (error) {+ if (!isSyntaxError(error)) throw error;+ response.body?.cancel().catch(() => {});+ return UNUSABLE_BODY;+ }+}++/**+ * **firebill saying "I do not know".** A publish whose broker confirm timed out+ * may have been taken anyway, so firebill answers `504` with+ * `{"success": false, "ambiguous": true}` rather than the `{"success": false}`+ * it sends when it is certain it took nothing.+ *+ * Read by field first and status second: the field is the contract, and the+ * status is what an older firebill and every proxy in between would give us.+ */+function saysAmbiguous(+ status: number,+ body: Record<string, unknown> | typeof UNUSABLE_BODY,+): boolean {+ return (body !== UNUSABLE_BODY && body.ambiguous === true) || status === 504;+}+// FIREBILL_ORG_IDS is decoded once at startup by the config schema; the Set is// built lazily on first use and cached, keyed on the decoded array reference.let allowlistCache: { source: string[] | undefined; ids: Set<string> } | null =@@ -162,12 +299,18 @@ export async function firebillTrack(params: TrackParams): Promise<boolean> {return true;}- // `refused` only for an explicit `success: false`, which is firebill saying it- // did not take the event. A transport failure is `ambiguous`: firebill may- // have accepted it and be delivering it right now, so this is not proof of- // lost usage. Either way the caller is told false and must not assume a- // charge landed. A counter rather than a throw: billing must not fail the- // customer's request, and a log alone is too quiet to alert on.+ // **`refused` only for an explicit `success: false`.** That is firebill+ // saying it did not take the event, and it is the one case that is proof the+ // usage is gone β which is what the alert on this label means. Everything+ // else is `ambiguous`: firebill may have accepted it and be delivering it+ // right now. A confirm timeout now says so itself (a 504, or `ambiguous:+ // true`); before that it arrived as a plain `success: false` and was logged+ // 2,100 times over eight days as usage that would never be billed, all of+ // which had in fact settled.+ //+ // Either way the caller is told false and must not assume a charge landed. A+ // counter rather than a throw: billing must not fail the customer's request,+ // and a log alone is too quiet to alert on.const outcome = last.reason === "not_success" ? "refused" : "ambiguous";logger.error(outcome === "refused"@@ -183,15 +326,35 @@ export async function firebillTrack(params: TrackParams): Promise<boolean> {path,attempts: FIREBILL_ATTEMPTS,reason: last.reason,+ cause: last.cause,+ ...(last.status !== undefined ? { status: last.status } : {}),+ ...(last.errorName ? { errorName: last.errorName } : {}),+ ...(last.errorCode ? { errorCode: last.errorCode } : {}),},);firebillTrackTotal.labels(operation, outcome).inc();+ // Beside the outcome, never inside it: the alerts read `outcome` with+ // `increase()`, which evaluates per series, so a `cause` label on that counter+ // would quietly turn one threshold into one threshold per cause.+ firebillFailureCauseTotal.labels(operation, last.cause).inc();return false;}type AttemptResult =| { ok: true }- | { ok: false; reason: "not_ok" | "not_success" | "exception" };+ | {+ ok: false;+ /**+ * Kept as-is for `firebillRetryTotal`, plus `ambiguous` for the answer+ * firebill did not used to be able to give.+ */+ reason: "not_ok" | "not_success" | "ambiguous" | "exception";+ /** Who failed. See {@link FirebillCause}. */+ cause: FirebillCause;+ status?: number;+ errorName?: string;+ errorCode?: string;+ };async function firebillAttempt(path: string,@@ -230,51 +393,85 @@ async function firebillAttempt(signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),});+ const context = { customerId, entityId, featureId, value, path };+if (!response.ok) {- logger.warn("firebill track attempt failed β non-OK response", {- customerId,- entityId,- featureId,- value,- path,+ const body = await readJson(response);+ if (saysAmbiguous(response.status, body)) {+ logger.warn("firebill did not know whether it took the event", {+ ...context,+ status: response.status,+ });+ return {+ ok: false,+ reason: "ambiguous",+ cause: "ambiguous",+ status: response.status,+ };+ }+ logger.warn(+ "firebill track attempt failed β firebill answered a non-OK status",+ {+ ...context,+ status: response.status,+ },+ );+ return {+ ok: false,+ reason: "not_ok",+ cause: "non_ok",status: response.status,- });- return { ok: false, reason: "not_ok" };+ };}- const body = (await response.json()) as { success?: boolean };- if (body.success !== true) {- logger.warn("firebill track attempt did not succeed", {- customerId,- entityId,- featureId,- value,- path,+ const body = await readJson(response);+ // Belt and braces: a 200 that still says it does not know is not a refusal.+ if (saysAmbiguous(response.status, body)) {+ logger.warn("firebill did not know whether it took the event", context);+ return { ok: false, reason: "ambiguous", cause: "ambiguous" };+ }+ // **Only `success: false` is a refusal.** It is the one answer that is proof+ // the event is gone, and it is what the `refused` alert means. A body we+ // could not read, or one with no `success` in it, may well be firebill+ // telling us it took the event in a shape we did not understand β so it is+ // ambiguous, and the caller retries under the same key rather than logging+ // usage as lost.+ if (body !== UNUSABLE_BODY && body.success === false) {+ logger.warn("firebill refused the event β it did not take it", context);+ return { ok: false, reason: "not_success", cause: "refused" };+ }+ if (body === UNUSABLE_BODY || body.success !== true) {+ logger.warn("firebill answered with a body we could not read", {+ ...context,+ status: response.status,});- return { ok: false, reason: "not_success" };+ return { ok: false, reason: "ambiguous", cause: "unusable" };}- logger.info("firebill track succeeded", {- customerId,- entityId,- featureId,- value,- path,- });+ logger.info("firebill track succeeded", context);return { ok: true };} catch (error) {// DO NOT fall back to Autumn directly: firebill may have accepted the event// before this failed, and the Autumn SDK sends no idempotency key, so the// pair could not be deduped and the customer would be billed twice.- logger.warn("firebill track attempt failed β firebill may be unavailable", {- customerId,- entityId,- featureId,- value,- path,- error,- });- return { ok: false, reason: "exception" };+ const failure = classifyTransportError(error);+ // **Not "firebill may be unavailable".** The request never completed, which+ // says nothing about firebill: these are almost all our own 5s deadline+ // firing against a service whose server-side p99 is 73ms.+ logger.warn(+ "firebill track request did not complete β client-side timeout or connection error; firebill did not answer",+ {+ customerId,+ entityId,+ featureId,+ value,+ path,+ timeoutMs: FIREBILL_TIMEOUT_MS,+ ...failure,+ error,+ },+ );+ return { ok: false, reason: "exception", ...failure };}}@@ -325,8 +522,12 @@ export async function firebillCheck({value: number;properties?: Record<string, unknown>;}): Promise<FirebillCheckResult> {+ // `reason` is always a literal from the call sites below, so the rendered+ // message stays a closed set that log search can group on; anything variable+ // goes in `extra`.const unavailable = (reason: string,+ cause: FirebillCause,extra?: Record<string, unknown>,): FirebillCheckResult => {logger.error(`firebill check unavailable β ${reason}`, {@@ -334,9 +535,11 @@ export async function firebillCheck({entityId,featureId,value,+ cause,...extra,});firebillCheckTotal.labels("unavailable").inc();+ firebillFailureCauseTotal.labels("check", cause).inc();return { status: "unavailable" };};@@ -358,17 +561,28 @@ export async function firebillCheck({});if (!response.ok) {- // Release the socket. Returning without reading or cancelling leaves the- // body unconsumed, and undici keeps the connection pinned until it is β- // so a firebill that is erroring would exhaust the pool and turn one- // failure into a run of them. Matters most here: this path is the one- // that fails open, so the leak would be silently widening the window in- // which credit checks are skipped.- response.body?.cancel().catch(() => {});- return unavailable("non-OK response", { status: response.status });+ // `readJson` releases the socket whatever the body turns out to be.+ // Returning without reading or cancelling leaves it unconsumed, and+ // undici keeps the connection pinned until it is β so a firebill that is+ // erroring would exhaust the pool and turn one failure into a run of+ // them. Matters most here: this path is the one that fails open, so the+ // leak would be silently widening the window in which credit checks are+ // skipped.+ const errorBody = await readJson(response);+ return unavailable(+ "firebill answered a non-OK status",+ saysAmbiguous(response.status, errorBody) ? "ambiguous" : "non_ok",+ { status: response.status },+ );}- const body = (await response.json()) as {+ const answered = await readJson(response);+ if (answered === UNUSABLE_BODY) {+ return unavailable("answered with a body we could not read", "unusable", {+ status: response.status,+ });+ }+ const body = answered as {success?: boolean;allowed?: boolean;remaining?: number;@@ -376,13 +590,20 @@ export async function firebillCheck({// `success: false` is firebill saying it does not know β an unanswered// balance, or a gateway lookup that failed. Never a denial.+ if (body.success === false) {+ return unavailable("firebill could not answer", "refused");+ }+ // Anything else that is not `success: true` is an answer we cannot read+ // rather than one firebill gave: `refused` is documented as an explicit+ // `success: false`, and labelling a shape we did not understand with it+ // would have anyone slicing by cause counting these as declines.if (body.success !== true) {- return unavailable("firebill could not answer");+ return unavailable("answered without a usable `success`", "unusable");}// A missing or mis-shaped `allowed` is not something firebill sends today.// Reading it as a denial would 402 a paying customer, so it fails open.if (typeof body.allowed !== "boolean") {- return unavailable("answered without a usable `allowed`");+ return unavailable("answered without a usable `allowed`", "unusable");}// `remaining` clamps downstream limits, and the safe default inverts with@@ -416,7 +637,15 @@ export async function firebillCheck({}return { status: "answered", allowed: body.allowed, remaining };} catch (error) {- return unavailable("request threw", { error });+ const failure = classifyTransportError(error);+ // **Not "firebill is unavailable".** The request never completed, which is+ // a statement about this process's socket, DNS or deadline β not about+ // firebill, which never got the chance to answer.+ return unavailable(+ "the request did not complete (client-side timeout or connection error); firebill did not answer",+ failure.cause,+ { timeoutMs: FIREBILL_TIMEOUT_MS, ...failure, error },+ );}}@@ -466,19 +695,31 @@ export async function firebillLock({),});+ const context = { customerId, entityId, featureId, value, lockId };+if (!response.ok) {- logger.error("firebill lock failed β non-OK response", {- customerId,- entityId,- featureId,- value,- lockId,+ response.body?.cancel().catch(() => {});+ logger.error("firebill lock failed β firebill answered a non-OK status", {+ ...context,status: response.status,});+ firebillFailureCauseTotal.labels("lock", "non_ok").inc();return { status: "unavailable" };}- const body = (await response.json()) as {+ const answered = await readJson(response);+ // An answer we could not parse is firebill answering, not the network+ // failing β so it must not fall through to the transport catch below and be+ // reported as a connection error.+ if (answered === UNUSABLE_BODY) {+ logger.error("firebill lock answered with a body we could not read", {+ ...context,+ status: response.status,+ });+ firebillFailureCauseTotal.labels("lock", "unusable").inc();+ return { status: "unavailable" };+ }+ const body = answered as {success?: boolean;allowed?: boolean;lock_id?: string;@@ -487,13 +728,10 @@ export async function firebillLock({};if (body.success !== true) {- logger.error("firebill lock did not succeed", {- customerId,- entityId,- featureId,- value,- lockId,- });+ logger.error("firebill lock did not succeed", context);+ firebillFailureCauseTotal+ .labels("lock", body.success === false ? "refused" : "unusable")+ .inc();return { status: "unavailable" };}@@ -515,13 +753,11 @@ export async function firebillLock({// treating it as a denial would hard-stop the check (skipped_no_credits),// so it maps to unavailable β proceed unlocked β instead.if (body.allowed !== true) {- logger.error("firebill lock answered without a usable `allowed`", {- customerId,- entityId,- featureId,- value,- lockId,- });+ logger.error(+ "firebill lock answered without a usable `allowed`",+ context,+ );+ firebillFailureCauseTotal.labels("lock", "unusable").inc();return { status: "unavailable" };}@@ -545,14 +781,23 @@ export async function firebillLock({...(operationToken ? { operationToken } : {}),};} catch (error) {- logger.error("firebill lock failed β firebill may be unavailable", {- customerId,- entityId,- featureId,- value,- lockId,- error,- });+ const failure = classifyTransportError(error);+ logger.error(+ "firebill lock request did not complete β client-side timeout or connection error; firebill did not answer",+ {+ customerId,+ entityId,+ featureId,+ value,+ lockId,+ timeoutMs: partnerJobToken+ ? FIREBILL_GATED_LOCK_TIMEOUT_MS+ : FIREBILL_TIMEOUT_MS,+ ...failure,+ error,+ },+ );+ firebillFailureCauseTotal.labels("lock", failure.cause).inc();return { status: "unavailable" };}}@@ -630,39 +875,62 @@ export async function firebillFinalize({signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),});+ const context = { lockId, action, overrideValue };+if (!response.ok) {- logger.error("firebill finalize failed β non-OK response", {- lockId,- action,- overrideValue,+ const errorBody = await readJson(response);+ // A settle firebill did not know it took is not a settle it refused. The+ // finalize contract is a boolean either way β the schedule is what+ // retries β but the log has to say which happened.+ const ambiguous = saysAmbiguous(response.status, errorBody);+ logger.error(+ ambiguous+ ? "firebill did not know whether it took the settle"+ : "firebill finalize failed β firebill answered a non-OK status",+ { ...context, status: response.status },+ );+ firebillFailureCauseTotal+ .labels("finalize", ambiguous ? "ambiguous" : "non_ok")+ .inc();+ return false;+ }++ const answered = await readJson(response);+ // Answered but unreadable, which is not the network failing β the transport+ // catch below must not claim it was.+ if (answered === UNUSABLE_BODY) {+ logger.error("firebill finalize answered with a body we could not read", {+ ...context,status: response.status,});+ firebillFailureCauseTotal.labels("finalize", "unusable").inc();return false;}-- const body = (await response.json()) as { success?: boolean };+ const body = answered as { success?: boolean };if (body.success !== true) {- logger.error("firebill finalize did not succeed", {- lockId,- action,- overrideValue,- });+ logger.error("firebill finalize did not succeed", context);+ firebillFailureCauseTotal+ .labels("finalize", body.success === false ? "refused" : "unusable")+ .inc();return false;}- logger.info("firebill finalize succeeded", {- lockId,- action,- overrideValue,- });+ logger.info("firebill finalize succeeded", context);return true;} catch (error) {- logger.error("firebill finalize failed β firebill may be unavailable", {- lockId,- action,- overrideValue,- error,- });+ const failure = classifyTransportError(error);+ logger.error(+ "firebill finalize request did not complete β client-side timeout or connection error; firebill did not answer",+ {+ lockId,+ action,+ overrideValue,+ timeoutMs: FIREBILL_TIMEOUT_MS,+ ...failure,+ error,+ },+ );+ firebillFailureCauseTotal.labels("finalize", failure.cause).inc();return false;}}diff --git a/apps/api/src/services/autumn/metrics.ts b/apps/api/src/services/autumn/metrics.tsindex 16ecf49c7..5f696fdef 100644--- a/apps/api/src/services/autumn/metrics.ts+++ b/apps/api/src/services/autumn/metrics.ts@@ -26,6 +26,39 @@ export const firebillTrackTotal = new Counter({labelNames: ["operation", "outcome"] as const,});+/**+ * **Who failed, when a firebill call did not produce a usable answer.**+ *+ * A separate series rather than a `cause` label on the counters above, and that+ * is the whole point. `increase()` evaluates per series, so+ * `increase(firecrawl_firebill_check_total{outcome="unavailable"}[15m]) > 5` β+ * the live `FirebillCheckUnavailable` rule β would silently start comparing 5+ * against each cause on its own: two causes sitting at 4 apiece would be 8+ * unanswered credit checks that nobody is paged about. The counters those rules+ * read stay byte-for-byte what they were; this one adds the detail beside them.+ *+ * - `timeout` / `connection` β the request never completed. **A client-side or+ * transport failure, not firebill**: it never answered, and its own+ * server-side p99 may be fine (73ms, while ~1,800 of these were logged per+ * 30h against a 5s client deadline).+ * - `non_ok` β firebill answered, with a status we cannot use.+ * - `unusable` β firebill answered, and the answer is not one we can read.+ * Never proof that anything was lost.+ * - `refused` β firebill answered `success: false`: on a track it did not take+ * the event, on a check it declined to answer.+ * - `ambiguous` β firebill answered "I do not know" (a 504, or+ * `ambiguous: true`).+ *+ * Only incremented on failure, so a healthy service produces no series at all.+ */+export const firebillFailureCauseTotal = new Counter({+ name: "firecrawl_firebill_failure_cause_total",+ help: "Why a firebill call did not produce a usable answer",+ // operation: track|refund|check|lock|finalize+ // cause: timeout|connection|non_ok|unusable|refused|ambiguous+ labelNames: ["operation", "cause"] as const,+});+/** Retries of a firebill call that answered `false` or threw. */export const firebillRetryTotal = new Counter({name: "firecrawl_firebill_retry_total",