Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { Job } from "bullmq";
import {
FIREBILL_DURABLE_RETRY_ATTEMPTS,
firebillCheck,
firebillFinalize,
firebillLock,
firebillTrack,
processFirebillTrackRetry,
} from "../firebill";
import { logger } from "../../../lib/logger";
import {
firebillCheckTotal,
firebillFailureCauseTotal,
firebillRetryTotal,
firebillTrackRetryExhaustedValueTotal,
firebillTrackRetryTotal,
firebillTrackTotal,
} from "../metrics";
import type { FirebillTrackRetryJobData } from "../../queue-service";
const { configState } = vi.hoisted(() => ({
configState: {
FIREBILL_URL: "https://firebill.test",
FIREBILL_SECRET: "secret",
FIRECRAWL_BUILD_SHA: undefined as string | undefined,
},
}));
vi.mock("../../../config", () => ({ config: configState }));
const { retryQueueAdd } = vi.hoisted(() => ({
retryQueueAdd: vi.fn(),
}));
vi.mock("../../queue-service", () => ({
getFirebillTrackRetryQueue: () => ({ add: retryQueueAdd }),
}));
vi.mock("../../../lib/logger", () => ({
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
// Built fresh per call: a Response body can only be read once, so a shared
// instance would make the second attempt look like a transport failure.
const ok = () =>
new Response(JSON.stringify({ success: true }), { status: 200 });
const refused = () =>
new Response(JSON.stringify({ success: false }), { status: 200 });
/** firebill's confirm timed out: it does not know whether it took the event. */
const ambiguous = () =>
new Response(JSON.stringify({ success: false, ambiguous: true }), {
status: 504,
});
/** What undici actually throws: the real error nested under a bare TypeError. */
const wrapped = (inner: Error) =>
Object.assign(new TypeError("fetch failed"), { cause: inner });
const abortError = () =>
Object.assign(new Error("The operation was aborted due to timeout"), {
name: "TimeoutError",
});
const connectionError = (code: string) =>
Object.assign(new Error(`connect ${code} 10.0.0.1:8080`), { code });
const params = {
customerId: "org-1",
entityId: "team-1",
featureId: "CREDITS",
value: 3,
properties: {},
idempotencyKey: "fc:track:scrape:job-1",
};
beforeEach(() => {
configState.FIREBILL_URL = "https://firebill.test";
configState.FIREBILL_SECRET = "secret";
configState.FIRECRAWL_BUILD_SHA = undefined;
firebillTrackTotal.reset();
firebillRetryTotal.reset();
firebillCheckTotal.reset();
firebillFailureCauseTotal.reset();
firebillTrackRetryTotal.reset();
firebillTrackRetryExhaustedValueTotal.reset();
retryQueueAdd.mockReset();
retryQueueAdd.mockResolvedValue(undefined);
vi.mocked(logger.info).mockClear();
vi.mocked(logger.warn).mockClear();
vi.mocked(logger.error).mockClear();
});
afterEach(() => vi.unstubAllGlobals());
/**
* The outcome counter alone — the one the alerts read, which must keep exactly
* the labels it always had.
*/
const trackOutcomes = async () =>
Object.fromEntries(
(await firebillTrackTotal.get()).values.map(v => [
v.labels.outcome,
v.value,
]),
);
const checkOutcomes = async () =>
Object.fromEntries(
(await firebillCheckTotal.get()).values.map(v => [
v.labels.outcome,
v.value,
]),
);
/** The separate cause counter, as `operation/cause`. */
const failureCauses = async () =>
Object.fromEntries(
(await firebillFailureCauseTotal.get()).values.map(v => [
`${v.labels.operation}/${v.labels.cause}`,
v.value,
]),
);
describe("firebillTrack", () => {
it("sends the caller's external request id with the charge, and no field without one", async () => {
const fetchMock = vi.fn().mockImplementation(async () => ok());
vi.stubGlobal("fetch", fetchMock);
await firebillTrack({ ...params, externalRequestId: "partner-op-42" });
expect(
JSON.parse(fetchMock.mock.calls[0][1].body).external_request_id,
).toBe("partner-op-42");
await firebillTrack(params);
expect(
Object.hasOwn(
JSON.parse(fetchMock.mock.calls[1][1].body),
"external_request_id",
),
).toBe(false);
});
it("retries a refusal and reports success if the retry lands", async () => {
const fetchMock = vi
.fn()
.mockImplementationOnce(async () => refused())
.mockImplementationOnce(async () => ok());
vi.stubGlobal("fetch", fetchMock);
await expect(firebillTrack(params)).resolves.toBe(true);
expect(fetchMock).toHaveBeenCalledTimes(2);
// Same key on both attempts, which is what makes the retry safe.
const keys = fetchMock.mock.calls.map(
c => JSON.parse(c[1].body).idempotency_key,
);
expect(keys).toEqual([params.idempotencyKey, params.idempotencyKey]);
});
it("gives up after the attempt limit and reports false", async () => {
const fetchMock = vi.fn().mockImplementation(async () => refused());
vi.stubGlobal("fetch", fetchMock);
await expect(firebillTrack(params)).resolves.toBe(false);
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it("reuses one key across attempts even when the caller supplies none", async () => {
// Without this the retry is a second event: firebill mints its own key per
// request, so an accepted-but-lost first attempt would be billed twice.
const fetchMock = vi
.fn()
.mockRejectedValueOnce(new Error("timeout"))
.mockImplementationOnce(async () => ok());
vi.stubGlobal("fetch", fetchMock);
await firebillTrack({ ...params, idempotencyKey: undefined });
const keys = fetchMock.mock.calls.map(
c => JSON.parse(c[1].body).idempotency_key,
);
expect(keys[0]).toBeTruthy();
expect(keys[0]).toBe(keys[1]);
});
it("mints a different key for each keyless call", async () => {
const fetchMock = vi.fn().mockImplementation(async () => ok());
vi.stubGlobal("fetch", fetchMock);
await firebillTrack({ ...params, idempotencyKey: undefined });
await firebillTrack({ ...params, idempotencyKey: undefined });
const [first, second] = fetchMock.mock.calls.map(
c => JSON.parse(c[1].body).idempotency_key,
);
expect(first).not.toBe(second);
});
it("separates an explicit refusal from an ambiguous transport failure", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => refused()),
);
await firebillTrack(params);
expect(await trackOutcomes()).toEqual({ refused: 1 });
expect(await failureCauses()).toEqual({ "track/refused": 1 });
firebillTrackTotal.reset();
firebillFailureCauseTotal.reset();
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("timeout")));
await firebillTrack(params);
// A timeout is not proof the usage was lost — firebill may have taken it.
expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
expect(await failureCauses()).toEqual({ "track/connection": 1 });
});
// -----------------------------------------------------------------------
// The alerts' own series
// -----------------------------------------------------------------------
it("leaves the alerted counters with exactly the labels they had", async () => {
// `increase()` evaluates per series, so a `cause` label here would turn
// `increase(firecrawl_firebill_track_total{outcome="refused"}[15m]) > 0`
// into one threshold per cause. The detail lives beside it instead.
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => refused()),
);
await firebillTrack(params);
const [series] = (await firebillTrackTotal.get()).values;
expect(Object.keys(series.labels).sort()).toEqual(["operation", "outcome"]);
const [cause] = (await firebillFailureCauseTotal.get()).values;
expect(Object.keys(cause.labels).sort()).toEqual(["cause", "operation"]);
expect(cause.labels).toMatchObject({
operation: "track",
cause: "refused",
});
});
it("counts a refund's cause under its own operation", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => refused()),
);
await firebillTrack({ ...params, value: -7 });
expect(await failureCauses()).toEqual({ "refund/refused": 1 });
});
// -----------------------------------------------------------------------
// firebill saying "I do not know"
//
// A publish whose broker confirm timed out may have been taken anyway. Over
// eight days ~2,100 of these were answered `{"success": false}` — the same
// body a refusal sends — and every one of those events had in fact settled,
// while this client logged each as usage that would never be billed.
// -----------------------------------------------------------------------
it.each([
["a 504 carrying the field", () => ambiguous()],
["a 504 with no body at all", () => new Response(null, { status: 504 })],
[
"a 200 that says so by field",
() =>
new Response(JSON.stringify({ success: false, ambiguous: true }), {
status: 200,
}),
],
])("reads %s as ambiguous, never as refused", async (_label, respond) => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => respond()),
);
await expect(firebillTrack(params)).resolves.toBe(false);
expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
expect(await failureCauses()).toEqual({ "track/ambiguous": 1 });
});
// **An answer we cannot read is not proof of anything.** firebill may have
// taken the event and said so in a shape we did not understand, so calling it
// `refused` would log billed usage as lost — the same class of untruth this
// PR exists to remove, one layer down.
it.each([
["an empty 200", () => new Response("", { status: 200 })],
[
"a 200 that is not JSON",
() => new Response("<html>502</html>", { status: 200 }),
],
[
"a 200 with no success field",
() => new Response(JSON.stringify({ queued: true }), { status: 200 }),
],
[
"a 200 whose success is not a boolean",
() => new Response(JSON.stringify({ success: "yes" }), { status: 200 }),
],
])("reads %s as unusable, not as a refusal", async (_label, respond) => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => respond()),
);
await expect(firebillTrack(params)).resolves.toBe(false);
expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
expect(await failureCauses()).toEqual({ "track/unusable": 1 });
expect(vi.mocked(logger.error).mock.calls.map(c => c[0])).not.toContain(
"firebill refused a usage event; it will not be billed",
);
});
it("still reads success:false as a refusal", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => refused()),
);
await expect(firebillTrack(params)).resolves.toBe(false);
expect(await trackOutcomes()).toEqual({ refused: 1 });
});
// A body that never finished arriving is the network failing, not firebill
// answering — it has to reach the transport classifier, not be swallowed as
// an unusable body.
it("treats a body that dies mid-read as a transport failure", async () => {
const dying = () => {
const response = new Response(JSON.stringify({ success: true }), {
status: 200,
});
vi.spyOn(response, "json").mockRejectedValue(
Object.assign(new TypeError("terminated"), {
cause: Object.assign(new Error("aborted"), {
code: "UND_ERR_BODY_TIMEOUT",
}),
}),
);
return response;
};
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => dying()),
);
await expect(firebillTrack(params)).resolves.toBe(false);
expect(await failureCauses()).toEqual({ "track/timeout": 1 });
expect(vi.mocked(logger.warn).mock.calls.map(c => c[0])).toContain(
"firebill track request did not complete — client-side timeout or connection error; firebill did not answer",
);
});
it("logs an ambiguous answer as unknown rather than as unbilled usage", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => ambiguous()),
);
await firebillTrack(params);
const messages = vi.mocked(logger.error).mock.calls.map(c => c[0]);
expect(messages).toContain(
"firebill did not answer; the event may or may not have been accepted",
);
expect(messages).not.toContain(
"firebill refused a usage event; it will not be billed",
);
});
// -----------------------------------------------------------------------
// Whose failure was it
//
// ~1,800 warnings per 30h said "firebill may be unavailable" against a
// firebill whose server-side p99 was 73ms. They were our own 5s deadline.
// -----------------------------------------------------------------------
it.each([
["our own deadline firing", () => abortError(), "timeout"],
["an undici-wrapped deadline", () => wrapped(abortError()), "timeout"],
[
"a refused connection",
() => wrapped(connectionError("ECONNREFUSED")),
"connection",
],
[
"a reset connection",
() => wrapped(connectionError("ECONNRESET")),
"connection",
],
[
"a DNS failure",
() => wrapped(connectionError("ENOTFOUND")),
"connection",
],
])("labels %s as a client-side %s", async (_label, error, cause) => {
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(error()));
await expect(firebillTrack(params)).resolves.toBe(false);
expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
expect(await failureCauses()).toEqual({ [`track/${cause}`]: 1 });
});
it("stops blaming firebill for a request that never reached it", async () => {
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(wrapped(abortError())));
await firebillTrack(params);
// The logger's own overloads make `calls` a one-element tuple to TS; every
// call site here passes (message, fields).
const warnings = vi.mocked(logger.warn).mock.calls as unknown as [
string,
Record<string, unknown>,
][];
const messages = warnings.map(c => c[0]);
expect(messages).toContain(
"firebill track request did not complete — client-side timeout or connection error; firebill did not answer",
);
expect(messages.join(" ")).not.toContain("firebill may be unavailable");
// The error's own identity, so the class of stall is greppable.
// The innermost error's identity, not the `TypeError: fetch failed` undici
// wraps it in — so the class of stall is greppable.
expect(warnings[0][1]).toMatchObject({
cause: "timeout",
errorName: "TimeoutError",
timeoutMs: 5000,
});
});
it("names the status when firebill really did answer", async () => {
vi.stubGlobal(
"fetch",
vi
.fn()
.mockImplementation(async () => new Response("nope", { status: 502 })),
);
await expect(firebillTrack(params)).resolves.toBe(false);
expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
expect(await failureCauses()).toEqual({ "track/non_ok": 1 });
// A static message; the status is context, so log search can group these.
const [message, fields] = vi.mocked(logger.warn).mock
.calls[0] as unknown as [string, Record<string, unknown>];
expect(message).toBe(
"firebill track attempt failed — firebill answered a non-OK status",
);
expect(fields).toMatchObject({ status: 502 });
});
it("retries a thrown error too", async () => {
const fetchMock = vi
.fn()
.mockRejectedValueOnce(new Error("connection refused"))
.mockImplementationOnce(async () => ok());
vi.stubGlobal("fetch", fetchMock);
await expect(firebillTrack(params)).resolves.toBe(true);
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it("sends a refund to /v1/refund with the value made positive", async () => {
const fetchMock = vi.fn().mockImplementation(async () => ok());
vi.stubGlobal("fetch", fetchMock);
await firebillTrack({ ...params, value: -7 });
expect(fetchMock.mock.calls[0][0]).toBe("https://firebill.test/v1/refund");
expect(JSON.parse(fetchMock.mock.calls[0][1].body).value).toBe(7);
});
it("logs one accepted line carrying the key, the amount and the build", async () => {
configState.FIRECRAWL_BUILD_SHA = "abc123";
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => ok()),
);
await firebillTrack({ ...params, properties: { jobId: "job-1" } });
expect(logger.info).toHaveBeenCalledTimes(1);
expect(logger.info).toHaveBeenCalledWith(
"firebill track succeeded",
expect.objectContaining({
operation: "track",
outcome: "accepted",
idempotencyKey: params.idempotencyKey,
expectedValue: 3,
jobId: "job-1",
buildVersion: "abc123",
}),
);
});
it("logs a refund under its own operation with the amount made positive", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => ok()),
);
await firebillTrack({ ...params, value: -7 });
expect(logger.info).toHaveBeenCalledWith(
"firebill track succeeded",
expect.objectContaining({
operation: "refund",
expectedValue: 7,
buildVersion: "unknown",
}),
);
});
it("logs no success line when firebill did not take the event", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => refused()),
);
await firebillTrack(params);
expect(logger.info).not.toHaveBeenCalled();
});
});
const retryOutcomes = async () =>
Object.fromEntries(
(await firebillTrackRetryTotal.get()).values.map(v => [
`${v.labels.operation}/${v.labels.outcome}`,
v.value,
]),
);
/** A retry job as the queue hands it to the worker on a given attempt. */
const retryJob = (
data: FirebillTrackRetryJobData,
attemptsMade: number,
): Job<FirebillTrackRetryJobData> =>
({
data,
attemptsMade,
opts: { attempts: FIREBILL_DURABLE_RETRY_ATTEMPTS },
}) as unknown as Job<FirebillTrackRetryJobData>;
/** Logger calls as [message, context], which the mocked signature hides. */
const logCalls = (method: "info" | "warn" | "error") =>
vi.mocked(logger[method]).mock.calls as unknown as [
string,
Record<string, unknown>,
][];
describe("firebillTrack durable retry", () => {
it("hands an unanswered track to the retry queue as the identical event", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => ambiguous()),
);
await expect(firebillTrack(params)).resolves.toBe(false);
expect(retryQueueAdd).toHaveBeenCalledTimes(1);
const [name, data, opts] = retryQueueAdd.mock.calls[0];
expect(name).toBe("track");
expect(data).toEqual(params);
expect(opts).toMatchObject({
jobId: "firebill|fc%3Atrack%3Ascrape%3Ajob-1",
attempts: FIREBILL_DURABLE_RETRY_ATTEMPTS,
backoff: { type: "exponential" },
});
// BullMQ waits backoff * 2^(n-1) after the nth failure: 30s to the first
// re-send, then doubling, which puts the last one about two hours out.
const gaps = [opts.delay];
for (let n = 1; n < FIREBILL_DURABLE_RETRY_ATTEMPTS; n++) {
gaps.push(opts.backoff.delay * 2 ** (n - 1));
}
expect(gaps.slice(0, 3)).toEqual([30_000, 60_000, 120_000]);
const total = gaps.reduce((a, b) => a + b, 0);
expect(total).toBeGreaterThan(2 * 60 * 60 * 1000);
expect(total).toBeLessThan(3 * 60 * 60 * 1000);
expect(await retryOutcomes()).toEqual({ "track/enqueued": 1 });
// The alerted counter still says what the caller saw.
expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
expect(logCalls("error")[0][1]).toMatchObject({
idempotencyKey: params.idempotencyKey,
handedToRetry: true,
});
});
it("enqueues a timed-out refund under the minted key it already sent", async () => {
const fetchMock = vi.fn().mockRejectedValue(wrapped(abortError()));
vi.stubGlobal("fetch", fetchMock);
await firebillTrack({ ...params, value: -3, idempotencyKey: undefined });
const sentKey = JSON.parse(fetchMock.mock.calls[0][1].body).idempotency_key;
const [name, data] = retryQueueAdd.mock.calls[0];
expect(name).toBe("refund");
expect(data).toMatchObject({ value: -3, idempotencyKey: sentKey });
expect(await retryOutcomes()).toEqual({ "refund/enqueued": 1 });
});
it("does not enqueue an explicit refusal or an accepted event", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => refused()),
);
await firebillTrack(params);
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => ok()),
);
await firebillTrack(params);
expect(retryQueueAdd).not.toHaveBeenCalled();
});
it("keeps an unanswered event out of the retry when the caller opts out", async () => {
const fetchMock = vi.fn().mockImplementation(async () => ambiguous());
vi.stubGlobal("fetch", fetchMock);
await expect(
firebillTrack({ ...params, value: -1, durableRetry: false }),
).resolves.toBe(false);
expect(retryQueueAdd).not.toHaveBeenCalled();
expect(await retryOutcomes()).toEqual({});
expect(logCalls("error")[0][1]).toMatchObject({ handedToRetry: false });
// The flag is ours; firebill never sees it.
expect(
Object.hasOwn(
JSON.parse(fetchMock.mock.calls[0][1].body),
"durableRetry",
),
).toBe(false);
});
it("gives distinct keys distinct job ids even when they differ only in delimiters", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => ambiguous()),
);
await firebillTrack({ ...params, idempotencyKey: "a:b" });
await firebillTrack({ ...params, idempotencyKey: "a|b" });
const [first, second] = retryQueueAdd.mock.calls.map(c => c[2].jobId);
expect(first).not.toBe(second);
});
it("counts an enqueue that does not confirm in time as unknown, not lost", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => ambiguous()),
);
retryQueueAdd.mockImplementationOnce(() => new Promise(() => {}));
await expect(firebillTrack(params)).resolves.toBe(false);
expect(await retryOutcomes()).toEqual({ "track/enqueue_timeout": 1 });
expect(logCalls("error").map(c => c[0])).toContain(
"firebill durable retry enqueue timed out; the retry may or may not have been queued",
);
});
it("logs and counts an enqueue that fails, still reporting false", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => ambiguous()),
);
retryQueueAdd.mockRejectedValueOnce(new Error("redis down"));
await expect(firebillTrack(params)).resolves.toBe(false);
expect(await retryOutcomes()).toEqual({ "track/enqueue_failed": 1 });
const messages = vi.mocked(logger.error).mock.calls.map(c => c[0]);
expect(messages).toContain(
"firebill durable retry could not be enqueued; the event will not be billed unless replayed",
);
expect(logCalls("error").at(-1)?.[1]).toMatchObject({
handedToRetry: false,
});
});
it("re-sends with the same key, customer, entity, feature and value", async () => {
const fetchMock = vi.fn().mockImplementation(async () => ok());
vi.stubGlobal("fetch", fetchMock);
const event = { ...params, externalRequestId: "partner-op-42" };
await processFirebillTrackRetry(retryJob(event, 0));
expect(fetchMock.mock.calls[0][0]).toBe("https://firebill.test/v1/track");
expect(JSON.parse(fetchMock.mock.calls[0][1].body)).toEqual({
customer_id: "org-1",
entity_id: "team-1",
feature_id: "CREDITS",
value: 3,
properties: {},
idempotency_key: params.idempotencyKey,
external_request_id: "partner-op-42",
});
expect(await retryOutcomes()).toEqual({
"track/retried": 1,
"track/settled": 1,
});
});
it("sends a refund retry to /v1/refund with the value made positive", async () => {
const fetchMock = vi.fn().mockImplementation(async () => ok());
vi.stubGlobal("fetch", fetchMock);
await processFirebillTrackRetry(retryJob({ ...params, value: -3 }, 2));
expect(fetchMock.mock.calls[0][0]).toBe("https://firebill.test/v1/refund");
expect(JSON.parse(fetchMock.mock.calls[0][1].body)).toMatchObject({
value: 3,
idempotency_key: params.idempotencyKey,
});
});
it("throws on a failed attempt so the queue backs off and tries again", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => ambiguous()),
);
await expect(
processFirebillTrackRetry(retryJob(params, 0)),
).rejects.toThrow(/retry 1\/8 failed/);
expect(await retryOutcomes()).toEqual({ "track/retried": 1 });
const messages = vi.mocked(logger.error).mock.calls.map(c => c[0]);
expect(messages).not.toContain(
"firebill durable retry exhausted; the event may not have been billed",
);
});
it("keeps retrying a refusal, since an earlier attempt may have landed", async () => {
vi.stubGlobal(
"fetch",
vi.fn().mockImplementation(async () => refused()),
);
await expect(
processFirebillTrackRetry(retryJob(params, 3)),
).rejects.toThrow();
expect(await retryOutcomes()).toEqual({ "track/retried": 1 });
});
it("logs the key and amount and counts the value when the last attempt fails", async () => {
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(wrapped(abortError())));
await expect(
processFirebillTrackRetry(
retryJob(params, FIREBILL_DURABLE_RETRY_ATTEMPTS - 1),
),
).rejects.toThrow(/retry 8\/8 failed/);
expect(await retryOutcomes()).toEqual({
"track/retried": 1,
"track/exhausted": 1,
});
const [value] = (await firebillTrackRetryExhaustedValueTotal.get()).values;
expect(value).toMatchObject({
value: 3,
labels: { operation: "track", feature: "CREDITS" },
});
const exhausted = logCalls("error").find(
c =>
c[0] ===
"firebill durable retry exhausted; the event may not have been billed",
);
expect(exhausted?.[1]).toMatchObject({
idempotencyKey: params.idempotencyKey,
value: 3,
attempt: 8,
attempts: 8,
cause: "timeout",
});
});
});
describe("firebillCheck", () => {
const checkParams = {
customerId: "org-1",
entityId: "team-1",
featureId: "CREDITS",
value: 10,
properties: { source: "checkCreditsMiddleware" },
};
const answer = (body: unknown, status = 200) =>
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue(new Response(JSON.stringify(body), { status })),
);
it("passes through a yes, with the funder's pool as remaining", async () => {
answer({ success: true, allowed: true, remaining: 500 });
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "answered",
allowed: true,
remaining: 500,
});
});
it("passes through a no — this is the only thing that may 402", async () => {
answer({ success: true, allowed: false, remaining: 3 });
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "answered",
allowed: false,
remaining: 3,
});
});
// The asymmetry with firebillTrack, which fails closed. Refusing to answer an
// authorization question must not become a 402.
it.each([
["firebill could not answer", { success: false }, 200],
["a non-OK response", { success: true, allowed: true }, 503],
["a missing allowed", { success: true }, 200],
["a non-boolean allowed", { success: true, allowed: "yes" }, 200],
])("fails open on %s", async (_label, body, status) => {
answer(body, status);
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "unavailable",
});
});
it("fails open when the request throws", async () => {
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("boom")));
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "unavailable",
});
});
// A missing figure must not read as zero: callers clamp crawl limits with
// it, so zero would silently shrink an allowed crawl to nothing.
// The inverse of the case below, and the dangerous direction: the middleware
// reads a denial with `remaining > 0` as a *partial* crawl and calls next(),
// so an unbounded default would turn a refusal into an unlimited crawl.
it("treats an absent remaining on a DENIAL as zero, not unbounded", async () => {
answer({ success: true, allowed: false });
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "answered",
allowed: false,
remaining: 0,
});
});
// A denial that *does* carry a figure keeps it: that is a legitimate partial
// crawl, and the existing middleware behaviour we must not change.
it("keeps a usable remaining on a denial", async () => {
answer({ success: true, allowed: false, remaining: 25 });
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "answered",
allowed: false,
remaining: 25,
});
});
// What a 402 names as the time credits come back.
it("passes through the balance's reset time", async () => {
answer({
success: true,
allowed: false,
remaining: 0,
resets_at: 1_798_761_600_000,
});
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "answered",
allowed: false,
remaining: 0,
resetsAt: 1_798_761_600_000,
});
});
// A reset time is decoration on the answer, never a reason to fail open.
it.each([
["a string", "2026-12-01T00:00:00Z"],
["a fraction", 1.5],
["zero", 0],
["past what a Date can hold", 8_640_000_000_000_001],
["null", null],
])(
"drops a resets_at that is %s and keeps the answer",
async (_label, resetsAt) => {
answer({
success: true,
allowed: false,
remaining: 0,
resets_at: resetsAt,
});
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "answered",
allowed: false,
remaining: 0,
});
},
);
it("treats an absent remaining on an ALLOWED check as unbounded, not zero", async () => {
answer({ success: true, allowed: true });
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "answered",
allowed: true,
remaining: Infinity,
});
});
it("sends the caller's context so Autumn records the decision", async () => {
const fetchMock = vi
.fn()
.mockResolvedValue(
new Response(
JSON.stringify({ success: true, allowed: true, remaining: 1 }),
{ status: 200 },
),
);
vi.stubGlobal("fetch", fetchMock);
await firebillCheck(checkParams);
const [url, init] = fetchMock.mock.calls[0];
expect(url).toBe("https://firebill.test/v1/check");
expect(JSON.parse(init.body)).toEqual({
customer_id: "org-1",
entity_id: "team-1",
feature_id: "CREDITS",
value: 10,
properties: { source: "checkCreditsMiddleware" },
});
});
it.each([
["our own deadline firing", () => wrapped(abortError()), "check/timeout"],
[
"a refused connection",
() => wrapped(connectionError("ECONNREFUSED")),
"check/connection",
],
])(
"blames the client, not firebill, for %s",
async (_label, error, expected) => {
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(error()));
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "unavailable",
});
expect(await checkOutcomes()).toEqual({ unavailable: 1 });
expect(await failureCauses()).toEqual({ [expected]: 1 });
expect(vi.mocked(logger.error).mock.calls.map(c => c[0])).toContain(
"firebill check unavailable — the request did not complete (client-side timeout or connection error); firebill did not answer",
);
},
);
it("names the status in context when firebill answered one", async () => {
answer({ success: true, allowed: true }, 503);
await firebillCheck(checkParams);
expect(await checkOutcomes()).toEqual({ unavailable: 1 });
expect(await failureCauses()).toEqual({ "check/non_ok": 1 });
const [message, fields] = vi.mocked(logger.error).mock
.calls[0] as unknown as [string, Record<string, unknown>];
expect(message).toBe(
"firebill check unavailable — firebill answered a non-OK status",
);
expect(fields).toMatchObject({ status: 503 });
});
it("records no cause at all for a real answer", async () => {
answer({ success: true, allowed: true, remaining: 500 });
await firebillCheck(checkParams);
expect(await checkOutcomes()).toEqual({ allowed: 1 });
expect(await failureCauses()).toEqual({});
});
// `refused` is documented as an explicit `success: false`. An answer we could
// not read is not one, and labelling it so would have anyone slicing by cause
// counting unreadable answers as declines.
it.each([
["a missing allowed", { success: true }, 200],
["a non-boolean allowed", { success: true, allowed: "yes" }, 200],
])("calls %s unusable rather than refused", async (_label, body, status) => {
answer(body, status);
await firebillCheck(checkParams);
expect(await failureCauses()).toEqual({ "check/unusable": 1 });
});
it("calls an unreadable body unusable rather than refused", async () => {
vi.stubGlobal(
"fetch",
vi
.fn()
.mockResolvedValue(new Response("<html>502</html>", { status: 200 })),
);
await expect(firebillCheck(checkParams)).resolves.toEqual({
status: "unavailable",
});
expect(await failureCauses()).toEqual({ "check/unusable": 1 });
});
it("keeps refused for an explicit success:false", async () => {
answer({ success: false });
await firebillCheck(checkParams);
expect(await failureCauses()).toEqual({ "check/refused": 1 });
});
it("does not retry — this is on the request path", async () => {
const fetchMock = vi.fn().mockRejectedValue(new Error("boom"));
vi.stubGlobal("fetch", fetchMock);
await firebillCheck(checkParams);
expect(fetchMock).toHaveBeenCalledTimes(1);
});
});
describe("firebillLock", () => {
const lockParams = {
customerId: "org-1",
entityId: "team-1",
featureId: "CREDITS",
value: 10,
lockId: "monitor_check-1",
expiresAt: Date.now() + 60_000,
};
const answer = (body: unknown, status = 200) =>
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue(new Response(JSON.stringify(body), { status })),
);
it("locks and carries the operation token back on a gated hold", async () => {
answer({
success: true,
allowed: true,
lock_id: "monitor_check-1",
operation_token: "op-tok",
});
await expect(firebillLock(lockParams)).resolves.toEqual({
status: "locked",
lockId: "monitor_check-1",
operationToken: "op-tok",
});
});
it("falls back to the caller's lock id when firebill echoes none", async () => {
answer({ success: true, allowed: true });
await expect(firebillLock(lockParams)).resolves.toEqual({
status: "locked",
lockId: "monitor_check-1",
});
});
it("reads a denial and its reason", async () => {
answer({ success: true, allowed: false, reason: "out_of_credits" });
await expect(firebillLock(lockParams)).resolves.toEqual({
status: "denied",
reason: "out_of_credits",
});
});
it("drops an unrecognised denial reason rather than passing it through", async () => {
answer({ success: true, allowed: false, reason: "some_new_reason" });
await expect(firebillLock(lockParams)).resolves.toEqual({
status: "denied",
});
});
// A shape firebill sent on purpose (`success: false`) is `refused`; one we
// could not read is `unusable`. Both proceed unlocked, but slicing the cause
// counter must tell them apart.
it("calls an explicit success:false refused", async () => {
answer({ success: false });
await expect(firebillLock(lockParams)).resolves.toEqual({
status: "unavailable",
});
expect(await failureCauses()).toEqual({ "lock/refused": 1 });
});
it.each([
["a missing allowed", { success: true }],
["a non-boolean allowed", { success: true, allowed: "yes" }],
["a missing success", { locked: true }],
])("calls %s unusable, not refused", async (_label, body) => {
answer(body);
await expect(firebillLock(lockParams)).resolves.toEqual({
status: "unavailable",
});
expect(await failureCauses()).toEqual({ "lock/unusable": 1 });
});
it("calls an unreadable body unusable", async () => {
vi.stubGlobal(
"fetch",
vi
.fn()
.mockResolvedValue(new Response("<html>502</html>", { status: 200 })),
);
await expect(firebillLock(lockParams)).resolves.toEqual({
status: "unavailable",
});
expect(await failureCauses()).toEqual({ "lock/unusable": 1 });
});
it("names a non-OK status as its own cause", async () => {
answer({ success: true, allowed: true }, 503);
await expect(firebillLock(lockParams)).resolves.toEqual({
status: "unavailable",
});
expect(await failureCauses()).toEqual({ "lock/non_ok": 1 });
});
});
describe("firebillFinalize", () => {
const finalizeParams = {
lockId: "monitor_check-1",
action: "confirm" as const,
};
const answer = (body: unknown, status = 200) =>
vi.stubGlobal(
"fetch",
vi.fn().mockResolvedValue(new Response(JSON.stringify(body), { status })),
);
it("returns true when firebill accepted the settle", async () => {
answer({ success: true });
await expect(firebillFinalize(finalizeParams)).resolves.toBe(true);
});
it("logs the settle's key and the amount it bills", async () => {
answer({ success: true });
await firebillFinalize({
...finalizeParams,
overrideValue: 2,
heldValue: 5,
});
expect(logger.info).toHaveBeenCalledWith(
"firebill finalize succeeded",
expect.objectContaining({
operation: "finalize",
outcome: "accepted",
idempotencyKey: "fc:finalize:confirm:monitor_check-1",
expectedValue: 2,
}),
);
vi.mocked(logger.info).mockClear();
answer({ success: true });
await firebillFinalize({ ...finalizeParams, heldValue: 5 });
expect(logger.info).toHaveBeenCalledWith(
"firebill finalize succeeded",
expect.objectContaining({ expectedValue: 5 }),
);
vi.mocked(logger.info).mockClear();
answer({ success: true });
await firebillFinalize({
...finalizeParams,
action: "release",
heldValue: 5,
});
expect(logger.info).toHaveBeenCalledWith(
"firebill finalize succeeded",
expect.objectContaining({
idempotencyKey: "fc:finalize:release:monitor_check-1",
expectedValue: 0,
}),
);
});
it("logs nothing as succeeded when the settle was refused", async () => {
answer({ success: false });
await firebillFinalize(finalizeParams);
expect(logger.info).not.toHaveBeenCalled();
});
it("calls an explicit success:false refused", async () => {
answer({ success: false });
await expect(firebillFinalize(finalizeParams)).resolves.toBe(false);
expect(await failureCauses()).toEqual({ "finalize/refused": 1 });
});
it.each([
["a missing success", { queued: true }],
["a non-boolean success", { success: "yes" }],
])("calls %s unusable, not refused", async (_label, body) => {
answer(body);
await expect(firebillFinalize(finalizeParams)).resolves.toBe(false);
expect(await failureCauses()).toEqual({ "finalize/unusable": 1 });
});
it("calls an unreadable body unusable", async () => {
vi.stubGlobal(
"fetch",
vi
.fn()
.mockResolvedValue(new Response("<html>502</html>", { status: 200 })),
);
await expect(firebillFinalize(finalizeParams)).resolves.toBe(false);
expect(await failureCauses()).toEqual({ "finalize/unusable": 1 });
});
it("reads a 504 as ambiguous rather than refused", async () => {
answer({ success: false, ambiguous: true }, 504);
await expect(firebillFinalize(finalizeParams)).resolves.toBe(false);
expect(await failureCauses()).toEqual({ "finalize/ambiguous": 1 });
});
});