Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

40.0 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { Job } from "bullmq";
import {
  FIREBILL_DURABLE_RETRY_ATTEMPTS,
  firebillCheck,
  firebillFinalize,
  firebillLock,
  firebillTrack,
  processFirebillTrackRetry,
} from "../firebill";
import { logger } from "../../../lib/logger";
import {
  firebillCheckTotal,
  firebillFailureCauseTotal,
  firebillRetryTotal,
  firebillTrackRetryExhaustedValueTotal,
  firebillTrackRetryTotal,
  firebillTrackTotal,
} from "../metrics";
import type { FirebillTrackRetryJobData } from "../../queue-service";

const { configState } = vi.hoisted(() => ({
  configState: {
    FIREBILL_URL: "https://firebill.test",
    FIREBILL_SECRET: "secret",
    FIRECRAWL_BUILD_SHA: undefined as string | undefined,
  },
}));

vi.mock("../../../config", () => ({ config: configState }));

const { retryQueueAdd } = vi.hoisted(() => ({
  retryQueueAdd: vi.fn(),
}));
vi.mock("../../queue-service", () => ({
  getFirebillTrackRetryQueue: () => ({ add: retryQueueAdd }),
}));
vi.mock("../../../lib/logger", () => ({
  logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));

// Built fresh per call: a Response body can only be read once, so a shared
// instance would make the second attempt look like a transport failure.
const ok = () =>
  new Response(JSON.stringify({ success: true }), { status: 200 });
const refused = () =>
  new Response(JSON.stringify({ success: false }), { status: 200 });
/** firebill's confirm timed out: it does not know whether it took the event. */
const ambiguous = () =>
  new Response(JSON.stringify({ success: false, ambiguous: true }), {
    status: 504,
  });

/** What undici actually throws: the real error nested under a bare TypeError. */
const wrapped = (inner: Error) =>
  Object.assign(new TypeError("fetch failed"), { cause: inner });

const abortError = () =>
  Object.assign(new Error("The operation was aborted due to timeout"), {
    name: "TimeoutError",
  });

const connectionError = (code: string) =>
  Object.assign(new Error(`connect ${code} 10.0.0.1:8080`), { code });

const params = {
  customerId: "org-1",
  entityId: "team-1",
  featureId: "CREDITS",
  value: 3,
  properties: {},
  idempotencyKey: "fc:track:scrape:job-1",
};

beforeEach(() => {
  configState.FIREBILL_URL = "https://firebill.test";
  configState.FIREBILL_SECRET = "secret";
  configState.FIRECRAWL_BUILD_SHA = undefined;
  firebillTrackTotal.reset();
  firebillRetryTotal.reset();
  firebillCheckTotal.reset();
  firebillFailureCauseTotal.reset();
  firebillTrackRetryTotal.reset();
  firebillTrackRetryExhaustedValueTotal.reset();
  retryQueueAdd.mockReset();
  retryQueueAdd.mockResolvedValue(undefined);
  vi.mocked(logger.info).mockClear();
  vi.mocked(logger.warn).mockClear();
  vi.mocked(logger.error).mockClear();
});
afterEach(() => vi.unstubAllGlobals());

/**
 * The outcome counter alone — the one the alerts read, which must keep exactly
 * the labels it always had.
 */
const trackOutcomes = async () =>
  Object.fromEntries(
    (await firebillTrackTotal.get()).values.map(v => [
      v.labels.outcome,
      v.value,
    ]),
  );

const checkOutcomes = async () =>
  Object.fromEntries(
    (await firebillCheckTotal.get()).values.map(v => [
      v.labels.outcome,
      v.value,
    ]),
  );

/** The separate cause counter, as `operation/cause`. */
const failureCauses = async () =>
  Object.fromEntries(
    (await firebillFailureCauseTotal.get()).values.map(v => [
      `${v.labels.operation}/${v.labels.cause}`,
      v.value,
    ]),
  );

describe("firebillTrack", () => {
  it("sends the caller's external request id with the charge, and no field without one", async () => {
    const fetchMock = vi.fn().mockImplementation(async () => ok());
    vi.stubGlobal("fetch", fetchMock);

    await firebillTrack({ ...params, externalRequestId: "partner-op-42" });
    expect(
      JSON.parse(fetchMock.mock.calls[0][1].body).external_request_id,
    ).toBe("partner-op-42");

    await firebillTrack(params);
    expect(
      Object.hasOwn(
        JSON.parse(fetchMock.mock.calls[1][1].body),
        "external_request_id",
      ),
    ).toBe(false);
  });

  it("retries a refusal and reports success if the retry lands", async () => {
    const fetchMock = vi
      .fn()
      .mockImplementationOnce(async () => refused())
      .mockImplementationOnce(async () => ok());
    vi.stubGlobal("fetch", fetchMock);

    await expect(firebillTrack(params)).resolves.toBe(true);
    expect(fetchMock).toHaveBeenCalledTimes(2);
    // Same key on both attempts, which is what makes the retry safe.
    const keys = fetchMock.mock.calls.map(
      c => JSON.parse(c[1].body).idempotency_key,
    );
    expect(keys).toEqual([params.idempotencyKey, params.idempotencyKey]);
  });

  it("gives up after the attempt limit and reports false", async () => {
    const fetchMock = vi.fn().mockImplementation(async () => refused());
    vi.stubGlobal("fetch", fetchMock);

    await expect(firebillTrack(params)).resolves.toBe(false);
    expect(fetchMock).toHaveBeenCalledTimes(2);
  });

  it("reuses one key across attempts even when the caller supplies none", async () => {
    // Without this the retry is a second event: firebill mints its own key per
    // request, so an accepted-but-lost first attempt would be billed twice.
    const fetchMock = vi
      .fn()
      .mockRejectedValueOnce(new Error("timeout"))
      .mockImplementationOnce(async () => ok());
    vi.stubGlobal("fetch", fetchMock);

    await firebillTrack({ ...params, idempotencyKey: undefined });

    const keys = fetchMock.mock.calls.map(
      c => JSON.parse(c[1].body).idempotency_key,
    );
    expect(keys[0]).toBeTruthy();
    expect(keys[0]).toBe(keys[1]);
  });

  it("mints a different key for each keyless call", async () => {
    const fetchMock = vi.fn().mockImplementation(async () => ok());
    vi.stubGlobal("fetch", fetchMock);

    await firebillTrack({ ...params, idempotencyKey: undefined });
    await firebillTrack({ ...params, idempotencyKey: undefined });

    const [first, second] = fetchMock.mock.calls.map(
      c => JSON.parse(c[1].body).idempotency_key,
    );
    expect(first).not.toBe(second);
  });

  it("separates an explicit refusal from an ambiguous transport failure", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => refused()),
    );
    await firebillTrack(params);
    expect(await trackOutcomes()).toEqual({ refused: 1 });
    expect(await failureCauses()).toEqual({ "track/refused": 1 });

    firebillTrackTotal.reset();
    firebillFailureCauseTotal.reset();
    vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("timeout")));
    await firebillTrack(params);
    // A timeout is not proof the usage was lost — firebill may have taken it.
    expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
    expect(await failureCauses()).toEqual({ "track/connection": 1 });
  });

  // -----------------------------------------------------------------------
  // The alerts' own series
  // -----------------------------------------------------------------------

  it("leaves the alerted counters with exactly the labels they had", async () => {
    // `increase()` evaluates per series, so a `cause` label here would turn
    // `increase(firecrawl_firebill_track_total{outcome="refused"}[15m]) > 0`
    // into one threshold per cause. The detail lives beside it instead.
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => refused()),
    );
    await firebillTrack(params);

    const [series] = (await firebillTrackTotal.get()).values;
    expect(Object.keys(series.labels).sort()).toEqual(["operation", "outcome"]);

    const [cause] = (await firebillFailureCauseTotal.get()).values;
    expect(Object.keys(cause.labels).sort()).toEqual(["cause", "operation"]);
    expect(cause.labels).toMatchObject({
      operation: "track",
      cause: "refused",
    });
  });

  it("counts a refund's cause under its own operation", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => refused()),
    );
    await firebillTrack({ ...params, value: -7 });
    expect(await failureCauses()).toEqual({ "refund/refused": 1 });
  });

  // -----------------------------------------------------------------------
  // firebill saying "I do not know"
  //
  // A publish whose broker confirm timed out may have been taken anyway. Over
  // eight days ~2,100 of these were answered `{"success": false}` — the same
  // body a refusal sends — and every one of those events had in fact settled,
  // while this client logged each as usage that would never be billed.
  // -----------------------------------------------------------------------

  it.each([
    ["a 504 carrying the field", () => ambiguous()],
    ["a 504 with no body at all", () => new Response(null, { status: 504 })],
    [
      "a 200 that says so by field",
      () =>
        new Response(JSON.stringify({ success: false, ambiguous: true }), {
          status: 200,
        }),
    ],
  ])("reads %s as ambiguous, never as refused", async (_label, respond) => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => respond()),
    );

    await expect(firebillTrack(params)).resolves.toBe(false);
    expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
    expect(await failureCauses()).toEqual({ "track/ambiguous": 1 });
  });

  // **An answer we cannot read is not proof of anything.** firebill may have
  // taken the event and said so in a shape we did not understand, so calling it
  // `refused` would log billed usage as lost — the same class of untruth this
  // PR exists to remove, one layer down.
  it.each([
    ["an empty 200", () => new Response("", { status: 200 })],
    [
      "a 200 that is not JSON",
      () => new Response("<html>502</html>", { status: 200 }),
    ],
    [
      "a 200 with no success field",
      () => new Response(JSON.stringify({ queued: true }), { status: 200 }),
    ],
    [
      "a 200 whose success is not a boolean",
      () => new Response(JSON.stringify({ success: "yes" }), { status: 200 }),
    ],
  ])("reads %s as unusable, not as a refusal", async (_label, respond) => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => respond()),
    );

    await expect(firebillTrack(params)).resolves.toBe(false);
    expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
    expect(await failureCauses()).toEqual({ "track/unusable": 1 });
    expect(vi.mocked(logger.error).mock.calls.map(c => c[0])).not.toContain(
      "firebill refused a usage event; it will not be billed",
    );
  });

  it("still reads success:false as a refusal", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => refused()),
    );

    await expect(firebillTrack(params)).resolves.toBe(false);
    expect(await trackOutcomes()).toEqual({ refused: 1 });
  });

  // A body that never finished arriving is the network failing, not firebill
  // answering — it has to reach the transport classifier, not be swallowed as
  // an unusable body.
  it("treats a body that dies mid-read as a transport failure", async () => {
    const dying = () => {
      const response = new Response(JSON.stringify({ success: true }), {
        status: 200,
      });
      vi.spyOn(response, "json").mockRejectedValue(
        Object.assign(new TypeError("terminated"), {
          cause: Object.assign(new Error("aborted"), {
            code: "UND_ERR_BODY_TIMEOUT",
          }),
        }),
      );
      return response;
    };
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => dying()),
    );

    await expect(firebillTrack(params)).resolves.toBe(false);
    expect(await failureCauses()).toEqual({ "track/timeout": 1 });
    expect(vi.mocked(logger.warn).mock.calls.map(c => c[0])).toContain(
      "firebill track request did not complete — client-side timeout or connection error; firebill did not answer",
    );
  });

  it("logs an ambiguous answer as unknown rather than as unbilled usage", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => ambiguous()),
    );

    await firebillTrack(params);

    const messages = vi.mocked(logger.error).mock.calls.map(c => c[0]);
    expect(messages).toContain(
      "firebill did not answer; the event may or may not have been accepted",
    );
    expect(messages).not.toContain(
      "firebill refused a usage event; it will not be billed",
    );
  });

  // -----------------------------------------------------------------------
  // Whose failure was it
  //
  // ~1,800 warnings per 30h said "firebill may be unavailable" against a
  // firebill whose server-side p99 was 73ms. They were our own 5s deadline.
  // -----------------------------------------------------------------------

  it.each([
    ["our own deadline firing", () => abortError(), "timeout"],
    ["an undici-wrapped deadline", () => wrapped(abortError()), "timeout"],
    [
      "a refused connection",
      () => wrapped(connectionError("ECONNREFUSED")),
      "connection",
    ],
    [
      "a reset connection",
      () => wrapped(connectionError("ECONNRESET")),
      "connection",
    ],
    [
      "a DNS failure",
      () => wrapped(connectionError("ENOTFOUND")),
      "connection",
    ],
  ])("labels %s as a client-side %s", async (_label, error, cause) => {
    vi.stubGlobal("fetch", vi.fn().mockRejectedValue(error()));

    await expect(firebillTrack(params)).resolves.toBe(false);
    expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
    expect(await failureCauses()).toEqual({ [`track/${cause}`]: 1 });
  });

  it("stops blaming firebill for a request that never reached it", async () => {
    vi.stubGlobal("fetch", vi.fn().mockRejectedValue(wrapped(abortError())));

    await firebillTrack(params);

    // The logger's own overloads make `calls` a one-element tuple to TS; every
    // call site here passes (message, fields).
    const warnings = vi.mocked(logger.warn).mock.calls as unknown as [
      string,
      Record<string, unknown>,
    ][];
    const messages = warnings.map(c => c[0]);
    expect(messages).toContain(
      "firebill track request did not complete — client-side timeout or connection error; firebill did not answer",
    );
    expect(messages.join(" ")).not.toContain("firebill may be unavailable");
    // The error's own identity, so the class of stall is greppable.
    // The innermost error's identity, not the `TypeError: fetch failed` undici
    // wraps it in — so the class of stall is greppable.
    expect(warnings[0][1]).toMatchObject({
      cause: "timeout",
      errorName: "TimeoutError",
      timeoutMs: 5000,
    });
  });

  it("names the status when firebill really did answer", async () => {
    vi.stubGlobal(
      "fetch",
      vi
        .fn()
        .mockImplementation(async () => new Response("nope", { status: 502 })),
    );

    await expect(firebillTrack(params)).resolves.toBe(false);
    expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
    expect(await failureCauses()).toEqual({ "track/non_ok": 1 });
    // A static message; the status is context, so log search can group these.
    const [message, fields] = vi.mocked(logger.warn).mock
      .calls[0] as unknown as [string, Record<string, unknown>];
    expect(message).toBe(
      "firebill track attempt failed — firebill answered a non-OK status",
    );
    expect(fields).toMatchObject({ status: 502 });
  });

  it("retries a thrown error too", async () => {
    const fetchMock = vi
      .fn()
      .mockRejectedValueOnce(new Error("connection refused"))
      .mockImplementationOnce(async () => ok());
    vi.stubGlobal("fetch", fetchMock);

    await expect(firebillTrack(params)).resolves.toBe(true);
    expect(fetchMock).toHaveBeenCalledTimes(2);
  });

  it("sends a refund to /v1/refund with the value made positive", async () => {
    const fetchMock = vi.fn().mockImplementation(async () => ok());
    vi.stubGlobal("fetch", fetchMock);

    await firebillTrack({ ...params, value: -7 });
    expect(fetchMock.mock.calls[0][0]).toBe("https://firebill.test/v1/refund");
    expect(JSON.parse(fetchMock.mock.calls[0][1].body).value).toBe(7);
  });

  it("logs one accepted line carrying the key, the amount and the build", async () => {
    configState.FIRECRAWL_BUILD_SHA = "abc123";
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => ok()),
    );

    await firebillTrack({ ...params, properties: { jobId: "job-1" } });

    expect(logger.info).toHaveBeenCalledTimes(1);
    expect(logger.info).toHaveBeenCalledWith(
      "firebill track succeeded",
      expect.objectContaining({
        operation: "track",
        outcome: "accepted",
        idempotencyKey: params.idempotencyKey,
        expectedValue: 3,
        jobId: "job-1",
        buildVersion: "abc123",
      }),
    );
  });

  it("logs a refund under its own operation with the amount made positive", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => ok()),
    );

    await firebillTrack({ ...params, value: -7 });

    expect(logger.info).toHaveBeenCalledWith(
      "firebill track succeeded",
      expect.objectContaining({
        operation: "refund",
        expectedValue: 7,
        buildVersion: "unknown",
      }),
    );
  });

  it("logs no success line when firebill did not take the event", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => refused()),
    );

    await firebillTrack(params);

    expect(logger.info).not.toHaveBeenCalled();
  });
});

const retryOutcomes = async () =>
  Object.fromEntries(
    (await firebillTrackRetryTotal.get()).values.map(v => [
      `${v.labels.operation}/${v.labels.outcome}`,
      v.value,
    ]),
  );

/** A retry job as the queue hands it to the worker on a given attempt. */
const retryJob = (
  data: FirebillTrackRetryJobData,
  attemptsMade: number,
): Job<FirebillTrackRetryJobData> =>
  ({
    data,
    attemptsMade,
    opts: { attempts: FIREBILL_DURABLE_RETRY_ATTEMPTS },
  }) as unknown as Job<FirebillTrackRetryJobData>;

/** Logger calls as [message, context], which the mocked signature hides. */
const logCalls = (method: "info" | "warn" | "error") =>
  vi.mocked(logger[method]).mock.calls as unknown as [
    string,
    Record<string, unknown>,
  ][];

describe("firebillTrack durable retry", () => {
  it("hands an unanswered track to the retry queue as the identical event", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => ambiguous()),
    );

    await expect(firebillTrack(params)).resolves.toBe(false);

    expect(retryQueueAdd).toHaveBeenCalledTimes(1);
    const [name, data, opts] = retryQueueAdd.mock.calls[0];
    expect(name).toBe("track");
    expect(data).toEqual(params);
    expect(opts).toMatchObject({
      jobId: "firebill|fc%3Atrack%3Ascrape%3Ajob-1",
      attempts: FIREBILL_DURABLE_RETRY_ATTEMPTS,
      backoff: { type: "exponential" },
    });
    // BullMQ waits backoff * 2^(n-1) after the nth failure: 30s to the first
    // re-send, then doubling, which puts the last one about two hours out.
    const gaps = [opts.delay];
    for (let n = 1; n < FIREBILL_DURABLE_RETRY_ATTEMPTS; n++) {
      gaps.push(opts.backoff.delay * 2 ** (n - 1));
    }
    expect(gaps.slice(0, 3)).toEqual([30_000, 60_000, 120_000]);
    const total = gaps.reduce((a, b) => a + b, 0);
    expect(total).toBeGreaterThan(2 * 60 * 60 * 1000);
    expect(total).toBeLessThan(3 * 60 * 60 * 1000);
    expect(await retryOutcomes()).toEqual({ "track/enqueued": 1 });
    // The alerted counter still says what the caller saw.
    expect(await trackOutcomes()).toEqual({ ambiguous: 1 });
    expect(logCalls("error")[0][1]).toMatchObject({
      idempotencyKey: params.idempotencyKey,
      handedToRetry: true,
    });
  });

  it("enqueues a timed-out refund under the minted key it already sent", async () => {
    const fetchMock = vi.fn().mockRejectedValue(wrapped(abortError()));
    vi.stubGlobal("fetch", fetchMock);

    await firebillTrack({ ...params, value: -3, idempotencyKey: undefined });

    const sentKey = JSON.parse(fetchMock.mock.calls[0][1].body).idempotency_key;
    const [name, data] = retryQueueAdd.mock.calls[0];
    expect(name).toBe("refund");
    expect(data).toMatchObject({ value: -3, idempotencyKey: sentKey });
    expect(await retryOutcomes()).toEqual({ "refund/enqueued": 1 });
  });

  it("does not enqueue an explicit refusal or an accepted event", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => refused()),
    );
    await firebillTrack(params);

    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => ok()),
    );
    await firebillTrack(params);

    expect(retryQueueAdd).not.toHaveBeenCalled();
  });

  it("keeps an unanswered event out of the retry when the caller opts out", async () => {
    const fetchMock = vi.fn().mockImplementation(async () => ambiguous());
    vi.stubGlobal("fetch", fetchMock);

    await expect(
      firebillTrack({ ...params, value: -1, durableRetry: false }),
    ).resolves.toBe(false);

    expect(retryQueueAdd).not.toHaveBeenCalled();
    expect(await retryOutcomes()).toEqual({});
    expect(logCalls("error")[0][1]).toMatchObject({ handedToRetry: false });
    // The flag is ours; firebill never sees it.
    expect(
      Object.hasOwn(
        JSON.parse(fetchMock.mock.calls[0][1].body),
        "durableRetry",
      ),
    ).toBe(false);
  });

  it("gives distinct keys distinct job ids even when they differ only in delimiters", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => ambiguous()),
    );

    await firebillTrack({ ...params, idempotencyKey: "a:b" });
    await firebillTrack({ ...params, idempotencyKey: "a|b" });

    const [first, second] = retryQueueAdd.mock.calls.map(c => c[2].jobId);
    expect(first).not.toBe(second);
  });

  it("counts an enqueue that does not confirm in time as unknown, not lost", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => ambiguous()),
    );
    retryQueueAdd.mockImplementationOnce(() => new Promise(() => {}));

    await expect(firebillTrack(params)).resolves.toBe(false);

    expect(await retryOutcomes()).toEqual({ "track/enqueue_timeout": 1 });
    expect(logCalls("error").map(c => c[0])).toContain(
      "firebill durable retry enqueue timed out; the retry may or may not have been queued",
    );
  });

  it("logs and counts an enqueue that fails, still reporting false", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => ambiguous()),
    );
    retryQueueAdd.mockRejectedValueOnce(new Error("redis down"));

    await expect(firebillTrack(params)).resolves.toBe(false);

    expect(await retryOutcomes()).toEqual({ "track/enqueue_failed": 1 });
    const messages = vi.mocked(logger.error).mock.calls.map(c => c[0]);
    expect(messages).toContain(
      "firebill durable retry could not be enqueued; the event will not be billed unless replayed",
    );
    expect(logCalls("error").at(-1)?.[1]).toMatchObject({
      handedToRetry: false,
    });
  });

  it("re-sends with the same key, customer, entity, feature and value", async () => {
    const fetchMock = vi.fn().mockImplementation(async () => ok());
    vi.stubGlobal("fetch", fetchMock);
    const event = { ...params, externalRequestId: "partner-op-42" };

    await processFirebillTrackRetry(retryJob(event, 0));

    expect(fetchMock.mock.calls[0][0]).toBe("https://firebill.test/v1/track");
    expect(JSON.parse(fetchMock.mock.calls[0][1].body)).toEqual({
      customer_id: "org-1",
      entity_id: "team-1",
      feature_id: "CREDITS",
      value: 3,
      properties: {},
      idempotency_key: params.idempotencyKey,
      external_request_id: "partner-op-42",
    });
    expect(await retryOutcomes()).toEqual({
      "track/retried": 1,
      "track/settled": 1,
    });
  });

  it("sends a refund retry to /v1/refund with the value made positive", async () => {
    const fetchMock = vi.fn().mockImplementation(async () => ok());
    vi.stubGlobal("fetch", fetchMock);

    await processFirebillTrackRetry(retryJob({ ...params, value: -3 }, 2));

    expect(fetchMock.mock.calls[0][0]).toBe("https://firebill.test/v1/refund");
    expect(JSON.parse(fetchMock.mock.calls[0][1].body)).toMatchObject({
      value: 3,
      idempotency_key: params.idempotencyKey,
    });
  });

  it("throws on a failed attempt so the queue backs off and tries again", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => ambiguous()),
    );

    await expect(
      processFirebillTrackRetry(retryJob(params, 0)),
    ).rejects.toThrow(/retry 1\/8 failed/);

    expect(await retryOutcomes()).toEqual({ "track/retried": 1 });
    const messages = vi.mocked(logger.error).mock.calls.map(c => c[0]);
    expect(messages).not.toContain(
      "firebill durable retry exhausted; the event may not have been billed",
    );
  });

  it("keeps retrying a refusal, since an earlier attempt may have landed", async () => {
    vi.stubGlobal(
      "fetch",
      vi.fn().mockImplementation(async () => refused()),
    );

    await expect(
      processFirebillTrackRetry(retryJob(params, 3)),
    ).rejects.toThrow();
    expect(await retryOutcomes()).toEqual({ "track/retried": 1 });
  });

  it("logs the key and amount and counts the value when the last attempt fails", async () => {
    vi.stubGlobal("fetch", vi.fn().mockRejectedValue(wrapped(abortError())));

    await expect(
      processFirebillTrackRetry(
        retryJob(params, FIREBILL_DURABLE_RETRY_ATTEMPTS - 1),
      ),
    ).rejects.toThrow(/retry 8\/8 failed/);

    expect(await retryOutcomes()).toEqual({
      "track/retried": 1,
      "track/exhausted": 1,
    });
    const [value] = (await firebillTrackRetryExhaustedValueTotal.get()).values;
    expect(value).toMatchObject({
      value: 3,
      labels: { operation: "track", feature: "CREDITS" },
    });
    const exhausted = logCalls("error").find(
      c =>
        c[0] ===
        "firebill durable retry exhausted; the event may not have been billed",
    );
    expect(exhausted?.[1]).toMatchObject({
      idempotencyKey: params.idempotencyKey,
      value: 3,
      attempt: 8,
      attempts: 8,
      cause: "timeout",
    });
  });
});

describe("firebillCheck", () => {
  const checkParams = {
    customerId: "org-1",
    entityId: "team-1",
    featureId: "CREDITS",
    value: 10,
    properties: { source: "checkCreditsMiddleware" },
  };

  const answer = (body: unknown, status = 200) =>
    vi.stubGlobal(
      "fetch",
      vi.fn().mockResolvedValue(new Response(JSON.stringify(body), { status })),
    );

  it("passes through a yes, with the funder's pool as remaining", async () => {
    answer({ success: true, allowed: true, remaining: 500 });
    await expect(firebillCheck(checkParams)).resolves.toEqual({
      status: "answered",
      allowed: true,
      remaining: 500,
    });
  });

  it("passes through a no — this is the only thing that may 402", async () => {
    answer({ success: true, allowed: false, remaining: 3 });
    await expect(firebillCheck(checkParams)).resolves.toEqual({
      status: "answered",
      allowed: false,
      remaining: 3,
    });
  });

  // The asymmetry with firebillTrack, which fails closed. Refusing to answer an
  // authorization question must not become a 402.
  it.each([
    ["firebill could not answer", { success: false }, 200],
    ["a non-OK response", { success: true, allowed: true }, 503],
    ["a missing allowed", { success: true }, 200],
    ["a non-boolean allowed", { success: true, allowed: "yes" }, 200],
  ])("fails open on %s", async (_label, body, status) => {
    answer(body, status);
    await expect(firebillCheck(checkParams)).resolves.toEqual({
      status: "unavailable",
    });
  });

  it("fails open when the request throws", async () => {
    vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("boom")));
    await expect(firebillCheck(checkParams)).resolves.toEqual({
      status: "unavailable",
    });
  });

  // A missing figure must not read as zero: callers clamp crawl limits with
  // it, so zero would silently shrink an allowed crawl to nothing.
  // The inverse of the case below, and the dangerous direction: the middleware
  // reads a denial with `remaining > 0` as a *partial* crawl and calls next(),
  // so an unbounded default would turn a refusal into an unlimited crawl.
  it("treats an absent remaining on a DENIAL as zero, not unbounded", async () => {
    answer({ success: true, allowed: false });
    await expect(firebillCheck(checkParams)).resolves.toEqual({
      status: "answered",
      allowed: false,
      remaining: 0,
    });
  });

  // A denial that *does* carry a figure keeps it: that is a legitimate partial
  // crawl, and the existing middleware behaviour we must not change.
  it("keeps a usable remaining on a denial", async () => {
    answer({ success: true, allowed: false, remaining: 25 });
    await expect(firebillCheck(checkParams)).resolves.toEqual({
      status: "answered",
      allowed: false,
      remaining: 25,
    });
  });

  // What a 402 names as the time credits come back.
  it("passes through the balance's reset time", async () => {
    answer({
      success: true,
      allowed: false,
      remaining: 0,
      resets_at: 1_798_761_600_000,
    });
    await expect(firebillCheck(checkParams)).resolves.toEqual({
      status: "answered",
      allowed: false,
      remaining: 0,
      resetsAt: 1_798_761_600_000,
    });
  });

  // A reset time is decoration on the answer, never a reason to fail open.
  it.each([
    ["a string", "2026-12-01T00:00:00Z"],
    ["a fraction", 1.5],
    ["zero", 0],
    ["past what a Date can hold", 8_640_000_000_000_001],
    ["null", null],
  ])(
    "drops a resets_at that is %s and keeps the answer",
    async (_label, resetsAt) => {
      answer({
        success: true,
        allowed: false,
        remaining: 0,
        resets_at: resetsAt,
      });
      await expect(firebillCheck(checkParams)).resolves.toEqual({
        status: "answered",
        allowed: false,
        remaining: 0,
      });
    },
  );

  it("treats an absent remaining on an ALLOWED check as unbounded, not zero", async () => {
    answer({ success: true, allowed: true });
    await expect(firebillCheck(checkParams)).resolves.toEqual({
      status: "answered",
      allowed: true,
      remaining: Infinity,
    });
  });

  it("sends the caller's context so Autumn records the decision", async () => {
    const fetchMock = vi
      .fn()
      .mockResolvedValue(
        new Response(
          JSON.stringify({ success: true, allowed: true, remaining: 1 }),
          { status: 200 },
        ),
      );
    vi.stubGlobal("fetch", fetchMock);

    await firebillCheck(checkParams);

    const [url, init] = fetchMock.mock.calls[0];
    expect(url).toBe("https://firebill.test/v1/check");
    expect(JSON.parse(init.body)).toEqual({
      customer_id: "org-1",
      entity_id: "team-1",
      feature_id: "CREDITS",
      value: 10,
      properties: { source: "checkCreditsMiddleware" },
    });
  });

  it.each([
    ["our own deadline firing", () => wrapped(abortError()), "check/timeout"],
    [
      "a refused connection",
      () => wrapped(connectionError("ECONNREFUSED")),
      "check/connection",
    ],
  ])(
    "blames the client, not firebill, for %s",
    async (_label, error, expected) => {
      vi.stubGlobal("fetch", vi.fn().mockRejectedValue(error()));

      await expect(firebillCheck(checkParams)).resolves.toEqual({
        status: "unavailable",
      });
      expect(await checkOutcomes()).toEqual({ unavailable: 1 });
      expect(await failureCauses()).toEqual({ [expected]: 1 });
      expect(vi.mocked(logger.error).mock.calls.map(c => c[0])).toContain(
        "firebill check unavailable — the request did not complete (client-side timeout or connection error); firebill did not answer",
      );
    },
  );

  it("names the status in context when firebill answered one", async () => {
    answer({ success: true, allowed: true }, 503);

    await firebillCheck(checkParams);

    expect(await checkOutcomes()).toEqual({ unavailable: 1 });
    expect(await failureCauses()).toEqual({ "check/non_ok": 1 });
    const [message, fields] = vi.mocked(logger.error).mock
      .calls[0] as unknown as [string, Record<string, unknown>];
    expect(message).toBe(
      "firebill check unavailable — firebill answered a non-OK status",
    );
    expect(fields).toMatchObject({ status: 503 });
  });

  it("records no cause at all for a real answer", async () => {
    answer({ success: true, allowed: true, remaining: 500 });
    await firebillCheck(checkParams);
    expect(await checkOutcomes()).toEqual({ allowed: 1 });
    expect(await failureCauses()).toEqual({});
  });

  // `refused` is documented as an explicit `success: false`. An answer we could
  // not read is not one, and labelling it so would have anyone slicing by cause
  // counting unreadable answers as declines.
  it.each([
    ["a missing allowed", { success: true }, 200],
    ["a non-boolean allowed", { success: true, allowed: "yes" }, 200],
  ])("calls %s unusable rather than refused", async (_label, body, status) => {
    answer(body, status);
    await firebillCheck(checkParams);
    expect(await failureCauses()).toEqual({ "check/unusable": 1 });
  });

  it("calls an unreadable body unusable rather than refused", async () => {
    vi.stubGlobal(
      "fetch",
      vi
        .fn()
        .mockResolvedValue(new Response("<html>502</html>", { status: 200 })),
    );
    await expect(firebillCheck(checkParams)).resolves.toEqual({
      status: "unavailable",
    });
    expect(await failureCauses()).toEqual({ "check/unusable": 1 });
  });

  it("keeps refused for an explicit success:false", async () => {
    answer({ success: false });
    await firebillCheck(checkParams);
    expect(await failureCauses()).toEqual({ "check/refused": 1 });
  });

  it("does not retry — this is on the request path", async () => {
    const fetchMock = vi.fn().mockRejectedValue(new Error("boom"));
    vi.stubGlobal("fetch", fetchMock);
    await firebillCheck(checkParams);
    expect(fetchMock).toHaveBeenCalledTimes(1);
  });
});

describe("firebillLock", () => {
  const lockParams = {
    customerId: "org-1",
    entityId: "team-1",
    featureId: "CREDITS",
    value: 10,
    lockId: "monitor_check-1",
    expiresAt: Date.now() + 60_000,
  };

  const answer = (body: unknown, status = 200) =>
    vi.stubGlobal(
      "fetch",
      vi.fn().mockResolvedValue(new Response(JSON.stringify(body), { status })),
    );

  it("locks and carries the operation token back on a gated hold", async () => {
    answer({
      success: true,
      allowed: true,
      lock_id: "monitor_check-1",
      operation_token: "op-tok",
    });
    await expect(firebillLock(lockParams)).resolves.toEqual({
      status: "locked",
      lockId: "monitor_check-1",
      operationToken: "op-tok",
    });
  });

  it("falls back to the caller's lock id when firebill echoes none", async () => {
    answer({ success: true, allowed: true });
    await expect(firebillLock(lockParams)).resolves.toEqual({
      status: "locked",
      lockId: "monitor_check-1",
    });
  });

  it("reads a denial and its reason", async () => {
    answer({ success: true, allowed: false, reason: "out_of_credits" });
    await expect(firebillLock(lockParams)).resolves.toEqual({
      status: "denied",
      reason: "out_of_credits",
    });
  });

  it("drops an unrecognised denial reason rather than passing it through", async () => {
    answer({ success: true, allowed: false, reason: "some_new_reason" });
    await expect(firebillLock(lockParams)).resolves.toEqual({
      status: "denied",
    });
  });

  // A shape firebill sent on purpose (`success: false`) is `refused`; one we
  // could not read is `unusable`. Both proceed unlocked, but slicing the cause
  // counter must tell them apart.
  it("calls an explicit success:false refused", async () => {
    answer({ success: false });
    await expect(firebillLock(lockParams)).resolves.toEqual({
      status: "unavailable",
    });
    expect(await failureCauses()).toEqual({ "lock/refused": 1 });
  });

  it.each([
    ["a missing allowed", { success: true }],
    ["a non-boolean allowed", { success: true, allowed: "yes" }],
    ["a missing success", { locked: true }],
  ])("calls %s unusable, not refused", async (_label, body) => {
    answer(body);
    await expect(firebillLock(lockParams)).resolves.toEqual({
      status: "unavailable",
    });
    expect(await failureCauses()).toEqual({ "lock/unusable": 1 });
  });

  it("calls an unreadable body unusable", async () => {
    vi.stubGlobal(
      "fetch",
      vi
        .fn()
        .mockResolvedValue(new Response("<html>502</html>", { status: 200 })),
    );
    await expect(firebillLock(lockParams)).resolves.toEqual({
      status: "unavailable",
    });
    expect(await failureCauses()).toEqual({ "lock/unusable": 1 });
  });

  it("names a non-OK status as its own cause", async () => {
    answer({ success: true, allowed: true }, 503);
    await expect(firebillLock(lockParams)).resolves.toEqual({
      status: "unavailable",
    });
    expect(await failureCauses()).toEqual({ "lock/non_ok": 1 });
  });
});

describe("firebillFinalize", () => {
  const finalizeParams = {
    lockId: "monitor_check-1",
    action: "confirm" as const,
  };

  const answer = (body: unknown, status = 200) =>
    vi.stubGlobal(
      "fetch",
      vi.fn().mockResolvedValue(new Response(JSON.stringify(body), { status })),
    );

  it("returns true when firebill accepted the settle", async () => {
    answer({ success: true });
    await expect(firebillFinalize(finalizeParams)).resolves.toBe(true);
  });

  it("logs the settle's key and the amount it bills", async () => {
    answer({ success: true });
    await firebillFinalize({
      ...finalizeParams,
      overrideValue: 2,
      heldValue: 5,
    });
    expect(logger.info).toHaveBeenCalledWith(
      "firebill finalize succeeded",
      expect.objectContaining({
        operation: "finalize",
        outcome: "accepted",
        idempotencyKey: "fc:finalize:confirm:monitor_check-1",
        expectedValue: 2,
      }),
    );

    vi.mocked(logger.info).mockClear();
    answer({ success: true });
    await firebillFinalize({ ...finalizeParams, heldValue: 5 });
    expect(logger.info).toHaveBeenCalledWith(
      "firebill finalize succeeded",
      expect.objectContaining({ expectedValue: 5 }),
    );

    vi.mocked(logger.info).mockClear();
    answer({ success: true });
    await firebillFinalize({
      ...finalizeParams,
      action: "release",
      heldValue: 5,
    });
    expect(logger.info).toHaveBeenCalledWith(
      "firebill finalize succeeded",
      expect.objectContaining({
        idempotencyKey: "fc:finalize:release:monitor_check-1",
        expectedValue: 0,
      }),
    );
  });

  it("logs nothing as succeeded when the settle was refused", async () => {
    answer({ success: false });
    await firebillFinalize(finalizeParams);
    expect(logger.info).not.toHaveBeenCalled();
  });

  it("calls an explicit success:false refused", async () => {
    answer({ success: false });
    await expect(firebillFinalize(finalizeParams)).resolves.toBe(false);
    expect(await failureCauses()).toEqual({ "finalize/refused": 1 });
  });

  it.each([
    ["a missing success", { queued: true }],
    ["a non-boolean success", { success: "yes" }],
  ])("calls %s unusable, not refused", async (_label, body) => {
    answer(body);
    await expect(firebillFinalize(finalizeParams)).resolves.toBe(false);
    expect(await failureCauses()).toEqual({ "finalize/unusable": 1 });
  });

  it("calls an unreadable body unusable", async () => {
    vi.stubGlobal(
      "fetch",
      vi
        .fn()
        .mockResolvedValue(new Response("<html>502</html>", { status: 200 })),
    );
    await expect(firebillFinalize(finalizeParams)).resolves.toBe(false);
    expect(await failureCauses()).toEqual({ "finalize/unusable": 1 });
  });

  it("reads a 504 as ambiguous rather than refused", async () => {
    answer({ success: false, ambiguous: true }, 504);
    await expect(firebillFinalize(finalizeParams)).resolves.toBe(false);
    expect(await failureCauses()).toEqual({ "finalize/ambiguous": 1 });
  });
});