Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Ask firebill whether a gateway org can afford it, not just the ghost (#4404)

Rakshith Ramprakash51202ab55fec
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index 67f97d354..a35d1378a 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -838,6 +838,105 @@ describe("firebill routing", () => {
});
});
+ // The gate, not the charge. Before this branch existed checkCredits asked
+ // Autumn about the ghost's balance alone β€” and a gateway ghost is designed to
+ // spend credits it does not have, so it 402'd exactly the requests the
+ // partner pool exists to fund.
+ it("routes checkCredits for an allowlisted org to /v1/check, not Autumn", async () => {
+ state.configRef = firebillConfig();
+ mockFetch.mockResolvedValue(
+ new Response(
+ JSON.stringify({ success: true, allowed: true, remaining: 500 }),
+ { status: 200 },
+ ),
+ );
+ const svc = makeService();
+
+ const result = await svc.checkCredits({
+ teamId: "team-1",
+ value: 100,
+ properties: { source: "checkCreditsMiddleware" },
+ });
+
+ // 500 is the funder's pool, not the ghost's balance β€” the whole point.
+ expect(result).toEqual({ allowed: true, remaining: 500 });
+ expect(mockCheck).not.toHaveBeenCalled();
+
+ const [url, init] = mockFetch.mock.calls[0];
+ expect(String(url)).toBe("http://firebill.test/v1/check");
+ expect(JSON.parse(init.body)).toEqual({
+ customer_id: "org-1",
+ entity_id: "team-1",
+ feature_id: "CREDITS",
+ value: 100,
+ properties: { source: "checkCreditsMiddleware" },
+ // A read: no lock, no idempotency key, nothing to dedupe.
+ });
+ });
+
+ // The same property as the charge paths, and it bites harder on the gate: a
+ // provisioned org that misses firebill on a charge is billed to the wrong
+ // account, but one that misses firebill on the *gate* is refused outright β€”
+ // Autumn is asked about a balance the org was never meant to pay from. So a
+ // sampling bucket must not decide it.
+ it("routes a gateway-provisioned team's CHECK off the allowlist and at 0 percent", async () => {
+ state.configRef = {
+ ...firebillConfig(),
+ FIREBILL_ORG_IDS: ["some-other-org"],
+ FIREBILL_ROLLOUT_PERCENT: 0,
+ };
+ state.gatewayStubRow = { team_id: "team-1" };
+ mockFetch.mockResolvedValue(
+ new Response(
+ JSON.stringify({ success: true, allowed: true, remaining: 500 }),
+ { status: 200 },
+ ),
+ );
+ const svc = makeService();
+
+ const result = await svc.checkCredits({
+ teamId: "team-1",
+ value: 100,
+ properties: {},
+ });
+
+ // 500 is the funder's pool. Straight to Autumn this would have been the
+ // ghost's own balance, and a drained ghost would have been a 402.
+ expect(result).toEqual({ allowed: true, remaining: 500 });
+ expect(mockCheck).not.toHaveBeenCalled();
+ expect(String(mockFetch.mock.calls[0][0])).toBe(
+ "http://firebill.test/v1/check",
+ );
+ });
+
+ // Fails OPEN, unlike the charge paths. `null` is the contract the middleware
+ // already treats as "proceed", so a firebill wobble cannot 402 a customer.
+ it("returns null when firebill cannot answer a check", async () => {
+ state.configRef = firebillConfig();
+ mockFetch.mockResolvedValue(
+ new Response(JSON.stringify({ success: false }), { status: 200 }),
+ );
+ const svc = makeService();
+
+ await expect(
+ svc.checkCredits({ teamId: "team-1", value: 100, properties: {} }),
+ ).resolves.toBeNull();
+ expect(mockCheck).not.toHaveBeenCalled();
+ });
+
+ it("checks directly in Autumn for orgs NOT on the allowlist", async () => {
+ state.configRef = {
+ ...firebillConfig(),
+ FIREBILL_ORG_IDS: ["some-other-org"],
+ };
+ const svc = makeService();
+
+ await svc.checkCredits({ teamId: "team-1", value: 1, properties: {} });
+
+ expect(mockCheck).toHaveBeenCalledTimes(1);
+ expect(mockFetch).not.toHaveBeenCalled();
+ });
+
it("routes refundCredits to /v1/refund with the POSITIVE value", async () => {
state.configRef = firebillConfig();
const svc = makeService();
diff --git a/apps/api/src/services/autumn/__tests__/firebill.test.ts b/apps/api/src/services/autumn/__tests__/firebill.test.ts
index cfa904854..006b41201 100644
--- a/apps/api/src/services/autumn/__tests__/firebill.test.ts
+++ b/apps/api/src/services/autumn/__tests__/firebill.test.ts
@@ -1,6 +1,14 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { firebillTrack, shouldRouteToFirebill } from "../firebill";
-import { firebillRetryTotal, firebillTrackTotal } from "../metrics";
+import {
+ firebillCheck,
+ firebillTrack,
+ shouldRouteToFirebill,
+} from "../firebill";
+import {
+ firebillCheckTotal,
+ firebillRetryTotal,
+ firebillTrackTotal,
+} from "../metrics";
const { configState } = vi.hoisted(() => ({
configState: {
@@ -39,6 +47,7 @@ beforeEach(() => {
configState.FIREBILL_ROLLOUT_PERCENT = 0;
firebillTrackTotal.reset();
firebillRetryTotal.reset();
+ firebillCheckTotal.reset();
});
afterEach(() => vi.unstubAllGlobals());
@@ -180,3 +189,123 @@ describe("firebillTrack", () => {
expect(JSON.parse(fetchMock.mock.calls[0][1].body).value).toBe(7);
});
});
+
+describe("firebillCheck", () => {
+ const checkParams = {
+ customerId: "org-1",
+ entityId: "team-1",
+ featureId: "CREDITS",
+ value: 10,
+ properties: { source: "checkCreditsMiddleware" },
+ };
+
+ const answer = (body: unknown, status = 200) =>
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockResolvedValue(new Response(JSON.stringify(body), { status })),
+ );
+
+ it("passes through a yes, with the funder's pool as remaining", async () => {
+ answer({ success: true, allowed: true, remaining: 500 });
+ await expect(firebillCheck(checkParams)).resolves.toEqual({
+ status: "answered",
+ allowed: true,
+ remaining: 500,
+ });
+ });
+
+ it("passes through a no β€” this is the only thing that may 402", async () => {
+ answer({ success: true, allowed: false, remaining: 3 });
+ await expect(firebillCheck(checkParams)).resolves.toEqual({
+ status: "answered",
+ allowed: false,
+ remaining: 3,
+ });
+ });
+
+ // The asymmetry with firebillTrack, which fails closed. Refusing to answer an
+ // authorization question must not become a 402.
+ it.each([
+ ["firebill could not answer", { success: false }, 200],
+ ["a non-OK response", { success: true, allowed: true }, 503],
+ ["a missing allowed", { success: true }, 200],
+ ["a non-boolean allowed", { success: true, allowed: "yes" }, 200],
+ ])("fails open on %s", async (_label, body, status) => {
+ answer(body, status);
+ await expect(firebillCheck(checkParams)).resolves.toEqual({
+ status: "unavailable",
+ });
+ });
+
+ it("fails open when the request throws", async () => {
+ vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("boom")));
+ await expect(firebillCheck(checkParams)).resolves.toEqual({
+ status: "unavailable",
+ });
+ });
+
+ // A missing figure must not read as zero: callers clamp crawl limits with
+ // it, so zero would silently shrink an allowed crawl to nothing.
+ // The inverse of the case below, and the dangerous direction: the middleware
+ // reads a denial with `remaining > 0` as a *partial* crawl and calls next(),
+ // so an unbounded default would turn a refusal into an unlimited crawl.
+ it("treats an absent remaining on a DENIAL as zero, not unbounded", async () => {
+ answer({ success: true, allowed: false });
+ await expect(firebillCheck(checkParams)).resolves.toEqual({
+ status: "answered",
+ allowed: false,
+ remaining: 0,
+ });
+ });
+
+ // A denial that *does* carry a figure keeps it: that is a legitimate partial
+ // crawl, and the existing middleware behaviour we must not change.
+ it("keeps a usable remaining on a denial", async () => {
+ answer({ success: true, allowed: false, remaining: 25 });
+ await expect(firebillCheck(checkParams)).resolves.toEqual({
+ status: "answered",
+ allowed: false,
+ remaining: 25,
+ });
+ });
+
+ it("treats an absent remaining on an ALLOWED check as unbounded, not zero", async () => {
+ answer({ success: true, allowed: true });
+ await expect(firebillCheck(checkParams)).resolves.toEqual({
+ status: "answered",
+ allowed: true,
+ remaining: Infinity,
+ });
+ });
+
+ it("sends the caller's context so Autumn records the decision", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValue(
+ new Response(
+ JSON.stringify({ success: true, allowed: true, remaining: 1 }),
+ { status: 200 },
+ ),
+ );
+ vi.stubGlobal("fetch", fetchMock);
+
+ await firebillCheck(checkParams);
+
+ const [url, init] = fetchMock.mock.calls[0];
+ expect(url).toBe("https://firebill.test/v1/check");
+ expect(JSON.parse(init.body)).toEqual({
+ customer_id: "org-1",
+ entity_id: "team-1",
+ feature_id: "CREDITS",
+ value: 10,
+ properties: { source: "checkCreditsMiddleware" },
+ });
+ });
+
+ it("does not retry β€” this is on the request path", async () => {
+ const fetchMock = vi.fn().mockRejectedValue(new Error("boom"));
+ vi.stubGlobal("fetch", fetchMock);
+ await firebillCheck(checkParams);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index 5b60be146..3b51268ce 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -9,6 +9,7 @@ import {
firebillFinalize,
firebillLock,
firebillTrack,
+ firebillCheck,
shouldRouteToFirebill,
} from "./firebill";
import { billingRouteTotal } from "./metrics";
@@ -470,6 +471,39 @@ export class AutumnService {
}
try {
const customerId = await this.ensureTrackingContext(teamId);
+
+ // Mirrors track() and lockCredits(). Without this branch the gate reads
+ // the ghost's balance alone, and a gateway ghost is designed to spend
+ // credits it does not have β€” so it would 402 exactly the requests the
+ // partner pool exists to fund. firebill answers with the same arithmetic
+ // settlement uses.
+ //
+ // `unavailable` becomes `null`, which this method's existing contract
+ // already means "fail open" β€” the same answer a null from Autumn gets
+ // below, for the same reason.
+ //
+ // `gatewayProvisioned` matters more here than on the charge paths. A
+ // partner-provisioned org that misses firebill on a *charge* is billed to
+ // the wrong account; one that misses firebill on the *gate* is refused
+ // outright, because Autumn is asked about a balance the org was never
+ // meant to pay from. So the org this most needs to reach firebill is
+ // exactly the one a sampling bucket might leave behind.
+ if (
+ shouldRouteToFirebill(customerId, {
+ gatewayProvisioned: await this.isGatewayProvisioned(teamId),
+ })
+ ) {
+ const result = await firebillCheck({
+ customerId,
+ entityId: teamId,
+ featureId,
+ value,
+ properties,
+ });
+ if (result.status === "unavailable") return null;
+ return { allowed: result.allowed, remaining: result.remaining };
+ }
+
const { allowed, balance } = await autumnClient.check({
customerId,
entityId: teamId,
diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.ts
index c756f8c5b..864894500 100644
--- a/apps/api/src/services/autumn/firebill.ts
+++ b/apps/api/src/services/autumn/firebill.ts
@@ -3,7 +3,11 @@ import { config } from "../../config";
import { logger } from "../../lib/logger";
import { sampled } from "../../lib/rollout";
import type { TrackParams } from "./types";
-import { firebillRetryTotal, firebillTrackTotal } from "./metrics";
+import {
+ firebillCheckTotal,
+ firebillRetryTotal,
+ firebillTrackTotal,
+} from "./metrics";
/**
* Outcome of a firebill `/v1/lock` call, mirroring firebill's three-state
@@ -158,8 +162,7 @@ export async function firebillTrack(params: TrackParams): Promise<boolean> {
}
type AttemptResult =
- | { ok: true }
- | { ok: false; reason: "not_ok" | "not_success" | "exception" };
+ { ok: true } | { ok: false; reason: "not_ok" | "not_success" | "exception" };
async function firebillAttempt(
path: string,
@@ -257,6 +260,137 @@ async function firebillAttempt(
* lock's lifecycle across two routes would let a firebill-side hold and a
* fallback hold coexist under retries.
*/
+/**
+ * Three outcomes, and callers must tell them apart: `answered` carries a real
+ * yes/no, `unavailable` means firebill could not answer at all.
+ */
+export type FirebillCheckResult =
+ | { status: "answered"; allowed: boolean; remaining: number }
+ | { status: "unavailable" };
+
+/**
+ * Asks firebill whether a customer can afford a charge.
+ *
+ * For a gateway-funded org this counts the funder's pool, which is the reason
+ * it exists: a ghost spends credits it does not have, so a gate reading the
+ * ghost's balance alone refuses the requests the partner pool is there to pay
+ * for. firebill answers with the same arithmetic settlement uses.
+ *
+ * **Fails open, unlike {@link firebillTrack}.** Every failure maps to
+ * `unavailable`, and the caller's contract is to let the request through.
+ * Declining to charge loses nothing that cannot be replayed; declining to
+ * *answer* an authorization question would turn a firebill blip into a
+ * customer-facing outage. There is deliberately no retry either β€” this sits on
+ * the request path, and a second round trip buys less than failing open fast.
+ */
+export async function firebillCheck({
+ customerId,
+ entityId,
+ featureId,
+ value,
+ properties,
+}: {
+ customerId: string;
+ entityId: string;
+ featureId: string;
+ value: number;
+ properties?: Record<string, unknown>;
+}): Promise<FirebillCheckResult> {
+ const unavailable = (
+ reason: string,
+ extra?: Record<string, unknown>,
+ ): FirebillCheckResult => {
+ logger.error(`firebill check unavailable β€” ${reason}`, {
+ customerId,
+ entityId,
+ featureId,
+ value,
+ ...extra,
+ });
+ firebillCheckTotal.labels("unavailable").inc();
+ return { status: "unavailable" };
+ };
+
+ try {
+ const response = await fetch(firebillUrl("/v1/check"), {
+ method: "POST",
+ headers: {
+ authorization: `Bearer ${config.FIREBILL_SECRET}`,
+ "content-type": "application/json",
+ },
+ body: JSON.stringify({
+ customer_id: customerId,
+ entity_id: entityId,
+ feature_id: featureId,
+ value,
+ properties,
+ }),
+ signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),
+ });
+
+ if (!response.ok) {
+ // Release the socket. Returning without reading or cancelling leaves the
+ // body unconsumed, and undici keeps the connection pinned until it is β€”
+ // so a firebill that is erroring would exhaust the pool and turn one
+ // failure into a run of them. Matters most here: this path is the one
+ // that fails open, so the leak would be silently widening the window in
+ // which credit checks are skipped.
+ response.body?.cancel().catch(() => {});
+ return unavailable("non-OK response", { status: response.status });
+ }
+
+ const body = (await response.json()) as {
+ success?: boolean;
+ allowed?: boolean;
+ remaining?: number;
+ };
+
+ // `success: false` is firebill saying it does not know β€” an unanswered
+ // balance, or a gateway lookup that failed. Never a denial.
+ if (body.success !== true) {
+ return unavailable("firebill could not answer");
+ }
+ // A missing or mis-shaped `allowed` is not something firebill sends today.
+ // Reading it as a denial would 402 a paying customer, so it fails open.
+ if (typeof body.allowed !== "boolean") {
+ return unavailable("answered without a usable `allowed`");
+ }
+
+ // `remaining` clamps downstream limits, and the safe default inverts with
+ // `allowed`, so there is no single one.
+ //
+ // `checkCreditsMiddleware` treats a denial with `remaining > 0` as a
+ // *partial* crawl: it rewrites `limit` to that figure and calls `next()`
+ // rather than returning 402. So defaulting a denial to `Infinity` would
+ // turn "cannot afford this" into an unbounded crawl β€” the opposite of a
+ // refusal, and worse than the 402 this endpoint exists to avoid.
+ //
+ // Allowed keeps `Infinity`, for the mirror-image reason: zero there would
+ // silently shrink a crawl the customer *can* pay for down to nothing.
+ // A usable figure is always preferred to either default.
+ const remaining =
+ typeof body.remaining === "number" && Number.isFinite(body.remaining)
+ ? body.remaining
+ : body.allowed
+ ? Infinity
+ : 0;
+
+ firebillCheckTotal.labels(body.allowed ? "allowed" : "denied").inc();
+ if (!body.allowed) {
+ logger.info("firebill check denied", {
+ customerId,
+ entityId,
+ featureId,
+ value,
+ remaining,
+ });
+ }
+ return { status: "answered", allowed: body.allowed, remaining };
+ } catch (error) {
+ return unavailable("request threw", { error });
+ }
+}
+
export async function firebillLock({
customerId,
entityId,
diff --git a/apps/api/src/services/autumn/metrics.ts b/apps/api/src/services/autumn/metrics.ts
index abff7e25a..16ecf49c7 100644
--- a/apps/api/src/services/autumn/metrics.ts
+++ b/apps/api/src/services/autumn/metrics.ts
@@ -32,3 +32,19 @@ export const firebillRetryTotal = new Counter({
help: "Retried firebill calls, by why the previous attempt failed",
labelNames: ["reason"] as const, // not_ok | not_success | exception
});
+
+/**
+ * What firebill's credit check answered. `denied` is a real "cannot afford it"
+ * and becomes a 402; `unavailable` means firebill could not answer, so the
+ * caller **failed open** and the request proceeded unauthorized.
+ *
+ * Watch `unavailable`: unlike a refused charge it is invisible to the customer
+ * and to the balance, so a firebill wobble here shows up nowhere else. And for
+ * a gateway org `denied` means a partner's pool ran dry, which is a commercial
+ * event rather than a fault.
+ */
+export const firebillCheckTotal = new Counter({
+ name: "firecrawl_firebill_check_total",
+ help: "Outcomes of credit checks sent to firebill",
+ labelNames: ["outcome"] as const, // allowed | denied | unavailable
+});