Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Ask firebill whether a gateway org can afford it, not just the ghost (#4404)
Rakshith Ramprakash51202ab55fec
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.tsindex 67f97d354..a35d1378a 100644--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts@@ -838,6 +838,105 @@ describe("firebill routing", () => {});});+ // The gate, not the charge. Before this branch existed checkCredits asked+ // Autumn about the ghost's balance alone β and a gateway ghost is designed to+ // spend credits it does not have, so it 402'd exactly the requests the+ // partner pool exists to fund.+ it("routes checkCredits for an allowlisted org to /v1/check, not Autumn", async () => {+ state.configRef = firebillConfig();+ mockFetch.mockResolvedValue(+ new Response(+ JSON.stringify({ success: true, allowed: true, remaining: 500 }),+ { status: 200 },+ ),+ );+ const svc = makeService();++ const result = await svc.checkCredits({+ teamId: "team-1",+ value: 100,+ properties: { source: "checkCreditsMiddleware" },+ });++ // 500 is the funder's pool, not the ghost's balance β the whole point.+ expect(result).toEqual({ allowed: true, remaining: 500 });+ expect(mockCheck).not.toHaveBeenCalled();++ const [url, init] = mockFetch.mock.calls[0];+ expect(String(url)).toBe("http://firebill.test/v1/check");+ expect(JSON.parse(init.body)).toEqual({+ customer_id: "org-1",+ entity_id: "team-1",+ feature_id: "CREDITS",+ value: 100,+ properties: { source: "checkCreditsMiddleware" },+ // A read: no lock, no idempotency key, nothing to dedupe.+ });+ });++ // The same property as the charge paths, and it bites harder on the gate: a+ // provisioned org that misses firebill on a charge is billed to the wrong+ // account, but one that misses firebill on the *gate* is refused outright β+ // Autumn is asked about a balance the org was never meant to pay from. So a+ // sampling bucket must not decide it.+ it("routes a gateway-provisioned team's CHECK off the allowlist and at 0 percent", async () => {+ state.configRef = {+ ...firebillConfig(),+ FIREBILL_ORG_IDS: ["some-other-org"],+ FIREBILL_ROLLOUT_PERCENT: 0,+ };+ state.gatewayStubRow = { team_id: "team-1" };+ mockFetch.mockResolvedValue(+ new Response(+ JSON.stringify({ success: true, allowed: true, remaining: 500 }),+ { status: 200 },+ ),+ );+ const svc = makeService();++ const result = await svc.checkCredits({+ teamId: "team-1",+ value: 100,+ properties: {},+ });++ // 500 is the funder's pool. Straight to Autumn this would have been the+ // ghost's own balance, and a drained ghost would have been a 402.+ expect(result).toEqual({ allowed: true, remaining: 500 });+ expect(mockCheck).not.toHaveBeenCalled();+ expect(String(mockFetch.mock.calls[0][0])).toBe(+ "http://firebill.test/v1/check",+ );+ });++ // Fails OPEN, unlike the charge paths. `null` is the contract the middleware+ // already treats as "proceed", so a firebill wobble cannot 402 a customer.+ it("returns null when firebill cannot answer a check", async () => {+ state.configRef = firebillConfig();+ mockFetch.mockResolvedValue(+ new Response(JSON.stringify({ success: false }), { status: 200 }),+ );+ const svc = makeService();++ await expect(+ svc.checkCredits({ teamId: "team-1", value: 100, properties: {} }),+ ).resolves.toBeNull();+ expect(mockCheck).not.toHaveBeenCalled();+ });++ it("checks directly in Autumn for orgs NOT on the allowlist", async () => {+ state.configRef = {+ ...firebillConfig(),+ FIREBILL_ORG_IDS: ["some-other-org"],+ };+ const svc = makeService();++ await svc.checkCredits({ teamId: "team-1", value: 1, properties: {} });++ expect(mockCheck).toHaveBeenCalledTimes(1);+ expect(mockFetch).not.toHaveBeenCalled();+ });+it("routes refundCredits to /v1/refund with the POSITIVE value", async () => {state.configRef = firebillConfig();const svc = makeService();diff --git a/apps/api/src/services/autumn/__tests__/firebill.test.ts b/apps/api/src/services/autumn/__tests__/firebill.test.tsindex cfa904854..006b41201 100644--- a/apps/api/src/services/autumn/__tests__/firebill.test.ts+++ b/apps/api/src/services/autumn/__tests__/firebill.test.ts@@ -1,6 +1,14 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";-import { firebillTrack, shouldRouteToFirebill } from "../firebill";-import { firebillRetryTotal, firebillTrackTotal } from "../metrics";+import {+ firebillCheck,+ firebillTrack,+ shouldRouteToFirebill,+} from "../firebill";+import {+ firebillCheckTotal,+ firebillRetryTotal,+ firebillTrackTotal,+} from "../metrics";const { configState } = vi.hoisted(() => ({configState: {@@ -39,6 +47,7 @@ beforeEach(() => {configState.FIREBILL_ROLLOUT_PERCENT = 0;firebillTrackTotal.reset();firebillRetryTotal.reset();+ firebillCheckTotal.reset();});afterEach(() => vi.unstubAllGlobals());@@ -180,3 +189,123 @@ describe("firebillTrack", () => {expect(JSON.parse(fetchMock.mock.calls[0][1].body).value).toBe(7);});});++describe("firebillCheck", () => {+ const checkParams = {+ customerId: "org-1",+ entityId: "team-1",+ featureId: "CREDITS",+ value: 10,+ properties: { source: "checkCreditsMiddleware" },+ };++ const answer = (body: unknown, status = 200) =>+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue(new Response(JSON.stringify(body), { status })),+ );++ it("passes through a yes, with the funder's pool as remaining", async () => {+ answer({ success: true, allowed: true, remaining: 500 });+ await expect(firebillCheck(checkParams)).resolves.toEqual({+ status: "answered",+ allowed: true,+ remaining: 500,+ });+ });++ it("passes through a no β this is the only thing that may 402", async () => {+ answer({ success: true, allowed: false, remaining: 3 });+ await expect(firebillCheck(checkParams)).resolves.toEqual({+ status: "answered",+ allowed: false,+ remaining: 3,+ });+ });++ // The asymmetry with firebillTrack, which fails closed. Refusing to answer an+ // authorization question must not become a 402.+ it.each([+ ["firebill could not answer", { success: false }, 200],+ ["a non-OK response", { success: true, allowed: true }, 503],+ ["a missing allowed", { success: true }, 200],+ ["a non-boolean allowed", { success: true, allowed: "yes" }, 200],+ ])("fails open on %s", async (_label, body, status) => {+ answer(body, status);+ await expect(firebillCheck(checkParams)).resolves.toEqual({+ status: "unavailable",+ });+ });++ it("fails open when the request throws", async () => {+ vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("boom")));+ await expect(firebillCheck(checkParams)).resolves.toEqual({+ status: "unavailable",+ });+ });++ // A missing figure must not read as zero: callers clamp crawl limits with+ // it, so zero would silently shrink an allowed crawl to nothing.+ // The inverse of the case below, and the dangerous direction: the middleware+ // reads a denial with `remaining > 0` as a *partial* crawl and calls next(),+ // so an unbounded default would turn a refusal into an unlimited crawl.+ it("treats an absent remaining on a DENIAL as zero, not unbounded", async () => {+ answer({ success: true, allowed: false });+ await expect(firebillCheck(checkParams)).resolves.toEqual({+ status: "answered",+ allowed: false,+ remaining: 0,+ });+ });++ // A denial that *does* carry a figure keeps it: that is a legitimate partial+ // crawl, and the existing middleware behaviour we must not change.+ it("keeps a usable remaining on a denial", async () => {+ answer({ success: true, allowed: false, remaining: 25 });+ await expect(firebillCheck(checkParams)).resolves.toEqual({+ status: "answered",+ allowed: false,+ remaining: 25,+ });+ });++ it("treats an absent remaining on an ALLOWED check as unbounded, not zero", async () => {+ answer({ success: true, allowed: true });+ await expect(firebillCheck(checkParams)).resolves.toEqual({+ status: "answered",+ allowed: true,+ remaining: Infinity,+ });+ });++ it("sends the caller's context so Autumn records the decision", async () => {+ const fetchMock = vi+ .fn()+ .mockResolvedValue(+ new Response(+ JSON.stringify({ success: true, allowed: true, remaining: 1 }),+ { status: 200 },+ ),+ );+ vi.stubGlobal("fetch", fetchMock);++ await firebillCheck(checkParams);++ const [url, init] = fetchMock.mock.calls[0];+ expect(url).toBe("https://firebill.test/v1/check");+ expect(JSON.parse(init.body)).toEqual({+ customer_id: "org-1",+ entity_id: "team-1",+ feature_id: "CREDITS",+ value: 10,+ properties: { source: "checkCreditsMiddleware" },+ });+ });++ it("does not retry β this is on the request path", async () => {+ const fetchMock = vi.fn().mockRejectedValue(new Error("boom"));+ vi.stubGlobal("fetch", fetchMock);+ await firebillCheck(checkParams);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ });+});diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex 5b60be146..3b51268ce 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -9,6 +9,7 @@ import {firebillFinalize,firebillLock,firebillTrack,+ firebillCheck,shouldRouteToFirebill,} from "./firebill";import { billingRouteTotal } from "./metrics";@@ -470,6 +471,39 @@ export class AutumnService {}try {const customerId = await this.ensureTrackingContext(teamId);++ // Mirrors track() and lockCredits(). Without this branch the gate reads+ // the ghost's balance alone, and a gateway ghost is designed to spend+ // credits it does not have β so it would 402 exactly the requests the+ // partner pool exists to fund. firebill answers with the same arithmetic+ // settlement uses.+ //+ // `unavailable` becomes `null`, which this method's existing contract+ // already means "fail open" β the same answer a null from Autumn gets+ // below, for the same reason.+ //+ // `gatewayProvisioned` matters more here than on the charge paths. A+ // partner-provisioned org that misses firebill on a *charge* is billed to+ // the wrong account; one that misses firebill on the *gate* is refused+ // outright, because Autumn is asked about a balance the org was never+ // meant to pay from. So the org this most needs to reach firebill is+ // exactly the one a sampling bucket might leave behind.+ if (+ shouldRouteToFirebill(customerId, {+ gatewayProvisioned: await this.isGatewayProvisioned(teamId),+ })+ ) {+ const result = await firebillCheck({+ customerId,+ entityId: teamId,+ featureId,+ value,+ properties,+ });+ if (result.status === "unavailable") return null;+ return { allowed: result.allowed, remaining: result.remaining };+ }+const { allowed, balance } = await autumnClient.check({customerId,entityId: teamId,diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.tsindex c756f8c5b..864894500 100644--- a/apps/api/src/services/autumn/firebill.ts+++ b/apps/api/src/services/autumn/firebill.ts@@ -3,7 +3,11 @@ import { config } from "../../config";import { logger } from "../../lib/logger";import { sampled } from "../../lib/rollout";import type { TrackParams } from "./types";-import { firebillRetryTotal, firebillTrackTotal } from "./metrics";+import {+ firebillCheckTotal,+ firebillRetryTotal,+ firebillTrackTotal,+} from "./metrics";/*** Outcome of a firebill `/v1/lock` call, mirroring firebill's three-state@@ -158,8 +162,7 @@ export async function firebillTrack(params: TrackParams): Promise<boolean> {}type AttemptResult =- | { ok: true }- | { ok: false; reason: "not_ok" | "not_success" | "exception" };+ { ok: true } | { ok: false; reason: "not_ok" | "not_success" | "exception" };async function firebillAttempt(path: string,@@ -257,6 +260,137 @@ async function firebillAttempt(* lock's lifecycle across two routes would let a firebill-side hold and a* fallback hold coexist under retries.*/+/**+ * Three outcomes, and callers must tell them apart: `answered` carries a real+ * yes/no, `unavailable` means firebill could not answer at all.+ */+export type FirebillCheckResult =+ | { status: "answered"; allowed: boolean; remaining: number }+ | { status: "unavailable" };++/**+ * Asks firebill whether a customer can afford a charge.+ *+ * For a gateway-funded org this counts the funder's pool, which is the reason+ * it exists: a ghost spends credits it does not have, so a gate reading the+ * ghost's balance alone refuses the requests the partner pool is there to pay+ * for. firebill answers with the same arithmetic settlement uses.+ *+ * **Fails open, unlike {@link firebillTrack}.** Every failure maps to+ * `unavailable`, and the caller's contract is to let the request through.+ * Declining to charge loses nothing that cannot be replayed; declining to+ * *answer* an authorization question would turn a firebill blip into a+ * customer-facing outage. There is deliberately no retry either β this sits on+ * the request path, and a second round trip buys less than failing open fast.+ */+export async function firebillCheck({+ customerId,+ entityId,+ featureId,+ value,+ properties,+}: {+ customerId: string;+ entityId: string;+ featureId: string;+ value: number;+ properties?: Record<string, unknown>;+}): Promise<FirebillCheckResult> {+ const unavailable = (+ reason: string,+ extra?: Record<string, unknown>,+ ): FirebillCheckResult => {+ logger.error(`firebill check unavailable β ${reason}`, {+ customerId,+ entityId,+ featureId,+ value,+ ...extra,+ });+ firebillCheckTotal.labels("unavailable").inc();+ return { status: "unavailable" };+ };++ try {+ const response = await fetch(firebillUrl("/v1/check"), {+ method: "POST",+ headers: {+ authorization: `Bearer ${config.FIREBILL_SECRET}`,+ "content-type": "application/json",+ },+ body: JSON.stringify({+ customer_id: customerId,+ entity_id: entityId,+ feature_id: featureId,+ value,+ properties,+ }),+ signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),+ });++ if (!response.ok) {+ // Release the socket. Returning without reading or cancelling leaves the+ // body unconsumed, and undici keeps the connection pinned until it is β+ // so a firebill that is erroring would exhaust the pool and turn one+ // failure into a run of them. Matters most here: this path is the one+ // that fails open, so the leak would be silently widening the window in+ // which credit checks are skipped.+ response.body?.cancel().catch(() => {});+ return unavailable("non-OK response", { status: response.status });+ }++ const body = (await response.json()) as {+ success?: boolean;+ allowed?: boolean;+ remaining?: number;+ };++ // `success: false` is firebill saying it does not know β an unanswered+ // balance, or a gateway lookup that failed. Never a denial.+ if (body.success !== true) {+ return unavailable("firebill could not answer");+ }+ // A missing or mis-shaped `allowed` is not something firebill sends today.+ // Reading it as a denial would 402 a paying customer, so it fails open.+ if (typeof body.allowed !== "boolean") {+ return unavailable("answered without a usable `allowed`");+ }++ // `remaining` clamps downstream limits, and the safe default inverts with+ // `allowed`, so there is no single one.+ //+ // `checkCreditsMiddleware` treats a denial with `remaining > 0` as a+ // *partial* crawl: it rewrites `limit` to that figure and calls `next()`+ // rather than returning 402. So defaulting a denial to `Infinity` would+ // turn "cannot afford this" into an unbounded crawl β the opposite of a+ // refusal, and worse than the 402 this endpoint exists to avoid.+ //+ // Allowed keeps `Infinity`, for the mirror-image reason: zero there would+ // silently shrink a crawl the customer *can* pay for down to nothing.+ // A usable figure is always preferred to either default.+ const remaining =+ typeof body.remaining === "number" && Number.isFinite(body.remaining)+ ? body.remaining+ : body.allowed+ ? Infinity+ : 0;++ firebillCheckTotal.labels(body.allowed ? "allowed" : "denied").inc();+ if (!body.allowed) {+ logger.info("firebill check denied", {+ customerId,+ entityId,+ featureId,+ value,+ remaining,+ });+ }+ return { status: "answered", allowed: body.allowed, remaining };+ } catch (error) {+ return unavailable("request threw", { error });+ }+}+export async function firebillLock({customerId,entityId,diff --git a/apps/api/src/services/autumn/metrics.ts b/apps/api/src/services/autumn/metrics.tsindex abff7e25a..16ecf49c7 100644--- a/apps/api/src/services/autumn/metrics.ts+++ b/apps/api/src/services/autumn/metrics.ts@@ -32,3 +32,19 @@ export const firebillRetryTotal = new Counter({help: "Retried firebill calls, by why the previous attempt failed",labelNames: ["reason"] as const, // not_ok | not_success | exception});++/**+ * What firebill's credit check answered. `denied` is a real "cannot afford it"+ * and becomes a 402; `unavailable` means firebill could not answer, so the+ * caller **failed open** and the request proceeded unauthorized.+ *+ * Watch `unavailable`: unlike a refused charge it is invisible to the customer+ * and to the balance, so a firebill wobble here shows up nowhere else. And for+ * a gateway org `denied` means a partner's pool ran dry, which is a commercial+ * event rather than a fault.+ */+export const firebillCheckTotal = new Counter({+ name: "firecrawl_firebill_check_total",+ help: "Outcomes of credit checks sent to firebill",+ labelNames: ["outcome"] as const, // allowed | denied | unavailable+});