Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(billing): route orgless lock finalizes through Firebill (#5479)
Rakshith Ramprakash6b2b053d16d8
diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex 70ec3a1f1..9949b6803 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -287,8 +287,8 @@ export async function retrieveProviders(input: {const finalize = async (credits: number) => {if (!lockId) return true;const settled = await autumnService.finalizeCreditsLock({- // Omitted without an org: the settle goes straight to Autumn, the route- // an unnameable org already took.+ // Omitted without an org: the settle goes through firebill when it is+ // configured, settled by lock id alone.team:input.orgId !== null? { teamId: input.teamId, orgId: input.orgId }diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.tsindex 518d7de2a..97a037ede 100644--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts@@ -133,6 +133,7 @@ import {import {autumnCustomerGetOrCreateTotal,autumnEntityCreatedInlineTotal,+ billingRouteTotal,} from "../metrics";// ---------------------------------------------------------------------------@@ -143,6 +144,15 @@ function makeService() {return new AutumnService();}+async function routeCount(route: string, operation: string): Promise<number> {+ const { values } = await billingRouteTotal.get();+ return (+ values.find(+ v => v.labels.route === route && v.labels.operation === operation,+ )?.value ?? 0+ );+}+function makeEntity(usage: number) {return { balances: { CREDITS: { usage } } };}@@ -2106,14 +2116,76 @@ describe("firebill routing", () => {expect(body.idempotency_key).toBe("fc:finalize:release:monitor_check-1");});- it("finalizes directly in Autumn when no teamId is supplied", async () => {+ // The monitor scheduler and runner and Alexandria pass no team when the org+ // cannot be named. Those settles used to be the last direct Autumn writes.+ it("routes an orgless finalize through /v1/finalize, keyed like any other", async () => {state.configRef = firebillConfig();const svc = makeService();+ const before = await routeCount("firebill", "finalize");++ const settled = await svc.finalizeCreditsLock({+ lockId: "monitor_check-1",+ action: "confirm",+ overrideValue: 7,+ properties: { source: "monitorCheck", endpoint: "monitor" },+ });++ expect(settled).toBe(true);+ expect(mockFinalize).not.toHaveBeenCalled();+ const finalizeCalls = mockFetch.mock.calls.filter(([url]) =>+ String(url).endsWith("/v1/finalize"),+ );+ expect(finalizeCalls).toHaveLength(1);+ expect(JSON.parse(finalizeCalls[0]![1].body)).toEqual({+ lock_id: "monitor_check-1",+ action: "confirm",+ override_value: 7,+ properties: { source: "monitorCheck", endpoint: "monitor" },+ idempotency_key: "fc:finalize:confirm:monitor_check-1",+ });+ expect(await routeCount("firebill", "finalize")).toBe(before + 1);+ });++ it("routes an orgless finalize through firebill even when no org is on the rollout", async () => {+ state.configRef = { ...firebillConfig(), FIREBILL_ORG_IDS: [] };+ const svc = makeService();++ await svc.finalizeCreditsLock({ lockId: "lock-1", action: "release" });++ expect(mockFinalize).not.toHaveBeenCalled();+ const body = JSON.parse(mockFetch.mock.calls[0]![1].body);+ expect(body.idempotency_key).toBe("fc:finalize:release:lock-1");+ });++ // Same contract as the direct settle it replaces: one attempt, and `false`+ // back to the caller when it did not land. No retry is added here.+ it("answers false for an orgless finalize firebill did not take, after one attempt", async () => {+ state.configRef = firebillConfig();+ mockFetch.mockResolvedValue(new Response("nope", { status: 503 }));+ const svc = makeService();++ const settled = await svc.finalizeCreditsLock({+ lockId: "lock-1",+ action: "confirm",+ overrideValue: 3,+ });++ expect(settled).toBe(false);+ expect(mockFetch).toHaveBeenCalledTimes(1);+ expect(mockFinalize).not.toHaveBeenCalled();+ expect(retryQueueAdd).not.toHaveBeenCalled();+ });++ it("finalizes an orgless lock directly in Autumn, and counts it, when firebill is not configured", async () => {+ state.configRef = {};+ const svc = makeService();+ const before = await routeCount("direct", "finalize");await svc.finalizeCreditsLock({ lockId: "lock-1", action: "release" });expect(mockFetch).not.toHaveBeenCalled();expect(mockFinalize).toHaveBeenCalledTimes(1);+ expect(await routeCount("direct", "finalize")).toBe(before + 1);});it("finalizes directly in Autumn when the team's org is NOT on the allowlist", async () => {diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex a6da2816a..f9877eea6 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -360,7 +360,7 @@ export class AutumnService {// to Autumn on failure β firebill may already own the event, and the SDK// sends no idempotency key, so the pair could not be deduped.if (routed) {- billingRouteTotal.labels("firebill").inc();+ billingRouteTotal.labels("firebill", "track").inc();return await firebillTrack({customerId,entityId,@@ -373,7 +373,7 @@ export class AutumnService {});}- billingRouteTotal.labels("direct").inc();+ billingRouteTotal.labels("direct", "track").inc();if (!autumnClient) return false;@@ -789,6 +789,13 @@ export class AutumnService {* just expires, leaving a confirm's work unbilled. Either route lands on the* same Autumn lock, so routing is a durability choice, not a correctness one.*+ * A caller that cannot name the org passes no team, and there is no org to+ * ask the rollout about. That settle goes through firebill whenever firebill+ * is configured: the lock id is all a finalize needs, so an ordinary settle+ * loses nothing by not knowing the org (a run-token settle loses its partner+ * report, as below), and a direct settle would be the one write to Autumn+ * left outside firebill.+ ** **Except with a run token in hand**, where it is a correctness choice: only* firebill reports the operation to the partner, and the routing predicate is* not stable across the hour between a lock and its finalize. The token is@@ -807,7 +814,9 @@ export class AutumnService {const gated = Boolean(externalRequestId) && firebillConfigured();if (gated ||- (team && (await this.isRoutedThroughFirebill(team.teamId, team.orgId)))+ (team+ ? await this.isRoutedThroughFirebill(team.teamId, team.orgId)+ : firebillConfigured())) {// Named only for a gated settle: firebill needs the org to split the// settle and to find the integration to report to. An ordinary finalize@@ -824,6 +833,7 @@ export class AutumnService {// Settling and losing the label is the lesser loss, and firebill counts// the lost label as `partner_events_total{outcome="no_customer"}`.const customerId = externalRequestId && team ? team.orgId : null;+ billingRouteTotal.labels("firebill", "finalize").inc();// Surfaced, not discarded. firebill answers `false` for a refusal, a// timeout, or a non-OK β none of which throw β so a caller that ignores// this records a run as billed that nobody billed.@@ -839,6 +849,8 @@ export class AutumnService {});}+ billingRouteTotal.labels("direct", "finalize").inc();+// No client means no hold was ever taken, so nothing can have gone// unsettled.if (!autumnClient) return true;diff --git a/apps/api/src/services/autumn/metrics.ts b/apps/api/src/services/autumn/metrics.tsindex 65ec216f1..678fcda66 100644--- a/apps/api/src/services/autumn/metrics.ts+++ b/apps/api/src/services/autumn/metrics.ts@@ -1,15 +1,18 @@import { Counter } from "prom-client";/**- * Which path a usage event took. The ratio of `firebill` to `direct` is the- * live read on the rollout percentage β during a ramp it is the first thing to- * look at, because a routing bug shows up here long before it shows up in a- * balance.+ * Which path a usage event or a lock settle took. The ratio of `firebill` to+ * `direct` is the live read on the rollout percentage β during a ramp it is the+ * first thing to look at, because a routing bug shows up here long before it+ * shows up in a balance. `route="direct"` staying at 0 is the signal that no+ * usage event or settle goes to Autumn directly any more; direct lock holds+ * are not counted here.*/export const billingRouteTotal = new Counter({name: "firecrawl_billing_route_total",- help: "Usage events by the path they took to Autumn",- labelNames: ["route"] as const, // firebill | direct+ help: "Usage events and lock settles by the path they took to Autumn",+ // route: firebill | direct operation: track | finalize+ labelNames: ["route", "operation"] as const,});/**diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.tsindex 759f3cad6..a2dcc585c 100644--- a/apps/api/src/services/autumn/types.ts+++ b/apps/api/src/services/autumn/types.ts@@ -112,12 +112,12 @@ export type FinalizeCreditsLockParams = {overrideValue?: number;properties?: Record<string, unknown>;/**- * The team the lock was taken for, and its org. Needed to route the settle- * through firebill for allowlisted orgs β a finalize carries no customer- * context of its own. When omitted, the settle goes directly to Autumn- * (which also works for a firebill-taken lock: the hold lives in Autumn- * either way, but loses firebill's durable retry). A caller that cannot name- * the org omits this, which is the route an unnameable org already took.+ * The team the lock was taken for, and its org. Used to ask the rollout+ * whether this org's settle goes through firebill β a finalize carries no+ * customer context of its own. A caller that cannot name the org omits this,+ * and the settle goes through firebill whenever firebill is configured: the+ * lock id is all a settle needs, and the hold lives in Autumn whichever+ * route took it.*/team?: { teamId: string; orgId: string };/** For a gated run, the `operationToken` its lock handed back. */diff --git a/apps/api/src/services/monitoring/runner.reservation.test.ts b/apps/api/src/services/monitoring/runner.reservation.test.tsindex 8d399d65f..3aa0cd438 100644--- a/apps/api/src/services/monitoring/runner.reservation.test.ts+++ b/apps/api/src/services/monitoring/runner.reservation.test.ts@@ -44,7 +44,7 @@ vi.mock("../autumn/firebill", () => ({vi.mock("../autumn/metrics", () => ({autumnCustomerGetOrCreateTotal: { inc: vi.fn() },autumnEntityCreatedInlineTotal: { inc: vi.fn() },- billingRouteTotal: { inc: vi.fn() },+ billingRouteTotal: { labels: () => ({ inc: vi.fn() }) },}));vi.mock("../../db/connection", () => ({dbRr: {diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 44ae3174d..c79361f32 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -375,8 +375,8 @@ async function billMonitorCheck(params: {check: MonitorCheckRow;actualCredits: number;lockId: string | null;- /** The team's org. Null means none could be named, which is how the settle- * already behaved then: straight to Autumn, and unreportable to a partner. */+ /** The team's org. Null means none could be named: the settle still goes+ * through firebill when it is configured, but is unreportable to a partner. */orgId: string | null;}): Promise<boolean> {let settled = true;diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.tsindex fadbbdedf..6560c8b36 100644--- a/apps/api/src/services/monitoring/scheduler.ts+++ b/apps/api/src/services/monitoring/scheduler.ts@@ -212,7 +212,8 @@ async function clearFinishedOrStaleCurrentCheck(let released = true;if (failed.autumn_lock_id) {// The billing service no longer finds the org for itself; without one- // the release goes straight to Autumn, as it already did then.+ // the release goes through firebill when it is configured, settled by+ // lock id alone.const orgId = await orgIdForTeam(monitor.team_id);released = await autumnService.finalizeCreditsLock({