Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(billing): route orgless lock finalizes through Firebill (#5479)

Rakshith Ramprakash6b2b053d16d8
diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.ts
index 70ec3a1f1..9949b6803 100644
--- a/apps/api/src/services/alexandria/retrieve.ts
+++ b/apps/api/src/services/alexandria/retrieve.ts
@@ -287,8 +287,8 @@ export async function retrieveProviders(input: {
const finalize = async (credits: number) => {
if (!lockId) return true;
const settled = await autumnService.finalizeCreditsLock({
- // Omitted without an org: the settle goes straight to Autumn, the route
- // an unnameable org already took.
+ // Omitted without an org: the settle goes through firebill when it is
+ // configured, settled by lock id alone.
team:
input.orgId !== null
? { teamId: input.teamId, orgId: input.orgId }
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index 518d7de2a..97a037ede 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -133,6 +133,7 @@ import {
import {
autumnCustomerGetOrCreateTotal,
autumnEntityCreatedInlineTotal,
+ billingRouteTotal,
} from "../metrics";
// ---------------------------------------------------------------------------
@@ -143,6 +144,15 @@ function makeService() {
return new AutumnService();
}
+async function routeCount(route: string, operation: string): Promise<number> {
+ const { values } = await billingRouteTotal.get();
+ return (
+ values.find(
+ v => v.labels.route === route && v.labels.operation === operation,
+ )?.value ?? 0
+ );
+}
+
function makeEntity(usage: number) {
return { balances: { CREDITS: { usage } } };
}
@@ -2106,14 +2116,76 @@ describe("firebill routing", () => {
expect(body.idempotency_key).toBe("fc:finalize:release:monitor_check-1");
});
- it("finalizes directly in Autumn when no teamId is supplied", async () => {
+ // The monitor scheduler and runner and Alexandria pass no team when the org
+ // cannot be named. Those settles used to be the last direct Autumn writes.
+ it("routes an orgless finalize through /v1/finalize, keyed like any other", async () => {
state.configRef = firebillConfig();
const svc = makeService();
+ const before = await routeCount("firebill", "finalize");
+
+ const settled = await svc.finalizeCreditsLock({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ overrideValue: 7,
+ properties: { source: "monitorCheck", endpoint: "monitor" },
+ });
+
+ expect(settled).toBe(true);
+ expect(mockFinalize).not.toHaveBeenCalled();
+ const finalizeCalls = mockFetch.mock.calls.filter(([url]) =>
+ String(url).endsWith("/v1/finalize"),
+ );
+ expect(finalizeCalls).toHaveLength(1);
+ expect(JSON.parse(finalizeCalls[0]![1].body)).toEqual({
+ lock_id: "monitor_check-1",
+ action: "confirm",
+ override_value: 7,
+ properties: { source: "monitorCheck", endpoint: "monitor" },
+ idempotency_key: "fc:finalize:confirm:monitor_check-1",
+ });
+ expect(await routeCount("firebill", "finalize")).toBe(before + 1);
+ });
+
+ it("routes an orgless finalize through firebill even when no org is on the rollout", async () => {
+ state.configRef = { ...firebillConfig(), FIREBILL_ORG_IDS: [] };
+ const svc = makeService();
+
+ await svc.finalizeCreditsLock({ lockId: "lock-1", action: "release" });
+
+ expect(mockFinalize).not.toHaveBeenCalled();
+ const body = JSON.parse(mockFetch.mock.calls[0]![1].body);
+ expect(body.idempotency_key).toBe("fc:finalize:release:lock-1");
+ });
+
+ // Same contract as the direct settle it replaces: one attempt, and `false`
+ // back to the caller when it did not land. No retry is added here.
+ it("answers false for an orgless finalize firebill did not take, after one attempt", async () => {
+ state.configRef = firebillConfig();
+ mockFetch.mockResolvedValue(new Response("nope", { status: 503 }));
+ const svc = makeService();
+
+ const settled = await svc.finalizeCreditsLock({
+ lockId: "lock-1",
+ action: "confirm",
+ overrideValue: 3,
+ });
+
+ expect(settled).toBe(false);
+ expect(mockFetch).toHaveBeenCalledTimes(1);
+ expect(mockFinalize).not.toHaveBeenCalled();
+ expect(retryQueueAdd).not.toHaveBeenCalled();
+ });
+
+ it("finalizes an orgless lock directly in Autumn, and counts it, when firebill is not configured", async () => {
+ state.configRef = {};
+ const svc = makeService();
+ const before = await routeCount("direct", "finalize");
await svc.finalizeCreditsLock({ lockId: "lock-1", action: "release" });
expect(mockFetch).not.toHaveBeenCalled();
expect(mockFinalize).toHaveBeenCalledTimes(1);
+ expect(await routeCount("direct", "finalize")).toBe(before + 1);
});
it("finalizes directly in Autumn when the team's org is NOT on the allowlist", async () => {
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index a6da2816a..f9877eea6 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -360,7 +360,7 @@ export class AutumnService {
// to Autumn on failure β€” firebill may already own the event, and the SDK
// sends no idempotency key, so the pair could not be deduped.
if (routed) {
- billingRouteTotal.labels("firebill").inc();
+ billingRouteTotal.labels("firebill", "track").inc();
return await firebillTrack({
customerId,
entityId,
@@ -373,7 +373,7 @@ export class AutumnService {
});
}
- billingRouteTotal.labels("direct").inc();
+ billingRouteTotal.labels("direct", "track").inc();
if (!autumnClient) return false;
@@ -789,6 +789,13 @@ export class AutumnService {
* just expires, leaving a confirm's work unbilled. Either route lands on the
* same Autumn lock, so routing is a durability choice, not a correctness one.
*
+ * A caller that cannot name the org passes no team, and there is no org to
+ * ask the rollout about. That settle goes through firebill whenever firebill
+ * is configured: the lock id is all a finalize needs, so an ordinary settle
+ * loses nothing by not knowing the org (a run-token settle loses its partner
+ * report, as below), and a direct settle would be the one write to Autumn
+ * left outside firebill.
+ *
* **Except with a run token in hand**, where it is a correctness choice: only
* firebill reports the operation to the partner, and the routing predicate is
* not stable across the hour between a lock and its finalize. The token is
@@ -807,7 +814,9 @@ export class AutumnService {
const gated = Boolean(externalRequestId) && firebillConfigured();
if (
gated ||
- (team && (await this.isRoutedThroughFirebill(team.teamId, team.orgId)))
+ (team
+ ? await this.isRoutedThroughFirebill(team.teamId, team.orgId)
+ : firebillConfigured())
) {
// Named only for a gated settle: firebill needs the org to split the
// settle and to find the integration to report to. An ordinary finalize
@@ -824,6 +833,7 @@ export class AutumnService {
// Settling and losing the label is the lesser loss, and firebill counts
// the lost label as `partner_events_total{outcome="no_customer"}`.
const customerId = externalRequestId && team ? team.orgId : null;
+ billingRouteTotal.labels("firebill", "finalize").inc();
// Surfaced, not discarded. firebill answers `false` for a refusal, a
// timeout, or a non-OK β€” none of which throw β€” so a caller that ignores
// this records a run as billed that nobody billed.
@@ -839,6 +849,8 @@ export class AutumnService {
});
}
+ billingRouteTotal.labels("direct", "finalize").inc();
+
// No client means no hold was ever taken, so nothing can have gone
// unsettled.
if (!autumnClient) return true;
diff --git a/apps/api/src/services/autumn/metrics.ts b/apps/api/src/services/autumn/metrics.ts
index 65ec216f1..678fcda66 100644
--- a/apps/api/src/services/autumn/metrics.ts
+++ b/apps/api/src/services/autumn/metrics.ts
@@ -1,15 +1,18 @@
import { Counter } from "prom-client";
/**
- * Which path a usage event took. The ratio of `firebill` to `direct` is the
- * live read on the rollout percentage β€” during a ramp it is the first thing to
- * look at, because a routing bug shows up here long before it shows up in a
- * balance.
+ * Which path a usage event or a lock settle took. The ratio of `firebill` to
+ * `direct` is the live read on the rollout percentage β€” during a ramp it is the
+ * first thing to look at, because a routing bug shows up here long before it
+ * shows up in a balance. `route="direct"` staying at 0 is the signal that no
+ * usage event or settle goes to Autumn directly any more; direct lock holds
+ * are not counted here.
*/
export const billingRouteTotal = new Counter({
name: "firecrawl_billing_route_total",
- help: "Usage events by the path they took to Autumn",
- labelNames: ["route"] as const, // firebill | direct
+ help: "Usage events and lock settles by the path they took to Autumn",
+ // route: firebill | direct operation: track | finalize
+ labelNames: ["route", "operation"] as const,
});
/**
diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.ts
index 759f3cad6..a2dcc585c 100644
--- a/apps/api/src/services/autumn/types.ts
+++ b/apps/api/src/services/autumn/types.ts
@@ -112,12 +112,12 @@ export type FinalizeCreditsLockParams = {
overrideValue?: number;
properties?: Record<string, unknown>;
/**
- * The team the lock was taken for, and its org. Needed to route the settle
- * through firebill for allowlisted orgs β€” a finalize carries no customer
- * context of its own. When omitted, the settle goes directly to Autumn
- * (which also works for a firebill-taken lock: the hold lives in Autumn
- * either way, but loses firebill's durable retry). A caller that cannot name
- * the org omits this, which is the route an unnameable org already took.
+ * The team the lock was taken for, and its org. Used to ask the rollout
+ * whether this org's settle goes through firebill β€” a finalize carries no
+ * customer context of its own. A caller that cannot name the org omits this,
+ * and the settle goes through firebill whenever firebill is configured: the
+ * lock id is all a settle needs, and the hold lives in Autumn whichever
+ * route took it.
*/
team?: { teamId: string; orgId: string };
/** For a gated run, the `operationToken` its lock handed back. */
diff --git a/apps/api/src/services/monitoring/runner.reservation.test.ts b/apps/api/src/services/monitoring/runner.reservation.test.ts
index 8d399d65f..3aa0cd438 100644
--- a/apps/api/src/services/monitoring/runner.reservation.test.ts
+++ b/apps/api/src/services/monitoring/runner.reservation.test.ts
@@ -44,7 +44,7 @@ vi.mock("../autumn/firebill", () => ({
vi.mock("../autumn/metrics", () => ({
autumnCustomerGetOrCreateTotal: { inc: vi.fn() },
autumnEntityCreatedInlineTotal: { inc: vi.fn() },
- billingRouteTotal: { inc: vi.fn() },
+ billingRouteTotal: { labels: () => ({ inc: vi.fn() }) },
}));
vi.mock("../../db/connection", () => ({
dbRr: {
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 44ae3174d..c79361f32 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -375,8 +375,8 @@ async function billMonitorCheck(params: {
check: MonitorCheckRow;
actualCredits: number;
lockId: string | null;
- /** The team's org. Null means none could be named, which is how the settle
- * already behaved then: straight to Autumn, and unreportable to a partner. */
+ /** The team's org. Null means none could be named: the settle still goes
+ * through firebill when it is configured, but is unreportable to a partner. */
orgId: string | null;
}): Promise<boolean> {
let settled = true;
diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.ts
index fadbbdedf..6560c8b36 100644
--- a/apps/api/src/services/monitoring/scheduler.ts
+++ b/apps/api/src/services/monitoring/scheduler.ts
@@ -212,7 +212,8 @@ async function clearFinishedOrStaleCurrentCheck(
let released = true;
if (failed.autumn_lock_id) {
// The billing service no longer finds the org for itself; without one
- // the release goes straight to Autumn, as it already did then.
+ // the release goes through firebill when it is configured, settled by
+ // lock id alone.
const orgId = await orgIdForTeam(monitor.team_id);
released = await autumnService
.finalizeCreditsLock({