Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Let the partner's gate decide whether a monitor run happens (#4428)

Rakshith Ramprakasha75e0e80694b
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index 7ed04b804..79a779c18 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -378,6 +378,7 @@ export const monitor_checks = pgTable("monitor_checks", {
reserved_credits: integer("reserved_credits"),
actual_credits: integer("actual_credits"),
autumn_lock_id: text("autumn_lock_id"),
+ partner_run_token: text("partner_run_token"),
billing_status: text("billing_status").notNull().default("not_applicable"),
total_pages: integer("total_pages").notNull().default(0),
same_count: integer("same_count").notNull().default(0),
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index a35d1378a..9a74bfc54 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -1209,6 +1209,404 @@ describe("firebill routing", () => {
expect(mockCheck).not.toHaveBeenCalled();
});
+ // -------------------------------------------------------------------------
+ // The partner's credit gate (firebill asks; we only carry the tokens)
+ // -------------------------------------------------------------------------
+
+ it("sends a partner job token so firebill can arm the gate, and brings the run token back", async () => {
+ state.configRef = firebillConfig();
+ mockFetch.mockResolvedValue(
+ lockResponse({
+ success: true,
+ allowed: true,
+ lock_id: "monitor_check-1",
+ operation_token: "run-42",
+ }),
+ );
+ const svc = makeService();
+
+ const result = await svc.lockCredits({
+ teamId: "team-1",
+ value: 10,
+ lockId: "monitor_check-1",
+ partnerJobToken: "job-token-abc",
+ });
+
+ expect(result).toEqual({
+ status: "locked",
+ lockId: "monitor_check-1",
+ operationToken: "run-42",
+ });
+ const lockCall = mockFetch.mock.calls.find(([url]) =>
+ String(url).endsWith("/v1/lock"),
+ );
+ expect(JSON.parse(lockCall![1].body).partner_job_token).toBe(
+ "job-token-abc",
+ );
+ });
+
+ // Absent, not null: firebill arms the gate on presence.
+ it("omits the token entirely for a monitor no partner created", async () => {
+ state.configRef = firebillConfig();
+ mockFetch.mockResolvedValue(
+ lockResponse({ success: true, allowed: true, lock_id: "lock-1" }),
+ );
+ const svc = makeService();
+
+ for (const partnerJobToken of [undefined, null]) {
+ await svc.lockCredits({
+ teamId: "team-1",
+ value: 10,
+ lockId: "lock-1",
+ partnerJobToken,
+ });
+ const lockCall = mockFetch.mock.calls
+ .filter(([url]) => String(url).endsWith("/v1/lock"))
+ .pop();
+ expect(JSON.parse(lockCall![1].body)).not.toHaveProperty(
+ "partner_job_token",
+ );
+ }
+ });
+
+ // Collapsing these would strand a revoked job or cancel a monitor over a
+ // customer topping up late.
+ // A gated lock waits longer because it does more: firebill asks the partner
+ // before taking the hold. Giving up at 5s would abandon a call that still
+ // takes the hold - run skipped here, balance reserved there.
+ it("waits longer for a gated lock than an ordinary one", async () => {
+ state.configRef = firebillConfig();
+ mockFetch.mockResolvedValue(
+ lockResponse({ success: true, allowed: true, lock_id: "lock-1" }),
+ );
+ const svc = makeService();
+
+ // Capture the deadline each call asks for; two AbortSignals always differ
+ // as objects, so only the number proves anything.
+ const timeouts: number[] = [];
+ const realTimeout = AbortSignal.timeout.bind(AbortSignal);
+ const spy = vi
+ .spyOn(AbortSignal, "timeout")
+ .mockImplementation((ms: number) => {
+ timeouts.push(ms);
+ return realTimeout(ms);
+ });
+
+ await svc.lockCredits({ teamId: "team-1", value: 1, lockId: "lock-1" });
+ const plain = mockFetch.mock.calls
+ .filter(([url]) => String(url).endsWith("/v1/lock"))
+ .pop();
+
+ await svc.lockCredits({
+ teamId: "team-1",
+ value: 1,
+ lockId: "lock-1",
+ partnerJobToken: "job-token-abc",
+ });
+ const gated = mockFetch.mock.calls
+ .filter(([url]) => String(url).endsWith("/v1/lock"))
+ .pop();
+
+ expect(timeouts.at(-1)).toBe(10000);
+ expect(timeouts.at(-2)).toBe(5000);
+ expect(gated![1].signal).toBeDefined();
+ expect(plain![1].signal).toBeDefined();
+ spy.mockRestore();
+ });
+
+ it("carries the partner's denial reason through, and only ones it knows", async () => {
+ state.configRef = firebillConfig();
+ const svc = makeService();
+
+ for (const reason of [
+ "out_of_credits",
+ "job_revoked",
+ "gate_unavailable",
+ ]) {
+ mockFetch.mockResolvedValueOnce(
+ lockResponse({ success: true, allowed: false, reason }),
+ );
+ expect(
+ await svc.lockCredits({
+ teamId: "team-1",
+ value: 10,
+ lockId: "lock-1",
+ partnerJobToken: "job-token-abc",
+ }),
+ ).toEqual({ status: "denied", reason });
+ }
+
+ // Reading an unknown reason as `job_revoked` would pause a monitor over a
+ // string; a plain denial is the safe reading.
+ mockFetch.mockResolvedValueOnce(
+ lockResponse({ success: true, allowed: false, reason: "who-knows" }),
+ );
+ expect(
+ await svc.lockCredits({
+ teamId: "team-1",
+ value: 10,
+ lockId: "lock-1",
+ partnerJobToken: "job-token-abc",
+ }),
+ ).toEqual({ status: "denied" });
+ });
+
+ // Unreachable (#4403), but a token here would silently skip the gate.
+ it("refuses a hold when a job token reaches the direct-Autumn path", async () => {
+ state.configRef = {
+ ...firebillConfig(),
+ FIREBILL_ORG_IDS: ["some-other-org"],
+ };
+ const svc = makeService();
+
+ expect(
+ await svc.lockCredits({
+ teamId: "team-1",
+ value: 10,
+ lockId: "lock-1",
+ partnerJobToken: "job-token-abc",
+ }),
+ ).toEqual({ status: "denied", reason: "gate_unavailable" });
+ expect(mockCheck).not.toHaveBeenCalled();
+ });
+
+ // firebill cannot fail closed on its own behalf when it is what is down.
+ it("denies a gated run when firebill itself cannot answer", async () => {
+ state.configRef = firebillConfig();
+ const svc = makeService();
+ const gated = {
+ teamId: "team-1",
+ value: 10,
+ lockId: "lock-1",
+ partnerJobToken: "job-token-abc",
+ };
+
+ mockFetch.mockResolvedValueOnce(lockResponse({ success: false }));
+ expect(await svc.lockCredits(gated)).toEqual({
+ status: "denied",
+ reason: "gate_unavailable",
+ });
+
+ mockFetch.mockRejectedValueOnce(new Error("ECONNREFUSED"));
+ expect(await svc.lockCredits(gated)).toEqual({
+ status: "denied",
+ reason: "gate_unavailable",
+ });
+
+ // An answer we cannot read is not an answer either.
+ mockFetch.mockResolvedValueOnce(lockResponse({ success: true }));
+ expect(await svc.lockCredits(gated)).toEqual({
+ status: "denied",
+ reason: "gate_unavailable",
+ });
+
+ expect(mockCheck).not.toHaveBeenCalled();
+ });
+
+ // What makes the above affordable: refusing ordinary holds would turn a
+ // firebill blip into a customer-facing outage.
+ it("still skips rather than denies when no partner gate is involved", async () => {
+ state.configRef = firebillConfig();
+ const svc = makeService();
+
+ mockFetch.mockResolvedValueOnce(lockResponse({ success: false }));
+ expect(
+ await svc.lockCredits({ teamId: "team-1", value: 10, lockId: "lock-1" }),
+ ).toEqual({ status: "skipped" });
+ });
+
+ // Routing can flip in the hour between a lock and its finalize; going direct
+ // would drop the partner's only report of the run.
+ it("follows the run token to firebill even when routing says otherwise", async () => {
+ state.configRef = {
+ ...firebillConfig(),
+ FIREBILL_ORG_IDS: ["some-other-org"],
+ };
+ const svc = makeService();
+
+ await svc.finalizeCreditsLock({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ overrideValue: 7,
+ teamId: "team-1",
+ externalRequestId: "run-42",
+ });
+
+ expect(mockFinalize).not.toHaveBeenCalled();
+ const finalizeCall = mockFetch.mock.calls.find(([url]) =>
+ String(url).endsWith("/v1/finalize"),
+ );
+ expect(finalizeCall).toBeDefined();
+ expect(JSON.parse(finalizeCall![1].body).external_request_id).toBe(
+ "run-42",
+ );
+ });
+
+ it("hands the run token back on the finalize as the operation id", async () => {
+ state.configRef = firebillConfig();
+ const svc = makeService();
+
+ await svc.finalizeCreditsLock({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ overrideValue: 7,
+ teamId: "team-1",
+ externalRequestId: "run-42",
+ });
+
+ const finalizeCall = mockFetch.mock.calls.find(([url]) =>
+ String(url).endsWith("/v1/finalize"),
+ );
+ expect(JSON.parse(finalizeCall![1].body).external_request_id).toBe(
+ "run-42",
+ );
+ });
+
+ // firebill has no lock table and the Autumn finalize body carries no
+ // customer, so without this it cannot find the integration to report to.
+ it("sends the customer alongside the run token so the run can be reported", async () => {
+ state.configRef = firebillConfig();
+ const svc = makeService();
+
+ await svc.finalizeCreditsLock({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ overrideValue: 7,
+ teamId: "team-1",
+ externalRequestId: "run-42",
+ heldValue: 12,
+ });
+
+ const body = JSON.parse(
+ mockFetch.mock.calls
+ .filter(([url]) => String(url).endsWith("/v1/finalize"))
+ .pop()![1].body,
+ );
+ expect(body.customer_id).toBe("org-1");
+ expect(body.external_request_id).toBe("run-42");
+ // The other two the split needs: which balance, and what was held β€”
+ // Autumn reports a balance net of that hold.
+ expect(body.feature_id).toBe("CREDITS");
+ expect(body.held_value).toBe(12);
+ });
+
+ // There is no durable retry here: billMonitorCheck's caller catches, writes
+ // billing_status "failed", and nothing reads that back. Throwing would
+ // abandon the finalize, so the hold expires and the run goes unbilled as
+ // well as unreported β€” a worse loss than the missing label.
+ it("still finalizes when it cannot name the org, rather than abandoning the settle", async () => {
+ state.configRef = firebillConfig();
+ // No org row for this team, so resolving the customer throws.
+ state.supabaseStubData = { data: null, error: null };
+ const svc = makeService();
+
+ await expect(
+ svc.finalizeCreditsLock({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ overrideValue: 7,
+ teamId: "team-99",
+ externalRequestId: "run-42",
+ }),
+ ).resolves.not.toThrow();
+
+ const body = JSON.parse(
+ mockFetch.mock.calls
+ .filter(([url]) => String(url).endsWith("/v1/finalize"))
+ .pop()![1].body,
+ );
+ expect(body.lock_id).toBe("monitor_check-1");
+ // No org, so firebill cannot split or report β€” and counts the omission.
+ expect(body).not.toHaveProperty("customer_id");
+ expect(body).not.toHaveProperty("feature_id");
+ expect(body).not.toHaveProperty("held_value");
+ });
+
+ // An ordinary settle reports to nobody, so it should not pay for the lookup
+ // or put a customer on the wire.
+ // firebill answers `false` for a refusal, a timeout, or a non-OK, and none of
+ // them throw. A caller that cannot see that records a run as billed that
+ // nobody billed, with no retry β€” so the verdict has to come back.
+ it("reports whether the settle actually landed", async () => {
+ state.configRef = firebillConfig();
+ const svc = makeService();
+
+ mockFetch.mockResolvedValueOnce(
+ new Response(JSON.stringify({ success: true }), { status: 200 }),
+ );
+ await expect(
+ svc.finalizeCreditsLock({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ overrideValue: 7,
+ teamId: "team-1",
+ }),
+ ).resolves.toBe(true);
+
+ // firebill took it but says it did not land.
+ mockFetch.mockResolvedValueOnce(
+ new Response(JSON.stringify({ success: false }), { status: 200 }),
+ );
+ await expect(
+ svc.finalizeCreditsLock({
+ lockId: "monitor_check-2",
+ action: "confirm",
+ overrideValue: 7,
+ teamId: "team-1",
+ }),
+ ).resolves.toBe(false);
+
+ // Never answered at all.
+ mockFetch.mockRejectedValueOnce(new Error("ECONNREFUSED"));
+ await expect(
+ svc.finalizeCreditsLock({
+ lockId: "monitor_check-3",
+ action: "confirm",
+ overrideValue: 7,
+ teamId: "team-1",
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it("omits the customer when the settle is not gated", async () => {
+ state.configRef = firebillConfig();
+ const svc = makeService();
+
+ await svc.finalizeCreditsLock({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ overrideValue: 7,
+ teamId: "team-1",
+ });
+
+ const body = JSON.parse(
+ mockFetch.mock.calls
+ .filter(([url]) => String(url).endsWith("/v1/finalize"))
+ .pop()![1].body,
+ );
+ expect(body).not.toHaveProperty("customer_id");
+ expect(body).not.toHaveProperty("feature_id");
+ });
+
+ it("omits the operation id when there is no partner to report to", async () => {
+ state.configRef = firebillConfig();
+ const svc = makeService();
+
+ await svc.finalizeCreditsLock({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ teamId: "team-1",
+ externalRequestId: null,
+ });
+
+ const finalizeCall = mockFetch.mock.calls.find(([url]) =>
+ String(url).endsWith("/v1/finalize"),
+ );
+ expect(JSON.parse(finalizeCall![1].body)).not.toHaveProperty(
+ "external_request_id",
+ );
+ });
+
it("uses the direct Autumn lock path for orgs NOT on the allowlist", async () => {
state.configRef = {
...firebillConfig(),
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index 3b51268ce..cfd6e3700 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -10,6 +10,7 @@ import {
firebillLock,
firebillTrack,
firebillCheck,
+ firebillConfigured,
shouldRouteToFirebill,
} from "./firebill";
import { billingRouteTotal } from "./metrics";
@@ -546,12 +547,23 @@ export class AutumnService {
expiresAt,
properties,
featureId = CREDITS_FEATURE_ID,
+ partnerJobToken,
}: LockCreditsParams): Promise<LockCreditsResult> {
if (!autumnClient || this.isPreviewTeam(teamId)) {
return { status: "skipped" };
}
const resolvedLockId = lockId ?? `billing_${randomUUID()}`;
+ // An ordinary hold proceeds unlocked; refusing would turn a firebill blip
+ // into a customer outage. A gated run is the opposite: no answer means no
+ // run token, so the work could never be billed. firebill fails closed when
+ // it cannot reach the partner, but it cannot do so when it is the thing
+ // that is down.
+ const unreachable = (): LockCreditsResult =>
+ partnerJobToken
+ ? { status: "denied", reason: "gate_unavailable" }
+ : { status: "skipped" };
+
try {
const customerId = await this.ensureTrackingContext(teamId);
@@ -559,7 +571,8 @@ export class AutumnService {
// their holds through firebill. The hold still lives in Autumn (firebill
// keeps no lock state), but only firebill pins the retry/timeout budget
// around the call. An unavailable answer maps to "skipped" β€” proceed
- // unlocked β€” exactly like a direct-Autumn check failure below.
+ // unlocked β€” like a direct-Autumn check failure below, except when a
+ // partner gate is involved; see `unreachable` above.
if (
shouldRouteToFirebill(customerId, {
gatewayProvisioned: await this.isGatewayProvisioned(teamId),
@@ -576,14 +589,34 @@ export class AutumnService {
// monitor runner's convention, when the caller sets none.
expiresAt: expiresAt ?? Date.now() + 60 * 60 * 1000,
properties,
+ partnerJobToken,
});
if (result.status === "locked") {
- return { status: "locked", lockId: result.lockId };
+ return {
+ status: "locked",
+ lockId: result.lockId,
+ ...(result.operationToken
+ ? { operationToken: result.operationToken }
+ : {}),
+ };
}
if (result.status === "denied") {
- return { status: "denied" };
+ return {
+ status: "denied",
+ ...(result.reason ? { reason: result.reason } : {}),
+ };
}
- return { status: "skipped" };
+ return unreachable();
+ }
+
+ // Unreachable β€” a partner-provisioned org always routes to firebill
+ // (#4403) β€” but a token here would silently skip the gate.
+ if (partnerJobToken) {
+ logger.error(
+ "A partner job token reached the direct-Autumn lock path, where no partner can be asked; refusing the hold",
+ { teamId, lockId: resolvedLockId },
+ );
+ return { status: "denied", reason: "gate_unavailable" };
}
const { allowed } = await autumnClient.check({
@@ -627,7 +660,8 @@ export class AutumnService {
error,
},
);
- return { status: "skipped" };
+ // A gated run that threw before asking anyone is still unauthorized.
+ return unreachable();
}
}
@@ -639,6 +673,11 @@ export class AutumnService {
* durably and retries delivery β€” a dropped direct finalize means the hold
* just expires, leaving a confirm's work unbilled. Either route lands on the
* same Autumn lock, so routing is a durability choice, not a correctness one.
+ *
+ * **Except with a run token in hand**, where it is a correctness choice: only
+ * firebill reports the operation to the partner, and the routing predicate is
+ * not stable across the hour between a lock and its finalize. The token is
+ * proof of the route the lock took, so it wins over asking again.
*/
async finalizeCreditsLock({
lockId,
@@ -646,13 +685,51 @@ export class AutumnService {
overrideValue,
properties,
teamId,
- }: FinalizeCreditsLockParams): Promise<void> {
- if (teamId && (await this.isRoutedThroughFirebill(teamId))) {
- await firebillFinalize({ lockId, action, overrideValue, properties });
- return;
+ externalRequestId,
+ featureId = CREDITS_FEATURE_ID,
+ heldValue,
+ }: FinalizeCreditsLockParams): Promise<boolean> {
+ const gated = Boolean(externalRequestId) && firebillConfigured();
+ if (gated || (teamId && (await this.isRoutedThroughFirebill(teamId)))) {
+ // Only resolved for a gated settle: firebill needs the org to split the
+ // settle and to find the integration to report to. An ordinary finalize
+ // does neither, so it does not pay for the lookup.
+ //
+ // A failure degrades to omitting it rather than throwing. There is no
+ // durable retry on this path β€” `billMonitorCheck`'s only caller catches,
+ // writes `billing_status: "failed"`, and moves on, and nothing ever reads
+ // that back β€” so throwing would abandon the finalize entirely: the hold
+ // expires and the run goes unbilled at Autumn as well as unreported.
+ // Settling and losing the label is the lesser loss, and firebill counts
+ // the lost label as `partner_events_total{outcome="no_customer"}`.
+ const customerId =
+ externalRequestId && teamId
+ ? await this.ensureTrackingContext(teamId).catch(error => {
+ logger.error(
+ "Could not resolve the org for a gated settle; finalizing anyway, but this run cannot be reported to its partner",
+ { teamId, lockId, error },
+ );
+ return null;
+ })
+ : null;
+ // Surfaced, not discarded. firebill answers `false` for a refusal, a
+ // timeout, or a non-OK β€” none of which throw β€” so a caller that ignores
+ // this records a run as billed that nobody billed.
+ return await firebillFinalize({
+ lockId,
+ action,
+ overrideValue,
+ properties,
+ externalRequestId,
+ customerId,
+ featureId: customerId ? featureId : null,
+ heldValue: customerId ? heldValue : null,
+ });
}
- if (!autumnClient) return;
+ // No client means no hold was ever taken, so nothing can have gone
+ // unsettled.
+ if (!autumnClient) return true;
try {
await autumnClient.balances.finalize({
@@ -666,6 +743,7 @@ export class AutumnService {
action,
overrideValue,
});
+ return true;
} catch (error) {
logger.error(
"Autumn finalizeCreditsLock failed β€” billing API may be unavailable",
@@ -676,6 +754,7 @@ export class AutumnService {
error,
},
);
+ return false;
}
}
diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.ts
index 0f00d6481..2b995aa8e 100644
--- a/apps/api/src/services/autumn/firebill.ts
+++ b/apps/api/src/services/autumn/firebill.ts
@@ -2,7 +2,7 @@ import { randomUUID } from "crypto";
import { config } from "../../config";
import { logger } from "../../lib/logger";
import { sampled } from "../../lib/rollout";
-import type { TrackParams } from "./types";
+import type { LockDeniedReason, TrackParams } from "./types";
import {
firebillCheckTotal,
firebillRetryTotal,
@@ -16,15 +16,35 @@ import {
* Autumn at all and the caller decides how its endpoint fails.
*/
type FirebillLockResult =
- | { status: "locked"; lockId: string }
- | { status: "denied" }
+ | { status: "locked"; lockId: string; operationToken?: string }
+ | { status: "denied"; reason?: LockDeniedReason }
| { status: "unavailable" };
+/**
+ * An unrecognised reason is dropped rather than passed through: reading one we
+ * do not understand as `job_revoked` would stop a customer's schedule.
+ */
+const LOCK_DENIED_REASONS: readonly LockDeniedReason[] = [
+ "out_of_credits",
+ "job_revoked",
+ "gate_unavailable",
+];
+
+function lockDeniedReason(value: unknown): LockDeniedReason | undefined {
+ return LOCK_DENIED_REASONS.find(reason => reason === value);
+}
+
// firebill's own internal budget (durable write + forward attempt) is up to
// ~3.5s worst case, so this is deliberately looser than the 2s timeout on the
// direct Autumn client.
const FIREBILL_TIMEOUT_MS = 5000;
+// A gated lock legitimately does more: firebill asks the partner (2.5s ceiling)
+// before the Autumn hold (2s), on top of the funding lookup. Giving up at 5s
+// would abandon a call that is still going to take the hold - the run marked
+// skipped here, the balance reserved there.
+const FIREBILL_GATED_LOCK_TIMEOUT_MS = 10000;
+
// Safe to retry because the idempotency key is stable across attempts: if the
// first attempt did land (ambiguous confirm timeout), Autumn dedupes the second.
// Small because a caller may be waiting, and a firebill refusing events usually
@@ -50,6 +70,14 @@ function firebillOrgIds(): Set<string> {
return allowlistCache.ids;
}
+/**
+ * Configured, without the rollout question β€” a finalize carrying a run token
+ * follows the lock through firebill whatever the ramp says.
+ */
+export function firebillConfigured(): boolean {
+ return Boolean(config.FIREBILL_URL && config.FIREBILL_SECRET);
+}
+
/**
* Whether this org's usage goes through firebill rather than straight to Autumn.
* Needs firebill configured, then any of: the allowlist, being
@@ -75,7 +103,7 @@ export function shouldRouteToFirebill(
orgId: string,
opts?: { gatewayProvisioned?: boolean },
): boolean {
- if (!config.FIREBILL_URL || !config.FIREBILL_SECRET) return false;
+ if (!firebillConfigured()) return false;
// Always-on set: test orgs stay routed even at 0 percent.
if (firebillOrgIds().has(orgId)) return true;
// A partner-provisioned org always routes: firebill is the only thing that
@@ -400,6 +428,7 @@ export async function firebillLock({
lockId,
expiresAt,
properties,
+ partnerJobToken,
}: {
customerId: string;
entityId: string;
@@ -408,6 +437,7 @@ export async function firebillLock({
lockId: string;
expiresAt: number;
properties?: Record<string, unknown>;
+ partnerJobToken?: string | null;
}): Promise<FirebillLockResult> {
const url = firebillUrl("/v1/lock");
try {
@@ -428,8 +458,12 @@ export async function firebillLock({
lock_id: lockId,
expires_at: expiresAt,
properties,
+ // Present arms firebill's partner gate; omitted is today's path.
+ ...(partnerJobToken ? { partner_job_token: partnerJobToken } : {}),
}),
- signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),
+ signal: AbortSignal.timeout(
+ partnerJobToken ? FIREBILL_GATED_LOCK_TIMEOUT_MS : FIREBILL_TIMEOUT_MS,
+ ),
});
if (!response.ok) {
@@ -448,6 +482,8 @@ export async function firebillLock({
success?: boolean;
allowed?: boolean;
lock_id?: string;
+ reason?: unknown;
+ operation_token?: unknown;
};
if (body.success !== true) {
@@ -462,14 +498,16 @@ export async function firebillLock({
}
if (body.allowed === false) {
+ const reason = lockDeniedReason(body.reason);
logger.info("firebill lock denied", {
customerId,
entityId,
featureId,
value,
lockId,
+ reason,
});
- return { status: "denied" };
+ return { status: "denied", ...(reason ? { reason } : {}) };
}
// Only an explicit `allowed: false` is a denial. `success: true` with a
@@ -487,14 +525,25 @@ export async function firebillLock({
return { status: "unavailable" };
}
+ const operationToken =
+ typeof body.operation_token === "string" &&
+ body.operation_token.length > 0
+ ? body.operation_token
+ : undefined;
+
logger.info("firebill lock succeeded", {
customerId,
entityId,
featureId,
value,
lockId,
+ gated: operationToken !== undefined,
});
- return { status: "locked", lockId: body.lock_id ?? lockId };
+ return {
+ status: "locked",
+ lockId: body.lock_id ?? lockId,
+ ...(operationToken ? { operationToken } : {}),
+ };
} catch (error) {
logger.error("firebill lock failed β€” firebill may be unavailable", {
customerId,
@@ -525,11 +574,19 @@ export async function firebillFinalize({
action,
overrideValue,
properties,
+ externalRequestId,
+ customerId,
+ featureId,
+ heldValue,
}: {
lockId: string;
action: "confirm" | "release";
overrideValue?: number;
properties?: Record<string, unknown>;
+ externalRequestId?: string | null;
+ customerId?: string | null;
+ featureId?: string | null;
+ heldValue?: number | null;
}): Promise<boolean> {
const url = firebillUrl("/v1/finalize");
try {
@@ -551,6 +608,24 @@ export async function firebillFinalize({
// settle β€” e.g. the reconciler re-running a check it raced β€” dedupes
// upstream instead of settling twice.
idempotency_key: `fc:finalize:${action}:${lockId}`,
+ // The run token, brought home as this settle's operation id.
+ ...(externalRequestId
+ ? { external_request_id: externalRequestId }
+ : {}),
+ // Who the lock was for. The Autumn finalize body carries no customer
+ // and firebill keeps no lock table, so without this it cannot find the
+ // integration to report the run to.
+ ...(customerId ? { customer_id: customerId } : {}),
+ // With customer_id, this is what lets firebill split the settle across
+ // the ghost and its funder β€” it reads what the ghost has left of this
+ // feature. Without it the whole cost falls on the ghost.
+ ...(featureId ? { feature_id: featureId } : {}),
+ // What the lock reserved. Autumn reports a balance net of outstanding
+ // holds, so firebill needs this to know what the ghost can actually
+ // pay for this run; without it the whole cost falls on the ghost.
+ ...(heldValue !== undefined && heldValue !== null
+ ? { held_value: heldValue }
+ : {}),
}),
signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),
});
diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.ts
index 18859d1bb..bfe84aa82 100644
--- a/apps/api/src/services/autumn/types.ts
+++ b/apps/api/src/services/autumn/types.ts
@@ -52,19 +52,31 @@ export type LockCreditsParams = {
expiresAt?: number;
properties?: Record<string, unknown>;
featureId?: string;
+ /** Arms firebill's partner credit gate, which is asked before Autumn holds anything. */
+ partnerJobToken?: string | null;
};
+/**
+ * Why a partner's gate refused, as opposed to Autumn. `job_revoked` is the only
+ * one that never resolves on its own, and so the only one a caller may answer
+ * by stopping a schedule.
+ */
+export type LockDeniedReason =
+ | "out_of_credits"
+ | "job_revoked"
+ | "gate_unavailable";
+
/**
* Outcome of an Autumn credit lock attempt.
*
- * - `denied`: Autumn refused (`allowed: false`); the caller must NOT proceed.
- * - `skipped`: billing not in effect (no client, preview team, or API fallback);
- * the caller should proceed without a lock.
- * - `locked`: reserved; `lockId` must be finalized later.
+ * - `denied`: Autumn refused, or a partner's gate did β€” see `reason`.
+ * - `skipped`: billing not in effect; proceed without a lock.
+ * - `locked`: reserved; finalize with `lockId`. `operationToken` is the
+ * partner's id for this occurrence β€” hand it back on the finalize.
*/
export type LockCreditsResult =
- | { status: "locked"; lockId: string }
- | { status: "denied" }
+ | { status: "locked"; lockId: string; operationToken?: string }
+ | { status: "denied"; reason?: LockDeniedReason }
| { status: "skipped" };
export type FinalizeCreditsLockParams = {
@@ -80,6 +92,13 @@ export type FinalizeCreditsLockParams = {
* loses firebill's durable retry).
*/
teamId?: string;
+ /** For a gated run, the `operationToken` its lock handed back. */
+ externalRequestId?: string | null;
+ /** Which balance the hold was against; lets firebill split the settle. */
+ featureId?: string;
+ /** What the lock reserved. Autumn nets outstanding holds out of a reported
+ * balance, so firebill adds this back to see what the ghost can really pay. */
+ heldValue?: number | null;
};
export type TrackCreditsParams = {
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index ca4c271b2..3b6c5b94e 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -36,6 +36,7 @@ import { sendMonitoringSlackSummary } from "../notification/monitoring_slack";
import {
bulkUpsertMonitorPages,
calculateMonitorCheckActualCredits,
+ countRecentConsecutiveSkippedForCredits,
getMonitorCheck,
getMonitorForUpdate,
countMonitorCheckPages,
@@ -45,6 +46,7 @@ import {
listMonitorCheckPages,
listRunningMonitorChecks,
markMonitorRunning,
+ pauseMonitor,
updateMonitorCheck,
updateMonitorCheckIfRunning,
updateMonitorScheduleAfterRun,
@@ -83,6 +85,16 @@ const MONITOR_NOTIFY_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;
const MONITOR_CHECK_PAGE_SCAN_LIMIT = 100_000;
const MONITOR_CHECK_NO_CREDITS_ERROR =
"Monitor check skipped: insufficient credits.";
+const MONITOR_CHECK_REVOKED_ERROR =
+ "Monitor check skipped: the partner has revoked this job.";
+
+/**
+ * Consecutive credit-skipped checks, this one included, before a `job_revoked`
+ * denial stops the schedule. A 410 is permanent, but pausing on one response
+ * would let a partner's bad build take down every monitor they fund. Never
+ * reached by a 402: the current denial must itself be `job_revoked`.
+ */
+const MONITOR_GATE_REVOKED_STREAK = 3;
const TERMINAL_CHECK_STATUSES = new Set([
"completed",
"partial",
@@ -342,14 +354,18 @@ function summarize(pages: PageResult[]) {
};
}
+/// `false` means the settle did not land: the hold is still out there and this
+/// run is not billed. Only the caller can decide what to record for that, so it
+/// is returned rather than swallowed.
async function billMonitorCheck(params: {
monitor: MonitorRow;
check: MonitorCheckRow;
actualCredits: number;
lockId: string | null;
-}): Promise<void> {
+}): Promise<boolean> {
+ let settled = true;
if (params.lockId) {
- await autumnService.finalizeCreditsLock({
+ settled = await autumnService.finalizeCreditsLock({
lockId: params.lockId,
action: "confirm",
overrideValue: params.actualCredits,
@@ -359,10 +375,35 @@ async function billMonitorCheck(params: {
jobId: params.check.id,
},
teamId: params.monitor.team_id,
+ // Confirm only: a release bills nothing, so there is nothing to report.
+ externalRequestId: params.check.partner_run_token,
+ // What the lock reserved. firebill adds it back to the ghost's reported
+ // balance, which Autumn returns net of this very hold.
+ heldValue: params.check.reserved_credits,
});
}
- if (params.actualCredits <= 0 || !config.USE_DB_AUTHENTICATION) return;
+ if (params.actualCredits <= 0 || !config.USE_DB_AUTHENTICATION)
+ return settled;
+
+ // The settle did not land, so Autumn has nothing and the hold expires by
+ // itself. Debiting the team's ledger anyway charges them for a run Autumn
+ // never billed, leaving the two ledgers disagreeing while the row already
+ // says `failed`. `autumnTrackInRequest` below would be a lie too: it tells
+ // the batch Autumn already has this, and it does not.
+ //
+ // So nothing is charged. Firecrawl absorbs the run β€” the direction this path
+ // errs everywhere else β€” and `billing_status: "failed"` plus the error beside
+ // it are how it is found.
+ if (!settled) {
+ logger.error("Not billing a monitor check whose settle did not land", {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ lockId: params.lockId,
+ actualCredits: params.actualCredits,
+ });
+ return false;
+ }
await getBillingQueue().add(
"bill_team",
@@ -384,6 +425,8 @@ async function billMonitorCheck(params: {
priority: 10,
},
);
+
+ return settled;
}
async function sendNotifications(params: {
@@ -944,23 +987,58 @@ export async function processMonitorCheckJob(
endpoint: "monitor",
jobId: check.id,
},
+ // Arms firebill's partner gate; NULL is today's lock. Withheld once this
+ // check holds a run token: a redelivery is the same occurrence, and
+ // asking twice would orphan the first token.
+ partnerJobToken: check.partner_run_token
+ ? null
+ : monitor.partner_job_token,
});
if (lock.status === "denied") {
+ const revoked = lock.reason === "job_revoked";
check = await updateMonitorCheck(check.id, {
status: "skipped_no_credits",
finished_at: new Date().toISOString(),
actual_credits: 0,
billing_status: "not_applicable",
- error: MONITOR_CHECK_NO_CREDITS_ERROR,
+ error: revoked
+ ? MONITOR_CHECK_REVOKED_ERROR
+ : MONITOR_CHECK_NO_CREDITS_ERROR,
});
- await updateMonitorScheduleAfterRun({ monitor, check });
+ // A revoked job never becomes unrevoked; see MONITOR_GATE_REVOKED_STREAK
+ // for why it is waited out rather than acted on at once.
+ const paused =
+ revoked &&
+ (await countRecentConsecutiveSkippedForCredits({
+ teamId: monitor.team_id,
+ monitorId: monitor.id,
+ limit: MONITOR_GATE_REVOKED_STREAK,
+ })) >= MONITOR_GATE_REVOKED_STREAK;
+
+ if (paused) {
+ await pauseMonitor(monitor.id);
+ logger.warn("Paused monitor: the partner has revoked this job", {
+ monitorId: monitor.id,
+ checkId: check.id,
+ teamId: monitor.team_id,
+ consecutiveSkips: MONITOR_GATE_REVOKED_STREAK,
+ });
+ }
+
+ // Reads status off the object it is given, not the row β€” a paused
+ // monitor must not be handed a next_run_at.
+ await updateMonitorScheduleAfterRun({
+ monitor: paused ? { ...monitor, status: "paused" } : monitor,
+ check,
+ });
- logger.info("Skipped monitor check: insufficient credits", {
+ logger.info("Skipped monitor check: no credit authority allowed it", {
monitorId: monitor.id,
checkId: check.id,
teamId: monitor.team_id,
+ reason: lock.reason,
});
return;
}
@@ -969,6 +1047,11 @@ export async function processMonitorCheckJob(
check = await updateMonitorCheck(check.id, {
autumn_lock_id: lockId,
+ // A token already on the row wins: the gate was not re-asked, so there
+ // is no newer one, and null would lose the authorized operation.
+ partner_run_token:
+ check.partner_run_token ??
+ (lock.status === "locked" ? (lock.operationToken ?? null) : null),
reserved_credits: lockId ? (check.estimated_credits ?? 1) : null,
billing_status: lockId ? "reserved" : "not_applicable",
});
@@ -1435,7 +1518,10 @@ export async function reconcileRunningMonitorChecks(
status: errorCount > 0 ? "partial" : "completed",
finished_at: new Date().toISOString(),
actual_credits: actualCredits,
- billing_status: check.autumn_lock_id ? "confirmed" : "not_applicable",
+ // Not `confirmed` yet β€” the settle has not been attempted. Claiming it
+ // here and then finding firebill refused leaves a run permanently
+ // recorded as billed that nobody billed, with no retry.
+ billing_status: check.autumn_lock_id ? "reserved" : "not_applicable",
total_pages: totalPages,
same_count: same,
changed_count: changed,
@@ -1445,8 +1531,9 @@ export async function reconcileRunningMonitorChecks(
target_results: targetResults,
});
+ let settled = false;
try {
- await billMonitorCheck({
+ settled = await billMonitorCheck({
monitor,
check: finalized,
actualCredits,
@@ -1458,10 +1545,27 @@ export async function reconcileRunningMonitorChecks(
checkId: finalized.id,
error,
});
+ }
+
+ // A refusal and a throw are the same fact β€” the settle did not land β€” and
+ // both must be recorded as such. `firebillFinalize` answers `false`
+ // without throwing, so the catch alone never saw them.
+ if (check.autumn_lock_id) {
+ if (!settled) {
+ logger.error(
+ "Monitor check settle did not land; the hold is unsettled and this run is unbilled",
+ {
+ monitorId: monitor.id,
+ checkId: finalized.id,
+ lockId: check.autumn_lock_id,
+ actualCredits,
+ },
+ );
+ }
finalized = await updateMonitorCheck(check.id, {
- billing_status: "failed",
+ billing_status: settled ? "confirmed" : "failed",
}).catch(updateError => {
- logger.warn("Failed to record monitor check billing failure", {
+ logger.warn("Failed to record monitor check billing outcome", {
monitorId: monitor.id,
checkId: finalized.id,
error: updateError,
diff --git a/apps/api/src/services/monitoring/store.gate.test.ts b/apps/api/src/services/monitoring/store.gate.test.ts
new file mode 100644
index 000000000..5a239be24
--- /dev/null
+++ b/apps/api/src/services/monitoring/store.gate.test.ts
@@ -0,0 +1,84 @@
+const { rows, limits, updates, chain } = vi.hoisted(() => {
+ const rows: { status: string }[] = [];
+ const limits: number[] = [];
+ const updates: Record<string, unknown>[] = [];
+
+ const chain: Record<string, unknown> = {
+ from: () => chain,
+ where: () => chain,
+ orderBy: () => chain,
+ limit: (n: number) => {
+ limits.push(n);
+ return Promise.resolve(rows.slice(0, n));
+ },
+ };
+
+ return { rows, limits, updates, chain };
+});
+
+vi.mock("../../db/connection", () => ({
+ dbRr: { select: () => chain },
+ db: {
+ update: () => ({
+ set: (patch: Record<string, unknown>) => {
+ updates.push(patch);
+ return { where: async () => undefined };
+ },
+ }),
+ },
+}));
+vi.mock("../../db/rpc", () => ({ monitoringClaimDueMonitors: vi.fn() }));
+
+import { countRecentConsecutiveSkippedForCredits, pauseMonitor } from "./store";
+
+function seed(...statuses: string[]) {
+ rows.length = 0;
+ limits.length = 0;
+ rows.push(...statuses.map(status => ({ status })));
+}
+
+async function streak(limit = 3) {
+ return countRecentConsecutiveSkippedForCredits({
+ teamId: "11111111-1111-1111-1111-111111111111",
+ monitorId: "22222222-2222-2222-2222-222222222222",
+ limit,
+ });
+}
+
+describe("counting a monitor's credit-skip streak", () => {
+ const skipped = "skipped_no_credits";
+
+ it("counts the run ending at the newest check", async () => {
+ seed(skipped, skipped, skipped);
+ expect(await streak()).toBe(3);
+ });
+
+ // A partner that recovered must not lose the monitor on its next blip.
+ it("stops at the first check that ran", async () => {
+ seed(skipped, "completed", skipped);
+ expect(await streak()).toBe(1);
+ });
+
+ it("is zero when the newest check is not a skip", async () => {
+ seed("completed", skipped, skipped);
+ expect(await streak()).toBe(0);
+ seed();
+ expect(await streak()).toBe(0);
+ });
+
+ it("reads no more rows than the caller's limit", async () => {
+ seed(skipped, skipped, skipped, skipped, skipped);
+ expect(await streak(3)).toBe(3);
+ expect(limits).toEqual([3]);
+ });
+});
+
+describe("pausing a monitor", () => {
+ // Clearing next_run_at is what actually stops the runs being claimed.
+ it("stops the schedule without destroying the monitor", async () => {
+ updates.length = 0;
+ await pauseMonitor("22222222-2222-2222-2222-222222222222");
+
+ expect(updates[0]).toMatchObject({ status: "paused", next_run_at: null });
+ });
+});
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index b68f02eb1..d6ddb13ef 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -802,6 +802,60 @@ export async function listMonitorChecks(params: {
return data as MonitorCheckRow[];
}
+/**
+ * The leading run of credit-skipped checks ending at the newest, so one
+ * successful check resets it. Derived rather than counted: a stored counter
+ * would be a second source of truth to keep correct across retries.
+ */
+export async function countRecentConsecutiveSkippedForCredits(params: {
+ teamId: string;
+ monitorId: string;
+ limit: number;
+}): Promise<number> {
+ const rows = await run(
+ () =>
+ dbRr
+ .select({ status: schema.monitor_checks.status })
+ .from(schema.monitor_checks)
+ .where(
+ and(
+ eq(schema.monitor_checks.monitor_id, params.monitorId),
+ eq(schema.monitor_checks.team_id, params.teamId),
+ ),
+ )
+ .orderBy(desc(schema.monitor_checks.created_at))
+ .limit(params.limit),
+ "Failed to count skipped monitor checks",
+ );
+
+ let streak = 0;
+ for (const row of rows) {
+ if (row.status !== "skipped_no_credits") break;
+ streak += 1;
+ }
+ return streak;
+}
+
+/**
+ * `paused`, not `deleted`: nothing a partner says should destroy a customer's
+ * configuration. `monitoring_claim_due_monitors` only claims `active` rows, so
+ * this is enough to stop the runs.
+ */
+export async function pauseMonitor(monitorId: string): Promise<void> {
+ await run(
+ () =>
+ db
+ .update(schema.monitors)
+ .set({
+ status: "paused",
+ next_run_at: null,
+ updated_at: new Date().toISOString(),
+ })
+ .where(eq(schema.monitors.id, monitorId)),
+ "Failed to pause monitor",
+ );
+}
+
export async function updateMonitorCheck(
checkId: string,
patch: Partial<MonitorCheckRow>,
diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.ts
index 79944d01d..c4574a2a9 100644
--- a/apps/api/src/services/monitoring/types.ts
+++ b/apps/api/src/services/monitoring/types.ts
@@ -163,7 +163,8 @@ const monitorNotificationInner = z
if (notification.slack?.enabled && !notification.slack.channelId) {
ctx.addIssue({
code: "custom",
- message: "A Slack channel is required when Slack notifications are enabled",
+ message:
+ "A Slack channel is required when Slack notifications are enabled",
path: ["slack", "channelId"],
});
}
@@ -353,6 +354,8 @@ export type MonitorCheckRow = {
reserved_credits: number | null;
actual_credits: number | null;
autumn_lock_id: string | null;
+ /** The partner's id for this occurrence; carried back on the finalize. */
+ partner_run_token: string | null;
billing_status:
| "not_applicable"
| "reserved"