Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Let the partner's gate decide whether a monitor run happens (#4428)
Rakshith Ramprakasha75e0e80694b
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 7ed04b804..79a779c18 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -378,6 +378,7 @@ export const monitor_checks = pgTable("monitor_checks", {reserved_credits: integer("reserved_credits"),actual_credits: integer("actual_credits"),autumn_lock_id: text("autumn_lock_id"),+ partner_run_token: text("partner_run_token"),billing_status: text("billing_status").notNull().default("not_applicable"),total_pages: integer("total_pages").notNull().default(0),same_count: integer("same_count").notNull().default(0),diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.tsindex a35d1378a..9a74bfc54 100644--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts@@ -1209,6 +1209,404 @@ describe("firebill routing", () => {expect(mockCheck).not.toHaveBeenCalled();});+ // -------------------------------------------------------------------------+ // The partner's credit gate (firebill asks; we only carry the tokens)+ // -------------------------------------------------------------------------++ it("sends a partner job token so firebill can arm the gate, and brings the run token back", async () => {+ state.configRef = firebillConfig();+ mockFetch.mockResolvedValue(+ lockResponse({+ success: true,+ allowed: true,+ lock_id: "monitor_check-1",+ operation_token: "run-42",+ }),+ );+ const svc = makeService();++ const result = await svc.lockCredits({+ teamId: "team-1",+ value: 10,+ lockId: "monitor_check-1",+ partnerJobToken: "job-token-abc",+ });++ expect(result).toEqual({+ status: "locked",+ lockId: "monitor_check-1",+ operationToken: "run-42",+ });+ const lockCall = mockFetch.mock.calls.find(([url]) =>+ String(url).endsWith("/v1/lock"),+ );+ expect(JSON.parse(lockCall![1].body).partner_job_token).toBe(+ "job-token-abc",+ );+ });++ // Absent, not null: firebill arms the gate on presence.+ it("omits the token entirely for a monitor no partner created", async () => {+ state.configRef = firebillConfig();+ mockFetch.mockResolvedValue(+ lockResponse({ success: true, allowed: true, lock_id: "lock-1" }),+ );+ const svc = makeService();++ for (const partnerJobToken of [undefined, null]) {+ await svc.lockCredits({+ teamId: "team-1",+ value: 10,+ lockId: "lock-1",+ partnerJobToken,+ });+ const lockCall = mockFetch.mock.calls+ .filter(([url]) => String(url).endsWith("/v1/lock"))+ .pop();+ expect(JSON.parse(lockCall![1].body)).not.toHaveProperty(+ "partner_job_token",+ );+ }+ });++ // Collapsing these would strand a revoked job or cancel a monitor over a+ // customer topping up late.+ // A gated lock waits longer because it does more: firebill asks the partner+ // before taking the hold. Giving up at 5s would abandon a call that still+ // takes the hold - run skipped here, balance reserved there.+ it("waits longer for a gated lock than an ordinary one", async () => {+ state.configRef = firebillConfig();+ mockFetch.mockResolvedValue(+ lockResponse({ success: true, allowed: true, lock_id: "lock-1" }),+ );+ const svc = makeService();++ // Capture the deadline each call asks for; two AbortSignals always differ+ // as objects, so only the number proves anything.+ const timeouts: number[] = [];+ const realTimeout = AbortSignal.timeout.bind(AbortSignal);+ const spy = vi+ .spyOn(AbortSignal, "timeout")+ .mockImplementation((ms: number) => {+ timeouts.push(ms);+ return realTimeout(ms);+ });++ await svc.lockCredits({ teamId: "team-1", value: 1, lockId: "lock-1" });+ const plain = mockFetch.mock.calls+ .filter(([url]) => String(url).endsWith("/v1/lock"))+ .pop();++ await svc.lockCredits({+ teamId: "team-1",+ value: 1,+ lockId: "lock-1",+ partnerJobToken: "job-token-abc",+ });+ const gated = mockFetch.mock.calls+ .filter(([url]) => String(url).endsWith("/v1/lock"))+ .pop();++ expect(timeouts.at(-1)).toBe(10000);+ expect(timeouts.at(-2)).toBe(5000);+ expect(gated![1].signal).toBeDefined();+ expect(plain![1].signal).toBeDefined();+ spy.mockRestore();+ });++ it("carries the partner's denial reason through, and only ones it knows", async () => {+ state.configRef = firebillConfig();+ const svc = makeService();++ for (const reason of [+ "out_of_credits",+ "job_revoked",+ "gate_unavailable",+ ]) {+ mockFetch.mockResolvedValueOnce(+ lockResponse({ success: true, allowed: false, reason }),+ );+ expect(+ await svc.lockCredits({+ teamId: "team-1",+ value: 10,+ lockId: "lock-1",+ partnerJobToken: "job-token-abc",+ }),+ ).toEqual({ status: "denied", reason });+ }++ // Reading an unknown reason as `job_revoked` would pause a monitor over a+ // string; a plain denial is the safe reading.+ mockFetch.mockResolvedValueOnce(+ lockResponse({ success: true, allowed: false, reason: "who-knows" }),+ );+ expect(+ await svc.lockCredits({+ teamId: "team-1",+ value: 10,+ lockId: "lock-1",+ partnerJobToken: "job-token-abc",+ }),+ ).toEqual({ status: "denied" });+ });++ // Unreachable (#4403), but a token here would silently skip the gate.+ it("refuses a hold when a job token reaches the direct-Autumn path", async () => {+ state.configRef = {+ ...firebillConfig(),+ FIREBILL_ORG_IDS: ["some-other-org"],+ };+ const svc = makeService();++ expect(+ await svc.lockCredits({+ teamId: "team-1",+ value: 10,+ lockId: "lock-1",+ partnerJobToken: "job-token-abc",+ }),+ ).toEqual({ status: "denied", reason: "gate_unavailable" });+ expect(mockCheck).not.toHaveBeenCalled();+ });++ // firebill cannot fail closed on its own behalf when it is what is down.+ it("denies a gated run when firebill itself cannot answer", async () => {+ state.configRef = firebillConfig();+ const svc = makeService();+ const gated = {+ teamId: "team-1",+ value: 10,+ lockId: "lock-1",+ partnerJobToken: "job-token-abc",+ };++ mockFetch.mockResolvedValueOnce(lockResponse({ success: false }));+ expect(await svc.lockCredits(gated)).toEqual({+ status: "denied",+ reason: "gate_unavailable",+ });++ mockFetch.mockRejectedValueOnce(new Error("ECONNREFUSED"));+ expect(await svc.lockCredits(gated)).toEqual({+ status: "denied",+ reason: "gate_unavailable",+ });++ // An answer we cannot read is not an answer either.+ mockFetch.mockResolvedValueOnce(lockResponse({ success: true }));+ expect(await svc.lockCredits(gated)).toEqual({+ status: "denied",+ reason: "gate_unavailable",+ });++ expect(mockCheck).not.toHaveBeenCalled();+ });++ // What makes the above affordable: refusing ordinary holds would turn a+ // firebill blip into a customer-facing outage.+ it("still skips rather than denies when no partner gate is involved", async () => {+ state.configRef = firebillConfig();+ const svc = makeService();++ mockFetch.mockResolvedValueOnce(lockResponse({ success: false }));+ expect(+ await svc.lockCredits({ teamId: "team-1", value: 10, lockId: "lock-1" }),+ ).toEqual({ status: "skipped" });+ });++ // Routing can flip in the hour between a lock and its finalize; going direct+ // would drop the partner's only report of the run.+ it("follows the run token to firebill even when routing says otherwise", async () => {+ state.configRef = {+ ...firebillConfig(),+ FIREBILL_ORG_IDS: ["some-other-org"],+ };+ const svc = makeService();++ await svc.finalizeCreditsLock({+ lockId: "monitor_check-1",+ action: "confirm",+ overrideValue: 7,+ teamId: "team-1",+ externalRequestId: "run-42",+ });++ expect(mockFinalize).not.toHaveBeenCalled();+ const finalizeCall = mockFetch.mock.calls.find(([url]) =>+ String(url).endsWith("/v1/finalize"),+ );+ expect(finalizeCall).toBeDefined();+ expect(JSON.parse(finalizeCall![1].body).external_request_id).toBe(+ "run-42",+ );+ });++ it("hands the run token back on the finalize as the operation id", async () => {+ state.configRef = firebillConfig();+ const svc = makeService();++ await svc.finalizeCreditsLock({+ lockId: "monitor_check-1",+ action: "confirm",+ overrideValue: 7,+ teamId: "team-1",+ externalRequestId: "run-42",+ });++ const finalizeCall = mockFetch.mock.calls.find(([url]) =>+ String(url).endsWith("/v1/finalize"),+ );+ expect(JSON.parse(finalizeCall![1].body).external_request_id).toBe(+ "run-42",+ );+ });++ // firebill has no lock table and the Autumn finalize body carries no+ // customer, so without this it cannot find the integration to report to.+ it("sends the customer alongside the run token so the run can be reported", async () => {+ state.configRef = firebillConfig();+ const svc = makeService();++ await svc.finalizeCreditsLock({+ lockId: "monitor_check-1",+ action: "confirm",+ overrideValue: 7,+ teamId: "team-1",+ externalRequestId: "run-42",+ heldValue: 12,+ });++ const body = JSON.parse(+ mockFetch.mock.calls+ .filter(([url]) => String(url).endsWith("/v1/finalize"))+ .pop()![1].body,+ );+ expect(body.customer_id).toBe("org-1");+ expect(body.external_request_id).toBe("run-42");+ // The other two the split needs: which balance, and what was held β+ // Autumn reports a balance net of that hold.+ expect(body.feature_id).toBe("CREDITS");+ expect(body.held_value).toBe(12);+ });++ // There is no durable retry here: billMonitorCheck's caller catches, writes+ // billing_status "failed", and nothing reads that back. Throwing would+ // abandon the finalize, so the hold expires and the run goes unbilled as+ // well as unreported β a worse loss than the missing label.+ it("still finalizes when it cannot name the org, rather than abandoning the settle", async () => {+ state.configRef = firebillConfig();+ // No org row for this team, so resolving the customer throws.+ state.supabaseStubData = { data: null, error: null };+ const svc = makeService();++ await expect(+ svc.finalizeCreditsLock({+ lockId: "monitor_check-1",+ action: "confirm",+ overrideValue: 7,+ teamId: "team-99",+ externalRequestId: "run-42",+ }),+ ).resolves.not.toThrow();++ const body = JSON.parse(+ mockFetch.mock.calls+ .filter(([url]) => String(url).endsWith("/v1/finalize"))+ .pop()![1].body,+ );+ expect(body.lock_id).toBe("monitor_check-1");+ // No org, so firebill cannot split or report β and counts the omission.+ expect(body).not.toHaveProperty("customer_id");+ expect(body).not.toHaveProperty("feature_id");+ expect(body).not.toHaveProperty("held_value");+ });++ // An ordinary settle reports to nobody, so it should not pay for the lookup+ // or put a customer on the wire.+ // firebill answers `false` for a refusal, a timeout, or a non-OK, and none of+ // them throw. A caller that cannot see that records a run as billed that+ // nobody billed, with no retry β so the verdict has to come back.+ it("reports whether the settle actually landed", async () => {+ state.configRef = firebillConfig();+ const svc = makeService();++ mockFetch.mockResolvedValueOnce(+ new Response(JSON.stringify({ success: true }), { status: 200 }),+ );+ await expect(+ svc.finalizeCreditsLock({+ lockId: "monitor_check-1",+ action: "confirm",+ overrideValue: 7,+ teamId: "team-1",+ }),+ ).resolves.toBe(true);++ // firebill took it but says it did not land.+ mockFetch.mockResolvedValueOnce(+ new Response(JSON.stringify({ success: false }), { status: 200 }),+ );+ await expect(+ svc.finalizeCreditsLock({+ lockId: "monitor_check-2",+ action: "confirm",+ overrideValue: 7,+ teamId: "team-1",+ }),+ ).resolves.toBe(false);++ // Never answered at all.+ mockFetch.mockRejectedValueOnce(new Error("ECONNREFUSED"));+ await expect(+ svc.finalizeCreditsLock({+ lockId: "monitor_check-3",+ action: "confirm",+ overrideValue: 7,+ teamId: "team-1",+ }),+ ).resolves.toBe(false);+ });++ it("omits the customer when the settle is not gated", async () => {+ state.configRef = firebillConfig();+ const svc = makeService();++ await svc.finalizeCreditsLock({+ lockId: "monitor_check-1",+ action: "confirm",+ overrideValue: 7,+ teamId: "team-1",+ });++ const body = JSON.parse(+ mockFetch.mock.calls+ .filter(([url]) => String(url).endsWith("/v1/finalize"))+ .pop()![1].body,+ );+ expect(body).not.toHaveProperty("customer_id");+ expect(body).not.toHaveProperty("feature_id");+ });++ it("omits the operation id when there is no partner to report to", async () => {+ state.configRef = firebillConfig();+ const svc = makeService();++ await svc.finalizeCreditsLock({+ lockId: "monitor_check-1",+ action: "confirm",+ teamId: "team-1",+ externalRequestId: null,+ });++ const finalizeCall = mockFetch.mock.calls.find(([url]) =>+ String(url).endsWith("/v1/finalize"),+ );+ expect(JSON.parse(finalizeCall![1].body)).not.toHaveProperty(+ "external_request_id",+ );+ });+it("uses the direct Autumn lock path for orgs NOT on the allowlist", async () => {state.configRef = {...firebillConfig(),diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex 3b51268ce..cfd6e3700 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -10,6 +10,7 @@ import {firebillLock,firebillTrack,firebillCheck,+ firebillConfigured,shouldRouteToFirebill,} from "./firebill";import { billingRouteTotal } from "./metrics";@@ -546,12 +547,23 @@ export class AutumnService {expiresAt,properties,featureId = CREDITS_FEATURE_ID,+ partnerJobToken,}: LockCreditsParams): Promise<LockCreditsResult> {if (!autumnClient || this.isPreviewTeam(teamId)) {return { status: "skipped" };}const resolvedLockId = lockId ?? `billing_${randomUUID()}`;+ // An ordinary hold proceeds unlocked; refusing would turn a firebill blip+ // into a customer outage. A gated run is the opposite: no answer means no+ // run token, so the work could never be billed. firebill fails closed when+ // it cannot reach the partner, but it cannot do so when it is the thing+ // that is down.+ const unreachable = (): LockCreditsResult =>+ partnerJobToken+ ? { status: "denied", reason: "gate_unavailable" }+ : { status: "skipped" };+try {const customerId = await this.ensureTrackingContext(teamId);@@ -559,7 +571,8 @@ export class AutumnService {// their holds through firebill. The hold still lives in Autumn (firebill// keeps no lock state), but only firebill pins the retry/timeout budget// around the call. An unavailable answer maps to "skipped" β proceed- // unlocked β exactly like a direct-Autumn check failure below.+ // unlocked β like a direct-Autumn check failure below, except when a+ // partner gate is involved; see `unreachable` above.if (shouldRouteToFirebill(customerId, {gatewayProvisioned: await this.isGatewayProvisioned(teamId),@@ -576,14 +589,34 @@ export class AutumnService {// monitor runner's convention, when the caller sets none.expiresAt: expiresAt ?? Date.now() + 60 * 60 * 1000,properties,+ partnerJobToken,});if (result.status === "locked") {- return { status: "locked", lockId: result.lockId };+ return {+ status: "locked",+ lockId: result.lockId,+ ...(result.operationToken+ ? { operationToken: result.operationToken }+ : {}),+ };}if (result.status === "denied") {- return { status: "denied" };+ return {+ status: "denied",+ ...(result.reason ? { reason: result.reason } : {}),+ };}- return { status: "skipped" };+ return unreachable();+ }++ // Unreachable β a partner-provisioned org always routes to firebill+ // (#4403) β but a token here would silently skip the gate.+ if (partnerJobToken) {+ logger.error(+ "A partner job token reached the direct-Autumn lock path, where no partner can be asked; refusing the hold",+ { teamId, lockId: resolvedLockId },+ );+ return { status: "denied", reason: "gate_unavailable" };}const { allowed } = await autumnClient.check({@@ -627,7 +660,8 @@ export class AutumnService {error,},);- return { status: "skipped" };+ // A gated run that threw before asking anyone is still unauthorized.+ return unreachable();}}@@ -639,6 +673,11 @@ export class AutumnService {* durably and retries delivery β a dropped direct finalize means the hold* just expires, leaving a confirm's work unbilled. Either route lands on the* same Autumn lock, so routing is a durability choice, not a correctness one.+ *+ * **Except with a run token in hand**, where it is a correctness choice: only+ * firebill reports the operation to the partner, and the routing predicate is+ * not stable across the hour between a lock and its finalize. The token is+ * proof of the route the lock took, so it wins over asking again.*/async finalizeCreditsLock({lockId,@@ -646,13 +685,51 @@ export class AutumnService {overrideValue,properties,teamId,- }: FinalizeCreditsLockParams): Promise<void> {- if (teamId && (await this.isRoutedThroughFirebill(teamId))) {- await firebillFinalize({ lockId, action, overrideValue, properties });- return;+ externalRequestId,+ featureId = CREDITS_FEATURE_ID,+ heldValue,+ }: FinalizeCreditsLockParams): Promise<boolean> {+ const gated = Boolean(externalRequestId) && firebillConfigured();+ if (gated || (teamId && (await this.isRoutedThroughFirebill(teamId)))) {+ // Only resolved for a gated settle: firebill needs the org to split the+ // settle and to find the integration to report to. An ordinary finalize+ // does neither, so it does not pay for the lookup.+ //+ // A failure degrades to omitting it rather than throwing. There is no+ // durable retry on this path β `billMonitorCheck`'s only caller catches,+ // writes `billing_status: "failed"`, and moves on, and nothing ever reads+ // that back β so throwing would abandon the finalize entirely: the hold+ // expires and the run goes unbilled at Autumn as well as unreported.+ // Settling and losing the label is the lesser loss, and firebill counts+ // the lost label as `partner_events_total{outcome="no_customer"}`.+ const customerId =+ externalRequestId && teamId+ ? await this.ensureTrackingContext(teamId).catch(error => {+ logger.error(+ "Could not resolve the org for a gated settle; finalizing anyway, but this run cannot be reported to its partner",+ { teamId, lockId, error },+ );+ return null;+ })+ : null;+ // Surfaced, not discarded. firebill answers `false` for a refusal, a+ // timeout, or a non-OK β none of which throw β so a caller that ignores+ // this records a run as billed that nobody billed.+ return await firebillFinalize({+ lockId,+ action,+ overrideValue,+ properties,+ externalRequestId,+ customerId,+ featureId: customerId ? featureId : null,+ heldValue: customerId ? heldValue : null,+ });}- if (!autumnClient) return;+ // No client means no hold was ever taken, so nothing can have gone+ // unsettled.+ if (!autumnClient) return true;try {await autumnClient.balances.finalize({@@ -666,6 +743,7 @@ export class AutumnService {action,overrideValue,});+ return true;} catch (error) {logger.error("Autumn finalizeCreditsLock failed β billing API may be unavailable",@@ -676,6 +754,7 @@ export class AutumnService {error,},);+ return false;}}diff --git a/apps/api/src/services/autumn/firebill.ts b/apps/api/src/services/autumn/firebill.tsindex 0f00d6481..2b995aa8e 100644--- a/apps/api/src/services/autumn/firebill.ts+++ b/apps/api/src/services/autumn/firebill.ts@@ -2,7 +2,7 @@ import { randomUUID } from "crypto";import { config } from "../../config";import { logger } from "../../lib/logger";import { sampled } from "../../lib/rollout";-import type { TrackParams } from "./types";+import type { LockDeniedReason, TrackParams } from "./types";import {firebillCheckTotal,firebillRetryTotal,@@ -16,15 +16,35 @@ import {* Autumn at all and the caller decides how its endpoint fails.*/type FirebillLockResult =- | { status: "locked"; lockId: string }- | { status: "denied" }+ | { status: "locked"; lockId: string; operationToken?: string }+ | { status: "denied"; reason?: LockDeniedReason }| { status: "unavailable" };+/**+ * An unrecognised reason is dropped rather than passed through: reading one we+ * do not understand as `job_revoked` would stop a customer's schedule.+ */+const LOCK_DENIED_REASONS: readonly LockDeniedReason[] = [+ "out_of_credits",+ "job_revoked",+ "gate_unavailable",+];++function lockDeniedReason(value: unknown): LockDeniedReason | undefined {+ return LOCK_DENIED_REASONS.find(reason => reason === value);+}+// firebill's own internal budget (durable write + forward attempt) is up to// ~3.5s worst case, so this is deliberately looser than the 2s timeout on the// direct Autumn client.const FIREBILL_TIMEOUT_MS = 5000;+// A gated lock legitimately does more: firebill asks the partner (2.5s ceiling)+// before the Autumn hold (2s), on top of the funding lookup. Giving up at 5s+// would abandon a call that is still going to take the hold - the run marked+// skipped here, the balance reserved there.+const FIREBILL_GATED_LOCK_TIMEOUT_MS = 10000;+// Safe to retry because the idempotency key is stable across attempts: if the// first attempt did land (ambiguous confirm timeout), Autumn dedupes the second.// Small because a caller may be waiting, and a firebill refusing events usually@@ -50,6 +70,14 @@ function firebillOrgIds(): Set<string> {return allowlistCache.ids;}+/**+ * Configured, without the rollout question β a finalize carrying a run token+ * follows the lock through firebill whatever the ramp says.+ */+export function firebillConfigured(): boolean {+ return Boolean(config.FIREBILL_URL && config.FIREBILL_SECRET);+}+/*** Whether this org's usage goes through firebill rather than straight to Autumn.* Needs firebill configured, then any of: the allowlist, being@@ -75,7 +103,7 @@ export function shouldRouteToFirebill(orgId: string,opts?: { gatewayProvisioned?: boolean },): boolean {- if (!config.FIREBILL_URL || !config.FIREBILL_SECRET) return false;+ if (!firebillConfigured()) return false;// Always-on set: test orgs stay routed even at 0 percent.if (firebillOrgIds().has(orgId)) return true;// A partner-provisioned org always routes: firebill is the only thing that@@ -400,6 +428,7 @@ export async function firebillLock({lockId,expiresAt,properties,+ partnerJobToken,}: {customerId: string;entityId: string;@@ -408,6 +437,7 @@ export async function firebillLock({lockId: string;expiresAt: number;properties?: Record<string, unknown>;+ partnerJobToken?: string | null;}): Promise<FirebillLockResult> {const url = firebillUrl("/v1/lock");try {@@ -428,8 +458,12 @@ export async function firebillLock({lock_id: lockId,expires_at: expiresAt,properties,+ // Present arms firebill's partner gate; omitted is today's path.+ ...(partnerJobToken ? { partner_job_token: partnerJobToken } : {}),}),- signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),+ signal: AbortSignal.timeout(+ partnerJobToken ? FIREBILL_GATED_LOCK_TIMEOUT_MS : FIREBILL_TIMEOUT_MS,+ ),});if (!response.ok) {@@ -448,6 +482,8 @@ export async function firebillLock({success?: boolean;allowed?: boolean;lock_id?: string;+ reason?: unknown;+ operation_token?: unknown;};if (body.success !== true) {@@ -462,14 +498,16 @@ export async function firebillLock({}if (body.allowed === false) {+ const reason = lockDeniedReason(body.reason);logger.info("firebill lock denied", {customerId,entityId,featureId,value,lockId,+ reason,});- return { status: "denied" };+ return { status: "denied", ...(reason ? { reason } : {}) };}// Only an explicit `allowed: false` is a denial. `success: true` with a@@ -487,14 +525,25 @@ export async function firebillLock({return { status: "unavailable" };}+ const operationToken =+ typeof body.operation_token === "string" &&+ body.operation_token.length > 0+ ? body.operation_token+ : undefined;+logger.info("firebill lock succeeded", {customerId,entityId,featureId,value,lockId,+ gated: operationToken !== undefined,});- return { status: "locked", lockId: body.lock_id ?? lockId };+ return {+ status: "locked",+ lockId: body.lock_id ?? lockId,+ ...(operationToken ? { operationToken } : {}),+ };} catch (error) {logger.error("firebill lock failed β firebill may be unavailable", {customerId,@@ -525,11 +574,19 @@ export async function firebillFinalize({action,overrideValue,properties,+ externalRequestId,+ customerId,+ featureId,+ heldValue,}: {lockId: string;action: "confirm" | "release";overrideValue?: number;properties?: Record<string, unknown>;+ externalRequestId?: string | null;+ customerId?: string | null;+ featureId?: string | null;+ heldValue?: number | null;}): Promise<boolean> {const url = firebillUrl("/v1/finalize");try {@@ -551,6 +608,24 @@ export async function firebillFinalize({// settle β e.g. the reconciler re-running a check it raced β dedupes// upstream instead of settling twice.idempotency_key: `fc:finalize:${action}:${lockId}`,+ // The run token, brought home as this settle's operation id.+ ...(externalRequestId+ ? { external_request_id: externalRequestId }+ : {}),+ // Who the lock was for. The Autumn finalize body carries no customer+ // and firebill keeps no lock table, so without this it cannot find the+ // integration to report the run to.+ ...(customerId ? { customer_id: customerId } : {}),+ // With customer_id, this is what lets firebill split the settle across+ // the ghost and its funder β it reads what the ghost has left of this+ // feature. Without it the whole cost falls on the ghost.+ ...(featureId ? { feature_id: featureId } : {}),+ // What the lock reserved. Autumn reports a balance net of outstanding+ // holds, so firebill needs this to know what the ghost can actually+ // pay for this run; without it the whole cost falls on the ghost.+ ...(heldValue !== undefined && heldValue !== null+ ? { held_value: heldValue }+ : {}),}),signal: AbortSignal.timeout(FIREBILL_TIMEOUT_MS),});diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.tsindex 18859d1bb..bfe84aa82 100644--- a/apps/api/src/services/autumn/types.ts+++ b/apps/api/src/services/autumn/types.ts@@ -52,19 +52,31 @@ export type LockCreditsParams = {expiresAt?: number;properties?: Record<string, unknown>;featureId?: string;+ /** Arms firebill's partner credit gate, which is asked before Autumn holds anything. */+ partnerJobToken?: string | null;};+/**+ * Why a partner's gate refused, as opposed to Autumn. `job_revoked` is the only+ * one that never resolves on its own, and so the only one a caller may answer+ * by stopping a schedule.+ */+export type LockDeniedReason =+ | "out_of_credits"+ | "job_revoked"+ | "gate_unavailable";+/*** Outcome of an Autumn credit lock attempt.*- * - `denied`: Autumn refused (`allowed: false`); the caller must NOT proceed.- * - `skipped`: billing not in effect (no client, preview team, or API fallback);- * the caller should proceed without a lock.- * - `locked`: reserved; `lockId` must be finalized later.+ * - `denied`: Autumn refused, or a partner's gate did β see `reason`.+ * - `skipped`: billing not in effect; proceed without a lock.+ * - `locked`: reserved; finalize with `lockId`. `operationToken` is the+ * partner's id for this occurrence β hand it back on the finalize.*/export type LockCreditsResult =- | { status: "locked"; lockId: string }- | { status: "denied" }+ | { status: "locked"; lockId: string; operationToken?: string }+ | { status: "denied"; reason?: LockDeniedReason }| { status: "skipped" };export type FinalizeCreditsLockParams = {@@ -80,6 +92,13 @@ export type FinalizeCreditsLockParams = {* loses firebill's durable retry).*/teamId?: string;+ /** For a gated run, the `operationToken` its lock handed back. */+ externalRequestId?: string | null;+ /** Which balance the hold was against; lets firebill split the settle. */+ featureId?: string;+ /** What the lock reserved. Autumn nets outstanding holds out of a reported+ * balance, so firebill adds this back to see what the ghost can really pay. */+ heldValue?: number | null;};export type TrackCreditsParams = {diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex ca4c271b2..3b6c5b94e 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -36,6 +36,7 @@ import { sendMonitoringSlackSummary } from "../notification/monitoring_slack";import {bulkUpsertMonitorPages,calculateMonitorCheckActualCredits,+ countRecentConsecutiveSkippedForCredits,getMonitorCheck,getMonitorForUpdate,countMonitorCheckPages,@@ -45,6 +46,7 @@ import {listMonitorCheckPages,listRunningMonitorChecks,markMonitorRunning,+ pauseMonitor,updateMonitorCheck,updateMonitorCheckIfRunning,updateMonitorScheduleAfterRun,@@ -83,6 +85,16 @@ const MONITOR_NOTIFY_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;const MONITOR_CHECK_PAGE_SCAN_LIMIT = 100_000;const MONITOR_CHECK_NO_CREDITS_ERROR ="Monitor check skipped: insufficient credits.";+const MONITOR_CHECK_REVOKED_ERROR =+ "Monitor check skipped: the partner has revoked this job.";++/**+ * Consecutive credit-skipped checks, this one included, before a `job_revoked`+ * denial stops the schedule. A 410 is permanent, but pausing on one response+ * would let a partner's bad build take down every monitor they fund. Never+ * reached by a 402: the current denial must itself be `job_revoked`.+ */+const MONITOR_GATE_REVOKED_STREAK = 3;const TERMINAL_CHECK_STATUSES = new Set(["completed","partial",@@ -342,14 +354,18 @@ function summarize(pages: PageResult[]) {};}+/// `false` means the settle did not land: the hold is still out there and this+/// run is not billed. Only the caller can decide what to record for that, so it+/// is returned rather than swallowed.async function billMonitorCheck(params: {monitor: MonitorRow;check: MonitorCheckRow;actualCredits: number;lockId: string | null;-}): Promise<void> {+}): Promise<boolean> {+ let settled = true;if (params.lockId) {- await autumnService.finalizeCreditsLock({+ settled = await autumnService.finalizeCreditsLock({lockId: params.lockId,action: "confirm",overrideValue: params.actualCredits,@@ -359,10 +375,35 @@ async function billMonitorCheck(params: {jobId: params.check.id,},teamId: params.monitor.team_id,+ // Confirm only: a release bills nothing, so there is nothing to report.+ externalRequestId: params.check.partner_run_token,+ // What the lock reserved. firebill adds it back to the ghost's reported+ // balance, which Autumn returns net of this very hold.+ heldValue: params.check.reserved_credits,});}- if (params.actualCredits <= 0 || !config.USE_DB_AUTHENTICATION) return;+ if (params.actualCredits <= 0 || !config.USE_DB_AUTHENTICATION)+ return settled;++ // The settle did not land, so Autumn has nothing and the hold expires by+ // itself. Debiting the team's ledger anyway charges them for a run Autumn+ // never billed, leaving the two ledgers disagreeing while the row already+ // says `failed`. `autumnTrackInRequest` below would be a lie too: it tells+ // the batch Autumn already has this, and it does not.+ //+ // So nothing is charged. Firecrawl absorbs the run β the direction this path+ // errs everywhere else β and `billing_status: "failed"` plus the error beside+ // it are how it is found.+ if (!settled) {+ logger.error("Not billing a monitor check whose settle did not land", {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ lockId: params.lockId,+ actualCredits: params.actualCredits,+ });+ return false;+ }await getBillingQueue().add("bill_team",@@ -384,6 +425,8 @@ async function billMonitorCheck(params: {priority: 10,},);++ return settled;}async function sendNotifications(params: {@@ -944,23 +987,58 @@ export async function processMonitorCheckJob(endpoint: "monitor",jobId: check.id,},+ // Arms firebill's partner gate; NULL is today's lock. Withheld once this+ // check holds a run token: a redelivery is the same occurrence, and+ // asking twice would orphan the first token.+ partnerJobToken: check.partner_run_token+ ? null+ : monitor.partner_job_token,});if (lock.status === "denied") {+ const revoked = lock.reason === "job_revoked";check = await updateMonitorCheck(check.id, {status: "skipped_no_credits",finished_at: new Date().toISOString(),actual_credits: 0,billing_status: "not_applicable",- error: MONITOR_CHECK_NO_CREDITS_ERROR,+ error: revoked+ ? MONITOR_CHECK_REVOKED_ERROR+ : MONITOR_CHECK_NO_CREDITS_ERROR,});- await updateMonitorScheduleAfterRun({ monitor, check });+ // A revoked job never becomes unrevoked; see MONITOR_GATE_REVOKED_STREAK+ // for why it is waited out rather than acted on at once.+ const paused =+ revoked &&+ (await countRecentConsecutiveSkippedForCredits({+ teamId: monitor.team_id,+ monitorId: monitor.id,+ limit: MONITOR_GATE_REVOKED_STREAK,+ })) >= MONITOR_GATE_REVOKED_STREAK;++ if (paused) {+ await pauseMonitor(monitor.id);+ logger.warn("Paused monitor: the partner has revoked this job", {+ monitorId: monitor.id,+ checkId: check.id,+ teamId: monitor.team_id,+ consecutiveSkips: MONITOR_GATE_REVOKED_STREAK,+ });+ }++ // Reads status off the object it is given, not the row β a paused+ // monitor must not be handed a next_run_at.+ await updateMonitorScheduleAfterRun({+ monitor: paused ? { ...monitor, status: "paused" } : monitor,+ check,+ });- logger.info("Skipped monitor check: insufficient credits", {+ logger.info("Skipped monitor check: no credit authority allowed it", {monitorId: monitor.id,checkId: check.id,teamId: monitor.team_id,+ reason: lock.reason,});return;}@@ -969,6 +1047,11 @@ export async function processMonitorCheckJob(check = await updateMonitorCheck(check.id, {autumn_lock_id: lockId,+ // A token already on the row wins: the gate was not re-asked, so there+ // is no newer one, and null would lose the authorized operation.+ partner_run_token:+ check.partner_run_token ??+ (lock.status === "locked" ? (lock.operationToken ?? null) : null),reserved_credits: lockId ? (check.estimated_credits ?? 1) : null,billing_status: lockId ? "reserved" : "not_applicable",});@@ -1435,7 +1518,10 @@ export async function reconcileRunningMonitorChecks(status: errorCount > 0 ? "partial" : "completed",finished_at: new Date().toISOString(),actual_credits: actualCredits,- billing_status: check.autumn_lock_id ? "confirmed" : "not_applicable",+ // Not `confirmed` yet β the settle has not been attempted. Claiming it+ // here and then finding firebill refused leaves a run permanently+ // recorded as billed that nobody billed, with no retry.+ billing_status: check.autumn_lock_id ? "reserved" : "not_applicable",total_pages: totalPages,same_count: same,changed_count: changed,@@ -1445,8 +1531,9 @@ export async function reconcileRunningMonitorChecks(target_results: targetResults,});+ let settled = false;try {- await billMonitorCheck({+ settled = await billMonitorCheck({monitor,check: finalized,actualCredits,@@ -1458,10 +1545,27 @@ export async function reconcileRunningMonitorChecks(checkId: finalized.id,error,});+ }++ // A refusal and a throw are the same fact β the settle did not land β and+ // both must be recorded as such. `firebillFinalize` answers `false`+ // without throwing, so the catch alone never saw them.+ if (check.autumn_lock_id) {+ if (!settled) {+ logger.error(+ "Monitor check settle did not land; the hold is unsettled and this run is unbilled",+ {+ monitorId: monitor.id,+ checkId: finalized.id,+ lockId: check.autumn_lock_id,+ actualCredits,+ },+ );+ }finalized = await updateMonitorCheck(check.id, {- billing_status: "failed",+ billing_status: settled ? "confirmed" : "failed",}).catch(updateError => {- logger.warn("Failed to record monitor check billing failure", {+ logger.warn("Failed to record monitor check billing outcome", {monitorId: monitor.id,checkId: finalized.id,error: updateError,diff --git a/apps/api/src/services/monitoring/store.gate.test.ts b/apps/api/src/services/monitoring/store.gate.test.tsnew file mode 100644index 000000000..5a239be24--- /dev/null+++ b/apps/api/src/services/monitoring/store.gate.test.ts@@ -0,0 +1,84 @@+const { rows, limits, updates, chain } = vi.hoisted(() => {+ const rows: { status: string }[] = [];+ const limits: number[] = [];+ const updates: Record<string, unknown>[] = [];++ const chain: Record<string, unknown> = {+ from: () => chain,+ where: () => chain,+ orderBy: () => chain,+ limit: (n: number) => {+ limits.push(n);+ return Promise.resolve(rows.slice(0, n));+ },+ };++ return { rows, limits, updates, chain };+});++vi.mock("../../db/connection", () => ({+ dbRr: { select: () => chain },+ db: {+ update: () => ({+ set: (patch: Record<string, unknown>) => {+ updates.push(patch);+ return { where: async () => undefined };+ },+ }),+ },+}));+vi.mock("../../db/rpc", () => ({ monitoringClaimDueMonitors: vi.fn() }));++import { countRecentConsecutiveSkippedForCredits, pauseMonitor } from "./store";++function seed(...statuses: string[]) {+ rows.length = 0;+ limits.length = 0;+ rows.push(...statuses.map(status => ({ status })));+}++async function streak(limit = 3) {+ return countRecentConsecutiveSkippedForCredits({+ teamId: "11111111-1111-1111-1111-111111111111",+ monitorId: "22222222-2222-2222-2222-222222222222",+ limit,+ });+}++describe("counting a monitor's credit-skip streak", () => {+ const skipped = "skipped_no_credits";++ it("counts the run ending at the newest check", async () => {+ seed(skipped, skipped, skipped);+ expect(await streak()).toBe(3);+ });++ // A partner that recovered must not lose the monitor on its next blip.+ it("stops at the first check that ran", async () => {+ seed(skipped, "completed", skipped);+ expect(await streak()).toBe(1);+ });++ it("is zero when the newest check is not a skip", async () => {+ seed("completed", skipped, skipped);+ expect(await streak()).toBe(0);+ seed();+ expect(await streak()).toBe(0);+ });++ it("reads no more rows than the caller's limit", async () => {+ seed(skipped, skipped, skipped, skipped, skipped);+ expect(await streak(3)).toBe(3);+ expect(limits).toEqual([3]);+ });+});++describe("pausing a monitor", () => {+ // Clearing next_run_at is what actually stops the runs being claimed.+ it("stops the schedule without destroying the monitor", async () => {+ updates.length = 0;+ await pauseMonitor("22222222-2222-2222-2222-222222222222");++ expect(updates[0]).toMatchObject({ status: "paused", next_run_at: null });+ });+});diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex b68f02eb1..d6ddb13ef 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -802,6 +802,60 @@ export async function listMonitorChecks(params: {return data as MonitorCheckRow[];}+/**+ * The leading run of credit-skipped checks ending at the newest, so one+ * successful check resets it. Derived rather than counted: a stored counter+ * would be a second source of truth to keep correct across retries.+ */+export async function countRecentConsecutiveSkippedForCredits(params: {+ teamId: string;+ monitorId: string;+ limit: number;+}): Promise<number> {+ const rows = await run(+ () =>+ dbRr+ .select({ status: schema.monitor_checks.status })+ .from(schema.monitor_checks)+ .where(+ and(+ eq(schema.monitor_checks.monitor_id, params.monitorId),+ eq(schema.monitor_checks.team_id, params.teamId),+ ),+ )+ .orderBy(desc(schema.monitor_checks.created_at))+ .limit(params.limit),+ "Failed to count skipped monitor checks",+ );++ let streak = 0;+ for (const row of rows) {+ if (row.status !== "skipped_no_credits") break;+ streak += 1;+ }+ return streak;+}++/**+ * `paused`, not `deleted`: nothing a partner says should destroy a customer's+ * configuration. `monitoring_claim_due_monitors` only claims `active` rows, so+ * this is enough to stop the runs.+ */+export async function pauseMonitor(monitorId: string): Promise<void> {+ await run(+ () =>+ db+ .update(schema.monitors)+ .set({+ status: "paused",+ next_run_at: null,+ updated_at: new Date().toISOString(),+ })+ .where(eq(schema.monitors.id, monitorId)),+ "Failed to pause monitor",+ );+}+export async function updateMonitorCheck(checkId: string,patch: Partial<MonitorCheckRow>,diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsindex 79944d01d..c4574a2a9 100644--- a/apps/api/src/services/monitoring/types.ts+++ b/apps/api/src/services/monitoring/types.ts@@ -163,7 +163,8 @@ const monitorNotificationInner = zif (notification.slack?.enabled && !notification.slack.channelId) {ctx.addIssue({code: "custom",- message: "A Slack channel is required when Slack notifications are enabled",+ message:+ "A Slack channel is required when Slack notifications are enabled",path: ["slack", "channelId"],});}@@ -353,6 +354,8 @@ export type MonitorCheckRow = {reserved_credits: number | null;actual_credits: number | null;autumn_lock_id: string | null;+ /** The partner's id for this occurrence; carried back on the finalize. */+ partner_run_token: string | null;billing_status:| "not_applicable"| "reserved"