Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(api): preserve monitor credit reservation ownership (#5031)

Rakshith Ramprakashe5df790e8326
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index f143e9816..a31b54e55 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -234,6 +234,7 @@ jobs:
run: >-
pnpm exec vitest run
src/services/monitoring/runner.finalization.test.ts
+ src/services/monitoring/runner.reservation.test.ts
src/services/monitoring/store.finalization.test.ts
src/services/monitoring/scheduler.test.ts
--reporter=default
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index b1a171c04..49e5bd5d6 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -10,6 +10,7 @@
*/
import { vi } from "vitest";
+import { AutumnDefaultError } from "autumn-js";
// ---------------------------------------------------------------------------
// Mocks β€” vi.mock is hoisted above the module body, so the mock backing objects
@@ -412,6 +413,74 @@ describe("team org change", () => {
// ---------------------------------------------------------------------------
describe("lockCredits", () => {
+ function sdkError(status: number, body: string) {
+ return new AutumnDefaultError("Reservation failed", {
+ request: new Request("https://example.com/check"),
+ response: new Response(body, { status }),
+ body,
+ });
+ }
+
+ it("rejects an existing direct reservation without claiming its hold", async () => {
+ mockCheck.mockRejectedValue(
+ sdkError(409, JSON.stringify({ code: "lock_already_exists" })),
+ );
+ await expect(
+ makeService().lockCredits({
+ teamId: "team-1",
+ orgId: "org-1",
+ value: 1,
+ lockId: "monitor_check-1",
+ }),
+ ).rejects.toMatchObject({
+ name: "ExistingCreditsLockError",
+ lockId: "monitor_check-1",
+ });
+ expect(mockFinalize).not.toHaveBeenCalled();
+ expect(mockTrack).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ [409, JSON.stringify({ code: "other_conflict" })],
+ [409, JSON.stringify({ message: "lock_already_exists" })],
+ [409, "lock_already_exists"],
+ [409, "null"],
+ [500, JSON.stringify({ code: "lock_already_exists" })],
+ ])(
+ "retains the unavailable fallback for status %s and body %s",
+ async (status, body) => {
+ mockCheck.mockRejectedValue(sdkError(status, body));
+ await expect(
+ makeService().lockCredits({
+ teamId: "team-1",
+ orgId: "org-1",
+ value: 1,
+ }),
+ ).resolves.toEqual({ status: "skipped" });
+ },
+ );
+
+ it("does not identify contention from a plain error message", async () => {
+ mockCheck.mockRejectedValue(new Error("409 lock_already_exists"));
+ await expect(
+ makeService().lockCredits({ teamId: "team-1", orgId: "org-1", value: 1 }),
+ ).resolves.toEqual({ status: "skipped" });
+ });
+
+ it("preserves successful zero-credit reservations", async () => {
+ await expect(
+ makeService().lockCredits({
+ teamId: "team-1",
+ orgId: "org-1",
+ value: 0,
+ lockId: "monitor_check-1",
+ }),
+ ).resolves.toEqual({ status: "locked", lockId: "monitor_check-1" });
+ expect(mockCheck).toHaveBeenCalledWith(
+ expect.objectContaining({ requiredBalance: 0 }),
+ );
+ });
+
it("returns skipped when autumnClient is null", async () => {
state.autumnClientRef = null;
const svc = makeService();
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index c94186989..b60c56d4f 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -1,4 +1,5 @@
import { randomUUID } from "crypto";
+import { AutumnError } from "autumn-js";
import { logger } from "../../lib/logger";
import { eq } from "drizzle-orm";
import { dbRr } from "../../db/connection";
@@ -31,6 +32,7 @@ import type {
TrackCreditsParams,
TrackParams,
} from "./types";
+import { ExistingCreditsLockError } from "./types";
export const TEAM_FEATURE_ID = "TEAM";
export const CREDITS_FEATURE_ID = "CREDITS";
@@ -645,18 +647,31 @@ export class AutumnService {
return { status: "denied", reason: "gate_unavailable" };
}
- const { allowed } = await autumnClient.check({
- customerId,
- entityId: teamId,
- featureId,
- requiredBalance: value,
- properties,
- lock: {
- enabled: true,
- lockId: resolvedLockId,
- expiresAt,
- },
- });
+ const { allowed } = await autumnClient
+ .check({
+ customerId,
+ entityId: teamId,
+ featureId,
+ requiredBalance: value,
+ properties,
+ lock: {
+ enabled: true,
+ lockId: resolvedLockId,
+ expiresAt,
+ },
+ })
+ .catch(error => {
+ if (error instanceof AutumnError && error.statusCode === 409) {
+ let body: { code?: unknown } | null = null;
+ try {
+ body = JSON.parse(error.body);
+ } catch {}
+ if (body?.code === "lock_already_exists") {
+ throw new ExistingCreditsLockError(resolvedLockId);
+ }
+ }
+ throw error;
+ });
if (!allowed) {
logger.info("Autumn lockCredits denied", {
@@ -677,6 +692,7 @@ export class AutumnService {
});
return { status: "locked", lockId: resolvedLockId };
} catch (error) {
+ if (error instanceof ExistingCreditsLockError) throw error;
logger.error(
"Autumn lockCredits failed β€” billing API may be unavailable, falling back",
{
diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.ts
index 8b5946302..0d709bfc2 100644
--- a/apps/api/src/services/autumn/types.ts
+++ b/apps/api/src/services/autumn/types.ts
@@ -1,3 +1,12 @@
+export class ExistingCreditsLockError extends Error {
+ constructor(readonly lockId: string) {
+ super(
+ "Credit reservation already exists; execution ownership is unresolved.",
+ );
+ this.name = "ExistingCreditsLockError";
+ }
+}
+
export type GetOrCreateCustomerParams = {
customerId: string;
name?: string | null;
diff --git a/apps/api/src/services/monitoring/runner.reservation.test.ts b/apps/api/src/services/monitoring/runner.reservation.test.ts
new file mode 100644
index 000000000..8d399d65f
--- /dev/null
+++ b/apps/api/src/services/monitoring/runner.reservation.test.ts
@@ -0,0 +1,634 @@
+import { AutumnDefaultError } from "autumn-js";
+import type { MonitorCheckRow, MonitorRow } from "./types";
+
+const { sdk, channel, logger, bill } = vi.hoisted(() => {
+ const logger = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ child: vi.fn(),
+ };
+ logger.child.mockReturnValue(logger);
+ return {
+ logger,
+ bill: vi.fn(),
+ sdk: {
+ check: vi.fn(),
+ track: vi.fn(),
+ balances: { finalize: vi.fn() },
+ customers: { getOrCreate: vi.fn(), get: vi.fn() },
+ entities: { get: vi.fn(), create: vi.fn() },
+ },
+ channel: {
+ assertExchange: vi.fn(),
+ assertQueue: vi.fn(),
+ bindQueue: vi.fn(),
+ prefetch: vi.fn(),
+ consume: vi.fn(),
+ ack: vi.fn(),
+ nack: vi.fn(),
+ on: vi.fn(),
+ },
+ };
+});
+
+vi.mock("../../config", () => ({
+ config: { USE_DB_AUTHENTICATION: true, NUQ_RABBITMQ_URL: "amqp://test" },
+}));
+vi.mock("../../lib/logger", () => ({ logger }));
+vi.mock("../autumn/client", () => ({ autumnClient: sdk }));
+vi.mock("../autumn/firebill", () => ({
+ shouldRouteToFirebill: () => false,
+ firebillConfigured: () => false,
+}));
+vi.mock("../autumn/metrics", () => ({
+ autumnCustomerGetOrCreateTotal: { inc: vi.fn() },
+ autumnEntityCreatedInlineTotal: { inc: vi.fn() },
+ billingRouteTotal: { inc: vi.fn() },
+}));
+vi.mock("../../db/connection", () => ({
+ dbRr: {
+ select: () => ({
+ from: () => ({ where: () => ({ limit: async () => [] }) }),
+ }),
+ },
+}));
+vi.mock("../../db/schema", () => ({ partner_provisioned_accounts: {} }));
+vi.mock("../logging/log_job", () => ({ logRequest: vi.fn() }));
+vi.mock("../../lib/gcs-monitoring", () => ({}));
+vi.mock("../worker/scrape-worker", () => ({}));
+vi.mock("../worker/nuq-router", () => ({}));
+vi.mock("./diff", () => ({
+ normalizeMonitorFormats: (formats: unknown) => formats,
+}));
+vi.mock("../../lib/crawl-redis", () => ({}));
+vi.mock("../queue-jobs", () => ({ addScrapeJob: vi.fn() }));
+vi.mock("../../controllers/v2/types", () => ({
+ scrapeRequestSchema: { parse: (value: unknown) => value },
+}));
+vi.mock("../webhook", () => ({}));
+vi.mock("./results", () => ({}));
+vi.mock("../notification/monitoring_email", () => ({
+ sendMonitoringEmailSummary: vi.fn(),
+}));
+vi.mock("../notification/monitoring_slack", () => ({}));
+vi.mock("../notification/monitoring_in_app", () => ({
+ recordMonitorInAppNotification: vi.fn(),
+}));
+vi.mock("./types", () => ({ withMarkdownFormat: (value: unknown) => value }));
+vi.mock("./interest", () => ({ trackMonitorCheckStartedInterest: vi.fn() }));
+vi.mock("./search/run", () => ({ runSearchTarget: vi.fn() }));
+vi.mock("./search/judge", () => ({}));
+vi.mock("./search/dedupe", () => ({}));
+vi.mock("./search/persist", () => ({}));
+vi.mock("../../scraper/WebScraper/utils/blocklist", () => ({}));
+vi.mock("../../controllers/auth", () => ({ getACUCTeam: vi.fn() }));
+vi.mock("./store", () => ({
+ getMonitorCheckForUpdate: vi.fn(),
+ getMonitorForUpdate: vi.fn(),
+ listRunningMonitorChecks: vi.fn(),
+ updateMonitorCheck: vi.fn(),
+ updateMonitorCheckIfRunning: vi.fn(),
+ updateMonitorCheckIfStatus: vi.fn(),
+ markMonitorRunning: vi.fn(),
+ countMonitorCheckPages: vi.fn(),
+ listMonitorCheckPages: vi.fn(),
+ calculateMonitorCheckActualCredits: vi.fn(),
+ updateMonitorScheduleAfterRun: vi.fn(),
+}));
+vi.mock("../queue-service", () => ({ getBillingQueue: () => ({ add: bill }) }));
+vi.mock("../redis", () => ({
+ redisEvictConnection: { set: vi.fn(), eval: vi.fn() },
+}));
+vi.mock("amqplib", () => ({
+ default: {
+ connect: async () => ({ createChannel: async () => channel, on: vi.fn() }),
+ },
+}));
+
+import {
+ processMonitorCheckJob,
+ reconcileRunningMonitorChecks,
+} from "./runner";
+import { consumeMonitorCheckJobs } from "./queue";
+import * as store from "./store";
+import { addScrapeJob } from "../queue-jobs";
+import { getACUCTeam } from "../../controllers/auth";
+import { redisEvictConnection } from "../redis";
+import { trackMonitorCheckStartedInterest } from "./interest";
+import { runSearchTarget } from "./search/run";
+import { config } from "../../config";
+import { autumnService } from "../autumn/autumn.service";
+
+function deferred<T = void>() {
+ let resolve!: (value: T) => void;
+ let reject!: (error: unknown) => void;
+ const promise = new Promise<T>((res, rej) => {
+ resolve = res;
+ reject = rej;
+ });
+ return { promise, resolve, reject };
+}
+
+function conflict() {
+ const body = JSON.stringify({ code: "lock_already_exists" });
+ return new AutumnDefaultError("Reservation failed", {
+ request: new Request("https://example.com/check"),
+ response: new Response(body, { status: 409 }),
+ body,
+ });
+}
+
+describe("direct monitor reservation ownership", () => {
+ let current: MonitorCheckRow;
+ let monitor: MonitorRow;
+ let recordedPages: Array<{
+ checkId: string;
+ targetId: string;
+ status: "same" | "changed" | "new" | "removed" | "error";
+ }>;
+ const job = { teamId: "team-1", monitorId: "monitor-1", checkId: "check-1" };
+ const effects = [
+ store.updateMonitorCheck,
+ store.updateMonitorCheckIfRunning,
+ store.updateMonitorCheckIfStatus,
+ store.markMonitorRunning,
+ store.updateMonitorScheduleAfterRun,
+ addScrapeJob,
+ runSearchTarget,
+ trackMonitorCheckStartedInterest,
+ sdk.balances.finalize,
+ sdk.track,
+ bill,
+ ];
+ const snapshot = () => ({
+ row: structuredClone(current),
+ calls: effects.map(fn => vi.mocked(fn).mock.calls.length),
+ });
+
+ beforeEach(() => {
+ vi.resetAllMocks();
+ logger.child.mockReturnValue(logger);
+ (config as { USE_DB_AUTHENTICATION: boolean }).USE_DB_AUTHENTICATION = true;
+ recordedPages = [];
+ current = {
+ id: "check-1",
+ monitor_id: "monitor-1",
+ team_id: "team-1",
+ status: "queued",
+ started_at: null,
+ estimated_credits: 1,
+ autumn_lock_id: null,
+ billing_status: "not_applicable",
+ target_results: [],
+ } as unknown as MonitorCheckRow;
+ monitor = {
+ id: "monitor-1",
+ team_id: "team-1",
+ current_check_id: "check-1",
+ targets: [
+ { id: "target-1", type: "scrape", urls: ["https://example.com"] },
+ ],
+ } as MonitorRow;
+ vi.mocked(getACUCTeam).mockResolvedValue({ org_id: "org-1" } as any);
+ sdk.check.mockResolvedValue({ allowed: true });
+ sdk.balances.finalize.mockResolvedValue(undefined);
+ sdk.customers.getOrCreate.mockResolvedValue({ id: "org-1" });
+ sdk.entities.get.mockResolvedValue({ id: "team-1" });
+ vi.mocked(trackMonitorCheckStartedInterest).mockResolvedValue(undefined);
+ vi.mocked(store.getMonitorForUpdate).mockImplementation(async () =>
+ structuredClone(monitor),
+ );
+ vi.mocked(store.getMonitorCheckForUpdate).mockImplementation(async () =>
+ structuredClone(current),
+ );
+ vi.mocked(store.listRunningMonitorChecks).mockImplementation(async () =>
+ current.status === "running" ? [structuredClone(current)] : [],
+ );
+ vi.mocked(store.updateMonitorCheck).mockImplementation(
+ async (_id, patch) => {
+ Object.assign(current, patch);
+ return structuredClone(current);
+ },
+ );
+ vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
+ async (_id, patch) => {
+ if (current.status !== "running") return null;
+ Object.assign(current, patch);
+ return structuredClone(current);
+ },
+ );
+ vi.mocked(store.updateMonitorCheckIfStatus).mockImplementation(
+ async (_id, status, patch) => {
+ if (current.status !== status) return null;
+ Object.assign(current, patch);
+ return structuredClone(current);
+ },
+ );
+ vi.mocked(store.countMonitorCheckPages).mockImplementation(
+ async ({ checkId, targetId, status }) =>
+ recordedPages.filter(
+ page =>
+ page.checkId === checkId &&
+ (!targetId || page.targetId === targetId) &&
+ (!status || page.status === status),
+ ).length,
+ );
+ vi.mocked(store.calculateMonitorCheckActualCredits).mockResolvedValue(1);
+ vi.mocked(store.listMonitorCheckPages).mockResolvedValue([]);
+ vi.mocked(redisEvictConnection.set).mockImplementation((async (
+ key: string,
+ ) => (key.startsWith("monitor-check-notify:") ? null : "OK")) as any);
+ vi.mocked(redisEvictConnection.eval).mockResolvedValue(1);
+ });
+
+ async function settleWinner(
+ credits = 1,
+ status: (typeof recordedPages)[number]["status"] = "same",
+ ) {
+ recordedPages = [{ checkId: current.id, targetId: "target-1", status }];
+ vi.mocked(store.calculateMonitorCheckActualCredits).mockResolvedValue(
+ credits,
+ );
+ await reconcileRunningMonitorChecks();
+ await reconcileRunningMonitorChecks();
+ expect(current).toMatchObject({
+ status: status === "error" ? "partial" : "completed",
+ billing_status: "confirmed",
+ autumn_lock_id: "monitor_check-1",
+ total_pages: 1,
+ same_count: Number(status === "same"),
+ changed_count: Number(status === "changed"),
+ new_count: Number(status === "new"),
+ removed_count: Number(status === "removed"),
+ error_count: Number(status === "error"),
+ });
+ expect(sdk.balances.finalize).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ overrideValue: credits,
+ }),
+ );
+ expect(sdk.track).not.toHaveBeenCalled();
+ if (credits > 0) {
+ expect(bill).toHaveBeenCalledExactlyOnceWith(
+ "bill_team",
+ expect.objectContaining({ credits, autumnTrackInRequest: true }),
+ { jobId: "monitor-bill-check-1", priority: 10 },
+ );
+ } else expect(bill).not.toHaveBeenCalled();
+ }
+
+ it.each([false, true])(
+ "leaves a live winner untouched with stored hold=%s",
+ async stored => {
+ const paused = deferred();
+ const resume = deferred();
+ if (stored) {
+ vi.mocked(store.updateMonitorCheckIfRunning)
+ .mockImplementationOnce(async (_id, patch) => {
+ Object.assign(current, patch);
+ return structuredClone(current);
+ })
+ .mockImplementationOnce(async (_id, patch) => {
+ paused.resolve();
+ await resume.promise;
+ Object.assign(current, patch);
+ return structuredClone(current);
+ });
+ } else {
+ sdk.check.mockImplementationOnce(async () => {
+ paused.resolve();
+ await resume.promise;
+ return { allowed: true };
+ });
+ }
+ if (stored) sdk.check.mockResolvedValueOnce({ allowed: true });
+ sdk.check.mockRejectedValueOnce(conflict());
+ const winner = processMonitorCheckJob(job);
+ await paused.promise;
+ const before = snapshot();
+ try {
+ await expect(processMonitorCheckJob(job)).rejects.toMatchObject({
+ name: "ExistingCreditsLockError",
+ lockId: "monitor_check-1",
+ });
+ expect(snapshot()).toEqual(before);
+ } finally {
+ resume.resolve();
+ await winner;
+ }
+ expect(addScrapeJob).toHaveBeenCalledTimes(1);
+ await settleWinner();
+ },
+ );
+
+ it("does not let the queued claimant overwrite a later reservation winner", async () => {
+ const reached = deferred();
+ const pending = deferred<{ allowed: boolean }>();
+ sdk.check
+ .mockImplementationOnce(() => {
+ reached.resolve();
+ return pending.promise;
+ })
+ .mockResolvedValueOnce({ allowed: true });
+ const claimant = processMonitorCheckJob(job);
+ const rejected = expect(claimant).rejects.toMatchObject({
+ name: "ExistingCreditsLockError",
+ });
+ await reached.promise;
+ await processMonitorCheckJob(job);
+ const before = snapshot();
+ pending.reject(conflict());
+ await rejected;
+ expect(snapshot()).toEqual(before);
+ expect(addScrapeJob).toHaveBeenCalledTimes(1);
+ await settleWinner();
+ });
+
+ it("leaves terminal billing untouched when a contender's conflict arrives late", async () => {
+ await processMonitorCheckJob(job);
+ const reached = deferred();
+ const pending = deferred<{ allowed: boolean }>();
+ sdk.check.mockImplementationOnce(() => {
+ reached.resolve();
+ return pending.promise;
+ });
+ const contender = processMonitorCheckJob(job);
+ const rejected = expect(contender).rejects.toMatchObject({
+ name: "ExistingCreditsLockError",
+ });
+ await reached.promise;
+ await settleWinner();
+ const before = snapshot();
+ pending.reject(conflict());
+ await rejected;
+ expect(snapshot()).toEqual(before);
+ });
+
+ it("reconciles a completed persisted search target without redelivery work", async () => {
+ current.status = "running";
+ current.started_at = new Date().toISOString();
+ current.autumn_lock_id = "monitor_check-1";
+ current.billing_status = "reserved";
+ monitor.targets = [
+ { id: "target-1", type: "search" },
+ ] as MonitorRow["targets"];
+ current.target_results = [
+ {
+ targetId: "target-1",
+ type: "search",
+ searchCompleted: true,
+ searchCredits: 1,
+ },
+ ];
+ sdk.check.mockRejectedValueOnce(conflict());
+ const before = snapshot();
+ await expect(processMonitorCheckJob(job)).rejects.toMatchObject({
+ name: "ExistingCreditsLockError",
+ });
+ expect(snapshot()).toEqual(before);
+ await settleWinner();
+ expect(runSearchTarget).not.toHaveBeenCalled();
+ expect(addScrapeJob).not.toHaveBeenCalled();
+ });
+
+ it("confirms a successful hold with zero actual credits without a ledger debit", async () => {
+ await processMonitorCheckJob(job);
+ await settleWinner(0);
+ });
+
+ it("preserves exempt execution without a reservation or debit", async () => {
+ monitor.team_id = "preview_team";
+ vi.mocked(store.calculateMonitorCheckActualCredits).mockResolvedValue(0);
+ await processMonitorCheckJob({ ...job, teamId: monitor.team_id });
+ recordedPages = [
+ { checkId: current.id, targetId: "target-1", status: "same" },
+ ];
+ await reconcileRunningMonitorChecks();
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "not_applicable",
+ autumn_lock_id: null,
+ });
+ expect(sdk.check).not.toHaveBeenCalled();
+ expect(sdk.balances.finalize).not.toHaveBeenCalled();
+ expect(bill).not.toHaveBeenCalled();
+ });
+
+ it("dead-letters recovery of an unlinked orphan hold and leaves expiry to the provider", async () => {
+ current.status = "running";
+ current.started_at = new Date().toISOString();
+ const held = new Set<string>();
+ sdk.check.mockImplementation(async ({ lock }) => {
+ if (held.has(lock.lockId)) throw conflict();
+ held.add(lock.lockId);
+ return { allowed: true };
+ });
+ await expect(
+ autumnService.lockCredits({
+ teamId: job.teamId,
+ orgId: "org-1",
+ value: 1,
+ lockId: "monitor_check-1",
+ }),
+ ).resolves.toEqual({ status: "locked", lockId: "monitor_check-1" });
+ expect(held.has("monitor_check-1")).toBe(true);
+ expect(current.autumn_lock_id).toBeNull();
+ await consumeMonitorCheckJobs(processMonitorCheckJob);
+ const message = { content: Buffer.from(JSON.stringify(job)) };
+ const before = snapshot();
+ await channel.consume.mock.calls[0][1](message);
+ expect(snapshot()).toEqual(before);
+ expect(channel.ack).not.toHaveBeenCalled();
+ expect(channel.nack).toHaveBeenCalledExactlyOnceWith(message, false, false);
+ expect(logger.error).toHaveBeenCalledWith(
+ "Monitor check job failed",
+ expect.objectContaining({
+ error: expect.objectContaining({ name: "ExistingCreditsLockError" }),
+ }),
+ );
+ expect(channel.assertExchange).toHaveBeenCalledWith(
+ "monitor.checks.dlx",
+ "direct",
+ { durable: true },
+ );
+ expect(channel.assertQueue).toHaveBeenCalledWith(
+ "monitor.checks",
+ expect.objectContaining({
+ arguments: expect.objectContaining({
+ "x-dead-letter-exchange": "monitor.checks.dlx",
+ "x-dead-letter-routing-key": "monitor.checks",
+ }),
+ }),
+ );
+ expect(channel.bindQueue).toHaveBeenCalledWith(
+ "monitor.checks.dlq",
+ "monitor.checks.dlx",
+ "monitor.checks",
+ );
+ current.started_at = new Date(
+ Date.now() - 2 * 60 * 60 * 1000,
+ ).toISOString();
+ await reconcileRunningMonitorChecks();
+ expect(current).toMatchObject({
+ status: "failed",
+ billing_status: "not_applicable",
+ });
+ expect(sdk.balances.finalize).not.toHaveBeenCalled();
+ expect(bill).not.toHaveBeenCalled();
+ });
+
+ it.each(["same", "changed", "new", "removed", "error"] as const)(
+ "settles the winner using recorded %s page counts",
+ async status => {
+ await processMonitorCheckJob(job);
+ recordedPages = [
+ { checkId: current.id, targetId: "other-target", status },
+ ];
+ await reconcileRunningMonitorChecks();
+ expect(current.status).toBe("running");
+ expect(sdk.balances.finalize).not.toHaveBeenCalled();
+ expect(bill).not.toHaveBeenCalled();
+ await settleWinner(status === "error" ? 0 : 1, status);
+ },
+ );
+
+ it("does not reserve a running delivery that became terminal after its initial read", async () => {
+ current.status = "running";
+ const initial = structuredClone(current);
+ current.status = "completed";
+ current.billing_status = "confirmed";
+ vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValueOnce(initial);
+ const before = snapshot();
+ await processMonitorCheckJob(job);
+ expect(snapshot()).toEqual(before);
+ expect(sdk.check).not.toHaveBeenCalled();
+ });
+
+ it("does not fail or release a winner when the admission reread is unavailable", async () => {
+ current.status = "running";
+ current.started_at = new Date().toISOString();
+ current.autumn_lock_id = "monitor_check-1";
+ current.billing_status = "reserved";
+ const failure = new Error("Primary read unavailable");
+ vi.mocked(store.getMonitorCheckForUpdate)
+ .mockResolvedValueOnce(structuredClone(current))
+ .mockRejectedValueOnce(failure);
+ const before = snapshot();
+ await expect(processMonitorCheckJob(job)).rejects.toThrow(failure);
+ expect(snapshot()).toEqual(before);
+ expect(sdk.check).not.toHaveBeenCalled();
+ });
+
+ it.each(["queued", "running"] as const)(
+ "does not reserve when a %s delivery completes during org resolution",
+ async status => {
+ current.status = status;
+ const reached = deferred();
+ const pending = deferred<any>();
+ vi.mocked(getACUCTeam).mockImplementationOnce(() => {
+ reached.resolve();
+ return pending.promise;
+ });
+ const delivery = processMonitorCheckJob(job);
+ await reached.promise;
+ current.status = "completed";
+ current.billing_status = "confirmed";
+ const before = snapshot();
+ pending.resolve({ org_id: "org-1" });
+ await delivery;
+ expect(snapshot()).toEqual(before);
+ expect(sdk.check).not.toHaveBeenCalled();
+ },
+ );
+
+ it("does not mark a terminal check when a successful reservation arrives late", async () => {
+ current.status = "running";
+ current.autumn_lock_id = "monitor_check-1";
+ current.billing_status = "reserved";
+ const reached = deferred();
+ const pending = deferred<{ allowed: boolean }>();
+ sdk.check.mockImplementationOnce(() => {
+ reached.resolve();
+ return pending.promise;
+ });
+ const delivery = processMonitorCheckJob(job);
+ await reached.promise;
+ current.status = "completed";
+ current.billing_status = "confirmed";
+ const terminal = structuredClone(current);
+ pending.resolve({ allowed: true });
+ await delivery;
+ expect(current).toEqual(terminal);
+ expect(store.markMonitorRunning).not.toHaveBeenCalled();
+ expect(trackMonitorCheckStartedInterest).not.toHaveBeenCalled();
+ expect(sdk.balances.finalize).not.toHaveBeenCalled();
+ expect(addScrapeJob).not.toHaveBeenCalled();
+ expect(bill).not.toHaveBeenCalled();
+ });
+
+ it("rejects linked orphan recovery and only stale reconciliation releases its hold", async () => {
+ current.status = "running";
+ current.started_at = new Date().toISOString();
+ current.autumn_lock_id = "monitor_check-1";
+ current.billing_status = "reserved";
+ sdk.check.mockRejectedValueOnce(conflict());
+ const before = snapshot();
+ await expect(processMonitorCheckJob(job)).rejects.toMatchObject({
+ name: "ExistingCreditsLockError",
+ });
+ expect(snapshot()).toEqual(before);
+ current.started_at = new Date(
+ Date.now() - 2 * 60 * 60 * 1000,
+ ).toISOString();
+ await reconcileRunningMonitorChecks();
+ expect(current).toMatchObject({
+ status: "failed",
+ billing_status: "released",
+ });
+ expect(sdk.balances.finalize).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ lockId: "monitor_check-1", action: "release" }),
+ );
+ expect(bill).not.toHaveBeenCalled();
+ });
+
+ it("releases a newly acquired unpersisted hold if later admission fails", async () => {
+ vi.mocked(store.updateMonitorCheckIfRunning).mockRejectedValueOnce(
+ new Error("Admission failed"),
+ );
+ await expect(processMonitorCheckJob(job)).rejects.toThrow(
+ "Admission failed",
+ );
+ expect(current).toMatchObject({
+ status: "failed",
+ billing_status: "released",
+ autumn_lock_id: null,
+ });
+ expect(sdk.balances.finalize).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ action: "release", lockId: "monitor_check-1" }),
+ );
+ expect(addScrapeJob).not.toHaveBeenCalled();
+ expect(bill).not.toHaveBeenCalled();
+ });
+
+ it("releases the acquired persisted hold if marking the monitor fails", async () => {
+ vi.mocked(store.markMonitorRunning).mockRejectedValueOnce(
+ new Error("Admission failed"),
+ );
+ await expect(processMonitorCheckJob(job)).rejects.toThrow(
+ "Admission failed",
+ );
+ expect(current).toMatchObject({
+ status: "failed",
+ billing_status: "released",
+ autumn_lock_id: "monitor_check-1",
+ });
+ expect(sdk.balances.finalize).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ action: "release", lockId: "monitor_check-1" }),
+ );
+ expect(addScrapeJob).not.toHaveBeenCalled();
+ expect(bill).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 2d2dd9219..20c33a36f 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -14,6 +14,7 @@ import { ScrapeJobData } from "../../types";
import { includesFormat } from "../../lib/format-utils";
import { normalizeMonitorFormats } from "./diff";
import { autumnService } from "../autumn/autumn.service";
+import { ExistingCreditsLockError } from "../autumn/types";
import { getBillingQueue } from "../queue-service";
import {
crawlToCrawler,
@@ -1040,35 +1041,27 @@ export async function processMonitorCheckJob(
return;
}
- const started = await updateMonitorCheckIfStatus(
- job.checkId,
- initialCheck.status,
- {
- status: "running",
- started_at: new Date().toISOString(),
- },
- );
+ const started =
+ initialCheck.status === "running"
+ ? initialCheck
+ : await updateMonitorCheckIfStatus(job.checkId, "queued", {
+ status: "running",
+ started_at: new Date().toISOString(),
+ });
if (!started) return;
let check: MonitorCheckRow = started;
- await markMonitorRunning({
- monitorId: monitor.id,
- checkId: job.checkId,
- });
-
- trackMonitorCheckStartedInterest({ monitor, check }).catch(error =>
- logger.warn("Failed to track monitor target interest", {
- error,
- monitorId: monitor.id,
- checkId: check.id,
- eventType: "check_started",
- }),
- );
-
// One org lookup for the whole check job β€” the billing service no longer
// makes it, so every hold, settle and release below shares this one. A
// failure answers null, which is what the lookup inside the biller did.
const orgId = await orgIdForTeam(monitor.team_id);
+ const admissionCheck = await getMonitorCheckForUpdate(
+ job.teamId,
+ job.monitorId,
+ job.checkId,
+ );
+ if (!admissionCheck || admissionCheck.status !== "running") return;
+ check = admissionCheck;
const partnerJobToken = check.partner_run_token
? null
: monitor.partner_job_token;
@@ -1169,6 +1162,20 @@ export async function processMonitorCheckJob(
}
check = reserved;
+ await markMonitorRunning({
+ monitorId: monitor.id,
+ checkId: job.checkId,
+ });
+
+ trackMonitorCheckStartedInterest({ monitor, check }).catch(error =>
+ logger.warn("Failed to track monitor target interest", {
+ error,
+ monitorId: monitor.id,
+ checkId: check.id,
+ eventType: "check_started",
+ }),
+ );
+
const targetResults = monitor.targets.map(createMonitorTargetRun);
const initialized = await updateMonitorCheckIfRunning(check.id, {
target_results: targetResults,
@@ -1230,6 +1237,7 @@ export async function processMonitorCheckJob(
target_results: targetResults,
});
} catch (error) {
+ if (error instanceof ExistingCreditsLockError) throw error;
// Atomically flip running -> failed. Returns null when the check already
// reached a terminal status β€” i.e. the reconciler finalized it (completed,
// billed, lock confirmed) before this late catch ran. In that case we must