Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

34.0 KB
Raw Blame
import type { MonitorCheckRow, MonitorRow } from "./types";

vi.mock("../../config", () => ({ config: { USE_DB_AUTHENTICATION: true } }));
vi.mock("../../lib/logger", () => {
  const logger = {
    info: vi.fn(),
    warn: vi.fn(),
    error: vi.fn(),
    child: (): any => logger,
  };
  return { logger };
});
vi.mock("../logging/log_job", () => ({}));
vi.mock("../../lib/gcs-monitoring", () => ({}));
vi.mock("../worker/scrape-worker", () => ({}));
vi.mock("../worker/nuq-router", () => ({}));
vi.mock("./diff", () => ({}));
vi.mock("../../lib/crawl-redis", () => ({}));
vi.mock("../queue-jobs", () => ({}));
vi.mock("../../controllers/v2/types", () => ({}));
vi.mock("../webhook", () => ({}));
vi.mock("./results", () => ({}));
vi.mock("../notification/monitoring_email", () => ({
  sendMonitoringEmailSummary: vi.fn(),
}));
vi.mock("../notification/monitoring_slack", () => ({}));
vi.mock("../notification/monitoring_in_app", () => ({
  recordMonitorInAppNotification: vi.fn(),
}));
vi.mock("./types", () => ({}));
vi.mock("./interest", () => ({
  trackMonitorCheckStartedInterest: async () => {},
}));
vi.mock("./search/run", () => ({ runSearchTarget: vi.fn() }));
vi.mock("./search/judge", () => ({}));
vi.mock("./search/dedupe", () => ({ computeGoalVersion: () => "goal-1" }));
vi.mock("./search/persist", () => ({
  reconstructKnownState: () => ({ knownPages: new Map(), knownEvents: [] }),
  searchStatusToPageStatus: (status: string) => status,
}));
vi.mock("../../scraper/WebScraper/utils/blocklist", () => ({}));
vi.mock("../../controllers/auth", () => ({ getACUCTeam: vi.fn() }));
vi.mock("./store", () => ({
  getMonitorCheckForUpdate: vi.fn(),
  getMonitorForUpdate: vi.fn(),
  listRunningMonitorChecks: vi.fn(),
  updateMonitorCheck: vi.fn(),
  updateMonitorCheckIfRunning: vi.fn(),
  updateMonitorCheckIfStatus: vi.fn(),
  markMonitorRunning: vi.fn(),
  countMonitorCheckPages: vi.fn(),
  listMonitorCheckPages: vi.fn(),
  calculateMonitorCheckActualCredits: vi.fn(),
  updateMonitorScheduleAfterRun: vi.fn(),
  listActiveMonitorPages: vi.fn(),
  deleteMonitorCheckPages: vi.fn(),
  insertMonitorCheckPages: vi.fn(),
  bulkUpsertMonitorPages: vi.fn(),
}));
vi.mock("../autumn/autumn.service", () => ({
  autumnService: { finalizeCreditsLock: vi.fn(), lockCredits: vi.fn() },
}));
vi.mock("../queue-service", () => ({ getBillingQueue: vi.fn() }));
vi.mock("../redis", () => ({
  redisEvictConnection: { set: vi.fn(), del: vi.fn(), eval: vi.fn() },
}));

import {
  processMonitorCheckJob,
  reconcileRunningMonitorChecks,
} from "./runner";
import * as store from "./store";
import { config } from "../../config";
import { getACUCTeam } from "../../controllers/auth";
import { autumnService } from "../autumn/autumn.service";
import { getBillingQueue } from "../queue-service";
import { redisEvictConnection } from "../redis";
import { sendMonitoringEmailSummary } from "../notification/monitoring_email";
import { recordMonitorInAppNotification } from "../notification/monitoring_in_app";
import { logger } from "../../lib/logger";
import { runSearchTarget } from "./search/run";

function deferred() {
  let resolve!: () => void;
  const promise = new Promise<void>(r => {
    resolve = r;
  });
  return { promise, resolve };
}

describe("monitor check finalization ownership", () => {
  let current: MonitorCheckRow;
  let snapshot: MonitorCheckRow;
  let monitor: MonitorRow;
  const locks = new Map<string, string>();
  const bill = vi.fn();
  const lockKey = "monitor-check-finalize:check-1";

  beforeEach(() => {
    vi.resetAllMocks();
    locks.clear();
    current = {
      id: "check-1",
      monitor_id: "monitor-1",
      team_id: "team-1",
      status: "running",
      billing_status: "reserved",
      autumn_lock_id: "monitor_check-1",
      reserved_credits: 1,
      started_at: new Date().toISOString(),
      target_results: [
        { targetId: "target-1", type: "scrape", expectedJobs: ["scrape-1"] },
      ],
    } as MonitorCheckRow;
    snapshot = structuredClone(current);
    monitor = {
      id: current.monitor_id,
      team_id: current.team_id,
      current_check_id: current.id,
      targets: [
        { id: "target-1", type: "scrape", urls: ["https://example.com"] },
      ],
    } as MonitorRow;
    vi.mocked(store.listRunningMonitorChecks).mockImplementation(async () => [
      structuredClone(snapshot),
    ]);
    vi.mocked(store.getMonitorCheckForUpdate).mockImplementation(async () =>
      structuredClone(current),
    );
    vi.mocked(store.getMonitorForUpdate).mockImplementation(
      async () => monitor,
    );
    vi.mocked(store.updateMonitorCheck).mockImplementation(
      async (_id, patch) => {
        current = { ...current, ...patch };
        return structuredClone(current);
      },
    );
    vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
      async (_id, patch) => {
        if (current.status !== "running") return null;
        current = { ...current, ...patch };
        return structuredClone(current);
      },
    );
    vi.mocked(store.updateMonitorCheckIfStatus).mockImplementation(
      async (_id, expectedStatus, patch) => {
        if (current.status !== expectedStatus) return null;
        current = { ...current, ...patch };
        return structuredClone(current);
      },
    );
    vi.mocked(store.countMonitorCheckPages).mockImplementation(
      async ({ status }) => (!status || status === "same" ? 1 : 0),
    );
    vi.mocked(store.calculateMonitorCheckActualCredits).mockResolvedValue(1);
    (config as { USE_DB_AUTHENTICATION?: boolean }).USE_DB_AUTHENTICATION =
      true;
    vi.mocked(getACUCTeam).mockResolvedValue({ org_id: "org-1" } as any);
    vi.mocked(autumnService.finalizeCreditsLock).mockResolvedValue(true);
    vi.mocked(getBillingQueue).mockReturnValue({
      add: bill,
    } as unknown as ReturnType<typeof getBillingQueue>);
    vi.mocked(redisEvictConnection.set).mockImplementation((async (
      key: string,
      token: string,
    ) => {
      // Notifications are tested independently; this suite exercises settlement and scheduling.
      if (key.startsWith("monitor-check-notify:")) return null;
      if (locks.has(key)) return null;
      locks.set(key, token);
      return "OK";
    }) as any);
    vi.mocked(redisEvictConnection.del).mockImplementation((async (
      key: string,
    ) => Number(locks.delete(key))) as any);
    vi.mocked(redisEvictConnection.eval).mockImplementation((async (
      _script: string,
      _keys: number,
      key: string,
      token: string,
    ) => {
      if (locks.get(key) !== token) return 0;
      return Number(locks.delete(key));
    }) as any);
  });

  it("requests the persisted credit estimate and skips the check when its hold is denied", async () => {
    current.status = "queued";
    current.autumn_lock_id = null;
    current.estimated_credits = 1009;
    vi.mocked(autumnService.lockCredits).mockResolvedValue({
      status: "denied",
    });
    await processMonitorCheckJob({
      checkId: current.id,
      monitorId: monitor.id,
      teamId: monitor.team_id,
    });
    expect(autumnService.lockCredits).toHaveBeenCalledWith(
      expect.objectContaining({
        value: 1009,
        lockId: "monitor_check-1",
      }),
    );
    expect(current).toMatchObject({
      status: "skipped_no_credits",
      actual_credits: 0,
      billing_status: "not_applicable",
    });
    expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
    expect(bill).not.toHaveBeenCalled();
  });

  it("settles and schedules a completed check once when another batch retained its running snapshot", async () => {
    await reconcileRunningMonitorChecks();
    await reconcileRunningMonitorChecks();
    expect(current).toMatchObject({
      status: "completed",
      billing_status: "confirmed",
    });
    expect(autumnService.finalizeCreditsLock).toHaveBeenCalledTimes(1);
    expect(autumnService.finalizeCreditsLock).toHaveBeenCalledWith(
      expect.objectContaining({
        lockId: "monitor_check-1",
        action: "confirm",
        overrideValue: 1,
      }),
    );
    expect(bill).toHaveBeenCalledTimes(1);
    expect(bill).toHaveBeenCalledWith(
      "bill_team",
      expect.objectContaining({ team_id: "team-1", org_id: "org-1" }),
      expect.anything(),
    );
    expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);
  });

  it("records the in-app notification even when the email channel throws", async () => {
    const lockSet = vi
      .mocked(redisEvictConnection.set)
      .getMockImplementation()!;
    vi.mocked(redisEvictConnection.set).mockImplementation(((
      key: string,
      ...rest: unknown[]
    ) =>
      key.startsWith("monitor-check-notify:")
        ? Promise.resolve("OK")
        : (lockSet as any)(key, ...rest)) as any);
    vi.mocked(store.listMonitorCheckPages).mockResolvedValue([] as any);
    vi.mocked(recordMonitorInAppNotification).mockResolvedValue({
      attempted: true,
      success: true,
    });
    vi.mocked(sendMonitoringEmailSummary).mockRejectedValue(
      new Error("recipient lookup failed"),
    );

    await reconcileRunningMonitorChecks();

    expect(sendMonitoringEmailSummary).toHaveBeenCalledTimes(1);
    expect(recordMonitorInAppNotification).toHaveBeenCalledWith(
      expect.objectContaining({
        check: expect.objectContaining({ id: "check-1" }),
      }),
    );
    expect(current.status).toBe("completed");
  });

  it("logs the counts it compared when a check is not ready to finalize", async () => {
    current.target_results = [
      {
        targetId: "target-1",
        type: "scrape",
        expectedJobs: ["scrape-1", "scrape-2"],
      },
    ];
    await reconcileRunningMonitorChecks();
    expect(current.status).toBe("running");
    expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
    expect(logger.info).toHaveBeenCalledWith(
      "Monitor check not ready to finalize",
      expect.objectContaining({
        checkId: "check-1",
        type: "scrape",
        targetId: "target-1",
        recorded: 1,
        expected: 2,
      }),
    );
  });

  describe("search-only checks", () => {
    const runJob = () =>
      processMonitorCheckJob({
        checkId: current.id,
        monitorId: monitor.id,
        teamId: monitor.team_id,
      });

    beforeEach(() => {
      current.target_results = [
        {
          type: "search",
          targetId: "target-1",
          searchCompleted: true,
          searchCredits: 1,
        },
      ];
      snapshot = structuredClone(current);
      monitor = {
        ...monitor,
        targets: [{ id: "target-1", type: "search", queries: ["firecrawl"] }],
      } as unknown as MonitorRow;
      vi.mocked(autumnService.lockCredits).mockResolvedValue({
        status: "locked",
        lockId: "monitor_check-1",
      });
    });

    it("runs the search and settles its stamped credits in the same job", async () => {
      current.status = "queued";
      current.target_results = [];
      vi.mocked(store.listActiveMonitorPages).mockResolvedValue([]);
      vi.mocked(runSearchTarget).mockResolvedValue({
        pageUpserts: [],
        resultCount: 5,
        matches: 0,
        summary: "",
        judgeDegraded: false,
        degradedReason: null,
        searchCredits: 3,
        judgeCredits: 2,
        resultsJudged: 2,
      } as unknown as Awaited<ReturnType<typeof runSearchTarget>>);

      await runJob();

      expect(runSearchTarget).toHaveBeenCalledTimes(1);
      expect(store.calculateMonitorCheckActualCredits).toHaveBeenCalledWith(
        expect.objectContaining({
          targetResults: [
            expect.objectContaining({
              searchCompleted: true,
              searchCredits: 3,
              judgeCredits: 2,
            }),
          ],
        }),
      );
      expect(current).toMatchObject({
        status: "completed",
        billing_status: "confirmed",
      });
      expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
        expect.objectContaining({
          lockId: "monitor_check-1",
          action: "confirm",
        }),
      );
      expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);
    });

    it("settles a redelivered search once without waiting for the reconciler", async () => {
      await runJob();
      expect(runSearchTarget).not.toHaveBeenCalled();
      expect(current).toMatchObject({
        status: "completed",
        billing_status: "confirmed",
      });
      await reconcileRunningMonitorChecks();
      expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
        expect.objectContaining({
          lockId: "monitor_check-1",
          action: "confirm",
        }),
      );
      expect(bill).toHaveBeenCalledTimes(1);
      expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);
    });

    it.each([
      {
        name: "another worker holds the finalize lease",
        setup: () => {
          locks.set(lockKey, "other-worker");
        },
      },
      {
        name: "taking the finalize lease throws",
        setup: () => {
          vi.mocked(redisEvictConnection.set).mockRejectedValueOnce(
            new Error("Redis unavailable"),
          );
        },
      },
      {
        name: "settlement preparation throws",
        setup: () => {
          vi.mocked(
            store.calculateMonitorCheckActualCredits,
          ).mockRejectedValueOnce(new Error("Page count failed"));
        },
      },
    ])(
      "leaves the check running for the reconciler when $name",
      async ({ setup }) => {
        setup();
        await expect(runJob()).resolves.toBeUndefined();
        expect(current).toMatchObject({
          status: "running",
          billing_status: "reserved",
        });
        expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();

        locks.delete(lockKey);
        await reconcileRunningMonitorChecks();
        expect(current).toMatchObject({
          status: "completed",
          billing_status: "confirmed",
        });
        expect(autumnService.finalizeCreditsLock).toHaveBeenCalledTimes(1);
      },
    );
  });

  it("uses the refreshed target state instead of completing an obsolete snapshot", async () => {
    current.target_results = [
      { type: "search", targetId: "target-1", searchCompleted: false },
    ];
    await reconcileRunningMonitorChecks();
    expect(current.status).toBe("running");
    expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
  });

  it.each(["completed", "partial", "failed"] as const)(
    "leaves an already %s check untouched",
    async status => {
      current.status = status;
      current.billing_status = "confirmed";
      await reconcileRunningMonitorChecks();
      expect(store.updateMonitorCheck).not.toHaveBeenCalled();
      expect(store.updateMonitorCheckIfRunning).not.toHaveBeenCalled();
      expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
    },
  );

  it("skips a deleted check and releases its Redis lock", async () => {
    vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValue(null);
    await reconcileRunningMonitorChecks();
    expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
    expect(locks.has(lockKey)).toBe(false);
  });

  it("allows only one settlement when the Redis lease expires while a worker is still preparing completion", async () => {
    const entered = deferred();
    const resume = deferred();
    vi.mocked(store.calculateMonitorCheckActualCredits).mockImplementationOnce(
      async () => {
        entered.resolve();
        await resume.promise;
        return 1;
      },
    );
    const first = reconcileRunningMonitorChecks();
    await entered.promise;
    locks.delete(lockKey); // The second worker can acquire an expired lease.
    vi.resetModules();
    const otherProcess = await import("./runner.js");
    await otherProcess.reconcileRunningMonitorChecks();
    resume.resolve();
    await first;
    expect(autumnService.finalizeCreditsLock).toHaveBeenCalledTimes(1);
    expect(bill).toHaveBeenCalledTimes(1);
    expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);
    expect(current.billing_status).toBe("confirmed");
  });

  it("does not delete a replacement owner's Redis lock", async () => {
    vi.mocked(store.calculateMonitorCheckActualCredits).mockImplementation(
      async () => {
        locks.set(lockKey, "replacement-owner");
        return 1;
      },
    );
    await reconcileRunningMonitorChecks();
    expect(locks.get(lockKey)).toBe("replacement-owner");
  });

  it.each(["stale", "orphan"])(
    "does not release a %s check whose terminal claim was won by another worker",
    async kind => {
      if (kind === "stale") {
        current.started_at = new Date(
          Date.now() - 2 * 60 * 60 * 1000,
        ).toISOString();
        snapshot = structuredClone(current);
      } else {
        vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);
      }
      vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
        async () => {
          current = {
            ...current,
            status: "completed",
            billing_status: "confirmed",
          };
          return null;
        },
      );
      await reconcileRunningMonitorChecks();
      expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
      expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();
      expect(current).toMatchObject({
        status: "completed",
        billing_status: "confirmed",
      });
    },
  );

  it("records a failed settlement without enqueueing billing", async () => {
    vi.mocked(autumnService.finalizeCreditsLock).mockResolvedValue(false);
    await reconcileRunningMonitorChecks();
    expect(current).toMatchObject({
      status: "completed",
      billing_status: "failed",
    });
    expect(bill).not.toHaveBeenCalled();
  });

  it.each(["stale", "orphan"])(
    "releases a %s hold after winning the failure transition",
    async kind => {
      if (kind === "stale") {
        current.started_at = new Date(
          Date.now() - 2 * 60 * 60 * 1000,
        ).toISOString();
      } else {
        vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);
      }
      await reconcileRunningMonitorChecks();
      expect(current).toMatchObject({
        status: "failed",
        billing_status: "released",
      });
      expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
        expect.objectContaining({
          action: "release",
          lockId: "monitor_check-1",
        }),
      );
      expect(
        vi.mocked(store.updateMonitorCheckIfRunning).mock
          .invocationCallOrder[0],
      ).toBeLessThan(
        vi.mocked(autumnService.finalizeCreditsLock).mock
          .invocationCallOrder[0],
      );
    },
  );

  it.each([
    {
      name: "answers no org",
      acuc: async () => ({ org_id: null }) as any,
    },
    {
      name: "cannot be read",
      acuc: async () => {
        throw new Error("ACUC unavailable");
      },
    },
  ])(
    "releases a stale hold with no team when the ACUC $name",
    async ({ acuc }) => {
      vi.mocked(getACUCTeam).mockImplementation(acuc);
      current.started_at = new Date(
        Date.now() - 2 * 60 * 60 * 1000,
      ).toISOString();

      await reconcileRunningMonitorChecks();

      expect(current).toMatchObject({
        status: "failed",
        billing_status: "released",
      });
      // Fail open: the release still happens, just without a team to name.
      const calls = vi.mocked(autumnService.finalizeCreditsLock).mock.calls;
      expect(calls).toHaveLength(1);
      expect(calls[0][0]).toMatchObject({
        action: "release",
        lockId: "monitor_check-1",
      });
      expect(calls[0][0].team).toBeUndefined();
    },
  );

  it("names no team when DB authentication is off", async () => {
    // The mock ACUC's org is the sentinel "bypass", so the ACUC is not even
    // read: releasing against "bypass" would name a customer that isn't one.
    (config as { USE_DB_AUTHENTICATION?: boolean }).USE_DB_AUTHENTICATION =
      false;
    vi.mocked(getACUCTeam).mockResolvedValue({
      team_id: "bypass",
      org_id: "bypass",
    } as any);
    current.started_at = new Date(
      Date.now() - 2 * 60 * 60 * 1000,
    ).toISOString();

    await reconcileRunningMonitorChecks();

    const calls = vi.mocked(autumnService.finalizeCreditsLock).mock.calls;
    expect(calls).toHaveLength(1);
    expect(calls[0][0].team).toBeUndefined();
    expect(getACUCTeam).not.toHaveBeenCalled();
  });

  it.each([
    { kind: "stale", throws: false },
    { kind: "stale", throws: true },
    { kind: "orphan", throws: false },
    { kind: "orphan", throws: true },
    { kind: "handler", throws: false },
    { kind: "handler", throws: true },
  ])(
    "records failed settlement for a $kind release that throws: $throws",
    async ({ kind, throws }) => {
      if (throws) {
        vi.mocked(autumnService.finalizeCreditsLock).mockRejectedValue(
          new Error("Release unavailable"),
        );
      } else {
        vi.mocked(autumnService.finalizeCreditsLock).mockResolvedValue(false);
      }
      if (kind === "handler") {
        monitor.targets = [];
        const failure = new Error("Target write failed");
        vi.mocked(autumnService.lockCredits).mockResolvedValue({
          status: "locked",
          lockId: "monitor_check-1",
        });
        const updateIfRunning = vi
          .mocked(store.updateMonitorCheckIfRunning)
          .getMockImplementation()!;
        vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
          async (id, patch) => {
            if (patch.target_results) throw failure;
            return updateIfRunning(id, patch);
          },
        );
        await expect(
          processMonitorCheckJob({
            checkId: current.id,
            monitorId: monitor.id,
            teamId: monitor.team_id,
          }),
        ).rejects.toThrow(failure);
      } else {
        if (kind === "stale") {
          current.started_at = new Date(
            Date.now() - 2 * 60 * 60 * 1000,
          ).toISOString();
        } else {
          vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);
        }
        await reconcileRunningMonitorChecks();
      }
      expect(current).toMatchObject({
        status: "failed",
        billing_status: "failed",
      });
      expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
        expect.objectContaining({
          lockId: "monitor_check-1",
          action: "release",
        }),
      );
      expect(bill).not.toHaveBeenCalled();
    },
  );

  it.each(["completed", "stale", "orphan"])(
    "settles the hold returned by the terminal claim for a %s check",
    async kind => {
      current.autumn_lock_id = null;
      current.billing_status = "not_applicable";
      if (kind === "stale") {
        current.started_at = new Date(
          Date.now() - 2 * 60 * 60 * 1000,
        ).toISOString();
      } else if (kind === "orphan") {
        vi.mocked(store.getMonitorForUpdate).mockResolvedValue(null);
      }
      const updateIfRunning = vi
        .mocked(store.updateMonitorCheckIfRunning)
        .getMockImplementation()!;
      vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
        async (id, patch) => {
          if (patch.status) {
            // Reservation persisted after the primary read, just before the claim.
            current = {
              ...current,
              autumn_lock_id: "late-lock",
              reserved_credits: 2,
              partner_run_token: "late-token",
              billing_status: "reserved",
            };
          }
          return updateIfRunning(id, patch);
        },
      );
      await reconcileRunningMonitorChecks();
      expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
        expect.objectContaining({
          lockId: "late-lock",
          action: kind === "completed" ? "confirm" : "release",
          ...(kind === "completed"
            ? { externalRequestId: "late-token", heldValue: 2 }
            : {}),
        }),
      );
      expect(current.billing_status).toBe(
        kind === "completed" ? "confirmed" : "released",
      );
    },
  );

  it("does not reopen a check that became terminal after the job handler read it", async () => {
    const queued = { ...current, status: "queued" as const };
    vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValue(queued);
    current.status = "completed";
    current.billing_status = "confirmed";
    await processMonitorCheckJob({
      checkId: current.id,
      monitorId: monitor.id,
      teamId: monitor.team_id,
    });
    expect(current).toMatchObject({
      status: "completed",
      billing_status: "confirmed",
    });
    expect(store.markMonitorRunning).not.toHaveBeenCalled();
    expect(autumnService.lockCredits).not.toHaveBeenCalled();
  });

  it.each([
    { status: "locked" as const, lockId: "monitor_check-1" },
    { status: "skipped" as const },
    { status: "denied" as const },
  ])(
    "preserves completion when a slow reservation returns $status",
    async result => {
      monitor.targets = [];
      vi.mocked(autumnService.lockCredits).mockImplementation(async () => {
        current = {
          ...current,
          status: "completed",
          billing_status: "confirmed",
        };
        return result;
      });
      await processMonitorCheckJob({
        checkId: current.id,
        monitorId: monitor.id,
        teamId: monitor.team_id,
      });
      expect(current).toMatchObject({
        status: "completed",
        billing_status: "confirmed",
      });
      expect(store.updateMonitorCheck).not.toHaveBeenCalled();
      expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();
      expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
    },
  );
  it.each([
    {
      name: "an unowned hold",
      persistedLock: null,
      deleted: false,
      release: true,
    },
    {
      name: "a different hold",
      persistedLock: "other-lock",
      deleted: false,
      release: true,
    },
    {
      name: "the finalizer's hold",
      persistedLock: "monitor_check-1",
      deleted: false,
      release: false,
    },
    {
      name: "a deleted check's hold",
      persistedLock: null,
      deleted: true,
      release: true,
    },
  ])(
    "cleans up $name when a reservation arrives after completion",
    async ({ persistedLock, deleted, release }) => {
      current.autumn_lock_id = null;
      current.billing_status = "not_applicable";
      monitor.targets = [];
      vi.mocked(autumnService.lockCredits).mockImplementation(async () => {
        current = {
          ...current,
          status: "completed",
          billing_status: "confirmed",
          autumn_lock_id: persistedLock,
        };
        if (deleted) {
          vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValueOnce(null);
        }
        return {
          status: "locked",
          lockId: "monitor_check-1",
          operationToken: "operation-1",
        };
      });
      await processMonitorCheckJob({
        checkId: current.id,
        monitorId: monitor.id,
        teamId: monitor.team_id,
      });
      expect(current).toMatchObject({
        status: "completed",
        billing_status: "confirmed",
        autumn_lock_id: persistedLock,
      });
      expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();
      if (release) {
        expect(
          autumnService.finalizeCreditsLock,
        ).toHaveBeenCalledExactlyOnceWith({
          lockId: "monitor_check-1",
          action: "release",
          properties: {
            source: "monitorCheck",
            endpoint: "monitor",
            jobId: current.id,
          },
          team: { teamId: monitor.team_id, orgId: "org-1" },
        });
      } else {
        expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
      }
    },
  );

  it.each(["queued", "running"] as const)(
    "does not release a late reservation whose current row is %s",
    async status => {
      current.autumn_lock_id = null;
      vi.mocked(autumnService.lockCredits).mockImplementation(async () => {
        current.status = "completed";
        vi.mocked(store.getMonitorCheckForUpdate).mockResolvedValueOnce({
          ...current,
          status,
        });
        return { status: "locked", lockId: "monitor_check-1" };
      });
      await processMonitorCheckJob({
        checkId: current.id,
        monitorId: monitor.id,
        teamId: monitor.team_id,
      });
      expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
    },
  );

  it("does not release a late reservation without an authoritative ownership read", async () => {
    current.autumn_lock_id = null;
    const failure = new Error("Primary read unavailable");
    vi.mocked(autumnService.lockCredits).mockImplementation(async () => {
      current.status = "completed";
      vi.mocked(store.getMonitorCheckForUpdate).mockRejectedValue(failure);
      return { status: "locked", lockId: "monitor_check-1" };
    });
    await expect(
      processMonitorCheckJob({
        checkId: current.id,
        monitorId: monitor.id,
        teamId: monitor.team_id,
      }),
    ).rejects.toThrow(failure);
    expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
  });

  it.each([
    { localLock: "monitor_check-1", persistedLock: "other-lock" },
    { localLock: null, persistedLock: "other-lock" },
    { localLock: "monitor_check-1", persistedLock: null },
    { localLock: "monitor_check-1", persistedLock: "monitor_check-1" },
  ])(
    "releases owned holds after a handler failure with $localLock and $persistedLock",
    async ({ localLock, persistedLock }) => {
      monitor.targets = [];
      const failure = new Error("Target write failed");
      vi.mocked(autumnService.lockCredits).mockResolvedValue(
        localLock
          ? { status: "locked", lockId: localLock }
          : { status: "skipped" },
      );
      const updateIfRunning = vi
        .mocked(store.updateMonitorCheckIfRunning)
        .getMockImplementation()!;
      vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
        async (id, patch) => {
          if (patch.target_results) {
            current.autumn_lock_id = persistedLock;
            throw failure;
          }
          return updateIfRunning(id, patch);
        },
      );
      await expect(
        processMonitorCheckJob({
          checkId: current.id,
          monitorId: monitor.id,
          teamId: monitor.team_id,
        }),
      ).rejects.toThrow(failure);
      const expectedLocks = [
        ...new Set([localLock, persistedLock].filter(Boolean)),
      ].sort();
      expect(
        vi
          .mocked(autumnService.finalizeCreditsLock)
          .mock.calls.map(([call]) => call.lockId)
          .sort(),
      ).toEqual(expectedLocks);
      expect(
        vi
          .mocked(autumnService.finalizeCreditsLock)
          .mock.calls.every(([call]) => call.action === "release"),
      ).toBe(true);
      expect(current).toMatchObject({
        status: "failed",
        billing_status: "released",
      });
    },
  );

  it.each([false, true])(
    "releases an unpersisted hold when reservation storage throws and another finalizer wins: %s",
    async terminal => {
      current.autumn_lock_id = null;
      const failure = new Error("Reservation write failed");
      vi.mocked(autumnService.lockCredits).mockResolvedValue({
        status: "locked",
        lockId: "monitor_check-1",
      });
      const updateIfRunning = vi
        .mocked(store.updateMonitorCheckIfRunning)
        .getMockImplementation()!;
      vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
        async (id, patch) => {
          if (Object.hasOwn(patch, "autumn_lock_id")) {
            if (terminal) {
              current.status = "completed";
              current.billing_status = "not_applicable";
            }
            throw failure;
          }
          return updateIfRunning(id, patch);
        },
      );
      await expect(
        processMonitorCheckJob({
          checkId: current.id,
          monitorId: monitor.id,
          teamId: monitor.team_id,
        }),
      ).rejects.toThrow(failure);
      expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
        expect.objectContaining({
          lockId: "monitor_check-1",
          action: "release",
        }),
      );
      expect(current.status).toBe(terminal ? "completed" : "failed");
      if (terminal)
        expect(store.updateMonitorScheduleAfterRun).not.toHaveBeenCalled();
    },
  );

  it.each([1, 2])(
    "preserves finalized results when target write %i arrives late",
    async lateWrite => {
      monitor.targets = [];
      vi.mocked(autumnService.lockCredits).mockResolvedValue({
        status: "locked",
        lockId: "monitor_check-1",
      });
      const finalizedResults = [
        {
          type: "scrape",
          targetId: "target-1",
          expectedJobs: ["finished-scrape"],
        },
      ];
      let targetWrites = 0;
      const finishBeforeWrite = (patch: Partial<MonitorCheckRow>) => {
        if (patch.target_results && ++targetWrites === lateWrite) {
          current = {
            ...current,
            status: "completed",
            billing_status: "confirmed",
            target_results: finalizedResults,
          };
        }
      };
      const update = vi
        .mocked(store.updateMonitorCheck)
        .getMockImplementation()!;
      const updateIfRunning = vi
        .mocked(store.updateMonitorCheckIfRunning)
        .getMockImplementation()!;
      vi.mocked(store.updateMonitorCheck).mockImplementation(
        async (id, patch) => {
          finishBeforeWrite(patch);
          return update(id, patch);
        },
      );
      vi.mocked(store.updateMonitorCheckIfRunning).mockImplementation(
        async (id, patch) => {
          finishBeforeWrite(patch);
          return updateIfRunning(id, patch);
        },
      );
      await processMonitorCheckJob({
        checkId: current.id,
        monitorId: monitor.id,
        teamId: monitor.team_id,
      });
      expect(current).toMatchObject({
        status: "completed",
        billing_status: "confirmed",
        target_results: finalizedResults,
      });
      expect(targetWrites).toBe(lateWrite);
    },
  );
});