Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): finalize search-only monitor checks inline and log unready checks (#5194)

Gergő Móriczdcfed9dab6ed
diff --git a/apps/api/src/services/monitoring/runner.finalization.test.ts b/apps/api/src/services/monitoring/runner.finalization.test.ts
index cbc2d07e4..7c7815ea9 100644
--- a/apps/api/src/services/monitoring/runner.finalization.test.ts
+++ b/apps/api/src/services/monitoring/runner.finalization.test.ts
@@ -31,10 +31,13 @@ vi.mock("./types", () => ({}));
vi.mock("./interest", () => ({
trackMonitorCheckStartedInterest: async () => {},
}));
-vi.mock("./search/run", () => ({}));
+vi.mock("./search/run", () => ({ runSearchTarget: vi.fn() }));
vi.mock("./search/judge", () => ({}));
-vi.mock("./search/dedupe", () => ({}));
-vi.mock("./search/persist", () => ({}));
+vi.mock("./search/dedupe", () => ({ computeGoalVersion: () => "goal-1" }));
+vi.mock("./search/persist", () => ({
+ reconstructKnownState: () => ({ knownPages: new Map(), knownEvents: [] }),
+ searchStatusToPageStatus: (status: string) => status,
+}));
vi.mock("../../scraper/WebScraper/utils/blocklist", () => ({}));
vi.mock("../../controllers/auth", () => ({ getACUCTeam: vi.fn() }));
vi.mock("./store", () => ({
@@ -49,6 +52,10 @@ vi.mock("./store", () => ({
listMonitorCheckPages: vi.fn(),
calculateMonitorCheckActualCredits: vi.fn(),
updateMonitorScheduleAfterRun: vi.fn(),
+ listActiveMonitorPages: vi.fn(),
+ deleteMonitorCheckPages: vi.fn(),
+ insertMonitorCheckPages: vi.fn(),
+ bulkUpsertMonitorPages: vi.fn(),
}));
vi.mock("../autumn/autumn.service", () => ({
autumnService: { finalizeCreditsLock: vi.fn(), lockCredits: vi.fn() },
@@ -70,6 +77,8 @@ import { getBillingQueue } from "../queue-service";
import { redisEvictConnection } from "../redis";
import { sendMonitoringEmailSummary } from "../notification/monitoring_email";
import { recordMonitorInAppNotification } from "../notification/monitoring_in_app";
+import { logger } from "../../lib/logger";
+import { runSearchTarget } from "./search/run";
function deferred() {
let resolve!: () => void;
@@ -258,6 +267,163 @@ describe("monitor check finalization ownership", () => {
expect(current.status).toBe("completed");
});
+ it("logs the counts it compared when a check is not ready to finalize", async () => {
+ current.target_results = [
+ {
+ targetId: "target-1",
+ type: "scrape",
+ expectedJobs: ["scrape-1", "scrape-2"],
+ },
+ ];
+ await reconcileRunningMonitorChecks();
+ expect(current.status).toBe("running");
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+ expect(logger.info).toHaveBeenCalledWith(
+ "Monitor check not ready to finalize",
+ expect.objectContaining({
+ checkId: "check-1",
+ type: "scrape",
+ targetId: "target-1",
+ recorded: 1,
+ expected: 2,
+ }),
+ );
+ });
+
+ describe("search-only checks", () => {
+ const runJob = () =>
+ processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ });
+
+ beforeEach(() => {
+ current.target_results = [
+ {
+ type: "search",
+ targetId: "target-1",
+ searchCompleted: true,
+ searchCredits: 1,
+ },
+ ];
+ snapshot = structuredClone(current);
+ monitor = {
+ ...monitor,
+ targets: [{ id: "target-1", type: "search", queries: ["firecrawl"] }],
+ } as unknown as MonitorRow;
+ vi.mocked(autumnService.lockCredits).mockResolvedValue({
+ status: "locked",
+ lockId: "monitor_check-1",
+ });
+ });
+
+ it("runs the search and settles its stamped credits in the same job", async () => {
+ current.status = "queued";
+ current.target_results = [];
+ vi.mocked(store.listActiveMonitorPages).mockResolvedValue([]);
+ vi.mocked(runSearchTarget).mockResolvedValue({
+ pageUpserts: [],
+ resultCount: 5,
+ matches: 0,
+ summary: "",
+ judgeDegraded: false,
+ degradedReason: null,
+ searchCredits: 3,
+ judgeCredits: 2,
+ resultsJudged: 2,
+ } as unknown as Awaited<ReturnType<typeof runSearchTarget>>);
+
+ await runJob();
+
+ expect(runSearchTarget).toHaveBeenCalledTimes(1);
+ expect(store.calculateMonitorCheckActualCredits).toHaveBeenCalledWith(
+ expect.objectContaining({
+ targetResults: [
+ expect.objectContaining({
+ searchCompleted: true,
+ searchCredits: 3,
+ judgeCredits: 2,
+ }),
+ ],
+ }),
+ );
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "confirmed",
+ });
+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ }),
+ );
+ expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);
+ });
+
+ it("settles a redelivered search once without waiting for the reconciler", async () => {
+ await runJob();
+ expect(runSearchTarget).not.toHaveBeenCalled();
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "confirmed",
+ });
+ await reconcileRunningMonitorChecks();
+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ lockId: "monitor_check-1",
+ action: "confirm",
+ }),
+ );
+ expect(bill).toHaveBeenCalledTimes(1);
+ expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);
+ });
+
+ it.each([
+ {
+ name: "another worker holds the finalize lease",
+ setup: () => {
+ locks.set(lockKey, "other-worker");
+ },
+ },
+ {
+ name: "taking the finalize lease throws",
+ setup: () => {
+ vi.mocked(redisEvictConnection.set).mockRejectedValueOnce(
+ new Error("Redis unavailable"),
+ );
+ },
+ },
+ {
+ name: "settlement preparation throws",
+ setup: () => {
+ vi.mocked(
+ store.calculateMonitorCheckActualCredits,
+ ).mockRejectedValueOnce(new Error("Page count failed"));
+ },
+ },
+ ])(
+ "leaves the check running for the reconciler when $name",
+ async ({ setup }) => {
+ setup();
+ await expect(runJob()).resolves.toBeUndefined();
+ expect(current).toMatchObject({
+ status: "running",
+ billing_status: "reserved",
+ });
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+
+ locks.delete(lockKey);
+ await reconcileRunningMonitorChecks();
+ expect(current).toMatchObject({
+ status: "completed",
+ billing_status: "confirmed",
+ });
+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledTimes(1);
+ },
+ );
+ });
+
it("uses the refreshed target state instead of completing an obsolete snapshot", async () => {
current.target_results = [
{ type: "search", targetId: "target-1", searchCompleted: false },
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 9968d3033..44ae3174d 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -1233,9 +1233,21 @@ export async function processMonitorCheckJob(
}
}
- await updateMonitorCheckIfRunning(check.id, {
+ const stored = await updateMonitorCheckIfRunning(check.id, {
target_results: targetResults,
});
+
+ // Search targets finish inline, so a search-only check is complete here.
+ // Any failure or lost lease leaves it to the reconciler, as before.
+ if (stored && monitor.targets.every(target => target.type === "search")) {
+ await reconcileMonitorCheckCandidate(stored, () => {}).catch(error =>
+ logger.warn("Failed to finalize monitor check inline", {
+ error,
+ monitorId: monitor.id,
+ checkId: check.id,
+ }),
+ );
+ }
} catch (error) {
if (error instanceof ExistingCreditsLockError) throw error;
// Atomically flip running -> failed. Returns null when the check already
@@ -1393,28 +1405,36 @@ async function processRemovedPagesForCompletedCrawls(params: {
}
}
-async function isMonitorCheckComplete(
+// The first target still in flight, with the figures that decided it, or null
+// once every target is done.
+async function findUnfinishedMonitorTarget(
check: MonitorCheckRow,
monitor?: MonitorRow,
-): Promise<boolean> {
+): Promise<Record<string, unknown> | null> {
let targetResults = Array.isArray(check.target_results)
? (check.target_results as any[])
: [];
if (targetResults.length === 0) {
- if (!monitor) return false;
+ if (!monitor) return { reason: "no_target_results" };
targetResults = await recoverTargetRunsFromRecordedPages({
monitor,
check,
});
- if (targetResults.length === 0) return false;
+ if (targetResults.length === 0) return { reason: "no_target_results" };
}
for (const target of targetResults) {
if (target?.type === "search") {
// Not complete until the inline search has stamped its credits.
- if (!target.searchCompleted) return false;
+ if (!target.searchCompleted) {
+ return {
+ type: "search",
+ targetId: target.targetId,
+ searchCompleted: false,
+ };
+ }
} else if (target?.type === "scrape") {
const expected = Array.isArray(target.expectedJobs)
? target.expectedJobs.length
@@ -1423,22 +1443,47 @@ async function isMonitorCheckComplete(
checkId: check.id,
targetId: target.targetId,
});
- if (recorded < expected) return false;
+ if (recorded < expected) {
+ return {
+ type: "scrape",
+ targetId: target.targetId,
+ recorded,
+ expected,
+ };
+ }
} else if (target?.type === "crawl") {
const group = await crawlGroup.getGroup(target.crawlId);
- if (!group || group.status === "active") return false;
+ if (!group) {
+ return {
+ type: "crawl",
+ targetId: target.targetId,
+ crawlId: target.crawlId,
+ groupStatus: null,
+ };
+ }
const stats = await scrapeQueue.getGroupNumericStats(
target.crawlId,
logger,
);
- const unfinished =
- (stats.active ?? 0) + (stats.queued ?? 0) + (stats.backlog ?? 0);
- if (unfinished > 0) return false;
+ const active = stats.active ?? 0;
+ const queued = stats.queued ?? 0;
+ const backlog = stats.backlog ?? 0;
+ if (group.status === "active" || active + queued + backlog > 0) {
+ return {
+ type: "crawl",
+ targetId: target.targetId,
+ crawlId: target.crawlId,
+ groupStatus: group.status,
+ active,
+ queued,
+ backlog,
+ };
+ }
}
}
- return true;
+ return null;
}
async function failStaleMonitorCheck(params: {
@@ -1733,15 +1778,22 @@ async function reconcileMonitorCheckCandidate(
targetResults,
});
- if (
- !(await isMonitorCheckComplete(
- {
- ...check,
- target_results: targetResults,
- },
- monitor,
- ))
- ) {
+ const unfinished = await findUnfinishedMonitorTarget(
+ {
+ ...check,
+ target_results: targetResults,
+ },
+ monitor,
+ );
+ if (unfinished) {
+ logger.info("Monitor check not ready to finalize", {
+ monitorId: monitor.id,
+ checkId: check.id,
+ runningForMs: check.started_at
+ ? Date.now() - Date.parse(check.started_at)
+ : null,
+ ...unfinished,
+ });
// Only persist target_results we recovered from an empty snapshot. Writing
// back a non-empty stale snapshot here can DOWNGRADE a searchCompleted=true
// that the inline handler persisted after this reconciler loaded its
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index c6bcefaad..18253c613 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -605,7 +605,7 @@ describe("FLAT deterministic search-monitor billing", () => {
it("a still-running search (no credits stamped yet) sums to 0 — the reconciler must NOT finalize here", () => {
// Pre-stamp shape the dispatcher writes before the inline search finishes.
- // The never-zero guarantee relies on isMonitorCheckComplete refusing to
+ // The never-zero guarantee relies on findUnfinishedMonitorTarget refusing to
// finalize a run whose searchCompleted flag is not yet true (see runner.ts).
const pending = { targetId: "search-1", type: "search" as const };
expect((pending as any).searchCompleted).toBeUndefined();