Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): finalize search-only monitor checks inline and log unready checks (#5194)
Gergő Móriczdcfed9dab6ed
diff --git a/apps/api/src/services/monitoring/runner.finalization.test.ts b/apps/api/src/services/monitoring/runner.finalization.test.tsindex cbc2d07e4..7c7815ea9 100644--- a/apps/api/src/services/monitoring/runner.finalization.test.ts+++ b/apps/api/src/services/monitoring/runner.finalization.test.ts@@ -31,10 +31,13 @@ vi.mock("./types", () => ({}));vi.mock("./interest", () => ({trackMonitorCheckStartedInterest: async () => {},}));-vi.mock("./search/run", () => ({}));+vi.mock("./search/run", () => ({ runSearchTarget: vi.fn() }));vi.mock("./search/judge", () => ({}));-vi.mock("./search/dedupe", () => ({}));-vi.mock("./search/persist", () => ({}));+vi.mock("./search/dedupe", () => ({ computeGoalVersion: () => "goal-1" }));+vi.mock("./search/persist", () => ({+ reconstructKnownState: () => ({ knownPages: new Map(), knownEvents: [] }),+ searchStatusToPageStatus: (status: string) => status,+}));vi.mock("../../scraper/WebScraper/utils/blocklist", () => ({}));vi.mock("../../controllers/auth", () => ({ getACUCTeam: vi.fn() }));vi.mock("./store", () => ({@@ -49,6 +52,10 @@ vi.mock("./store", () => ({listMonitorCheckPages: vi.fn(),calculateMonitorCheckActualCredits: vi.fn(),updateMonitorScheduleAfterRun: vi.fn(),+ listActiveMonitorPages: vi.fn(),+ deleteMonitorCheckPages: vi.fn(),+ insertMonitorCheckPages: vi.fn(),+ bulkUpsertMonitorPages: vi.fn(),}));vi.mock("../autumn/autumn.service", () => ({autumnService: { finalizeCreditsLock: vi.fn(), lockCredits: vi.fn() },@@ -70,6 +77,8 @@ import { getBillingQueue } from "../queue-service";import { redisEvictConnection } from "../redis";import { sendMonitoringEmailSummary } from "../notification/monitoring_email";import { recordMonitorInAppNotification } from "../notification/monitoring_in_app";+import { logger } from "../../lib/logger";+import { runSearchTarget } from "./search/run";function deferred() {let resolve!: () => void;@@ -258,6 +267,163 @@ describe("monitor check finalization ownership", () => {expect(current.status).toBe("completed");});+ it("logs the counts it compared when a check is not ready to finalize", async () => {+ current.target_results = [+ {+ targetId: "target-1",+ type: "scrape",+ expectedJobs: ["scrape-1", "scrape-2"],+ },+ ];+ await reconcileRunningMonitorChecks();+ expect(current.status).toBe("running");+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();+ expect(logger.info).toHaveBeenCalledWith(+ "Monitor check not ready to finalize",+ expect.objectContaining({+ checkId: "check-1",+ type: "scrape",+ targetId: "target-1",+ recorded: 1,+ expected: 2,+ }),+ );+ });++ describe("search-only checks", () => {+ const runJob = () =>+ processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ });++ beforeEach(() => {+ current.target_results = [+ {+ type: "search",+ targetId: "target-1",+ searchCompleted: true,+ searchCredits: 1,+ },+ ];+ snapshot = structuredClone(current);+ monitor = {+ ...monitor,+ targets: [{ id: "target-1", type: "search", queries: ["firecrawl"] }],+ } as unknown as MonitorRow;+ vi.mocked(autumnService.lockCredits).mockResolvedValue({+ status: "locked",+ lockId: "monitor_check-1",+ });+ });++ it("runs the search and settles its stamped credits in the same job", async () => {+ current.status = "queued";+ current.target_results = [];+ vi.mocked(store.listActiveMonitorPages).mockResolvedValue([]);+ vi.mocked(runSearchTarget).mockResolvedValue({+ pageUpserts: [],+ resultCount: 5,+ matches: 0,+ summary: "",+ judgeDegraded: false,+ degradedReason: null,+ searchCredits: 3,+ judgeCredits: 2,+ resultsJudged: 2,+ } as unknown as Awaited<ReturnType<typeof runSearchTarget>>);++ await runJob();++ expect(runSearchTarget).toHaveBeenCalledTimes(1);+ expect(store.calculateMonitorCheckActualCredits).toHaveBeenCalledWith(+ expect.objectContaining({+ targetResults: [+ expect.objectContaining({+ searchCompleted: true,+ searchCredits: 3,+ judgeCredits: 2,+ }),+ ],+ }),+ );+ expect(current).toMatchObject({+ status: "completed",+ billing_status: "confirmed",+ });+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({+ lockId: "monitor_check-1",+ action: "confirm",+ }),+ );+ expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);+ });++ it("settles a redelivered search once without waiting for the reconciler", async () => {+ await runJob();+ expect(runSearchTarget).not.toHaveBeenCalled();+ expect(current).toMatchObject({+ status: "completed",+ billing_status: "confirmed",+ });+ await reconcileRunningMonitorChecks();+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({+ lockId: "monitor_check-1",+ action: "confirm",+ }),+ );+ expect(bill).toHaveBeenCalledTimes(1);+ expect(store.updateMonitorScheduleAfterRun).toHaveBeenCalledTimes(1);+ });++ it.each([+ {+ name: "another worker holds the finalize lease",+ setup: () => {+ locks.set(lockKey, "other-worker");+ },+ },+ {+ name: "taking the finalize lease throws",+ setup: () => {+ vi.mocked(redisEvictConnection.set).mockRejectedValueOnce(+ new Error("Redis unavailable"),+ );+ },+ },+ {+ name: "settlement preparation throws",+ setup: () => {+ vi.mocked(+ store.calculateMonitorCheckActualCredits,+ ).mockRejectedValueOnce(new Error("Page count failed"));+ },+ },+ ])(+ "leaves the check running for the reconciler when $name",+ async ({ setup }) => {+ setup();+ await expect(runJob()).resolves.toBeUndefined();+ expect(current).toMatchObject({+ status: "running",+ billing_status: "reserved",+ });+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();++ locks.delete(lockKey);+ await reconcileRunningMonitorChecks();+ expect(current).toMatchObject({+ status: "completed",+ billing_status: "confirmed",+ });+ expect(autumnService.finalizeCreditsLock).toHaveBeenCalledTimes(1);+ },+ );+ });+it("uses the refreshed target state instead of completing an obsolete snapshot", async () => {current.target_results = [{ type: "search", targetId: "target-1", searchCompleted: false },diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 9968d3033..44ae3174d 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -1233,9 +1233,21 @@ export async function processMonitorCheckJob(}}- await updateMonitorCheckIfRunning(check.id, {+ const stored = await updateMonitorCheckIfRunning(check.id, {target_results: targetResults,});++ // Search targets finish inline, so a search-only check is complete here.+ // Any failure or lost lease leaves it to the reconciler, as before.+ if (stored && monitor.targets.every(target => target.type === "search")) {+ await reconcileMonitorCheckCandidate(stored, () => {}).catch(error =>+ logger.warn("Failed to finalize monitor check inline", {+ error,+ monitorId: monitor.id,+ checkId: check.id,+ }),+ );+ }} catch (error) {if (error instanceof ExistingCreditsLockError) throw error;// Atomically flip running -> failed. Returns null when the check already@@ -1393,28 +1405,36 @@ async function processRemovedPagesForCompletedCrawls(params: {}}-async function isMonitorCheckComplete(+// The first target still in flight, with the figures that decided it, or null+// once every target is done.+async function findUnfinishedMonitorTarget(check: MonitorCheckRow,monitor?: MonitorRow,-): Promise<boolean> {+): Promise<Record<string, unknown> | null> {let targetResults = Array.isArray(check.target_results)? (check.target_results as any[]): [];if (targetResults.length === 0) {- if (!monitor) return false;+ if (!monitor) return { reason: "no_target_results" };targetResults = await recoverTargetRunsFromRecordedPages({monitor,check,});- if (targetResults.length === 0) return false;+ if (targetResults.length === 0) return { reason: "no_target_results" };}for (const target of targetResults) {if (target?.type === "search") {// Not complete until the inline search has stamped its credits.- if (!target.searchCompleted) return false;+ if (!target.searchCompleted) {+ return {+ type: "search",+ targetId: target.targetId,+ searchCompleted: false,+ };+ }} else if (target?.type === "scrape") {const expected = Array.isArray(target.expectedJobs)? target.expectedJobs.length@@ -1423,22 +1443,47 @@ async function isMonitorCheckComplete(checkId: check.id,targetId: target.targetId,});- if (recorded < expected) return false;+ if (recorded < expected) {+ return {+ type: "scrape",+ targetId: target.targetId,+ recorded,+ expected,+ };+ }} else if (target?.type === "crawl") {const group = await crawlGroup.getGroup(target.crawlId);- if (!group || group.status === "active") return false;+ if (!group) {+ return {+ type: "crawl",+ targetId: target.targetId,+ crawlId: target.crawlId,+ groupStatus: null,+ };+ }const stats = await scrapeQueue.getGroupNumericStats(target.crawlId,logger,);- const unfinished =- (stats.active ?? 0) + (stats.queued ?? 0) + (stats.backlog ?? 0);- if (unfinished > 0) return false;+ const active = stats.active ?? 0;+ const queued = stats.queued ?? 0;+ const backlog = stats.backlog ?? 0;+ if (group.status === "active" || active + queued + backlog > 0) {+ return {+ type: "crawl",+ targetId: target.targetId,+ crawlId: target.crawlId,+ groupStatus: group.status,+ active,+ queued,+ backlog,+ };+ }}}- return true;+ return null;}async function failStaleMonitorCheck(params: {@@ -1733,15 +1778,22 @@ async function reconcileMonitorCheckCandidate(targetResults,});- if (- !(await isMonitorCheckComplete(- {- ...check,- target_results: targetResults,- },- monitor,- ))- ) {+ const unfinished = await findUnfinishedMonitorTarget(+ {+ ...check,+ target_results: targetResults,+ },+ monitor,+ );+ if (unfinished) {+ logger.info("Monitor check not ready to finalize", {+ monitorId: monitor.id,+ checkId: check.id,+ runningForMs: check.started_at+ ? Date.now() - Date.parse(check.started_at)+ : null,+ ...unfinished,+ });// Only persist target_results we recovered from an empty snapshot. Writing// back a non-empty stale snapshot here can DOWNGRADE a searchCompleted=true// that the inline handler persisted after this reconciler loaded itsdiff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex c6bcefaad..18253c613 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -605,7 +605,7 @@ describe("FLAT deterministic search-monitor billing", () => {it("a still-running search (no credits stamped yet) sums to 0 — the reconciler must NOT finalize here", () => {// Pre-stamp shape the dispatcher writes before the inline search finishes.- // The never-zero guarantee relies on isMonitorCheckComplete refusing to+ // The never-zero guarantee relies on findUnfinishedMonitorTarget refusing to// finalize a run whose searchCompleted flag is not yet true (see runner.ts).const pending = { targetId: "search-1", type: "search" as const };expect((pending as any).searchCompleted).toBeUndefined();