Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

monitor search: route deep-mode scrape through the shared scrape path

Developers Digest43a50c5d3865
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 61c443ccf..a1bf99320 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -65,7 +65,8 @@ import {
MONITOR_CHECK_STALE_TIMEOUT_MS,
} from "./stale";
import { trackMonitorCheckStartedInterest } from "./interest";
-import { runSearchTarget } from "./search/run";
+import { runSearchTarget, type ScrapeSearchResult } from "./search/run";
+import { verdictJsonSchema } from "./search/judge";
import { computeGoalVersion } from "./search/dedupe";
import {
reconstructKnownState,
@@ -329,6 +330,77 @@ async function runSingleScrape(params: {
};
}
+// Deep-mode search-monitor page scrape. Routes through the same internal
+// scrape path as every other monitor scrape (processJobInternal, skipNuq,
+// bypassBilling) instead of calling scrapeURL directly, so it respects the
+// team's concurrency queue and is never billed per-page (search monitors bill
+// flat at the check level).
+async function scrapeSearchMonitorPage(params: {
+ teamId: string;
+ checkId: string;
+ url: string;
+ judgePrompt: string;
+}): Promise<ScrapeSearchResult | null> {
+ const scrapeId = uuidv7();
+ const scrapeOptions = scrapeRequestSchema.parse({
+ url: params.url,
+ formats: [
+ { type: "markdown" },
+ { type: "json", schema: verdictJsonSchema, prompt: params.judgePrompt },
+ ],
+ timeout: 20000,
+ origin: "monitor",
+ });
+
+ await logRequest({
+ id: scrapeId,
+ kind: "scrape",
+ api_version: "v2",
+ team_id: params.teamId,
+ origin: "monitor",
+ integration: null,
+ target_hint: params.url,
+ zeroDataRetention: false,
+ api_key_id: null,
+ });
+
+ const job: NuQJob<ScrapeJobData> = {
+ id: scrapeId,
+ status: "active",
+ createdAt: new Date(),
+ priority: 20,
+ data: {
+ mode: "single_urls",
+ url: params.url,
+ team_id: params.teamId,
+ scrapeOptions,
+ internalOptions: {
+ teamId: params.teamId,
+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
+ bypassBilling: true,
+ zeroDataRetention: false,
+ },
+ skipNuq: true,
+ origin: "monitor",
+ integration: null,
+ billing: { endpoint: "monitor", jobId: params.checkId },
+ zeroDataRetention: false,
+ apiKeyId: null,
+ },
+ };
+
+ const doc = await processJobInternal(job);
+ if (!doc) return null;
+ return {
+ json: doc.json ?? null,
+ markdown: doc.markdown ?? "",
+ metadata: {
+ publishedTime: doc.metadata?.publishedTime ?? null,
+ modifiedTime: doc.metadata?.modifiedTime ?? null,
+ },
+ };
+}
+
async function diffAndPersistPage(params: {
monitor: MonitorRow;
check: MonitorCheckRow;
@@ -1038,6 +1110,13 @@ async function runMonitorSearchTarget(params: {
depth: target.depth,
},
monitorCheckId: check.id,
+ scrapePage: ({ url, judgePrompt }) =>
+ scrapeSearchMonitorPage({
+ teamId: monitor.team_id,
+ checkId: check.id,
+ url,
+ judgePrompt,
+ }),
goalVersion,
knownPages,
knownEvents,
diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.ts
index 9b56146f1..f96f0f69e 100644
--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts
+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts
@@ -34,9 +34,6 @@ vi.mock("../../../search/v2", () => ({
return Array.isArray(r) ? { web: r } : r;
},
}));
-vi.mock("../../../scraper/scrapeURL", () => ({
- scrapeURL: (...a: unknown[]) => scrapeURLMock(...a),
-}));
vi.mock("./llm", () => ({
resolveEvent: (...a: unknown[]) => resolveEventMock(...a),
summarizeRun: (...a: unknown[]) => summarizeRunMock(...a),
@@ -102,6 +99,25 @@ function runParams(
...targetOver,
},
monitorCheckId: "check-1",
+ // scrapeURLMock keeps the legacy { success, document } shape; adapt it to the
+ // ScrapeSearchResult the injected scrapePage now returns.
+ scrapePage: async (...a: unknown[]) => {
+ const r = (await scrapeURLMock(...a)) as {
+ success?: boolean;
+ document?: {
+ json: unknown;
+ markdown?: string;
+ metadata?: { publishedTime?: string; modifiedTime?: string };
+ };
+ } | null;
+ return r && r.success && r.document
+ ? {
+ json: r.document.json ?? null,
+ markdown: r.document.markdown ?? "",
+ metadata: r.document.metadata ?? {},
+ }
+ : null;
+ },
goalVersion: "v1",
knownPages: new Map(),
knownEvents: [],
@@ -150,7 +166,7 @@ describe("router gating (deep)", () => {
const result = await runSearchTarget(runParams());
expect(scrapeURLMock).toHaveBeenCalledTimes(1);
- expect(scrapeURLMock.mock.calls[0][1]).toBe(serpRow(2).url);
+ expect(scrapeURLMock.mock.calls[0][0].url).toBe(serpRow(2).url);
expect(result.matches).toBe(1);
});
@@ -483,7 +499,7 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",
});
it("NOT degraded when at least one scrape succeeds and is judged", async () => {
- scrapeURLMock.mockImplementation((_id: string, url: string) =>
+ scrapeURLMock.mockImplementation(({ url }: { url: string }) =>
url === serpRow(1).url
? Promise.resolve({
success: true,
diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.ts
index 4124a6e72..168b5a5d2 100644
--- a/apps/api/src/services/monitoring/search/run.test.ts
+++ b/apps/api/src/services/monitoring/search/run.test.ts
@@ -2,14 +2,14 @@ import type { Logger } from "winston";
import { type SearchVerdict, verdictJsonSchema, buildJudgePrompt } from "./judge";
import type { EventResolution } from "./llm";
import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";
-import { scrapeOptions } from "../../../controllers/v2/types";
+import { scrapeRequestSchema } from "../../../controllers/v2/types";
// vi.mock is hoisted above declarations, so the mocks its factories reference
// are created in vi.hoisted() (also hoisted) to avoid any TDZ surprises.
-const { searchMock, scrapeURLMock, resolveEventMock, summarizeRunMock, materialDevMock } =
+const { searchMock, scrapePageMock, resolveEventMock, summarizeRunMock, materialDevMock } =
vi.hoisted(() => ({
searchMock: vi.fn(),
- scrapeURLMock: vi.fn(),
+ scrapePageMock: vi.fn(),
resolveEventMock: vi.fn(),
summarizeRunMock: vi.fn(),
materialDevMock: vi.fn(),
@@ -24,9 +24,6 @@ vi.mock("../../../search/v2", () => ({
return Array.isArray(r) ? { web: r } : r;
},
}));
-vi.mock("../../../scraper/scrapeURL", () => ({
- scrapeURL: (...a: unknown[]) => scrapeURLMock(...a),
-}));
vi.mock("./llm", () => ({
resolveEvent: (...a: unknown[]) => resolveEventMock(...a),
summarizeRun: (...a: unknown[]) => summarizeRunMock(...a),
@@ -54,8 +51,9 @@ const verdict = (over: Partial<SearchVerdict> = {}): SearchVerdict => ({
});
const okScrape = (v: SearchVerdict) => ({
- success: true,
- document: { json: v },
+ json: v,
+ markdown: "",
+ metadata: {},
});
function setSearchResults(
@@ -65,9 +63,9 @@ function setSearchResults(
}
function setVerdictsByUrl(map: Record<string, SearchVerdict>) {
- scrapeURLMock.mockImplementation((_id: string, url: string) => {
+ scrapePageMock.mockImplementation(({ url }: { url: string }) => {
const v = map[url];
- if (!v) return Promise.resolve({ success: false });
+ if (!v) return Promise.resolve(null);
return Promise.resolve(okScrape(v));
});
}
@@ -107,6 +105,7 @@ function run(
alertMode: over.alertMode ?? baseTarget.alertMode,
},
monitorCheckId: "check-1",
+ scrapePage: (...a: unknown[]) => scrapePageMock(...a),
goalVersion: over.goalVersion ?? "gv1",
knownPages: over.knownPages ?? new Map(),
knownEvents: over.knownEvents ?? [],
@@ -148,7 +147,7 @@ describe("runSearchTarget orchestration", () => {
expect(out.sources[0].eventKey).toBeTruthy();
expect(out.pageUpserts[0].status).toBe("alert");
expect(out.summary).toBe("OpenAI filed for IPO.");
- expect(scrapeURLMock).toHaveBeenCalledTimes(1);
+ expect(scrapePageMock).toHaveBeenCalledTimes(1);
});
it("skips scrape/judge for an already-seen unchanged page (same goalVersion)", async () => {
@@ -173,7 +172,7 @@ describe("runSearchTarget orchestration", () => {
expect(out.sources[0].status).toBe("already_seen");
expect(out.pageUpserts[0].status).toBe("already_seen");
- expect(scrapeURLMock).not.toHaveBeenCalled();
+ expect(scrapePageMock).not.toHaveBeenCalled();
expect(out.matches).toBe(0);
});
@@ -213,7 +212,7 @@ describe("runSearchTarget orchestration", () => {
const out = await run({ knownPages });
- expect(scrapeURLMock).not.toHaveBeenCalled();
+ expect(scrapePageMock).not.toHaveBeenCalled();
expect(out.pageUpserts[0].status).toBe("already_seen");
expect(out.pageUpserts[0].metadata).toMatchObject({
eventKey: "evt-1",
@@ -281,7 +280,7 @@ describe("runSearchTarget orchestration", () => {
const out = await run({ knownPages, goalVersion: "gv2" });
- expect(scrapeURLMock).toHaveBeenCalledTimes(1);
+ expect(scrapePageMock).toHaveBeenCalledTimes(1);
expect(out.sources[0].status).toBe("alert");
});
@@ -397,7 +396,7 @@ describe("runSearchTarget orchestration", () => {
setSearchResults([
{ url: "https://dead.com/x", title: "x", description: "y" },
]);
- scrapeURLMock.mockResolvedValue({ success: false });
+ scrapePageMock.mockResolvedValue(null);
const out = await run();
@@ -424,13 +423,14 @@ describe("runSearchTarget orchestration", () => {
const out = await run();
expect(out.resultCount).toBe(1);
- expect(scrapeURLMock).toHaveBeenCalledTimes(1);
+ expect(scrapePageMock).toHaveBeenCalledTimes(1);
});
});
describe("scrape payload validity (real validator, no mocks)", () => {
- it("scrapeOptions.parse accepts the verdict json format", () => {
- const parsed = scrapeOptions.parse({
+ it("scrapeRequestSchema accepts the verdict json format the runner sends", () => {
+ const parsed = scrapeRequestSchema.parse({
+ url: "https://example.com",
formats: [
{
type: "json",
@@ -439,6 +439,7 @@ describe("scrape payload validity (real validator, no mocks)", () => {
},
],
timeout: 20000,
+ origin: "monitor",
});
const jsonFormat = (parsed.formats as Array<{ type: string }>).find(
f => f.type === "json",
@@ -517,7 +518,7 @@ describe("re-judge cadence", () => {
],
]);
const out = await run({ target: { recheckAfter: "24h" }, knownPages });
- expect(scrapeURLMock).toHaveBeenCalledTimes(1);
+ expect(scrapePageMock).toHaveBeenCalledTimes(1);
expect(out.sources[0].status).toBe("alert");
});
@@ -536,7 +537,7 @@ describe("re-judge cadence", () => {
],
]);
const out = await run({ target: { recheckAfter: "24h" }, knownPages });
- expect(scrapeURLMock).not.toHaveBeenCalled();
+ expect(scrapePageMock).not.toHaveBeenCalled();
expect(out.sources[0].status).toBe("watching");
});
});
@@ -582,6 +583,6 @@ describe("domain scoping", () => {
});
expect(out.resultCount).toBe(1);
expect(out.sources.map(s => s.url)).toEqual(["https://news.com/openai"]);
- expect(scrapeURLMock).toHaveBeenCalledTimes(1);
+ expect(scrapePageMock).toHaveBeenCalledTimes(1);
});
});
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index b211c982a..0e8789ba7 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -2,15 +2,12 @@ import { v7 as uuidv7 } from "uuid";
import type { Logger } from "winston";
import { search } from "../../../search/v2";
import { buildSearchQuery } from "../../../lib/search-query-builder";
-import { scrapeURL } from "../../../scraper/scrapeURL";
-import { scrapeOptions } from "../../../controllers/v2/types";
import { CostTracking } from "../../../lib/cost-tracking";
import { hashMonitorUrl } from "../store";
import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";
import {
buildJudgePrompt,
parseVerdict,
- verdictJsonSchema,
verdictToDecision,
windowToMs,
type SearchVerdict,
@@ -125,6 +122,20 @@ export type KnownPage = {
metadata?: Record<string, unknown>;
};
+// Deep-mode page scrape. Injected by the caller so it can route through the
+// shared monitor scrape path (concurrency queue + bypassBilling), keeping this
+// module free of worker/queue dependencies. Resolves to null on scrape failure.
+export type ScrapeSearchResult = {
+ json: unknown;
+ markdown: string;
+ metadata: { publishedTime?: string | null; modifiedTime?: string | null };
+};
+
+type ScrapeSearchPage = (args: {
+ url: string;
+ judgePrompt: string;
+}) => Promise<ScrapeSearchResult | null>;
+
type SearchSource = {
url: string;
title: string;
@@ -175,6 +186,7 @@ export async function runSearchTarget(params: {
};
target: SearchTargetInput;
monitorCheckId: string;
+ scrapePage: ScrapeSearchPage;
goalVersion: string;
knownPages: Map<string, KnownPage>;
knownEvents: KnownEvent[];
@@ -557,27 +569,12 @@ export async function runSearchTarget(params: {
}
pagesChecked += 1;
} else {
- let res;
- const scrapeParsedOptions = scrapeOptions.parse({
- formats: [
- { type: "markdown" },
- {
- type: "json",
- schema: verdictJsonSchema,
- prompt: buildJudgePrompt(judgeGoal, subject, target.searchWindow),
- },
- ],
- timeout: 20000,
- });
- const scrapeCostTracking = new CostTracking();
+ let doc: ScrapeSearchResult | null;
try {
- res = await scrapeURL(
- "search-monitor;" + uuidv7(),
- c.url,
- scrapeParsedOptions,
- { teamId, zeroDataRetention: params.zeroDataRetention },
- scrapeCostTracking,
- );
+ doc = await params.scrapePage({
+ url: c.url,
+ judgePrompt: buildJudgePrompt(judgeGoal, subject, target.searchWindow),
+ });
} catch (error) {
logger.warn("search monitor scrape threw", { url: c.url, error });
judgeScrapeFailures += 1;
@@ -587,7 +584,7 @@ export async function runSearchTarget(params: {
continue;
}
- if (!res.success) {
+ if (!doc) {
logger.warn("search monitor scrape failed", { url: c.url });
judgeScrapeFailures += 1;
pushUnevaluatedPage("scrape failed");
@@ -595,17 +592,14 @@ export async function runSearchTarget(params: {
}
pagesChecked += 1;
- verdict = parseVerdict(res.document.json);
+ verdict = parseVerdict(doc.json);
if (!verdict) {
judgeScrapeFailures += 1;
pushUnevaluatedPage("verdict unparseable");
continue;
}
- pageText = res.document.markdown ?? "";
- realDate =
- res.document.metadata?.publishedTime ??
- res.document.metadata?.modifiedTime ??
- null;
+ pageText = doc.markdown ?? "";
+ realDate = doc.metadata?.publishedTime ?? doc.metadata?.modifiedTime ?? null;
}
resultsJudged += 1;