Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
monitor search: route deep-mode scrape through the shared scrape path
Developers Digest43a50c5d3865
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 61c443ccf..a1bf99320 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -65,7 +65,8 @@ import {MONITOR_CHECK_STALE_TIMEOUT_MS,} from "./stale";import { trackMonitorCheckStartedInterest } from "./interest";-import { runSearchTarget } from "./search/run";+import { runSearchTarget, type ScrapeSearchResult } from "./search/run";+import { verdictJsonSchema } from "./search/judge";import { computeGoalVersion } from "./search/dedupe";import {reconstructKnownState,@@ -329,6 +330,77 @@ async function runSingleScrape(params: {};}+// Deep-mode search-monitor page scrape. Routes through the same internal+// scrape path as every other monitor scrape (processJobInternal, skipNuq,+// bypassBilling) instead of calling scrapeURL directly, so it respects the+// team's concurrency queue and is never billed per-page (search monitors bill+// flat at the check level).+async function scrapeSearchMonitorPage(params: {+ teamId: string;+ checkId: string;+ url: string;+ judgePrompt: string;+}): Promise<ScrapeSearchResult | null> {+ const scrapeId = uuidv7();+ const scrapeOptions = scrapeRequestSchema.parse({+ url: params.url,+ formats: [+ { type: "markdown" },+ { type: "json", schema: verdictJsonSchema, prompt: params.judgePrompt },+ ],+ timeout: 20000,+ origin: "monitor",+ });++ await logRequest({+ id: scrapeId,+ kind: "scrape",+ api_version: "v2",+ team_id: params.teamId,+ origin: "monitor",+ integration: null,+ target_hint: params.url,+ zeroDataRetention: false,+ api_key_id: null,+ });++ const job: NuQJob<ScrapeJobData> = {+ id: scrapeId,+ status: "active",+ createdAt: new Date(),+ priority: 20,+ data: {+ mode: "single_urls",+ url: params.url,+ team_id: params.teamId,+ scrapeOptions,+ internalOptions: {+ teamId: params.teamId,+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,+ bypassBilling: true,+ zeroDataRetention: false,+ },+ skipNuq: true,+ origin: "monitor",+ integration: null,+ billing: { endpoint: "monitor", jobId: params.checkId },+ zeroDataRetention: false,+ apiKeyId: null,+ },+ };++ const doc = await processJobInternal(job);+ if (!doc) return null;+ return {+ json: doc.json ?? null,+ markdown: doc.markdown ?? "",+ metadata: {+ publishedTime: doc.metadata?.publishedTime ?? null,+ modifiedTime: doc.metadata?.modifiedTime ?? null,+ },+ };+}+async function diffAndPersistPage(params: {monitor: MonitorRow;check: MonitorCheckRow;@@ -1038,6 +1110,13 @@ async function runMonitorSearchTarget(params: {depth: target.depth,},monitorCheckId: check.id,+ scrapePage: ({ url, judgePrompt }) =>+ scrapeSearchMonitorPage({+ teamId: monitor.team_id,+ checkId: check.id,+ url,+ judgePrompt,+ }),goalVersion,knownPages,knownEvents,diff --git a/apps/api/src/services/monitoring/search/run-defenses.test.ts b/apps/api/src/services/monitoring/search/run-defenses.test.tsindex 9b56146f1..f96f0f69e 100644--- a/apps/api/src/services/monitoring/search/run-defenses.test.ts+++ b/apps/api/src/services/monitoring/search/run-defenses.test.ts@@ -34,9 +34,6 @@ vi.mock("../../../search/v2", () => ({return Array.isArray(r) ? { web: r } : r;},}));-vi.mock("../../../scraper/scrapeURL", () => ({- scrapeURL: (...a: unknown[]) => scrapeURLMock(...a),-}));vi.mock("./llm", () => ({resolveEvent: (...a: unknown[]) => resolveEventMock(...a),summarizeRun: (...a: unknown[]) => summarizeRunMock(...a),@@ -102,6 +99,25 @@ function runParams(...targetOver,},monitorCheckId: "check-1",+ // scrapeURLMock keeps the legacy { success, document } shape; adapt it to the+ // ScrapeSearchResult the injected scrapePage now returns.+ scrapePage: async (...a: unknown[]) => {+ const r = (await scrapeURLMock(...a)) as {+ success?: boolean;+ document?: {+ json: unknown;+ markdown?: string;+ metadata?: { publishedTime?: string; modifiedTime?: string };+ };+ } | null;+ return r && r.success && r.document+ ? {+ json: r.document.json ?? null,+ markdown: r.document.markdown ?? "",+ metadata: r.document.metadata ?? {},+ }+ : null;+ },goalVersion: "v1",knownPages: new Map(),knownEvents: [],@@ -150,7 +166,7 @@ describe("router gating (deep)", () => {const result = await runSearchTarget(runParams());expect(scrapeURLMock).toHaveBeenCalledTimes(1);- expect(scrapeURLMock.mock.calls[0][1]).toBe(serpRow(2).url);+ expect(scrapeURLMock.mock.calls[0][0].url).toBe(serpRow(2).url);expect(result.matches).toBe(1);});@@ -483,7 +499,7 @@ describe("deep-path scrape failures mark the check degraded (no silent empty)",});it("NOT degraded when at least one scrape succeeds and is judged", async () => {- scrapeURLMock.mockImplementation((_id: string, url: string) =>+ scrapeURLMock.mockImplementation(({ url }: { url: string }) =>url === serpRow(1).url? Promise.resolve({success: true,diff --git a/apps/api/src/services/monitoring/search/run.test.ts b/apps/api/src/services/monitoring/search/run.test.tsindex 4124a6e72..168b5a5d2 100644--- a/apps/api/src/services/monitoring/search/run.test.ts+++ b/apps/api/src/services/monitoring/search/run.test.ts@@ -2,14 +2,14 @@ import type { Logger } from "winston";import { type SearchVerdict, verdictJsonSchema, buildJudgePrompt } from "./judge";import type { EventResolution } from "./llm";import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";-import { scrapeOptions } from "../../../controllers/v2/types";+import { scrapeRequestSchema } from "../../../controllers/v2/types";// vi.mock is hoisted above declarations, so the mocks its factories reference// are created in vi.hoisted() (also hoisted) to avoid any TDZ surprises.-const { searchMock, scrapeURLMock, resolveEventMock, summarizeRunMock, materialDevMock } =+const { searchMock, scrapePageMock, resolveEventMock, summarizeRunMock, materialDevMock } =vi.hoisted(() => ({searchMock: vi.fn(),- scrapeURLMock: vi.fn(),+ scrapePageMock: vi.fn(),resolveEventMock: vi.fn(),summarizeRunMock: vi.fn(),materialDevMock: vi.fn(),@@ -24,9 +24,6 @@ vi.mock("../../../search/v2", () => ({return Array.isArray(r) ? { web: r } : r;},}));-vi.mock("../../../scraper/scrapeURL", () => ({- scrapeURL: (...a: unknown[]) => scrapeURLMock(...a),-}));vi.mock("./llm", () => ({resolveEvent: (...a: unknown[]) => resolveEventMock(...a),summarizeRun: (...a: unknown[]) => summarizeRunMock(...a),@@ -54,8 +51,9 @@ const verdict = (over: Partial<SearchVerdict> = {}): SearchVerdict => ({});const okScrape = (v: SearchVerdict) => ({- success: true,- document: { json: v },+ json: v,+ markdown: "",+ metadata: {},});function setSearchResults(@@ -65,9 +63,9 @@ function setSearchResults(}function setVerdictsByUrl(map: Record<string, SearchVerdict>) {- scrapeURLMock.mockImplementation((_id: string, url: string) => {+ scrapePageMock.mockImplementation(({ url }: { url: string }) => {const v = map[url];- if (!v) return Promise.resolve({ success: false });+ if (!v) return Promise.resolve(null);return Promise.resolve(okScrape(v));});}@@ -107,6 +105,7 @@ function run(alertMode: over.alertMode ?? baseTarget.alertMode,},monitorCheckId: "check-1",+ scrapePage: (...a: unknown[]) => scrapePageMock(...a),goalVersion: over.goalVersion ?? "gv1",knownPages: over.knownPages ?? new Map(),knownEvents: over.knownEvents ?? [],@@ -148,7 +147,7 @@ describe("runSearchTarget orchestration", () => {expect(out.sources[0].eventKey).toBeTruthy();expect(out.pageUpserts[0].status).toBe("alert");expect(out.summary).toBe("OpenAI filed for IPO.");- expect(scrapeURLMock).toHaveBeenCalledTimes(1);+ expect(scrapePageMock).toHaveBeenCalledTimes(1);});it("skips scrape/judge for an already-seen unchanged page (same goalVersion)", async () => {@@ -173,7 +172,7 @@ describe("runSearchTarget orchestration", () => {expect(out.sources[0].status).toBe("already_seen");expect(out.pageUpserts[0].status).toBe("already_seen");- expect(scrapeURLMock).not.toHaveBeenCalled();+ expect(scrapePageMock).not.toHaveBeenCalled();expect(out.matches).toBe(0);});@@ -213,7 +212,7 @@ describe("runSearchTarget orchestration", () => {const out = await run({ knownPages });- expect(scrapeURLMock).not.toHaveBeenCalled();+ expect(scrapePageMock).not.toHaveBeenCalled();expect(out.pageUpserts[0].status).toBe("already_seen");expect(out.pageUpserts[0].metadata).toMatchObject({eventKey: "evt-1",@@ -281,7 +280,7 @@ describe("runSearchTarget orchestration", () => {const out = await run({ knownPages, goalVersion: "gv2" });- expect(scrapeURLMock).toHaveBeenCalledTimes(1);+ expect(scrapePageMock).toHaveBeenCalledTimes(1);expect(out.sources[0].status).toBe("alert");});@@ -397,7 +396,7 @@ describe("runSearchTarget orchestration", () => {setSearchResults([{ url: "https://dead.com/x", title: "x", description: "y" },]);- scrapeURLMock.mockResolvedValue({ success: false });+ scrapePageMock.mockResolvedValue(null);const out = await run();@@ -424,13 +423,14 @@ describe("runSearchTarget orchestration", () => {const out = await run();expect(out.resultCount).toBe(1);- expect(scrapeURLMock).toHaveBeenCalledTimes(1);+ expect(scrapePageMock).toHaveBeenCalledTimes(1);});});describe("scrape payload validity (real validator, no mocks)", () => {- it("scrapeOptions.parse accepts the verdict json format", () => {- const parsed = scrapeOptions.parse({+ it("scrapeRequestSchema accepts the verdict json format the runner sends", () => {+ const parsed = scrapeRequestSchema.parse({+ url: "https://example.com",formats: [{type: "json",@@ -439,6 +439,7 @@ describe("scrape payload validity (real validator, no mocks)", () => {},],timeout: 20000,+ origin: "monitor",});const jsonFormat = (parsed.formats as Array<{ type: string }>).find(f => f.type === "json",@@ -517,7 +518,7 @@ describe("re-judge cadence", () => {],]);const out = await run({ target: { recheckAfter: "24h" }, knownPages });- expect(scrapeURLMock).toHaveBeenCalledTimes(1);+ expect(scrapePageMock).toHaveBeenCalledTimes(1);expect(out.sources[0].status).toBe("alert");});@@ -536,7 +537,7 @@ describe("re-judge cadence", () => {],]);const out = await run({ target: { recheckAfter: "24h" }, knownPages });- expect(scrapeURLMock).not.toHaveBeenCalled();+ expect(scrapePageMock).not.toHaveBeenCalled();expect(out.sources[0].status).toBe("watching");});});@@ -582,6 +583,6 @@ describe("domain scoping", () => {});expect(out.resultCount).toBe(1);expect(out.sources.map(s => s.url)).toEqual(["https://news.com/openai"]);- expect(scrapeURLMock).toHaveBeenCalledTimes(1);+ expect(scrapePageMock).toHaveBeenCalledTimes(1);});});diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex b211c982a..0e8789ba7 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -2,15 +2,12 @@ import { v7 as uuidv7 } from "uuid";import type { Logger } from "winston";import { search } from "../../../search/v2";import { buildSearchQuery } from "../../../lib/search-query-builder";-import { scrapeURL } from "../../../scraper/scrapeURL";-import { scrapeOptions } from "../../../controllers/v2/types";import { CostTracking } from "../../../lib/cost-tracking";import { hashMonitorUrl } from "../store";import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";import {buildJudgePrompt,parseVerdict,- verdictJsonSchema,verdictToDecision,windowToMs,type SearchVerdict,@@ -125,6 +122,20 @@ export type KnownPage = {metadata?: Record<string, unknown>;};+// Deep-mode page scrape. Injected by the caller so it can route through the+// shared monitor scrape path (concurrency queue + bypassBilling), keeping this+// module free of worker/queue dependencies. Resolves to null on scrape failure.+export type ScrapeSearchResult = {+ json: unknown;+ markdown: string;+ metadata: { publishedTime?: string | null; modifiedTime?: string | null };+};++type ScrapeSearchPage = (args: {+ url: string;+ judgePrompt: string;+}) => Promise<ScrapeSearchResult | null>;+type SearchSource = {url: string;title: string;@@ -175,6 +186,7 @@ export async function runSearchTarget(params: {};target: SearchTargetInput;monitorCheckId: string;+ scrapePage: ScrapeSearchPage;goalVersion: string;knownPages: Map<string, KnownPage>;knownEvents: KnownEvent[];@@ -557,27 +569,12 @@ export async function runSearchTarget(params: {}pagesChecked += 1;} else {- let res;- const scrapeParsedOptions = scrapeOptions.parse({- formats: [- { type: "markdown" },- {- type: "json",- schema: verdictJsonSchema,- prompt: buildJudgePrompt(judgeGoal, subject, target.searchWindow),- },- ],- timeout: 20000,- });- const scrapeCostTracking = new CostTracking();+ let doc: ScrapeSearchResult | null;try {- res = await scrapeURL(- "search-monitor;" + uuidv7(),- c.url,- scrapeParsedOptions,- { teamId, zeroDataRetention: params.zeroDataRetention },- scrapeCostTracking,- );+ doc = await params.scrapePage({+ url: c.url,+ judgePrompt: buildJudgePrompt(judgeGoal, subject, target.searchWindow),+ });} catch (error) {logger.warn("search monitor scrape threw", { url: c.url, error });judgeScrapeFailures += 1;@@ -587,7 +584,7 @@ export async function runSearchTarget(params: {continue;}- if (!res.success) {+ if (!doc) {logger.warn("search monitor scrape failed", { url: c.url });judgeScrapeFailures += 1;pushUnevaluatedPage("scrape failed");@@ -595,17 +592,14 @@ export async function runSearchTarget(params: {}pagesChecked += 1;- verdict = parseVerdict(res.document.json);+ verdict = parseVerdict(doc.json);if (!verdict) {judgeScrapeFailures += 1;pushUnevaluatedPage("verdict unparseable");continue;}- pageText = res.document.markdown ?? "";- realDate =- res.document.metadata?.publishedTime ??- res.document.metadata?.modifiedTime ??- null;+ pageText = doc.markdown ?? "";+ realDate = doc.metadata?.publishedTime ?? doc.metadata?.modifiedTime ?? null;}resultsJudged += 1;