Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
add search monitor target type and pipeline scaffolding
Developers Digest61f686e6927c
diff --git a/apps/api/src/lib/tracking.ts b/apps/api/src/lib/tracking.tsindex a58466c1e..0fd13ee78 100644--- a/apps/api/src/lib/tracking.ts+++ b/apps/api/src/lib/tracking.ts@@ -116,6 +116,9 @@ function estimateMonitorTargetCredits(target: MonitorTarget): number {if (target.type === "scrape") {return target.urls.length;}+ if (target.type === "search") {+ return target.maxResults;+ }return positiveIntegerOrNull(target.crawlOptions?.limit) ?? 10000;}@@ -127,6 +130,14 @@ function monitorTargetSignature(target: MonitorTarget): string {urls: target.urls.map(normalizeTrackingUrl).sort(),});}+ if (target.type === "search") {+ return hashSignature({+ type: "search",+ queries: [...target.queries].sort(),+ searchWindow: target.searchWindow,+ alertMode: target.alertMode,+ });+ }return hashSignature({type: "crawl",diff --git a/apps/api/src/services/monitoring/search/dedupe.ts b/apps/api/src/services/monitoring/search/dedupe.tsnew file mode 100644index 000000000..71037fc83--- /dev/null+++ b/apps/api/src/services/monitoring/search/dedupe.ts@@ -0,0 +1,25 @@+import { createHash } from "node:crypto";+import { normalizeUrl } from "../../../lib/canonical-url";++// Canonical key for URL dedup (reuses prod normalization; pair with hashMonitorUrl for url_hash).+export function canonicalizeUrl(raw: string): string {+ return normalizeUrl(String(raw || "")).toLowerCase();+}++// Stable pre-scrape fingerprint from SERP fields only (title + snippet). We never hash scraped+// page content β it churns between scrapes and would falsely re-flag the same source as changed.+export function stableSerpFingerprint(src: {+ url?: string;+ title?: string;+ snippet?: string;+ description?: string;+}): string {+ const text = [src.title, src.snippet ?? src.description]+ .filter(Boolean)+ .join("\n")+ .trim();+ return createHash("sha256")+ .update(text || canonicalizeUrl(src.url ?? ""))+ .digest("hex")+ .slice(0, 24);+}diff --git a/apps/api/src/services/monitoring/search/judge.ts b/apps/api/src/services/monitoring/search/judge.tsnew file mode 100644index 000000000..030b2fee5--- /dev/null+++ b/apps/api/src/services/monitoring/search/judge.ts@@ -0,0 +1,98 @@+// Page judge runs INSIDE the Firecrawl scrape (the `json` format) β the verdict comes back+// on document.json. No separate Gemini call: Gemini is only for event-resolution + summary.++export type SearchVerdict = {+ relevant: boolean;+ alertAction: "alert" | "watch" | "ignore";+ freshness: "fresh" | "stale" | "unknown";+ sourceQuality:+ | "first-party"+ | "authoritative"+ | "unverified"+ | "resale"+ | "unclear";+ concept: string;+ rationale: string;+};++// JSON schema passed to the scrape `json` format (Firecrawl runs the extraction internally).+export const verdictJsonSchema = {+ type: "object",+ additionalProperties: false,+ required: [+ "relevant",+ "alertAction",+ "freshness",+ "sourceQuality",+ "concept",+ "rationale",+ ],+ properties: {+ relevant: { type: "boolean" },+ alertAction: { type: "string", enum: ["alert", "watch", "ignore"] },+ freshness: { type: "string", enum: ["fresh", "stale", "unknown"] },+ sourceQuality: {+ type: "string",+ enum: ["first-party", "authoritative", "unverified", "resale", "unclear"],+ },+ concept: { type: "string" },+ rationale: { type: "string" },+ },+} as const;++export function buildJudgePrompt(+ goal: string,+ subject: string,+ searchWindow: string,+): string {+ return [+ `Monitor goal: ${goal}`,+ subject ? `Monitored subject: ${subject}.` : "",+ `Search window: ${searchWindow}.`,+ "Judge ONLY this page's visible content against the goal, not the query wording.",+ "Set relevant true and alertAction alert only when the page materially satisfies the exact goal and is fresh for the window. Competitors, look-alikes, listings, stale or unconfirmed pages are watch/ignore.",+ "Never pair freshness stale/unknown with alertAction alert.",+ "concept: a short reusable label naming the real-world event (company/product/event).",+ ]+ .filter(Boolean)+ .join("\n");+}++// Validate/coerce the scrape's document.json into a verdict.+export function parseVerdict(json: unknown): SearchVerdict | null {+ if (!json || typeof json !== "object") return null;+ const v = json as Record<string, unknown>;+ if (typeof v.relevant !== "boolean") return null;+ return {+ relevant: v.relevant,+ alertAction: (["alert", "watch", "ignore"].includes(v.alertAction as string)+ ? v.alertAction+ : "watch") as SearchVerdict["alertAction"],+ freshness: (["fresh", "stale", "unknown"].includes(v.freshness as string)+ ? v.freshness+ : "unknown") as SearchVerdict["freshness"],+ sourceQuality: ([+ "first-party",+ "authoritative",+ "unverified",+ "resale",+ "unclear",+ ].includes(v.sourceQuality as string)+ ? v.sourceQuality+ : "unclear") as SearchVerdict["sourceQuality"],+ concept: typeof v.concept === "string" ? v.concept : "",+ rationale: typeof v.rationale === "string" ? v.rationale : "",+ };+}++// "alert" only when relevant, fresh, trusted-enough, and the judge asked to alert.+export function verdictToDecision(+ v: SearchVerdict,+): "notify" | "watch" | "ignore" {+ if (!v.relevant || v.alertAction === "ignore") return "ignore";+ if (v.alertAction === "watch") return "watch";+ if (v.freshness !== "fresh") return "watch";+ if (v.sourceQuality === "unverified" || v.sourceQuality === "unclear")+ return "watch";+ return "notify";+}diff --git a/apps/api/src/services/monitoring/search/llm.ts b/apps/api/src/services/monitoring/search/llm.tsnew file mode 100644index 000000000..07bf7d552--- /dev/null+++ b/apps/api/src/services/monitoring/search/llm.ts@@ -0,0 +1,96 @@+import { generateObject } from "ai";+import { google } from "@ai-sdk/google";+import { z } from "zod";++// Gemini LLM calls for the search monitor (mirrors services/monitoring/judgeChange.ts).+// Cost: Gemini 3 Flash reasoning tokens dominate output β use a low thinking budget and+// batch the resolver when scaling.++const EVENT_MODEL =+ process.env.SEARCH_MONITOR_EVENT_MODEL ?? "gemini-3-flash-preview";+const SUMMARY_MODEL =+ process.env.SEARCH_MONITOR_SUMMARY_MODEL ?? "gemini-3-flash-preview";++export type KnownEvent = { key: string; label: string };++export type EventResolution = {+ matchedKey: string | null;+ isNew: boolean;+ label: string;+ reason: string;+};++const eventResolverSchema = z.object({+ matchedKey: z.string().nullable(),+ isNew: z.boolean(),+ label: z.string(),+ reason: z.string(),+});++export async function resolveEvent(params: {+ goal: string;+ subject: string;+ result: { title: string; url: string; evidence: string };+ candidates: KnownEvent[];+}): Promise<EventResolution> {+ const { object } = await generateObject({+ model: google(EVENT_MODEL),+ schema: eventResolverSchema,+ system:+ "You deduplicate real-world events for a monitoring product. Given a new meaningful result and a list of already-known events, decide if the result describes the SAME underlying event as one of them (a different outlet, headline, or wording still counts as the same event) or a genuinely NEW event. Match to an existing event whenever it is the same development. Return structured JSON only.",+ prompt: JSON.stringify({+ monitor: { goal: params.goal, subject: params.subject },+ knownEvents: params.candidates+ .slice(0, 20)+ .map(c => ({ key: c.key, label: c.label })),+ newResult: {+ title: params.result.title,+ url: params.result.url,+ evidence: (params.result.evidence || "").slice(0, 500),+ },+ instructions: [+ "If newResult is the same underlying event as a knownEvent, set matchedKey to that event's key and isNew false.",+ "Only set isNew true when this is a distinct development not represented by any knownEvent.",+ "label: when matched, reuse the matched event's label; when new, write a short reusable label naming the company/product/event.",+ "A new article from a different publisher about the same filing/launch/recall is NOT a new event.",+ ],+ }),+ temperature: 0,+ });++ return {+ matchedKey: object.matchedKey ?? null,+ isNew: object.matchedKey ? false : object.isNew !== false,+ label: object.label ?? "",+ reason: object.reason ?? "",+ };+}++const runSummarySchema = z.object({+ label: z.enum(["meaningful", "already_satisfied", "not_meaningful"]),+ summary: z.string(),+});++export async function summarizeRun(params: {+ goal: string;+ subject: string;+ evidence: Array<{ title: string; url: string; rationale: string }>;+}): Promise<{ label: string; summary: string }> {+ const { object } = await generateObject({+ model: google(SUMMARY_MODEL),+ schema: runSummarySchema,+ system:+ "You write concise user-facing search-monitor check summaries. Use only the provided monitor goal and meaningful source evidence. Do not introduce facts not present in the evidence. Return structured JSON only.",+ prompt: JSON.stringify({+ monitor: { goal: params.goal, subject: params.subject },+ evidence: params.evidence.slice(0, 8),+ instructions: [+ "Summarize why the meaningful sources satisfy the goal as a group, in one short sentence.",+ "If sources are meaningful but already reported, say they are related evidence, not a new notification.",+ ],+ }),+ temperature: 0,+ });++ return { label: object.label, summary: object.summary };+}diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsnew file mode 100644index 000000000..bdf1d1d8b--- /dev/null+++ b/apps/api/src/services/monitoring/search/run.ts@@ -0,0 +1,288 @@+import { v7 as uuidv7 } from "uuid";+import type { Logger } from "winston";+import { search } from "../../../search";+import { scrapeURL } from "../../../scraper/scrapeURL";+import { scrapeOptions } from "../../../controllers/v2/types";+import { CostTracking } from "../../../lib/cost-tracking";+import { hashMonitorUrl } from "../store";+import { canonicalizeUrl, stableSerpFingerprint } from "./dedupe";+import {+ buildJudgePrompt,+ parseVerdict,+ verdictJsonSchema,+ verdictToDecision,+} from "./judge";+import { resolveEvent, summarizeRun, type KnownEvent } from "./llm";++// Unified schedule/window step β Firecrawl tbs window.+function windowToTbs(window: string): string {+ if (window === "5m" || window === "15m" || window === "1h") return "qdr:h";+ if (window === "6h" || window === "24h") return "qdr:d";+ return "qdr:w"; // 7d+}++function slugifyEvent(label: string): string {+ return (+ label+ .toLowerCase()+ .replace(/[^a-z0-9]+/g, "-")+ .replace(/^-+|-+$/g, "")+ .slice(0, 64) || "event"+ );+}++export type SearchTargetInput = {+ id: string;+ queries: string[];+ searchWindow: string;+ alertMode: "first_match" | "every_new_result";+ includeDomains?: string[];+ maxResults: number;+};++// Per-URL dedup memory, reconstructed from monitor_pages.metadata by the runner. Stale-goal+// rows are ignored (goalVersion gate) so a changed goal re-evaluates instead of staying quiet.+export type KnownPage = {+ fingerprint: string;+ goalVersion: string;+};++export type SearchSource = {+ url: string;+ title: string;+ status: "alert" | "already_seen" | "watching" | "ignored" | "skipped";+ alertAction?: string;+ freshness?: string;+ sourceQuality?: string;+ concept?: string;+ eventKey?: string;+ rationale?: string;+};++export type SearchTargetRunResult = {+ targetId: string;+ type: "search";+ resultCount: number;+ pagesChecked: number;+ skipped: number;+ meaningful: number;+ matches: number; // new alerts this run+ summary: string;+ sources: SearchSource[];+ // β upsert into monitor_pages; metadata carries fingerprint, goalVersion, verdict, and+ // (for alert/already_seen) eventKey+eventLabel. The event index is reconstructed from+ // these page rows β no separate event table/column.+ pageUpserts: Array<{+ url: string;+ urlHash: Buffer;+ status: string;+ metadata: Record<string, unknown>;+ }>;+};++export async function runSearchTarget(params: {+ monitor: { id: string; teamId: string; goal: string | null; subject: string };+ target: SearchTargetInput;+ goalVersion: string;+ // Same-goalVersion dedup memory + events, derived from monitor_pages by the runner.+ knownPages: Map<string, KnownPage>;+ knownEvents: KnownEvent[];+ zeroDataRetention: boolean;+ logger: Logger;+}): Promise<SearchTargetRunResult> {+ const { target, knownPages, goalVersion, logger } = params;++ // P0(#4): a search monitor cannot judge without a goal.+ const goal = params.monitor.goal?.trim();+ if (!goal) {+ throw new Error("search monitor target requires a non-empty monitor goal");+ }+ const subject = params.monitor.subject ?? "";+ const teamId = params.monitor.teamId;++ const tbs = windowToTbs(target.searchWindow);+ const events: KnownEvent[] = [...params.knownEvents];+ const satisfied = new Set(events.map(e => e.key)); // first_match suppression set+ const sources: SearchSource[] = [];+ const pageUpserts: SearchTargetRunResult["pageUpserts"] = [];+ let resultCount = 0;+ let pagesChecked = 0;+ let skipped = 0;+ let matches = 0;++ // 1. Search each query; flatten + dedup by canonical URL within this run.+ const seenThisRun = new Set<string>();+ const candidates: Array<{ url: string; title: string; description: string }> =+ [];+ for (const query of target.queries) {+ const results = await search({+ query,+ logger,+ num_results: target.maxResults,+ tbs,+ });+ for (const r of results) {+ if (!r.url) continue;+ const canonical = canonicalizeUrl(r.url);+ if (seenThisRun.has(canonical)) continue;+ seenThisRun.add(canonical);+ candidates.push({ url: r.url, title: r.title, description: r.description });+ }+ }+ resultCount = candidates.length;++ // 2. Per candidate: dedup β (judge if new/changed) β decide β event-resolve.+ for (const c of candidates.slice(0, target.maxResults)) {+ const canonical = canonicalizeUrl(c.url);+ const fingerprint = stableSerpFingerprint({+ url: c.url,+ title: c.title,+ snippet: c.description,+ });+ const known = knownPages.get(canonical);+ // P0(#1): a page judged under a different goal is stale β treat it as new.+ const knownCurrent = known && known.goalVersion === goalVersion ? known : undefined;+ const isNewOrChanged =+ !knownCurrent || knownCurrent.fingerprint !== fingerprint;++ // Already-seen + unchanged under the current goal β reuse, no scrape/judge/LLM.+ if (!isNewOrChanged) {+ sources.push({ url: c.url, title: c.title, status: "already_seen" });+ pageUpserts.push({+ url: canonical,+ urlHash: hashMonitorUrl(canonical),+ status: "already_seen",+ metadata: { fingerprint, goalVersion },+ });+ continue;+ }++ // Judge the page INSIDE the scrape via the `json` format β Firecrawl runs the extraction+ // (the page judge), the verdict comes back on document.json. No separate Gemini call.+ let res;+ try {+ res = await scrapeURL(+ "search-monitor;" + uuidv7(),+ c.url,+ scrapeOptions.parse({+ formats: [+ {+ type: "json",+ schema: verdictJsonSchema,+ prompt: buildJudgePrompt(goal, subject, target.searchWindow),+ },+ ],+ timeout: 20000,+ }),+ { teamId, zeroDataRetention: params.zeroDataRetention },+ new CostTracking(),+ );+ } catch (error) {+ logger.warn("search monitor scrape threw", { url: c.url, error });+ skipped += 1;+ sources.push({ url: c.url, title: c.title, status: "skipped" });+ continue;+ }++ // P0(#2): scrapeURL returns a discriminated result β handle the failure arm.+ if (!res.success) {+ logger.warn("search monitor scrape failed", { url: c.url });+ skipped += 1;+ sources.push({ url: c.url, title: c.title, status: "skipped" });+ continue;+ }+ pagesChecked += 1;++ const verdict = parseVerdict(res.document.json);+ if (!verdict) {+ skipped += 1;+ sources.push({ url: c.url, title: c.title, status: "skipped" });+ continue;+ }++ const decision = verdictToDecision(verdict);+ const baseMeta = {+ fingerprint,+ goalVersion,+ alertAction: verdict.alertAction,+ freshness: verdict.freshness,+ sourceQuality: verdict.sourceQuality,+ concept: verdict.concept,+ rationale: verdict.rationale,+ };++ if (decision === "ignore") {+ sources.push({ url: c.url, title: c.title, status: "ignored", ...baseMeta });+ pageUpserts.push({ url: canonical, urlHash: hashMonitorUrl(canonical), status: "ignored", metadata: baseMeta });+ continue;+ }+ if (decision === "watch") {+ sources.push({ url: c.url, title: c.title, status: "watching", ...baseMeta });+ pageUpserts.push({ url: canonical, urlHash: hashMonitorUrl(canonical), status: "watching", metadata: baseMeta });+ continue;+ }++ // notify β resolve the real-world event against recent known events.+ const resolution = await resolveEvent({+ goal,+ subject,+ result: { title: c.title, url: c.url, evidence: verdict.rationale },+ candidates: events,+ });+ const eventKey =+ resolution.matchedKey ??+ (resolution.label ? slugifyEvent(resolution.label) : slugifyEvent(verdict.concept));+ const eventLabel =+ events.find(e => e.key === eventKey)?.label ||+ resolution.label ||+ verdict.concept;++ const alreadySatisfied =+ target.alertMode === "first_match" && satisfied.has(eventKey);+ const eventMeta = { ...baseMeta, eventKey, eventLabel };++ if (alreadySatisfied) {+ sources.push({ url: c.url, title: c.title, status: "already_seen", eventKey, ...baseMeta });+ pageUpserts.push({ url: canonical, urlHash: hashMonitorUrl(canonical), status: "already_seen", metadata: eventMeta });+ continue;+ }++ // New, meaningful, not-yet-satisfied β alert.+ matches += 1;+ satisfied.add(eventKey);+ if (!events.some(e => e.key === eventKey)) {+ events.push({ key: eventKey, label: eventLabel });+ }+ sources.push({ url: c.url, title: c.title, status: "alert", eventKey, ...baseMeta });+ pageUpserts.push({ url: canonical, urlHash: hashMonitorUrl(canonical), status: "alert", metadata: eventMeta });+ }++ const meaningful = sources.filter(+ s => s.status === "alert" || s.status === "already_seen",+ ).length;++ let summary = "";+ if (matches > 0) {+ const out = await summarizeRun({+ goal,+ subject,+ evidence: sources+ .filter(s => s.status === "alert")+ .map(s => ({ title: s.title, url: s.url, rationale: s.rationale ?? "" })),+ });+ summary = out.summary;+ }++ return {+ targetId: target.id,+ type: "search",+ resultCount,+ pagesChecked,+ skipped,+ meaningful,+ matches,+ summary,+ sources,+ pageUpserts,+ };+}diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 7994a0721..5442c4941 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -143,6 +143,9 @@ function estimateTargetBaseCredits(target: MonitorTarget): number {if (target.type === "scrape") {return target.urls.length * creditsPerPage;}+ if (target.type === "search") {+ return target.maxResults * creditsPerPage;+ }const limit =typeof target.crawlOptions?.limit === "number"@@ -155,6 +158,9 @@ function estimateTargetPageCount(target: MonitorTarget): number {if (target.type === "scrape") {return target.urls.length;}+ if (target.type === "search") {+ return target.maxResults;+ }const limit =typeof target.crawlOptions?.limit === "number"diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsindex 75f0e8957..85bf510c5 100644--- a/apps/api/src/services/monitoring/types.ts+++ b/apps/api/src/services/monitoring/types.ts@@ -36,7 +36,38 @@ const crawlTargetSchema = z.strictObject({scrapeOptions: scrapeOptionsSchema,});-const monitorTargetSchema = z.union([scrapeTargetSchema, crawlTargetSchema]);+// Search monitor target. Instead of watching fixed URLs (scrape/crawl), this runs a set+// of web-search queries on the monitor's schedule, judges each result against the goal,+// dedupes by URL and by real-world event, and alerts on genuinely-new meaningful matches.+// Search-specific state rides existing jsonb columns (config here; URL dedup memory in+// monitor_pages.metadata; event index in monitors.last_check_summary jsonb for v1) β no+// new tables/columns required to ship.+const searchTargetSchema = z.strictObject({+ id: z.string().uuid().optional(),+ type: z.literal("search"),+ // The web-search queries. (v1: caller/LLM-supplied; see search/queries.ts.)+ queries: z.array(z.string().min(1).max(256)).min(1).max(10),+ // Unified schedule/window step β mapped to a Firecrawl `tbs` window in search/run.ts.+ searchWindow: z+ .enum(["5m", "15m", "1h", "6h", "24h", "7d"])+ .optional()+ .default("24h"),+ // first_match = alert once per real-world event; every_new_result = alert on each new URL.+ alertMode: z+ .enum(["first_match", "every_new_result"])+ .optional()+ .default("first_match"),+ includeDomains: z.array(z.string().min(1)).max(50).optional(),+ // Max results judged per run (caps Firecrawl scrape + LLM cost).+ maxResults: z.number().int().min(1).max(50).optional().default(10),+ scrapeOptions: scrapeOptionsSchema,+});++const monitorTargetSchema = z.union([+ scrapeTargetSchema,+ crawlTargetSchema,+ searchTargetSchema,+]);const monitorWebhookSchema = createWebhookSchema(["monitor.page",@@ -119,15 +150,39 @@ const createMonitorBaseSchema = z.strictObject({origin: z.string().optional().prefault("api"),});-export const createMonitorSchema = createMonitorBaseSchema.transform(- applyJudgeEnabledDefault,-);+// A search target can only be judged with a goal β enforce it whenever one is present.+function requireGoalForSearchTargets(+ input: { targets?: unknown; goal?: unknown },+ ctx: z.RefinementCtx,+): void {+ const targets = input.targets;+ if (!Array.isArray(targets)) return;+ const hasSearchTarget = targets.some(+ t => t && typeof t === "object" && (t as { type?: unknown }).type === "search",+ );+ const goal = input.goal;+ if (+ hasSearchTarget &&+ (typeof goal !== "string" || goal.trim().length === 0)+ ) {+ ctx.addIssue({+ code: "custom",+ message: "A search target requires a non-empty goal",+ path: ["goal"],+ });+ }+}++export const createMonitorSchema = createMonitorBaseSchema+ .superRefine(requireGoalForSearchTargets)+ .transform(applyJudgeEnabledDefault);export const updateMonitorSchema = createMonitorBaseSchema.partial().extend({status: z.enum(["active", "paused"]).optional(),})+ .superRefine(requireGoalForSearchTargets).refine(x => Object.keys(x).length > 0, "Update body cannot be empty").transform(applyJudgeEnabledDefault);