Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add search support to tracking and cleanups
Developers Digest61912923f0b3
diff --git a/apps/api/src/lib/tracking.ts b/apps/api/src/lib/tracking.tsindex a58466c1e..0fd13ee78 100644--- a/apps/api/src/lib/tracking.ts+++ b/apps/api/src/lib/tracking.ts@@ -116,6 +116,9 @@ function estimateMonitorTargetCredits(target: MonitorTarget): number {if (target.type === "scrape") {return target.urls.length;}+ if (target.type === "search") {+ return target.maxResults;+ }return positiveIntegerOrNull(target.crawlOptions?.limit) ?? 10000;}@@ -127,6 +130,14 @@ function monitorTargetSignature(target: MonitorTarget): string {urls: target.urls.map(normalizeTrackingUrl).sort(),});}+ if (target.type === "search") {+ return hashSignature({+ type: "search",+ queries: [...target.queries].sort(),+ searchWindow: target.searchWindow,+ alertMode: target.alertMode,+ });+ }return hashSignature({type: "crawl",diff --git a/apps/api/src/services/monitoring/search/persist.ts b/apps/api/src/services/monitoring/search/persist.tsindex 582cb5ff6..be29adc3f 100644--- a/apps/api/src/services/monitoring/search/persist.ts+++ b/apps/api/src/services/monitoring/search/persist.ts@@ -2,9 +2,7 @@ import { canonicalizeUrl } from "./dedupe";import type { KnownPage } from "./run";import type { KnownEvent } from "./llm";-// Search verdict status β the monitor page-status enum the reconciler counts.-// alert = a genuinely-new meaningful match (new); seen/watching/ignored = no new alert (same);-// a failed scrape = error. This keeps search runs visible in the same new/same/error tallies.+// Search verdict status β the page-status enum the reconciler tallies (new/same/error).export function searchStatusToPageStatus(status: string,): "same" | "new" | "changed" | "removed" | "error" {@@ -13,8 +11,7 @@ export function searchStatusToPageStatus(return "same";}-// A scraped search result ran a json verdict extraction; already-seen results skip the scrape.-// The runner maps these flags onto the canonical credit estimator (no new constants here).+// True for statuses that came from a successful scrape+judge (used for credit attribution).export function searchPageWasScraped(status: string): boolean {return status === "alert" || status === "watching" || status === "ignored";}diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex f6f8659ae..6b59719d0 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -42,8 +42,6 @@ type SearchTargetInput = {maxResults: number;};-// Per-URL dedup memory, reconstructed from monitor_pages.metadata by the runner. Stale-goal-// rows are ignored (goalVersion gate) so a changed goal re-evaluates instead of staying quiet.export type KnownPage = {fingerprint: string;goalVersion: string;@@ -71,9 +69,6 @@ type SearchTargetRunResult = {matches: number; // new alerts this runsummary: string;sources: SearchSource[];- // β upsert into monitor_pages; metadata carries fingerprint, goalVersion, verdict, and- // (for alert/already_seen) eventKey+eventLabel. The event index is reconstructed from- // these page rows β no separate event table/column.pageUpserts: Array<{url: string;urlHash: Buffer;@@ -86,7 +81,6 @@ export async function runSearchTarget(params: {monitor: { id: string; teamId: string; goal: string | null; subject: string };target: SearchTargetInput;goalVersion: string;- // Same-goalVersion dedup memory + events, derived from monitor_pages by the runner.knownPages: Map<string, KnownPage>;knownEvents: KnownEvent[];zeroDataRetention: boolean;@@ -94,7 +88,6 @@ export async function runSearchTarget(params: {}): Promise<SearchTargetRunResult> {const { target, knownPages, goalVersion, logger } = params;- // P0(#4): a search monitor cannot judge without a goal.const goal = params.monitor.goal?.trim();if (!goal) {throw new Error("search monitor target requires a non-empty monitor goal");@@ -151,7 +144,6 @@ export async function runSearchTarget(params: {snippet: c.description,});const known = knownPages.get(canonical);- // P0(#1): a page judged under a different goal is stale β treat it as new.const knownCurrent =known && known.goalVersion === goalVersion ? known : undefined;const isNewOrChanged =@@ -169,8 +161,7 @@ export async function runSearchTarget(params: {continue;}- // Judge the page INSIDE the scrape via the `json` format β Firecrawl runs the extraction- // (the page judge), the verdict comes back on document.json. No separate Gemini call.+ // Judge in-scrape via the json format; verdict returns on document.json.let res;try {res = await scrapeURL(@@ -196,7 +187,6 @@ export async function runSearchTarget(params: {continue;}- // P0(#2): scrapeURL returns a discriminated result β handle the failure arm.if (!res.success) {logger.warn("search monitor scrape failed", { url: c.url });skipped += 1;