Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add 'raw' search depth and aggregate query hits
Developers Digest6095d1069ffb
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex a208c454f..2f4bbf63a 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -77,7 +77,7 @@ type SearchTargetInput = {excludeDomains?: string[];recheckAfter?: string;maxResults: number;- depth?: "standard" | "deep";+ depth?: "raw" | "standard" | "deep";};export type KnownPage = {@@ -153,12 +153,13 @@ export async function runSearchTarget(params: {let skipped = 0;let matches = 0;- const seenThisRun = new Set<string>();+ const seenThisRun = new Map<string, number>();const candidates: Array<{url: string;title: string;description: string;query: string;+ matchedQueries: string[];}> = [];for (const query of target.queries) {const { query: scopedQuery } = buildSearchQuery(query, undefined, {@@ -175,28 +176,42 @@ export async function runSearchTarget(params: {if (!r.url) continue;if (isExcludedDomain(r.url, target.excludeDomains)) continue;const canonical = canonicalizeUrl(r.url);- if (seenThisRun.has(canonical)) continue;- seenThisRun.add(canonical);+ const existingIdx = seenThisRun.get(canonical);+ if (existingIdx !== undefined) {+ // Same URL surfaced by another query this run — record the extra query+ // (mirrors the POC's queryHits) instead of dropping it.+ const existing = candidates[existingIdx];+ if (!existing.matchedQueries.includes(query)) {+ existing.matchedQueries.push(query);+ }+ continue;+ }+ seenThisRun.set(canonical, candidates.length);candidates.push({url: r.url,title: r.title,description: r.description,query,+ matchedQueries: [query],});}}resultCount = candidates.length;const llmStagesAvailable = hasGeminiKey();- const depth: "standard" | "deep" =- target.depth === "standard" && llmStagesAvailable ? "standard" : "deep";+ const depth: "raw" | "standard" | "deep" =+ target.depth === "raw"+ ? "raw"+ : target.depth === "standard" && llmStagesAvailable+ ? "standard"+ : "deep";let criteria: GoalCriteria = compileGoalCriteria({goal,subject,goalVersion,});- if (llmStagesAvailable) {+ if (llmStagesAvailable && depth !== "raw") {try {criteria = await compileGoalCriteriaWithLlm({goal,@@ -365,6 +380,79 @@ export async function runSearchTarget(params: {continue;}+ if (depth === "raw") {+ const nowIso = new Date().toISOString();+ const rawKey = canonical;+ const alreadyAlerted =+ satisfied.has(rawKey) && target.alertMode !== "every_new_result";+ const rawMeta = {+ fingerprint,+ goalVersion,+ searchStatus: alreadyAlerted ? "already_seen" : "alert",+ eventKey: rawKey,+ eventLabel: c.title || canonical,+ query: c.query,+ matchedQueries: c.matchedQueries,+ };+ if (alreadyAlerted) {+ sources.push({+ url: c.url,+ title: c.title,+ status: "already_seen",+ eventKey: rawKey,+ });+ pageUpserts.push({+ url: canonical,+ urlHash: hashMonitorUrl(canonical),+ status: "already_seen",+ scraped: false,+ metadata: rawMeta,+ });+ continue;+ }+ matches += 1;+ satisfied.add(rawKey);+ const existingEvent = events.find(e => e.key === rawKey);+ const eventSatisfiedAt = existingEvent?.satisfiedAt ?? nowIso;+ const eventAlertCount = (existingEvent?.alertCount ?? 0) + 1;+ if (existingEvent) {+ existingEvent.satisfiedAt = eventSatisfiedAt;+ existingEvent.alertCount = eventAlertCount;+ } else {+ events.unshift({+ key: rawKey,+ label: c.title || canonical,+ satisfiedAt: eventSatisfiedAt,+ alertCount: eventAlertCount,+ });+ }+ sources.push({+ url: c.url,+ title: c.title,+ status: "alert",+ eventKey: rawKey,+ });+ pageUpserts.push({+ url: canonical,+ urlHash: hashMonitorUrl(canonical),+ status: "alert",+ scraped: false,+ metadata: {+ ...rawMeta,+ eventSatisfiedAt,+ eventAlertCount,+ eventLastAlertAt: nowIso,+ },+ judgment: {+ meaningful: true,+ confidence: "low",+ reason: `New search result for "${c.query}"`,+ meaningfulChanges: [],+ },+ });+ continue;+ }+let verdict: SearchVerdict | null = null;let pageText = "";let realDate: string | null = null;@@ -482,6 +570,7 @@ export async function runSearchTarget(params: {publishedAt: realDate,concept: verdict.concept,rationale: verdict.rationale,+ matchedQueries: c.matchedQueries,...(verification && !verification.pass ? { verification } : {}),};@@ -638,7 +727,9 @@ export async function runSearchTarget(params: {).length;let summary = "";- if (matches > 0) {+ if (matches > 0 && depth === "raw") {+ summary = `${matches} new search result${matches === 1 ? "" : "s"} matched the monitor queries.`;+ } else if (matches > 0) {try {const out = await summarizeRun({goal,diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsindex 6f895d338..d113ef1b4 100644--- a/apps/api/src/services/monitoring/types.ts+++ b/apps/api/src/services/monitoring/types.ts@@ -62,7 +62,7 @@ const searchTargetSchema = z.strictObject({.default("first_match"),// "deep" scrapes + judges routed pages (extract tier); "standard" judges// from SERP snippets only — no page fetches, the cheap tier.- depth: z.enum(["standard", "deep"]).optional().default("deep"),+ depth: z.enum(["raw", "standard", "deep"]).optional().default("deep"),includeDomains: z.array(monitorDomainSchema).max(50).optional(),excludeDomains: z.array(monitorDomainSchema).max(50).optional(),recheckAfter: z.enum(["1h", "6h", "24h", "7d"]).optional(),