Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Add search support to tracking and cleanups

Developers Digest61912923f0b3
diff --git a/apps/api/src/lib/tracking.ts b/apps/api/src/lib/tracking.ts
index a58466c1e..0fd13ee78 100644
--- a/apps/api/src/lib/tracking.ts
+++ b/apps/api/src/lib/tracking.ts
@@ -116,6 +116,9 @@ function estimateMonitorTargetCredits(target: MonitorTarget): number {
if (target.type === "scrape") {
return target.urls.length;
}
+ if (target.type === "search") {
+ return target.maxResults;
+ }
return positiveIntegerOrNull(target.crawlOptions?.limit) ?? 10000;
}
@@ -127,6 +130,14 @@ function monitorTargetSignature(target: MonitorTarget): string {
urls: target.urls.map(normalizeTrackingUrl).sort(),
});
}
+ if (target.type === "search") {
+ return hashSignature({
+ type: "search",
+ queries: [...target.queries].sort(),
+ searchWindow: target.searchWindow,
+ alertMode: target.alertMode,
+ });
+ }
return hashSignature({
type: "crawl",
diff --git a/apps/api/src/services/monitoring/search/persist.ts b/apps/api/src/services/monitoring/search/persist.ts
index 582cb5ff6..be29adc3f 100644
--- a/apps/api/src/services/monitoring/search/persist.ts
+++ b/apps/api/src/services/monitoring/search/persist.ts
@@ -2,9 +2,7 @@ import { canonicalizeUrl } from "./dedupe";
import type { KnownPage } from "./run";
import type { KnownEvent } from "./llm";
-// Search verdict status β†’ the monitor page-status enum the reconciler counts.
-// alert = a genuinely-new meaningful match (new); seen/watching/ignored = no new alert (same);
-// a failed scrape = error. This keeps search runs visible in the same new/same/error tallies.
+// Search verdict status β†’ the page-status enum the reconciler tallies (new/same/error).
export function searchStatusToPageStatus(
status: string,
): "same" | "new" | "changed" | "removed" | "error" {
@@ -13,8 +11,7 @@ export function searchStatusToPageStatus(
return "same";
}
-// A scraped search result ran a json verdict extraction; already-seen results skip the scrape.
-// The runner maps these flags onto the canonical credit estimator (no new constants here).
+// True for statuses that came from a successful scrape+judge (used for credit attribution).
export function searchPageWasScraped(status: string): boolean {
return status === "alert" || status === "watching" || status === "ignored";
}
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index f6f8659ae..6b59719d0 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -42,8 +42,6 @@ type SearchTargetInput = {
maxResults: number;
};
-// Per-URL dedup memory, reconstructed from monitor_pages.metadata by the runner. Stale-goal
-// rows are ignored (goalVersion gate) so a changed goal re-evaluates instead of staying quiet.
export type KnownPage = {
fingerprint: string;
goalVersion: string;
@@ -71,9 +69,6 @@ type SearchTargetRunResult = {
matches: number; // new alerts this run
summary: string;
sources: SearchSource[];
- // β†’ upsert into monitor_pages; metadata carries fingerprint, goalVersion, verdict, and
- // (for alert/already_seen) eventKey+eventLabel. The event index is reconstructed from
- // these page rows β€” no separate event table/column.
pageUpserts: Array<{
url: string;
urlHash: Buffer;
@@ -86,7 +81,6 @@ export async function runSearchTarget(params: {
monitor: { id: string; teamId: string; goal: string | null; subject: string };
target: SearchTargetInput;
goalVersion: string;
- // Same-goalVersion dedup memory + events, derived from monitor_pages by the runner.
knownPages: Map<string, KnownPage>;
knownEvents: KnownEvent[];
zeroDataRetention: boolean;
@@ -94,7 +88,6 @@ export async function runSearchTarget(params: {
}): Promise<SearchTargetRunResult> {
const { target, knownPages, goalVersion, logger } = params;
- // P0(#4): a search monitor cannot judge without a goal.
const goal = params.monitor.goal?.trim();
if (!goal) {
throw new Error("search monitor target requires a non-empty monitor goal");
@@ -151,7 +144,6 @@ export async function runSearchTarget(params: {
snippet: c.description,
});
const known = knownPages.get(canonical);
- // P0(#1): a page judged under a different goal is stale β€” treat it as new.
const knownCurrent =
known && known.goalVersion === goalVersion ? known : undefined;
const isNewOrChanged =
@@ -169,8 +161,7 @@ export async function runSearchTarget(params: {
continue;
}
- // Judge the page INSIDE the scrape via the `json` format β€” Firecrawl runs the extraction
- // (the page judge), the verdict comes back on document.json. No separate Gemini call.
+ // Judge in-scrape via the json format; verdict returns on document.json.
let res;
try {
res = await scrapeURL(
@@ -196,7 +187,6 @@ export async function runSearchTarget(params: {
continue;
}
- // P0(#2): scrapeURL returns a discriminated result β€” handle the failure arm.
if (!res.success) {
logger.warn("search monitor scrape failed", { url: c.url });
skipped += 1;