Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
trim comments in search runner and persist helpers
Developers Digest6115668d9129
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex f53b80d67..2eff5d857 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -946,8 +946,8 @@ async function enqueueMonitorCrawlTarget(params: {return params.targetRun;}-// Reuse the canonical credit logic (no new constants): a search result that was scraped ran a-// json verdict extraction (5 credits); already-seen results skip the scrape (0); failures bill base.+// Credit attribution via the canonical estimator: scraped results ran a json extraction,+// already-seen results skip the scrape, failures bill the base scrape credit.const SEARCH_JSON_DOC = { json: {} } as const;const SEARCH_JSON_OPTS = { formats: [{ type: "json" }] } as const;function searchPageCredits(status: string): number {@@ -955,12 +955,11 @@ function searchPageCredits(status: string): number {if (searchPageWasScraped(status)) {return estimateActualCredits(SEARCH_JSON_DOC, SEARCH_JSON_OPTS);}- return estimateActualCredits({}, SEARCH_JSON_OPTS); // skipped/other → base scrape credit+ return estimateActualCredits({}, SEARCH_JSON_OPTS);}-// Search targets run inline (they own their search + scrape + judge). We reconstruct the dedup-// memory from prior monitor_pages, run the target, then persist results onto the same-// monitor_pages / monitor_check_pages tables the reconciler tallies — no new storage.+// Runs inline (owns its search + scrape + judge), then persists onto the same+// monitor_pages / monitor_check_pages tables the reconciler tallies.async function runMonitorSearchTarget(params: {monitor: MonitorRow;check: MonitorCheckRow;diff --git a/apps/api/src/services/monitoring/search/persist.ts b/apps/api/src/services/monitoring/search/persist.tsindex be29adc3f..831b1b8c9 100644--- a/apps/api/src/services/monitoring/search/persist.ts+++ b/apps/api/src/services/monitoring/search/persist.ts@@ -18,9 +18,8 @@ export function searchPageWasScraped(status: string): boolean {type PriorPage = { url: string; metadata: unknown | null };-// Rebuild the per-URL dedup memory and the event index from prior monitor_pages of this target.-// Stale-goal fingerprints are still loaded (the runner's goalVersion gate decides freshness),-// but the event index only carries events from the CURRENT goalVersion so a goal change starts clean.+// Rebuild dedup memory + event index from prior pages. Fingerprints load regardless of+// goalVersion (the runner gates freshness); events only carry the current goalVersion.export function reconstructKnownState(priorPages: PriorPage[],goalVersion: string,