Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Clarify search billing/completion comments
Developers Digest63a2625ed48e
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 0a0247f46..3f4b5dba4 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -115,22 +115,16 @@ type MonitorTargetRun =crawlId: string;}| {- // Search runs inline during dispatch; counts/credits filled in after.targetId: string;type: "search";- // False/undefined until the inline search finishes and stamps its credits.- // The reconciler must NOT finalize a check whose search run is still- // pending β otherwise it sums credits that haven't been written yet and- // bills 0. Set true ONLY once searchCredits/judgeCredits are populated.+ // Set true only after the inline search stamps its credits; the reconciler+ // waits on this so it never finalizes with credits still at 0.searchCompleted?: boolean;resultCount?: number;matches?: number;summary?: string;judgeDegraded?: boolean;degradedReason?: string | null;- // Flat, deterministic credits recorded onto target_results when the search- // completes; actual_credits is summed from these, not reconstructed from- // per-page metadata, so a missing page / reconciler race can't zero them.searchCredits?: number;judgeCredits?: number;resultsJudged?: number;@@ -1054,15 +1048,11 @@ async function runMonitorSearchTarget(params: {}),});- // Flat, deterministic credits, returned for recording onto target_results.- // actual_credits sums from target_results (not per-page metadata), so these- // are reliable regardless of page count/ordering/reconciler timing β closing- // the old "completed deep check with actual_credits = 0" hole.+ // Flat credits recorded onto target_results (summed there at finalization).const searchCredits = result.searchCredits;const judgeCredits = result.judgeCredits;- // Search pages carry no per-page credit (cost is the flat figure above, billed- // once at check level); writing creditsUsed here would double-count.+ // Search pages carry no per-page credit β billed once at check level.const pages: PageResult[] = result.pageUpserts.map(upsert => {const status = searchStatusToPageStatus(upsert.status);return {@@ -1214,8 +1204,7 @@ export async function processMonitorCheckJob(} else if (target.type === "crawl" && targetRun.type === "crawl") {await enqueueMonitorCrawlTarget({ monitor, check, target, targetRun });} else if (target.type === "search" && targetRun.type === "search") {- // Search is synchronous: run now and fold the outcome into target_results- // so the reconciler (which treats search as complete) finalizes the check.+ // Search runs synchronously; fold its outcome into target_results.const searchResult = await runMonitorSearchTarget({monitor,check,@@ -1226,14 +1215,11 @@ export async function processMonitorCheckJob(targetRun.summary = searchResult.summary;targetRun.judgeDegraded = searchResult.judgeDegraded;targetRun.degradedReason = searchResult.degradedReason;- // Persisted onto target_results below so they survive to finalization.targetRun.searchCredits = searchResult.searchCredits;targetRun.judgeCredits = searchResult.judgeCredits;targetRun.resultsJudged = searchResult.resultsJudged;- // Mark complete LAST, after credits are populated: the reconciler keys- // its completeness check on this flag, so it can never finalize (and- // bill the as-yet-unstamped, zero credits) while the search β possibly- // mid rate-limit retry β is still running.+ // Set LAST, after credits are stamped, so the reconciler never finalizes+ // with credits still at 0.targetRun.searchCompleted = true;}}@@ -1387,11 +1373,7 @@ async function isMonitorCheckComplete(for (const target of targetResults) {if (target?.type === "search") {- // Search runs inline during dispatch and stamps searchCompleted=true only- // after its flat credits are written. Until then the run is in flight- // (e.g. retrying a rate-limited query), so treating it as complete would- // let the reconciler finalize with credits still at 0 β the never-zero- // guarantee depends on waiting here.+ // Not complete until the inline search has stamped its credits.if (!target.searchCompleted) return false;} else if (target?.type === "scrape") {const expected = Array.isArray(target.expectedJobs)@@ -1615,8 +1597,7 @@ export async function reconcileRunningMonitorChecks(const actualCredits = await calculateMonitorCheckActualCredits({checkId: check.id,targets: monitor.targets,- // Flat search credits come from target_results, not page metadata, so a- // completed deep check that judged always records them (fixes the old bug).+ // Flat search credits come from target_results, not page metadata.targetResults,});diff --git a/apps/api/src/services/monitoring/search/cost.test.ts b/apps/api/src/services/monitoring/search/cost.test.tsindex 4b75f86db..3077faf5a 100644--- a/apps/api/src/services/monitoring/search/cost.test.ts+++ b/apps/api/src/services/monitoring/search/cost.test.ts@@ -1,16 +1,8 @@import { CostTracking } from "../../../lib/cost-tracking";import { recordLlmCall } from "./cost";-// Mirror of JUDGE_CREDITS_PER_RESULT in ./run.ts. We intentionally do NOT import-// from ./run here: run.ts pulls in the search/scrape stack (ESM `uuid`, native-// modules) which the jest transform can't load in this suite. The value is a-// stable billing constant; this test pins it and the flat-billing arithmetic.const JUDGE_CREDITS_PER_RESULT = 5;-// Monitor judge billing is now a FLAT, deterministic figure (5 credits per-// result the judge evaluates). recordLlmCall still records token usage into a-// shared CostTracking, but that is ONLY for observability β it no longer feeds-// the credits a team is charged.describe("search-monitor LLM cost recording (observability only)", () => {it("records token usage into the shared CostTracking with no dollar cost", () => {const ct = new CostTracking();@@ -52,23 +44,15 @@ describe("search-monitor FLAT judge billing rate", () => {});});-// The CANONICAL "judged result" definition, pinned so billing and the eval stay-// in lockstep. A judged result is one scraped+judged (deep) or snippet-judged-// (standard) THIS run β run.ts increments resultsJudged exactly once per such-// result and stamps metadata.judgedThisRun=true on its persisted page. The-// billed count therefore equals the number of pages with judgedThisRun=true.-// Reused (unchanged), raw, and skipped pages are NOT judged this run and carry-// judgedThisRun=false / absent β even if they retain a stale `concept`.+// Billed count == pages with judgedThisRun=true (not every page, not every page+// with a concept β reused/skipped pages keep a stale concept but aren't judged).describe("canonical judged-result count (billing == persisted signal)", () => {type Page = { searchStatus: string; judgedThisRun?: boolean; concept?: string };const billedJudgedCount = (pages: Page[]) =>pages.filter(p => p.judgedThisRun === true).length;it("counts judgedThisRun=true, NOT every page nor every page with a concept", () => {- // The judged#1 shape: 6 results, only 1 re-judged this run; the other two- // "concept" pages are REUSED from a prior run (judgedThisRun=false) and a- // skipped page got no verdict. Counting pages-with-concept gives 3 (the old- // eval undercount mismatch); the canonical count is 1.+ // 1 judged this run; 2 reused (stale concept, not judged); 1 skipped.const pages: Page[] = [{ searchStatus: "alert", judgedThisRun: true, concept: "fresh-verdict" },{ searchStatus: "already_seen", judgedThisRun: false, concept: "stale-1" },diff --git a/apps/api/src/services/monitoring/search/criteria.ts b/apps/api/src/services/monitoring/search/criteria.tsindex 68b695375..ab1c6982d 100644--- a/apps/api/src/services/monitoring/search/criteria.ts+++ b/apps/api/src/services/monitoring/search/criteria.ts@@ -4,8 +4,7 @@ import { googleModel, googleProviderOptions } from "./tuning";import { recordLlmCall } from "./cost";import type { CostTracking } from "../../../lib/cost-tracking";-const CRITERIA_MODEL =- process.env.SEARCH_MONITOR_CRITERIA_MODEL ?? "gemini-flash-lite-latest";+const CRITERIA_MODEL = "gemini-flash-lite-latest";export type GoalCriteria = {goalVersion: string;diff --git a/apps/api/src/services/monitoring/search/llm.ts b/apps/api/src/services/monitoring/search/llm.tsindex ec83c185f..3c68f2022 100644--- a/apps/api/src/services/monitoring/search/llm.ts+++ b/apps/api/src/services/monitoring/search/llm.ts@@ -5,14 +5,10 @@ import type { GoalCriteria } from "./criteria";import { recordLlmCall } from "./cost";import type { CostTracking } from "../../../lib/cost-tracking";-const EVENT_MODEL =- process.env.SEARCH_MONITOR_EVENT_MODEL ?? "gemini-flash-lite-latest";-const SUMMARY_MODEL =- process.env.SEARCH_MONITOR_SUMMARY_MODEL ?? "gemini-flash-lite-latest";-const SKEPTIC_MODEL =- process.env.SEARCH_MONITOR_SKEPTIC_MODEL ?? "gemini-flash-lite-latest";-const ROUTER_MODEL =- process.env.SEARCH_MONITOR_ROUTER_MODEL ?? "gemini-3-flash-preview";+const EVENT_MODEL = "gemini-flash-lite-latest";+const SUMMARY_MODEL = "gemini-flash-lite-latest";+const SKEPTIC_MODEL = "gemini-flash-lite-latest";+const ROUTER_MODEL = "gemini-3-flash-preview";export type KnownEvent = {key: string;@@ -57,12 +53,10 @@ export async function resolveEvent(params: {url: params.result.url,evidence: (params.result.evidence || "").slice(0, 500),},- instructions: [- "If newResult is the same underlying event as a knownEvent, set matchedKey to that event's key and isNew false.",- "Only set isNew true when this is a distinct development not represented by any knownEvent.",- "label: when matched, reuse the matched event's label; when new, write a short reusable label naming the company/product/event.",- "A new article from a different publisher about the same filing/launch/recall is NOT a new event.",- ],+ instructions: `If newResult is the same underlying event as a knownEvent, set matchedKey to that event's key and isNew false.+Only set isNew true when this is a distinct development not represented by any knownEvent.+label: when matched, reuse the matched event's label; when new, write a short reusable label naming the company/product/event.+A new article from a different publisher about the same filing/launch/recall is NOT a new event.`,}),temperature: 0,...googleProviderOptions(),@@ -109,10 +103,8 @@ export async function judgeMaterialDevelopment(params: {title: params.result.title,evidence: (params.result.evidence || "").slice(0, 500),},- instructions: [- "Set material true only when the new result reports a genuinely new development of the known event.",- "Set material false when it restates already-known facts, even from a new source or headline.",- ],+ instructions: `Set material true only when the new result reports a genuinely new development of the known event.+Set material false when it restates already-known facts, even from a new source or headline.`,}),temperature: 0,...googleProviderOptions(),@@ -239,12 +231,10 @@ export async function routeSearchResults(params: {monitor: { goal: params.goal, subject: params.subject },searchWindow: params.searchWindow,maxResults: params.maxResults,- instructions: [- "Select scrape only when the result could plausibly contain primary evidence or substantive discussion satisfying the monitor goal.",- "Skip obvious jobs, directories, tag pages, search pages, marketplaces, profiles, irrelevant foreign-language spam, generic tool lists, or pages where the monitored subject appears only as a skill, integration, or incidental keyword.",- "Do not use keyword overlap alone. Explain the user-facing reason.",- "Never select more than maxResults.",- ],+ instructions: `Select scrape only when the result could plausibly contain primary evidence or substantive discussion satisfying the monitor goal.+Skip obvious jobs, directories, tag pages, search pages, marketplaces, profiles, irrelevant foreign-language spam, generic tool lists, or pages where the monitored subject appears only as a skill, integration, or incidental keyword.+Do not use keyword overlap alone. Explain the user-facing reason.+Never select more than maxResults.`,candidates,}),temperature: 0,@@ -343,10 +333,8 @@ export async function summarizeRun(params: {prompt: JSON.stringify({monitor: { goal: params.goal, subject: params.subject },evidence: params.evidence.slice(0, 8),- instructions: [- "Summarize why the meaningful sources satisfy the goal as a group, in one short sentence.",- "If sources are meaningful but already reported, say they are related evidence, not a new notification.",- ],+ instructions: `Summarize why the meaningful sources satisfy the goal as a group, in one short sentence.+If sources are meaningful but already reported, say they are related evidence, not a new notification.`,}),temperature: 0,...googleProviderOptions(),diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex 8d30936c8..9f0208ed8 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -301,12 +301,8 @@ describe("FLAT deterministic search-monitor billing", () => {});});- // Regression guard for the FLAT search billing contract that eval H asserts:- // actual_credits = 2*ceil(totalResults/10) + 5*resultsJudged (search-only- // when judge is off). These pin the deterministic sum, the never-zero-with-- // results invariant, and that the retry path still bills.+ // Flat billing contract: actual = 2*ceil(totalResults/10) + 5*resultsJudged.describe("flat search billing contract (eval H regression guard)", () => {- // The deterministic figures the runner stamps onto a search target run.const searchCreditsFor = (totalResults: number) =>2 * Math.ceil(totalResults / 10);const stampedSearchRun = (totalResults: number, resultsJudged: number) => ({diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 667c6de2a..dd39502d1 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -173,8 +173,6 @@ function estimateTargetBaseCredits(return target.urls.length * creditsPerPage;}if (target.type === "search") {- // Full estimate (search + flat judge); excluded from the global per-page- // judge loop below to avoid double-counting.return estimateSearchTargetCredits(target, judgeEnabled);}@@ -208,8 +206,7 @@ export function estimateMonitorCreditsPerRun((sum, target) => sum + estimateTargetBaseCredits(target, judgeEnabled),0,);- // Search judging is already folded into estimateTargetBaseCredits above; the- // per-page allowance applies only to scrape/crawl, so exclude search here.+ // Per-page judge allowance is scrape/crawl only (search judging is folded in above).const judgeCredits = judgeEnabled? targets.reduce((sum, target) =>@@ -295,21 +292,16 @@ export function calculateMonitorCheckActualCreditsFromPages(return 0;}- // Search judging is billed at the check level; the per-page judge credit- // would double-count it. Only scrape/crawl get it.+ // Search is billed at the check level (see flatSearchTargetCredits).const target = targetsById.get(page.target_id ?? "");if (target?.type === "search") {return 0;}-- // A persisted judgment means the judge ran for this page; charge for it- // whether or not the verdict was meaningful.return JUDGE_CREDITS_PER_PAGE;}return pages.reduce((total, page) => {- // Search pages carry no per-page credit (billed at check level via- // flatSearchTargetCredits); summing a per-page base would double-bill.+ // Search pages carry no per-page credit β billed at check level.const target = targetsById.get(page.target_id ?? "");if (target?.type === "search") {return total;@@ -331,12 +323,7 @@ export function calculateMonitorCheckActualCreditsFromPages(}, 0);}-/**- * Sum the flat credits recorded on a check's target_results for every search- * target (searchCredits + judgeCredits). Persisted when the search completes, so- * they're always present at finalization regardless of which pages persisted β- * guaranteeing a deep check that judged can never record actual_credits = 0.- */+// Sum the flat searchCredits + judgeCredits recorded on a check's target_results.export function flatSearchTargetCredits(targetResults: unknown): number {if (!Array.isArray(targetResults)) return 0;return targetResults.reduce((total: number, run: unknown) => {@@ -930,11 +917,8 @@ export async function countMonitorCheckPages(params: {export async function calculateMonitorCheckActualCredits(params: {checkId: string;targets: MonitorTarget[];- // The check's persisted target_results; search credits are summed from here so- // an empty/missing pages table can't zero them out.targetResults?: unknown;}): Promise<number> {- // Flat search credits first: deterministic and independent of pages.let total = flatSearchTargetCredits(params.targetResults);let offset = 0;diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsindex 58dd8a160..2dbd63c5d 100644--- a/apps/api/src/services/monitoring/types.ts+++ b/apps/api/src/services/monitoring/types.ts@@ -48,12 +48,8 @@ const monitorDomainSchema = z"Domain must be a valid hostname without protocol or path",);-// `depth`/`alertMode`/`recheckAfter` are internal-only (derived at runtime /-// defaulted by the runner). `scrapeOptions` is meaningless for search: the runner-// builds its own scrape format (markdown + verdict-json) per result, so a client-// value would be ignored. Strip all of these BEFORE strictObject validation-// rather than 400ing, so older clients that still send them keep working (values-// ignored, not persisted).+// depth/alertMode/recheckAfter are internal-only; scrapeOptions is unused for+// search. Strip them before validation so older clients sending them don't 400.const searchTargetSchema = z.preprocess(value => {if (value && typeof value === "object" && !Array.isArray(value)) {@@ -265,9 +261,7 @@ export type MonitorTarget = z.infer<typeof monitorTargetSchema> & {depth?: "raw" | "standard" | "deep";alertMode?: "first_match" | "every_new_result" | "material_dev";recheckAfter?: "1h" | "6h" | "24h" | "7d";- // Present on scrape/crawl targets; absent on search (the search runner builds- // its own per-result scrape format). Optional here so union-level reads in- // runner/store/results keep compiling.+ // Present on scrape/crawl, absent on search; optional so union reads compile.scrapeOptions?: z.infer<typeof scrapeOptionsSchema>;};export type CreateMonitorRequest = z.infer<typeof createMonitorSchema>;