Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(monitor): search billing never-zero + correct judged-result count under retries
Developers Digestc4b3aeffcb22
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 6abfa8ddf..719edcf9a 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -113,6 +113,11 @@ type MonitorTargetRun =// Search runs inline during dispatch; counts/credits filled in after.targetId: string;type: "search";+ // False/undefined until the inline search finishes and stamps its credits.+ // The reconciler must NOT finalize a check whose search run is still+ // pending β otherwise it sums credits that haven't been written yet and+ // bills 0. Set true ONLY once searchCredits/judgeCredits are populated.+ searchCompleted?: boolean;resultCount?: number;matches?: number;summary?: string;@@ -1203,6 +1208,11 @@ export async function processMonitorCheckJob(targetRun.searchCredits = searchResult.searchCredits;targetRun.judgeCredits = searchResult.judgeCredits;targetRun.resultsJudged = searchResult.resultsJudged;+ // Mark complete LAST, after credits are populated: the reconciler keys+ // its completeness check on this flag, so it can never finalize (and+ // bill the as-yet-unstamped, zero credits) while the search β possibly+ // mid rate-limit retry β is still running.+ targetRun.searchCompleted = true;}}@@ -1354,7 +1364,14 @@ async function isMonitorCheckComplete(}for (const target of targetResults) {- if (target?.type === "scrape") {+ if (target?.type === "search") {+ // Search runs inline during dispatch and stamps searchCompleted=true only+ // after its flat credits are written. Until then the run is in flight+ // (e.g. retrying a rate-limited query), so treating it as complete would+ // let the reconciler finalize with credits still at 0 β the never-zero+ // guarantee depends on waiting here.+ if (!target.searchCompleted) return false;+ } else if (target?.type === "scrape") {const expected = Array.isArray(target.expectedJobs)? target.expectedJobs.length: 0;diff --git a/apps/api/src/services/monitoring/search/cost.test.ts b/apps/api/src/services/monitoring/search/cost.test.tsindex 522e3bc69..e08dc6155 100644--- a/apps/api/src/services/monitoring/search/cost.test.ts+++ b/apps/api/src/services/monitoring/search/cost.test.ts@@ -60,3 +60,43 @@ describe("search-monitor FLAT judge billing rate", () => {expect(judgeCreditsFor(10)).toBe(50);});});++// The CANONICAL "judged result" definition, pinned so billing and the eval stay+// in lockstep. A judged result is one scraped+judged (deep) or snippet-judged+// (standard) THIS run β run.ts increments resultsJudged exactly once per such+// result and stamps metadata.judgedThisRun=true on its persisted page. The+// billed count therefore equals the number of pages with judgedThisRun=true.+// Reused (unchanged), raw, and skipped pages are NOT judged this run and carry+// judgedThisRun=false / absent β even if they retain a stale `concept`.+describe("canonical judged-result count (billing == persisted signal)", () => {+ type Page = { searchStatus: string; judgedThisRun?: boolean; concept?: string };+ const billedJudgedCount = (pages: Page[]) =>+ pages.filter(p => p.judgedThisRun === true).length;++ it("counts judgedThisRun=true, NOT every page nor every page with a concept", () => {+ // The judged#1 shape: 6 results, only 1 re-judged this run; the other two+ // "concept" pages are REUSED from a prior run (judgedThisRun=false) and a+ // skipped page got no verdict. Counting pages-with-concept gives 3 (the old+ // eval undercount mismatch); the canonical count is 1.+ const pages: Page[] = [+ { searchStatus: "alert", judgedThisRun: true, concept: "fresh-verdict" },+ { searchStatus: "already_seen", judgedThisRun: false, concept: "stale-1" },+ { searchStatus: "watching", judgedThisRun: false, concept: "stale-2" },+ { searchStatus: "skipped" }, // no verdict, no judgedThisRun+ ];+ expect(pages.filter(p => p.concept).length).toBe(3); // the misleading count+ expect(billedJudgedCount(pages)).toBe(1); // the canonical, billed count+ expect(billedJudgedCount(pages) * JUDGE_CREDITS_PER_RESULT).toBe(5);+ });++ it("counts judged results across every billed outcome (alert/watch/ignore/already_seen)", () => {+ const pages: Page[] = [+ { searchStatus: "alert", judgedThisRun: true },+ { searchStatus: "watching", judgedThisRun: true },+ { searchStatus: "ignored", judgedThisRun: true },+ { searchStatus: "already_seen", judgedThisRun: true },+ { searchStatus: "skipped" }, // verdict failed -> not billed+ ];+ expect(billedJudgedCount(pages)).toBe(4); // -> 5*4 = 20 judge credits+ });+});diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.tsindex f8a1bf369..833a044c5 100644--- a/apps/api/src/services/monitoring/search/run.ts+++ b/apps/api/src/services/monitoring/search/run.ts@@ -486,11 +486,16 @@ export async function runSearchTarget(params: {status: reusedStatus,// Upserts replace metadata wholesale; spread prior metadata to preserve// event stamps (eventKey/eventLabel/...) that reconstruction needs.+ // Force judgedThisRun=false: this page is REUSED unchanged, not+ // re-judged, so it must not be counted/billed again even though prior+ // metadata may carry a stale concept/judgedThisRun from when it was+ // first judged.metadata: {...(knownCurrent?.metadata ?? {}),fingerprint,goalVersion,searchStatus: reusedStatus,+ judgedThisRun: false,},});continue;@@ -630,7 +635,23 @@ export async function runSearchTarget(params: {null;}- // Judged this result β the unit of flat billing, regardless of outcome.+ // Canonical "judged result" β the unit of flat billing.+ //+ // A judged result is a result we scraped (deep) or snippet-judged+ // (standard) and obtained a parsed AI verdict for THIS run, regardless of+ // the verdict's outcome (alert/watch/ignore/already_seen). This is exactly+ // what costs money β the scrape + the verdict LLM call β and exactly what+ // the docs price: "5 credits per result judged β covers scraping and+ // evaluating the result".+ //+ // NOT counted (and not billed): the router-skipped subset (never scraped),+ // results whose scrape/verdict failed (status "skipped", no verdict), raw /+ // judge-off results, and unchanged results REUSED from a prior run (those+ // were billed when first judged β see the !isNewOrChanged branch above,+ // which carries a stale `concept` over but must not re-bill). Reaching this+ // line is the single source of truth; every billed page is stamped+ // judgedThisRun=true in baseMeta so the billed count == count of pages with+ // judgedThisRun, with no ambiguity from leaked prior-run concepts.resultsJudged += 1;let decision = verdictToDecision(verdict);@@ -693,6 +714,11 @@ export async function runSearchTarget(params: {concept: verdict.concept,rationale: verdict.rationale,matchedQueries: c.matchedQueries,+ // Stamped iff resultsJudged was incremented just above β the canonical,+ // unambiguous billing signal. `concept` alone is unreliable: the reuse+ // path copies a prior run's concept onto an unchanged page that was NOT+ // re-judged this run. judgedThisRun is true ONLY on results billed now.+ judgedThisRun: true,...(verification && !verification.pass ? { verification } : {}),};diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex 4008913f4..fe8dd3be5 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -300,6 +300,61 @@ describe("FLAT deterministic search-monitor billing", () => {});});+ // Regression guard for the FLAT search billing contract that eval H asserts:+ // actual_credits = 2*ceil(totalResults/10) + 5*resultsJudged (search-only+ // when judge is off). These pin the deterministic sum, the never-zero-with-+ // results invariant, and that the retry path still bills.+ describe("flat search billing contract (eval H regression guard)", () => {+ // The deterministic figures the runner stamps onto a search target run.+ const searchCreditsFor = (totalResults: number) =>+ 2 * Math.ceil(totalResults / 10);+ const stampedSearchRun = (totalResults: number, resultsJudged: number) => ({+ targetId: "search-1",+ type: "search" as const,+ searchCompleted: true,+ resultCount: totalResults,+ searchCredits: searchCreditsFor(totalResults),+ judgeCredits: resultsJudged * 5,+ resultsJudged,+ });++ it("is the exact deterministic flat sum (the headline 17-credit trace)", () => {+ // results=6 -> search 2; judged=3 -> 15; total 17.+ expect(flatSearchTargetCredits([stampedSearchRun(6, 3)])).toBe(17);+ });++ it("never bills 0 on a check that returned results and judged them", () => {+ // results=6, judged=4 -> 2 + 20 = 22 (the judged#2 case that landed on 0).+ const credits = flatSearchTargetCredits([stampedSearchRun(6, 4)]);+ expect(credits).toBe(22);+ expect(credits).toBeGreaterThan(0);+ });++ it("the RETRY path still bills: rate-limited-then-succeeded run is summed", () => {+ // A run that hit rate-limiting, retried, and finally returned 6 results ++ // judged 4 must record the SAME deterministic figure as a clean run β the+ // searchCompleted=true guard ensures credits are stamped before summing.+ const retried = stampedSearchRun(6, 4);+ expect(retried.searchCompleted).toBe(true);+ expect(flatSearchTargetCredits([retried])).toBe(22);+ });++ it("search-only (judge off) bills just the search portion, never the judge", () => {+ // results=6, judged=0 -> 2 + 0 = 2 (the raw case, which is already correct).+ expect(flatSearchTargetCredits([stampedSearchRun(6, 0)])).toBe(2);+ });++ it("a still-running search (no credits stamped yet) sums to 0 β the reconciler must NOT finalize here", () => {+ // This is the pre-stamp shape the dispatcher writes before the inline+ // search finishes. flatSearchTargetCredits sees 0; the never-zero+ // guarantee instead relies on isMonitorCheckComplete refusing to finalize+ // a run whose searchCompleted flag is not yet true (see runner.ts).+ const pending = { targetId: "search-1", type: "search" as const };+ expect((pending as any).searchCompleted).toBeUndefined();+ expect(flatSearchTargetCredits([pending])).toBe(0);+ });+ });+it("page-sum never bills search-target pages (cost is at the check level)", () => {const targets: MonitorTarget[] = [searchTarget()];// Even with judgment + status, a search page contributes 0 to the page sum;