Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(monitor): search billing never-zero + correct judged-result count under retries

Developers Digestc4b3aeffcb22
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 6abfa8ddf..719edcf9a 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -113,6 +113,11 @@ type MonitorTargetRun =
// Search runs inline during dispatch; counts/credits filled in after.
targetId: string;
type: "search";
+ // False/undefined until the inline search finishes and stamps its credits.
+ // The reconciler must NOT finalize a check whose search run is still
+ // pending β€” otherwise it sums credits that haven't been written yet and
+ // bills 0. Set true ONLY once searchCredits/judgeCredits are populated.
+ searchCompleted?: boolean;
resultCount?: number;
matches?: number;
summary?: string;
@@ -1203,6 +1208,11 @@ export async function processMonitorCheckJob(
targetRun.searchCredits = searchResult.searchCredits;
targetRun.judgeCredits = searchResult.judgeCredits;
targetRun.resultsJudged = searchResult.resultsJudged;
+ // Mark complete LAST, after credits are populated: the reconciler keys
+ // its completeness check on this flag, so it can never finalize (and
+ // bill the as-yet-unstamped, zero credits) while the search β€” possibly
+ // mid rate-limit retry β€” is still running.
+ targetRun.searchCompleted = true;
}
}
@@ -1354,7 +1364,14 @@ async function isMonitorCheckComplete(
}
for (const target of targetResults) {
- if (target?.type === "scrape") {
+ if (target?.type === "search") {
+ // Search runs inline during dispatch and stamps searchCompleted=true only
+ // after its flat credits are written. Until then the run is in flight
+ // (e.g. retrying a rate-limited query), so treating it as complete would
+ // let the reconciler finalize with credits still at 0 β€” the never-zero
+ // guarantee depends on waiting here.
+ if (!target.searchCompleted) return false;
+ } else if (target?.type === "scrape") {
const expected = Array.isArray(target.expectedJobs)
? target.expectedJobs.length
: 0;
diff --git a/apps/api/src/services/monitoring/search/cost.test.ts b/apps/api/src/services/monitoring/search/cost.test.ts
index 522e3bc69..e08dc6155 100644
--- a/apps/api/src/services/monitoring/search/cost.test.ts
+++ b/apps/api/src/services/monitoring/search/cost.test.ts
@@ -60,3 +60,43 @@ describe("search-monitor FLAT judge billing rate", () => {
expect(judgeCreditsFor(10)).toBe(50);
});
});
+
+// The CANONICAL "judged result" definition, pinned so billing and the eval stay
+// in lockstep. A judged result is one scraped+judged (deep) or snippet-judged
+// (standard) THIS run β€” run.ts increments resultsJudged exactly once per such
+// result and stamps metadata.judgedThisRun=true on its persisted page. The
+// billed count therefore equals the number of pages with judgedThisRun=true.
+// Reused (unchanged), raw, and skipped pages are NOT judged this run and carry
+// judgedThisRun=false / absent β€” even if they retain a stale `concept`.
+describe("canonical judged-result count (billing == persisted signal)", () => {
+ type Page = { searchStatus: string; judgedThisRun?: boolean; concept?: string };
+ const billedJudgedCount = (pages: Page[]) =>
+ pages.filter(p => p.judgedThisRun === true).length;
+
+ it("counts judgedThisRun=true, NOT every page nor every page with a concept", () => {
+ // The judged#1 shape: 6 results, only 1 re-judged this run; the other two
+ // "concept" pages are REUSED from a prior run (judgedThisRun=false) and a
+ // skipped page got no verdict. Counting pages-with-concept gives 3 (the old
+ // eval undercount mismatch); the canonical count is 1.
+ const pages: Page[] = [
+ { searchStatus: "alert", judgedThisRun: true, concept: "fresh-verdict" },
+ { searchStatus: "already_seen", judgedThisRun: false, concept: "stale-1" },
+ { searchStatus: "watching", judgedThisRun: false, concept: "stale-2" },
+ { searchStatus: "skipped" }, // no verdict, no judgedThisRun
+ ];
+ expect(pages.filter(p => p.concept).length).toBe(3); // the misleading count
+ expect(billedJudgedCount(pages)).toBe(1); // the canonical, billed count
+ expect(billedJudgedCount(pages) * JUDGE_CREDITS_PER_RESULT).toBe(5);
+ });
+
+ it("counts judged results across every billed outcome (alert/watch/ignore/already_seen)", () => {
+ const pages: Page[] = [
+ { searchStatus: "alert", judgedThisRun: true },
+ { searchStatus: "watching", judgedThisRun: true },
+ { searchStatus: "ignored", judgedThisRun: true },
+ { searchStatus: "already_seen", judgedThisRun: true },
+ { searchStatus: "skipped" }, // verdict failed -> not billed
+ ];
+ expect(billedJudgedCount(pages)).toBe(4); // -> 5*4 = 20 judge credits
+ });
+});
diff --git a/apps/api/src/services/monitoring/search/run.ts b/apps/api/src/services/monitoring/search/run.ts
index f8a1bf369..833a044c5 100644
--- a/apps/api/src/services/monitoring/search/run.ts
+++ b/apps/api/src/services/monitoring/search/run.ts
@@ -486,11 +486,16 @@ export async function runSearchTarget(params: {
status: reusedStatus,
// Upserts replace metadata wholesale; spread prior metadata to preserve
// event stamps (eventKey/eventLabel/...) that reconstruction needs.
+ // Force judgedThisRun=false: this page is REUSED unchanged, not
+ // re-judged, so it must not be counted/billed again even though prior
+ // metadata may carry a stale concept/judgedThisRun from when it was
+ // first judged.
metadata: {
...(knownCurrent?.metadata ?? {}),
fingerprint,
goalVersion,
searchStatus: reusedStatus,
+ judgedThisRun: false,
},
});
continue;
@@ -630,7 +635,23 @@ export async function runSearchTarget(params: {
null;
}
- // Judged this result β€” the unit of flat billing, regardless of outcome.
+ // Canonical "judged result" β€” the unit of flat billing.
+ //
+ // A judged result is a result we scraped (deep) or snippet-judged
+ // (standard) and obtained a parsed AI verdict for THIS run, regardless of
+ // the verdict's outcome (alert/watch/ignore/already_seen). This is exactly
+ // what costs money β€” the scrape + the verdict LLM call β€” and exactly what
+ // the docs price: "5 credits per result judged β€” covers scraping and
+ // evaluating the result".
+ //
+ // NOT counted (and not billed): the router-skipped subset (never scraped),
+ // results whose scrape/verdict failed (status "skipped", no verdict), raw /
+ // judge-off results, and unchanged results REUSED from a prior run (those
+ // were billed when first judged β€” see the !isNewOrChanged branch above,
+ // which carries a stale `concept` over but must not re-bill). Reaching this
+ // line is the single source of truth; every billed page is stamped
+ // judgedThisRun=true in baseMeta so the billed count == count of pages with
+ // judgedThisRun, with no ambiguity from leaked prior-run concepts.
resultsJudged += 1;
let decision = verdictToDecision(verdict);
@@ -693,6 +714,11 @@ export async function runSearchTarget(params: {
concept: verdict.concept,
rationale: verdict.rationale,
matchedQueries: c.matchedQueries,
+ // Stamped iff resultsJudged was incremented just above β€” the canonical,
+ // unambiguous billing signal. `concept` alone is unreliable: the reuse
+ // path copies a prior run's concept onto an unchanged page that was NOT
+ // re-judged this run. judgedThisRun is true ONLY on results billed now.
+ judgedThisRun: true,
...(verification && !verification.pass ? { verification } : {}),
};
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index 4008913f4..fe8dd3be5 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -300,6 +300,61 @@ describe("FLAT deterministic search-monitor billing", () => {
});
});
+ // Regression guard for the FLAT search billing contract that eval H asserts:
+ // actual_credits = 2*ceil(totalResults/10) + 5*resultsJudged (search-only
+ // when judge is off). These pin the deterministic sum, the never-zero-with-
+ // results invariant, and that the retry path still bills.
+ describe("flat search billing contract (eval H regression guard)", () => {
+ // The deterministic figures the runner stamps onto a search target run.
+ const searchCreditsFor = (totalResults: number) =>
+ 2 * Math.ceil(totalResults / 10);
+ const stampedSearchRun = (totalResults: number, resultsJudged: number) => ({
+ targetId: "search-1",
+ type: "search" as const,
+ searchCompleted: true,
+ resultCount: totalResults,
+ searchCredits: searchCreditsFor(totalResults),
+ judgeCredits: resultsJudged * 5,
+ resultsJudged,
+ });
+
+ it("is the exact deterministic flat sum (the headline 17-credit trace)", () => {
+ // results=6 -> search 2; judged=3 -> 15; total 17.
+ expect(flatSearchTargetCredits([stampedSearchRun(6, 3)])).toBe(17);
+ });
+
+ it("never bills 0 on a check that returned results and judged them", () => {
+ // results=6, judged=4 -> 2 + 20 = 22 (the judged#2 case that landed on 0).
+ const credits = flatSearchTargetCredits([stampedSearchRun(6, 4)]);
+ expect(credits).toBe(22);
+ expect(credits).toBeGreaterThan(0);
+ });
+
+ it("the RETRY path still bills: rate-limited-then-succeeded run is summed", () => {
+ // A run that hit rate-limiting, retried, and finally returned 6 results +
+ // judged 4 must record the SAME deterministic figure as a clean run β€” the
+ // searchCompleted=true guard ensures credits are stamped before summing.
+ const retried = stampedSearchRun(6, 4);
+ expect(retried.searchCompleted).toBe(true);
+ expect(flatSearchTargetCredits([retried])).toBe(22);
+ });
+
+ it("search-only (judge off) bills just the search portion, never the judge", () => {
+ // results=6, judged=0 -> 2 + 0 = 2 (the raw case, which is already correct).
+ expect(flatSearchTargetCredits([stampedSearchRun(6, 0)])).toBe(2);
+ });
+
+ it("a still-running search (no credits stamped yet) sums to 0 β€” the reconciler must NOT finalize here", () => {
+ // This is the pre-stamp shape the dispatcher writes before the inline
+ // search finishes. flatSearchTargetCredits sees 0; the never-zero
+ // guarantee instead relies on isMonitorCheckComplete refusing to finalize
+ // a run whose searchCompleted flag is not yet true (see runner.ts).
+ const pending = { targetId: "search-1", type: "search" as const };
+ expect((pending as any).searchCompleted).toBeUndefined();
+ expect(flatSearchTargetCredits([pending])).toBe(0);
+ });
+ });
+
it("page-sum never bills search-target pages (cost is at the check level)", () => {
const targets: MonitorTarget[] = [searchTarget()];
// Even with judgment + status, a search page contributes 0 to the page sum;