Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Retry monitor judge when a true verdict has no changes (#5634)
Micah Stairsc513e8b7e640
diff --git a/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts b/apps/api/src/services/monitoring/judgeChange.telemetry.test.tsindex a718f93bb..68d653ef4 100644--- a/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts+++ b/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts@@ -215,9 +215,46 @@ describe("judgeChange telemetry", () => {});});+function contradictoryVerdict(overrides: Record<string, unknown> = {}) {+ return JSON.stringify({+ meaningful: true,+ confidence: "high",+ reason: "There are no meaningful new announcements added.",+ meaningfulChanges: [],+ ...overrides,+ });+}++function capturingLogger() {+ const warnings: Array<{ message: string; meta?: unknown }> = [];+ const logger = {+ warn: (message: string, meta?: unknown) => {+ warnings.push({ message, meta });+ },+ info: () => {},+ error: () => {},+ debug: () => {},+ child() {+ return logger;+ },+ };+ return { logger: logger as any, warnings };+}++function assertLogsOmitEvidence(+ warnings: Array<{ message: string; meta?: unknown }>,+) {+ const serialized = JSON.stringify(warnings);+ expect(serialized).not.toContain("Pro plan");+ expect(serialized).not.toContain("Track the Pro plan price");+ expect(serialized).not.toContain("capture the price table");+ expect(serialized).not.toContain("no meaningful new announcements");+}+describe("judgeChange behaviour", () => {beforeEach(() => {mocks.doGenerate.mockReset();+ (generateText as Mock).mockClear();vi.spyOn(Math, "random").mockReturnValue(0);});@@ -241,6 +278,33 @@ describe("judgeChange behaviour", () => {},],});+ expect(mocks.doGenerate).toHaveBeenCalledTimes(1);+ });++ it("does not let reason wording override a true verdict that still has a valid change", async () => {+ mocks.doGenerate.mockResolvedValue(+ providerResult(+ JSON.stringify({+ meaningful: true,+ confidence: "high",+ reason: "There are no meaningful new announcements added.",+ meaningfulChanges: [+ {+ type: "changed",+ before: "Pro plan: $19",+ after: "Pro plan: $24",+ reason: "The goal tracks the Pro price.",+ },+ ],+ }),+ ),+ );++ const result = await judge();++ expect(mocks.doGenerate).toHaveBeenCalledTimes(1);+ expect(result.meaningful).toBe(true);+ expect(result.meaningfulChanges).toHaveLength(1);});it("defaults to meaningful when the response is not JSON", async () => {@@ -268,4 +332,211 @@ describe("judgeChange behaviour", () => {expect(mocks.doGenerate).not.toHaveBeenCalled();});++ it("retries when every change entry is removed by sanitization", async () => {+ const { logger, warnings } = capturingLogger();+ mocks.doGenerate+ .mockResolvedValueOnce(+ providerResult(+ contradictoryVerdict({+ meaningfulChanges: [+ {+ type: "added",+ before: "x",+ after: "y",+ reason: "invalid pair",+ },+ ],+ }),+ ),+ )+ .mockResolvedValueOnce(providerResult(PRICE_CHANGED));++ const result = await judge({+ logger,+ extractionPrompt: "capture the price table",+ });++ expect(mocks.doGenerate).toHaveBeenCalledTimes(2);+ expect(result.meaningful).toBe(true);+ expect(result.meaningfulChanges).toEqual([+ {+ type: "changed",+ before: "Pro plan: $19",+ after: "Pro plan: $24",+ reason: "The goal tracks the Pro price.",+ },+ ]);+ const retryBlob = JSON.stringify(mocks.doGenerate.mock.calls[1][0]);+ expect(retryBlob).toContain("MONITOR GOAL");+ expect(retryBlob).toContain("Track the Pro plan price");+ expect(retryBlob).toContain("Pro plan: $19");+ expect(retryBlob).toContain("capture the price table");+ expect(retryBlob).toContain("no valid meaningfulChanges remained");+ expect(mocks.doGenerate.mock.calls[1][0].providerOptions).toEqual({+ vertex: {+ thinkingConfig: { thinkingLevel: "minimal" },+ labels: { functionId: "judgeChangeValidationRetry", ...LABELS },+ },+ });+ expect(telemetryOfCalls()[1]).toMatchObject({+ functionId: "monitor/judgeChangeValidationRetry",+ metadata: expect.objectContaining({+ jobId: "test-check",+ feature: "monitor_judge_validation_retry",+ }),+ });+ expect(warnings.map(warning => warning.message)).toEqual([+ "Judge validation failed; retrying once",+ ]);+ assertLogsOmitEvidence(warnings);+ });++ it("accepts a retry that returns meaningful false", async () => {+ const { logger, warnings } = capturingLogger();+ mocks.doGenerate+ .mockResolvedValueOnce(providerResult(contradictoryVerdict()))+ .mockResolvedValueOnce(+ providerResult(+ JSON.stringify({+ meaningful: false,+ confidence: "medium",+ reason: "There are no meaningful new announcements added.",+ meaningfulChanges: [+ { type: "added", before: "x", after: "y", reason: "invalid" },+ ],+ }),+ ),+ );++ const result = await judge({ logger });++ expect(mocks.doGenerate).toHaveBeenCalledTimes(2);+ expect(result).toMatchObject({+ meaningful: false,+ confidence: "medium",+ meaningfulChanges: [],+ });+ expect(warnings).toHaveLength(1);+ assertLogsOmitEvidence(warnings);+ });++ it("fails open when the validation retry is still contradictory", async () => {+ const { logger, warnings } = capturingLogger();+ mocks.doGenerate.mockResolvedValue(providerResult(contradictoryVerdict()));++ const result = await judge({ logger });++ expect(mocks.doGenerate).toHaveBeenCalledTimes(2);+ expect(result).toEqual({+ meaningful: true,+ confidence: "high",+ reason: "There are no meaningful new announcements added.",+ meaningfulChanges: [],+ });+ expect(warnings.map(warning => warning.message)).toEqual([+ "Judge validation failed; retrying once",+ "Judge validation retry was still invalid; failing open",+ ]);+ expect(warnings[1].meta).toMatchObject({+ validation: "meaningful_without_changes",+ outcome: "contradictory",+ });+ assertLogsOmitEvidence(warnings);+ });++ it("fails open when the validation retry errors", async () => {+ const { logger, warnings } = capturingLogger();+ mocks.doGenerate+ .mockResolvedValueOnce(providerResult(contradictoryVerdict()))+ .mockRejectedValue(new Error("provider unavailable"));++ const result = await judge({ logger });++ expect(mocks.doGenerate).toHaveBeenCalledTimes(4);+ expect(result).toMatchObject({ meaningful: true, confidence: "low" });+ expect(result.reason).toContain("provider unavailable");+ expect(warnings.map(warning => warning.message)).toEqual([+ "Judge validation failed; retrying once",+ "Judge validation retry failed; failing open",+ ]);+ expect(warnings[1].meta).toEqual({+ validation: "meaningful_without_changes",+ errorName: "Error",+ });+ assertLogsOmitEvidence(warnings);+ });++ it("fails open when the validation retry is unparseable", async () => {+ const { logger, warnings } = capturingLogger();+ mocks.doGenerate+ .mockResolvedValueOnce(providerResult(contradictoryVerdict()))+ .mockResolvedValueOnce(+ providerResult("no idea, the diff was Pro plan: $19"),+ );++ const result = await judge({ logger });++ expect(mocks.doGenerate).toHaveBeenCalledTimes(2);+ expect(result).toMatchObject({+ meaningful: true,+ confidence: "low",+ reason: "Judge response unparseable β defaulting to meaningful.",+ meaningfulChanges: [],+ });+ expect(warnings.map(warning => warning.message)).toEqual([+ "Judge validation failed; retrying once",+ "Judge returned unparseable response",+ "Judge validation retry was still invalid; failing open",+ ]);+ expect(warnings[2].meta).toMatchObject({ outcome: "unparseable" });+ assertLogsOmitEvidence(warnings);+ });++ it("fails open when the validation retry is not valid JSON", async () => {+ const { logger, warnings } = capturingLogger();+ mocks.doGenerate+ .mockResolvedValueOnce(providerResult(contradictoryVerdict()))+ .mockResolvedValueOnce(+ providerResult('{ "meaningful": true, "reason": "Pro plan: $19", }'),+ );++ const result = await judge({ logger });++ expect(mocks.doGenerate).toHaveBeenCalledTimes(2);+ expect(result).toMatchObject({+ meaningful: true,+ confidence: "low",+ reason: "Judge response not valid JSON β defaulting to meaningful.",+ meaningfulChanges: [],+ });+ expect(warnings.map(warning => warning.message)).toEqual([+ "Judge validation failed; retrying once",+ "Judge JSON parse failed β defaulting to meaningful",+ "Judge validation retry was still invalid; failing open",+ ]);+ expect(warnings[1].meta).toEqual({});+ expect(warnings[2].meta).toMatchObject({ outcome: "unparseable" });+ assertLogsOmitEvidence(warnings);+ });++ it("does not retry when the meaningful boolean is missing", async () => {+ mocks.doGenerate.mockResolvedValue(+ providerResult(+ JSON.stringify({+ confidence: "high",+ reason: "There are no meaningful new announcements added.",+ meaningfulChanges: [],+ }),+ ),+ );++ const result = await judge();++ expect(mocks.doGenerate).toHaveBeenCalledTimes(1);+ expect(result).toMatchObject({+ meaningful: true,+ meaningfulChanges: [],+ });+ });});diff --git a/apps/api/src/services/monitoring/judgeChange.ts b/apps/api/src/services/monitoring/judgeChange.tsindex 21cb0102c..beeaeac81 100644--- a/apps/api/src/services/monitoring/judgeChange.ts+++ b/apps/api/src/services/monitoring/judgeChange.ts@@ -132,15 +132,30 @@ const JUDGE_ATTEMPT_TIMEOUT_MS = 30_000;const JUDGE_MAX_ATTEMPTS = 3;const JUDGE_BACKOFF_MS = [300, 800];const JUDGE_FUNCTION_ID = "judgeChange";+const JUDGE_VALIDATION_RETRY_FUNCTION_ID = "judgeChangeValidationRetry";+// One logical retry. Each logical call still uses JUDGE_MAX_ATTEMPTS provider tries.+const JUDGE_VALIDATION_RETRIES = 1;+const VALIDATION_FAILURE = "meaningful_without_changes";++const VALIDATION_RETRY_NOTE = `The previous JSON verdict was invalid: "meaningful" was true, but no valid meaningfulChanges remained. Return one JSON object only, using the same MONITOR GOAL and PAGE DIFF / FIELD DIFFS above.+- If something matching the goal changed, set meaningful to true and include at least one valid meaningfulChanges entry with verbatim before/after evidence from the diff.+- If nothing matching the goal changed, set meaningful to false and set meaningfulChanges to [].+The meaningful boolean and meaningfulChanges must agree. Do not add prose outside the JSON.`;function judgeTelemetry(labels: LlmUsageLabels | undefined,zeroDataRetention: boolean | undefined,+ functionId: string,) {return {isEnabled: !(zeroDataRetention || isZeroDataRetentionActive()),- functionId: `monitor/${JUDGE_FUNCTION_ID}`,- metadata: monitorTelemetryMetadata("monitor_judge", labels),+ functionId: `monitor/${functionId}`,+ metadata: monitorTelemetryMetadata(+ functionId === JUDGE_FUNCTION_ID+ ? "monitor_judge"+ : "monitor_judge_validation_retry",+ labels,+ ),};}@@ -149,7 +164,9 @@ async function callGemini(args: {labels?: LlmUsageLabels;zeroDataRetention?: boolean;teamFlags?: TeamFlags | null;+ functionId?: string;}): Promise<{ text: string }> {+ const functionId = args.functionId ?? JUDGE_FUNCTION_ID;const judgeModel = withUsageTelemetry(googleModel(JUDGE_MODEL_NAME, {teamFlags: args.teamFlags,@@ -174,8 +191,9 @@ async function callGemini(args: {experimental_telemetry: judgeTelemetry(args.labels,args.zeroDataRetention,+ functionId,),- ...googleProviderOptions(JUDGE_FUNCTION_ID, args.labels),+ ...googleProviderOptions(functionId, args.labels),});return { text: result.text?.trim() ?? "" };} catch (error) {@@ -191,6 +209,114 @@ async function callGemini(args: {throw lastError;}+type ParsedJudgeText = {+ judgment: JudgmentResult;+ // Explicit meaningful=true whose change list is empty after sanitization.+ contradictory: boolean;+ // Unparseable or invalid JSON. Already a fail-open judgment.+ fallback: boolean;+};++function parseJudgeText(+ text: string,+ logger: Logger,+ options: { includeResponsePeek: boolean },+): ParsedJudgeText {+ const peek = options.includeResponsePeek+ ? (value: string) => value.slice(0, 200)+ : undefined;+ const match = text.match(/\{[\s\S]*\}/);+ if (!match) {+ logger.warn("Judge returned unparseable response", {+ ...(peek ? { textPeek: peek(text) } : {}),+ });+ return {+ contradictory: false,+ fallback: true,+ judgment: {+ meaningful: true,+ confidence: "low",+ reason: "Judge response unparseable β defaulting to meaningful.",+ meaningfulChanges: [],+ },+ };+ }++ let parsed: Partial<JudgmentResult>;+ try {+ parsed = JSON.parse(match[0]) as Partial<JudgmentResult>;+ } catch (parseError) {+ logger.warn("Judge JSON parse failed β defaulting to meaningful", {+ ...(peek+ ? {+ textPeek: peek(match[0]),+ parseError:+ parseError instanceof Error+ ? parseError.message+ : String(parseError),+ }+ : {}),+ });+ return {+ contradictory: false,+ fallback: true,+ judgment: {+ meaningful: true,+ confidence: "low",+ reason: "Judge response not valid JSON β defaulting to meaningful.",+ meaningfulChanges: [],+ },+ };+ }++ // A stored boolean does not narrow `parsed.meaningful` under strictNullChecks,+ // so resolve the three cases explicitly. Missing or non-boolean stays the+ // existing fail-open true and is not a validation retry.+ const parsedMeaningful = parsed.meaningful;+ const meaningfulExplicit =+ parsedMeaningful === true || parsedMeaningful === false;+ const meaningful: boolean =+ parsedMeaningful === true+ ? true+ : parsedMeaningful === false+ ? false+ : true;+ const meaningfulChanges = sanitizeMeaningfulChanges(+ parsed.meaningfulChanges,+ meaningful,+ );+ return {+ contradictory:+ meaningful === true &&+ meaningfulChanges.length === 0 &&+ meaningfulExplicit,+ fallback: false,+ judgment: {+ meaningful,+ confidence:+ parsed.confidence === "high" ||+ parsed.confidence === "medium" ||+ parsed.confidence === "low"+ ? parsed.confidence+ : "low",+ reason:+ typeof parsed.reason === "string" && parsed.reason.length > 0+ ? parsed.reason+ : "No reason provided.",+ meaningfulChanges,+ },+ };+}++function failOpenAfterCallError(error: unknown): JudgmentResult {+ return {+ meaningful: true,+ confidence: "low",+ reason: `Judge call failed β defaulting to meaningful. (${error instanceof Error ? error.message : "unknown"})`,+ meaningfulChanges: [],+ };+}+export async function judgeChange(args: JudgeChangeArgs,): Promise<JudgmentResult> {@@ -230,72 +356,53 @@ export async function judgeChange(};}const userBlock = parts.join("\n\n");+ const callArgs = { labels, zeroDataRetention, teamFlags };try {- const { text } = await callGemini({- userBlock,- labels,- zeroDataRetention,- teamFlags,- });- const match = text.match(/\{[\s\S]*\}/);- if (!match) {- logger.warn("Judge returned unparseable response", {- textPeek: text.slice(0, 200),+ const { text } = await callGemini({ userBlock, ...callArgs });+ let outcome = parseJudgeText(text, logger, { includeResponsePeek: true });++ // Not noise: an explicit true verdict with no surviving changes is invalid.+ // Ask once for a consistent verdict, keeping the original goal and diff.+ // Reason text is never used to flip the boolean.+ let validationRetriesLeft = JUDGE_VALIDATION_RETRIES;+ while (outcome.contradictory && validationRetriesLeft > 0) {+ validationRetriesLeft -= 1;+ logger.warn("Judge validation failed; retrying once", {+ validation: VALIDATION_FAILURE,});- return {- meaningful: true,- confidence: "low",- reason: "Judge response unparseable β defaulting to meaningful.",- meaningfulChanges: [],- };+ try {+ const retry = await callGemini({+ ...callArgs,+ userBlock: `${userBlock}\n\n${VALIDATION_RETRY_NOTE}`,+ functionId: JUDGE_VALIDATION_RETRY_FUNCTION_ID,+ });+ outcome = parseJudgeText(retry.text, logger, {+ includeResponsePeek: false,+ });+ } catch (error) {+ if (error instanceof UsOnlyUnavailableError) throw error;+ logger.warn("Judge validation retry failed; failing open", {+ validation: VALIDATION_FAILURE,+ errorName: error instanceof Error ? error.name : "unknown",+ });+ return failOpenAfterCallError(error);+ }}- let parsed: Partial<JudgmentResult>;- try {- parsed = JSON.parse(match[0]) as Partial<JudgmentResult>;- } catch (parseError) {- logger.warn("Judge JSON parse failed β defaulting to meaningful", {- textPeek: match[0].slice(0, 200),- parseError:- parseError instanceof Error ? parseError.message : String(parseError),+ if (+ outcome.contradictory ||+ (outcome.fallback && validationRetriesLeft < JUDGE_VALIDATION_RETRIES)+ ) {+ logger.warn("Judge validation retry was still invalid; failing open", {+ validation: VALIDATION_FAILURE,+ outcome: outcome.contradictory ? "contradictory" : "unparseable",});- return {- meaningful: true,- confidence: "low",- reason: "Judge response not valid JSON β defaulting to meaningful.",- meaningfulChanges: [],- };}- const meaningful =- parsed.meaningful === true || parsed.meaningful === false- ? parsed.meaningful- : true;- return {- meaningful,- confidence:- parsed.confidence === "high" ||- parsed.confidence === "medium" ||- parsed.confidence === "low"- ? parsed.confidence- : "low",- reason:- typeof parsed.reason === "string" && parsed.reason.length > 0- ? parsed.reason- : "No reason provided.",- meaningfulChanges: sanitizeMeaningfulChanges(- parsed.meaningfulChanges,- meaningful,- ),- };+ return outcome.judgment;} catch (error) {if (error instanceof UsOnlyUnavailableError) throw error;logger.error("Judge call failed", { error });- return {- meaningful: true,- confidence: "low",- reason: `Judge call failed β defaulting to meaningful. (${error instanceof Error ? error.message : "unknown"})`,- meaningfulChanges: [],- };+ return failOpenAfterCallError(error);}}