Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Retry monitor judge when a true verdict has no changes (#5634)

Micah Stairsc513e8b7e640
diff --git a/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts b/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts
index a718f93bb..68d653ef4 100644
--- a/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts
+++ b/apps/api/src/services/monitoring/judgeChange.telemetry.test.ts
@@ -215,9 +215,46 @@ describe("judgeChange telemetry", () => {
});
});
+function contradictoryVerdict(overrides: Record<string, unknown> = {}) {
+ return JSON.stringify({
+ meaningful: true,
+ confidence: "high",
+ reason: "There are no meaningful new announcements added.",
+ meaningfulChanges: [],
+ ...overrides,
+ });
+}
+
+function capturingLogger() {
+ const warnings: Array<{ message: string; meta?: unknown }> = [];
+ const logger = {
+ warn: (message: string, meta?: unknown) => {
+ warnings.push({ message, meta });
+ },
+ info: () => {},
+ error: () => {},
+ debug: () => {},
+ child() {
+ return logger;
+ },
+ };
+ return { logger: logger as any, warnings };
+}
+
+function assertLogsOmitEvidence(
+ warnings: Array<{ message: string; meta?: unknown }>,
+) {
+ const serialized = JSON.stringify(warnings);
+ expect(serialized).not.toContain("Pro plan");
+ expect(serialized).not.toContain("Track the Pro plan price");
+ expect(serialized).not.toContain("capture the price table");
+ expect(serialized).not.toContain("no meaningful new announcements");
+}
+
describe("judgeChange behaviour", () => {
beforeEach(() => {
mocks.doGenerate.mockReset();
+ (generateText as Mock).mockClear();
vi.spyOn(Math, "random").mockReturnValue(0);
});
@@ -241,6 +278,33 @@ describe("judgeChange behaviour", () => {
},
],
});
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not let reason wording override a true verdict that still has a valid change", async () => {
+ mocks.doGenerate.mockResolvedValue(
+ providerResult(
+ JSON.stringify({
+ meaningful: true,
+ confidence: "high",
+ reason: "There are no meaningful new announcements added.",
+ meaningfulChanges: [
+ {
+ type: "changed",
+ before: "Pro plan: $19",
+ after: "Pro plan: $24",
+ reason: "The goal tracks the Pro price.",
+ },
+ ],
+ }),
+ ),
+ );
+
+ const result = await judge();
+
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(1);
+ expect(result.meaningful).toBe(true);
+ expect(result.meaningfulChanges).toHaveLength(1);
});
it("defaults to meaningful when the response is not JSON", async () => {
@@ -268,4 +332,211 @@ describe("judgeChange behaviour", () => {
expect(mocks.doGenerate).not.toHaveBeenCalled();
});
+
+ it("retries when every change entry is removed by sanitization", async () => {
+ const { logger, warnings } = capturingLogger();
+ mocks.doGenerate
+ .mockResolvedValueOnce(
+ providerResult(
+ contradictoryVerdict({
+ meaningfulChanges: [
+ {
+ type: "added",
+ before: "x",
+ after: "y",
+ reason: "invalid pair",
+ },
+ ],
+ }),
+ ),
+ )
+ .mockResolvedValueOnce(providerResult(PRICE_CHANGED));
+
+ const result = await judge({
+ logger,
+ extractionPrompt: "capture the price table",
+ });
+
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(2);
+ expect(result.meaningful).toBe(true);
+ expect(result.meaningfulChanges).toEqual([
+ {
+ type: "changed",
+ before: "Pro plan: $19",
+ after: "Pro plan: $24",
+ reason: "The goal tracks the Pro price.",
+ },
+ ]);
+ const retryBlob = JSON.stringify(mocks.doGenerate.mock.calls[1][0]);
+ expect(retryBlob).toContain("MONITOR GOAL");
+ expect(retryBlob).toContain("Track the Pro plan price");
+ expect(retryBlob).toContain("Pro plan: $19");
+ expect(retryBlob).toContain("capture the price table");
+ expect(retryBlob).toContain("no valid meaningfulChanges remained");
+ expect(mocks.doGenerate.mock.calls[1][0].providerOptions).toEqual({
+ vertex: {
+ thinkingConfig: { thinkingLevel: "minimal" },
+ labels: { functionId: "judgeChangeValidationRetry", ...LABELS },
+ },
+ });
+ expect(telemetryOfCalls()[1]).toMatchObject({
+ functionId: "monitor/judgeChangeValidationRetry",
+ metadata: expect.objectContaining({
+ jobId: "test-check",
+ feature: "monitor_judge_validation_retry",
+ }),
+ });
+ expect(warnings.map(warning => warning.message)).toEqual([
+ "Judge validation failed; retrying once",
+ ]);
+ assertLogsOmitEvidence(warnings);
+ });
+
+ it("accepts a retry that returns meaningful false", async () => {
+ const { logger, warnings } = capturingLogger();
+ mocks.doGenerate
+ .mockResolvedValueOnce(providerResult(contradictoryVerdict()))
+ .mockResolvedValueOnce(
+ providerResult(
+ JSON.stringify({
+ meaningful: false,
+ confidence: "medium",
+ reason: "There are no meaningful new announcements added.",
+ meaningfulChanges: [
+ { type: "added", before: "x", after: "y", reason: "invalid" },
+ ],
+ }),
+ ),
+ );
+
+ const result = await judge({ logger });
+
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(2);
+ expect(result).toMatchObject({
+ meaningful: false,
+ confidence: "medium",
+ meaningfulChanges: [],
+ });
+ expect(warnings).toHaveLength(1);
+ assertLogsOmitEvidence(warnings);
+ });
+
+ it("fails open when the validation retry is still contradictory", async () => {
+ const { logger, warnings } = capturingLogger();
+ mocks.doGenerate.mockResolvedValue(providerResult(contradictoryVerdict()));
+
+ const result = await judge({ logger });
+
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(2);
+ expect(result).toEqual({
+ meaningful: true,
+ confidence: "high",
+ reason: "There are no meaningful new announcements added.",
+ meaningfulChanges: [],
+ });
+ expect(warnings.map(warning => warning.message)).toEqual([
+ "Judge validation failed; retrying once",
+ "Judge validation retry was still invalid; failing open",
+ ]);
+ expect(warnings[1].meta).toMatchObject({
+ validation: "meaningful_without_changes",
+ outcome: "contradictory",
+ });
+ assertLogsOmitEvidence(warnings);
+ });
+
+ it("fails open when the validation retry errors", async () => {
+ const { logger, warnings } = capturingLogger();
+ mocks.doGenerate
+ .mockResolvedValueOnce(providerResult(contradictoryVerdict()))
+ .mockRejectedValue(new Error("provider unavailable"));
+
+ const result = await judge({ logger });
+
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(4);
+ expect(result).toMatchObject({ meaningful: true, confidence: "low" });
+ expect(result.reason).toContain("provider unavailable");
+ expect(warnings.map(warning => warning.message)).toEqual([
+ "Judge validation failed; retrying once",
+ "Judge validation retry failed; failing open",
+ ]);
+ expect(warnings[1].meta).toEqual({
+ validation: "meaningful_without_changes",
+ errorName: "Error",
+ });
+ assertLogsOmitEvidence(warnings);
+ });
+
+ it("fails open when the validation retry is unparseable", async () => {
+ const { logger, warnings } = capturingLogger();
+ mocks.doGenerate
+ .mockResolvedValueOnce(providerResult(contradictoryVerdict()))
+ .mockResolvedValueOnce(
+ providerResult("no idea, the diff was Pro plan: $19"),
+ );
+
+ const result = await judge({ logger });
+
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(2);
+ expect(result).toMatchObject({
+ meaningful: true,
+ confidence: "low",
+ reason: "Judge response unparseable β€” defaulting to meaningful.",
+ meaningfulChanges: [],
+ });
+ expect(warnings.map(warning => warning.message)).toEqual([
+ "Judge validation failed; retrying once",
+ "Judge returned unparseable response",
+ "Judge validation retry was still invalid; failing open",
+ ]);
+ expect(warnings[2].meta).toMatchObject({ outcome: "unparseable" });
+ assertLogsOmitEvidence(warnings);
+ });
+
+ it("fails open when the validation retry is not valid JSON", async () => {
+ const { logger, warnings } = capturingLogger();
+ mocks.doGenerate
+ .mockResolvedValueOnce(providerResult(contradictoryVerdict()))
+ .mockResolvedValueOnce(
+ providerResult('{ "meaningful": true, "reason": "Pro plan: $19", }'),
+ );
+
+ const result = await judge({ logger });
+
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(2);
+ expect(result).toMatchObject({
+ meaningful: true,
+ confidence: "low",
+ reason: "Judge response not valid JSON β€” defaulting to meaningful.",
+ meaningfulChanges: [],
+ });
+ expect(warnings.map(warning => warning.message)).toEqual([
+ "Judge validation failed; retrying once",
+ "Judge JSON parse failed β€” defaulting to meaningful",
+ "Judge validation retry was still invalid; failing open",
+ ]);
+ expect(warnings[1].meta).toEqual({});
+ expect(warnings[2].meta).toMatchObject({ outcome: "unparseable" });
+ assertLogsOmitEvidence(warnings);
+ });
+
+ it("does not retry when the meaningful boolean is missing", async () => {
+ mocks.doGenerate.mockResolvedValue(
+ providerResult(
+ JSON.stringify({
+ confidence: "high",
+ reason: "There are no meaningful new announcements added.",
+ meaningfulChanges: [],
+ }),
+ ),
+ );
+
+ const result = await judge();
+
+ expect(mocks.doGenerate).toHaveBeenCalledTimes(1);
+ expect(result).toMatchObject({
+ meaningful: true,
+ meaningfulChanges: [],
+ });
+ });
});
diff --git a/apps/api/src/services/monitoring/judgeChange.ts b/apps/api/src/services/monitoring/judgeChange.ts
index 21cb0102c..beeaeac81 100644
--- a/apps/api/src/services/monitoring/judgeChange.ts
+++ b/apps/api/src/services/monitoring/judgeChange.ts
@@ -132,15 +132,30 @@ const JUDGE_ATTEMPT_TIMEOUT_MS = 30_000;
const JUDGE_MAX_ATTEMPTS = 3;
const JUDGE_BACKOFF_MS = [300, 800];
const JUDGE_FUNCTION_ID = "judgeChange";
+const JUDGE_VALIDATION_RETRY_FUNCTION_ID = "judgeChangeValidationRetry";
+// One logical retry. Each logical call still uses JUDGE_MAX_ATTEMPTS provider tries.
+const JUDGE_VALIDATION_RETRIES = 1;
+const VALIDATION_FAILURE = "meaningful_without_changes";
+
+const VALIDATION_RETRY_NOTE = `The previous JSON verdict was invalid: "meaningful" was true, but no valid meaningfulChanges remained. Return one JSON object only, using the same MONITOR GOAL and PAGE DIFF / FIELD DIFFS above.
+- If something matching the goal changed, set meaningful to true and include at least one valid meaningfulChanges entry with verbatim before/after evidence from the diff.
+- If nothing matching the goal changed, set meaningful to false and set meaningfulChanges to [].
+The meaningful boolean and meaningfulChanges must agree. Do not add prose outside the JSON.`;
function judgeTelemetry(
labels: LlmUsageLabels | undefined,
zeroDataRetention: boolean | undefined,
+ functionId: string,
) {
return {
isEnabled: !(zeroDataRetention || isZeroDataRetentionActive()),
- functionId: `monitor/${JUDGE_FUNCTION_ID}`,
- metadata: monitorTelemetryMetadata("monitor_judge", labels),
+ functionId: `monitor/${functionId}`,
+ metadata: monitorTelemetryMetadata(
+ functionId === JUDGE_FUNCTION_ID
+ ? "monitor_judge"
+ : "monitor_judge_validation_retry",
+ labels,
+ ),
};
}
@@ -149,7 +164,9 @@ async function callGemini(args: {
labels?: LlmUsageLabels;
zeroDataRetention?: boolean;
teamFlags?: TeamFlags | null;
+ functionId?: string;
}): Promise<{ text: string }> {
+ const functionId = args.functionId ?? JUDGE_FUNCTION_ID;
const judgeModel = withUsageTelemetry(
googleModel(JUDGE_MODEL_NAME, {
teamFlags: args.teamFlags,
@@ -174,8 +191,9 @@ async function callGemini(args: {
experimental_telemetry: judgeTelemetry(
args.labels,
args.zeroDataRetention,
+ functionId,
),
- ...googleProviderOptions(JUDGE_FUNCTION_ID, args.labels),
+ ...googleProviderOptions(functionId, args.labels),
});
return { text: result.text?.trim() ?? "" };
} catch (error) {
@@ -191,6 +209,114 @@ async function callGemini(args: {
throw lastError;
}
+type ParsedJudgeText = {
+ judgment: JudgmentResult;
+ // Explicit meaningful=true whose change list is empty after sanitization.
+ contradictory: boolean;
+ // Unparseable or invalid JSON. Already a fail-open judgment.
+ fallback: boolean;
+};
+
+function parseJudgeText(
+ text: string,
+ logger: Logger,
+ options: { includeResponsePeek: boolean },
+): ParsedJudgeText {
+ const peek = options.includeResponsePeek
+ ? (value: string) => value.slice(0, 200)
+ : undefined;
+ const match = text.match(/\{[\s\S]*\}/);
+ if (!match) {
+ logger.warn("Judge returned unparseable response", {
+ ...(peek ? { textPeek: peek(text) } : {}),
+ });
+ return {
+ contradictory: false,
+ fallback: true,
+ judgment: {
+ meaningful: true,
+ confidence: "low",
+ reason: "Judge response unparseable β€” defaulting to meaningful.",
+ meaningfulChanges: [],
+ },
+ };
+ }
+
+ let parsed: Partial<JudgmentResult>;
+ try {
+ parsed = JSON.parse(match[0]) as Partial<JudgmentResult>;
+ } catch (parseError) {
+ logger.warn("Judge JSON parse failed β€” defaulting to meaningful", {
+ ...(peek
+ ? {
+ textPeek: peek(match[0]),
+ parseError:
+ parseError instanceof Error
+ ? parseError.message
+ : String(parseError),
+ }
+ : {}),
+ });
+ return {
+ contradictory: false,
+ fallback: true,
+ judgment: {
+ meaningful: true,
+ confidence: "low",
+ reason: "Judge response not valid JSON β€” defaulting to meaningful.",
+ meaningfulChanges: [],
+ },
+ };
+ }
+
+ // A stored boolean does not narrow `parsed.meaningful` under strictNullChecks,
+ // so resolve the three cases explicitly. Missing or non-boolean stays the
+ // existing fail-open true and is not a validation retry.
+ const parsedMeaningful = parsed.meaningful;
+ const meaningfulExplicit =
+ parsedMeaningful === true || parsedMeaningful === false;
+ const meaningful: boolean =
+ parsedMeaningful === true
+ ? true
+ : parsedMeaningful === false
+ ? false
+ : true;
+ const meaningfulChanges = sanitizeMeaningfulChanges(
+ parsed.meaningfulChanges,
+ meaningful,
+ );
+ return {
+ contradictory:
+ meaningful === true &&
+ meaningfulChanges.length === 0 &&
+ meaningfulExplicit,
+ fallback: false,
+ judgment: {
+ meaningful,
+ confidence:
+ parsed.confidence === "high" ||
+ parsed.confidence === "medium" ||
+ parsed.confidence === "low"
+ ? parsed.confidence
+ : "low",
+ reason:
+ typeof parsed.reason === "string" && parsed.reason.length > 0
+ ? parsed.reason
+ : "No reason provided.",
+ meaningfulChanges,
+ },
+ };
+}
+
+function failOpenAfterCallError(error: unknown): JudgmentResult {
+ return {
+ meaningful: true,
+ confidence: "low",
+ reason: `Judge call failed β€” defaulting to meaningful. (${error instanceof Error ? error.message : "unknown"})`,
+ meaningfulChanges: [],
+ };
+}
+
export async function judgeChange(
args: JudgeChangeArgs,
): Promise<JudgmentResult> {
@@ -230,72 +356,53 @@ export async function judgeChange(
};
}
const userBlock = parts.join("\n\n");
+ const callArgs = { labels, zeroDataRetention, teamFlags };
try {
- const { text } = await callGemini({
- userBlock,
- labels,
- zeroDataRetention,
- teamFlags,
- });
- const match = text.match(/\{[\s\S]*\}/);
- if (!match) {
- logger.warn("Judge returned unparseable response", {
- textPeek: text.slice(0, 200),
+ const { text } = await callGemini({ userBlock, ...callArgs });
+ let outcome = parseJudgeText(text, logger, { includeResponsePeek: true });
+
+ // Not noise: an explicit true verdict with no surviving changes is invalid.
+ // Ask once for a consistent verdict, keeping the original goal and diff.
+ // Reason text is never used to flip the boolean.
+ let validationRetriesLeft = JUDGE_VALIDATION_RETRIES;
+ while (outcome.contradictory && validationRetriesLeft > 0) {
+ validationRetriesLeft -= 1;
+ logger.warn("Judge validation failed; retrying once", {
+ validation: VALIDATION_FAILURE,
});
- return {
- meaningful: true,
- confidence: "low",
- reason: "Judge response unparseable β€” defaulting to meaningful.",
- meaningfulChanges: [],
- };
+ try {
+ const retry = await callGemini({
+ ...callArgs,
+ userBlock: `${userBlock}\n\n${VALIDATION_RETRY_NOTE}`,
+ functionId: JUDGE_VALIDATION_RETRY_FUNCTION_ID,
+ });
+ outcome = parseJudgeText(retry.text, logger, {
+ includeResponsePeek: false,
+ });
+ } catch (error) {
+ if (error instanceof UsOnlyUnavailableError) throw error;
+ logger.warn("Judge validation retry failed; failing open", {
+ validation: VALIDATION_FAILURE,
+ errorName: error instanceof Error ? error.name : "unknown",
+ });
+ return failOpenAfterCallError(error);
+ }
}
- let parsed: Partial<JudgmentResult>;
- try {
- parsed = JSON.parse(match[0]) as Partial<JudgmentResult>;
- } catch (parseError) {
- logger.warn("Judge JSON parse failed β€” defaulting to meaningful", {
- textPeek: match[0].slice(0, 200),
- parseError:
- parseError instanceof Error ? parseError.message : String(parseError),
+ if (
+ outcome.contradictory ||
+ (outcome.fallback && validationRetriesLeft < JUDGE_VALIDATION_RETRIES)
+ ) {
+ logger.warn("Judge validation retry was still invalid; failing open", {
+ validation: VALIDATION_FAILURE,
+ outcome: outcome.contradictory ? "contradictory" : "unparseable",
});
- return {
- meaningful: true,
- confidence: "low",
- reason: "Judge response not valid JSON β€” defaulting to meaningful.",
- meaningfulChanges: [],
- };
}
- const meaningful =
- parsed.meaningful === true || parsed.meaningful === false
- ? parsed.meaningful
- : true;
- return {
- meaningful,
- confidence:
- parsed.confidence === "high" ||
- parsed.confidence === "medium" ||
- parsed.confidence === "low"
- ? parsed.confidence
- : "low",
- reason:
- typeof parsed.reason === "string" && parsed.reason.length > 0
- ? parsed.reason
- : "No reason provided.",
- meaningfulChanges: sanitizeMeaningfulChanges(
- parsed.meaningfulChanges,
- meaningful,
- ),
- };
+ return outcome.judgment;
} catch (error) {
if (error instanceof UsOnlyUnavailableError) throw error;
logger.error("Judge call failed", { error });
- return {
- meaningful: true,
- confidence: "low",
- reason: `Judge call failed β€” defaulting to meaningful. (${error instanceof Error ? error.message : "unknown"})`,
- meaningfulChanges: [],
- };
+ return failOpenAfterCallError(error);
}
}