Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(evaluation): own transcript result parsing and error certainty

morluto79dea89f1f0f
diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.ts
index 744524067..8dfc14df2 100644
--- a/src/evaluation/CodexAgentEval.ts
+++ b/src/evaluation/CodexAgentEval.ts
@@ -58,64 +58,18 @@ const textValue = (value: unknown): string | undefined =>
const numberValue = (value: unknown): number =>
typeof value === "number" && Number.isFinite(value) ? value : 0;
-const toolResultError = (
- item: Record<string, unknown>,
-): Record<string, unknown> | undefined => {
- const result = record(item.result ?? item.output);
- const structured = record(
- result?.structured_content ?? result?.structuredContent,
- );
- const direct = record(structured?.error ?? result?.error ?? item.error);
- if (direct !== undefined || structured !== undefined) return direct;
- // Codex preserves some REA failures only as one complete JSON text block.
- // A client-truncated preview or SDK prose is not a structured error record.
- const content = result?.content;
- if (!Array.isArray(content) || content.length !== 1) return undefined;
- const block = record(content[0]);
- if (block?.type !== "text" || typeof block.text !== "string")
- return undefined;
- try {
- return record(record(JSON.parse(block.text))?.error);
- } catch (cause) {
- if (cause instanceof SyntaxError || cause instanceof RangeError)
- return undefined;
- throw cause;
- }
-};
-
-const toolResultFailed = (
- item: Record<string, unknown>,
- error: Record<string, unknown> | undefined,
-): boolean => {
- if (
- item.status === "failed" ||
- (item.error !== undefined && item.error !== null)
- )
- return true;
- const result = record(item.result ?? item.output);
- const structured = record(
- result?.structured_content ?? result?.structuredContent,
- );
- return (
- result?.isError === true ||
- structured?.error !== undefined ||
- error !== undefined
- );
-};
-
const callFromParsed = (
parsed: ParsedCodexToolCall | undefined,
): CodexMcpCall | undefined => {
if (parsed === undefined) return undefined;
- const error = toolResultError(parsed.item);
return {
id: parsed.id,
server: parsed.server,
tool: parsed.tool,
arguments: parsed.arguments,
evidenceIds: parsed.evidence.map(({ evidence_id }) => evidence_id),
- error: toolResultFailed(parsed.item, error),
- errorCode: textValue(error?.code) ?? null,
+ error: parsed.error,
+ errorCode: parsed.errorCode,
};
};
diff --git a/src/evaluation/KnownAnswerEvaluation.ts b/src/evaluation/KnownAnswerEvaluation.ts
index fe71302eb..6b51d4284 100644
--- a/src/evaluation/KnownAnswerEvaluation.ts
+++ b/src/evaluation/KnownAnswerEvaluation.ts
@@ -1,7 +1,12 @@
import { visit } from "jsonc-parser";
import { z } from "zod";
-import { parseEvidence, type Evidence } from "../domain/evidence.js";
+import {
+ parseEvidence,
+ evidenceAuthoritySchema,
+ evidenceSchema,
+ type Evidence,
+} from "../domain/evidence.js";
import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js";
import { canonicalJson } from "../domain/comparisonSemantics.js";
@@ -65,14 +70,6 @@ export interface KnownAnswerAssessment {
readonly claims: readonly FixtureClaimAssessment[];
}
-const authoritySchema = z.enum([
- "shipped-artifact",
- "controlled-replay",
- "historical-reference",
- "external-service",
- "analyst-inference",
-]);
-const confidenceSchema = z.enum(["observed", "derived", "inferred"]);
const subjectSchema = z.strictObject({
path: z.string().min(1),
sha256: z
@@ -85,8 +82,8 @@ const expectationSchema = z.strictObject({
expectedValue: jsonValueSchema,
source: z.strictObject({
operation: z.string().min(1),
- authority: authoritySchema,
- confidence: confidenceSchema,
+ authority: evidenceAuthoritySchema,
+ confidence: evidenceSchema.shape.confidence,
subject: subjectSchema.optional(),
parameters: z.record(z.string(), jsonValueSchema).optional(),
linkedSubjects: z
@@ -103,8 +100,8 @@ const finalAnswerSchema = z.strictObject({
id: z.string().min(1),
value: jsonValueSchema,
evidence_id: z.string().regex(/^ev_[a-f0-9]{64}$/u),
- confidence: confidenceSchema,
- authority: authoritySchema,
+ confidence: evidenceSchema.shape.confidence,
+ authority: evidenceAuthoritySchema,
}),
),
});
@@ -123,9 +120,15 @@ const sameJson = (left: unknown, right: unknown): boolean => {
);
};
-const parseClosedJson = (text: string): unknown => {
+interface TranscriptJson {
+ readonly value: unknown;
+ readonly hasDuplicateMembers: boolean;
+}
+
+const parseTranscriptJson = (text: string): TranscriptJson | undefined => {
const objects: Set<string>[] = [];
let invalid = false;
+ let hasDuplicateMembers = false;
try {
visit(
text,
@@ -135,7 +138,8 @@ const parseClosedJson = (text: string): unknown => {
},
onObjectProperty: (key) => {
const keys = objects.at(-1);
- if (keys === undefined || keys.has(key)) invalid = true;
+ if (keys === undefined) invalid = true;
+ else if (keys.has(key)) hasDuplicateMembers = true;
keys?.add(key);
},
onObjectEnd: () => {
@@ -152,8 +156,9 @@ const parseClosedJson = (text: string): unknown => {
},
);
if (invalid) return undefined;
- // The JSON visitor checked syntax and duplicate members before JSON.parse can fold them.
- return JSON.parse(text);
+ // Keep ambiguity separate from syntax: a failure can be observed without
+ // choosing one contradictory code or authenticating ambiguous Evidence.
+ return { value: JSON.parse(text) as unknown, hasDuplicateMembers };
} catch (error) {
// Syntax and nesting failures are malformed transcript input, not an evaluation crash.
if (error instanceof SyntaxError || error instanceof RangeError)
@@ -169,7 +174,8 @@ interface DeliveredEvidence {
/** One completed tool call, with aliases normalized and successful Evidence authenticated. */
export interface ParsedCodexToolCall {
- readonly item: Record<string, unknown>;
+ readonly error: boolean;
+ readonly errorCode: string | null;
readonly id: string | null;
readonly server: string | null;
readonly tool: string;
@@ -206,35 +212,66 @@ const consistentJsonAliases = (
value[right] === null ||
sameJson(value[left], value[right]);
-const evidenceFromCall = (
+/** Parse result representations once; authentication and diagnostic certainty stay distinct. */
+const parseToolResult = (
item: Record<string, unknown>,
- tool: string,
-): Evidence | undefined => {
- if (!consistentJsonAliases(item, "result", "output")) return undefined;
+ server: string | null,
+): {
+ readonly failed: boolean;
+ readonly errorCode: string | null;
+ readonly evidenceValue: unknown;
+} => {
const result = record(item.result ?? item.output);
- if (
- result === undefined ||
- result.isError === true ||
- (result.error !== undefined && result.error !== null)
- )
- return undefined;
- if (
- result.structuredContent !== undefined &&
- result.structured_content !== undefined &&
- !sameJson(result.structuredContent, result.structured_content)
- )
- return undefined;
- let value = result.structuredContent ?? result.structured_content;
- if (value === undefined || value === null) {
- // Some Codex transcripts retain only the complete MCP JSON text block.
- // Prose, multiple blocks, and truncated previews cannot establish delivery.
- if (!Array.isArray(result.content) || result.content.length !== 1)
- return undefined;
- const block = record(result.content[0]);
- if (block?.type !== "text" || typeof block.text !== "string")
- return undefined;
- value = parseClosedJson(block.text);
+ const structured = record(
+ result?.structured_content ?? result?.structuredContent,
+ );
+ const direct = record(structured?.error ?? result?.error ?? item.error);
+ let text: TranscriptJson | undefined;
+ if (direct === undefined && structured === undefined) {
+ // Only a complete single JSON block can supply a missing structured result.
+ const content = result?.content;
+ const block =
+ Array.isArray(content) && content.length === 1
+ ? record(content[0])
+ : undefined;
+ if (block?.type === "text" && typeof block.text === "string")
+ text = parseTranscriptJson(block.text);
}
+ const error = direct ?? record(record(text?.value)?.error);
+ const failed =
+ item.status === "failed" ||
+ (item.error !== undefined && item.error !== null) ||
+ result?.isError === true ||
+ structured?.error !== undefined ||
+ error !== undefined;
+ const errorCode =
+ (direct !== undefined || text?.hasDuplicateMembers === false) &&
+ typeof error?.code === "string" &&
+ error.code.length > 0
+ ? error.code
+ : null;
+ const evidenceValue =
+ server?.toLowerCase() === "rea" &&
+ item.status !== "failed" &&
+ (item.error === undefined || item.error === null) &&
+ consistentJsonAliases(item, "result", "output") &&
+ result !== undefined &&
+ result.isError !== true &&
+ (result.error === undefined || result.error === null) &&
+ (result.structuredContent === undefined ||
+ result.structured_content === undefined ||
+ sameJson(result.structuredContent, result.structured_content))
+ ? (result.structuredContent ??
+ result.structured_content ??
+ (text?.hasDuplicateMembers === false ? text.value : undefined))
+ : undefined;
+ return { failed, errorCode, evidenceValue };
+};
+
+const evidenceFromCall = (
+ value: unknown,
+ tool: string,
+): Evidence | undefined => {
const structured = record(value);
if (
structured === undefined ||
@@ -313,7 +350,6 @@ export const parseCodexToolCall = (
): ParsedCodexToolCall | undefined => {
const item = completedToolItem(event);
if (item === undefined) return undefined;
- const status = item.status;
const server = item.server ?? item.server_name;
const tool = item.tool ?? item.name;
if (
@@ -331,21 +367,18 @@ export const parseCodexToolCall = (
: {};
const parsedArguments = jsonValueSchema.safeParse(argumentsValue);
if (!parsedArguments.success) return undefined;
+ const serverName = typeof server === "string" ? server : null;
+ const result = parseToolResult(item, serverName);
const parsed: ParsedCodexToolCall = {
- item,
+ error: result.failed,
+ errorCode: result.errorCode,
id: typeof item.id === "string" && item.id.length > 0 ? item.id : null,
- server: typeof server === "string" ? server : null,
+ server: serverName,
tool,
arguments: parsedArguments.data,
evidence: [],
};
- if (
- parsed.server?.toLowerCase() !== "rea" ||
- status === "failed" ||
- (item.error !== undefined && item.error !== null)
- )
- return parsed;
- const evidence = evidenceFromCall(item, tool);
+ const evidence = evidenceFromCall(result.evidenceValue, tool);
if (evidence === undefined) return parsed;
return { ...parsed, evidence: artifactDelivery(evidence) };
};
@@ -486,7 +519,10 @@ export const evaluateKnownAnswers = (
manifestFailures: ["duplicate_fixture_claim_ids"],
claims: [],
};
- const manifest = finalAnswerSchema.safeParse(parseClosedJson(finalMessage));
+ const answer = parseTranscriptJson(finalMessage);
+ const manifest = finalAnswerSchema.safeParse(
+ answer?.hasDuplicateMembers === false ? answer.value : undefined,
+ );
if (!manifest.success)
return {
scope,
diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts
index 910fd7ae6..b72bf2ff4 100644
--- a/tests/evaluation/model-evals/codexAgentEval.test.ts
+++ b/tests/evaluation/model-evals/codexAgentEval.test.ts
@@ -473,6 +473,46 @@ describe("Codex agent routing and error evaluation", () => {
expect(metrics.requiredToolSubsequenceMet).toBe(false);
});
+ it.each(["failed", undefined])(
+ "preserves failure without choosing duplicate JSON error codes with status %s",
+ (status) => {
+ const texts = [
+ '{"error":{"code":"bridge_timeout","code":"invalid_request"}}',
+ '{"error":{"code":"invalid_request","code":"bridge_timeout"}}',
+ '{"error":{"code":"bridge_timeout"},"error":{"code":"invalid_request"}}',
+ '{"error":{"code":"invalid_request"}}',
+ ];
+ const metrics = evaluateCodexEvents(
+ texts.map((text, index) => ({
+ type: "item.completed",
+ item: {
+ id: `failure-${String(index)}`,
+ type: "mcp_tool_call",
+ server: "rea",
+ tool: "close_binary",
+ arguments: {},
+ ...(status === undefined ? {} : { status }),
+ result: { content: [{ type: "text", text }] },
+ },
+ })),
+ "close_binary",
+ );
+ expect(
+ metrics.reaCalls.map(({ error, errorCode, evidenceIds }) => ({
+ error,
+ errorCode,
+ evidenceIds,
+ })),
+ ).toEqual([
+ { error: true, errorCode: null, evidenceIds: [] },
+ { error: true, errorCode: null, evidenceIds: [] },
+ { error: true, errorCode: null, evidenceIds: [] },
+ { error: true, errorCode: "invalid_request", evidenceIds: [] },
+ ]);
+ expect(metrics.inputValidationFailureCount).toBe(1);
+ },
+ );
+
it("requires no REA call when target selection is missing", () => {
const events = [
{