Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(evaluation): share completed transcript facts across graders

morluto1447fd2f990c
diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.ts
index bac8d8191..55ec8333e 100644
--- a/src/evaluation/CodexAgentEval.ts
+++ b/src/evaluation/CodexAgentEval.ts
@@ -1,10 +1,11 @@
import {
- deliveredCodexEvidence,
evaluateKnownAnswers,
+ parseCodexToolCall,
type FixtureClaimExpectation,
type KnownAnswerAssessment,
+ type ParsedCodexToolCall,
} from "./KnownAnswerEvaluation.js";
-import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+import { canonicalJson } from "../domain/comparisonSemantics.js";
import { TOOL_CONTRACTS } from "../contracts/toolContracts.js";
/** One REA MCP invocation observed in a Codex JSONL transcript. */
@@ -57,17 +58,6 @@ const textValue = (value: unknown): string | undefined =>
const numberValue = (value: unknown): number =>
typeof value === "number" && Number.isFinite(value) ? value : 0;
-const canonicalValue = (value: unknown): unknown => {
- if (Array.isArray(value)) return value.map(canonicalValue);
- const object = record(value);
- if (object === undefined) return value;
- return Object.fromEntries(
- Object.entries(object)
- .sort(([left], [right]) => compareUnicodeCodePoints(left, right))
- .map(([key, child]) => [key, canonicalValue(child)]),
- );
-};
-
const toolResultError = (
item: Record<string, unknown>,
): Record<string, unknown> | undefined => {
@@ -113,29 +103,18 @@ const toolResultFailed = (
);
};
-const callFromEvent = (
- event: Record<string, unknown>,
+const callFromParsed = (
+ parsed: ParsedCodexToolCall | undefined,
): CodexMcpCall | undefined => {
- const item = record(
- event.type === "item.completed"
- ? event.item
- : event.type === "mcp_tool_call"
- ? event
- : undefined,
- );
- if (item?.type !== "mcp_tool_call") return undefined;
- const tool = textValue(item.tool) ?? textValue(item.name);
- if (tool === undefined) return undefined;
- const error = toolResultError(item);
+ if (parsed === undefined) return undefined;
+ const error = toolResultError(parsed.item);
return {
- id: textValue(item.id) ?? null,
- server: textValue(item.server) ?? textValue(item.server_name) ?? null,
- tool,
- arguments: item.arguments ?? item.input ?? {},
- evidenceIds: deliveredCodexEvidence(event).map(
- ({ evidence_id }) => evidence_id,
- ),
- error: toolResultFailed(item, error),
+ id: parsed.id,
+ server: parsed.server,
+ tool: parsed.tool,
+ arguments: parsed.arguments,
+ evidenceIds: parsed.evidence.map(({ evidence_id }) => evidence_id),
+ error: toolResultFailed(parsed.item, error),
errorCode: textValue(error?.code) ?? null,
};
};
@@ -163,16 +142,20 @@ interface EvaluationState {
outputTokens: number;
}
-const consumeEvent = (state: EvaluationState, value: unknown): void => {
+const consumeEvent = (
+ state: EvaluationState,
+ value: unknown,
+ parsedCall: ParsedCodexToolCall | undefined,
+): void => {
const event = record(value);
if (event === undefined) return;
const completedItem =
event.type === "item.completed" ? event.item : undefined;
- const call = callFromEvent(event);
+ const call = callFromParsed(parsedCall);
if (call !== undefined) {
const identity =
call.id ??
- `${call.server ?? ""}:${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`;
+ `${call.server ?? ""}:${call.tool}:${canonicalJson(call.arguments)}`;
if (!state.seenCallIds.has(identity)) {
state.seenCallIds.add(identity);
state.calls.push(call);
@@ -216,7 +199,10 @@ export const evaluateCodexEvents = (
cachedInputTokens: 0,
outputTokens: 0,
};
- for (const event of events) consumeEvent(state, event);
+ const parsedCalls = events.map(parseCodexToolCall);
+ events.forEach((event, index) =>
+ consumeEvent(state, event, parsedCalls[index]),
+ );
const reaCalls = state.calls.filter(
({ server }) => server === null || server.toLowerCase() === "rea",
@@ -228,7 +214,7 @@ export const evaluateCodexEvents = (
// useful verification, not a duplicate analysis or unchanged failed retry.
const context =
call.tool === "binary_session" ? `${String(targetRevision)}:` : "";
- const signature = `${context}${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`;
+ const signature = `${context}${call.tool}:${canonicalJson(call.arguments)}`;
signatures.set(signature, (signatures.get(signature) ?? 0) + 1);
if (
!call.error &&
@@ -271,7 +257,7 @@ export const evaluateCodexEvents = (
finalMessage,
);
const factualAssessment = evaluateKnownAnswers(
- events,
+ parsedCalls,
finalMessage,
options.fixtureClaims,
);
diff --git a/src/evaluation/KnownAnswerEvaluation.ts b/src/evaluation/KnownAnswerEvaluation.ts
index 807308b59..c6e4af72b 100644
--- a/src/evaluation/KnownAnswerEvaluation.ts
+++ b/src/evaluation/KnownAnswerEvaluation.ts
@@ -3,7 +3,7 @@ import { z } from "zod";
import { parseEvidence, type Evidence } from "../domain/evidence.js";
import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js";
-import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+import { canonicalJson } from "../domain/comparisonSemantics.js";
/** One closed fixture assertion and the producer facts that must support it. */
export interface FixtureClaimExpectation {
@@ -104,23 +104,13 @@ const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const record = (value: unknown): Record<string, unknown> | undefined =>
isRecord(value) ? value : undefined;
-const canonicalValue = (value: JsonValue): JsonValue => {
- if (Array.isArray(value)) return value.map(canonicalValue);
- if (value === null || typeof value !== "object") return value;
- return Object.fromEntries(
- Object.entries(value)
- .sort(([left], [right]) => compareUnicodeCodePoints(left, right))
- .map(([key, child]) => [key, canonicalValue(child)]),
- );
-};
const sameJson = (left: unknown, right: unknown): boolean => {
const parsedLeft = jsonValueSchema.safeParse(left);
const parsedRight = jsonValueSchema.safeParse(right);
return (
parsedLeft.success &&
parsedRight.success &&
- JSON.stringify(canonicalValue(parsedLeft.data)) ===
- JSON.stringify(canonicalValue(parsedRight.data))
+ canonicalJson(parsedLeft.data) === canonicalJson(parsedRight.data)
);
};
@@ -168,6 +158,16 @@ interface DeliveredEvidence {
readonly conflicts: Set<string>;
}
+/** One completed tool call, with aliases normalized and successful Evidence authenticated. */
+export interface ParsedCodexToolCall {
+ readonly item: Record<string, unknown>;
+ readonly id: string | null;
+ readonly server: string | null;
+ readonly tool: string;
+ readonly arguments: JsonValue;
+ readonly evidence: readonly Evidence[];
+}
+
const consistentAliases = (
value: Record<string, unknown>,
left: string,
@@ -177,30 +177,14 @@ const consistentAliases = (
value[right] === undefined ||
value[left] === value[right];
-const completedCall = (event: unknown): Record<string, unknown> | undefined => {
- const value = record(event);
- const item =
- value?.type === "item.completed"
- ? record(value.item)
- : value?.type === "mcp_tool_call" && value.status === "completed"
- ? value
- : undefined;
- if (item?.type !== "mcp_tool_call") return undefined;
- const server = item.server ?? item.server_name;
- if (
- typeof server !== "string" ||
- server.toLowerCase() !== "rea" ||
- (item.status !== undefined && item.status !== "completed") ||
- (item.error !== undefined && item.error !== null)
- )
- return undefined;
- if (
- !consistentAliases(item, "server", "server_name") ||
- !consistentAliases(item, "tool", "name")
- )
- return undefined;
- return item;
-};
+const consistentJsonFields = (
+ value: Record<string, unknown>,
+ left: string,
+ right: string,
+): boolean =>
+ value[left] === undefined ||
+ value[right] === undefined ||
+ sameJson(value[left], value[right]);
const consistentJsonAliases = (
value: Record<string, unknown>,
@@ -215,6 +199,7 @@ const consistentJsonAliases = (
const evidenceFromCall = (
item: Record<string, unknown>,
+ tool: string,
): Evidence | undefined => {
if (!consistentJsonAliases(item, "result", "output")) return undefined;
const result = record(item.result ?? item.output);
@@ -254,22 +239,42 @@ const evidenceFromCall = (
// Transcript output is untrusted; invalid Evidence cannot support a claim.
return undefined;
}
- if ((item.tool ?? item.name) !== evidence.operation) return undefined;
+ if (tool !== evidence.operation) return undefined;
return evidence;
};
-/** Authenticate completed REA deliveries for both citations and fixture grading. */
-export const deliveredCodexEvidence = (event: unknown): readonly Evidence[] => {
- const item = completedCall(event);
- if (item === undefined) return [];
- const evidence = evidenceFromCall(item);
- if (evidence === undefined) return [];
+const completedToolItem = (
+ event: unknown,
+): Record<string, unknown> | undefined => {
+ const value = record(event);
+ const wrapped = value?.type === "item.completed";
+ const item = wrapped
+ ? record(value.item)
+ : value?.type === "mcp_tool_call"
+ ? value
+ : undefined;
+ if (item?.type !== "mcp_tool_call") return undefined;
+ const terminalStatus =
+ item.status === "completed" || item.status === "failed";
+ if (
+ (wrapped
+ ? item.status !== undefined && !terminalStatus
+ : !terminalStatus) ||
+ !consistentAliases(item, "server", "server_name") ||
+ !consistentAliases(item, "tool", "name") ||
+ !consistentJsonFields(item, "arguments", "input")
+ )
+ return undefined;
+ return item;
+};
+
+const artifactDelivery = (evidence: Evidence): readonly Evidence[] => {
const records = [evidence];
- // Artifact inspection delivers its inventory Evidence inline. Bare links and
- // arbitrary IDs in normalized facts are references, not delivered records.
if (evidence.operation !== "inspect_artifact") return records;
const substeps = record(evidence.normalized_result)?.substeps;
if (!Array.isArray(substeps)) return records;
+ // Artifact inspection delivers linked inventory Evidence inline; bare IDs in
+ // normalized facts are references, not authenticated deliveries.
for (const value of substeps) {
const substep = record(value);
if (
@@ -293,13 +298,56 @@ export const deliveredCodexEvidence = (event: unknown): readonly Evidence[] => {
return records;
};
+/** Parse one completed transcript call and authenticate any successful REA Evidence. */
+export const parseCodexToolCall = (
+ event: unknown,
+): ParsedCodexToolCall | undefined => {
+ const item = completedToolItem(event);
+ if (item === undefined) return undefined;
+ const status = item.status;
+ const server = item.server ?? item.server_name;
+ const tool = item.tool ?? item.name;
+ if (
+ ((item.server !== undefined || item.server_name !== undefined) &&
+ typeof server !== "string") ||
+ typeof tool !== "string" ||
+ tool.length === 0
+ )
+ return undefined;
+ const argumentsValue =
+ item.arguments !== undefined
+ ? item.arguments
+ : item.input !== undefined
+ ? item.input
+ : {};
+ const parsedArguments = jsonValueSchema.safeParse(argumentsValue);
+ if (!parsedArguments.success) return undefined;
+ const parsed: ParsedCodexToolCall = {
+ item,
+ id: typeof item.id === "string" && item.id.length > 0 ? item.id : null,
+ server: typeof server === "string" ? server : null,
+ tool,
+ arguments: parsedArguments.data,
+ evidence: [],
+ };
+ if (
+ parsed.server?.toLowerCase() !== "rea" ||
+ status === "failed" ||
+ (item.error !== undefined && item.error !== null)
+ )
+ return parsed;
+ const evidence = evidenceFromCall(item, tool);
+ if (evidence === undefined) return parsed;
+ return { ...parsed, evidence: artifactDelivery(evidence) };
+};
+
const collectDeliveredEvidence = (
- events: readonly unknown[],
+ calls: readonly (ParsedCodexToolCall | undefined)[],
): DeliveredEvidence => {
const records = new Map<string, Evidence>();
const conflicts = new Set<string>();
- for (const event of events) {
- for (const evidence of deliveredCodexEvidence(event)) {
+ for (const call of calls) {
+ for (const evidence of call?.evidence ?? []) {
const previous = records.get(evidence.evidence_id);
if (previous !== undefined && !sameJson(previous, evidence))
conflicts.add(evidence.evidence_id);
@@ -401,7 +449,7 @@ const assessClaim = (
/** Grade closed answers only against configured fixture truth and delivered authenticated Evidence. */
export const evaluateKnownAnswers = (
- events: readonly unknown[],
+ calls: readonly (ParsedCodexToolCall | undefined)[],
finalMessage: string,
expectations: readonly FixtureClaimExpectation[] | undefined,
): KnownAnswerAssessment => {
@@ -441,7 +489,7 @@ export const evaluateKnownAnswers = (
manifestFailures.push(`unknown_claim:${claim.id}`);
submitted.set(claim.id, claim);
}
- const delivered = collectDeliveredEvidence(events);
+ const delivered = collectDeliveredEvidence(calls);
const claims = expectations.map((expected) =>
assessClaim(expected, submitted.get(expected.id), delivered),
);
diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts
index d96265202..6b46d9b1d 100644
--- a/tests/evaluation/model-evals/codexAgentEval.test.ts
+++ b/tests/evaluation/model-evals/codexAgentEval.test.ts
@@ -134,6 +134,114 @@ describe("Codex agent release evaluation", () => {
});
});
+describe("Codex completed tool call parsing", () => {
+ it("does not let an unfinished direct event reserve a completed call identity", () => {
+ const evidence = observation();
+ const metrics = evaluateCodexEvents(
+ [
+ {
+ type: "mcp_tool_call",
+ id: "same-call",
+ status: "in_progress",
+ server: "rea",
+ tool: evidence.operation,
+ arguments: {},
+ },
+ {
+ type: "item.completed",
+ item: {
+ id: "same-call",
+ type: "mcp_tool_call",
+ status: "completed",
+ server: "rea",
+ tool: evidence.operation,
+ arguments: {},
+ result: { structuredContent: evidence },
+ },
+ },
+ {
+ type: "item.completed",
+ item: {
+ type: "agent_message",
+ text: `Observed Evidence ${evidence.evidence_id} supports the artifact claim; runtime behavior remains unknown without runtime authority.`,
+ },
+ },
+ ],
+ evidence.operation,
+ { requireEvidence: true },
+ );
+
+ expect(metrics).toMatchObject({
+ naturalUse: true,
+ correctFirstTool: true,
+ reaCalls: [{ id: "same-call" }],
+ evidenceIds: [evidence.evidence_id],
+ finalCitesEvidence: true,
+ answerHeuristicsMet: true,
+ });
+ });
+
+ it.each([
+ { server: "rea", server_name: "other" },
+ { tool: "inspect_managed_artifact", name: "other_tool" },
+ { server: 42 },
+ { arguments: { value: 1 }, input: { value: 2 } },
+ ])(
+ "rejects completed calls with invalid or conflicting fields: %j",
+ (aliases) => {
+ const evidence = observation();
+ const metrics = evaluateCodexEvents(
+ [
+ {
+ type: "item.completed",
+ item: {
+ type: "mcp_tool_call",
+ status: "completed",
+ server: "rea",
+ tool: evidence.operation,
+ arguments: {},
+ result: { structuredContent: evidence },
+ ...aliases,
+ },
+ },
+ ],
+ evidence.operation,
+ { requiredToolSubsequence: [evidence.operation] },
+ );
+
+ expect(metrics).toMatchObject({
+ naturalUse: false,
+ correctFirstTool: false,
+ requiredToolSubsequenceMet: false,
+ evidenceIds: [],
+ });
+ },
+ );
+
+ it("preserves legacy completed calls with no server field", () => {
+ const metrics = evaluateCodexEvents(
+ [
+ {
+ type: "item.completed",
+ item: {
+ type: "mcp_tool_call",
+ status: "completed",
+ tool: "inspect_managed_artifact",
+ arguments: {},
+ },
+ },
+ ],
+ "inspect_managed_artifact",
+ );
+
+ expect(metrics).toMatchObject({
+ naturalUse: true,
+ correctFirstTool: true,
+ reaCalls: [{ server: null }],
+ });
+ });
+});
+
describe("Codex Evidence delivery", () => {
it("rejects diagnostic IDs and failed results as delivered Evidence", () => {
const evidenceId = `ev_${"a".repeat(64)}`;
@@ -524,28 +632,32 @@ describe("Codex agent completion evaluation", () => {
});
});
- it("recognizes Codex MCP failed status when the error field is null", () => {
- const metrics = evaluateCodexEvents(
- [
- {
- type: "item.completed",
- item: {
- id: "item-failed",
- type: "mcp_tool_call",
- server: "rea",
- tool: "analyze_javascript_application",
- arguments: { input_path: "/tmp/app.asar" },
- result: { structured_content: { error: { code: "denied" } } },
- error: null,
- status: "failed",
- },
- },
- ],
- "analyze_javascript_application",
- );
+ it.each(["item.completed", "mcp_tool_call"] as const)(
+ "counts a failed completed invocation from %s without delivering Evidence",
+ (type) => {
+ const item = {
+ id: "item-failed",
+ type: "mcp_tool_call",
+ server: "rea",
+ tool: "analyze_javascript_application",
+ arguments: { input_path: "/tmp/app.asar" },
+ result: { structured_content: { error: { code: "denied" } } },
+ error: null,
+ status: "failed",
+ };
+ const metrics = evaluateCodexEvents(
+ [type === "item.completed" ? { type, item } : item],
+ "analyze_javascript_application",
+ );
- expect(metrics.reaCalls[0]?.error).toBe(true);
- });
+ expect(metrics).toMatchObject({
+ naturalUse: true,
+ correctFirstTool: true,
+ evidenceIds: [],
+ reaCalls: [{ error: true }],
+ });
+ },
+ );
});
describe("Codex agent workflow evaluation", () => {