Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(evaluation): authenticate delivered Evidence citations

Ysopwnable844fa52772c1
diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.ts
index ba5c500d0..bac8d8191 100644
--- a/src/evaluation/CodexAgentEval.ts
+++ b/src/evaluation/CodexAgentEval.ts
@@ -1,4 +1,5 @@
import {
+ deliveredCodexEvidence,
evaluateKnownAnswers,
type FixtureClaimExpectation,
type KnownAnswerAssessment,
@@ -67,12 +68,6 @@ const canonicalValue = (value: unknown): unknown => {
);
};
-const evidenceIdsFrom = (value: unknown): readonly string[] => {
- const encoded = JSON.stringify(value);
- if (encoded === undefined) return [];
- return [...new Set(encoded.match(/ev_[a-f0-9]{64}/gu) ?? [])];
-};
-
const toolResultError = (
item: Record<string, unknown>,
): Record<string, unknown> | undefined => {
@@ -118,8 +113,16 @@ const toolResultFailed = (
);
};
-const callFromItem = (value: unknown): CodexMcpCall | undefined => {
- const item = record(value);
+const callFromEvent = (
+ event: Record<string, unknown>,
+): CodexMcpCall | undefined => {
+ const item = record(
+ event.type === "item.completed"
+ ? event.item
+ : event.type === "mcp_tool_call"
+ ? event
+ : undefined,
+ );
if (item?.type !== "mcp_tool_call") return undefined;
const tool = textValue(item.tool) ?? textValue(item.name);
if (tool === undefined) return undefined;
@@ -129,7 +132,9 @@ const callFromItem = (value: unknown): CodexMcpCall | undefined => {
server: textValue(item.server) ?? textValue(item.server_name) ?? null,
tool,
arguments: item.arguments ?? item.input ?? {},
- evidenceIds: evidenceIdsFrom(item.result ?? item.output),
+ evidenceIds: deliveredCodexEvidence(event).map(
+ ({ evidence_id }) => evidence_id,
+ ),
error: toolResultFailed(item, error),
errorCode: textValue(error?.code) ?? null,
};
@@ -163,9 +168,7 @@ const consumeEvent = (state: EvaluationState, value: unknown): void => {
if (event === undefined) return;
const completedItem =
event.type === "item.completed" ? event.item : undefined;
- const call =
- callFromItem(completedItem) ??
- (event.type === "mcp_tool_call" ? callFromItem(event) : undefined);
+ const call = callFromEvent(event);
if (call !== undefined) {
const identity =
call.id ??
diff --git a/src/evaluation/KnownAnswerEvaluation.ts b/src/evaluation/KnownAnswerEvaluation.ts
index 77fbd9718..807308b59 100644
--- a/src/evaluation/KnownAnswerEvaluation.ts
+++ b/src/evaluation/KnownAnswerEvaluation.ts
@@ -124,7 +124,7 @@ const sameJson = (left: unknown, right: unknown): boolean => {
);
};
-const parseClosedAnswer = (text: string): unknown => {
+const parseClosedJson = (text: string): unknown => {
const objects: Set<string>[] = [];
let invalid = false;
try {
@@ -156,7 +156,7 @@ const parseClosedAnswer = (text: string): unknown => {
// The JSON visitor checked syntax and duplicate members before JSON.parse can fold them.
return JSON.parse(text);
} catch (error) {
- // Syntax and nesting failures are malformed final-answer input, not an evaluation crash.
+ // Syntax and nesting failures are malformed transcript input, not an evaluation crash.
if (error instanceof SyntaxError || error instanceof RangeError)
return undefined;
throw error;
@@ -224,20 +224,29 @@ const evidenceFromCall = (
(result.error !== undefined && result.error !== null)
)
return undefined;
- const structured = record(
- result.structuredContent ?? result.structured_content,
- );
- if (
- structured === undefined ||
- (structured.error !== undefined && structured.error !== null)
- )
- return undefined;
if (
result.structuredContent !== undefined &&
result.structured_content !== undefined &&
!sameJson(result.structuredContent, result.structured_content)
)
return undefined;
+ let value = result.structuredContent ?? result.structured_content;
+ if (value === undefined || value === null) {
+ // Some Codex transcripts retain only the complete MCP JSON text block.
+ // Prose, multiple blocks, and truncated previews cannot establish delivery.
+ if (!Array.isArray(result.content) || result.content.length !== 1)
+ return undefined;
+ const block = record(result.content[0]);
+ if (block?.type !== "text" || typeof block.text !== "string")
+ return undefined;
+ value = parseClosedJson(block.text);
+ }
+ const structured = record(value);
+ if (
+ structured === undefined ||
+ (structured.error !== undefined && structured.error !== null)
+ )
+ return undefined;
let evidence: Evidence;
try {
evidence = parseEvidence(structured);
@@ -249,20 +258,53 @@ const evidenceFromCall = (
return evidence;
};
+/** Authenticate completed REA deliveries for both citations and fixture grading. */
+export const deliveredCodexEvidence = (event: unknown): readonly Evidence[] => {
+ const item = completedCall(event);
+ if (item === undefined) return [];
+ const evidence = evidenceFromCall(item);
+ if (evidence === undefined) return [];
+ const records = [evidence];
+ // Artifact inspection delivers its inventory Evidence inline. Bare links and
+ // arbitrary IDs in normalized facts are references, not delivered records.
+ if (evidence.operation !== "inspect_artifact") return records;
+ const substeps = record(evidence.normalized_result)?.substeps;
+ if (!Array.isArray(substeps)) return records;
+ for (const value of substeps) {
+ const substep = record(value);
+ if (
+ substep?.status !== "completed" ||
+ substep.operation !== "inventory_artifact" ||
+ typeof substep.evidence_id !== "string" ||
+ !evidence.evidence_links.includes(substep.evidence_id)
+ )
+ continue;
+ try {
+ const nested = parseEvidence(substep.evidence);
+ if (
+ nested.evidence_id === substep.evidence_id &&
+ nested.operation === substep.operation
+ )
+ records.push(nested);
+ } catch {
+ // An authenticated parent does not authenticate an embedded record's ID.
+ }
+ }
+ return records;
+};
+
const collectDeliveredEvidence = (
events: readonly unknown[],
): DeliveredEvidence => {
const records = new Map<string, Evidence>();
const conflicts = new Set<string>();
for (const event of events) {
- const item = completedCall(event);
- if (item === undefined) continue;
- const evidence = evidenceFromCall(item);
- if (evidence === undefined) continue;
- const previous = records.get(evidence.evidence_id);
- if (previous !== undefined && !sameJson(previous, evidence))
- conflicts.add(evidence.evidence_id);
- else records.set(evidence.evidence_id, evidence);
+ for (const evidence of deliveredCodexEvidence(event)) {
+ const previous = records.get(evidence.evidence_id);
+ if (previous !== undefined && !sameJson(previous, evidence))
+ conflicts.add(evidence.evidence_id);
+ else records.set(evidence.evidence_id, evidence);
+ }
}
return { records, conflicts };
};
@@ -382,7 +424,7 @@ export const evaluateKnownAnswers = (
manifestFailures: ["duplicate_fixture_claim_ids"],
claims: [],
};
- const manifest = finalAnswerSchema.safeParse(parseClosedAnswer(finalMessage));
+ const manifest = finalAnswerSchema.safeParse(parseClosedJson(finalMessage));
if (!manifest.success)
return {
scope,
diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts
index f802d68d2..d96265202 100644
--- a/tests/evaluation/model-evals/codexAgentEval.test.ts
+++ b/tests/evaluation/model-evals/codexAgentEval.test.ts
@@ -1,7 +1,56 @@
import { describe, expect, it } from "vitest";
+import { createEvidence } from "../../../src/domain/evidence.js";
+import type { JsonValue } from "../../../src/domain/jsonValue.js";
import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js";
+const observation = (
+ operation = "inspect_managed_artifact",
+ result: JsonValue = { observed: true },
+ evidenceLinks: readonly string[] = [],
+) =>
+ createEvidence(
+ { path: "/tmp/app.dll", sha256: "1".repeat(64), format: "pe" },
+ { id: "managed", name: "Managed static analysis", version: null },
+ {
+ operation,
+ parameters: {},
+ result,
+ evidenceLinks,
+ authority: "shipped-artifact",
+ },
+ );
+
+const citationMetrics = (
+ result: unknown,
+ evidenceId: string,
+ tool = "inspect_managed_artifact",
+) =>
+ evaluateCodexEvents(
+ [
+ {
+ type: "item.completed",
+ item: {
+ type: "mcp_tool_call",
+ status: "completed",
+ server: "rea",
+ tool,
+ arguments: {},
+ result,
+ },
+ },
+ {
+ type: "item.completed",
+ item: {
+ type: "agent_message",
+ text: `Observed artifact evidence ${evidenceId} is cited here; runtime behavior remains unknown because no runtime authority was supplied.`,
+ },
+ },
+ ],
+ tool,
+ { requireEvidence: true },
+ );
+
describe("Codex agent release evaluation", () => {
it("measures natural routing, repeated calls, tokens, and answer heuristics", () => {
const call = {
@@ -50,7 +99,8 @@ describe("Codex agent release evaluation", () => {
});
it("requires the final answer to cite produced Evidence when requested", () => {
- const evidenceId = `ev_${"a".repeat(64)}`;
+ const evidence = observation();
+ const evidenceId = evidence.evidence_id;
const events = [
{
type: "item.completed",
@@ -60,7 +110,7 @@ describe("Codex agent release evaluation", () => {
server: "rea",
tool: "inspect_managed_artifact",
arguments: { path: "/tmp/app.dll" },
- result: { evidence_id: evidenceId },
+ result: { structuredContent: evidence },
},
},
{
@@ -82,7 +132,126 @@ describe("Codex agent release evaluation", () => {
answerHeuristicsMet: true,
});
});
+});
+
+describe("Codex Evidence delivery", () => {
+ it("rejects diagnostic IDs and failed results as delivered Evidence", () => {
+ const evidenceId = `ev_${"a".repeat(64)}`;
+ for (const result of [
+ { content: [{ type: "text", text: `diagnostic: ${evidenceId}` }] },
+ { evidence_id: evidenceId },
+ { structuredContent: { metadata: { evidence_id: evidenceId } } },
+ {
+ isError: true,
+ content: [
+ {
+ type: "text",
+ text: JSON.stringify({ error: { message: evidenceId } }),
+ },
+ ],
+ },
+ ]) {
+ expect(citationMetrics(result, evidenceId)).toMatchObject({
+ evidenceIds: [],
+ finalCitesEvidence: false,
+ answerHeuristicsMet: false,
+ });
+ }
+ });
+
+ it("accepts complete text-only Evidence while rejecting truncated and tampered copies", () => {
+ const evidence = observation();
+ const text = JSON.stringify(evidence);
+ const textResult = (value: string) => ({
+ content: [{ type: "text", text: value }],
+ structured_content: null,
+ });
+ expect(
+ citationMetrics(textResult(text), evidence.evidence_id),
+ ).toMatchObject({
+ evidenceIds: [evidence.evidence_id],
+ finalCitesEvidence: true,
+ answerHeuristicsMet: true,
+ });
+ for (const result of [
+ textResult(text.slice(0, -1)),
+ textResult(
+ JSON.stringify({
+ ...evidence,
+ normalized_result: { fabricated: true },
+ }),
+ ),
+ textResult(
+ text.replace(
+ `"evidence_id":`,
+ `"evidence_id":"${evidence.evidence_id}","evidence_id":`,
+ ),
+ ),
+ ]) {
+ expect(citationMetrics(result, evidence.evidence_id).evidenceIds).toEqual(
+ [],
+ );
+ }
+ });
+
+ it("retains authenticated nested inspection Evidence without accepting bare or tampered links", () => {
+ const nested = observation("inventory_artifact");
+ const substep = {
+ status: "completed",
+ operation: nested.operation,
+ evidence_id: nested.evidence_id,
+ evidence: nested,
+ };
+ const inspection = observation(
+ "inspect_artifact",
+ { substeps: [substep] },
+ [nested.evidence_id],
+ );
+ expect(
+ citationMetrics(
+ { structuredContent: inspection },
+ nested.evidence_id,
+ "inspect_artifact",
+ ),
+ ).toMatchObject({
+ evidenceIds: [inspection.evidence_id, nested.evidence_id],
+ finalCitesEvidence: true,
+ });
+ for (const [step, links] of [
+ [substep, []],
+ [
+ {
+ ...substep,
+ evidence: { ...nested, normalized_result: { fabricated: true } },
+ },
+ [nested.evidence_id],
+ ],
+ [{ ...substep, operation: "other_tool" }, [nested.evidence_id]],
+ [
+ { ...substep, evidence_id: `ev_${"b".repeat(64)}` },
+ [nested.evidence_id],
+ ],
+ ] as const) {
+ const parent = observation(
+ "inspect_artifact",
+ { substeps: [step] },
+ links,
+ );
+ expect(
+ citationMetrics(
+ { structuredContent: parent },
+ nested.evidence_id,
+ "inspect_artifact",
+ ),
+ ).toMatchObject({
+ evidenceIds: [parent.evidence_id],
+ finalCitesEvidence: false,
+ });
+ }
+ });
+});
+describe("Codex request accounting", () => {
it("flags identical calls with distinct item identities", () => {
const event = (id: string) => ({
type: "item.completed",
@@ -269,7 +438,8 @@ describe("Codex agent routing and error evaluation", () => {
describe("Codex agent completion evaluation", () => {
it("does not present matching answer terms as factual correctness or honesty", () => {
- const evidenceId = `ev_${"a".repeat(64)}`;
+ const evidence = observation("analyze_javascript_application");
+ const evidenceId = evidence.evidence_id;
const metrics = evaluateCodexEvents(
[
{
@@ -280,7 +450,7 @@ describe("Codex agent completion evaluation", () => {
server: "rea",
tool: "analyze_javascript_application",
arguments: { input_path: "/tmp/app.asar" },
- result: { evidence_id: evidenceId },
+ result: { structuredContent: evidence },
},
},
{
@@ -380,11 +550,15 @@ describe("Codex agent completion evaluation", () => {
describe("Codex agent workflow evaluation", () => {
it("requires an ordered tool subsequence and rejects input validation failures", () => {
- const evidenceId = `ev_${"b".repeat(64)}`;
+ const comparison = observation("compare_javascript_export_shapes");
+ const evidenceId = comparison.evidence_id;
const call = (
id: string,
tool: string,
- result: unknown = { evidence_id: evidenceId },
+ result: unknown = {
+ structuredContent:
+ tool === comparison.operation ? comparison : observation(tool),
+ },
) => ({
type: "item.completed",
item: {
diff --git a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts
index eeb1b9b5d..cbe59dd7b 100644
--- a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts
+++ b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts
@@ -351,6 +351,19 @@ describe("successful Evidence grounding", () => {
).toBe("passed");
});
+ it("grades a complete text-only production Evidence response", () => {
+ const evidence = createObservation();
+ const event = delivered(evidence, {
+ result: {
+ content: resultFor(evidence).content,
+ structured_content: null,
+ },
+ });
+ expect(
+ assess([event], answer([claimFor(evidence)])).factualCorrectness,
+ ).toBe("passed");
+ });
+
it("rejects authentic unrelated Evidence containing the same value elsewhere", () => {
const evidence = createObservation({ unrelated: expected().expectedValue });
expect(