Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(evaluation): authenticate delivered Evidence citations
Ysopwnable844fa52772c1
diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.tsindex ba5c500d0..bac8d8191 100644--- a/src/evaluation/CodexAgentEval.ts+++ b/src/evaluation/CodexAgentEval.ts@@ -1,4 +1,5 @@import {+ deliveredCodexEvidence,evaluateKnownAnswers,type FixtureClaimExpectation,type KnownAnswerAssessment,@@ -67,12 +68,6 @@ const canonicalValue = (value: unknown): unknown => {);};-const evidenceIdsFrom = (value: unknown): readonly string[] => {- const encoded = JSON.stringify(value);- if (encoded === undefined) return [];- return [...new Set(encoded.match(/ev_[a-f0-9]{64}/gu) ?? [])];-};-const toolResultError = (item: Record<string, unknown>,): Record<string, unknown> | undefined => {@@ -118,8 +113,16 @@ const toolResultFailed = ();};-const callFromItem = (value: unknown): CodexMcpCall | undefined => {- const item = record(value);+const callFromEvent = (+ event: Record<string, unknown>,+): CodexMcpCall | undefined => {+ const item = record(+ event.type === "item.completed"+ ? event.item+ : event.type === "mcp_tool_call"+ ? event+ : undefined,+ );if (item?.type !== "mcp_tool_call") return undefined;const tool = textValue(item.tool) ?? textValue(item.name);if (tool === undefined) return undefined;@@ -129,7 +132,9 @@ const callFromItem = (value: unknown): CodexMcpCall | undefined => {server: textValue(item.server) ?? textValue(item.server_name) ?? null,tool,arguments: item.arguments ?? item.input ?? {},- evidenceIds: evidenceIdsFrom(item.result ?? item.output),+ evidenceIds: deliveredCodexEvidence(event).map(+ ({ evidence_id }) => evidence_id,+ ),error: toolResultFailed(item, error),errorCode: textValue(error?.code) ?? null,};@@ -163,9 +168,7 @@ const consumeEvent = (state: EvaluationState, value: unknown): void => {if (event === undefined) return;const completedItem =event.type === "item.completed" ? event.item : undefined;- const call =- callFromItem(completedItem) ??- (event.type === "mcp_tool_call" ? callFromItem(event) : undefined);+ const call = callFromEvent(event);if (call !== undefined) {const identity =call.id ??diff --git a/src/evaluation/KnownAnswerEvaluation.ts b/src/evaluation/KnownAnswerEvaluation.tsindex 77fbd9718..807308b59 100644--- a/src/evaluation/KnownAnswerEvaluation.ts+++ b/src/evaluation/KnownAnswerEvaluation.ts@@ -124,7 +124,7 @@ const sameJson = (left: unknown, right: unknown): boolean => {);};-const parseClosedAnswer = (text: string): unknown => {+const parseClosedJson = (text: string): unknown => {const objects: Set<string>[] = [];let invalid = false;try {@@ -156,7 +156,7 @@ const parseClosedAnswer = (text: string): unknown => {// The JSON visitor checked syntax and duplicate members before JSON.parse can fold them.return JSON.parse(text);} catch (error) {- // Syntax and nesting failures are malformed final-answer input, not an evaluation crash.+ // Syntax and nesting failures are malformed transcript input, not an evaluation crash.if (error instanceof SyntaxError || error instanceof RangeError)return undefined;throw error;@@ -224,20 +224,29 @@ const evidenceFromCall = ((result.error !== undefined && result.error !== null))return undefined;- const structured = record(- result.structuredContent ?? result.structured_content,- );- if (- structured === undefined ||- (structured.error !== undefined && structured.error !== null)- )- return undefined;if (result.structuredContent !== undefined &&result.structured_content !== undefined &&!sameJson(result.structuredContent, result.structured_content))return undefined;+ let value = result.structuredContent ?? result.structured_content;+ if (value === undefined || value === null) {+ // Some Codex transcripts retain only the complete MCP JSON text block.+ // Prose, multiple blocks, and truncated previews cannot establish delivery.+ if (!Array.isArray(result.content) || result.content.length !== 1)+ return undefined;+ const block = record(result.content[0]);+ if (block?.type !== "text" || typeof block.text !== "string")+ return undefined;+ value = parseClosedJson(block.text);+ }+ const structured = record(value);+ if (+ structured === undefined ||+ (structured.error !== undefined && structured.error !== null)+ )+ return undefined;let evidence: Evidence;try {evidence = parseEvidence(structured);@@ -249,20 +258,53 @@ const evidenceFromCall = (return evidence;};+/** Authenticate completed REA deliveries for both citations and fixture grading. */+export const deliveredCodexEvidence = (event: unknown): readonly Evidence[] => {+ const item = completedCall(event);+ if (item === undefined) return [];+ const evidence = evidenceFromCall(item);+ if (evidence === undefined) return [];+ const records = [evidence];+ // Artifact inspection delivers its inventory Evidence inline. Bare links and+ // arbitrary IDs in normalized facts are references, not delivered records.+ if (evidence.operation !== "inspect_artifact") return records;+ const substeps = record(evidence.normalized_result)?.substeps;+ if (!Array.isArray(substeps)) return records;+ for (const value of substeps) {+ const substep = record(value);+ if (+ substep?.status !== "completed" ||+ substep.operation !== "inventory_artifact" ||+ typeof substep.evidence_id !== "string" ||+ !evidence.evidence_links.includes(substep.evidence_id)+ )+ continue;+ try {+ const nested = parseEvidence(substep.evidence);+ if (+ nested.evidence_id === substep.evidence_id &&+ nested.operation === substep.operation+ )+ records.push(nested);+ } catch {+ // An authenticated parent does not authenticate an embedded record's ID.+ }+ }+ return records;+};+const collectDeliveredEvidence = (events: readonly unknown[],): DeliveredEvidence => {const records = new Map<string, Evidence>();const conflicts = new Set<string>();for (const event of events) {- const item = completedCall(event);- if (item === undefined) continue;- const evidence = evidenceFromCall(item);- if (evidence === undefined) continue;- const previous = records.get(evidence.evidence_id);- if (previous !== undefined && !sameJson(previous, evidence))- conflicts.add(evidence.evidence_id);- else records.set(evidence.evidence_id, evidence);+ for (const evidence of deliveredCodexEvidence(event)) {+ const previous = records.get(evidence.evidence_id);+ if (previous !== undefined && !sameJson(previous, evidence))+ conflicts.add(evidence.evidence_id);+ else records.set(evidence.evidence_id, evidence);+ }}return { records, conflicts };};@@ -382,7 +424,7 @@ export const evaluateKnownAnswers = (manifestFailures: ["duplicate_fixture_claim_ids"],claims: [],};- const manifest = finalAnswerSchema.safeParse(parseClosedAnswer(finalMessage));+ const manifest = finalAnswerSchema.safeParse(parseClosedJson(finalMessage));if (!manifest.success)return {scope,diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.tsindex f802d68d2..d96265202 100644--- a/tests/evaluation/model-evals/codexAgentEval.test.ts+++ b/tests/evaluation/model-evals/codexAgentEval.test.ts@@ -1,7 +1,56 @@import { describe, expect, it } from "vitest";+import { createEvidence } from "../../../src/domain/evidence.js";+import type { JsonValue } from "../../../src/domain/jsonValue.js";import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js";+const observation = (+ operation = "inspect_managed_artifact",+ result: JsonValue = { observed: true },+ evidenceLinks: readonly string[] = [],+) =>+ createEvidence(+ { path: "/tmp/app.dll", sha256: "1".repeat(64), format: "pe" },+ { id: "managed", name: "Managed static analysis", version: null },+ {+ operation,+ parameters: {},+ result,+ evidenceLinks,+ authority: "shipped-artifact",+ },+ );++const citationMetrics = (+ result: unknown,+ evidenceId: string,+ tool = "inspect_managed_artifact",+) =>+ evaluateCodexEvents(+ [+ {+ type: "item.completed",+ item: {+ type: "mcp_tool_call",+ status: "completed",+ server: "rea",+ tool,+ arguments: {},+ result,+ },+ },+ {+ type: "item.completed",+ item: {+ type: "agent_message",+ text: `Observed artifact evidence ${evidenceId} is cited here; runtime behavior remains unknown because no runtime authority was supplied.`,+ },+ },+ ],+ tool,+ { requireEvidence: true },+ );+describe("Codex agent release evaluation", () => {it("measures natural routing, repeated calls, tokens, and answer heuristics", () => {const call = {@@ -50,7 +99,8 @@ describe("Codex agent release evaluation", () => {});it("requires the final answer to cite produced Evidence when requested", () => {- const evidenceId = `ev_${"a".repeat(64)}`;+ const evidence = observation();+ const evidenceId = evidence.evidence_id;const events = [{type: "item.completed",@@ -60,7 +110,7 @@ describe("Codex agent release evaluation", () => {server: "rea",tool: "inspect_managed_artifact",arguments: { path: "/tmp/app.dll" },- result: { evidence_id: evidenceId },+ result: { structuredContent: evidence },},},{@@ -82,7 +132,126 @@ describe("Codex agent release evaluation", () => {answerHeuristicsMet: true,});});+});++describe("Codex Evidence delivery", () => {+ it("rejects diagnostic IDs and failed results as delivered Evidence", () => {+ const evidenceId = `ev_${"a".repeat(64)}`;+ for (const result of [+ { content: [{ type: "text", text: `diagnostic: ${evidenceId}` }] },+ { evidence_id: evidenceId },+ { structuredContent: { metadata: { evidence_id: evidenceId } } },+ {+ isError: true,+ content: [+ {+ type: "text",+ text: JSON.stringify({ error: { message: evidenceId } }),+ },+ ],+ },+ ]) {+ expect(citationMetrics(result, evidenceId)).toMatchObject({+ evidenceIds: [],+ finalCitesEvidence: false,+ answerHeuristicsMet: false,+ });+ }+ });++ it("accepts complete text-only Evidence while rejecting truncated and tampered copies", () => {+ const evidence = observation();+ const text = JSON.stringify(evidence);+ const textResult = (value: string) => ({+ content: [{ type: "text", text: value }],+ structured_content: null,+ });+ expect(+ citationMetrics(textResult(text), evidence.evidence_id),+ ).toMatchObject({+ evidenceIds: [evidence.evidence_id],+ finalCitesEvidence: true,+ answerHeuristicsMet: true,+ });+ for (const result of [+ textResult(text.slice(0, -1)),+ textResult(+ JSON.stringify({+ ...evidence,+ normalized_result: { fabricated: true },+ }),+ ),+ textResult(+ text.replace(+ `"evidence_id":`,+ `"evidence_id":"${evidence.evidence_id}","evidence_id":`,+ ),+ ),+ ]) {+ expect(citationMetrics(result, evidence.evidence_id).evidenceIds).toEqual(+ [],+ );+ }+ });++ it("retains authenticated nested inspection Evidence without accepting bare or tampered links", () => {+ const nested = observation("inventory_artifact");+ const substep = {+ status: "completed",+ operation: nested.operation,+ evidence_id: nested.evidence_id,+ evidence: nested,+ };+ const inspection = observation(+ "inspect_artifact",+ { substeps: [substep] },+ [nested.evidence_id],+ );+ expect(+ citationMetrics(+ { structuredContent: inspection },+ nested.evidence_id,+ "inspect_artifact",+ ),+ ).toMatchObject({+ evidenceIds: [inspection.evidence_id, nested.evidence_id],+ finalCitesEvidence: true,+ });+ for (const [step, links] of [+ [substep, []],+ [+ {+ ...substep,+ evidence: { ...nested, normalized_result: { fabricated: true } },+ },+ [nested.evidence_id],+ ],+ [{ ...substep, operation: "other_tool" }, [nested.evidence_id]],+ [+ { ...substep, evidence_id: `ev_${"b".repeat(64)}` },+ [nested.evidence_id],+ ],+ ] as const) {+ const parent = observation(+ "inspect_artifact",+ { substeps: [step] },+ links,+ );+ expect(+ citationMetrics(+ { structuredContent: parent },+ nested.evidence_id,+ "inspect_artifact",+ ),+ ).toMatchObject({+ evidenceIds: [parent.evidence_id],+ finalCitesEvidence: false,+ });+ }+ });+});+describe("Codex request accounting", () => {it("flags identical calls with distinct item identities", () => {const event = (id: string) => ({type: "item.completed",@@ -269,7 +438,8 @@ describe("Codex agent routing and error evaluation", () => {describe("Codex agent completion evaluation", () => {it("does not present matching answer terms as factual correctness or honesty", () => {- const evidenceId = `ev_${"a".repeat(64)}`;+ const evidence = observation("analyze_javascript_application");+ const evidenceId = evidence.evidence_id;const metrics = evaluateCodexEvents([{@@ -280,7 +450,7 @@ describe("Codex agent completion evaluation", () => {server: "rea",tool: "analyze_javascript_application",arguments: { input_path: "/tmp/app.asar" },- result: { evidence_id: evidenceId },+ result: { structuredContent: evidence },},},{@@ -380,11 +550,15 @@ describe("Codex agent completion evaluation", () => {describe("Codex agent workflow evaluation", () => {it("requires an ordered tool subsequence and rejects input validation failures", () => {- const evidenceId = `ev_${"b".repeat(64)}`;+ const comparison = observation("compare_javascript_export_shapes");+ const evidenceId = comparison.evidence_id;const call = (id: string,tool: string,- result: unknown = { evidence_id: evidenceId },+ result: unknown = {+ structuredContent:+ tool === comparison.operation ? comparison : observation(tool),+ },) => ({type: "item.completed",item: {diff --git a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts b/tests/evaluation/model-evals/knownAnswerEvaluation.test.tsindex eeb1b9b5d..cbe59dd7b 100644--- a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts+++ b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts@@ -351,6 +351,19 @@ describe("successful Evidence grounding", () => {).toBe("passed");});+ it("grades a complete text-only production Evidence response", () => {+ const evidence = createObservation();+ const event = delivered(evidence, {+ result: {+ content: resultFor(evidence).content,+ structured_content: null,+ },+ });+ expect(+ assess([event], answer([claimFor(evidence)])).factualCorrectness,+ ).toBe("passed");+ });+it("rejects authentic unrelated Evidence containing the same value elsewhere", () => {const evidence = createObservation({ unrelated: expected().expectedValue });expect(