Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
test(evaluation): grade managed agent fixture facts (#1924)
Huamnanxffd8c9c03ddf
diff --git a/docs/client-verification.md b/docs/client-verification.mdindex d9ece9aad..38d0bf399 100644--- a/docs/client-verification.md+++ b/docs/client-verification.md@@ -458,7 +458,7 @@ artifact path. Its `expectedFirstTool` is `null`, and`targetClarificationPassed` grades that limited routing outcome independentlyof analysis prose heuristics. Analyzing a nearby example cannot pass this case.-The packaged desktop and parser-comparison scenarios now assess a closed set of+The packaged desktop, parser-comparison, and managed scenarios assess a closed set ofknown fixture claims. Each final answer must be a strict JSON manifest containingexactly the requested claim IDs, values, Evidence IDs, and Evidence authority andconfidence. Golden values are not included in the prompt. The evaluator checks@@ -475,15 +475,24 @@ analyses and use the requested module and export selectors. Boundary tests packathe same source-owned desktop fixture and compare the same source-owned parserfiles through actual MCP tool results, then verify that fabricated answers fail.+The managed rubric checks the exact public top-level type set and the CLI entry+point's declaring type and method name. The exact type-set claim requires an+unfiltered member inventory: `selection` must be null and the Evidence parameters+must omit `type_names`. A selected inventory can still ground the entry-point+claim. Both inspections must describe the freshly built DLL's SHA-256. Entry-point+resolution additionally requires complete artifact and member metadata, matching+non-null module MVIDs, and a unique method at the header's metadata token. These are+static metadata claims, not runtime behavior;+compiler-generated member counts, token values, and MVIDs are not golden answers.+Per-scenario `factualCorrectness` is `passed`, `failed`, or `not_assessed` withinthe `configured_fixture_claims` scope. A pass establishes only the configured-claims, not unrestricted factual correctness. Native, managed, browser,+claims, not unrestricted factual correctness. Native, browser,navigation-context, and address-context scenarios have no factual rubric and-remain `not_assessed`; the managed workflow requires both artifact and member-inspection to answer its type and entry-point question.+remain `not_assessed`.The closed factual scenarios request complete producer Evidence on the initial-analysis so their source selectors can authenticate all configured facts.+inspection or analysis so their source selectors can authenticate all configured facts.Summary/page/item workflows are exercised separately by `javascript-module-view`;its heuristic gate does not establish full-producer factual correctness.diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjsindex 99ab060dd..71cd6383c 100644--- a/scripts/verify-agent-experience.mjs+++ b/scripts/verify-agent-experience.mjs@@ -1,4 +1,5 @@import { spawn } from "node:child_process";+import { createHash } from "node:crypto";import {chmod,copyFile,@@ -259,7 +260,7 @@ try {const execution = await runCodex(answerInstructions.length === 0? scenario.prompt- : `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial application analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,+ : `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial inspection or analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,scenario.approvedTools ?? [],);const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR;@@ -440,6 +441,13 @@ async function createTargets(root, includeManaged) {...fixtures,largeJavascript,managed: join(managedOutput, "AgentEval.dll"),+ ...(includeManaged+ ? {+ managedSha256: createHash("sha256")+ .update(await readFile(join(managedOutput, "AgentEval.dll")))+ .digest("hex"),+ }+ : {}),};}diff --git a/src/evaluation/AgentEvaluationFixtures.ts b/src/evaluation/AgentEvaluationFixtures.tsindex b7ef17efe..b0535a96e 100644--- a/src/evaluation/AgentEvaluationFixtures.ts+++ b/src/evaluation/AgentEvaluationFixtures.ts@@ -8,7 +8,19 @@ import type { JsonValue } from "../domain/jsonValue.js";import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js";import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraphSchemas.js";import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js";-import type { FixtureClaimExpectation } from "./KnownAnswerEvaluation.js";+import {+ managedArtifactInspectionSchema,+ managedMemberInspectionSchema,+} from "../domain/managed/managedArtifact.js";+import {+ managedTokenScopesMatch,+ parseManagedInspectionEvidence,+} from "../domain/managed/managedInspectionEvidence.js";+import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";+import type {+ FixtureClaimContext,+ FixtureClaimExpectation,+} from "./KnownAnswerEvaluation.js";/** Source-owned artifact paths used by the configured known-answer benchmarks. */export interface AgentEvaluationFixtureTargets {@@ -16,6 +28,8 @@ export interface AgentEvaluationFixtureTargets {readonly javascriptSha256: string;readonly javascriptShapeLeft: string;readonly javascriptShapeRight: string;+ readonly managed?: string;+ readonly managedSha256?: string;}const desktopFixtureFiles = {@@ -277,17 +291,122 @@ const parserClaims = (];};+const managedMembers = ({ evidence }: FixtureClaimContext) => {+ const { result } = parseManagedInspectionEvidence(+ evidence,+ "inspect_managed_members",+ managedMemberInspectionSchema,+ );+ if (+ result.coverage.state !== "complete" ||+ result.metadata.status !== "complete" ||+ !managedTokenScopesMatch(+ result.identity_scope.requires_mvid,+ result.module?.mvid ?? null,+ )+ )+ return undefined;+ return result;+};++const managedClaims = (+ targets: AgentEvaluationFixtureTargets,+): readonly FixtureClaimExpectation[] => {+ if (targets.managed === undefined || targets.managedSha256 === undefined)+ throw new TypeError(+ "Managed fixture grading requires the built DLL path and SHA-256",+ );+ const source = {+ operation: "inspect_managed_members",+ authority: "shipped-artifact",+ confidence: "observed",+ subject: { path: targets.managed, sha256: targets.managedSha256 },+ } satisfies Omit<FixtureClaimExpectation["source"], "select">;+ return [+ {+ id: "public_types",+ expectedValue: ["AgentEval.Profile", "AgentEval.Program"],+ source: {+ ...source,+ select: (_value, context) => {+ const members = managedMembers(context);+ if (+ members === undefined ||+ members.selection !== null ||+ Object.hasOwn(context.evidence.parameters, "type_names")+ )+ return undefined;+ return members.types+ .filter((type) => (type.flags & 7) === 1)+ .map((type) => type.full_name)+ .sort(compareUnicodeCodePoints);+ },+ },+ },+ {+ id: "entry_point",+ expectedValue: { declaring_type: "AgentEval.Program", name: "Main" },+ source: {+ ...source,+ select: (_value, context) => {+ const members = managedMembers(context);+ if (members === undefined) return undefined;+ const artifacts = context.deliveredEvidence.filter(+ (evidence) =>+ evidence.operation === "inspect_managed_artifact" &&+ evidence.authority === source.authority &&+ evidence.confidence === source.confidence &&+ evidence.subject?.local_path === source.subject.path &&+ evidence.subject.digest.sha256 === source.subject.sha256,+ );+ if (artifacts.length !== 1) return undefined;+ const { result: artifact } = parseManagedInspectionEvidence(+ artifacts[0],+ "inspect_managed_artifact",+ managedArtifactInspectionSchema,+ );+ if (+ artifact.coverage.state !== "complete" ||+ artifact.metadata.status !== "complete" ||+ !managedTokenScopesMatch(+ artifact.module?.mvid ?? null,+ members.module?.mvid ?? null,+ ) ||+ artifact.pe.cli?.entry_point.kind !== "metadata-token"+ )+ return undefined;+ const methods = members.methods.filter(+ (method) => method.token === artifact.pe.cli?.entry_point.value,+ );+ const method = methods.length === 1 ? methods[0] : undefined;+ return method === undefined+ ? undefined+ : {+ declaring_type: method.declaring_type,+ name: method.name,+ };+ },+ },+ },+ ];+};+/** Return the closed known-answer rubric, or undefined for routing-only scenarios. */export const agentFixtureClaims = (scenarioId: string,targets: AgentEvaluationFixtureTargets,): readonly FixtureClaimExpectation[] | undefined => {if (scenarioId === "asar") return asarClaims(targets);+ if (scenarioId === "managed") return managedClaims(targets);if (scenarioId === "javascript-export-shape") return parserClaims(targets);return undefined;};const answerValueDescriptions: Readonly<Record<string, readonly string[]>> = {+ managed: [+ "public_types: string[], listing the exact public top-level type full names in Unicode code-point order from unfiltered member Evidence (omit type_names).",+ 'entry_point: {"declaring_type": string, "name": string}, resolving the CLI metadata entry-point token against the member inventory from the same artifact SHA-256 and module MVID. Cite member Evidence; both artifact and member Evidence must be delivered.',+ ],asar: ['renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.','renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.',diff --git a/src/evaluation/KnownAnswerEvaluation.ts b/src/evaluation/KnownAnswerEvaluation.tsindex c6e4af72b..fe71302eb 100644--- a/src/evaluation/KnownAnswerEvaluation.ts+++ b/src/evaluation/KnownAnswerEvaluation.ts@@ -5,6 +5,12 @@ import { parseEvidence, type Evidence } from "../domain/evidence.js";import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js";import { canonicalJson } from "../domain/comparisonSemantics.js";+/** Only authenticated, successful, non-conflicting transcript deliveries. */+export interface FixtureClaimContext {+ readonly evidence: Evidence;+ readonly deliveredEvidence: readonly Evidence[];+}+/** One closed fixture assertion and the producer facts that must support it. */export interface FixtureClaimExpectation {readonly id: string;@@ -20,7 +26,10 @@ export interface FixtureClaimExpectation {readonly path: string;readonly sha256?: string;}[];- readonly select: (normalizedResult: JsonValue) => JsonValue | undefined;+ readonly select: (+ normalizedResult: JsonValue,+ context: FixtureClaimContext,+ ) => JsonValue | undefined;};}@@ -430,7 +439,12 @@ const assessClaim = (try {if (!sameJson(- expected.source.select(evidence.normalized_result),+ expected.source.select(evidence.normalized_result, {+ evidence,+ deliveredEvidence: [...delivered.records.values()].filter(+ (record) => !delivered.conflicts.has(record.evidence_id),+ ),+ }),expected.expectedValue,))diff --git a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts b/tests/evaluation/model-evals/knownAnswerEvaluation.test.tsindex cbe59dd7b..daed23cac 100644--- a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts+++ b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts@@ -5,6 +5,16 @@ import { toolContract } from "../../../src/contracts/toolContracts.js";import { createEvidence, type Evidence } from "../../../src/domain/evidence.js";import type { JsonValue } from "../../../src/domain/jsonValue.js";import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js";+import {+ agentFixtureAnswerInstructions,+ agentFixtureClaims,+} from "../../../src/evaluation/AgentEvaluationFixtures.js";+import { inspectManagedArtifactBytes } from "../../../src/dotnet/ManagedArtifactInspector.js";+import { inspectManagedMembersBytes } from "../../../src/dotnet/ManagedMemberInspector.js";+import {+ buildManagedPeFixture,+ managedPeFixtureTarget,+} from "../../../src/dotnet/ManagedPe.fixture.js";import type { FixtureClaimExpectation } from "../../../src/evaluation/KnownAnswerEvaluation.js";import { ToolResultDelivery } from "../../../src/server/toolResult.js";@@ -33,7 +43,13 @@ const createObservation = (const resultFor = (evidence: Evidence) =>delivery.toEvidenceToolResult(evidence,- toolContract("analyze_javascript_application"),+ toolContract(+ evidence.operation === "inspect_managed_artifact"+ ? "inspect_managed_artifact"+ : evidence.operation === "inspect_managed_members"+ ? "inspect_managed_members"+ : "analyze_javascript_application",+ ),undefined,);const delivered = (@@ -95,6 +111,336 @@ const assess = ();const answer = (claims: readonly unknown[]) => JSON.stringify({ claims });+// Synthetic transcript cases exercise grading, not an SDK-built assembly oracle.+const managedTranscript = () => {+ const bytes = buildManagedPeFixture({+ typeNamespace: "AgentEval",+ typeName: "Program",+ methodName: "Main",+ });+ const target = managedPeFixtureTarget(bytes, "/targets/AgentEval.dll");+ const artifact = inspectManagedArtifactBytes(bytes, target);+ const members = inspectManagedMembersBytes(bytes, target);+ const program = members.types[0];+ if (program === undefined) throw new Error("Fixture type missing");+ const result = {+ ...members,+ types: [+ program,+ {+ ...program,+ token: "0x02000002",+ name: "Profile",+ full_name: "AgentEval.Profile",+ },+ ],+ };+ const rubric = agentFixtureClaims("managed", {+ javascript: "",+ javascriptSha256: "",+ javascriptShapeLeft: "",+ javascriptShapeRight: "",+ managed: target.path,+ managedSha256: target.sha256,+ });+ if (rubric === undefined) throw new Error("Managed rubric missing");+ const evidenceFor = (+ operation: string,+ value: JsonValue,+ overrides: Record<string, unknown> = {},+ ) =>+ createEvidence(target, provider, {+ operation,+ parameters: {},+ result: value,+ confidence: "observed",+ authority: "shipped-artifact",+ ...overrides,+ });+ const artifactEvidence = evidenceFor("inspect_managed_artifact", artifact);+ const memberEvidence = evidenceFor("inspect_managed_members", result);+ const textFor = (evidence: Evidence) =>+ answer([+ {+ id: "public_types",+ value: ["AgentEval.Profile", "AgentEval.Program"],+ evidence_id: evidence.evidence_id,+ confidence: "observed",+ authority: "shipped-artifact",+ },+ {+ id: "entry_point",+ value: { declaring_type: "AgentEval.Program", name: "Main" },+ evidence_id: evidence.evidence_id,+ confidence: "observed",+ authority: "shipped-artifact",+ },+ ]);+ const grade = (+ events = [delivered(artifactEvidence), delivered(memberEvidence)],+ text = textFor(memberEvidence),+ ) =>+ evaluateCodexEvents([...events, final(text)], "inspect_managed_artifact", {+ fixtureClaims: rubric,+ });+ return {+ artifact,+ result,+ artifactEvidence,+ memberEvidence,+ evidenceFor,+ textFor,+ grade,+ };+};++describe("managed known-answer fixture grading", () => {+ it("grades public types independent of row order and resolves the header token", () => {+ expect(managedTranscript().grade().factualCorrectness).toBe("passed");+ });++ it("does not pin compiler-local entry tokens or MVIDs", () => {+ const fixture = managedTranscript();+ const artifact = structuredClone(fixture.artifact);+ const members = structuredClone(fixture.result);+ if (+ artifact.pe.cli === null ||+ artifact.module === null ||+ members.module === null ||+ members.methods[0] === undefined+ )+ throw new Error("Fixture identity missing");+ artifact.pe.cli.entry_point.value = "0x06000009";+ members.methods[0].token = "0x06000009";+ const mvid = "11111111-2222-3333-4444-555555555555";+ artifact.module.mvid = mvid;+ members.module.mvid = mvid;+ members.identity_scope.requires_mvid = mvid;+ members.types.reverse();+ const artifactEvidence = fixture.evidenceFor(+ "inspect_managed_artifact",+ artifact,+ );+ const memberEvidence = fixture.evidenceFor(+ "inspect_managed_members",+ members,+ );+ expect(+ fixture.grade(+ [delivered(artifactEvidence), delivered(memberEvidence)],+ fixture.textFor(memberEvidence),+ ).factualCorrectness,+ ).toBe("passed");+ });++ it.each([+ "wrong-token",+ "native-rva",+ "mvid",+ "null-mvid",+ "digest",+ "partial",+ ])("rejects an unsupported artifact entry point: %s", (kind) => {+ const fixture = managedTranscript();+ const artifact = structuredClone(fixture.artifact);+ if (artifact.pe.cli === null || artifact.module === null)+ throw new Error("Fixture identity missing");+ if (kind === "wrong-token")+ artifact.pe.cli.entry_point.value = "0x06000002";+ if (kind === "native-rva") artifact.pe.cli.entry_point.kind = "native-rva";+ if (kind === "mvid")+ artifact.module.mvid = "11111111-2222-3333-4444-555555555555";+ if (kind === "null-mvid") artifact.module.mvid = null;+ if (kind === "digest") artifact.artifact.sha256 = "b".repeat(64);+ if (kind === "partial") artifact.coverage.state = "partial";+ const evidence = fixture.evidenceFor("inspect_managed_artifact", artifact);+ expect(+ fixture.grade([delivered(evidence), delivered(fixture.memberEvidence)])+ .factualCorrectness,+ ).toBe("failed");+ });+});++describe("managed inventory scope", () => {+ it.each(["selection", "parameters", "both"])(+ "rejects selected public-type inventories while retaining entry-point grounding: %s",+ (kind) => {+ const fixture = managedTranscript();+ const typeNames = ["AgentEval.Profile", "AgentEval.Program"];+ const members = {+ ...fixture.result,+ selection:+ kind === "parameters"+ ? null+ : { type_names: typeNames, unmatched_type_names: [] },+ };+ const evidence = fixture.evidenceFor("inspect_managed_members", members, {+ parameters: kind === "selection" ? {} : { type_names: typeNames },+ });+ expect(+ fixture.grade(+ [delivered(fixture.artifactEvidence), delivered(evidence)],+ fixture.textFor(evidence),+ ),+ ).toMatchObject({+ factualCorrectness: "failed",+ factualAssessment: {+ claims: [+ { id: "public_types", status: "failed" },+ { id: "entry_point", status: "passed" },+ ],+ },+ });+ },+ );+});++describe("managed member grounding", () => {+ it.each([+ "extra-public-type",+ "missing-public-type",+ "duplicate-method",+ "decoy-main",+ "scope-digest",+ "scope-mvid",+ "null-scope",+ "partial",+ "unavailable",+ ])("rejects unsupported member facts: %s", (kind) => {+ const fixture = managedTranscript();+ const members = structuredClone(fixture.result);+ const method = members.methods[0];+ const type = members.types[0];+ if (method === undefined || type === undefined)+ throw new Error("Fixture members missing");+ if (kind === "extra-public-type")+ members.types.push({+ ...type,+ name: "Extra",+ full_name: "AgentEval.Extra",+ });+ if (kind === "missing-public-type") members.types.pop();+ if (kind === "duplicate-method") members.methods.push(method);+ if (kind === "decoy-main") {+ method.name = "Other";+ members.methods.push({ ...method, token: "0x06000002", name: "Main" });+ }+ if (kind === "scope-digest")+ members.identity_scope.requires_artifact_sha256 = "b".repeat(64);+ if (kind === "scope-mvid")+ members.identity_scope.requires_mvid =+ "11111111-2222-3333-4444-555555555555";+ if (kind === "null-scope") members.identity_scope.requires_mvid = null;+ if (kind === "partial") members.metadata.status = "partial";+ if (kind === "unavailable") members.coverage.state = "unavailable";+ const evidence = fixture.evidenceFor("inspect_managed_members", members);+ expect(+ fixture.grade(+ [delivered(fixture.artifactEvidence), delivered(evidence)],+ fixture.textFor(evidence),+ ).factualCorrectness,+ ).toBe("failed");+ });++ it("rejects distinct authenticated matching artifact deliveries", () => {+ const fixture = managedTranscript();+ const secondArtifact = fixture.evidenceFor(+ "inspect_managed_artifact",+ fixture.artifact,+ { parameters: { path: "/targets/AgentEval.dll" } },+ );+ expect(secondArtifact.evidence_id).not.toBe(+ fixture.artifactEvidence.evidence_id,+ );+ const member = delivered(fixture.memberEvidence);+ expect(+ fixture.grade([delivered(secondArtifact), member]).factualCorrectness,+ ).toBe("passed");+ expect(+ fixture.grade([+ delivered(fixture.artifactEvidence),+ delivered(secondArtifact),+ member,+ ]).factualCorrectness,+ ).toBe("failed");+ });++ it("accepts identical repeated artifact deliveries", () => {+ const fixture = managedTranscript();+ expect(+ fixture.grade([+ delivered(fixture.artifactEvidence),+ delivered(fixture.artifactEvidence),+ delivered(fixture.memberEvidence),+ ]).factualCorrectness,+ ).toBe("passed");+ });++ it("requires companion delivery and rejects conflicting or failed artifact deliveries", () => {+ const fixture = managedTranscript();+ const member = delivered(fixture.memberEvidence);+ expect(fixture.grade([member]).factualCorrectness).toBe("failed");+ expect(+ fixture.grade([+ delivered(fixture.artifactEvidence, { status: "failed" }),+ member,+ ]).factualCorrectness,+ ).toBe("failed");+ const conflicting = {+ ...fixture.artifactEvidence,+ subject:+ fixture.memberEvidence.subject === null+ ? null+ : { ...fixture.memberEvidence.subject, local_path: "/other.dll" },+ };+ expect(+ fixture.grade([+ delivered(fixture.artifactEvidence),+ delivered(conflicting),+ member,+ ]).factualCorrectness,+ ).toBe("failed");+ });++ it("binds the digest, rejects fabricated values and cites the supporting operation", () => {+ const fixture = managedTranscript();+ expect(+ fixture.grade(undefined, fixture.textFor(fixture.artifactEvidence))+ .factualCorrectness,+ ).toBe("failed");+ expect(+ fixture.grade(+ undefined,+ fixture.textFor(fixture.memberEvidence).replace('"Main"', '"Invented"'),+ ).factualCorrectness,+ ).toBe("failed");+ const other = createEvidence(+ { path: "/targets/AgentEval.dll", sha256: "b".repeat(64), format: "pe" },+ provider,+ {+ operation: "inspect_managed_members",+ parameters: {},+ result: fixture.result,+ confidence: "observed",+ authority: "shipped-artifact",+ },+ );+ expect(+ fixture.grade(+ [delivered(fixture.artifactEvidence), delivered(other)],+ fixture.textFor(other),+ ).factualCorrectness,+ ).toBe("failed");+ });++ it("keeps golden names out of instructions and preserves routing-only scenarios", () => {+ const instructions = agentFixtureAnswerInstructions("managed");+ expect(instructions).toContain("do not read target files directly");+ expect(instructions).not.toMatch(/AgentEval|Profile|Program|Main/u);+ expect(agentFixtureAnswerInstructions("native")).toBe("");+ });+});+// Removing exact fixture comparison or accepted-source validation must break these regressions.describe("known-answer fixture grading", () => {it("passes closed claims against authenticated production MCP Evidence", () => {