Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(comparison): validate function result projections
morluto5f4e39fea263
diff --git a/src/domain/functionComparisonSchemas.ts b/src/domain/functionComparisonSchemas.tsindex 75957a7b7..7c4e1dfbb 100644--- a/src/domain/functionComparisonSchemas.ts+++ b/src/domain/functionComparisonSchemas.ts@@ -1,5 +1,6 @@import { z } from "zod";-import { overallStatus } from "./functionComparisonResults.js";+import { overallStatus, summarize } from "./functionComparisonResults.js";+import { canonicalJson } from "./comparisonSemantics.js";import { evidenceInputSchema } from "./evidenceReference.js";import { digestSchema } from "./../domain/digests.js";import { prefixedDigestSchema } from "./../domain/digests.js";@@ -91,11 +92,60 @@ export const functionComparisonResultSchema = zchanged: z.int().min(0),unknown: z.int().min(0),}),- dimensions: z.array(functionDimensionSchema).length(8),+ dimensions: z+ .array(functionDimensionSchema)+ .length(functionComparisonDimensionNameSchema.options.length),changes: z.array(functionDimensionSchema),limitations: z.array(z.string()),}).superRefine((result, context) => {+ const dimensions = new Map<DimensionName, FunctionDimension>();+ for (const [index, dimension] of result.dimensions.entries()) {+ if (dimensions.has(dimension.dimension))+ context.addIssue({+ code: "custom",+ path: ["dimensions", index, "dimension"],+ message: "Function comparison dimensions must occur exactly once.",+ });+ dimensions.set(dimension.dimension, dimension);+ }+ const expectedSummary = summarize(result.dimensions);+ for (const status of comparisonStatusSchema.options)+ if (result.summary[status] !== expectedSummary[status])+ context.addIssue({+ code: "custom",+ path: ["summary", status],+ message: "Function comparison summary must count its dimensions.",+ });+ const changes = new Set<DimensionName>();+ for (const [index, change] of result.changes.entries()) {+ const dimension = dimensions.get(change.dimension);+ if (+ changes.has(change.dimension) ||+ dimension === undefined ||+ change.status === "unchanged" ||+ canonicalJson(change) !== canonicalJson(dimension)+ )+ context.addIssue({+ code: "custom",+ path: ["changes", index],+ message:+ "Function comparison changes must cite each changed or unknown dimension exactly as recorded.",+ });+ changes.add(change.dimension);+ }+ if (+ result.dimensions.some(+ ({ dimension, status }) =>+ status !== "unchanged" && !changes.has(dimension),+ )+ )+ context.addIssue({+ code: "custom",+ path: ["changes"],+ message:+ "Function comparison changes must include every changed or unknown dimension.",+ });if (result.status !==overallStatus(result.dimensions, result.function_match.status)diff --git a/src/domain/reconstructionVerification.functionSources.test.ts b/src/domain/reconstructionVerification.functionSources.test.tsindex eadb957c4..c05373081 100644--- a/src/domain/reconstructionVerification.functionSources.test.ts+++ b/src/domain/reconstructionVerification.functionSources.test.ts@@ -120,23 +120,27 @@ const comparison = (target?: EvidenceSubjectTarget;} = {},resultInput = baselineResult,-): Evidence =>- createEvidence(identity.target, identity.provider ?? provider, {+): Evidence => {+ const links = sources.map(({ evidence_id: id }) => id);+ const dimensions = resultInput.dimensions.map((dimension) => ({+ ...dimension,+ evidence_links: links,+ }));+ return createEvidence(identity.target, identity.provider ?? provider, {predicateType: "rea.function-comparison",operation: "compare_functions",parameters,result: jsonValueSchema.parse({...resultInput,- dimensions: resultInput.dimensions.map((dimension) => ({- ...dimension,- evidence_links: sources.map(({ evidence_id: id }) => id),- })),+ dimensions,+ changes: dimensions.filter(({ status }) => status !== "unchanged"),}),confidence: "derived",authority: "analyst-inference",- evidenceLinks: sources.map(({ evidence_id: id }) => id),+ evidenceLinks: links,...observation,});+};const specification = (evidence: Evidence, dimension = "pseudocode") => ({name: "Function reconstruction",diff --git a/src/domain/reconstructionVerificationSchemas.ts b/src/domain/reconstructionVerificationSchemas.tsindex 1910b055e..2cf3a2f9f 100644--- a/src/domain/reconstructionVerificationSchemas.ts+++ b/src/domain/reconstructionVerificationSchemas.ts@@ -4,6 +4,7 @@ import { IDENTIFIER_PATTERN } from "./stringPatterns.js";import { digestSchema } from "./../domain/digests.js";import { prefixedDigestSchema } from "./../domain/digests.js";import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js";+import { functionComparisonDimensionNameSchema } from "./functionComparisonSchemas.js";const evidenceIdSchema = prefixedDigestSchema("ev");const unknownIdSchema = prefixedDigestSchema("unk");@@ -26,14 +27,7 @@ const functionClaimSchema = z.object({kind: z.literal("structural-function"),dimension: z.enum(["overall",- "identity",- "pseudocode",- "assembly",- "comments",- "calls",- "references",- "strings_names",- "cfg",+ ...functionComparisonDimensionNameSchema.options,]),});const artifactClaimSchema = z.object({diff --git a/src/domain/staticRuntimeCorrelation.ts b/src/domain/staticRuntimeCorrelation.tsindex db82a04c8..8eed65a48 100644--- a/src/domain/staticRuntimeCorrelation.ts+++ b/src/domain/staticRuntimeCorrelation.ts@@ -14,7 +14,10 @@ import {} from "./comparisonSourceEvidence.js";import { evidenceSchema, type Evidence } from "./evidence.js";import { hasComparisonEvidenceIdentity } from "./comparisonEvidenceIdentity.js";-import { functionComparisonResultSchema } from "./functionComparisonSchemas.js";+import {+ functionComparisonDimensionNameSchema,+ functionComparisonResultSchema,+} from "./functionComparisonSchemas.js";import type { JsonValue } from "./jsonValue.js";import {comparisonStatusSchema,@@ -24,16 +27,6 @@ import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js";import { prefixedDigestSchema } from "./../domain/digests.js";const evidenceIdSchema = prefixedDigestSchema("ev");-const functionDimensionSchema = z.enum([- "identity",- "pseudocode",- "assembly",- "comments",- "calls",- "references",- "strings_names",- "cfg",-]);const runtimeDimensionSchema = z.enum(PROCESS_COMPARISON_DIMENSIONS);const expectedPatternSchema = z.enum(["cochanged",@@ -54,7 +47,10 @@ const artifactSelectorSchema = z}).strict();const functionSelectorSchema = z- .object({ kind: z.literal("function"), dimension: functionDimensionSchema })+ .object({+ kind: z.literal("function"),+ dimension: functionComparisonDimensionNameSchema,+ }).strict();const staticSelectorSchema = z.discriminatedUnion("kind", [artifactSelectorSchema,@@ -333,16 +329,11 @@ const selectStatic = ("Function selector requires function comparison Evidence",);const result = comparison.result;- const matches = result.dimensions.filter(+ const match = result.dimensions.find(({ dimension }) => dimension === selector.dimension,);- const match = matches.at(0);- if (matches.length !== 1 || match === undefined)- throw callerInputError(- input,- path,- "Function selector must resolve exactly once",- );+ if (match === undefined)+ throw evidenceIntegrityError("Function comparison dimension missing");return {status: match.status,conclusion_kind: match.conclusion_kind,diff --git a/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts b/tests/boundary/mcp/functionComparisonIntegrityMcp.test.tsnew file mode 100644index 000000000..8615d92d7--- /dev/null+++ b/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts@@ -0,0 +1,262 @@+import { expect, it, onTestFinished } from "vitest";++import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";+import {+ FUNCTION_COMPARISON_EVIDENCE,+ INVESTIGATION_EXAMPLES,+ PROCESS_CAPTURE_REFERENCE,+ PROCESS_CAPTURE_RECONSTRUCTION,+ PROCESS_COMPARISON_EVIDENCE,+} from "../../../src/contracts/investigationExamples.js";+import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";+import {+ functionComparisonResultSchema,+ type FunctionComparisonResult,+} from "../../../src/domain/functionComparisonSchemas.js";+import { summarize } from "../../../src/domain/functionComparisonResults.js";+import { jsonValueSchema } from "../../../src/domain/jsonValue.js";+import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";+import { createTestBinarySession } from "../../fixtures/binarySession.js";+import { parseMcpToolError } from "../../fixtures/mcpToolError.js";++const conflictingDimensions = (result: FunctionComparisonResult) => {+ const changed = result.dimensions.find(+ (dimension) => dimension.dimension === "pseudocode",+ );+ if (changed?.status !== "changed")+ throw new Error("Fixture requires a known pseudocode change");+ const dimensions = result.dimensions.map((dimension) =>+ dimension.dimension === "pseudocode"+ ? {+ ...changed,+ status: "unchanged" as const,+ conclusion_kind: "derived_relationship" as const,+ }+ : dimension.dimension === "cfg"+ ? changed+ : dimension,+ );+ return {+ ...result,+ dimensions,+ summary: summarize(dimensions),+ changes: dimensions.filter(({ status }) => status !== "unchanged"),+ };+};++const unknownDimension = (result: FunctionComparisonResult) => {+ const dimensions = result.dimensions.map((dimension) =>+ dimension.dimension === "cfg"+ ? {+ ...dimension,+ status: "unknown" as const,+ left_digest: null,+ right_digest: null,+ text_delta: null,+ conclusion_kind: "unresolved_branch" as const,+ }+ : dimension,+ );+ return {+ ...result,+ dimensions,+ summary: summarize(dimensions),+ changes: dimensions.filter(({ status }) => status !== "unchanged"),+ };+};++it.each([+ {+ name: "canonical",+ valid: true,+ result: (value: FunctionComparisonResult) => value,+ },+ {+ name: "reordered canonical",+ valid: true,+ result: (value: FunctionComparisonResult) => ({+ ...value,+ dimensions: [...value.dimensions].reverse(),+ changes: [...value.changes].reverse(),+ }),+ },+ {+ name: "unknown dimension with complete changes",+ valid: true,+ result: unknownDimension,+ },+ {+ name: "omitted unknown dimension with known changes retained",+ valid: false,+ result: (value: FunctionComparisonResult) => {+ const unknown = unknownDimension(value);+ return {+ ...unknown,+ changes: unknown.changes.filter(({ dimension }) => dimension !== "cfg"),+ };+ },+ },+ {+ name: "conflicting dimensions",+ valid: false,+ result: conflictingDimensions,+ },+ {+ name: "reversed conflicting dimensions",+ valid: false,+ result: (value: FunctionComparisonResult) => {+ const malformed = conflictingDimensions(value);+ return {+ ...malformed,+ dimensions: [...malformed.dimensions].reverse(),+ changes: [...malformed.changes].reverse(),+ };+ },+ },+ {+ name: "incorrect summary counts with the correct total",+ valid: false,+ result: (value: FunctionComparisonResult) => ({+ ...value,+ summary: {+ ...value.summary,+ changed: value.summary.changed - 1,+ unchanged: value.summary.unchanged + 1,+ },+ }),+ },+ {+ name: "missing changes",+ valid: false,+ result: (value: FunctionComparisonResult) => ({ ...value, changes: [] }),+ },+ {+ name: "altered change provenance",+ valid: false,+ result: (value: FunctionComparisonResult) => ({+ ...value,+ changes: value.changes.map((dimension, index) =>+ index === 0+ ? {+ ...dimension,+ limitations: [+ ...dimension.limitations,+ "Unattributed change description.",+ ],+ }+ : dimension,+ ),+ }),+ },+ {+ name: "duplicate changes",+ valid: false,+ result: (value: FunctionComparisonResult) => ({+ ...value,+ changes: [...value.changes, ...value.changes],+ }),+ },+])(+ "validates $name function comparisons before investigation publication",+ async (scenario) => {+ const base = FUNCTION_COMPARISON_EVIDENCE;+ const result = functionComparisonResultSchema.parse(base.normalized_result);+ const comparison = createEvidence(undefined, base.provider, {+ predicateType: base.predicate_type,+ operation: base.operation,+ parameters: base.parameters,+ result: jsonValueSchema.parse(scenario.result(result)),+ confidence: base.confidence,+ authority: base.authority,+ limitations: base.limitations,+ evidenceLinks: base.evidence_links,+ });+ let providerLaunches = 0;+ const session = createTestBinarySession(() => {+ providerLaunches += 1;+ throw new Error(+ "Retained comparison inspection must not launch a provider",+ );+ });+ const harness = await createApplicationMcpHarness({ session });+ onTestFinished(harness.close);+ for (const record of [+ FUNCTION_COMPARISON_EXAMPLE.left,+ FUNCTION_COMPARISON_EXAMPLE.right,+ PROCESS_CAPTURE_REFERENCE,+ PROCESS_CAPTURE_RECONSTRUCTION,+ PROCESS_COMPARISON_EVIDENCE,+ comparison,+ ])+ expect(session.recordEvidence(record).ok).toBe(true);+ const recordCount = session.exportEvidenceBundle().records.length;+ const correlation = INVESTIGATION_EXAMPLES.correlate_static_and_runtime;+ const calls = [+ {+ name: "find_changed_behavior",+ arguments: { comparisons: [comparison] },+ },+ {+ name: "correlate_static_and_runtime",+ arguments: {+ ...correlation,+ static_comparisons: [comparison],+ mappings: correlation.mappings.map((mapping) => ({+ ...mapping,+ static: {+ ...mapping.static,+ comparison_evidence_id: comparison.evidence_id,+ },+ })),+ },+ },+ {+ name: "verify_reconstruction",+ arguments: {+ specification: {+ name: "Function comparison consistency",+ claims: [+ {+ kind: "structural-function",+ claim_id: "pseudocode",+ title: "Selected pseudocode remains equivalent",+ comparison_evidence_id: comparison.evidence_id,+ dimension: "pseudocode",+ },+ ],+ },+ },+ },+ ];+ const responses = [];+ for (const call of calls)+ responses.push({ call, response: await harness.client.callTool(call) });+ expect(+ Object.fromEntries(+ responses.map(({ call, response }) => [+ call.name,+ response.isError === true,+ ]),+ ),+ ).toEqual(+ Object.fromEntries(calls.map(({ name }) => [name, !scenario.valid])),+ );+ for (const { call, response } of responses) {+ if (scenario.valid)+ expect(parseEvidence(response.structuredContent).operation).toBe(+ call.name,+ );+ else+ expect(parseMcpToolError(response)).toMatchObject({+ error: {+ code: "evidence_integrity_mismatch",+ category: "integrity_mismatch",+ },+ });+ }+ if (!scenario.valid)+ expect(session.exportEvidenceBundle().records).toHaveLength(recordCount);+ expect(session.evidenceById(comparison.evidence_id)).toEqual(comparison);+ expect(providerLaunches).toBe(0);+ },+);