Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(comparison): validate function result projections

morluto5f4e39fea263
diff --git a/src/domain/functionComparisonSchemas.ts b/src/domain/functionComparisonSchemas.ts
index 75957a7b7..7c4e1dfbb 100644
--- a/src/domain/functionComparisonSchemas.ts
+++ b/src/domain/functionComparisonSchemas.ts
@@ -1,5 +1,6 @@
import { z } from "zod";
-import { overallStatus } from "./functionComparisonResults.js";
+import { overallStatus, summarize } from "./functionComparisonResults.js";
+import { canonicalJson } from "./comparisonSemantics.js";
import { evidenceInputSchema } from "./evidenceReference.js";
import { digestSchema } from "./../domain/digests.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
@@ -91,11 +92,60 @@ export const functionComparisonResultSchema = z
changed: z.int().min(0),
unknown: z.int().min(0),
}),
- dimensions: z.array(functionDimensionSchema).length(8),
+ dimensions: z
+ .array(functionDimensionSchema)
+ .length(functionComparisonDimensionNameSchema.options.length),
changes: z.array(functionDimensionSchema),
limitations: z.array(z.string()),
})
.superRefine((result, context) => {
+ const dimensions = new Map<DimensionName, FunctionDimension>();
+ for (const [index, dimension] of result.dimensions.entries()) {
+ if (dimensions.has(dimension.dimension))
+ context.addIssue({
+ code: "custom",
+ path: ["dimensions", index, "dimension"],
+ message: "Function comparison dimensions must occur exactly once.",
+ });
+ dimensions.set(dimension.dimension, dimension);
+ }
+ const expectedSummary = summarize(result.dimensions);
+ for (const status of comparisonStatusSchema.options)
+ if (result.summary[status] !== expectedSummary[status])
+ context.addIssue({
+ code: "custom",
+ path: ["summary", status],
+ message: "Function comparison summary must count its dimensions.",
+ });
+ const changes = new Set<DimensionName>();
+ for (const [index, change] of result.changes.entries()) {
+ const dimension = dimensions.get(change.dimension);
+ if (
+ changes.has(change.dimension) ||
+ dimension === undefined ||
+ change.status === "unchanged" ||
+ canonicalJson(change) !== canonicalJson(dimension)
+ )
+ context.addIssue({
+ code: "custom",
+ path: ["changes", index],
+ message:
+ "Function comparison changes must cite each changed or unknown dimension exactly as recorded.",
+ });
+ changes.add(change.dimension);
+ }
+ if (
+ result.dimensions.some(
+ ({ dimension, status }) =>
+ status !== "unchanged" && !changes.has(dimension),
+ )
+ )
+ context.addIssue({
+ code: "custom",
+ path: ["changes"],
+ message:
+ "Function comparison changes must include every changed or unknown dimension.",
+ });
if (
result.status !==
overallStatus(result.dimensions, result.function_match.status)
diff --git a/src/domain/reconstructionVerification.functionSources.test.ts b/src/domain/reconstructionVerification.functionSources.test.ts
index eadb957c4..c05373081 100644
--- a/src/domain/reconstructionVerification.functionSources.test.ts
+++ b/src/domain/reconstructionVerification.functionSources.test.ts
@@ -120,23 +120,27 @@ const comparison = (
target?: EvidenceSubjectTarget;
} = {},
resultInput = baselineResult,
-): Evidence =>
- createEvidence(identity.target, identity.provider ?? provider, {
+): Evidence => {
+ const links = sources.map(({ evidence_id: id }) => id);
+ const dimensions = resultInput.dimensions.map((dimension) => ({
+ ...dimension,
+ evidence_links: links,
+ }));
+ return createEvidence(identity.target, identity.provider ?? provider, {
predicateType: "rea.function-comparison",
operation: "compare_functions",
parameters,
result: jsonValueSchema.parse({
...resultInput,
- dimensions: resultInput.dimensions.map((dimension) => ({
- ...dimension,
- evidence_links: sources.map(({ evidence_id: id }) => id),
- })),
+ dimensions,
+ changes: dimensions.filter(({ status }) => status !== "unchanged"),
}),
confidence: "derived",
authority: "analyst-inference",
- evidenceLinks: sources.map(({ evidence_id: id }) => id),
+ evidenceLinks: links,
...observation,
});
+};
const specification = (evidence: Evidence, dimension = "pseudocode") => ({
name: "Function reconstruction",
diff --git a/src/domain/reconstructionVerificationSchemas.ts b/src/domain/reconstructionVerificationSchemas.ts
index 1910b055e..2cf3a2f9f 100644
--- a/src/domain/reconstructionVerificationSchemas.ts
+++ b/src/domain/reconstructionVerificationSchemas.ts
@@ -4,6 +4,7 @@ import { IDENTIFIER_PATTERN } from "./stringPatterns.js";
import { digestSchema } from "./../domain/digests.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js";
+import { functionComparisonDimensionNameSchema } from "./functionComparisonSchemas.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
const unknownIdSchema = prefixedDigestSchema("unk");
@@ -26,14 +27,7 @@ const functionClaimSchema = z.object({
kind: z.literal("structural-function"),
dimension: z.enum([
"overall",
- "identity",
- "pseudocode",
- "assembly",
- "comments",
- "calls",
- "references",
- "strings_names",
- "cfg",
+ ...functionComparisonDimensionNameSchema.options,
]),
});
const artifactClaimSchema = z.object({
diff --git a/src/domain/staticRuntimeCorrelation.ts b/src/domain/staticRuntimeCorrelation.ts
index db82a04c8..8eed65a48 100644
--- a/src/domain/staticRuntimeCorrelation.ts
+++ b/src/domain/staticRuntimeCorrelation.ts
@@ -14,7 +14,10 @@ import {
} from "./comparisonSourceEvidence.js";
import { evidenceSchema, type Evidence } from "./evidence.js";
import { hasComparisonEvidenceIdentity } from "./comparisonEvidenceIdentity.js";
-import { functionComparisonResultSchema } from "./functionComparisonSchemas.js";
+import {
+ functionComparisonDimensionNameSchema,
+ functionComparisonResultSchema,
+} from "./functionComparisonSchemas.js";
import type { JsonValue } from "./jsonValue.js";
import {
comparisonStatusSchema,
@@ -24,16 +27,6 @@ import { PROCESS_COMPARISON_DIMENSIONS } from "./process/processComparison.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
-const functionDimensionSchema = z.enum([
- "identity",
- "pseudocode",
- "assembly",
- "comments",
- "calls",
- "references",
- "strings_names",
- "cfg",
-]);
const runtimeDimensionSchema = z.enum(PROCESS_COMPARISON_DIMENSIONS);
const expectedPatternSchema = z.enum([
"cochanged",
@@ -54,7 +47,10 @@ const artifactSelectorSchema = z
})
.strict();
const functionSelectorSchema = z
- .object({ kind: z.literal("function"), dimension: functionDimensionSchema })
+ .object({
+ kind: z.literal("function"),
+ dimension: functionComparisonDimensionNameSchema,
+ })
.strict();
const staticSelectorSchema = z.discriminatedUnion("kind", [
artifactSelectorSchema,
@@ -333,16 +329,11 @@ const selectStatic = (
"Function selector requires function comparison Evidence",
);
const result = comparison.result;
- const matches = result.dimensions.filter(
+ const match = result.dimensions.find(
({ dimension }) => dimension === selector.dimension,
);
- const match = matches.at(0);
- if (matches.length !== 1 || match === undefined)
- throw callerInputError(
- input,
- path,
- "Function selector must resolve exactly once",
- );
+ if (match === undefined)
+ throw evidenceIntegrityError("Function comparison dimension missing");
return {
status: match.status,
conclusion_kind: match.conclusion_kind,
diff --git a/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts b/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts
new file mode 100644
index 000000000..8615d92d7
--- /dev/null
+++ b/tests/boundary/mcp/functionComparisonIntegrityMcp.test.ts
@@ -0,0 +1,262 @@
+import { expect, it, onTestFinished } from "vitest";
+
+import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
+import {
+ FUNCTION_COMPARISON_EVIDENCE,
+ INVESTIGATION_EXAMPLES,
+ PROCESS_CAPTURE_REFERENCE,
+ PROCESS_CAPTURE_RECONSTRUCTION,
+ PROCESS_COMPARISON_EVIDENCE,
+} from "../../../src/contracts/investigationExamples.js";
+import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
+import {
+ functionComparisonResultSchema,
+ type FunctionComparisonResult,
+} from "../../../src/domain/functionComparisonSchemas.js";
+import { summarize } from "../../../src/domain/functionComparisonResults.js";
+import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
+import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
+import { createTestBinarySession } from "../../fixtures/binarySession.js";
+import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
+
+const conflictingDimensions = (result: FunctionComparisonResult) => {
+ const changed = result.dimensions.find(
+ (dimension) => dimension.dimension === "pseudocode",
+ );
+ if (changed?.status !== "changed")
+ throw new Error("Fixture requires a known pseudocode change");
+ const dimensions = result.dimensions.map((dimension) =>
+ dimension.dimension === "pseudocode"
+ ? {
+ ...changed,
+ status: "unchanged" as const,
+ conclusion_kind: "derived_relationship" as const,
+ }
+ : dimension.dimension === "cfg"
+ ? changed
+ : dimension,
+ );
+ return {
+ ...result,
+ dimensions,
+ summary: summarize(dimensions),
+ changes: dimensions.filter(({ status }) => status !== "unchanged"),
+ };
+};
+
+const unknownDimension = (result: FunctionComparisonResult) => {
+ const dimensions = result.dimensions.map((dimension) =>
+ dimension.dimension === "cfg"
+ ? {
+ ...dimension,
+ status: "unknown" as const,
+ left_digest: null,
+ right_digest: null,
+ text_delta: null,
+ conclusion_kind: "unresolved_branch" as const,
+ }
+ : dimension,
+ );
+ return {
+ ...result,
+ dimensions,
+ summary: summarize(dimensions),
+ changes: dimensions.filter(({ status }) => status !== "unchanged"),
+ };
+};
+
+it.each([
+ {
+ name: "canonical",
+ valid: true,
+ result: (value: FunctionComparisonResult) => value,
+ },
+ {
+ name: "reordered canonical",
+ valid: true,
+ result: (value: FunctionComparisonResult) => ({
+ ...value,
+ dimensions: [...value.dimensions].reverse(),
+ changes: [...value.changes].reverse(),
+ }),
+ },
+ {
+ name: "unknown dimension with complete changes",
+ valid: true,
+ result: unknownDimension,
+ },
+ {
+ name: "omitted unknown dimension with known changes retained",
+ valid: false,
+ result: (value: FunctionComparisonResult) => {
+ const unknown = unknownDimension(value);
+ return {
+ ...unknown,
+ changes: unknown.changes.filter(({ dimension }) => dimension !== "cfg"),
+ };
+ },
+ },
+ {
+ name: "conflicting dimensions",
+ valid: false,
+ result: conflictingDimensions,
+ },
+ {
+ name: "reversed conflicting dimensions",
+ valid: false,
+ result: (value: FunctionComparisonResult) => {
+ const malformed = conflictingDimensions(value);
+ return {
+ ...malformed,
+ dimensions: [...malformed.dimensions].reverse(),
+ changes: [...malformed.changes].reverse(),
+ };
+ },
+ },
+ {
+ name: "incorrect summary counts with the correct total",
+ valid: false,
+ result: (value: FunctionComparisonResult) => ({
+ ...value,
+ summary: {
+ ...value.summary,
+ changed: value.summary.changed - 1,
+ unchanged: value.summary.unchanged + 1,
+ },
+ }),
+ },
+ {
+ name: "missing changes",
+ valid: false,
+ result: (value: FunctionComparisonResult) => ({ ...value, changes: [] }),
+ },
+ {
+ name: "altered change provenance",
+ valid: false,
+ result: (value: FunctionComparisonResult) => ({
+ ...value,
+ changes: value.changes.map((dimension, index) =>
+ index === 0
+ ? {
+ ...dimension,
+ limitations: [
+ ...dimension.limitations,
+ "Unattributed change description.",
+ ],
+ }
+ : dimension,
+ ),
+ }),
+ },
+ {
+ name: "duplicate changes",
+ valid: false,
+ result: (value: FunctionComparisonResult) => ({
+ ...value,
+ changes: [...value.changes, ...value.changes],
+ }),
+ },
+])(
+ "validates $name function comparisons before investigation publication",
+ async (scenario) => {
+ const base = FUNCTION_COMPARISON_EVIDENCE;
+ const result = functionComparisonResultSchema.parse(base.normalized_result);
+ const comparison = createEvidence(undefined, base.provider, {
+ predicateType: base.predicate_type,
+ operation: base.operation,
+ parameters: base.parameters,
+ result: jsonValueSchema.parse(scenario.result(result)),
+ confidence: base.confidence,
+ authority: base.authority,
+ limitations: base.limitations,
+ evidenceLinks: base.evidence_links,
+ });
+ let providerLaunches = 0;
+ const session = createTestBinarySession(() => {
+ providerLaunches += 1;
+ throw new Error(
+ "Retained comparison inspection must not launch a provider",
+ );
+ });
+ const harness = await createApplicationMcpHarness({ session });
+ onTestFinished(harness.close);
+ for (const record of [
+ FUNCTION_COMPARISON_EXAMPLE.left,
+ FUNCTION_COMPARISON_EXAMPLE.right,
+ PROCESS_CAPTURE_REFERENCE,
+ PROCESS_CAPTURE_RECONSTRUCTION,
+ PROCESS_COMPARISON_EVIDENCE,
+ comparison,
+ ])
+ expect(session.recordEvidence(record).ok).toBe(true);
+ const recordCount = session.exportEvidenceBundle().records.length;
+ const correlation = INVESTIGATION_EXAMPLES.correlate_static_and_runtime;
+ const calls = [
+ {
+ name: "find_changed_behavior",
+ arguments: { comparisons: [comparison] },
+ },
+ {
+ name: "correlate_static_and_runtime",
+ arguments: {
+ ...correlation,
+ static_comparisons: [comparison],
+ mappings: correlation.mappings.map((mapping) => ({
+ ...mapping,
+ static: {
+ ...mapping.static,
+ comparison_evidence_id: comparison.evidence_id,
+ },
+ })),
+ },
+ },
+ {
+ name: "verify_reconstruction",
+ arguments: {
+ specification: {
+ name: "Function comparison consistency",
+ claims: [
+ {
+ kind: "structural-function",
+ claim_id: "pseudocode",
+ title: "Selected pseudocode remains equivalent",
+ comparison_evidence_id: comparison.evidence_id,
+ dimension: "pseudocode",
+ },
+ ],
+ },
+ },
+ },
+ ];
+ const responses = [];
+ for (const call of calls)
+ responses.push({ call, response: await harness.client.callTool(call) });
+ expect(
+ Object.fromEntries(
+ responses.map(({ call, response }) => [
+ call.name,
+ response.isError === true,
+ ]),
+ ),
+ ).toEqual(
+ Object.fromEntries(calls.map(({ name }) => [name, !scenario.valid])),
+ );
+ for (const { call, response } of responses) {
+ if (scenario.valid)
+ expect(parseEvidence(response.structuredContent).operation).toBe(
+ call.name,
+ );
+ else
+ expect(parseMcpToolError(response)).toMatchObject({
+ error: {
+ code: "evidence_integrity_mismatch",
+ category: "integrity_mismatch",
+ },
+ });
+ }
+ if (!scenario.valid)
+ expect(session.exportEvidenceBundle().records).toHaveLength(recordCount);
+ expect(session.evidenceById(comparison.evidence_id)).toEqual(comparison);
+ expect(providerLaunches).toBe(0);
+ },
+);