Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(evidence): preserve call coverage in function comparisons

morluto049132ab43fa
diff --git a/src/contracts/functionComparisonExample.ts b/src/contracts/functionComparisonExample.ts
index 2ba8581d9..39dfe6f92 100644
--- a/src/contracts/functionComparisonExample.ts
+++ b/src/contracts/functionComparisonExample.ts
@@ -50,6 +50,7 @@ const observe = (digit: string, text: string) =>
result: dossier(text),
confidence: "derived",
authority: "shipped-artifact",
+ relationshipCoverage: { status: "complete" },
},
);
diff --git a/src/domain/functionCallCoverage.ts b/src/domain/functionCallCoverage.ts
new file mode 100644
index 000000000..eb2c05a13
--- /dev/null
+++ b/src/domain/functionCallCoverage.ts
@@ -0,0 +1,78 @@
+import type { RelationshipCoverage } from "./relationshipCoverage.js";
+import type { FunctionDossier } from "./hopperValues.js";
+import { uniqueSorted } from "./canonicalOrdering.js";
+
+/** Combine caller and callee read outcomes without losing either explanation. */
+export const combineFunctionRelationshipCoverage = (
+ callers: RelationshipCoverage | undefined,
+ callees: RelationshipCoverage | undefined,
+): RelationshipCoverage => {
+ const reads = [
+ { direction: "callers" as const, coverage: callers },
+ { direction: "callees" as const, coverage: callees },
+ ];
+ const diagnostics = uniqueSorted(
+ reads.flatMap(({ coverage }) =>
+ coverage !== undefined && "diagnostics" in coverage
+ ? coverage.diagnostics
+ : [],
+ ),
+ );
+ const details = diagnostics.length > 0 ? { diagnostics } : {};
+ const reasons = reads.flatMap(({ direction, coverage }) => {
+ const label = direction === "callers" ? "Callers" : "Callees";
+ const limitations =
+ coverage === undefined
+ ? ["Provider did not report relationship coverage."]
+ : coverage.status === "unknown"
+ ? [coverage.reason]
+ : coverage.status === "partial"
+ ? coverage.limitations
+ : [];
+ return limitations.map((reason) => `${label}: ${reason}`);
+ });
+ if (
+ reads.some(
+ ({ coverage }) => coverage === undefined || coverage.status === "unknown",
+ )
+ )
+ return {
+ status: "unknown",
+ reason: uniqueSorted(reasons).join(" "),
+ ...details,
+ };
+ if (reasons.length > 0)
+ return {
+ status: "partial",
+ limitations: uniqueSorted(reasons),
+ ...details,
+ };
+ return { status: "complete", ...details };
+};
+
+/** Explain why two resolved call sets cannot establish call-set equality. */
+export const functionCallCoverageLimitations = (
+ coverage: RelationshipCoverage | null,
+ dossier: Pick<FunctionDossier, "unresolved_calls">,
+): string[] => {
+ const limitations =
+ coverage === null
+ ? ["Provider did not report relationship coverage."]
+ : coverage.status === "unknown"
+ ? [
+ coverage.reason,
+ ...("diagnostics" in coverage ? coverage.diagnostics : []),
+ ]
+ : coverage.status === "partial"
+ ? [
+ ...coverage.limitations,
+ ...("diagnostics" in coverage ? coverage.diagnostics : []),
+ ]
+ : [];
+ limitations.push(
+ ...(dossier.unresolved_calls ?? []).map(
+ ({ address, reason }) => `Unresolved call at ${address}: ${reason}`,
+ ),
+ );
+ return uniqueSorted(limitations);
+};
diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts
index f846217c5..acb505dac 100644
--- a/src/domain/functionComparison.test.ts
+++ b/src/domain/functionComparison.test.ts
@@ -11,6 +11,7 @@ import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { jsonValueSchema } from "./jsonValue.js";
import { compareFunctions } from "./functionComparison.js";
+import type { RelationshipCoverage } from "./relationshipCoverage.js";
const procedure = (address: string, name: string) => ({
address,
name,
@@ -96,6 +97,7 @@ const observe = (
digit: string,
value: ReturnType<typeof dossier>,
providerId = "rea-workflow",
+ relationshipCoverage: RelationshipCoverage | null = { status: "complete" },
): Evidence =>
createEvidence(
{
@@ -112,6 +114,7 @@ const observe = (
result: jsonValueSchema.parse(value),
confidence: "derived",
authority: "shipped-artifact",
+ ...(relationshipCoverage === null ? {} : { relationshipCoverage }),
},
);
@@ -187,6 +190,100 @@ describe("function comparison normalized identity", () => {
});
});
+describe("function call relationship coverage", () => {
+ const coverageCases: {
+ readonly label: string;
+ readonly coverage: RelationshipCoverage | null;
+ readonly limitation: string;
+ }[] = [
+ {
+ label: "missing",
+ coverage: null,
+ limitation: "Provider did not report relationship coverage.",
+ },
+ {
+ label: "partial",
+ coverage: {
+ status: "partial",
+ limitations: ["Call graph stopped at the read limit."],
+ },
+ limitation: "Call graph stopped at the read limit.",
+ },
+ ];
+ it.each(coverageCases)(
+ "keeps equal empty calls unknown for $label coverage",
+ ({ coverage, limitation }) => {
+ const left = observe("a", dossier("return 0;", "0x1000"));
+ const right = observe(
+ "b",
+ dossier("return 0;", "0x2000"),
+ "rea-workflow",
+ coverage,
+ );
+ const result = compareFunctions(left, right);
+ const calls = result.dimensions.find(
+ ({ dimension }) => dimension === "calls",
+ );
+ expect(calls).toMatchObject({
+ status: "unknown",
+ conclusion_kind: "unresolved_branch",
+ left_count: 0,
+ right_count: 0,
+ limitations: [`Right: ${limitation}`],
+ });
+ expect(result.status).toBe("unknown");
+ },
+ );
+
+ it("retains complete observations with execution diagnostics as comparable", () => {
+ const coverage = {
+ status: "complete" as const,
+ diagnostics: ["Provider reported a non-call xref warning."],
+ };
+ const left = observe(
+ "c",
+ dossier("return 0;", "0x1000"),
+ "rea-workflow",
+ coverage,
+ );
+ const right = observe(
+ "d",
+ dossier("return 0;", "0x2000"),
+ "rea-workflow",
+ coverage,
+ );
+ expect(
+ compareFunctions(left, right).dimensions.find(
+ ({ dimension }) => dimension === "calls",
+ ),
+ ).toMatchObject({ status: "unchanged", left_count: 0, right_count: 0 });
+ });
+
+ it("keeps unresolved call sites unknown despite complete resolved-set coverage", () => {
+ const left = observe("e", dossier("return 0;", "0x1000"));
+ const rightDossier = functionDossierSchema.parse({
+ ...dossier("return 0;", "0x2000"),
+ unresolved_calls: [
+ {
+ address: "0x2010",
+ reason: "Indirect dispatch target could not be resolved.",
+ },
+ ],
+ });
+ const right = observe("f", rightDossier);
+ expect(
+ compareFunctions(left, right).dimensions.find(
+ ({ dimension }) => dimension === "calls",
+ ),
+ ).toMatchObject({
+ status: "unknown",
+ limitations: [
+ "Right: Unresolved call at 0x2010: Indirect dispatch target could not be resolved.",
+ ],
+ });
+ });
+});
+
describe("function collection collation ties", () => {
const composed = procedure("0x2000", "caf\u00e9");
const decomposed = procedure("0x3000", "cafe\u0301");
@@ -342,6 +439,7 @@ describe("function comparison ordering controls", () => {
result: jsonValueSchema.parse(result),
confidence: "derived",
authority: "shipped-artifact",
+ relationshipCoverage: { status: "complete" },
},
);
const left = controlled("0x1000", false);
@@ -525,7 +623,9 @@ describe("function comparison", () => {
limitations: [expect.stringContaining("did not expose reference kinds")],
});
});
+});
+describe("function comparison identity and malformed CFG", () => {
it("does not promote address-derived names through equal signatures", () => {
const autoDossier = (base: "0x1000" | "0x2000") =>
functionDossierSchema.parse({
@@ -546,6 +646,22 @@ describe("function comparison", () => {
comparison.dimensions.find(({ dimension }) => dimension === "identity"),
).toMatchObject({ status: "unknown" });
expect(comparison.status).toBe("unknown");
+ const changedText = compareFunctions(
+ observe("b", autoDossier("0x1000")),
+ observe(
+ "c",
+ functionDossierSchema.parse({
+ ...autoDossier("0x2000"),
+ pseudocode: "return 1;",
+ }),
+ ),
+ );
+ expect(changedText.dimensions).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({ dimension: "pseudocode", status: "changed" }),
+ ]),
+ );
+ expect(changedText.status).toBe("unknown");
});
it("rejects invalid CFG normalization and preserves newline differences", () => {
diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts
index f4d7d86a1..27bb7b176 100644
--- a/src/domain/functionComparisonDimensions.ts
+++ b/src/domain/functionComparisonDimensions.ts
@@ -233,6 +233,18 @@ const compareCalls = (
): FunctionDimension => {
const leftValues = callProjection(left);
const rightValues = callProjection(right);
+ const coverageLimitations = [
+ ...left.callCoverageLimitations.map((reason) => `Left: ${reason}`),
+ ...right.callCoverageLimitations.map((reason) => `Right: ${reason}`),
+ ];
+ if (coverageLimitations.length > 0)
+ return unresolvedDimension({
+ dimension: "calls",
+ links: context.links,
+ leftCount: leftValues.length,
+ rightCount: rightValues.length,
+ limitations: coverageLimitations,
+ });
if ([...leftValues, ...rightValues].some(({ name }) => isAutoName(name)))
return unresolvedDimension({
dimension: "calls",
diff --git a/src/domain/functionComparisonResults.ts b/src/domain/functionComparisonResults.ts
index 95465c3e7..93eb9d752 100644
--- a/src/domain/functionComparisonResults.ts
+++ b/src/domain/functionComparisonResults.ts
@@ -69,15 +69,11 @@ export const overallStatus = (
dimensions: readonly FunctionDimension[],
match: FunctionComparisonResult["function_match"]["status"],
): FunctionComparisonResult["status"] => {
- if (
- match === "ambiguous" ||
- dimensions.some(({ status }) => status === "unknown")
- )
- return "unknown";
if (match === "mismatched") return "changed";
- return dimensions.some(({ status }) => status === "changed")
- ? "changed"
- : "unchanged";
+ if (match === "ambiguous") return "unknown";
+ if (dimensions.some(({ status }) => status === "changed")) return "changed";
+ if (dimensions.some(({ status }) => status === "unknown")) return "unknown";
+ return "unchanged";
};
export const summarize = (dimensions: readonly FunctionDimension[]) => ({
diff --git a/src/domain/functionComparisonSchemas.ts b/src/domain/functionComparisonSchemas.ts
index 26c10b1fb..bafc3def1 100644
--- a/src/domain/functionComparisonSchemas.ts
+++ b/src/domain/functionComparisonSchemas.ts
@@ -4,7 +4,7 @@ import { digestSchema } from "./../domain/digests.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
const comparisonStatusSchema = z.enum(["unchanged", "changed", "unknown"]);
-const dimensionNameSchema = z.enum([
+export const functionComparisonDimensionNameSchema = z.enum([
"identity",
"pseudocode",
"assembly",
@@ -29,7 +29,7 @@ const textDeltaSchema = z.object({
});
const functionDimensionContextShape = {
- dimension: dimensionNameSchema,
+ dimension: functionComparisonDimensionNameSchema,
left_count: z.number().int().min(0).nullable(),
right_count: z.number().int().min(0).nullable(),
evidence_links: z.array(evidenceIdSchema).min(2),
@@ -98,4 +98,6 @@ export type FunctionComparisonResult = z.infer<
typeof functionComparisonResultSchema
>;
export type FunctionDimension = z.infer<typeof functionDimensionSchema>;
-export type DimensionName = z.infer<typeof dimensionNameSchema>;
+export type DimensionName = z.infer<
+ typeof functionComparisonDimensionNameSchema
+>;
diff --git a/src/domain/functionDossierEvidence.ts b/src/domain/functionDossierEvidence.ts
index 5a1443c1c..35f5c4b8b 100644
--- a/src/domain/functionDossierEvidence.ts
+++ b/src/domain/functionDossierEvidence.ts
@@ -1,6 +1,7 @@
import { parseEvidence, type Evidence } from "./evidence.js";
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
import { functionDossierSchema, type FunctionDossier } from "./hopperValues.js";
+import { functionCallCoverageLimitations } from "./functionCallCoverage.js";
/** Complete function observation parsed from one analyze_function Evidence record. */
export interface FunctionSnapshot {
@@ -8,6 +9,7 @@ export interface FunctionSnapshot {
readonly subject: NonNullable<Evidence["subject"]>;
readonly dossier: FunctionDossier;
readonly limitations: readonly string[];
+ readonly callCoverageLimitations: readonly string[];
}
/** Parse one complete analyze_function Evidence record for comparison workflows. */
@@ -20,6 +22,10 @@ export const parseFunctionEvidence = (input: unknown): FunctionSnapshot => {
if (evidence.subject === null)
throw new TypeError("Function comparison requires artifact-bound Evidence");
const dossier = functionDossierSchema.parse(evidence.normalized_result);
+ const callCoverageLimitations = functionCallCoverageLimitations(
+ evidence.relationship_coverage,
+ dossier,
+ );
return {
evidence,
subject: evidence.subject,
@@ -27,5 +33,6 @@ export const parseFunctionEvidence = (input: unknown): FunctionSnapshot => {
limitations: [
...new Set([...evidence.limitations, ...dossier.limitations]),
].sort((left, right) => compareUnicodeCodePoints(left, right)),
+ callCoverageLimitations,
};
};
diff --git a/src/domain/reconstructionUnknowns.test.ts b/src/domain/reconstructionUnknowns.test.ts
new file mode 100644
index 000000000..9fa406b19
--- /dev/null
+++ b/src/domain/reconstructionUnknowns.test.ts
@@ -0,0 +1,73 @@
+import { expect, it } from "vitest";
+
+import { createResidualUnknown } from "./residualUnknown.js";
+import { reconstructionClaimUnknowns } from "./reconstructionUnknowns.js";
+
+const ev = (digit: string) => `ev_${digit.repeat(64)}`;
+const base = {
+ severity: "high" as const,
+ domain: "function-comparison",
+ supporting_evidence_ids: [ev("a"), ev("b")],
+ contradicting_evidence_ids: [],
+ required_authority: null,
+ required_confidence: "derived" as const,
+ required_environment: null,
+ recommended_probes: [],
+};
+
+it("keeps explicit dependencies while excluding unrelated scoped unknowns", () => {
+ const comparisonId = ev("a");
+ const other = createResidualUnknown(
+ {
+ ...base,
+ question: "Other dimension",
+ comparison_conclusion_scope: {
+ comparison_evidence_id: comparisonId,
+ dimensions: ["pseudocode"],
+ },
+ relationships: [],
+ },
+ ev("c"),
+ null,
+ );
+ const calls = createResidualUnknown(
+ {
+ ...base,
+ question: "Call dimension",
+ comparison_conclusion_scope: {
+ comparison_evidence_id: comparisonId,
+ dimensions: ["calls"],
+ },
+ relationships: [{ type: "depends-on", unknown_id: other.unknown_id }],
+ },
+ ev("d"),
+ null,
+ );
+ const unrelated = createResidualUnknown(
+ {
+ ...base,
+ question: "Unrelated dimension",
+ comparison_conclusion_scope: {
+ comparison_evidence_id: comparisonId,
+ dimensions: ["pseudocode"],
+ },
+ relationships: [],
+ },
+ ev("e"),
+ null,
+ );
+ const selected = reconstructionClaimUnknowns(
+ new Map([
+ [calls.unknown_id, calls],
+ [other.unknown_id, other],
+ [unrelated.unknown_id, unrelated],
+ ]),
+ comparisonId,
+ [ev("b")],
+ "calls",
+ );
+ expect(selected.map(({ unknown_id }) => unknown_id)).toEqual([
+ calls.unknown_id,
+ other.unknown_id,
+ ]);
+});
diff --git a/src/domain/reconstructionUnknowns.ts b/src/domain/reconstructionUnknowns.ts
index d05f96fe9..c09e814eb 100644
--- a/src/domain/reconstructionUnknowns.ts
+++ b/src/domain/reconstructionUnknowns.ts
@@ -28,13 +28,26 @@ export const reconstructionClaimUnknowns = (
heads: ReadonlyMap<string, ResidualUnknown>,
comparisonId: string,
sourceIds: readonly string[],
+ dimension?: string,
): ResidualUnknown[] => {
+ const appliesToClaim = (item: ResidualUnknown): boolean => {
+ const scope = item.comparison_conclusion_scope;
+ return (
+ scope === undefined ||
+ (scope.comparison_evidence_id === comparisonId &&
+ (dimension === undefined ||
+ scope.dimensions.some(
+ (scopedDimension) => scopedDimension === dimension,
+ )))
+ );
+ };
const evidenceIds = new Set([comparisonId, ...sourceIds]);
const relevantIds = new Set(
[...heads.values()]
.filter(
(item) =>
item.status !== "resolved" &&
+ appliesToClaim(item) &&
[
...item.supporting_evidence_ids,
...item.contradicting_evidence_ids,
diff --git a/src/domain/reconstructionVerification.functionSources.test.ts b/src/domain/reconstructionVerification.functionSources.test.ts
index 4c4c04f04..5dc8db589 100644
--- a/src/domain/reconstructionVerification.functionSources.test.ts
+++ b/src/domain/reconstructionVerification.functionSources.test.ts
@@ -69,6 +69,7 @@ const source = (
result: dossier,
confidence: "derived",
authority: "shipped-artifact",
+ relationshipCoverage: { status: "complete" },
...observation,
},
);
@@ -99,14 +100,15 @@ const comparison = (
provider?: EvidenceProvider;
target?: EvidenceSubjectTarget;
} = {},
+ resultInput = baselineResult,
): Evidence =>
createEvidence(identity.target, identity.provider ?? provider, {
predicateType: "rea.function-comparison",
operation: "compare_functions",
parameters,
result: jsonValueSchema.parse({
- ...baselineResult,
- dimensions: baselineResult.dimensions.map((dimension) => ({
+ ...resultInput,
+ dimensions: resultInput.dimensions.map((dimension) => ({
...dimension,
evidence_links: sources.map(({ evidence_id: id }) => id),
})),
@@ -195,6 +197,100 @@ describe("function reconstruction source parameters", () => {
});
});
+describe("function reconstruction call coverage", () => {
+ it("rechecks old unchanged comparisons without affecting unrelated finite dimensions", () => {
+ const incomplete = source("7", {
+ relationshipCoverage: {
+ status: "unknown",
+ reason: "Call graph unavailable.",
+ },
+ });
+ const incompleteSides = {
+ left_evidence_id: left.evidence_id,
+ right_evidence_id: incomplete.evidence_id,
+ };
+ const oldUnchanged = comparison(incompleteSides, {}, [left, incomplete]);
+ for (const dimension of ["calls", "overall"] as const) {
+ const result = admittedVerification(
+ oldUnchanged,
+ [left, incomplete],
+ dimension,
+ )();
+ expect(result).toMatchObject({
+ status: "unknown",
+ summary: { total: 1, passed: 0, failed: 0, unknown: 1 },
+ claims: {
+ items: [
+ expect.objectContaining({
+ dimension,
+ status: "unknown",
+ observed_status: "unchanged",
+ limitations: expect.arrayContaining([
+ `Source ${incomplete.evidence_id}: Call graph unavailable.`,
+ ]),
+ }),
+ ],
+ },
+ });
+ }
+ expect(
+ admittedVerification(oldUnchanged, [left, incomplete], "pseudocode")(),
+ ).toMatchObject({
+ status: "pass",
+ claims: {
+ items: [
+ expect.objectContaining({
+ dimension: "pseudocode",
+ status: "pass",
+ observed_status: "unchanged",
+ }),
+ ],
+ },
+ });
+ });
+
+ it("keeps a confirmed historical overall change as a failure despite unknown calls", () => {
+ const changedDossier = jsonValueSchema.parse({
+ ...functionDossierSchema.parse(dossier),
+ pseudocode: "return 1;",
+ });
+ const changedComplete = source("8", { result: changedDossier });
+ const incomplete = source("9", {
+ result: changedDossier,
+ relationshipCoverage: {
+ status: "unknown",
+ reason: "Call graph unavailable.",
+ },
+ });
+ const changedResult = compareFunctions(left, changedComplete);
+ const oldChanged = comparison(
+ {
+ left_evidence_id: left.evidence_id,
+ right_evidence_id: incomplete.evidence_id,
+ },
+ {},
+ [left, incomplete],
+ {},
+ changedResult,
+ );
+ expect(oldChanged.normalized_result).toMatchObject({ status: "changed" });
+ expect(
+ admittedVerification(oldChanged, [left, incomplete], "overall")(),
+ ).toMatchObject({
+ status: "fail",
+ summary: { total: 1, passed: 0, failed: 1, unknown: 0 },
+ claims: {
+ items: [
+ expect.objectContaining({
+ status: "fail",
+ observed_status: "changed",
+ }),
+ ],
+ },
+ });
+ });
+});
+
describe("function reconstruction source aliases", () => {
it.each([
["no source declarations", {}],
diff --git a/src/domain/reconstructionVerification.ts b/src/domain/reconstructionVerification.ts
index 3dd9e3baf..dd300a0a2 100644
--- a/src/domain/reconstructionVerification.ts
+++ b/src/domain/reconstructionVerification.ts
@@ -16,6 +16,7 @@ import {
isProcessEvidenceProvider,
} from "./process/processEvidenceProvider.js";
import { functionDossierSchema } from "./hopperValues.js";
+import { functionCallCoverageLimitations } from "./functionCallCoverage.js";
import {
deriveProcessComparisonStatus,
PROCESS_COMPARISON_DIMENSIONS,
@@ -58,6 +59,11 @@ const providers = {
},
} as const;
+interface FunctionSourceFacts {
+ readonly evidence_id: string;
+ readonly callCoverageLimitations: readonly string[];
+}
+
const acceptsPredicate = (
expected: readonly string[],
actual: string,
@@ -137,14 +143,19 @@ const evaluateClaim = (
throw new TypeError(`Missing source Evidence ${id}`);
return source;
});
- validateSourceKinds(claim, sources);
+ const functionSourceFacts = readValidatedSourceFacts(claim, sources);
const observed = observedStatus(claim, comparison);
- const limitations = authorityLimitations(claim, sources, sides);
- const relevantUnknowns = reconstructionClaimUnknowns(
+ const limitations = [
+ ...authorityLimitations(claim, sources, sides),
+ ...sourceCallCoverageLimitations(claim, functionSourceFacts, observed),
+ ];
+ const relatedUnknowns = reconstructionClaimUnknowns(
heads,
comparison.evidence_id,
allSourceIds,
+ claim.dimension,
);
+ const relevantUnknowns = relatedUnknowns;
const status = classify(observed, limitations, relevantUnknowns);
const displayedUnknowns = relevantUnknowns;
return reconstructionClaimResultSchema.parse({
@@ -168,10 +179,10 @@ const evaluateClaim = (
});
};
-const validateSourceKinds = (
+const readValidatedSourceFacts = (
claim: Claim,
sources: readonly Evidence[],
-): void => {
+): readonly FunctionSourceFacts[] => {
const expected =
claim.kind === "behavioral"
? ["capture_process_scenario", "rea.process-capture"]
@@ -188,13 +199,22 @@ const validateSourceKinds = (
throw new TypeError(
"Comparison source Evidence has an unexpected observation type",
);
+ const functionSources: FunctionSourceFacts[] = [];
for (const source of sources) {
if (claim.kind === "behavioral")
parseProcessCapture(source.normalized_result);
- else if (claim.kind === "structural-function")
- functionDossierSchema.parse(source.normalized_result);
- else artifactInventoryResultSchema.parse(source.normalized_result);
+ else if (claim.kind === "structural-function") {
+ const dossier = functionDossierSchema.parse(source.normalized_result);
+ functionSources.push({
+ evidence_id: source.evidence_id,
+ callCoverageLimitations: functionCallCoverageLimitations(
+ source.relationship_coverage,
+ dossier,
+ ),
+ });
+ } else artifactInventoryResultSchema.parse(source.normalized_result);
}
+ return functionSources;
};
const validateComparisonIdentity = (claim: Claim, evidence: Evidence): void => {
@@ -317,6 +337,22 @@ const authorityLimitations = (
return limitations;
};
+const sourceCallCoverageLimitations = (
+ claim: Claim,
+ functionSources: readonly FunctionSourceFacts[],
+ observed: ObservedStatus,
+): string[] => {
+ if (
+ claim.kind !== "structural-function" ||
+ (claim.dimension !== "calls" &&
+ !(claim.dimension === "overall" && observed !== "changed"))
+ )
+ return [];
+ return functionSources.flatMap(({ evidence_id, callCoverageLimitations }) =>
+ callCoverageLimitations.map((reason) => `Source ${evidence_id}: ${reason}`),
+ );
+};
+
const classify = (
observed: ObservedStatus,
limitations: readonly string[],
diff --git a/src/domain/residualUnknown.test.ts b/src/domain/residualUnknown.test.ts
index f48e80376..5f07078f2 100644
--- a/src/domain/residualUnknown.test.ts
+++ b/src/domain/residualUnknown.test.ts
@@ -97,6 +97,49 @@ it("normalizes producer text before identity and revision authentication", () =>
expect(residualUnknownSchema.parse(created)).toEqual(created);
});
+it("authenticates typed comparison scope and preserves it across revisions", () => {
+ const comparisonId = `ev_${"c".repeat(64)}`;
+ const scopedInput = recordUnknownInputSchema.parse({
+ ...input,
+ domain: "function-comparison",
+ supporting_evidence_ids: [comparisonId],
+ comparison_conclusion_scope: {
+ comparison_evidence_id: comparisonId,
+ dimensions: ["calls"],
+ },
+ });
+ const created = createResidualUnknown(scopedInput, mutationId, scopeDigest);
+ const updated = updateResidualUnknown(
+ created,
+ {
+ ...input,
+ unknown_id: created.unknown_id,
+ expected_revision: 1,
+ status: "investigating",
+ severity: "high",
+ supporting_evidence_ids: [comparisonId],
+ required_confidence: "observed",
+ recommended_probes: [],
+ resolution: null,
+ },
+ mutationId,
+ );
+ expect(updated.comparison_conclusion_scope).toEqual({
+ comparison_evidence_id: comparisonId,
+ dimensions: ["calls"],
+ });
+ expect(residualUnknownSchema.parse(updated)).toEqual(updated);
+ expect(
+ residualUnknownSchema.safeParse({
+ ...created,
+ comparison_conclusion_scope: {
+ comparison_evidence_id: `ev_${"d".repeat(64)}`,
+ dimensions: ["calls"],
+ },
+ }).success,
+ ).toBe(false);
+});
+
it("keeps generated revisions subject to state and relationship validation", () => {
const created = createResidualUnknown(input, mutationId, scopeDigest);
const command = {
diff --git a/src/domain/residualUnknown.ts b/src/domain/residualUnknown.ts
index 88e0da49e..0819a3964 100644
--- a/src/domain/residualUnknown.ts
+++ b/src/domain/residualUnknown.ts
@@ -7,6 +7,7 @@ import {
compositeKey,
} from "./unicodeCodePointOrder.js";
import { prefixedDigestSchema } from "./../domain/digests.js";
+import { functionComparisonDimensionNameSchema } from "./functionComparisonSchemas.js";
const evidenceIdSchema = prefixedDigestSchema("ev");
const unknownIdSchema = prefixedDigestSchema("unk");
@@ -23,6 +24,7 @@ const stableUnknownId = (
question: string,
domain: string,
scopeDigest: string | null,
+ comparisonConclusionScope?: z.infer<typeof comparisonConclusionScopeSchema>,
): string =>
`unk_${createHash("sha256")
.update(
@@ -30,6 +32,9 @@ const stableUnknownId = (
domain: domain.trim(),
question: question.replace(/\s+/gu, " ").trim(),
scope_digest: scopeDigest,
+ ...(comparisonConclusionScope === undefined
+ ? {}
+ : { comparison_conclusion_scope: comparisonConclusionScope }),
}),
)
.digest("hex")}`;
@@ -38,6 +43,23 @@ const probeSchema = z.object({
operation: z.string().min(1),
rationale: text,
});
+const comparisonConclusionScopeSchema = z
+ .object({
+ comparison_evidence_id: evidenceIdSchema,
+ dimensions: z
+ .array(
+ z.union([z.literal("overall"), functionComparisonDimensionNameSchema]),
+ )
+ .min(1),
+ })
+ .strict()
+ .superRefine(({ dimensions }, context) => {
+ if (!sameOrderedValues(dimensions, sortedUnique(dimensions)))
+ context.addIssue({
+ code: "custom",
+ message: "Comparison dimensions must be sorted and unique",
+ });
+ });
const relationshipSchema = z.object({
type: z.enum(["depends-on", "related-to", "contradicts"]),
unknown_id: unknownIdSchema,
@@ -78,6 +100,7 @@ const residualUnknownObjectSchema = z.object({
]),
severity: z.enum(["low", "medium", "high", "critical"]),
domain: text,
+ comparison_conclusion_scope: comparisonConclusionScopeSchema.exactOptional(),
supporting_evidence_ids: z.array(evidenceIdSchema),
contradicting_evidence_ids: z.array(evidenceIdSchema),
required_authority: z
@@ -97,6 +120,8 @@ const residualUnknownObjectSchema = z.object({
mutation_evidence_ids: z.array(evidenceIdSchema).min(1),
});
+export type ResidualUnknown = z.infer<typeof residualUnknownSchema>;
+
type UnknownRevision = Omit<
z.infer<typeof residualUnknownObjectSchema>,
"revision_digest"
@@ -105,7 +130,12 @@ type UnknownRevision = Omit<
const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => {
if (
value.unknown_id !==
- stableUnknownId(value.question, value.domain, value.scope_digest)
+ stableUnknownId(
+ value.question,
+ value.domain,
+ value.scope_digest,
+ value.comparison_conclusion_scope,
+ )
)
context.addIssue({
code: "custom",
@@ -153,6 +183,29 @@ const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => {
code: "custom",
message: "Evidence IDs must be unique",
});
+ if (
+ value.comparison_conclusion_scope !== undefined &&
+ ![
+ ...value.supporting_evidence_ids,
+ ...value.contradicting_evidence_ids,
+ ].includes(value.comparison_conclusion_scope.comparison_evidence_id)
+ )
+ context.addIssue({
+ code: "custom",
+ message:
+ "Scoped comparison Evidence must support or contradict the unknown",
+ path: ["comparison_conclusion_scope"],
+ });
+ if (
+ value.comparison_conclusion_scope !== undefined &&
+ value.domain !== "function-comparison"
+ )
+ context.addIssue({
+ code: "custom",
+ message:
+ "Comparison conclusion scope requires function-comparison domain",
+ path: ["comparison_conclusion_scope"],
+ });
const contradictingEvidence = new Set(value.contradicting_evidence_ids);
if (value.supporting_evidence_ids.some((id) => contradictingEvidence.has(id)))
context.addIssue({
@@ -170,7 +223,7 @@ const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => {
});
};
-export const residualUnknownSchema = residualUnknownObjectSchema.superRefine(
+const residualUnknownValidatedSchema = residualUnknownObjectSchema.superRefine(
(value, context) => {
if (computeRevisionDigest(value) !== value.revision_digest)
context.addIssue({
@@ -182,11 +235,12 @@ export const residualUnknownSchema = residualUnknownObjectSchema.superRefine(
},
);
+export const residualUnknownSchema = residualUnknownValidatedSchema;
+
const generatedRevisionSchema = residualUnknownObjectSchema
.omit({ revision_digest: true })
.superRefine(refineRevision);
-export type ResidualUnknown = z.infer<typeof residualUnknownSchema>;
export type UnknownStatus = ResidualUnknown["status"];
/** Input for deterministic creation of one residual unknown. */
@@ -237,8 +291,21 @@ export const createResidualUnknown = (
): ResidualUnknown => {
const question = input.question.replace(/\s+/gu, " ").trim();
const domain = input.domain.trim();
+ const comparisonConclusionScope =
+ input.comparison_conclusion_scope === undefined
+ ? undefined
+ : {
+ comparison_evidence_id:
+ input.comparison_conclusion_scope.comparison_evidence_id,
+ dimensions: input.comparison_conclusion_scope.dimensions,
+ };
const revision = {
- unknown_id: stableUnknownId(question, domain, scopeDigest),
+ unknown_id: stableUnknownId(
+ question,
+ domain,
+ scopeDigest,
+ comparisonConclusionScope,
+ ),
revision: 1,
previous_revision_digest: null,
scope_digest: scopeDigest,
@@ -246,6 +313,9 @@ export const createResidualUnknown = (
status:
input.contradicting_evidence_ids.length > 0 ? "contradicted" : "open",
domain,
+ ...(comparisonConclusionScope === undefined
+ ? {}
+ : { comparison_conclusion_scope: comparisonConclusionScope }),
...projectRevisionDetails(input),
resolution: null,
mutation_evidence_ids: [mutationEvidenceId],
@@ -267,6 +337,9 @@ export const updateResidualUnknown = (
question: current.question,
status: input.status,
domain: current.domain,
+ ...(current.comparison_conclusion_scope === undefined
+ ? {}
+ : { comparison_conclusion_scope: current.comparison_conclusion_scope }),
...projectRevisionDetails(input),
resolution: input.resolution,
mutation_evidence_ids: sortedUnique([
@@ -307,7 +380,7 @@ const projectRevisionDetails = (input: RevisionDetailsInput) => ({
});
const computeRevisionDigest = (
- value: Omit<ResidualUnknown, "revision_digest"> | ResidualUnknown,
+ value: UnknownRevision | ResidualUnknown,
): string => {
const { revision_digest: _revisionDigest, ...semantic } =
"revision_digest" in value ? value : { ...value, revision_digest: "" };
@@ -323,6 +396,12 @@ const canonicalJson = (value: unknown): string => {
const sortedUnique = (values: readonly string[]): string[] =>
[...new Set(values)].sort(compareUnicodeCodePoints);
+const sameOrderedValues = (
+ left: readonly string[],
+ right: readonly string[],
+): boolean =>
+ left.length === right.length &&
+ left.every((value, index) => value === right[index]);
const sortedRelationships = (
values: readonly z.infer<typeof relationshipSchema>[],
diff --git a/src/ida/IdaNativeProtocol.test.ts b/src/ida/IdaNativeProtocol.test.ts
index 0e17167b3..f90673031 100644
--- a/src/ida/IdaNativeProtocol.test.ts
+++ b/src/ida/IdaNativeProtocol.test.ts
@@ -223,11 +223,22 @@ describe("native IDA provider observations", () => {
procedure: "main",
});
if (!dossier.ok) throw dossier.error;
- expect(dossier.value.relationshipCoverage).toEqual({
+ expect(dossier.value.relationshipCoverage).toMatchObject({
status: "unknown",
- reason:
- "Upstream xrefs do not establish direct caller relationships.",
+ reason: more
+ ? expect.stringContaining(
+ "Callees: Upstream callees capped results at 500; additional callees remain unknown.",
+ )
+ : expect.stringContaining(
+ "Callers: Upstream xrefs do not establish direct caller relationships.",
+ ),
});
+ if (more)
+ expect(dossier.value.relationshipCoverage).toMatchObject({
+ reason: expect.stringContaining(
+ "Callers: Upstream xrefs do not establish direct caller relationships.",
+ ),
+ });
expect(dossier.value.result).toMatchObject({
callers: [],
callees: [{ address: "0x2000", name: "puts" }],
diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts
index 530376458..c30ec81bc 100644
--- a/src/ida/IdaOperationRunner.ts
+++ b/src/ida/IdaOperationRunner.ts
@@ -19,6 +19,7 @@ import {
} from "../domain/hopperValues.js";
import type { JsonValue } from "../domain/jsonValue.js";
import type { RelationshipCoverage } from "../domain/relationshipCoverage.js";
+import { combineFunctionRelationshipCoverage } from "../domain/functionCallCoverage.js";
import { readCallGraphCallees, unavailableCallGraph } from "./IdaCallGraph.js";
import type { IdaMcpConnection } from "./IdaMcpConnection.js";
import type { IdaOperation } from "./IdaProviderCapabilities.js";
@@ -70,33 +71,6 @@ interface IdaRelationships {
readonly coverage: RelationshipCoverage;
}
-const combinedRelationshipCoverage = (
- callers: RelationshipCoverage,
- callees: RelationshipCoverage,
-): RelationshipCoverage => {
- const coverage = [callers, callees];
- const diagnostics = [
- ...new Set(
- coverage.flatMap((item) =>
- "diagnostics" in item ? item.diagnostics : [],
- ),
- ),
- ];
- const details = diagnostics.length > 0 ? { diagnostics } : {};
- const unknown = coverage.filter((item) => item.status === "unknown");
- if (unknown.length > 0)
- return {
- status: "unknown",
- reason: [...new Set(unknown.map((item) => item.reason))].join(" "),
- ...details,
- };
- const limitations = coverage.flatMap((item) =>
- item.status === "partial" ? item.limitations : [],
- );
- return limitations.length > 0
- ? { status: "partial", limitations, ...details }
- : { status: "complete", ...details };
-};
/** The only lookup answers upstream reports for a name or address that is not a function. */
const MISSING_PROCEDURE_ANSWERS: ReadonlySet<string> = new Set([
"Not found",
@@ -632,7 +606,7 @@ export class IdaOperationRunner {
const pseudocode = await this.pseudocode(fn);
const callers = await this.relationships(fn, "callers");
const callees = await this.relationships(fn, "callees");
- this.relationshipCoverage = combinedRelationshipCoverage(
+ this.relationshipCoverage = combineFunctionRelationshipCoverage(
callers.coverage,
callees.coverage,
);
diff --git a/src/server/registerFunctionComparisonTool.ts b/src/server/registerFunctionComparisonTool.ts
index b968972ed..405efd713 100644
--- a/src/server/registerFunctionComparisonTool.ts
+++ b/src/server/registerFunctionComparisonTool.ts
@@ -20,6 +20,7 @@ import { withAdmittedAnalysis } from "./analysisAdmission.js";
import { runAdmittedToolOperation } from "./admittedToolOperation.js";
import { compareFunctions } from "../domain/functionComparison.js";
+import { uniqueSorted } from "../domain/canonicalOrdering.js";
/** Register explicit Evidence-backed function comparison. */
export const registerFunctionComparisonTool = (
server: EvidenceMcpServer,
@@ -102,12 +103,12 @@ export const registerFunctionComparisonTool = (
const recorded = recordDerivedEvidence(
session,
evidence,
- functionUnknownInput({
- status: comparison.status,
+ comparisonUnknownInput(
+ comparison,
leftIds,
rightIds,
- comparisonEvidenceId: evidence.evidence_id,
- }),
+ evidence.evidence_id,
+ ),
);
return server.delivery.toEvidenceToolResult(
evidence,
@@ -119,26 +120,63 @@ export const registerFunctionComparisonTool = (
);
};
+const comparisonUnknownInput = (
+ comparison: ReturnType<typeof compareFunctions>,
+ leftIds: readonly string[],
+ rightIds: readonly string[],
+ comparisonEvidenceId: string,
+): RecordUnknownInput | undefined =>
+ functionUnknownInput({
+ status: comparison.status,
+ dimensions: comparison.dimensions
+ .filter(({ status }) =>
+ comparison.status === "unknown"
+ ? status === "unknown"
+ : status === "changed",
+ )
+ .map(({ dimension }) => dimension),
+ leftIds,
+ rightIds,
+ comparisonEvidenceId,
+ });
+
const functionUnknownInput = ({
status,
+ dimensions,
leftIds,
rightIds,
comparisonEvidenceId,
}: {
status: ReturnType<typeof compareFunctions>["status"];
+ dimensions: readonly ReturnType<
+ typeof compareFunctions
+ >["dimensions"][number]["dimension"][];
leftIds: readonly string[];
rightIds: readonly string[];
comparisonEvidenceId: string;
}): RecordUnknownInput | undefined => {
if (status === "unchanged") return undefined;
- const supportingIds = [...new Set([...leftIds, ...rightIds])];
+ const supportingIds = [
+ ...new Set([...leftIds, ...rightIds, comparisonEvidenceId]),
+ ];
const contradictingIds = rightIds.filter((id) => !leftIds.includes(id));
const contradictory = status === "changed";
return {
question: `Function comparison is ${status} (comparison ${comparisonEvidenceId})`,
severity: status === "changed" ? "medium" : "high",
domain: "function-comparison",
- supporting_evidence_ids: contradictory ? [...leftIds] : supportingIds,
+ comparison_conclusion_scope: {
+ comparison_evidence_id: comparisonEvidenceId,
+ dimensions: uniqueSorted([
+ ...(status === "unknown" || dimensions.length === 0
+ ? ["overall" as const]
+ : []),
+ ...dimensions,
+ ]),
+ },
+ supporting_evidence_ids: contradictory
+ ? [...leftIds, comparisonEvidenceId]
+ : supportingIds,
contradicting_evidence_ids: contradictory ? contradictingIds : [],
required_authority: "shipped-artifact",
required_confidence: "observed",
diff --git a/tests/boundary/mcp/functionComparisonCoverageMcp.test.ts b/tests/boundary/mcp/functionComparisonCoverageMcp.test.ts
new file mode 100644
index 000000000..3d3295fc4
--- /dev/null
+++ b/tests/boundary/mcp/functionComparisonCoverageMcp.test.ts
@@ -0,0 +1,233 @@
+import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
+import { expect, it } from "vitest";
+
+import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js";
+import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
+import { functionDossierSchema } from "../../../src/domain/hopperValues.js";
+import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
+import { createServer } from "../../../src/server/createServer.js";
+import { createTestBinarySession } from "../../fixtures/binarySession.js";
+
+const sourceWithCoverage = (
+ template: typeof FUNCTION_COMPARISON_EXAMPLE.left,
+ observation: string,
+ result: typeof template.normalized_result,
+ relationshipCoverage:
+ | { readonly status: "complete" }
+ | { readonly status: "unknown"; readonly reason: string },
+) => {
+ if (template.subject === null)
+ throw new Error("Function comparison fixtures must be artifact-bound");
+ return createEvidence(
+ {
+ path: `${template.subject.local_path}/${observation}`,
+ sha256: template.subject.digest.sha256,
+ format: template.subject.format,
+ },
+ template.provider,
+ {
+ operation: "analyze_function",
+ parameters: { ...template.parameters, observation },
+ result,
+ confidence: template.confidence,
+ authority: template.authority,
+ relationshipCoverage,
+ },
+ );
+};
+
+const verify = async (
+ call: (
+ name: string,
+ arguments_: Record<string, unknown>,
+ ) => Promise<{
+ readonly isError?: boolean | undefined;
+ readonly structuredContent?: unknown | undefined;
+ }>,
+ comparison: ReturnType<typeof parseEvidence>,
+ dimensions: readonly ("calls" | "overall" | "pseudocode")[],
+) => {
+ const response = await call("verify_reconstruction", {
+ specification: {
+ name: "Function call coverage verification",
+ claims: dimensions.map((dimension) => ({
+ kind: "structural-function",
+ claim_id: dimension,
+ title: `Check ${dimension}`,
+ comparison_evidence_id: comparison.evidence_id,
+ dimension,
+ })),
+ },
+ });
+ expect(response.isError, JSON.stringify(response.structuredContent)).not.toBe(
+ true,
+ );
+ return parseEvidence(response.structuredContent);
+};
+
+const assertChangedDespiteUnknownCalls = async (
+ call: Parameters<typeof verify>[0],
+ left: ReturnType<typeof parseEvidence>,
+): Promise<void> => {
+ const changedRight = sourceWithCoverage(
+ FUNCTION_COMPARISON_EXAMPLE.right,
+ "coverage-changed-right",
+ jsonValueSchema.parse({
+ ...functionDossierSchema.parse(
+ FUNCTION_COMPARISON_EXAMPLE.right.normalized_result,
+ ),
+ pseudocode: "return 1;",
+ }),
+ { status: "unknown", reason: "Call graph unavailable." },
+ );
+ const changed = await call("compare_functions", {
+ left,
+ right: changedRight,
+ });
+ expect(changed.isError, JSON.stringify(changed.structuredContent)).not.toBe(
+ true,
+ );
+ const changedComparison = parseEvidence(changed.structuredContent);
+ expect(changedComparison.normalized_result).toMatchObject({
+ status: "changed",
+ dimensions: expect.arrayContaining([
+ expect.objectContaining({ dimension: "calls", status: "unknown" }),
+ expect.objectContaining({ dimension: "pseudocode", status: "changed" }),
+ ]),
+ });
+ const verification = await verify(call, changedComparison, ["overall"]);
+ expect(verification.normalized_result).toMatchObject({
+ status: "fail",
+ summary: { total: 1, passed: 0, failed: 1, unknown: 0 },
+ claims: {
+ items: [
+ expect.objectContaining({
+ claim_id: "overall",
+ status: "fail",
+ observed_status: "changed",
+ }),
+ ],
+ },
+ });
+};
+
+it("keeps incomplete calls unknown through public comparison and verification while preserving confirmed changes", async () => {
+ const session = createTestBinarySession(() => {
+ throw new Error("Function comparison must not launch a provider");
+ });
+ const server = createServer({ kind: "session", session });
+ const client = new Client({ name: "function-coverage-test", version: "1" });
+ const [clientTransport, serverTransport] =
+ InMemoryTransport.createLinkedPair();
+ try {
+ await server.connect(serverTransport);
+ await client.connect(clientTransport);
+ const left = sourceWithCoverage(
+ FUNCTION_COMPARISON_EXAMPLE.left,
+ "coverage-left",
+ FUNCTION_COMPARISON_EXAMPLE.left.normalized_result,
+ { status: "complete" },
+ );
+ const right = sourceWithCoverage(
+ FUNCTION_COMPARISON_EXAMPLE.right,
+ "coverage-right",
+ FUNCTION_COMPARISON_EXAMPLE.left.normalized_result,
+ { status: "unknown", reason: "Call graph unavailable." },
+ );
+ expect(left.subject).not.toBeNull();
+ expect(right.subject).not.toBeNull();
+ expect(left.limitations).not.toContain(
+ "Artifact identity is unavailable for this observation.",
+ );
+ expect(right.limitations).not.toContain(
+ "Artifact identity is unavailable for this observation.",
+ );
+ const compared = await client.callTool({
+ name: "compare_functions",
+ arguments: { left, right },
+ });
+ expect(
+ compared.isError,
+ JSON.stringify(compared.structuredContent),
+ ).not.toBe(true);
+ const comparison = parseEvidence(compared.structuredContent);
+ expect(session.evidenceById(left.evidence_id)?.subject).not.toBeNull();
+ expect(session.evidenceById(right.evidence_id)?.subject).not.toBeNull();
+ expect(comparison.normalized_result).toMatchObject({
+ status: "unknown",
+ summary: { unchanged: 7, changed: 0, unknown: 1 },
+ dimensions: expect.arrayContaining([
+ expect.objectContaining({
+ dimension: "calls",
+ status: "unknown",
+ conclusion_kind: "unresolved_branch",
+ left_count: 0,
+ right_count: 0,
+ limitations: ["Right: Call graph unavailable."],
+ evidence_links: [left.evidence_id, right.evidence_id],
+ }),
+ ]),
+ });
+ expect(comparison.evidence_links).toEqual(
+ expect.arrayContaining([left.evidence_id, right.evidence_id]),
+ );
+
+ const unknowns = await client.callTool({
+ name: "list_unknowns",
+ arguments: { domain: "function-comparison" },
+ });
+ expect(unknowns.isError).not.toBe(true);
+ expect(unknowns.structuredContent).toMatchObject({
+ result: {
+ items: [
+ expect.objectContaining({
+ domain: "function-comparison",
+ status: "open",
+ supporting_evidence_ids: expect.arrayContaining([
+ left.evidence_id,
+ right.evidence_id,
+ ]),
+ }),
+ ],
+ },
+ });
+
+ const verification = await verify(
+ async (name, arguments_) =>
+ client.callTool({ name, arguments: arguments_ }),
+ comparison,
+ ["calls", "overall", "pseudocode"],
+ );
+ expect(verification.normalized_result).toMatchObject({
+ status: "unknown",
+ summary: { total: 3, passed: 1, failed: 0, unknown: 2 },
+ claims: {
+ items: expect.arrayContaining([
+ expect.objectContaining({
+ claim_id: "calls",
+ status: "unknown",
+ observed_status: "unknown",
+ }),
+ expect.objectContaining({
+ claim_id: "overall",
+ status: "unknown",
+ observed_status: "unknown",
+ }),
+ expect.objectContaining({
+ claim_id: "pseudocode",
+ status: "pass",
+ observed_status: "unchanged",
+ }),
+ ]),
+ },
+ });
+
+ await assertChangedDespiteUnknownCalls(
+ async (name, arguments_) =>
+ client.callTool({ name, arguments: arguments_ }),
+ left,
+ );
+ } finally {
+ await Promise.allSettled([client.close(), server.close(), session.close()]);
+ }
+});