Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

refactor(evidence)!: require current coverage commitments

morlutodb8128dc0d1c
diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts
index 91b9f8807..a4025bcca 100644
--- a/src/application/binary/AnalysisSnapshotCache.test.ts
+++ b/src/application/binary/AnalysisSnapshotCache.test.ts
@@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest";
import { createAnalysisProfile } from "../../domain/analysisProfile.js";
+import type { RelationshipCoverage } from "../../domain/relationshipCoverage.js";
import {
ANALYSIS_SNAPSHOT_PROFILE,
ANALYSIS_SNAPSHOT_PROVIDER,
@@ -502,57 +503,62 @@ describe("analysis snapshot cache partitioning", () => {
});
});
+const createRelationshipCoverageSnapshot = (
+ relationshipCoverage?: RelationshipCoverage,
+) => {
+ const operation = "procedure_callees" as const;
+ const parameters = { procedure: "0x1000", document: "fixture" };
+ const cache = new AnalysisSnapshotCache();
+ const execution = createAnalysisExecution(
+ ["0x2000"],
+ ANALYSIS_SNAPSHOT_PROVIDER,
+ {
+ analysisProfile: ANALYSIS_SNAPSHOT_PROFILE,
+ rawResult: ["0x2000"],
+ subject: ANALYSIS_SNAPSHOT_TARGET,
+ ...(relationshipCoverage === undefined
+ ? {}
+ : { relationshipCoverage }),
+ },
+ );
+ cache.record({
+ target: ANALYSIS_SNAPSHOT_TARGET,
+ profile: ANALYSIS_SNAPSHOT_PROFILE,
+ operation,
+ parameters,
+ execution,
+ });
+ const evidence = createEvidence(
+ ANALYSIS_SNAPSHOT_TARGET,
+ ANALYSIS_SNAPSHOT_PROVIDER,
+ {
+ operation,
+ parameters,
+ result: ["0x2000"],
+ rawResult: ["0x2000"],
+ analysisProfile: ANALYSIS_SNAPSHOT_PROFILE,
+ ...(relationshipCoverage === undefined
+ ? {}
+ : { relationshipCoverage }),
+ },
+ );
+ const exported = cache.export(
+ ANALYSIS_SNAPSHOT_TARGET,
+ ANALYSIS_SNAPSHOT_PROFILE,
+ createEvidenceBundle([evidence]),
+ );
+ if (!exported.ok) throw exported.error;
+ return { cache, operation, parameters, snapshot: exported.value };
+};
+
describe("relationship coverage snapshot replay", () => {
- it("authenticates relationship coverage and replays legacy reads as unknown", () => {
- const cache = new AnalysisSnapshotCache();
- const operation = "procedure_callees" as const;
- const parameters = { procedure: "0x1000", document: "fixture" };
- const coverage = {
- status: "complete" as const,
+ it("authenticates coverage and rejects missing serialized fields", () => {
+ const coverage: RelationshipCoverage = {
+ status: "complete",
diagnostics: ["The provider retained an unresolved call target."],
};
- const execution = createAnalysisExecution(
- ["0x2000"],
- ANALYSIS_SNAPSHOT_PROVIDER,
- {
- analysisProfile: ANALYSIS_SNAPSHOT_PROFILE,
- rawResult: ["0x2000"],
- relationshipCoverage: coverage,
- subject: {
- path: ANALYSIS_SNAPSHOT_TARGET.path,
- sha256: ANALYSIS_SNAPSHOT_TARGET.sha256,
- format: ANALYSIS_SNAPSHOT_TARGET.format,
- ...(ANALYSIS_SNAPSHOT_TARGET.architecture === undefined
- ? {}
- : { architecture: ANALYSIS_SNAPSHOT_TARGET.architecture }),
- },
- },
- );
- cache.record({
- target: ANALYSIS_SNAPSHOT_TARGET,
- profile: ANALYSIS_SNAPSHOT_PROFILE,
- operation,
- parameters,
- execution,
- });
- const evidence = createEvidence(
- ANALYSIS_SNAPSHOT_TARGET,
- ANALYSIS_SNAPSHOT_PROVIDER,
- {
- operation,
- parameters,
- result: ["0x2000"],
- rawResult: ["0x2000"],
- analysisProfile: ANALYSIS_SNAPSHOT_PROFILE,
- relationshipCoverage: coverage,
- },
- );
- const exported = cache.export(
- ANALYSIS_SNAPSHOT_TARGET,
- ANALYSIS_SNAPSHOT_PROFILE,
- createEvidenceBundle([evidence]),
- );
- if (!exported.ok) throw exported.error;
+ const { cache, operation, parameters, snapshot } =
+ createRelationshipCoverageSnapshot(coverage);
expect(
cache.lookup(
ANALYSIS_SNAPSHOT_TARGET,
@@ -562,7 +568,7 @@ describe("relationship coverage snapshot replay", () => {
)?.relationshipCoverage,
).toEqual(coverage);
- const tampered = structuredClone(exported.value);
+ const tampered = structuredClone(snapshot);
const tamperedEntry = tampered.entries[0];
if (tamperedEntry === undefined) throw new Error("snapshot entry missing");
tamperedEntry.execution.relationship_coverage = {
@@ -573,26 +579,34 @@ describe("relationship coverage snapshot replay", () => {
new AnalysisSnapshotCache().import(tampered, undefined, () => ok(0)).ok,
).toBe(false);
- const legacy = structuredClone(exported.value);
- const legacyEntry = legacy.entries[0];
- if (legacyEntry === undefined) throw new Error("snapshot entry missing");
- Reflect.deleteProperty(legacyEntry.execution, "relationship_coverage");
- const legacyEvidence = createEvidence(
- ANALYSIS_SNAPSHOT_TARGET,
- ANALYSIS_SNAPSHOT_PROVIDER,
- {
- operation,
- parameters,
- result: ["0x2000"],
- rawResult: ["0x2000"],
- analysisProfile: ANALYSIS_SNAPSHOT_PROFILE,
- },
+ const missingCoverage = structuredClone(snapshot);
+ const missingCoverageEntry = missingCoverage.entries[0];
+ if (missingCoverageEntry === undefined)
+ throw new Error("snapshot entry missing");
+ Reflect.deleteProperty(
+ missingCoverageEntry.execution,
+ "relationship_coverage",
);
- Reflect.deleteProperty(legacyEvidence, "relationship_coverage");
- legacy.evidence_bundle = createEvidenceBundle([legacyEvidence]);
- parseAnalysisSnapshot(legacy);
+ const missingCoverageEvidence = missingCoverage.evidence_bundle.records[0];
+ if (missingCoverageEvidence === undefined)
+ throw new Error("snapshot Evidence missing");
+ Reflect.deleteProperty(missingCoverageEvidence, "relationship_coverage");
+ expect(() => parseAnalysisSnapshot(missingCoverage)).toThrow();
+ expect(
+ new AnalysisSnapshotCache().import(missingCoverage, undefined, () =>
+ ok(0),
+ ).ok,
+ ).toBe(false);
+ });
+
+ it("replays absent coverage as explicitly unknown", () => {
+ const { operation, parameters, snapshot } =
+ createRelationshipCoverageSnapshot();
+ expect(snapshot.entries[0]?.execution.relationship_coverage).toBeNull();
+ expect(snapshot.evidence_bundle.records[0]?.relationship_coverage).toBeNull();
+
const replayCache = new AnalysisSnapshotCache();
- expect(replayCache.import(legacy, undefined, () => ok(0)).ok).toBe(true);
+ expect(replayCache.import(snapshot, undefined, () => ok(0)).ok).toBe(true);
expect(
replayCache.lookup(
ANALYSIS_SNAPSHOT_TARGET,
diff --git a/src/contracts/analysisViewApiSurfaceExample.ts b/src/contracts/analysisViewApiSurfaceExample.ts
index 389ac7268..5879637c2 100644
--- a/src/contracts/analysisViewApiSurfaceExample.ts
+++ b/src/contracts/analysisViewApiSurfaceExample.ts
@@ -4,7 +4,7 @@ export const API_SURFACE_EXAMPLE = {
source: {
kind: "retained-evidence",
evidence_id:
- "ev_9e218c18bc6159ec9c8492a9130c286496fb1b6d9f03ec2f4ea3321ce2b15097",
+ "ev_90073e9478eada1e2cc351b523e6ba9231c9b4bc9708773567b880094db9d4c7",
},
view: {
kind: "api-surface",
@@ -14,12 +14,12 @@ export const API_SURFACE_EXAMPLE = {
},
output: {
parent_evidence_id:
- "ev_9e218c18bc6159ec9c8492a9130c286496fb1b6d9f03ec2f4ea3321ce2b15097",
+ "ev_90073e9478eada1e2cc351b523e6ba9231c9b4bc9708773567b880094db9d4c7",
parent_operation: "inspect_web_network_capture",
parent_digest:
- "9e218c18bc6159ec9c8492a9130c286496fb1b6d9f03ec2f4ea3321ce2b15097",
+ "90073e9478eada1e2cc351b523e6ba9231c9b4bc9708773567b880094db9d4c7",
view_digest:
- "47c1859b7a86dd10770061767300bdb6ec177a9254e8dc7863d5ea8603343da6",
+ "c9494341920dc4cff6ecc6d8ed2c5e3abf15976642c253248e220d45f290ca7b",
artifact: {
path: "/fixtures/api-surface.har",
sha256:
diff --git a/src/domain/analysisSnapshot.test.ts b/src/domain/analysisSnapshot.test.ts
index d83ddac30..6ea0a1b5f 100644
--- a/src/domain/analysisSnapshot.test.ts
+++ b/src/domain/analysisSnapshot.test.ts
@@ -234,6 +234,7 @@ describe("analysis snapshot contract", () => {
raw_result: null,
provider: binding.provider,
limitations: [],
+ relationship_coverage: null,
locations: [],
subject: target,
},
@@ -262,6 +263,7 @@ describe("analysis snapshot contract", () => {
raw_result: null,
provider: binding.provider,
limitations: [],
+ relationship_coverage: null,
locations: [],
subject: {
path: "a\u0000b",
@@ -293,6 +295,7 @@ describe("analysis snapshot contract", () => {
raw_result: null,
provider: binding.provider,
limitations: [],
+ relationship_coverage: null,
locations: [],
subject: null,
},
@@ -456,6 +459,7 @@ describe("analysis snapshot Evidence binding", () => {
raw_result: rawResult,
provider: binding.provider,
limitations: ["fixture limitation"],
+ relationship_coverage: null,
locations: [{ kind: "address" as const, address: "0x1000" }],
subject,
},
@@ -480,11 +484,21 @@ describe("analysis snapshot Evidence binding", () => {
workflow_entries: [],
evidence_bundle: createEvidenceBundle([evidence]),
};
- const parsedLegacySnapshot = parseAnalysisSnapshot(snapshot);
+ const parsedSnapshot = parseAnalysisSnapshot(snapshot);
expect(
- parsedLegacySnapshot.entries[0]?.execution.relationship_coverage,
+ parsedSnapshot.entries[0]?.execution.relationship_coverage,
).toBeNull();
+ const missingCoverage = structuredClone(snapshot);
+ const missingCoverageEntry = missingCoverage.entries[0];
+ if (missingCoverageEntry === undefined)
+ throw new Error("snapshot entry missing");
+ Reflect.deleteProperty(
+ missingCoverageEntry.execution,
+ "relationship_coverage",
+ );
+ expect(() => parseAnalysisSnapshot(missingCoverage)).toThrow();
+
expect(() =>
parseAnalysisSnapshot({
...snapshot,
diff --git a/src/domain/analysisSnapshot.ts b/src/domain/analysisSnapshot.ts
index 4e1af5fa8..e0766ba15 100644
--- a/src/domain/analysisSnapshot.ts
+++ b/src/domain/analysisSnapshot.ts
@@ -145,7 +145,7 @@ const entrySchema = z.object({
raw_result: jsonValueSchema.nullable(),
provider: committedProviderSchema,
limitations: z.array(z.string()),
- relationship_coverage: relationshipCoverageSchema.nullable().default(null),
+ relationship_coverage: relationshipCoverageSchema.nullable(),
locations: z.array(evidenceLocationSchema),
subject: subjectSchema.nullable(),
}),
@@ -160,7 +160,7 @@ const workflowEntrySchema = z.object({
provider: committedProviderSchema,
analysis_profile: analysisProfileSchema,
limitations: z.array(z.string()),
- relationship_coverage: relationshipCoverageSchema.nullable().default(null),
+ relationship_coverage: relationshipCoverageSchema.nullable(),
locations: z.array(evidenceLocationSchema),
subject: subjectSchema.nullable(),
}),
@@ -305,9 +305,7 @@ const snapshotContentDigest = (
format: execution.subject.format,
architecture: execution.subject.architecture,
},
- ...(execution.relationship_coverage === null
- ? {}
- : { relationship_coverage: execution.relationship_coverage }),
+ relationship_coverage: execution.relationship_coverage,
});
return contentDigest({
...metadata,
@@ -872,9 +870,7 @@ const evidenceMatchesWorkflowEntry = (
const evidenceContentMetadata = (evidence: Evidence) => {
return {
limitations: evidence.limitations,
- ...(evidence.relationship_coverage === null
- ? {}
- : { relationship_coverage: evidence.relationship_coverage }),
+ relationship_coverage: evidence.relationship_coverage,
locations: evidence.locations,
subject:
evidence.subject === null
diff --git a/src/domain/evidence.test.ts b/src/domain/evidence.test.ts
index 4b33beb5c..82ef1dc7b 100644
--- a/src/domain/evidence.test.ts
+++ b/src/domain/evidence.test.ts
@@ -79,6 +79,21 @@ it("preserves omitted relationship diagnostics and authenticates reported diagno
).toThrow("semantic identifier");
});
+it("requires explicit coverage in serialized Evidence while producers encode omission as unknown", () => {
+ const evidence = createEvidence(TARGET, PROVIDER, {
+ operation: "procedure_callees",
+ parameters: { procedure: "0x1000" },
+ result: ["0x2000"],
+ });
+
+ expect(evidence.relationship_coverage).toBeNull();
+ expect(parseEvidence(JSON.parse(JSON.stringify(evidence)))).toEqual(evidence);
+ const oldEnvelope = { ...evidence } as Record<string, unknown>;
+ delete oldEnvelope.relationship_coverage;
+ expect(evidenceSchema.safeParse(oldEnvelope).success).toBe(false);
+ expect(() => parseEvidence(oldEnvelope)).toThrow();
+});
+
it("projects known provider fields and preserves open JSON payloads", () => {
const liveProvider = { ...PROVIDER, transient_status: "starting" };
const evidence = createEvidence(TARGET, liveProvider, {
diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts
index 49cb2a229..6205c9c13 100644
--- a/src/domain/evidence.ts
+++ b/src/domain/evidence.ts
@@ -135,7 +135,7 @@ const evidenceBaseSchema = z
authority: evidenceAuthoritySchema,
environment: executionEnvironmentSchema.nullable(),
limitations: z.array(z.string()),
- relationship_coverage: relationshipCoverageSchema.nullable().default(null),
+ relationship_coverage: relationshipCoverageSchema.nullable(),
locations: z.array(evidenceLocationSchema),
evidence_links: z.array(prefixedDigestSchema("ev")),
})
@@ -264,9 +264,7 @@ const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({
authority: evidence.authority,
environment: evidence.environment,
limitations: evidence.limitations,
- ...(evidence.relationship_coverage === null
- ? {}
- : { relationship_coverage: evidence.relationship_coverage }),
+ relationship_coverage: evidence.relationship_coverage,
locations: evidence.locations,
evidence_links: evidence.evidence_links,
});
diff --git a/tests/conformance/providers/conformancePackage.test.ts b/tests/conformance/providers/conformancePackage.test.ts
index ebfbb8368..977fe2521 100644
--- a/tests/conformance/providers/conformancePackage.test.ts
+++ b/tests/conformance/providers/conformancePackage.test.ts
@@ -5,6 +5,7 @@ import {
parseConformancePackage,
type ConformancePackageInput,
} from "../../../src/domain/conformancePackage.js";
+import { createEvidence } from "../../../src/domain/evidence.js";
const validPackageInput: ConformancePackageInput = {
name: "test-fixture",
@@ -204,23 +205,19 @@ describe("conformance package construction", () => {
scenario_id,
bundle: null,
required_dimensions: [],
- envelopes: Array.from({ length: 101 }, (_, index) => ({
- evidence_id: `ev_${String(index).padStart(64, "0")}`,
- analysis_profile: null,
- subject: null,
- provider: { id: "fixture", name: "Fixture", version: null },
- predicate_type: "observation",
- operation: "inspect",
- parameters: {},
- raw_result: null,
- normalized_result: null,
- confidence: "observed" as const,
- authority: "shipped-artifact" as const,
- environment: null,
- limitations: [],
- locations: [],
- evidence_links: [],
- })),
+ envelopes: Array.from({ length: 101 }, (_, index) =>
+ createEvidence(
+ undefined,
+ { id: "fixture", name: "Fixture", version: null },
+ {
+ operation: "inspect",
+ parameters: { index },
+ result: { index },
+ confidence: "observed",
+ authority: "shipped-artifact",
+ },
+ ),
+ ),
})),
verifier_contracts: scenarioIds.map((scenario_id) => ({
scenario_id,