Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(environment): report capture payload failures at their field paths (#6247)

Kaoru0822c537c52c96b7
diff --git a/src/application/environment/environmentSource.test.ts b/src/application/environment/environmentSource.test.ts
index 69858623b..c1312eb87 100644
--- a/src/application/environment/environmentSource.test.ts
+++ b/src/application/environment/environmentSource.test.ts
@@ -5,20 +5,51 @@ import {
analysisViewCaptureFixture,
environmentManifestCaptureFixture,
} from "../../../tests/fixtures/analysisView.js";
+import { scenarioCaptureEvidence } from "../../../tests/fixtures/browserScenarioCapture.js";
import {
environmentServeFixture,
environmentServeSource,
} from "../../../tests/fixtures/environmentServe.js";
+import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
+import { createEvidence, type Evidence } from "../../domain/evidence.js";
import { inspectAnalysisView } from "../analysisView/AnalysisViewService.js";
import { loadEnvironmentSource } from "./environmentSource.js";
-const refusal = (manifest: unknown, evidence: unknown) => {
- const loaded = loadEnvironmentSource(manifest, evidence);
+const refusal = (manifest: unknown, evidence: unknown, operation?: string) => {
+ const loaded = loadEnvironmentSource(manifest, evidence, operation);
if (loaded.ok) throw new Error("Expected the pair to be refused");
return projectAnalysisError(loaded.error);
};
+/** Authenticated Evidence whose payload is rewritten; createEvidence recomputes the digest. */
+const resealed = (
+ evidence: Evidence,
+ edit: (result: Record<string, unknown>) => void,
+ operation: string = evidence.operation,
+): Evidence => {
+ const result = structuredClone(evidence.normalized_result) as Record<
+ string,
+ unknown
+ >;
+ edit(result);
+ return createEvidence(undefined, evidence.provider, {
+ operation,
+ parameters: structuredClone(evidence.parameters),
+ result: result as never,
+ confidence: evidence.confidence,
+ });
+};
+
+/** Re-cite a manifest's parent so pairing passes and the capture parse alone decides. */
+const citing = (manifest: Evidence, capture: Evidence): Evidence =>
+ resealed(manifest, (result) => {
+ result.parent_evidence_id = capture.evidence_id;
+ });
+
+const record = (value: unknown, key: string): Record<string, unknown> =>
+ (value as Record<string, Record<string, unknown>>)[key] ?? {};
+
describe("loadEnvironmentSource", () => {
it("authenticates the pair and indexes every cited record as a replayable exchange", () => {
const fixture = environmentServeFixture();
@@ -165,6 +196,111 @@ describe("loadEnvironmentSource", () => {
});
});
+describe("loadEnvironmentSource payload attribution", () => {
+ it("reports capture payload failures beneath normalized_result instead of the supported operation", () => {
+ const fixture = environmentServeFixture();
+ for (const operation of [
+ "inspect_web_network_capture",
+ "capture_browser_scenario",
+ ]) {
+ const capture = createEvidence(undefined, fixture.capture.provider, {
+ operation,
+ parameters: {},
+ result: {},
+ confidence: "observed",
+ });
+ for (const command of ["environment-serve", "environment-export"]) {
+ const loaded = loadEnvironmentSource(
+ citing(fixture.manifest, capture),
+ capture,
+ command,
+ );
+ if (loaded.ok || !(loaded.error instanceof AnalysisInputError))
+ throw new Error("Expected an input refusal");
+ expect(loaded.error.issues.length).toBeGreaterThan(0);
+ for (const issue of loaded.error.issues)
+ expect(issue.path.slice(0, 2)).toEqual([
+ "evidence",
+ "normalized_result",
+ ]);
+ expect(loaded.error.cause).toBeInstanceOf(Error);
+ expect(projectAnalysisError(loaded.error)).toMatchObject({
+ code: "invalid_request",
+ category: "invalid_input",
+ details: { operation: command },
+ });
+ }
+ }
+ });
+
+ it("reports the full path of a nested capture payload field", () => {
+ const fixture = environmentServeFixture();
+ const har = resealed(fixture.capture, (result) => {
+ record(result, "artifact").sha256 = 42;
+ });
+ expect(refusal(citing(fixture.manifest, har), har).details?.issues).toEqual(
+ [
+ expect.objectContaining({
+ path: ["evidence", "normalized_result", "artifact", "sha256"],
+ reason: "invalid_type",
+ }),
+ ],
+ );
+ const scenario = resealed(scenarioCaptureEvidence(), (result) => {
+ record(result, "browser").product = 7;
+ });
+ expect(
+ refusal(citing(fixture.manifest, scenario), scenario).details?.issues,
+ ).toEqual([
+ expect.objectContaining({
+ path: ["evidence", "normalized_result", "browser", "product"],
+ reason: "invalid_type",
+ }),
+ ]);
+ });
+
+ it("still blames the operation of Evidence no capture schema owns", () => {
+ const fixture = environmentServeFixture();
+ const other = resealed(fixture.capture, () => {}, "inspect_other_capture");
+ expect(
+ refusal(citing(fixture.manifest, other), other).details?.issues,
+ ).toEqual([
+ expect.objectContaining({
+ path: ["evidence", "operation"],
+ expected: "inspect_web_network_capture or capture_browser_scenario",
+ }),
+ ]);
+ });
+
+ it("separates the manifest operation, view kind and view payload failures", () => {
+ const fixture = environmentServeFixture();
+ expect(refusal(fixture.capture, fixture.capture).details?.issues).toEqual([
+ expect.objectContaining({
+ path: ["manifest", "operation"],
+ expected: "inspect_analysis_view",
+ }),
+ ]);
+ const malformed = resealed(fixture.manifest, (result) => {
+ result.name = 7;
+ });
+ expect(refusal(malformed, fixture.capture).details?.issues).toEqual([
+ expect.objectContaining({
+ path: ["manifest", "normalized_result", "name"],
+ reason: "invalid_type",
+ }),
+ ]);
+ const unknownKind = resealed(fixture.manifest, (result) => {
+ result.kind = "not-a-view";
+ });
+ expect(refusal(unknownKind, fixture.capture).details?.issues).toEqual([
+ expect.objectContaining({
+ path: ["manifest", "normalized_result", "kind"],
+ expected: "environment-manifest",
+ }),
+ ]);
+ });
+});
+
describe("loadEnvironmentSource recorded request lines", () => {
it("keeps the delimiter of an explicit empty query in the recorded exchange", () => {
const capture = environmentManifestCaptureFixture();
diff --git a/src/application/environment/environmentSource.ts b/src/application/environment/environmentSource.ts
index f7ec97321..ff6d7b264 100644
--- a/src/application/environment/environmentSource.ts
+++ b/src/application/environment/environmentSource.ts
@@ -143,18 +143,34 @@ export const parseEvidenceFile = (
} catch (cause: unknown) {
if (!(cause instanceof z.ZodError)) throw cause;
return err(
- inputError(
+ schemaInputError(
operation,
- projectInputIssues(cause.issues, value).map((issue) => ({
- ...issue,
- path: [role, ...issue.path],
- })),
+ { prefix: [role], cause, value },
`The ${role} file is not authenticated REA Evidence JSON.`,
),
);
}
};
+/** A schema refusal of `value`, found at `prefix`; issue paths extend the prefix and the schema failure is retained. */
+const schemaInputError = (
+ operation: string,
+ failure: {
+ readonly prefix: readonly (string | number)[];
+ readonly cause: z.ZodError;
+ readonly value: unknown;
+ },
+ userMessage: string,
+): AnalysisInputError =>
+ new AnalysisInputError(
+ operation,
+ { userMessage, cause: failure.cause },
+ projectInputIssues(failure.cause.issues, failure.value).map((issue) => ({
+ ...issue,
+ path: [...failure.prefix, ...issue.path],
+ })),
+ );
+
const recordedExchange = (
record: WebNetworkCaptureRecord,
): RecordedExchange | undefined => {
@@ -185,14 +201,27 @@ export const parseViewEvidence = <Kind extends AnalysisViewResult["kind"]>(
> => {
const evidence = parseEvidenceFile(operation, role, value);
if (!evidence.ok) return evidence;
- const view = analysisViewResultSchema.safeParse(
- evidence.value.normalized_result,
+ const message = `The ${role} file is not ${kind} view Evidence.`;
+ if (evidence.value.operation !== "inspect_analysis_view")
+ return err(
+ inputError(
+ operation,
+ [
+ {
+ path: [role, "operation"],
+ reason: "invalid_value",
+ expected: "inspect_analysis_view",
+ message: `The ${role} file must be inspect_analysis_view Evidence of kind ${kind}.`,
+ },
+ ],
+ message,
+ ),
+ );
+ const normalized = immutableJsonSnapshotMember(
+ evidence.value,
+ "normalized_result",
);
- if (
- evidence.value.operation !== "inspect_analysis_view" ||
- !view.success ||
- view.data.kind !== kind
- )
+ if (immutableJsonSnapshotMember(normalized, "kind") !== kind)
return err(
inputError(
operation,
@@ -204,7 +233,21 @@ export const parseViewEvidence = <Kind extends AnalysisViewResult["kind"]>(
message: `The ${role} file must be inspect_analysis_view Evidence of kind ${kind}.`,
},
],
- `The ${role} file is not ${kind} view Evidence.`,
+ message,
+ ),
+ );
+ // The discriminator selected the requested view, so any failure is in that view's payload.
+ const view = analysisViewResultSchema.safeParse(normalized);
+ if (!view.success)
+ return err(
+ schemaInputError(
+ operation,
+ {
+ prefix: [role, "normalized_result"],
+ cause: view.error,
+ value: normalized,
+ },
+ message,
),
);
return ok({
@@ -213,89 +256,132 @@ export const parseViewEvidence = <Kind extends AnalysisViewResult["kind"]>(
});
};
-/** Authenticate the evidence file and require complete capture Evidence of either kind. */
+/** What replay needs from a parent capture once its payload has been validated. */
+interface CaptureFacts {
+ readonly records: readonly WebNetworkCaptureRecord[];
+ readonly captureKind: EnvironmentCaptureKind;
+ readonly captureSha256: string | null;
+}
+
+/** Issue paths of a capture payload refusal start here. */
+const CAPTURE_PAYLOAD_PATH = ["evidence", "normalized_result"] as const;
+
+/** Validate a capture_browser_scenario payload; its failures name the payload fields. */
+const parseScenarioCapture = (
+ operation: string,
+ normalized: unknown,
+ parameters: unknown,
+): Result<CaptureFacts, AnalysisError> => {
+ const scenario = browserScenarioCaptureSchema.safeParse(normalized);
+ if (!scenario.success)
+ return err(
+ schemaInputError(
+ operation,
+ {
+ prefix: CAPTURE_PAYLOAD_PATH,
+ cause: scenario.error,
+ value: normalized,
+ },
+ "The evidence file is not a complete browser scenario capture.",
+ ),
+ );
+ const definition = z
+ .object({ scenario_sha256: z.string() })
+ .safeParse(parameters);
+ return ok({
+ records: scenarioCaptureRecords(scenario.data, "replay").records,
+ captureKind: "browser-scenario",
+ captureSha256: definition.success ? definition.data.scenario_sha256 : null,
+ });
+};
+
+/** Validate an inspect_web_network_capture payload as HAR; its failures name the payload fields. */
+const parseHarCapture = (
+ operation: string,
+ normalized: unknown,
+): Result<CaptureFacts, AnalysisError> => {
+ const records = immutableJsonSnapshotMember(normalized, "records");
+ if (Array.isArray(records))
+ for (let index = 0; index < records.length; index += 1) {
+ const record = immutableJsonSnapshotMember(records, String(index));
+ immutableJsonSnapshotMember(record, "reported");
+ }
+ // An owned snapshot is validated once and borrowed, so deriving the state reuses this validation.
+ const capture = safeParseOwnedWebNetworkCapture(normalized);
+ if (!capture.success)
+ return err(
+ schemaInputError(
+ operation,
+ {
+ prefix: CAPTURE_PAYLOAD_PATH,
+ cause: capture.error,
+ value: normalized,
+ },
+ "The evidence file is not a complete web network capture.",
+ ),
+ );
+ // The manifest view projects HAR captures only, and replay reads HAR's response fields.
+ if (capture.data.format !== "har")
+ return err(
+ inputError(
+ operation,
+ [
+ {
+ path: [...CAPTURE_PAYLOAD_PATH, "format"],
+ reason: "invalid_value",
+ expected: "har",
+ message: `Agent environments replay HAR captures; this capture's format is ${capture.data.format}.`,
+ },
+ ],
+ "The evidence file is not a HAR capture.",
+ ),
+ );
+ return ok({
+ records: capture.data.records,
+ captureKind: "web-network-capture",
+ captureSha256: capture.data.artifact.sha256,
+ });
+};
+
+/**
+ * Authenticate the evidence file and require complete capture Evidence of
+ * either kind. Only an operation no capture schema owns blames the operation;
+ * a recognized operation's payload failures name the payload fields.
+ */
const parseCaptureEvidence = (
operation: string,
value: unknown,
-): Result<
- {
- readonly records: readonly WebNetworkCaptureRecord[];
- readonly evidenceId: string;
- readonly captureKind: EnvironmentCaptureKind;
- readonly captureSha256: string | null;
- },
- AnalysisError
-> => {
+): Result<CaptureFacts & { readonly evidenceId: string }, AnalysisError> => {
const evidence = parseEvidenceFile(operation, "evidence", value);
if (!evidence.ok) return evidence;
- const { evidence_id: evidenceId } = evidence.value;
const normalized = immutableJsonSnapshotMember(
evidence.value,
"normalized_result",
);
- if (evidence.value.operation === "capture_browser_scenario") {
- const scenario = browserScenarioCaptureSchema.safeParse(normalized);
- const definition = z
- .object({ scenario_sha256: z.string() })
- .safeParse(evidence.value.parameters);
- if (scenario.success)
- return ok({
- records: scenarioCaptureRecords(scenario.data, "replay").records,
- evidenceId,
- captureKind: "browser-scenario",
- captureSha256: definition.success
- ? definition.data.scenario_sha256
- : null,
- });
- } else if (evidence.value.operation === "inspect_web_network_capture") {
- const records = immutableJsonSnapshotMember(normalized, "records");
- if (Array.isArray(records))
- for (let index = 0; index < records.length; index += 1) {
- const record = immutableJsonSnapshotMember(records, String(index));
- immutableJsonSnapshotMember(record, "reported");
- }
- // An owned snapshot is validated once and borrowed, so deriving the state reuses this validation.
- const capture = safeParseOwnedWebNetworkCapture(normalized);
- if (capture.success) {
- // The manifest view projects HAR captures only, and replay reads HAR's response fields.
- if (capture.data.format !== "har")
- return err(
- inputError(
- operation,
- [
- {
- path: ["evidence", "normalized_result", "format"],
- reason: "invalid_value",
- expected: "har",
- message: `Agent environments replay HAR captures; this capture's format is ${capture.data.format}.`,
- },
- ],
- "The evidence file is not a HAR capture.",
- ),
- );
- return ok({
- records: capture.data.records,
- evidenceId,
- captureKind: "web-network-capture",
- captureSha256: capture.data.artifact.sha256,
- });
- }
- }
- return err(
- inputError(
- operation,
- [
- {
- path: ["evidence", "operation"],
- reason: "invalid_value",
- expected: "inspect_web_network_capture or capture_browser_scenario",
- message:
- "The evidence file must be complete inspect_web_network_capture or capture_browser_scenario Evidence.",
- },
- ],
- "The evidence file is not web network capture or browser scenario Evidence.",
- ),
- );
+ const capture =
+ evidence.value.operation === "capture_browser_scenario"
+ ? parseScenarioCapture(operation, normalized, evidence.value.parameters)
+ : evidence.value.operation === "inspect_web_network_capture"
+ ? parseHarCapture(operation, normalized)
+ : err(
+ inputError(
+ operation,
+ [
+ {
+ path: ["evidence", "operation"],
+ reason: "invalid_value",
+ expected:
+ "inspect_web_network_capture or capture_browser_scenario",
+ message:
+ "The evidence file must be inspect_web_network_capture or capture_browser_scenario Evidence.",
+ },
+ ],
+ "The evidence file is not web network capture or browser scenario Evidence.",
+ ),
+ );
+ return capture.ok
+ ? ok({ ...capture.value, evidenceId: evidence.value.evidence_id })
+ : capture;
};
/**