Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(cli): accept JSON files for web capture and screenshot comparisons (#1397)

Kaoru08226875d4a64c0d
diff --git a/docs/browser-observation.md b/docs/browser-observation.md
index 0cebe1649..35cc1e620 100644
--- a/docs/browser-observation.md
+++ b/docs/browser-observation.md
@@ -18,6 +18,7 @@ REA can attach to a user-owned Chrome-family browser through the Chrome DevTools
- `compare_web_captures` / `rea compare-web-captures` compares passive captures or exact-step-aligned browser scenarios. Scenario results commit reproducible literal normalization and expose alignment failures plus artifact-level action, screenshot, DOM, accessibility, URL, history, storage, and event differences. Missing or truncated evidence is never treated as equivalence.
- `capture_web_screenshot` / `rea capture-web-screenshot` returns a content-addressed visible-viewport PNG.
- `compare_web_screenshots` / `rea compare-web-screenshots` performs bounded local PNG pixel comparison without OCR or external services.
+- The CLI comparison commands accept each capture, screenshot artifact or normalization policy as inline JSON or a JSON file path. Use files for screenshot artifacts and scenario captures, whose embedded PNG bytes can exceed the host's command-line length limit.
- Every surface has equivalent CLI and MCP contracts and returns Evidence provenance.
- MCP tools return complete results inline and include the Evidence ID. Session evidence can be exported with `export_evidence_bundle`.
diff --git a/src/cli/browserAdvancedCommands.ts b/src/cli/browserAdvancedCommands.ts
index 63cbbd756..1eff33f4b 100644
--- a/src/cli/browserAdvancedCommands.ts
+++ b/src/cli/browserAdvancedCommands.ts
@@ -15,13 +15,14 @@ import {
type AnalysisInputIssue,
} from "../domain/analysisErrorCore.js";
import { browserCaptureComparisonInputSchema } from "../domain/browserCaptureComparison.js";
+import { projectInputIssues } from "../domain/inputIssueProjection.js";
import { discoverWebMcpToolsInputSchema } from "../domain/webMcpDiscovery.js";
import {
captureWebScreenshotInputSchema,
compareWebScreenshotsInputSchema,
} from "../domain/webScreenshot.js";
import type { JsonValue } from "../domain/jsonValue.js";
-import { safeParseJson } from "../domain/safeJson.js";
+import { parseCliJsonInput } from "../cliJsonInput.js";
import type { Logger } from "pino";
import { CLI_COMMANDS } from "../cliCommandNames.js";
@@ -79,46 +80,39 @@ const registerCaptureDiff = (
description:
"Compare two normalized passive captures or recorded browser scenarios",
args: z.object({
- beforeJson: z.string().describe("Earlier normalized web capture JSON"),
- afterJson: z.string().describe("Later normalized web capture JSON"),
+ beforeJson: z
+ .string()
+ .describe("Earlier normalized web capture JSON or JSON file path"),
+ afterJson: z
+ .string()
+ .describe("Later normalized web capture JSON or JSON file path"),
}),
options: z.object({
normalizationJson: z
.string()
.default('{"rules":[]}')
.describe(
- "Recorded literal normalization policy for browser scenarios",
+ "Recorded literal normalization policy JSON or JSON file path for browser scenarios",
),
}),
run: ({ args, options }) =>
logCliCommand(logger, "compare-web-captures", async () => {
- const before = safeParseJson(args.beforeJson);
- const after = safeParseJson(args.afterJson);
- const normalization = safeParseJson(options.normalizationJson);
- if (!before.ok)
- return inputError("compare_web_captures", [
- invalidJsonIssue("before"),
- ]);
- if (!after.ok)
- return inputError("compare_web_captures", [
- invalidJsonIssue("after"),
- ]);
- if (!normalization.ok)
- return inputError("compare_web_captures", [
- invalidJsonIssue("normalization"),
- ]);
+ const inputs = await parseJsonArguments("compare_web_captures", {
+ before: args.beforeJson,
+ after: args.afterJson,
+ normalization: options.normalizationJson,
+ });
+ if (!inputs.ok) return inputs.error;
+ const { before, after, normalization } = inputs.value;
const scenarioComparison =
browserCaptureComparisonInputSchema.safeParse({
- before_scenario: before.value,
- after_scenario: after.value,
- normalization: normalization.value,
+ before_scenario: before,
+ after_scenario: after,
+ normalization,
});
const parsed = scenarioComparison.success
? scenarioComparison
- : browserCaptureComparisonInputSchema.safeParse({
- before: before.value,
- after: after.value,
- });
+ : browserCaptureComparisonInputSchema.safeParse({ before, after });
if (!parsed.success) return inputError("compare_web_captures");
const result = await compareWebCaptureEvidence(
new CdpBrowserProvider(),
@@ -168,8 +162,12 @@ const registerScreenshotDiff = (
cli.command(CLI_COMMANDS.compareWebScreenshots, {
description: "Compare two self-verifying PNG artifact JSON values",
args: z.object({
- beforeJson: z.string().describe("Earlier screenshot artifact JSON"),
- afterJson: z.string().describe("Later screenshot artifact JSON"),
+ beforeJson: z
+ .string()
+ .describe("Earlier screenshot artifact JSON or JSON file path"),
+ afterJson: z
+ .string()
+ .describe("Later screenshot artifact JSON or JSON file path"),
}),
options: z.object({
channelThreshold: z
@@ -182,12 +180,27 @@ const registerScreenshotDiff = (
}),
run: ({ args, options }) =>
logCliCommand(logger, "compare-web-screenshots", async () => {
- const parsed = compareWebScreenshotsInputSchema.safeParse({
- before: parseJson(args.beforeJson),
- after: parseJson(args.afterJson),
- channel_threshold: options.channelThreshold,
+ const operation = "compare_web_screenshots";
+ const inputs = await parseJsonArguments(operation, {
+ before: args.beforeJson,
+ after: args.afterJson,
});
- if (!parsed.success) return inputError("compare_web_screenshots");
+ if (!inputs.ok) return inputs.error;
+ const { before, after } = inputs.value;
+ const input = {
+ before,
+ after,
+ channel_threshold: options.channelThreshold,
+ };
+ const parsed = compareWebScreenshotsInputSchema.safeParse(input);
+ if (!parsed.success)
+ return browserCliError(
+ new AnalysisInputError(
+ operation,
+ { cause: parsed.error },
+ projectInputIssues(parsed.error.issues, input),
+ ),
+ );
const result = await compareWebScreenshotEvidence(
new CdpBrowserProvider(),
parsed.data,
@@ -197,11 +210,40 @@ const registerScreenshotDiff = (
});
};
-const parseJson = (value: string): unknown => {
- const parsed = safeParseJson(value);
- return parsed.ok ? parsed.value : undefined;
+/**
+ * Parse each named inline JSON argument or local JSON file, like other
+ * JSON-input commands. Screenshot artifacts and scenario captures embed PNG
+ * bytes that can exceed the host's command-line length limit.
+ */
+const parseJsonArguments = async (
+ operation: string,
+ fields: Readonly<Record<string, string>>,
+): Promise<
+ | { readonly ok: true; readonly value: Readonly<Record<string, unknown>> }
+ | { readonly ok: false; readonly error: JsonValue }
+> => {
+ const values: Record<string, unknown> = {};
+ for (const [field, value] of Object.entries(fields)) {
+ const parsed = await parseCliJsonInput(value, operation);
+ if (!parsed.ok)
+ return isFileInputFailure(parsed.error)
+ ? parsed
+ : {
+ ok: false,
+ error: inputError(operation, [invalidJsonIssue(field)]),
+ };
+ values[field] = parsed.value;
+ }
+ return { ok: true, value: values };
};
+/** File read and decode failures name the selected path; inline JSON does not. */
+const isFileInputFailure = (error: JsonValue): boolean =>
+ typeof error === "object" &&
+ error !== null &&
+ !Array.isArray(error) &&
+ "input_path" in error;
+
const invalidJsonIssue = (field: string): AnalysisInputIssue => ({
path: [field],
reason: "invalid_format",
diff --git a/tests/boundary/cli/browserCli.test.ts b/tests/boundary/cli/browserCli.test.ts
index 72c9e851d..bfd167a7e 100644
--- a/tests/boundary/cli/browserCli.test.ts
+++ b/tests/boundary/cli/browserCli.test.ts
@@ -1,15 +1,12 @@
-import { execFile } from "node:child_process";
-import { promisify } from "node:util";
-
import { afterEach, describe, expect, it } from "vitest";
+import { runCliJson as runCli } from "../../fixtures/cliJsonProcess.js";
import {
startFakeCdpBrowser,
type FakeCdpBrowser,
type FakeOptions,
} from "../../fixtures/fakeCdpBrowser.js";
-const execute = promisify(execFile);
const browsers: FakeCdpBrowser[] = [];
const INTEGRATION_TEST_TIMEOUT_MS = 60_000;
@@ -288,33 +285,6 @@ describe("browser CLI capture parity", () => {
);
});
-const runCli = async (
- arguments_: readonly string[],
- environment: NodeJS.ProcessEnv,
-): Promise<unknown> => {
- try {
- const { stdout } = await execute(
- process.execPath,
- ["scripts/rea.mjs", ...arguments_],
- {
- cwd: process.cwd(),
- env: environment,
- maxBuffer: 16 * 1_024 * 1_024,
- },
- );
- return JSON.parse(stdout);
- } catch (cause: unknown) {
- if (
- typeof cause === "object" &&
- cause !== null &&
- "stdout" in cause &&
- typeof cause.stdout === "string"
- )
- return JSON.parse(cause.stdout);
- throw cause;
- }
-};
-
const screenshotArtifact = (value: unknown): Record<string, unknown> => {
if (
!isRecord(value) ||
diff --git a/tests/boundary/cli/browserCompareJsonFiles.test.ts b/tests/boundary/cli/browserCompareJsonFiles.test.ts
new file mode 100644
index 000000000..0f067175f
--- /dev/null
+++ b/tests/boundary/cli/browserCompareJsonFiles.test.ts
@@ -0,0 +1,125 @@
+import { randomBytes } from "node:crypto";
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { crc32, deflateSync } from "node:zlib";
+
+import { describe, expect, it } from "vitest";
+
+import { createWebScreenshotArtifact } from "../../../src/domain/webScreenshot.js";
+import { runCliJson as runCli } from "../../fixtures/cliJsonProcess.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+const INTEGRATION_TEST_TIMEOUT_MS = 60_000;
+// Linux limits one argument to 128 KiB and macOS all arguments to 1 MiB.
+const COMMAND_LINE_LIMIT_BYTES = 1024 * 1024;
+
+const pngChunk = (type: string, data: Buffer): Buffer => {
+ const typed = Buffer.concat([Buffer.from(type, "latin1"), data]);
+ const length = Buffer.alloc(4);
+ length.writeUInt32BE(data.length);
+ const checksum = Buffer.alloc(4);
+ checksum.writeUInt32BE(crc32(typed));
+ return Buffer.concat([length, typed, checksum]);
+};
+
+/** An incompressible RGB PNG whose artifact JSON exceeds command-line limits. */
+const noisePng = (width: number, height: number): Buffer => {
+ const header = Buffer.alloc(13);
+ header.writeUInt32BE(width, 0);
+ header.writeUInt32BE(height, 4);
+ header.set([8, 2, 0, 0, 0], 8);
+ const rows = Array.from({ length: height }, () =>
+ Buffer.concat([Buffer.of(0), randomBytes(width * 3)]),
+ );
+ return Buffer.concat([
+ Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+ pngChunk("IHDR", header),
+ pngChunk("IDAT", deflateSync(Buffer.concat(rows))),
+ pngChunk("IEND", Buffer.alloc(0)),
+ ]);
+};
+
+describe("browser comparison JSON file inputs", () => {
+ it(
+ "compares screenshot artifacts too large for a command-line argument",
+ async () => {
+ const root = await createTestTempDirectory("rea-screenshot-files-");
+ const artifact = JSON.stringify(
+ createWebScreenshotArtifact(noisePng(640, 640)),
+ );
+ expect(Buffer.byteLength(artifact)).toBeGreaterThan(
+ COMMAND_LINE_LIMIT_BYTES,
+ );
+ const before = join(root, "before.json");
+ const after = join(root, "after.json");
+ await writeFile(before, artifact);
+ await writeFile(after, artifact);
+
+ expect(
+ await runCli(["compare-web-screenshots", before, after, "--json"]),
+ ).toMatchObject({
+ operation: "compare_web_screenshots",
+ normalized_result: { status: "identical", changed_pixels: 0 },
+ });
+ },
+ INTEGRATION_TEST_TIMEOUT_MS,
+ );
+
+ it(
+ "reports which screenshot artifact failed validation",
+ async () => {
+ const root = await createTestTempDirectory("rea-screenshot-invalid-");
+ const valid = createWebScreenshotArtifact(noisePng(2, 2));
+ const before = join(root, "before.json");
+ await writeFile(before, JSON.stringify({ ...valid, bytes: 1 }));
+
+ expect(
+ await runCli([
+ "compare-web-screenshots",
+ before,
+ JSON.stringify(valid),
+ "--json",
+ ]),
+ ).toMatchObject({
+ error: "Browser observation failed",
+ code: "invalid_request",
+ details: {
+ operation: "compare_web_screenshots",
+ issues: [
+ {
+ path: ["before"],
+ message: "Screenshot artifact digest or size mismatch",
+ },
+ ],
+ },
+ });
+ },
+ INTEGRATION_TEST_TIMEOUT_MS,
+ );
+
+ it(
+ "identifies a missing capture file instead of reporting malformed JSON",
+ async () => {
+ const root = await createTestTempDirectory("rea-capture-missing-");
+ const missing = join(root, "before.json");
+
+ expect(
+ await runCli(["compare-web-captures", missing, "{}", "--json"]),
+ ).toMatchObject({
+ code: "invalid_request",
+ details: {
+ operation: "compare_web_captures",
+ issues: [
+ {
+ reason: "invalid_value",
+ message: expect.stringContaining(`(ENOENT): ${missing}`),
+ },
+ ],
+ },
+ input_path: missing,
+ input_reason: "read-failed",
+ });
+ },
+ INTEGRATION_TEST_TIMEOUT_MS,
+ );
+});
diff --git a/tests/boundary/cli/browserRedirectShapeCompleteness.test.ts b/tests/boundary/cli/browserRedirectShapeCompleteness.test.ts
index 6608ce08d..6340c5c34 100644
--- a/tests/boundary/cli/browserRedirectShapeCompleteness.test.ts
+++ b/tests/boundary/cli/browserRedirectShapeCompleteness.test.ts
@@ -1,14 +1,11 @@
-import { execFile } from "node:child_process";
-import { promisify } from "node:util";
-
import { afterEach, expect, it } from "vitest";
+import { runCliJson as runCli } from "../../fixtures/cliJsonProcess.js";
import {
startFakeCdpBrowser,
type FakeCdpBrowser,
} from "../../fixtures/fakeCdpBrowser.js";
-const execute = promisify(execFile);
const browsers: FakeCdpBrowser[] = [];
afterEach(async () => {
@@ -79,30 +76,6 @@ it("reports missing JSON shapes only when body shapes are selected", async () =>
});
});
-const runCli = async (arguments_: readonly string[]): Promise<unknown> => {
- try {
- const { stdout } = await execute(
- process.execPath,
- ["scripts/rea.mjs", ...arguments_],
- {
- cwd: process.cwd(),
- env: process.env,
- maxBuffer: 16 * 1_024 * 1_024,
- },
- );
- return JSON.parse(stdout);
- } catch (cause: unknown) {
- if (
- typeof cause === "object" &&
- cause !== null &&
- "stdout" in cause &&
- typeof cause.stdout === "string"
- )
- return JSON.parse(cause.stdout);
- throw cause;
- }
-};
-
const normalizedResult = (value: unknown): Record<string, unknown> => {
if (!isRecord(value) || !isRecord(value.normalized_result))
throw new TypeError("Missing CLI normalized result");
diff --git a/tests/fixtures/cliJsonProcess.ts b/tests/fixtures/cliJsonProcess.ts
new file mode 100644
index 000000000..c66c82d4e
--- /dev/null
+++ b/tests/fixtures/cliJsonProcess.ts
@@ -0,0 +1,32 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+const execute = promisify(execFile);
+
+/** Run the built CLI and parse its JSON stdout, including a failing command's. */
+export const runCliJson = async (
+ arguments_: readonly string[],
+ environment: NodeJS.ProcessEnv = process.env,
+): Promise<unknown> => {
+ try {
+ const { stdout } = await execute(
+ process.execPath,
+ ["scripts/rea.mjs", ...arguments_],
+ {
+ cwd: process.cwd(),
+ env: environment,
+ maxBuffer: 16 * 1_024 * 1_024,
+ },
+ );
+ return JSON.parse(stdout);
+ } catch (cause: unknown) {
+ if (
+ typeof cause === "object" &&
+ cause !== null &&
+ "stdout" in cause &&
+ typeof cause.stdout === "string"
+ )
+ return JSON.parse(cause.stdout);
+ throw cause;
+ }
+};