Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

test(web): cover nullable HTTP captures through CDP and CLI (#1834)

Yann Roberto5d12a0b4b613
diff --git a/tests/boundary/browser/cdpHttpComparison.test.ts b/tests/boundary/browser/cdpHttpComparison.test.ts
new file mode 100644
index 000000000..2f4ad689c
--- /dev/null
+++ b/tests/boundary/browser/cdpHttpComparison.test.ts
@@ -0,0 +1,255 @@
+import { describe, expect, it } from "vitest";
+
+import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js";
+import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";
+import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js";
+import { compareWebCaptures } from "../../../src/domain/webCaptureDiff.js";
+import { compareWebCapturesInputSchema } from "../../../src/domain/webCaptureDiffSchemas.js";
+import type { FakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";
+import {
+ startHttpComparisonBrowser,
+ type HttpRequestObservation,
+} from "../../fixtures/httpComparisonBrowser.js";
+import { trackBrowser } from "./cdpBrowserProvider.support.js";
+
+const inspect = async (browser: FakeCdpBrowser): Promise<WebPageInspection> => {
+ const result = await new CdpBrowserProvider().inspectPage(
+ inspectWebPageInputSchema.parse({
+ cdp_endpoint: browser.endpoint,
+ allowed_origins: [browser.allowedOrigin],
+ target_id: "allowed-page",
+ observation_ms: 0,
+ }),
+ );
+ if (!result.ok) throw result.error;
+ return result.value;
+};
+
+const capturePair = async (
+ before: readonly HttpRequestObservation[],
+ after: readonly HttpRequestObservation[],
+): Promise<readonly [WebPageInspection, WebPageInspection]> => {
+ const fixture = await startHttpComparisonBrowser(before);
+ trackBrowser(fixture.browser);
+ const left = await inspect(fixture.browser);
+ fixture.setRequests(after);
+ return [left, await inspect(fixture.browser)];
+};
+
+const compareNetwork = (before: WebPageInspection, after: WebPageInspection) =>
+ compareWebCaptures(
+ compareWebCapturesInputSchema.parse({
+ before: { inspection: before },
+ after: { inspection: after },
+ }),
+ ).dimensions.network;
+
+const response = {
+ requestId: "request-before",
+ status: 200,
+ mimeType: "application/json",
+} as const;
+
+describe("HTTP comparison response coverage", () => {
+ it("does not compare an unavailable byte count with a completed response", async () => {
+ const [before, after] = await capturePair(
+ [response],
+ [{ ...response, requestId: "request-after", encodedDataLength: 100 }],
+ );
+ expect(before.network.requests[0]?.encoded_data_length).toBeNull();
+ expect(after.network.requests[0]?.encoded_data_length).toBe(100);
+ for (const [left, right] of [
+ [before, after],
+ [after, before],
+ ] as const)
+ expect(compareNetwork(left, right)).toMatchObject({
+ status: "unknown",
+ total_changes: 0,
+ changes: [],
+ });
+ });
+
+ it("does not treat an unobserved response as a changed response", async () => {
+ const [before, after] = await capturePair(
+ [{ requestId: "request-before" }],
+ [{ ...response, encodedDataLength: 100 }],
+ );
+ expect(before.network.requests).toEqual([
+ expect.objectContaining({
+ status: null,
+ mime_type: null,
+ encoded_data_length: null,
+ }),
+ ]);
+ for (const [left, right] of [
+ [before, after],
+ [after, before],
+ ] as const)
+ expect(compareNetwork(left, right)).toMatchObject({
+ status: "unknown",
+ total_changes: 0,
+ });
+ });
+
+ it("retains observed status changes when response bytes are unavailable", async () => {
+ const [before, after] = await capturePair(
+ [response],
+ [{ ...response, status: 500 }],
+ );
+ expect(compareNetwork(before, after)).toMatchObject({
+ status: "changed",
+ changes: [expect.objectContaining({ change: "modified" })],
+ });
+ });
+
+ it.each([
+ { mimeType: "text/plain", encodedDataLength: 100 },
+ { mimeType: "application/json", encodedDataLength: 200 },
+ ])("retains observed response changes: %j", async (change) => {
+ const [before, after] = await capturePair(
+ [{ ...response, encodedDataLength: 100 }],
+ [{ ...response, ...change }],
+ );
+ expect(compareNetwork(before, after).status).toBe("changed");
+ });
+});
+
+describe("HTTP comparison repeated request coverage", () => {
+ it("does not assign missing evidence by request order or transient ID", async () => {
+ const [before, after] = await capturePair(
+ [
+ { ...response, encodedDataLength: 100 },
+ { ...response, requestId: "second-before" },
+ ],
+ [
+ { ...response, requestId: "second-after", encodedDataLength: 200 },
+ { ...response, requestId: "first-after", encodedDataLength: 100 },
+ ],
+ );
+ expect(before.network.requests).toHaveLength(2);
+ expect(after.network.requests).toHaveLength(2);
+ expect(compareNetwork(before, after)).toMatchObject({
+ status: "unknown",
+ total_changes: 0,
+ });
+ });
+
+ it("compares complete byte counts even when another field is incomplete", async () => {
+ const [before, after] = await capturePair(
+ [
+ { ...response, encodedDataLength: 100 },
+ { requestId: "second-before", encodedDataLength: 200 },
+ ],
+ [
+ { requestId: "second-after", encodedDataLength: 300 },
+ { ...response, requestId: "first-after", encodedDataLength: 100 },
+ ],
+ );
+ expect(compareNetwork(before, after).status).toBe("changed");
+ });
+
+ it("preserves the association between complete response fields", async () => {
+ const [before, after] = await capturePair(
+ [
+ { ...response, encodedDataLength: 100 },
+ {
+ ...response,
+ requestId: "second-before",
+ status: 201,
+ encodedDataLength: 200,
+ },
+ ],
+ [
+ { ...response, requestId: "first-after", encodedDataLength: 200 },
+ {
+ ...response,
+ requestId: "second-after",
+ status: 201,
+ encodedDataLength: 100,
+ },
+ ],
+ );
+ expect(compareNetwork(before, after).status).toBe("changed");
+ });
+});
+
+it.each([
+ {
+ name: "a pending request in an empty inventory",
+ before: [],
+ after: [{ requestId: "pending" }],
+ },
+ {
+ name: "a known request beside an existing pending request",
+ before: [{ requestId: "pending" }],
+ after: [
+ { requestId: "pending" },
+ {
+ ...response,
+ requestId: "new",
+ path: "/api/new",
+ encodedDataLength: 100,
+ },
+ ],
+ },
+])("preserves additions and removals of $name", async (observations) => {
+ const [before, after] = await capturePair(
+ observations.before,
+ observations.after,
+ );
+ // Live CDP observations have attach-limited inventories. Their absence
+ // remains unknown even when a request is seen in the other capture.
+ for (const [left, right] of [
+ [before, after],
+ [after, before],
+ ] as const)
+ expect(compareNetwork(left, right)).toMatchObject({
+ status: "unknown",
+ total_changes: 0,
+ changes: [],
+ });
+
+ // Project explicitly complete inventory coverage without changing the
+ // producer's request facts; response fields remain independently unknown.
+ markNetworkInventoriesComplete(before, after);
+ for (const [left, right, change] of [
+ [before, after, "added"],
+ [after, before, "removed"],
+ ] as const)
+ expect(compareNetwork(left, right)).toMatchObject({
+ status: "changed",
+ total_changes: 1,
+ changes: [expect.objectContaining({ change })],
+ });
+});
+
+it("requires response field coverage even when capture sections are complete", async () => {
+ const [before, after] = await capturePair(
+ [response],
+ [{ ...response, encodedDataLength: 100 }],
+ );
+ markNetworkInventoriesComplete(before, after);
+ expect(compareNetwork(before, before).status).toBe("unknown");
+ expect(compareNetwork(before, after).status).toBe("unknown");
+ expect(compareNetwork(after, after).status).toBe("unchanged");
+});
+
+const markNetworkInventoriesComplete = (
+ ...inspections: WebPageInspection[]
+): void => {
+ const sections = new Set([
+ "network_requests",
+ "websocket_connections",
+ "websocket_frames",
+ ]);
+ for (const inspection of inspections)
+ for (const key of [
+ "policy_filtered_sections",
+ "attach_limited_sections",
+ "truncated_sections",
+ "unavailable_sections",
+ ] as const)
+ inspection.completeness[key] = inspection.completeness[key].filter(
+ (section) => !sections.has(section),
+ );
+};
diff --git a/tests/boundary/browser/webCaptureDiff.test.ts b/tests/boundary/browser/webCaptureDiff.test.ts
index 5c2ce8785..3680f7dfa 100644
--- a/tests/boundary/browser/webCaptureDiff.test.ts
+++ b/tests/boundary/browser/webCaptureDiff.test.ts
@@ -446,91 +446,50 @@ describe("web capture WebSocket payload byte coverage", () => {
});
describe("web capture diff", () => {
- it("treats unavailable HTTP response facts as unknown while retaining observed changes", async () => {
- const browser = await startFakeCdpBrowser();
- browsers.push(browser);
- const captured = await new CdpBrowserProvider().inspectPage(
- inspectWebPageInputSchema.parse({
- cdp_endpoint: browser.endpoint,
- allowed_origins: [browser.allowedOrigin],
- target_id: "allowed-page",
- observation_ms: 0,
- }),
- );
- if (!captured.ok) throw captured.error;
- const before = structuredClone(captured.value);
- const request = before.network.requests[0];
- if (request === undefined) throw new Error("Missing network request");
- request.status = null;
- request.mime_type = null;
- request.encoded_data_length = null;
- markSectionsComplete(before, [
- "network_requests",
- "websocket_connections",
- "websocket_frames",
- "json_body_shapes",
- ]);
- for (const item of before.network.requests)
- item.body_shapes.status = "not_approved";
+ it.each(["status", "mime_type"] as const)(
+ "keeps an independently unavailable HTTP %s unknown",
+ async (field) => {
+ const browser = await startFakeCdpBrowser();
+ browsers.push(browser);
+ const captured = await new CdpBrowserProvider().inspectPage(
+ inspectWebPageInputSchema.parse({
+ cdp_endpoint: browser.endpoint,
+ allowed_origins: [browser.allowedOrigin],
+ target_id: "allowed-page",
+ observation_ms: 0,
+ }),
+ );
+ if (!captured.ok) throw captured.error;
+ const observed = structuredClone(captured.value);
+ const request = observed.network.requests[0];
+ if (request === undefined) throw new Error("Missing network request");
+ request.status = 200;
+ request.mime_type = "application/json";
+ request.encoded_data_length = 100;
+ markSectionsComplete(observed, [
+ "network_requests",
+ "websocket_connections",
+ "websocket_frames",
+ "json_body_shapes",
+ ]);
+ for (const item of observed.network.requests)
+ item.body_shapes.status = "not_approved";
+ expect(compareNetwork(observed, observed).status).toBe("unchanged");
- const after = structuredClone(before);
- const responseObserved = after.network.requests[0];
- if (responseObserved === undefined)
- throw new Error("Missing network request");
- responseObserved.status = 200;
- responseObserved.mime_type = "application/json";
- responseObserved.encoded_data_length = 100;
- expect(compareNetwork(before, after)).toMatchObject({
- status: "unknown",
- total_changes: 0,
- changes: [],
- });
- expect(compareNetwork(after, before)).toMatchObject({
- status: "unknown",
- total_changes: 0,
- });
- expect(compareNetwork(before, before).status).toBe("unknown");
-
- for (const field of [
- "status",
- "mime_type",
- "encoded_data_length",
- ] as const) {
- const partial = structuredClone(after);
+ const partial = structuredClone(observed);
partial.network.requests[0]![field] = null;
- expect(compareNetwork(partial, after).status).toBe("unknown");
- expect(compareNetwork(after, partial).status).toBe("unknown");
- }
-
- const inventoryBefore = structuredClone(before);
- inventoryBefore.network.requests = [];
- expect(compareNetwork(inventoryBefore, before)).toMatchObject({
- status: "changed",
- total_changes: 1,
- changes: [expect.objectContaining({ change: "added" })],
- });
-
- const statusChanged = structuredClone(after);
- statusChanged.network.requests[0]!.status = 500;
- expect(compareNetwork(after, statusChanged)).toMatchObject({
- status: "changed",
- changes: [expect.objectContaining({ change: "modified" })],
- });
- after.network.requests[0]!.encoded_data_length = null;
- expect(compareNetwork(after, statusChanged).status).toBe("changed");
-
- const repeated = structuredClone(statusChanged);
- const duplicate = structuredClone(repeated.network.requests[0]!);
- duplicate.status = 200;
- duplicate.mime_type = "text/plain";
- repeated.network.requests.push(duplicate);
- const reassociated = structuredClone(repeated);
- reassociated.network.requests[0]!.mime_type = "text/plain";
- reassociated.network.requests.at(-1)!.mime_type = "application/json";
- expect(compareNetwork(repeated, reassociated).status).toBe("changed");
- repeated.network.requests[0]!.status = null;
- expect(compareNetwork(repeated, reassociated).status).toBe("unknown");
- });
+ for (const [left, right] of [
+ [partial, observed],
+ [observed, partial],
+ [partial, partial],
+ ] as const)
+ expect(compareNetwork(left, right)).toMatchObject({
+ status: "unknown",
+ total_changes: 0,
+ changes: [],
+ });
+ },
+ );
it("ignores transient request IDs and capture-approval state", async () => {
const browser = await startFakeCdpBrowser();
diff --git a/tests/boundary/cli/httpCaptureComparison.test.ts b/tests/boundary/cli/httpCaptureComparison.test.ts
new file mode 100644
index 000000000..10d373351
--- /dev/null
+++ b/tests/boundary/cli/httpCaptureComparison.test.ts
@@ -0,0 +1,77 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { expect, it } from "vitest";
+
+import { webPageInspectionSchema } from "../../../src/domain/browserObservationSchemas.js";
+import { parseEvidence } from "../../../src/domain/evidence.js";
+import { runCliJson } from "../../fixtures/cliJsonProcess.js";
+import { startHttpComparisonBrowser } from "../../fixtures/httpComparisonBrowser.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+it("CLI compares incomplete HTTP responses without inventing changes", async () => {
+ const root = await createTestTempDirectory("rea-http-comparison-");
+ const response = {
+ requestId: "request-1",
+ status: 200,
+ mimeType: "application/json",
+ };
+ const fixture = await startHttpComparisonBrowser([response]);
+ try {
+ const inspect = async () =>
+ webPageInspectionSchema.parse(
+ parseEvidence(
+ await runCliJson([
+ "inspect-web-page",
+ fixture.browser.endpoint,
+ "allowed-page",
+ "--observation-ms",
+ "0",
+ "--json",
+ ]),
+ ).normalized_result,
+ );
+ const pending = await inspect();
+ fixture.setRequests([{ ...response, encodedDataLength: 100 }]);
+ const finished = await inspect();
+ fixture.setRequests([{ ...response, status: 500, encodedDataLength: 100 }]);
+ const changed = await inspect();
+ expect(pending.network.requests).toMatchObject([
+ { status: 200, mime_type: "application/json", encoded_data_length: null },
+ ]);
+ expect(finished.network.requests).toMatchObject([
+ { status: 200, mime_type: "application/json", encoded_data_length: 100 },
+ ]);
+ const paths = [pending, finished, changed].map((_, index) =>
+ join(root, `capture-${String(index)}.json`),
+ );
+ await Promise.all(
+ [pending, finished, changed].map((inspection, index) =>
+ writeFile(paths[index]!, JSON.stringify({ inspection })),
+ ),
+ );
+ for (const [left, right, status, count] of [
+ [0, 1, "unknown", 0],
+ [1, 0, "unknown", 0],
+ [0, 2, "changed", 1],
+ ] as const) {
+ const compared = await runCliJson([
+ "compare-web-captures",
+ paths[left]!,
+ paths[right]!,
+ "--json",
+ ]);
+ expect(compared).toMatchObject({
+ operation: "compare_web_captures",
+ normalized_result: {
+ overall_status: status,
+ dimensions: {
+ network: { status, total_changes: count },
+ },
+ },
+ });
+ }
+ } finally {
+ await fixture.browser.close();
+ }
+}, 60_000);
diff --git a/tests/fixtures/httpComparisonBrowser.ts b/tests/fixtures/httpComparisonBrowser.ts
new file mode 100644
index 000000000..94c7ed856
--- /dev/null
+++ b/tests/fixtures/httpComparisonBrowser.ts
@@ -0,0 +1,83 @@
+import { startFakeCdpBrowser, type FakeCdpBrowser } from "./fakeCdpBrowser.js";
+
+export type HttpRequestObservation = {
+ readonly requestId: string;
+ readonly path?: string;
+ readonly status?: number;
+ readonly mimeType?: string;
+ readonly encodedDataLength?: number;
+};
+
+type CdpEvent = Parameters<FakeCdpBrowser["emitEvent"]>[0];
+
+const requestEvents = (
+ observation: HttpRequestObservation,
+ origin: string,
+ sessionId: string | undefined,
+): CdpEvent[] => {
+ const session = sessionId === undefined ? {} : { sessionId };
+ const url = `${origin}${observation.path ?? "/api/items"}`;
+ const { requestId, status, mimeType, encodedDataLength } = observation;
+ const events: CdpEvent[] = [
+ {
+ ...session,
+ method: "Network.requestWillBeSent",
+ params: {
+ requestId,
+ timestamp: 1,
+ type: "Fetch",
+ request: { url, method: "GET", headers: {} },
+ initiator: { type: "other" },
+ },
+ },
+ ];
+ if (status !== undefined || mimeType !== undefined)
+ events.push({
+ ...session,
+ method: "Network.responseReceived",
+ params: {
+ requestId,
+ timestamp: 2,
+ type: "Fetch",
+ response: {
+ url,
+ headers: {},
+ ...(status === undefined ? {} : { status }),
+ ...(mimeType === undefined ? {} : { mimeType }),
+ },
+ },
+ });
+ if (encodedDataLength !== undefined)
+ events.push({
+ ...session,
+ method: "Network.loadingFinished",
+ params: { requestId, timestamp: 3, encodedDataLength },
+ });
+ return events;
+};
+
+/** Capture changing response coverage through the real CDP wire boundary. */
+export const startHttpComparisonBrowser = async (
+ initial: readonly HttpRequestObservation[],
+): Promise<{
+ readonly browser: FakeCdpBrowser;
+ setRequests(next: readonly HttpRequestObservation[]): void;
+}> => {
+ let requests = initial;
+ const browser = await startFakeCdpBrowser({
+ eventsBeforeReply: ["Network.enable"],
+ commandEvents(command, origin) {
+ return command.method === "Network.enable"
+ ? requests.flatMap((request) =>
+ requestEvents(request, origin, command.sessionId),
+ )
+ : undefined;
+ },
+ });
+ return {
+ browser,
+ setRequests(next) {
+ requests = next;
+ },
+ };
+};