Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
test(web): cover nullable HTTP captures through CDP and CLI (#1834)
Yann Roberto5d12a0b4b613
diff --git a/tests/boundary/browser/cdpHttpComparison.test.ts b/tests/boundary/browser/cdpHttpComparison.test.tsnew file mode 100644index 000000000..2f4ad689c--- /dev/null+++ b/tests/boundary/browser/cdpHttpComparison.test.ts@@ -0,0 +1,255 @@+import { describe, expect, it } from "vitest";++import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js";+import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";+import type { WebPageInspection } from "../../../src/domain/browserObservationSchemas.js";+import { compareWebCaptures } from "../../../src/domain/webCaptureDiff.js";+import { compareWebCapturesInputSchema } from "../../../src/domain/webCaptureDiffSchemas.js";+import type { FakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";+import {+ startHttpComparisonBrowser,+ type HttpRequestObservation,+} from "../../fixtures/httpComparisonBrowser.js";+import { trackBrowser } from "./cdpBrowserProvider.support.js";++const inspect = async (browser: FakeCdpBrowser): Promise<WebPageInspection> => {+ const result = await new CdpBrowserProvider().inspectPage(+ inspectWebPageInputSchema.parse({+ cdp_endpoint: browser.endpoint,+ allowed_origins: [browser.allowedOrigin],+ target_id: "allowed-page",+ observation_ms: 0,+ }),+ );+ if (!result.ok) throw result.error;+ return result.value;+};++const capturePair = async (+ before: readonly HttpRequestObservation[],+ after: readonly HttpRequestObservation[],+): Promise<readonly [WebPageInspection, WebPageInspection]> => {+ const fixture = await startHttpComparisonBrowser(before);+ trackBrowser(fixture.browser);+ const left = await inspect(fixture.browser);+ fixture.setRequests(after);+ return [left, await inspect(fixture.browser)];+};++const compareNetwork = (before: WebPageInspection, after: WebPageInspection) =>+ compareWebCaptures(+ compareWebCapturesInputSchema.parse({+ before: { inspection: before },+ after: { inspection: after },+ }),+ ).dimensions.network;++const response = {+ requestId: "request-before",+ status: 200,+ mimeType: "application/json",+} as const;++describe("HTTP comparison response coverage", () => {+ it("does not compare an unavailable byte count with a completed response", async () => {+ const [before, after] = await capturePair(+ [response],+ [{ ...response, requestId: "request-after", encodedDataLength: 100 }],+ );+ expect(before.network.requests[0]?.encoded_data_length).toBeNull();+ expect(after.network.requests[0]?.encoded_data_length).toBe(100);+ for (const [left, right] of [+ [before, after],+ [after, before],+ ] as const)+ expect(compareNetwork(left, right)).toMatchObject({+ status: "unknown",+ total_changes: 0,+ changes: [],+ });+ });++ it("does not treat an unobserved response as a changed response", async () => {+ const [before, after] = await capturePair(+ [{ requestId: "request-before" }],+ [{ ...response, encodedDataLength: 100 }],+ );+ expect(before.network.requests).toEqual([+ expect.objectContaining({+ status: null,+ mime_type: null,+ encoded_data_length: null,+ }),+ ]);+ for (const [left, right] of [+ [before, after],+ [after, before],+ ] as const)+ expect(compareNetwork(left, right)).toMatchObject({+ status: "unknown",+ total_changes: 0,+ });+ });++ it("retains observed status changes when response bytes are unavailable", async () => {+ const [before, after] = await capturePair(+ [response],+ [{ ...response, status: 500 }],+ );+ expect(compareNetwork(before, after)).toMatchObject({+ status: "changed",+ changes: [expect.objectContaining({ change: "modified" })],+ });+ });++ it.each([+ { mimeType: "text/plain", encodedDataLength: 100 },+ { mimeType: "application/json", encodedDataLength: 200 },+ ])("retains observed response changes: %j", async (change) => {+ const [before, after] = await capturePair(+ [{ ...response, encodedDataLength: 100 }],+ [{ ...response, ...change }],+ );+ expect(compareNetwork(before, after).status).toBe("changed");+ });+});++describe("HTTP comparison repeated request coverage", () => {+ it("does not assign missing evidence by request order or transient ID", async () => {+ const [before, after] = await capturePair(+ [+ { ...response, encodedDataLength: 100 },+ { ...response, requestId: "second-before" },+ ],+ [+ { ...response, requestId: "second-after", encodedDataLength: 200 },+ { ...response, requestId: "first-after", encodedDataLength: 100 },+ ],+ );+ expect(before.network.requests).toHaveLength(2);+ expect(after.network.requests).toHaveLength(2);+ expect(compareNetwork(before, after)).toMatchObject({+ status: "unknown",+ total_changes: 0,+ });+ });++ it("compares complete byte counts even when another field is incomplete", async () => {+ const [before, after] = await capturePair(+ [+ { ...response, encodedDataLength: 100 },+ { requestId: "second-before", encodedDataLength: 200 },+ ],+ [+ { requestId: "second-after", encodedDataLength: 300 },+ { ...response, requestId: "first-after", encodedDataLength: 100 },+ ],+ );+ expect(compareNetwork(before, after).status).toBe("changed");+ });++ it("preserves the association between complete response fields", async () => {+ const [before, after] = await capturePair(+ [+ { ...response, encodedDataLength: 100 },+ {+ ...response,+ requestId: "second-before",+ status: 201,+ encodedDataLength: 200,+ },+ ],+ [+ { ...response, requestId: "first-after", encodedDataLength: 200 },+ {+ ...response,+ requestId: "second-after",+ status: 201,+ encodedDataLength: 100,+ },+ ],+ );+ expect(compareNetwork(before, after).status).toBe("changed");+ });+});++it.each([+ {+ name: "a pending request in an empty inventory",+ before: [],+ after: [{ requestId: "pending" }],+ },+ {+ name: "a known request beside an existing pending request",+ before: [{ requestId: "pending" }],+ after: [+ { requestId: "pending" },+ {+ ...response,+ requestId: "new",+ path: "/api/new",+ encodedDataLength: 100,+ },+ ],+ },+])("preserves additions and removals of $name", async (observations) => {+ const [before, after] = await capturePair(+ observations.before,+ observations.after,+ );+ // Live CDP observations have attach-limited inventories. Their absence+ // remains unknown even when a request is seen in the other capture.+ for (const [left, right] of [+ [before, after],+ [after, before],+ ] as const)+ expect(compareNetwork(left, right)).toMatchObject({+ status: "unknown",+ total_changes: 0,+ changes: [],+ });++ // Project explicitly complete inventory coverage without changing the+ // producer's request facts; response fields remain independently unknown.+ markNetworkInventoriesComplete(before, after);+ for (const [left, right, change] of [+ [before, after, "added"],+ [after, before, "removed"],+ ] as const)+ expect(compareNetwork(left, right)).toMatchObject({+ status: "changed",+ total_changes: 1,+ changes: [expect.objectContaining({ change })],+ });+});++it("requires response field coverage even when capture sections are complete", async () => {+ const [before, after] = await capturePair(+ [response],+ [{ ...response, encodedDataLength: 100 }],+ );+ markNetworkInventoriesComplete(before, after);+ expect(compareNetwork(before, before).status).toBe("unknown");+ expect(compareNetwork(before, after).status).toBe("unknown");+ expect(compareNetwork(after, after).status).toBe("unchanged");+});++const markNetworkInventoriesComplete = (+ ...inspections: WebPageInspection[]+): void => {+ const sections = new Set([+ "network_requests",+ "websocket_connections",+ "websocket_frames",+ ]);+ for (const inspection of inspections)+ for (const key of [+ "policy_filtered_sections",+ "attach_limited_sections",+ "truncated_sections",+ "unavailable_sections",+ ] as const)+ inspection.completeness[key] = inspection.completeness[key].filter(+ (section) => !sections.has(section),+ );+};diff --git a/tests/boundary/browser/webCaptureDiff.test.ts b/tests/boundary/browser/webCaptureDiff.test.tsindex 5c2ce8785..3680f7dfa 100644--- a/tests/boundary/browser/webCaptureDiff.test.ts+++ b/tests/boundary/browser/webCaptureDiff.test.ts@@ -446,91 +446,50 @@ describe("web capture WebSocket payload byte coverage", () => {});describe("web capture diff", () => {- it("treats unavailable HTTP response facts as unknown while retaining observed changes", async () => {- const browser = await startFakeCdpBrowser();- browsers.push(browser);- const captured = await new CdpBrowserProvider().inspectPage(- inspectWebPageInputSchema.parse({- cdp_endpoint: browser.endpoint,- allowed_origins: [browser.allowedOrigin],- target_id: "allowed-page",- observation_ms: 0,- }),- );- if (!captured.ok) throw captured.error;- const before = structuredClone(captured.value);- const request = before.network.requests[0];- if (request === undefined) throw new Error("Missing network request");- request.status = null;- request.mime_type = null;- request.encoded_data_length = null;- markSectionsComplete(before, [- "network_requests",- "websocket_connections",- "websocket_frames",- "json_body_shapes",- ]);- for (const item of before.network.requests)- item.body_shapes.status = "not_approved";+ it.each(["status", "mime_type"] as const)(+ "keeps an independently unavailable HTTP %s unknown",+ async (field) => {+ const browser = await startFakeCdpBrowser();+ browsers.push(browser);+ const captured = await new CdpBrowserProvider().inspectPage(+ inspectWebPageInputSchema.parse({+ cdp_endpoint: browser.endpoint,+ allowed_origins: [browser.allowedOrigin],+ target_id: "allowed-page",+ observation_ms: 0,+ }),+ );+ if (!captured.ok) throw captured.error;+ const observed = structuredClone(captured.value);+ const request = observed.network.requests[0];+ if (request === undefined) throw new Error("Missing network request");+ request.status = 200;+ request.mime_type = "application/json";+ request.encoded_data_length = 100;+ markSectionsComplete(observed, [+ "network_requests",+ "websocket_connections",+ "websocket_frames",+ "json_body_shapes",+ ]);+ for (const item of observed.network.requests)+ item.body_shapes.status = "not_approved";+ expect(compareNetwork(observed, observed).status).toBe("unchanged");- const after = structuredClone(before);- const responseObserved = after.network.requests[0];- if (responseObserved === undefined)- throw new Error("Missing network request");- responseObserved.status = 200;- responseObserved.mime_type = "application/json";- responseObserved.encoded_data_length = 100;- expect(compareNetwork(before, after)).toMatchObject({- status: "unknown",- total_changes: 0,- changes: [],- });- expect(compareNetwork(after, before)).toMatchObject({- status: "unknown",- total_changes: 0,- });- expect(compareNetwork(before, before).status).toBe("unknown");-- for (const field of [- "status",- "mime_type",- "encoded_data_length",- ] as const) {- const partial = structuredClone(after);+ const partial = structuredClone(observed);partial.network.requests[0]![field] = null;- expect(compareNetwork(partial, after).status).toBe("unknown");- expect(compareNetwork(after, partial).status).toBe("unknown");- }-- const inventoryBefore = structuredClone(before);- inventoryBefore.network.requests = [];- expect(compareNetwork(inventoryBefore, before)).toMatchObject({- status: "changed",- total_changes: 1,- changes: [expect.objectContaining({ change: "added" })],- });-- const statusChanged = structuredClone(after);- statusChanged.network.requests[0]!.status = 500;- expect(compareNetwork(after, statusChanged)).toMatchObject({- status: "changed",- changes: [expect.objectContaining({ change: "modified" })],- });- after.network.requests[0]!.encoded_data_length = null;- expect(compareNetwork(after, statusChanged).status).toBe("changed");-- const repeated = structuredClone(statusChanged);- const duplicate = structuredClone(repeated.network.requests[0]!);- duplicate.status = 200;- duplicate.mime_type = "text/plain";- repeated.network.requests.push(duplicate);- const reassociated = structuredClone(repeated);- reassociated.network.requests[0]!.mime_type = "text/plain";- reassociated.network.requests.at(-1)!.mime_type = "application/json";- expect(compareNetwork(repeated, reassociated).status).toBe("changed");- repeated.network.requests[0]!.status = null;- expect(compareNetwork(repeated, reassociated).status).toBe("unknown");- });+ for (const [left, right] of [+ [partial, observed],+ [observed, partial],+ [partial, partial],+ ] as const)+ expect(compareNetwork(left, right)).toMatchObject({+ status: "unknown",+ total_changes: 0,+ changes: [],+ });+ },+ );it("ignores transient request IDs and capture-approval state", async () => {const browser = await startFakeCdpBrowser();diff --git a/tests/boundary/cli/httpCaptureComparison.test.ts b/tests/boundary/cli/httpCaptureComparison.test.tsnew file mode 100644index 000000000..10d373351--- /dev/null+++ b/tests/boundary/cli/httpCaptureComparison.test.ts@@ -0,0 +1,77 @@+import { writeFile } from "node:fs/promises";+import { join } from "node:path";++import { expect, it } from "vitest";++import { webPageInspectionSchema } from "../../../src/domain/browserObservationSchemas.js";+import { parseEvidence } from "../../../src/domain/evidence.js";+import { runCliJson } from "../../fixtures/cliJsonProcess.js";+import { startHttpComparisonBrowser } from "../../fixtures/httpComparisonBrowser.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";++it("CLI compares incomplete HTTP responses without inventing changes", async () => {+ const root = await createTestTempDirectory("rea-http-comparison-");+ const response = {+ requestId: "request-1",+ status: 200,+ mimeType: "application/json",+ };+ const fixture = await startHttpComparisonBrowser([response]);+ try {+ const inspect = async () =>+ webPageInspectionSchema.parse(+ parseEvidence(+ await runCliJson([+ "inspect-web-page",+ fixture.browser.endpoint,+ "allowed-page",+ "--observation-ms",+ "0",+ "--json",+ ]),+ ).normalized_result,+ );+ const pending = await inspect();+ fixture.setRequests([{ ...response, encodedDataLength: 100 }]);+ const finished = await inspect();+ fixture.setRequests([{ ...response, status: 500, encodedDataLength: 100 }]);+ const changed = await inspect();+ expect(pending.network.requests).toMatchObject([+ { status: 200, mime_type: "application/json", encoded_data_length: null },+ ]);+ expect(finished.network.requests).toMatchObject([+ { status: 200, mime_type: "application/json", encoded_data_length: 100 },+ ]);+ const paths = [pending, finished, changed].map((_, index) =>+ join(root, `capture-${String(index)}.json`),+ );+ await Promise.all(+ [pending, finished, changed].map((inspection, index) =>+ writeFile(paths[index]!, JSON.stringify({ inspection })),+ ),+ );+ for (const [left, right, status, count] of [+ [0, 1, "unknown", 0],+ [1, 0, "unknown", 0],+ [0, 2, "changed", 1],+ ] as const) {+ const compared = await runCliJson([+ "compare-web-captures",+ paths[left]!,+ paths[right]!,+ "--json",+ ]);+ expect(compared).toMatchObject({+ operation: "compare_web_captures",+ normalized_result: {+ overall_status: status,+ dimensions: {+ network: { status, total_changes: count },+ },+ },+ });+ }+ } finally {+ await fixture.browser.close();+ }+}, 60_000);diff --git a/tests/fixtures/httpComparisonBrowser.ts b/tests/fixtures/httpComparisonBrowser.tsnew file mode 100644index 000000000..94c7ed856--- /dev/null+++ b/tests/fixtures/httpComparisonBrowser.ts@@ -0,0 +1,83 @@+import { startFakeCdpBrowser, type FakeCdpBrowser } from "./fakeCdpBrowser.js";++export type HttpRequestObservation = {+ readonly requestId: string;+ readonly path?: string;+ readonly status?: number;+ readonly mimeType?: string;+ readonly encodedDataLength?: number;+};++type CdpEvent = Parameters<FakeCdpBrowser["emitEvent"]>[0];++const requestEvents = (+ observation: HttpRequestObservation,+ origin: string,+ sessionId: string | undefined,+): CdpEvent[] => {+ const session = sessionId === undefined ? {} : { sessionId };+ const url = `${origin}${observation.path ?? "/api/items"}`;+ const { requestId, status, mimeType, encodedDataLength } = observation;+ const events: CdpEvent[] = [+ {+ ...session,+ method: "Network.requestWillBeSent",+ params: {+ requestId,+ timestamp: 1,+ type: "Fetch",+ request: { url, method: "GET", headers: {} },+ initiator: { type: "other" },+ },+ },+ ];+ if (status !== undefined || mimeType !== undefined)+ events.push({+ ...session,+ method: "Network.responseReceived",+ params: {+ requestId,+ timestamp: 2,+ type: "Fetch",+ response: {+ url,+ headers: {},+ ...(status === undefined ? {} : { status }),+ ...(mimeType === undefined ? {} : { mimeType }),+ },+ },+ });+ if (encodedDataLength !== undefined)+ events.push({+ ...session,+ method: "Network.loadingFinished",+ params: { requestId, timestamp: 3, encodedDataLength },+ });+ return events;+};++/** Capture changing response coverage through the real CDP wire boundary. */+export const startHttpComparisonBrowser = async (+ initial: readonly HttpRequestObservation[],+): Promise<{+ readonly browser: FakeCdpBrowser;+ setRequests(next: readonly HttpRequestObservation[]): void;+}> => {+ let requests = initial;+ const browser = await startFakeCdpBrowser({+ eventsBeforeReply: ["Network.enable"],+ commandEvents(command, origin) {+ return command.method === "Network.enable"+ ? requests.flatMap((request) =>+ requestEvents(request, origin, command.sessionId),+ )+ : undefined;+ },+ });+ return {+ browser,+ setRequests(next) {+ requests = next;+ },+ };+};