Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(browser): preserve captures when response bodies are unavailable

morluto96af2fa7444a
diff --git a/scripts/fixtures/browser-verifier-site.mjs b/scripts/fixtures/browser-verifier-site.mjs
index 841e7f900..d5ccb7923 100644
--- a/scripts/fixtures/browser-verifier-site.mjs
+++ b/scripts/fixtures/browser-verifier-site.mjs
@@ -7,6 +7,13 @@ export async function startBrowserVerifierSite() {
let port = 0;
let sessionGeneration = 0;
const server = createServer((request, response) => {
+ if (request.url === "/slow-json") {
+ response.setHeader("content-type", "application/json");
+ response.write('{"pending":');
+ const timer = setTimeout(() => response.end("true}"), 350);
+ response.once("close", () => clearTimeout(timer));
+ return;
+ }
if (request.url === "/screenshot-noise") {
response.setHeader("content-type", "text/html");
response.end(`<!doctype html><html><body style="margin:0"><canvas width="2048" height="1536"></canvas><script>
@@ -174,6 +181,11 @@ const browserScript = (port) => `
},
body: JSON.stringify({ token: "request-body-secret-value", active: true })
});
+ fetch("/slow-json", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ pending: true })
+ });
const socket = new WebSocket("ws://127.0.0.1:${String(port)}/live?token=websocket-url-secret");
socket.addEventListener("open", () => socket.send(JSON.stringify({ token: "websocket-secret-value" })));
socket.addEventListener("message", () => socket.close());
diff --git a/scripts/lib/browser-verifier-assertions.mjs b/scripts/lib/browser-verifier-assertions.mjs
index 514fbf267..b0c9b88d5 100644
--- a/scripts/lib/browser-verifier-assertions.mjs
+++ b/scripts/lib/browser-verifier-assertions.mjs
@@ -217,6 +217,19 @@ export function assertSensitiveShapes(result) {
);
if (request === undefined)
throw new Error("Real Chrome JSON request/response shapes were missing");
+ if (
+ !result.network.requests.some(
+ (item) =>
+ item.url.endsWith("/slow-json") &&
+ item.body_shapes.status === "partial" &&
+ item.body_shapes.request !== null &&
+ item.body_shapes.response === null,
+ ) ||
+ !result.completeness.unavailable_sections.includes("json_body_shapes")
+ )
+ throw new Error(
+ "Real Chrome unfinished JSON bodies were not reported as unavailable",
+ );
const consoleText = result.console.events.flatMap(
(event) => event.text_capture.values,
);
diff --git a/src/browser/CdpPageCapture.ts b/src/browser/CdpPageCapture.ts
index d8c275494..279253046 100644
--- a/src/browser/CdpPageCapture.ts
+++ b/src/browser/CdpPageCapture.ts
@@ -12,6 +12,7 @@ import { CdpCaptureEvents } from "./CdpCaptureEvents.js";
import { authorizedMainFrame } from "./CdpAuthorizedMainFrame.js";
import { captureStorage } from "./CdpCaptureStorage.js";
import { optionalCdpCommand } from "./CdpOptionalCommand.js";
+import { CdpCommandRejection } from "./CdpCommandRejection.js";
import {
captureAccessibility,
captureDom,
@@ -185,12 +186,34 @@ const captureJsonResponseBodies = async (
for (let index = 0; index < requestIds.length; index += 1) {
const requestId = requestIds[index];
if (requestId === undefined) continue;
- const result = await optionalCdpCommand(
- state.context,
- "Network.getResponseBody",
- { requestId },
- state.limitations,
- );
+ const request = state.events.network.get(requestId);
+ if (request === undefined) continue;
+ if (request.encoded_data_length === null) {
+ state.events.responseBodyUnavailable(requestId);
+ continue;
+ }
+ let result: unknown;
+ try {
+ result = await optionalCdpCommand(
+ state.context,
+ "Network.getResponseBody",
+ { requestId },
+ state.limitations,
+ );
+ } catch (cause: unknown) {
+ if (
+ !(cause instanceof CdpCommandRejection) ||
+ cause.command !== "Network.getResponseBody" ||
+ cause.code !== -32_000 ||
+ cause.reportedMessage === null
+ )
+ throw cause;
+ state.events.responseBodyUnavailable(requestId);
+ state.limitations.push(
+ `Response body unavailable for request ${requestId}: ${cause.userMessage}`,
+ );
+ continue;
+ }
if (result !== undefined) {
state.events.ingestResponseBody(requestId, result);
continue;
diff --git a/tests/boundary/browser/cdpBrowserProvider.network.test.ts b/tests/boundary/browser/cdpBrowserProvider.network.test.ts
index 152b2ef14..6534a39f0 100644
--- a/tests/boundary/browser/cdpBrowserProvider.network.test.ts
+++ b/tests/boundary/browser/cdpBrowserProvider.network.test.ts
@@ -5,6 +5,170 @@ import { inspectWebPageInputSchema } from "../../../src/domain/browserObservatio
import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";
import { trackBrowser } from "./cdpBrowserProvider.support.js";
+it("preserves unavailable response observations and captures completed peers", async () => {
+ const browser = await startFakeCdpBrowser({
+ commandEvents(command, origin) {
+ if (command.method !== "Network.enable") return undefined;
+ return ["unfinished", "missing-body", "available"].flatMap(
+ (requestId) => {
+ const session =
+ command.sessionId === undefined
+ ? {}
+ : { sessionId: command.sessionId };
+ const url = `${origin}/${requestId}`;
+ return [
+ {
+ ...session,
+ method: "Network.requestWillBeSent",
+ params: {
+ requestId,
+ type: "Fetch",
+ request: {
+ url,
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ postData: '{"selected":true}',
+ },
+ },
+ },
+ {
+ ...session,
+ method: "Network.responseReceived",
+ params: {
+ requestId,
+ response: {
+ url,
+ status: 200,
+ mimeType: "application/json",
+ headers: {},
+ },
+ },
+ },
+ ...(requestId === "unfinished"
+ ? []
+ : [
+ {
+ ...session,
+ method: "Network.loadingFinished",
+ params: { requestId, encodedDataLength: 0 },
+ },
+ ]),
+ ];
+ },
+ );
+ },
+ commandError(command) {
+ if (
+ command.method === "Network.getResponseBody" &&
+ command.params.requestId === "missing-body"
+ )
+ return {
+ code: -32_000,
+ message: "No data found for resource with given identifier",
+ };
+ return undefined;
+ },
+ commandResult(command) {
+ return command.method === "Network.getResponseBody"
+ ? { body: '{"retained":true}', base64Encoded: false }
+ : undefined;
+ },
+ });
+ trackBrowser(browser);
+ const result = await new CdpBrowserProvider().inspectPage(
+ inspectWebPageInputSchema.parse({
+ cdp_endpoint: browser.endpoint,
+ allowed_origins: [browser.allowedOrigin],
+ target_id: "allowed-page",
+ observation_ms: 20,
+ include_json_body_shapes: true,
+ }),
+ );
+ if (!result.ok) throw result.error;
+ expect(result.value.network.requests).toEqual([
+ expect.objectContaining({
+ request_id: "unfinished",
+ status: 200,
+ encoded_data_length: null,
+ body_shapes: {
+ status: "partial",
+ request: expect.any(Object),
+ response: null,
+ },
+ }),
+ expect.objectContaining({
+ request_id: "missing-body",
+ status: 200,
+ encoded_data_length: 0,
+ body_shapes: {
+ status: "partial",
+ request: expect.any(Object),
+ response: null,
+ },
+ }),
+ expect.objectContaining({
+ request_id: "available",
+ body_shapes: {
+ status: "included",
+ request: expect.any(Object),
+ response: expect.any(Object),
+ },
+ }),
+ ]);
+ expect(result.value.completeness.unavailable_sections).toContain(
+ "json_body_shapes",
+ );
+ expect(result.value.limitations).toContainEqual(
+ expect.stringContaining(
+ "missing-body: CDP Network.getResponseBody failed (-32000): No data found",
+ ),
+ );
+ expect(
+ browser.commands
+ .filter(({ method }) => method === "Network.getResponseBody")
+ .map(({ params }) => params.requestId),
+ ).toEqual(["missing-body", "available"]);
+});
+
+it.each([
+ {
+ options: {
+ commandError: (command: { method: string }) =>
+ command.method === "Network.getResponseBody"
+ ? { code: -32_602, message: "Invalid parameters" }
+ : undefined,
+ },
+ reason: "protocol_error",
+ },
+ {
+ options: { malformedMessageOnMethod: "Network.getResponseBody" },
+ reason: "protocol_error",
+ },
+ {
+ options: { closeOnMethod: "Network.getResponseBody" },
+ reason: "disconnected",
+ },
+])(
+ "preserves fatal response capture errors ($reason)",
+ async ({ options, reason }) => {
+ const browser = await startFakeCdpBrowser(options);
+ trackBrowser(browser);
+ const result = await new CdpBrowserProvider().inspectPage(
+ inspectWebPageInputSchema.parse({
+ cdp_endpoint: browser.endpoint,
+ allowed_origins: [browser.allowedOrigin],
+ target_id: "allowed-page",
+ observation_ms: 20,
+ include_json_body_shapes: true,
+ }),
+ );
+ expect(result).toMatchObject({
+ ok: false,
+ error: { _tag: "BrowserObservationError", reason },
+ });
+ },
+);
+
describe("CdpBrowserProvider: network 1", () => {
it("retains same-origin redirect hops on the one final request", async () => {
const browser = await startFakeCdpBrowser({ redirectWithinOrigin: true });