Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(browser): keep omitted JSON request bodies out of complete shape coverage (#4476)
Adolanium7e8ec0e4566a
diff --git a/docs/browser-observation.md b/docs/browser-observation.mdindex 389dcfde7..ff527a694 100644--- a/docs/browser-observation.md+++ b/docs/browser-observation.md@@ -340,7 +340,7 @@ REA retains local observation data and removes only URL userinfo credentials, ex- DOM snapshots retain node types, node names, value lengths, and attribute names, but not text or attribute values.- Accessibility structure and roles are retained by default, while names and descriptions require `include_accessibility_text: true`.- Network observations retain method, status, MIME type, size, type, initiator stack location, and complete URLs with userinfo credentials removed. Redirects reported by CDP are attached as ordered `redirects` on the one request ID, with the prior request URL, CDP response URL, method/type, response status/MIME/encoded length, request timestamp, and the next `requestWillBeSent` event timestamp. CDP delivers `redirectResponse` with that next request event, so the latter is an event boundary rather than an exact response timestamp. Redirect response headers and bodies are discarded; hops outside the selected origins are discarded with the request chain. Headers are discarded after an allowlisted projection of length/encoding, structured CSP/Link/policy fields, and untrusted agent hints. Cookies and authorization headers are never retained.-- Request/response bodies are not requested or parsed by default. When selected, only allowed-origin JSON media types are read, converted immediately to complete value-free typed paths, types and observation counts, and discarded. Each path segment is either `{ "kind": "property", "name": "raw property name" }` or `{ "kind": "array-element" }`; array-element types include primitives and nested arrays. Literal `*` properties remain distinct from array elements. `Network.getResponseBody` is never sent unless JSON body shape capture is selected. Comparison uses body shapes only when both captures selected them, and compares request/response shapes only where both observations provide those facts. Unavailable or truncated selected coverage makes equivalence unknown; observed differences still report changed.+- Request/response bodies are not requested or parsed by default. When selected, only allowed-origin JSON media types are read, converted immediately to complete value-free typed paths, types and observation counts, and discarded. Each path segment is either `{ "kind": "property", "name": "raw property name" }` or `{ "kind": "array-element" }`; array-element types include primitives and nested arrays. Literal `*` properties remain distinct from array elements. `Network.getResponseBody` is never sent unless JSON body shape capture is selected. A JSON-typed request body that CDP reports through `hasPostData` but omits from `postData`, or that cannot be parsed as JSON, makes `json_body_shapes` unavailable. That request's `body_shapes.status` is `partial`, not `included`, when only its response shape was captured. Comparison uses body shapes only when both captures selected them, and compares request/response shapes only where both observations provide those facts. Unavailable or truncated selected coverage makes equivalence unknown; observed differences still report changed.For normalized passive inputs with complete request inventories on both sides,request additions and removals remain observable even when selected body shapes,response metadata, or WebSocket coverage are unavailable. HTTP request anddiff --git a/src/browser/CdpCaptureEventBodyShapes.ts b/src/browser/CdpCaptureEventBodyShapes.tsindex d12f21c2f..0c3af407f 100644--- a/src/browser/CdpCaptureEventBodyShapes.ts+++ b/src/browser/CdpCaptureEventBodyShapes.ts@@ -35,15 +35,22 @@ export const ingestResponseBodyShape = (export const requestBodyShape = (state: CdpCaptureEventsState,+ requestId: string,request: UnknownRecord,): NetworkState["body_shapes"] => {+ state.missingRequestBodyShapeIds.delete(requestId);if (!state.input.include_json_body_shapes)return { status: "not_approved", request: null, response: null };+ const headers = recordValue(request.headers);+ const json = isJsonContentType(headers);const body = cdpStringValue(request.postData);- if (!isJsonContentType(recordValue(request.headers)) || body === undefined)- return { status: "unavailable", request: null, response: null };- const shape = inferJsonShape(body);- if (shape === null) state.completeness.unavailable("json_body_shapes");+ // CDP may omit a long body from postData while reporting hasPostData.+ const shape = json && body !== undefined ? inferJsonShape(body) : null;+ const expected = json && (body !== undefined || request.hasPostData === true);+ if (expected && shape === null) {+ state.missingRequestBodyShapeIds.add(requestId);+ state.completeness.unavailable("json_body_shapes");+ }return {status: shape === null ? "unavailable" : "included",request: shape,@@ -61,7 +68,9 @@ export const updateResponseBodyShape = (const request = current.body_shapes.request;const status =response !== null- ? "included"+ ? state.missingRequestBodyShapeIds.has(requestId)+ ? "partial"+ : "included": request !== null? "partial": "unavailable";diff --git a/src/browser/CdpCaptureEventHandlers.ts b/src/browser/CdpCaptureEventHandlers.tsindex bf0d36d4d..d11652b18 100644--- a/src/browser/CdpCaptureEventHandlers.ts+++ b/src/browser/CdpCaptureEventHandlers.ts@@ -334,7 +334,7 @@ export const handleRequestWillBeSent = (line: integerOrNull(initiatorFrame?.lineNumber),column: integerOrNull(initiatorFrame?.columnNumber),},- body_shapes: requestBodyShape(state, request),+ body_shapes: requestBodyShape(state, requestId, request),});const timestamp = numberValue(params.timestamp);if (timestamp === undefined) state.networkRequestTimestamps.delete(requestId);diff --git a/src/browser/CdpCaptureEventState.ts b/src/browser/CdpCaptureEventState.tsindex 2e72bd37c..778c54812 100644--- a/src/browser/CdpCaptureEventState.ts+++ b/src/browser/CdpCaptureEventState.ts@@ -16,6 +16,8 @@ export interface CdpCaptureEventsState {readonly network: Map<string, NetworkState>;readonly networkRequestTimestamps: Map<string, number>;readonly malformedRedirectRequestIds: Set<string>;+ /** Requests whose selected JSON request-body shape was expected but not obtained. */+ readonly missingRequestBodyShapeIds: Set<string>;readonly webSocketConnectionsById: Map<string,WebPageInspection["network"]["websocket_connections"][number]diff --git a/src/browser/CdpCaptureEvents.ts b/src/browser/CdpCaptureEvents.tsindex 96b8bba7b..c29df633d 100644--- a/src/browser/CdpCaptureEvents.ts+++ b/src/browser/CdpCaptureEvents.ts@@ -29,6 +29,7 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {readonly network = new Map<string, NetworkState>();readonly networkRequestTimestamps = new Map<string, number>();readonly malformedRedirectRequestIds = new Set<string>();+ readonly missingRequestBodyShapeIds = new Set<string>();readonly webSocketConnectionsById = new Map<string,WebPageInspection["network"]["websocket_connections"][number]@@ -76,6 +77,7 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {this.#networkRequestOccurrences.clear();this.#responseBodyExaminedOccurrences.clear();this.malformedRedirectRequestIds.clear();+ this.missingRequestBodyShapeIds.clear();this.webSocketConnectionsById.clear();this.console.length = 0;this.responseMetadata.length = 0;diff --git a/tests/boundary/browser/cdpJsonShapeComparison.test.ts b/tests/boundary/browser/cdpJsonShapeComparison.test.tsindex 677e8bd04..7f2f8d864 100644--- a/tests/boundary/browser/cdpJsonShapeComparison.test.ts+++ b/tests/boundary/browser/cdpJsonShapeComparison.test.ts@@ -213,6 +213,84 @@ describe("selected JSON body-shape coverage", () => {},);+ it.each([+ {+ name: "an omitted JSON body",+ request: {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ hasPostData: true,+ },+ },+ {+ name: "an unparseable JSON body",+ request: {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ postData: "{",+ },+ },+ ])(+ "keeps $name partial after a response shape arrives",+ async ({ request }) => {+ const before = captureBody("response", "[123]", request);+ const after = captureBody("response", "[123]", request);+ for (const capture of [before, after]) {+ capture.completeness.attach_limited_sections = [];+ expect(capture.completeness.unavailable_sections).toContain(+ "json_body_shapes",+ );+ expect(capture.network.requests[0]?.body_shapes).toMatchObject({+ status: "partial",+ request: null,+ response: { root_type: "array" },+ });+ }+ expect(+ (await compareCaptures(before, after)).dimensions.network,+ ).toMatchObject({ status: "unknown", total_changes: 0 });+ },+ );++ it.each([+ { name: "a bodiless GET", request: { method: "GET", headers: {} } },+ {+ name: "a bodiless JSON-typed GET",+ request: {+ method: "GET",+ headers: { "Content-Type": "application/json" },+ },+ },+ {+ name: "an omitted body without a media type",+ request: { method: "POST", headers: {}, hasPostData: true },+ },+ {+ name: "an omitted form body",+ request: {+ method: "POST",+ headers: { "Content-Type": "application/x-www-form-urlencoded" },+ hasPostData: true,+ },+ },+ ])("includes the response shape for $name", async ({ request }) => {+ const before = captureBody("response", "[123]", request);+ const after = captureBody("response", "[123]", request);+ for (const capture of [before, after]) {+ capture.completeness.attach_limited_sections = [];+ expect(capture.completeness.unavailable_sections).not.toContain(+ "json_body_shapes",+ );+ expect(capture.network.requests[0]?.body_shapes).toMatchObject({+ status: "included",+ request: null,+ });+ }+ expect(+ (await compareCaptures(before, after)).dimensions.network,+ ).toMatchObject({ status: "unchanged", total_changes: 0 });+ });+it("ignores shapes when either capture did not request them", async () => {const before = captureBody("request", "[123]");const after = structuredClone(before);@@ -293,7 +371,15 @@ const compareCaptures = async (return webCaptureDiffSchema.parse(evidence.normalized_result);};-const captureBody = (source: BodySource, body: string): WebPageInspection => {+const captureBody = (+ source: BodySource,+ body: string,+ request: Record<string, unknown> = {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ postData: source === "request" ? body : "{}",+ },+): WebPageInspection => {const events = new CdpCaptureEvents(inspectWebPageInputSchema.parse({cdp_endpoint: "http://127.0.0.1:9222",@@ -310,12 +396,7 @@ const captureBody = (source: BodySource, body: string): WebPageInspection => {params: {requestId: "request-1",type: "Fetch",- request: {- url: `${ORIGIN}/api`,- method: "POST",- headers: { "Content-Type": "application/json" },- postData: source === "request" ? body : "{}",- },+ request: { url: `${ORIGIN}/api`, ...request },initiator: { type: "script" },},});