Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(protobuf): preserve partial facts for malformed web trailers
morluto9cf7a7cb0d3c
diff --git a/src/browser/history/protobuf/ProtobufBody.ts b/src/browser/history/protobuf/ProtobufBody.tsindex 7d040fd9c..eb3d91777 100644--- a/src/browser/history/protobuf/ProtobufBody.ts+++ b/src/browser/history/protobuf/ProtobufBody.ts@@ -3,7 +3,7 @@ import {prepareProtobufBody,type PreparedProtobufBody,} from "./ProtobufFraming.js";-import { CaptureFormatError } from "../CaptureFormatError.js";+import { parseGrpcWebTrailers } from "./ProtobufTrailers.js";import type {ProtobufDecoding,ProtobufFrame,@@ -155,33 +155,22 @@ export const decodeProtobufBody = async (};} else {budget.charge(length * 8);- try {- const text = new TextDecoder("utf-8", {- fatal: true,- ignoreBOM: true,- }).decode(data.subarray(at + 5, at + 5 + length));- frame.trailers = [];- for (const line of text.split("\r\n")) {- if (line === "") continue;- const colon = line.indexOf(":");- if (- colon <= 0 ||- !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(line.slice(0, colon))- )- throw new Error("Malformed trailer header");- budget.charge(512);- frame.trailers.push({- name: redact(line.slice(0, colon)),- value: redact(line.slice(colon + 1).replace(/^[ \t]+/, "")),- });- }- frame.examined_bytes = length;- } catch (cause) {- if (cause instanceof CaptureFormatError) throw cause;+ const parsed = parseGrpcWebTrailers(+ data.subarray(at + 5, at + 5 + length),+ budget,+ );+ const facts = parsed.ok ? parsed.value : parsed.error;+ frame.examined_bytes = facts.examined_bytes;+ if (parsed.ok || parsed.error.kind !== "invalid-utf8")+ frame.trailers = facts.trailers.map(({ name, value }) => ({+ name: redact(name),+ value: redact(value),+ }));+ if (!parsed.ok) {frame.status = "partial";frame.error = {offset: at + 5,- reason: "Invalid gRPC-web trailer header block",+ reason: parsed.error.reason,};}}diff --git a/src/browser/history/protobuf/ProtobufBodyTrailer.test.ts b/src/browser/history/protobuf/ProtobufBodyTrailer.test.tsnew file mode 100644index 000000000..71dbb3030--- /dev/null+++ b/src/browser/history/protobuf/ProtobufBodyTrailer.test.ts@@ -0,0 +1,111 @@+import { expect, it } from "vitest";+import { decodeProtobufBody } from "./ProtobufBody.js";+import { ProtobufBudget } from "./ProtobufWire.js";++const source = {+ pointer: "/response/content/text",+ content_type: "application/grpc-web+proto",+ body_sha256: "a".repeat(64),+};+const same = (value: string) => value;+const trailerFrame = (text: string) => {+ const payload = Buffer.from(text, "utf8");+ const frame = Buffer.alloc(5 + payload.length);+ frame[0] = 128;+ frame.writeUInt32BE(payload.length, 1);+ payload.copy(frame, 5);+ return frame;+};++it("rejects forbidden trailer controls while retaining valid earlier trailers", async () => {+ for (const code of [+ ...Array.from({ length: 32 }, (_, index) => index),+ 127,+ ]) {+ if (code === 9) continue;+ const control = String.fromCharCode(code);+ const text = `grpc-message: kept\r\ngrpc-status: 0${control}bad\r\n`;+ const input = trailerFrame(text);+ const result = await decodeProtobufBody(+ input,+ "grpc-web",+ source,+ same,+ new ProtobufBudget(),+ );+ expect(result.status, JSON.stringify(control)).toBe("partial");+ expect(result.frames[0]?.error).toMatchObject({ offset: 5 });+ expect(result.examined_bytes).toBe(5 + Buffer.byteLength(text));+ expect(result.frames[0]).toMatchObject({+ status: "partial",+ examined_bytes: Buffer.byteLength(text),+ });+ expect(result.frames[0]?.trailers).toEqual([+ { name: "grpc-message", value: "kept" },+ ]);+ expect(input.subarray(5)).toEqual(Buffer.from(text, "utf8"));+ }+});++it("retains allowed whitespace, visible text, and valid UTF-8 obs-text", async () => {+ const text = "grpc-message: alpha\tbeta \u00e9\r\n";+ const result = await decodeProtobufBody(+ trailerFrame(text),+ "grpc-web",+ source,+ same,+ new ProtobufBudget(),+ );+ expect(result.status).toBe("complete");+ expect(result.frames[0]?.trailers).toEqual([+ { name: "grpc-message", value: "alpha\tbeta \u00e9" },+ ]);+ expect(result.examined_bytes).toBe(5 + Buffer.byteLength(text));+});++it.each(["Grpc-Status", "GRPC-STATUS"])(+ "retains a valid prefix when the trailer name %s violates gRPC-Web casing",+ async (name) => {+ const text = `grpc-message: kept\r\n${name}: 0\r\n`;+ const result = await decodeProtobufBody(+ trailerFrame(text),+ "grpc-web",+ source,+ same,+ new ProtobufBudget(),+ );+ expect(result.status).toBe("partial");+ expect(result.examined_bytes).toBe(5 + Buffer.byteLength(text));+ expect(result.frames[0]?.trailers).toEqual([+ { name: "grpc-message", value: "kept" },+ ]);+ },+);++it("leaves undecodable trailer fields unknown and preserves unexpected failures", async () => {+ const input = trailerFrame("grpc-status: 0\r\n");+ input[5] = 255;+ const malformed = await decodeProtobufBody(+ input,+ "grpc-web",+ source,+ same,+ new ProtobufBudget(),+ );+ expect(malformed.status).toBe("partial");+ expect(malformed.frames[0]?.examined_bytes).toBe(0);+ expect(malformed.frames[0]?.trailers).toBeUndefined();++ const failure = { kind: "controlled-redaction-failure" };+ await expect(+ decodeProtobufBody(+ trailerFrame("grpc-status: 0\r\n"),+ "grpc-web",+ source,+ () => {+ throw failure;+ },+ new ProtobufBudget(),+ ),+ ).rejects.toBe(failure);+});diff --git a/src/browser/history/protobuf/ProtobufCapture.test.ts b/src/browser/history/protobuf/ProtobufCapture.test.tsindex 7e711471b..a0d7e6d60 100644--- a/src/browser/history/protobuf/ProtobufCapture.test.ts+++ b/src/browser/history/protobuf/ProtobufCapture.test.ts@@ -10,47 +10,78 @@ import {decodeCaptureProtobuf,} from "./ProtobufCapture.js";-it("restricts framing overrides to native body facts while preserving header bytes", async () => {- const binary = (pointer: string, bytes: Buffer) => ({- pointer,- representation: "producer-bytes" as const,- state: "retained" as const,- bytes: bytes.length,- sha256: createHash("sha256").update(bytes).digest("hex"),- content_base64: bytes.toString("base64"),- });- const headers = [- binary("/response/headers/0/0", Buffer.from("content-type")),- binary("/response/headers/0/1", Buffer.from("application/grpc")),- ];- const report: WebNetworkCapture = {- artifact: { path: "/fixture.mitm", bytes: 100, sha256: "a".repeat(64) },- decoder: { id: "fixture", name: "Native producer facts", version: "1" },- container: {- reported: null,+const binary = (pointer: string, bytes: Buffer) => ({+ pointer,+ representation: "producer-bytes" as const,+ state: "retained" as const,+ bytes: bytes.length,+ sha256: createHash("sha256").update(bytes).digest("hex"),+ content_base64: bytes.toString("base64"),+});++const captureWithBody = (+ contentType: string,+ body: Buffer,+): WebNetworkCapture => ({+ artifact: { path: "/fixture.mitm", bytes: 100, sha256: "a".repeat(64) },+ decoder: { id: "fixture", name: "Native producer facts", version: "1" },+ container: {+ reported: null,+ numeric_literals: [],+ redactions: [],+ records_pointer: "/flows",+ },+ format: "mitmproxy",+ total_records: 1,+ runtime_attribution: "unknown",+ limitations: [],+ records: [+ {+ ordinal: 0,+ location: { kind: "byte-range", offset: 0, bytes: 100 },+ reported: { response: { headers: [[null, null]], content: null } },+ binary_fields: [+ binary("/response/headers/0/0", Buffer.from("content-type")),+ binary("/response/headers/0/1", Buffer.from(contentType)),+ binary("/response/content", body),+ ],numeric_literals: [],redactions: [],- records_pointer: "/flows",+ limitations: [],},- format: "mitmproxy",- total_records: 1,- runtime_attribution: "unknown",- limitations: [],- records: [- {- ordinal: 0,- location: { kind: "byte-range", offset: 0, bytes: 100 },- reported: { response: { headers: [[null, null]], content: null } },- binary_fields: [- ...headers,- binary("/response/content", Buffer.from([0, 0, 0, 0, 2, 8, 1])),- ],- numeric_literals: [],- redactions: [],- limitations: [],- },- ],- };+ ],+});++const decode = async (capture: WebNetworkCapture) => {+ const input = inspectWebNetworkCaptureInputSchema.parse({+ capture_path: capture.artifact.path,+ format: capture.format,+ record_ordinals: [0],+ });+ const resources = new ArtifactResourceScope();+ try {+ const prepared = await prepareCaptureProtobuf(capture, input);+ await decodeCaptureProtobuf(capture, input, resources, prepared);+ } finally {+ await resources.close();+ }+};++const grpcWebTrailerFrame = (text: Buffer) => {+ const frame = Buffer.alloc(5 + text.length);+ frame[0] = 128;+ frame.writeUInt32BE(text.length, 1);+ text.copy(frame, 5);+ return frame;+};++it("restricts framing overrides to native body facts while preserving header bytes", async () => {+ const payload = Buffer.from([8, 1]);+ const body = Buffer.alloc(5 + payload.length);+ body.writeUInt32BE(payload.length, 1);+ payload.copy(body, 5);+ const report = captureWithBody("application/grpc", body);+ const headers = report.records[0]!.binary_fields.slice(0, 2);const input = inspectWebNetworkCaptureInputSchema.parse({capture_path: report.artifact.path,format: report.format,@@ -75,3 +106,20 @@ it("restricts framing overrides to native body facts while preserving header bytexpect(report.records[0]?.binary_fields.slice(0, 2)).toEqual(headers);expect(report.protobuf_coverage?.status).toBe("complete");});++it("keeps capture coverage partial for a malformed gRPC-web trailer", async () => {+ const trailerText = Buffer.from("grpc-status: 0\u0000bad\r\n");+ const body = grpcWebTrailerFrame(trailerText);+ const report = captureWithBody("application/grpc-web", body);+ const retainedBody = report.records[0]!.binary_fields[2];+ await decode(report);++ expect(report.records[0]?.protobuf?.[0]).toMatchObject({+ framing: "grpc-web",+ status: "partial",+ examined_bytes: body.length,+ frames: [{ status: "partial", examined_bytes: trailerText.length }],+ });+ expect(report.protobuf_coverage?.status).toBe("partial");+ expect(report.records[0]?.binary_fields[2]).toEqual(retainedBody);+});diff --git a/src/browser/history/protobuf/ProtobufTrailers.ts b/src/browser/history/protobuf/ProtobufTrailers.tsnew file mode 100644index 000000000..040923202--- /dev/null+++ b/src/browser/history/protobuf/ProtobufTrailers.ts@@ -0,0 +1,60 @@+import { err, ok, type Result } from "../../../domain/result.js";+import type { ProtobufBudget } from "./ProtobufWire.js";++interface TrailerFacts {+ readonly trailers: { name: string; value: string }[];+ readonly examined_bytes: number;+}++type TrailerFailure = TrailerFacts & {+ readonly kind: "invalid-utf8" | "invalid-name" | "invalid-value";+ readonly reason: string;+};++/** Parse the gRPC-Web HTTP/1 field block without discarding its valid prefix. */+export const parseGrpcWebTrailers = (+ bytes: Uint8Array,+ budget: ProtobufBudget,+): Result<TrailerFacts, TrailerFailure> => {+ let text: string;+ try {+ text = new TextDecoder("utf-8", {+ fatal: true,+ ignoreBOM: true,+ }).decode(bytes);+ } catch (cause) {+ if (!(cause instanceof TypeError)) throw cause;+ return err({+ kind: "invalid-utf8",+ reason: "gRPC-web trailer header block is not valid UTF-8",+ trailers: [],+ examined_bytes: 0,+ });+ }+ // The whole UTF-8 block has been examined even when field validation stops.+ const facts: TrailerFacts = { trailers: [], examined_bytes: bytes.length };+ for (const line of text.split("\r\n")) {+ if (line === "") continue;+ const colon = line.indexOf(":");+ // Unlike ordinary HTTP/1 headers, in-body gRPC-Web names must be lower-case.+ if (colon <= 0 || !/^[!#$%&'*+.^_`|~0-9a-z-]+$/.test(line.slice(0, colon)))+ return err({+ ...facts,+ kind: "invalid-name",+ reason:+ "gRPC-web trailer header names must be lower-case HTTP field names",+ });+ const value = line.slice(colon + 1).replace(/^[ \t]+/, "");+ // RFC 9110 permits HTAB but excludes other C0 controls and DEL.+ if (/[\u0000-\u0008\u000a-\u001f\u007f]/u.test(value))+ return err({+ ...facts,+ kind: "invalid-value",+ reason:+ "gRPC-web trailer field value contains a forbidden control character",+ });+ budget.charge(512);+ facts.trailers.push({ name: line.slice(0, colon), value });+ }+ return ok(facts);+};diff --git a/tests/acceptance/tools/protobufCapture.test.ts b/tests/acceptance/tools/protobufCapture.test.tsindex 921c1b13d..10e6b821c 100644--- a/tests/acceptance/tools/protobufCapture.test.ts+++ b/tests/acceptance/tools/protobufCapture.test.ts@@ -556,3 +556,71 @@ captureTest(});},);++captureTest(+ "keeps malformed gRPC-web trailers partial through CLI and MCP without changing capture bytes",+ async ({ cli, mcp }) => {+ const root = await createTestTempDirectory("rea-grpc-web-trailers-");+ for (const framing of ["grpc-web", "grpc-web-text"] as const)+ for (const [name, headerName, value, status] of [+ ["valid", "grpc-status", "0", "complete"],+ ["uppercase", "Grpc-Status", "0", "partial"],+ ["nul", "grpc-status", "0\u0000bad", "partial"],+ ["lf", "grpc-status", "0\nbad", "partial"],+ ["cr", "grpc-status", "0\rbad", "partial"],+ ["vt", "grpc-status", "0\u000bbad", "partial"],+ ["del", "grpc-status", "0\u007fbad", "partial"],+ ] as const) {+ const path = join(root, `${framing}-${name}.har`);+ const trailer = Buffer.from(`${headerName}: ${value}\r\n`);+ const body = Buffer.concat([+ protobufFrame(Buffer.from([8, 1])),+ protobufFrame(trailer, 128),+ ]);+ const original = protobufHar(+ framing === "grpc-web-text"+ ? Buffer.from(body.toString("base64"))+ : body,+ `application/${framing}`,+ );+ await writeFile(path, original);+ const response = await cli.run({+ arguments: [+ "inspect-web-network-capture",+ path,+ "har",+ "--record",+ "0",+ "--protobuf-framing",+ "auto",+ "--json",+ ],+ });+ expect(response.exitCode, response.stdout + response.stderr).toBe(0);+ const evidence = parseEvidence(response.json);+ const report = webNetworkCaptureSchema.parse(+ evidence.normalized_result,+ );+ expect(report.protobuf_coverage?.status, name).toBe(status);+ expect(report.records[0]?.protobuf?.[0]).toMatchObject({+ status,+ framing,+ examined_bytes: body.length,+ frames: [+ { kind: "data", status: "complete", examined_bytes: 2 },+ { kind: "trailer", status, examined_bytes: trailer.length },+ ],+ fields: [{ number: 1, wire_type: "varint", value: "1" }],+ });+ const reply = await mcp.call("inspect_web_network_capture", {+ capture_path: path,+ format: "har",+ record_ordinals: [0],+ protobuf: { framing: "auto" },+ });+ expect(reply.isError, JSON.stringify(reply)).not.toBe(true);+ expect(reply.structuredContent).toEqual(response.json);+ expect(await readFile(path, "utf8")).toBe(original);+ }+ },+);