Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(protobuf): preserve partial facts for malformed web trailers

morluto9cf7a7cb0d3c
diff --git a/src/browser/history/protobuf/ProtobufBody.ts b/src/browser/history/protobuf/ProtobufBody.ts
index 7d040fd9c..eb3d91777 100644
--- a/src/browser/history/protobuf/ProtobufBody.ts
+++ b/src/browser/history/protobuf/ProtobufBody.ts
@@ -3,7 +3,7 @@ import {
prepareProtobufBody,
type PreparedProtobufBody,
} from "./ProtobufFraming.js";
-import { CaptureFormatError } from "../CaptureFormatError.js";
+import { parseGrpcWebTrailers } from "./ProtobufTrailers.js";
import type {
ProtobufDecoding,
ProtobufFrame,
@@ -155,33 +155,22 @@ export const decodeProtobufBody = async (
};
} else {
budget.charge(length * 8);
- try {
- const text = new TextDecoder("utf-8", {
- fatal: true,
- ignoreBOM: true,
- }).decode(data.subarray(at + 5, at + 5 + length));
- frame.trailers = [];
- for (const line of text.split("\r\n")) {
- if (line === "") continue;
- const colon = line.indexOf(":");
- if (
- colon <= 0 ||
- !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(line.slice(0, colon))
- )
- throw new Error("Malformed trailer header");
- budget.charge(512);
- frame.trailers.push({
- name: redact(line.slice(0, colon)),
- value: redact(line.slice(colon + 1).replace(/^[ \t]+/, "")),
- });
- }
- frame.examined_bytes = length;
- } catch (cause) {
- if (cause instanceof CaptureFormatError) throw cause;
+ const parsed = parseGrpcWebTrailers(
+ data.subarray(at + 5, at + 5 + length),
+ budget,
+ );
+ const facts = parsed.ok ? parsed.value : parsed.error;
+ frame.examined_bytes = facts.examined_bytes;
+ if (parsed.ok || parsed.error.kind !== "invalid-utf8")
+ frame.trailers = facts.trailers.map(({ name, value }) => ({
+ name: redact(name),
+ value: redact(value),
+ }));
+ if (!parsed.ok) {
frame.status = "partial";
frame.error = {
offset: at + 5,
- reason: "Invalid gRPC-web trailer header block",
+ reason: parsed.error.reason,
};
}
}
diff --git a/src/browser/history/protobuf/ProtobufBodyTrailer.test.ts b/src/browser/history/protobuf/ProtobufBodyTrailer.test.ts
new file mode 100644
index 000000000..71dbb3030
--- /dev/null
+++ b/src/browser/history/protobuf/ProtobufBodyTrailer.test.ts
@@ -0,0 +1,111 @@
+import { expect, it } from "vitest";
+import { decodeProtobufBody } from "./ProtobufBody.js";
+import { ProtobufBudget } from "./ProtobufWire.js";
+
+const source = {
+ pointer: "/response/content/text",
+ content_type: "application/grpc-web+proto",
+ body_sha256: "a".repeat(64),
+};
+const same = (value: string) => value;
+const trailerFrame = (text: string) => {
+ const payload = Buffer.from(text, "utf8");
+ const frame = Buffer.alloc(5 + payload.length);
+ frame[0] = 128;
+ frame.writeUInt32BE(payload.length, 1);
+ payload.copy(frame, 5);
+ return frame;
+};
+
+it("rejects forbidden trailer controls while retaining valid earlier trailers", async () => {
+ for (const code of [
+ ...Array.from({ length: 32 }, (_, index) => index),
+ 127,
+ ]) {
+ if (code === 9) continue;
+ const control = String.fromCharCode(code);
+ const text = `grpc-message: kept\r\ngrpc-status: 0${control}bad\r\n`;
+ const input = trailerFrame(text);
+ const result = await decodeProtobufBody(
+ input,
+ "grpc-web",
+ source,
+ same,
+ new ProtobufBudget(),
+ );
+ expect(result.status, JSON.stringify(control)).toBe("partial");
+ expect(result.frames[0]?.error).toMatchObject({ offset: 5 });
+ expect(result.examined_bytes).toBe(5 + Buffer.byteLength(text));
+ expect(result.frames[0]).toMatchObject({
+ status: "partial",
+ examined_bytes: Buffer.byteLength(text),
+ });
+ expect(result.frames[0]?.trailers).toEqual([
+ { name: "grpc-message", value: "kept" },
+ ]);
+ expect(input.subarray(5)).toEqual(Buffer.from(text, "utf8"));
+ }
+});
+
+it("retains allowed whitespace, visible text, and valid UTF-8 obs-text", async () => {
+ const text = "grpc-message: alpha\tbeta \u00e9\r\n";
+ const result = await decodeProtobufBody(
+ trailerFrame(text),
+ "grpc-web",
+ source,
+ same,
+ new ProtobufBudget(),
+ );
+ expect(result.status).toBe("complete");
+ expect(result.frames[0]?.trailers).toEqual([
+ { name: "grpc-message", value: "alpha\tbeta \u00e9" },
+ ]);
+ expect(result.examined_bytes).toBe(5 + Buffer.byteLength(text));
+});
+
+it.each(["Grpc-Status", "GRPC-STATUS"])(
+ "retains a valid prefix when the trailer name %s violates gRPC-Web casing",
+ async (name) => {
+ const text = `grpc-message: kept\r\n${name}: 0\r\n`;
+ const result = await decodeProtobufBody(
+ trailerFrame(text),
+ "grpc-web",
+ source,
+ same,
+ new ProtobufBudget(),
+ );
+ expect(result.status).toBe("partial");
+ expect(result.examined_bytes).toBe(5 + Buffer.byteLength(text));
+ expect(result.frames[0]?.trailers).toEqual([
+ { name: "grpc-message", value: "kept" },
+ ]);
+ },
+);
+
+it("leaves undecodable trailer fields unknown and preserves unexpected failures", async () => {
+ const input = trailerFrame("grpc-status: 0\r\n");
+ input[5] = 255;
+ const malformed = await decodeProtobufBody(
+ input,
+ "grpc-web",
+ source,
+ same,
+ new ProtobufBudget(),
+ );
+ expect(malformed.status).toBe("partial");
+ expect(malformed.frames[0]?.examined_bytes).toBe(0);
+ expect(malformed.frames[0]?.trailers).toBeUndefined();
+
+ const failure = { kind: "controlled-redaction-failure" };
+ await expect(
+ decodeProtobufBody(
+ trailerFrame("grpc-status: 0\r\n"),
+ "grpc-web",
+ source,
+ () => {
+ throw failure;
+ },
+ new ProtobufBudget(),
+ ),
+ ).rejects.toBe(failure);
+});
diff --git a/src/browser/history/protobuf/ProtobufCapture.test.ts b/src/browser/history/protobuf/ProtobufCapture.test.ts
index 7e711471b..a0d7e6d60 100644
--- a/src/browser/history/protobuf/ProtobufCapture.test.ts
+++ b/src/browser/history/protobuf/ProtobufCapture.test.ts
@@ -10,47 +10,78 @@ import {
decodeCaptureProtobuf,
} from "./ProtobufCapture.js";
-it("restricts framing overrides to native body facts while preserving header bytes", async () => {
- const binary = (pointer: string, bytes: Buffer) => ({
- pointer,
- representation: "producer-bytes" as const,
- state: "retained" as const,
- bytes: bytes.length,
- sha256: createHash("sha256").update(bytes).digest("hex"),
- content_base64: bytes.toString("base64"),
- });
- const headers = [
- binary("/response/headers/0/0", Buffer.from("content-type")),
- binary("/response/headers/0/1", Buffer.from("application/grpc")),
- ];
- const report: WebNetworkCapture = {
- artifact: { path: "/fixture.mitm", bytes: 100, sha256: "a".repeat(64) },
- decoder: { id: "fixture", name: "Native producer facts", version: "1" },
- container: {
- reported: null,
+const binary = (pointer: string, bytes: Buffer) => ({
+ pointer,
+ representation: "producer-bytes" as const,
+ state: "retained" as const,
+ bytes: bytes.length,
+ sha256: createHash("sha256").update(bytes).digest("hex"),
+ content_base64: bytes.toString("base64"),
+});
+
+const captureWithBody = (
+ contentType: string,
+ body: Buffer,
+): WebNetworkCapture => ({
+ artifact: { path: "/fixture.mitm", bytes: 100, sha256: "a".repeat(64) },
+ decoder: { id: "fixture", name: "Native producer facts", version: "1" },
+ container: {
+ reported: null,
+ numeric_literals: [],
+ redactions: [],
+ records_pointer: "/flows",
+ },
+ format: "mitmproxy",
+ total_records: 1,
+ runtime_attribution: "unknown",
+ limitations: [],
+ records: [
+ {
+ ordinal: 0,
+ location: { kind: "byte-range", offset: 0, bytes: 100 },
+ reported: { response: { headers: [[null, null]], content: null } },
+ binary_fields: [
+ binary("/response/headers/0/0", Buffer.from("content-type")),
+ binary("/response/headers/0/1", Buffer.from(contentType)),
+ binary("/response/content", body),
+ ],
numeric_literals: [],
redactions: [],
- records_pointer: "/flows",
+ limitations: [],
},
- format: "mitmproxy",
- total_records: 1,
- runtime_attribution: "unknown",
- limitations: [],
- records: [
- {
- ordinal: 0,
- location: { kind: "byte-range", offset: 0, bytes: 100 },
- reported: { response: { headers: [[null, null]], content: null } },
- binary_fields: [
- ...headers,
- binary("/response/content", Buffer.from([0, 0, 0, 0, 2, 8, 1])),
- ],
- numeric_literals: [],
- redactions: [],
- limitations: [],
- },
- ],
- };
+ ],
+});
+
+const decode = async (capture: WebNetworkCapture) => {
+ const input = inspectWebNetworkCaptureInputSchema.parse({
+ capture_path: capture.artifact.path,
+ format: capture.format,
+ record_ordinals: [0],
+ });
+ const resources = new ArtifactResourceScope();
+ try {
+ const prepared = await prepareCaptureProtobuf(capture, input);
+ await decodeCaptureProtobuf(capture, input, resources, prepared);
+ } finally {
+ await resources.close();
+ }
+};
+
+const grpcWebTrailerFrame = (text: Buffer) => {
+ const frame = Buffer.alloc(5 + text.length);
+ frame[0] = 128;
+ frame.writeUInt32BE(text.length, 1);
+ text.copy(frame, 5);
+ return frame;
+};
+
+it("restricts framing overrides to native body facts while preserving header bytes", async () => {
+ const payload = Buffer.from([8, 1]);
+ const body = Buffer.alloc(5 + payload.length);
+ body.writeUInt32BE(payload.length, 1);
+ payload.copy(body, 5);
+ const report = captureWithBody("application/grpc", body);
+ const headers = report.records[0]!.binary_fields.slice(0, 2);
const input = inspectWebNetworkCaptureInputSchema.parse({
capture_path: report.artifact.path,
format: report.format,
@@ -75,3 +106,20 @@ it("restricts framing overrides to native body facts while preserving header byt
expect(report.records[0]?.binary_fields.slice(0, 2)).toEqual(headers);
expect(report.protobuf_coverage?.status).toBe("complete");
});
+
+it("keeps capture coverage partial for a malformed gRPC-web trailer", async () => {
+ const trailerText = Buffer.from("grpc-status: 0\u0000bad\r\n");
+ const body = grpcWebTrailerFrame(trailerText);
+ const report = captureWithBody("application/grpc-web", body);
+ const retainedBody = report.records[0]!.binary_fields[2];
+ await decode(report);
+
+ expect(report.records[0]?.protobuf?.[0]).toMatchObject({
+ framing: "grpc-web",
+ status: "partial",
+ examined_bytes: body.length,
+ frames: [{ status: "partial", examined_bytes: trailerText.length }],
+ });
+ expect(report.protobuf_coverage?.status).toBe("partial");
+ expect(report.records[0]?.binary_fields[2]).toEqual(retainedBody);
+});
diff --git a/src/browser/history/protobuf/ProtobufTrailers.ts b/src/browser/history/protobuf/ProtobufTrailers.ts
new file mode 100644
index 000000000..040923202
--- /dev/null
+++ b/src/browser/history/protobuf/ProtobufTrailers.ts
@@ -0,0 +1,60 @@
+import { err, ok, type Result } from "../../../domain/result.js";
+import type { ProtobufBudget } from "./ProtobufWire.js";
+
+interface TrailerFacts {
+ readonly trailers: { name: string; value: string }[];
+ readonly examined_bytes: number;
+}
+
+type TrailerFailure = TrailerFacts & {
+ readonly kind: "invalid-utf8" | "invalid-name" | "invalid-value";
+ readonly reason: string;
+};
+
+/** Parse the gRPC-Web HTTP/1 field block without discarding its valid prefix. */
+export const parseGrpcWebTrailers = (
+ bytes: Uint8Array,
+ budget: ProtobufBudget,
+): Result<TrailerFacts, TrailerFailure> => {
+ let text: string;
+ try {
+ text = new TextDecoder("utf-8", {
+ fatal: true,
+ ignoreBOM: true,
+ }).decode(bytes);
+ } catch (cause) {
+ if (!(cause instanceof TypeError)) throw cause;
+ return err({
+ kind: "invalid-utf8",
+ reason: "gRPC-web trailer header block is not valid UTF-8",
+ trailers: [],
+ examined_bytes: 0,
+ });
+ }
+ // The whole UTF-8 block has been examined even when field validation stops.
+ const facts: TrailerFacts = { trailers: [], examined_bytes: bytes.length };
+ for (const line of text.split("\r\n")) {
+ if (line === "") continue;
+ const colon = line.indexOf(":");
+ // Unlike ordinary HTTP/1 headers, in-body gRPC-Web names must be lower-case.
+ if (colon <= 0 || !/^[!#$%&'*+.^_`|~0-9a-z-]+$/.test(line.slice(0, colon)))
+ return err({
+ ...facts,
+ kind: "invalid-name",
+ reason:
+ "gRPC-web trailer header names must be lower-case HTTP field names",
+ });
+ const value = line.slice(colon + 1).replace(/^[ \t]+/, "");
+ // RFC 9110 permits HTAB but excludes other C0 controls and DEL.
+ if (/[\u0000-\u0008\u000a-\u001f\u007f]/u.test(value))
+ return err({
+ ...facts,
+ kind: "invalid-value",
+ reason:
+ "gRPC-web trailer field value contains a forbidden control character",
+ });
+ budget.charge(512);
+ facts.trailers.push({ name: line.slice(0, colon), value });
+ }
+ return ok(facts);
+};
diff --git a/tests/acceptance/tools/protobufCapture.test.ts b/tests/acceptance/tools/protobufCapture.test.ts
index 921c1b13d..10e6b821c 100644
--- a/tests/acceptance/tools/protobufCapture.test.ts
+++ b/tests/acceptance/tools/protobufCapture.test.ts
@@ -556,3 +556,71 @@ captureTest(
});
},
);
+
+captureTest(
+ "keeps malformed gRPC-web trailers partial through CLI and MCP without changing capture bytes",
+ async ({ cli, mcp }) => {
+ const root = await createTestTempDirectory("rea-grpc-web-trailers-");
+ for (const framing of ["grpc-web", "grpc-web-text"] as const)
+ for (const [name, headerName, value, status] of [
+ ["valid", "grpc-status", "0", "complete"],
+ ["uppercase", "Grpc-Status", "0", "partial"],
+ ["nul", "grpc-status", "0\u0000bad", "partial"],
+ ["lf", "grpc-status", "0\nbad", "partial"],
+ ["cr", "grpc-status", "0\rbad", "partial"],
+ ["vt", "grpc-status", "0\u000bbad", "partial"],
+ ["del", "grpc-status", "0\u007fbad", "partial"],
+ ] as const) {
+ const path = join(root, `${framing}-${name}.har`);
+ const trailer = Buffer.from(`${headerName}: ${value}\r\n`);
+ const body = Buffer.concat([
+ protobufFrame(Buffer.from([8, 1])),
+ protobufFrame(trailer, 128),
+ ]);
+ const original = protobufHar(
+ framing === "grpc-web-text"
+ ? Buffer.from(body.toString("base64"))
+ : body,
+ `application/${framing}`,
+ );
+ await writeFile(path, original);
+ const response = await cli.run({
+ arguments: [
+ "inspect-web-network-capture",
+ path,
+ "har",
+ "--record",
+ "0",
+ "--protobuf-framing",
+ "auto",
+ "--json",
+ ],
+ });
+ expect(response.exitCode, response.stdout + response.stderr).toBe(0);
+ const evidence = parseEvidence(response.json);
+ const report = webNetworkCaptureSchema.parse(
+ evidence.normalized_result,
+ );
+ expect(report.protobuf_coverage?.status, name).toBe(status);
+ expect(report.records[0]?.protobuf?.[0]).toMatchObject({
+ status,
+ framing,
+ examined_bytes: body.length,
+ frames: [
+ { kind: "data", status: "complete", examined_bytes: 2 },
+ { kind: "trailer", status, examined_bytes: trailer.length },
+ ],
+ fields: [{ number: 1, wire_type: "varint", value: "1" }],
+ });
+ const reply = await mcp.call("inspect_web_network_capture", {
+ capture_path: path,
+ format: "har",
+ record_ordinals: [0],
+ protobuf: { framing: "auto" },
+ });
+ expect(reply.isError, JSON.stringify(reply)).not.toBe(true);
+ expect(reply.structuredContent).toEqual(response.json);
+ expect(await readFile(path, "utf8")).toBe(original);
+ }
+ },
+);