Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

feat(protocol): add gRPC/Protobuf and JSON-RPC/MessagePack capture

morluto3f336fefb132
diff --git a/src/domain/protocolCapture.ts b/src/domain/protocolCapture.ts
new file mode 100644
index 000000000..9c6dedd2c
--- /dev/null
+++ b/src/domain/protocolCapture.ts
@@ -0,0 +1,272 @@
+import { z } from "zod";
+
+import { jsonValueSchema } from "./jsonValue.js";
+
+/** Supported protocol families for structured protocol capture. */
+export const protocolFamilySchema = z.enum([
+ "grpc",
+ "protobuf",
+ "json-rpc",
+ "messagepack",
+]);
+export type ProtocolFamily = z.infer<typeof protocolFamilySchema>;
+
+/** A decoded field from a protocol message. */
+export const decodedFieldSchema = z.strictObject({
+ /** Field name or path within the message structure. */
+ path: z.string().min(1),
+ /** Wire type or encoding hint. */
+ wire_type: z.string().nullable(),
+ /** Decoded field value. */
+ value: jsonValueSchema.nullable(),
+ /** Whether this field value was inferred rather than decoded. */
+ inferred: z.boolean().default(false),
+});
+export type DecodedField = z.infer<typeof decodedFieldSchema>;
+
+/** Schema inference hypothesis for a repeated message pattern. */
+export const schemaHypothesisSchema = z.strictObject({
+ /** Hypothesized field name. */
+ field_name: z.string().min(1),
+ /** Inferred type. */
+ inferred_type: z.string().min(1),
+ /** Confidence: 0-1 */
+ confidence: z.number().min(0).max(1),
+ /** Supporting evidence count. */
+ evidence_count: z.number().int().nonnegative(),
+});
+export type SchemaHypothesis = z.infer<typeof schemaHypothesisSchema>;
+
+/** Result of decoding a single protocol message. */
+export const protocolMessageSchema = z.strictObject({
+ /** Monotonic sequence number. */
+ sequence: z.number().int().nonnegative(),
+ /** Timestamp in milliseconds. */
+ at_ms: z.number().int().nonnegative(),
+ /** Protocol family. */
+ family: protocolFamilySchema,
+ /** Direction of the message. */
+ direction: z.enum(["request", "response"]),
+ /** Transport endpoint or path (e.g. gRPC method path). */
+ endpoint: z.string().nullable(),
+ /** Content type of the raw payload. */
+ content_type: z.string().nullable(),
+ /** Raw payload bytes as base64. */
+ raw_payload: z.string().nullable(),
+ /** Decoded fields from the payload. */
+ decoded_fields: z.array(decodedFieldSchema).default([]),
+ /** Inferred schema hypotheses. */
+ schema_hypotheses: z.array(schemaHypothesisSchema).default([]),
+ /** Whether decoding was truncated. */
+ truncated: z.boolean().default(false),
+ /** Whether credentials were detected and redacted. */
+ credentials_redacted: z.boolean().default(false),
+});
+export type ProtocolMessage = z.infer<typeof protocolMessageSchema>;
+
+/** A captured protocol session with messages and inferred schemas. */
+export const protocolCaptureSchema = z.strictObject({
+ /** Protocol family for this capture. */
+ family: protocolFamilySchema,
+ /** Captured messages in order. */
+ messages: z.array(protocolMessageSchema).min(0).max(10_000),
+ /** Aggregated schema hypotheses across all messages. */
+ inferred_schema: z.array(schemaHypothesisSchema).default([]),
+ /** Whether any message was truncated. */
+ has_truncated: z.boolean().default(false),
+ /** Whether credentials were detected and redacted. */
+ credentials_detected: z.boolean().default(false),
+});
+export type ProtocolCapture = z.infer<typeof protocolCaptureSchema>;
+
+/** Classification of a byte sequence as a known protocol family. */
+export function classifyProtocolFamily(
+ contentType: string | null,
+ path: string | null,
+): ProtocolFamily | null {
+ if (contentType) {
+ if (contentType.includes("application/grpc")) return "grpc";
+ if (contentType.includes("application/x-protobuf")) return "protobuf";
+ if (contentType.includes("application/json-rpc")) return "json-rpc";
+ if (
+ contentType.includes("application/msgpack") ||
+ contentType.includes("application/x-msgpack")
+ )
+ return "messagepack";
+ }
+ if (path) {
+ if (path.startsWith("/grpc.") || path.includes("/grpc/")) return "grpc";
+ if (path.includes("/json-rpc") || path.includes("/jsonrpc"))
+ return "json-rpc";
+ }
+ return null;
+}
+
+/** Credential-like field names that should be redacted. */
+const CREDENTIAL_PATTERNS = [
+ "password",
+ "passwd",
+ "secret",
+ "token",
+ "api_key",
+ "apikey",
+ "authorization",
+ "auth",
+ "credential",
+ "private_key",
+];
+
+/** Check if a field path looks like it may contain credentials. */
+export function looksLikeCredential(path: string): boolean {
+ const lower = path.toLowerCase();
+ return CREDENTIAL_PATTERNS.some((p) => lower.includes(p));
+}
+
+/** Decode a JSON-RPC message from raw payload bytes. */
+export function decodeJsonRpc(rawPayload: Uint8Array): {
+ decoded_fields: DecodedField[];
+ truncated: boolean;
+} {
+ const text = new TextDecoder().decode(rawPayload);
+ const parsed = JSON.parse(text);
+ const fields: DecodedField[] = [];
+
+ if (typeof parsed === "object" && parsed !== null) {
+ for (const [key, value] of Object.entries(parsed)) {
+ if (looksLikeCredential(key)) {
+ fields.push({
+ path: key,
+ wire_type: "json",
+ value: "[REDACTED]",
+ inferred: false,
+ });
+ } else {
+ fields.push({
+ path: key,
+ wire_type: "json",
+ value: value as never,
+ inferred: false,
+ });
+ }
+ }
+ }
+
+ return { decoded_fields: fields, truncated: false };
+}
+
+/** Decode a MessagePack message (simplified: only checks basic structure). */
+export function decodeMessagePack(rawPayload: Uint8Array): {
+ decoded_fields: DecodedField[];
+ truncated: boolean;
+} {
+ const fields: DecodedField[] = [];
+ // MessagePack uses a single-byte type tag
+ if (rawPayload.length === 0) {
+ return { decoded_fields: [], truncated: false };
+ }
+ const tag = rawPayload[0]!;
+ // Map type: 0x80-0x8f (fixmap), 0xde (map16), 0xdf (map32)
+ if ((tag >= 0x80 && tag <= 0x8f) || tag === 0xde || tag === 0xdf) {
+ fields.push({
+ path: "type",
+ wire_type: "msgpack",
+ value: "map",
+ inferred: false,
+ });
+ }
+ // Array type: 0x90-0x9f (fixarray), 0xdc (array16), 0xdd (array32)
+ else if ((tag >= 0x90 && tag <= 0x9f) || tag === 0xdc || tag === 0xdd) {
+ fields.push({
+ path: "type",
+ wire_type: "msgpack",
+ value: "array",
+ inferred: false,
+ });
+ } else {
+ fields.push({
+ path: "type",
+ wire_type: "msgpack",
+ value: "unknown",
+ inferred: true,
+ });
+ }
+ return {
+ decoded_fields: fields,
+ truncated: rawPayload.length > 10_000,
+ };
+}
+
+/** Decode a gRPC frame (5-byte header + protobuf payload). */
+export function decodeGrpcFrame(rawPayload: Uint8Array): {
+ decoded_fields: DecodedField[];
+ truncated: boolean;
+} {
+ // gRPC uses a 5-byte prefix: 1 byte compressed flag + 4 byte length
+ if (rawPayload.length < 5) {
+ return {
+ decoded_fields: [
+ {
+ path: "error",
+ wire_type: "grpc",
+ value: "frame too short",
+ inferred: false,
+ },
+ ],
+ truncated: false,
+ };
+ }
+ const compressed = rawPayload[0] !== 0;
+ const length =
+ (rawPayload[1]! << 24) |
+ (rawPayload[2]! << 16) |
+ (rawPayload[3]! << 8) |
+ rawPayload[4]!;
+ return {
+ decoded_fields: [
+ {
+ path: "compressed",
+ wire_type: "grpc",
+ value: compressed,
+ inferred: false,
+ },
+ {
+ path: "message_length",
+ wire_type: "grpc",
+ value: length,
+ inferred: false,
+ },
+ ],
+ truncated: length > 10_000,
+ };
+}
+
+/** Infer schema hypotheses from a set of decoded messages. */
+export function inferMessageSchema(
+ messages: readonly ProtocolMessage[],
+): SchemaHypothesis[] {
+ const fieldCounts = new Map<string, { count: number; types: Set<string> }>();
+ for (const msg of messages) {
+ for (const field of msg.decoded_fields) {
+ const existing = fieldCounts.get(field.path);
+ if (existing) {
+ existing.count++;
+ if (field.wire_type) existing.types.add(field.wire_type);
+ } else {
+ fieldCounts.set(field.path, {
+ count: 1,
+ types: new Set(field.wire_type ?? "unknown"),
+ });
+ }
+ }
+ }
+ const hypotheses: SchemaHypothesis[] = [];
+ for (const [field_name, data] of fieldCounts) {
+ hypotheses.push({
+ field_name,
+ inferred_type: [...data.types].join("|"),
+ confidence: Math.min(1, data.count / messages.length || 0),
+ evidence_count: data.count,
+ });
+ }
+ return hypotheses;
+}
diff --git a/tests/protocolCapture.test.ts b/tests/protocolCapture.test.ts
new file mode 100644
index 000000000..23dac0e3a
--- /dev/null
+++ b/tests/protocolCapture.test.ts
@@ -0,0 +1,207 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ classifyProtocolFamily,
+ decodeGrpcFrame,
+ decodeJsonRpc,
+ decodeMessagePack,
+ inferMessageSchema,
+ looksLikeCredential,
+ protocolCaptureSchema,
+ type ProtocolMessage,
+ type ProtocolCapture,
+} from "../src/domain/protocolCapture.js";
+
+describe("protocol capture", () => {
+ it("classifies gRPC from content type", () => {
+ expect(classifyProtocolFamily("application/grpc+proto", null)).toBe("grpc");
+ });
+
+ it("classifies protobuf from content type", () => {
+ expect(classifyProtocolFamily("application/x-protobuf", null)).toBe(
+ "protobuf",
+ );
+ });
+
+ it("classifies json-rpc from content type", () => {
+ expect(classifyProtocolFamily("application/json-rpc", null)).toBe(
+ "json-rpc",
+ );
+ });
+
+ it("classifies msgpack from content type", () => {
+ expect(classifyProtocolFamily("application/msgpack", null)).toBe(
+ "messagepack",
+ );
+ });
+
+ it("classifies from path when content type is null", () => {
+ expect(classifyProtocolFamily(null, "/some/grpc/method")).toBe("grpc");
+ expect(classifyProtocolFamily(null, "/api/json-rpc/2.0")).toBe("json-rpc");
+ });
+
+ it("returns null for unknown protocols", () => {
+ expect(classifyProtocolFamily("text/plain", null)).toBeNull();
+ expect(classifyProtocolFamily(null, null)).toBeNull();
+ });
+
+ it("detects credential-like field names", () => {
+ expect(looksLikeCredential("password")).toBe(true);
+ expect(looksLikeCredential("api_key")).toBe(true);
+ expect(looksLikeCredential("authorization")).toBe(true);
+ expect(looksLikeCredential("username")).toBe(false);
+ });
+});
+
+describe("JSON-RPC decoding", () => {
+ it("decodes a simple JSON-RPC request", () => {
+ const payload = new TextEncoder().encode(
+ JSON.stringify({
+ jsonrpc: "2.0",
+ method: "add",
+ params: [1, 2],
+ id: 1,
+ }),
+ );
+ const result = decodeJsonRpc(payload);
+ expect(result.decoded_fields.length).toBe(4);
+ expect(result.truncated).toBe(false);
+ });
+
+ it("redacts credential-like fields", () => {
+ const payload = new TextEncoder().encode(
+ JSON.stringify({
+ password: "secret123",
+ username: "admin",
+ }),
+ );
+ const result = decodeJsonRpc(payload);
+ const passwordField = result.decoded_fields.find(
+ (f) => f.path === "password",
+ );
+ expect(passwordField?.value).toBe("[REDACTED]");
+ const usernameField = result.decoded_fields.find(
+ (f) => f.path === "username",
+ );
+ expect(usernameField?.value).toBe("admin");
+ });
+});
+
+describe("MessagePack decoding", () => {
+ it("detects map type", () => {
+ const payload = new Uint8Array([0x82]);
+ const result = decodeMessagePack(payload);
+ expect(result.decoded_fields[0]?.value).toBe("map");
+ });
+
+ it("detects array type", () => {
+ const payload = new Uint8Array([0x92]);
+ const result = decodeMessagePack(payload);
+ expect(result.decoded_fields[0]?.value).toBe("array");
+ });
+
+ it("reports unknown type for other tags", () => {
+ const payload = new Uint8Array([0xff]);
+ const result = decodeMessagePack(payload);
+ expect(result.decoded_fields[0]?.value).toBe("unknown");
+ expect(result.decoded_fields[0]?.inferred).toBe(true);
+ });
+});
+
+describe("gRPC frame decoding", () => {
+ it("decodes a valid gRPC frame header", () => {
+ const payload = new Uint8Array([
+ 0x00, 0x00, 0x00, 0x00, 0x05, 1, 2, 3, 4, 5,
+ ]);
+ const result = decodeGrpcFrame(payload);
+ expect(result.decoded_fields).toHaveLength(2);
+ expect(result.decoded_fields[0]?.path).toBe("compressed");
+ expect(result.decoded_fields[0]?.value).toBe(false);
+ expect(result.decoded_fields[1]?.path).toBe("message_length");
+ expect(result.decoded_fields[1]?.value).toBe(5);
+ });
+
+ it("detects compressed flag", () => {
+ const payload = new Uint8Array([0x01, 0x00, 0x00, 0x00, 0x0a]);
+ const result = decodeGrpcFrame(payload);
+ expect(result.decoded_fields[0]?.value).toBe(true);
+ });
+
+ it("handles too-short frames", () => {
+ const payload = new Uint8Array([0x00, 0x01]);
+ const result = decodeGrpcFrame(payload);
+ expect(result.decoded_fields[0]?.path).toBe("error");
+ });
+});
+
+describe("schema inference", () => {
+ it("infers schema from repeated messages", () => {
+ const messages: ProtocolMessage[] = [
+ {
+ sequence: 0,
+ at_ms: 0,
+ family: "json-rpc",
+ direction: "request",
+ endpoint: "/api",
+ content_type: "application/json",
+ raw_payload: null,
+ decoded_fields: [
+ { path: "method", wire_type: "json", value: "add", inferred: false },
+ { path: "id", wire_type: "json", value: 1, inferred: false },
+ ],
+ schema_hypotheses: [],
+ truncated: false,
+ credentials_redacted: false,
+ },
+ {
+ sequence: 1,
+ at_ms: 10,
+ family: "json-rpc",
+ direction: "response",
+ endpoint: "/api",
+ content_type: "application/json",
+ raw_payload: null,
+ decoded_fields: [
+ { path: "result", wire_type: "json", value: 3, inferred: false },
+ { path: "id", wire_type: "json", value: 1, inferred: false },
+ ],
+ schema_hypotheses: [],
+ truncated: false,
+ credentials_redacted: false,
+ },
+ ];
+ const hypotheses = inferMessageSchema(messages);
+ expect(hypotheses).toHaveLength(3);
+ const idHypothesis = hypotheses.find((h) => h.field_name === "id");
+ expect(idHypothesis).toBeDefined();
+ expect(idHypothesis!.evidence_count).toBe(2);
+ expect(idHypothesis!.confidence).toBe(1);
+ });
+});
+
+describe("uses ProtocolCapture type", () => {
+ it("validates a well-formed capture typed value", () => {
+ const capture: ProtocolCapture = {
+ family: "grpc",
+ messages: [],
+ inferred_schema: [],
+ has_truncated: false,
+ credentials_detected: false,
+ };
+ expect(capture.family).toBe("grpc");
+ });
+});
+
+describe("protocol capture schema validation", () => {
+ it("validates a well-formed capture", () => {
+ const capture = {
+ family: "grpc" as const,
+ messages: [],
+ inferred_schema: [],
+ has_truncated: false,
+ credentials_detected: false,
+ };
+ const result = protocolCaptureSchema.safeParse(capture);
+ expect(result.success).toBe(true);
+ });
+});