Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
feat(protocol): add gRPC/Protobuf and JSON-RPC/MessagePack capture
morluto3f336fefb132
diff --git a/src/domain/protocolCapture.ts b/src/domain/protocolCapture.tsnew file mode 100644index 000000000..9c6dedd2c--- /dev/null+++ b/src/domain/protocolCapture.ts@@ -0,0 +1,272 @@+import { z } from "zod";++import { jsonValueSchema } from "./jsonValue.js";++/** Supported protocol families for structured protocol capture. */+export const protocolFamilySchema = z.enum([+ "grpc",+ "protobuf",+ "json-rpc",+ "messagepack",+]);+export type ProtocolFamily = z.infer<typeof protocolFamilySchema>;++/** A decoded field from a protocol message. */+export const decodedFieldSchema = z.strictObject({+ /** Field name or path within the message structure. */+ path: z.string().min(1),+ /** Wire type or encoding hint. */+ wire_type: z.string().nullable(),+ /** Decoded field value. */+ value: jsonValueSchema.nullable(),+ /** Whether this field value was inferred rather than decoded. */+ inferred: z.boolean().default(false),+});+export type DecodedField = z.infer<typeof decodedFieldSchema>;++/** Schema inference hypothesis for a repeated message pattern. */+export const schemaHypothesisSchema = z.strictObject({+ /** Hypothesized field name. */+ field_name: z.string().min(1),+ /** Inferred type. */+ inferred_type: z.string().min(1),+ /** Confidence: 0-1 */+ confidence: z.number().min(0).max(1),+ /** Supporting evidence count. */+ evidence_count: z.number().int().nonnegative(),+});+export type SchemaHypothesis = z.infer<typeof schemaHypothesisSchema>;++/** Result of decoding a single protocol message. */+export const protocolMessageSchema = z.strictObject({+ /** Monotonic sequence number. */+ sequence: z.number().int().nonnegative(),+ /** Timestamp in milliseconds. */+ at_ms: z.number().int().nonnegative(),+ /** Protocol family. */+ family: protocolFamilySchema,+ /** Direction of the message. */+ direction: z.enum(["request", "response"]),+ /** Transport endpoint or path (e.g. gRPC method path). */+ endpoint: z.string().nullable(),+ /** Content type of the raw payload. */+ content_type: z.string().nullable(),+ /** Raw payload bytes as base64. */+ raw_payload: z.string().nullable(),+ /** Decoded fields from the payload. */+ decoded_fields: z.array(decodedFieldSchema).default([]),+ /** Inferred schema hypotheses. */+ schema_hypotheses: z.array(schemaHypothesisSchema).default([]),+ /** Whether decoding was truncated. */+ truncated: z.boolean().default(false),+ /** Whether credentials were detected and redacted. */+ credentials_redacted: z.boolean().default(false),+});+export type ProtocolMessage = z.infer<typeof protocolMessageSchema>;++/** A captured protocol session with messages and inferred schemas. */+export const protocolCaptureSchema = z.strictObject({+ /** Protocol family for this capture. */+ family: protocolFamilySchema,+ /** Captured messages in order. */+ messages: z.array(protocolMessageSchema).min(0).max(10_000),+ /** Aggregated schema hypotheses across all messages. */+ inferred_schema: z.array(schemaHypothesisSchema).default([]),+ /** Whether any message was truncated. */+ has_truncated: z.boolean().default(false),+ /** Whether credentials were detected and redacted. */+ credentials_detected: z.boolean().default(false),+});+export type ProtocolCapture = z.infer<typeof protocolCaptureSchema>;++/** Classification of a byte sequence as a known protocol family. */+export function classifyProtocolFamily(+ contentType: string | null,+ path: string | null,+): ProtocolFamily | null {+ if (contentType) {+ if (contentType.includes("application/grpc")) return "grpc";+ if (contentType.includes("application/x-protobuf")) return "protobuf";+ if (contentType.includes("application/json-rpc")) return "json-rpc";+ if (+ contentType.includes("application/msgpack") ||+ contentType.includes("application/x-msgpack")+ )+ return "messagepack";+ }+ if (path) {+ if (path.startsWith("/grpc.") || path.includes("/grpc/")) return "grpc";+ if (path.includes("/json-rpc") || path.includes("/jsonrpc"))+ return "json-rpc";+ }+ return null;+}++/** Credential-like field names that should be redacted. */+const CREDENTIAL_PATTERNS = [+ "password",+ "passwd",+ "secret",+ "token",+ "api_key",+ "apikey",+ "authorization",+ "auth",+ "credential",+ "private_key",+];++/** Check if a field path looks like it may contain credentials. */+export function looksLikeCredential(path: string): boolean {+ const lower = path.toLowerCase();+ return CREDENTIAL_PATTERNS.some((p) => lower.includes(p));+}++/** Decode a JSON-RPC message from raw payload bytes. */+export function decodeJsonRpc(rawPayload: Uint8Array): {+ decoded_fields: DecodedField[];+ truncated: boolean;+} {+ const text = new TextDecoder().decode(rawPayload);+ const parsed = JSON.parse(text);+ const fields: DecodedField[] = [];++ if (typeof parsed === "object" && parsed !== null) {+ for (const [key, value] of Object.entries(parsed)) {+ if (looksLikeCredential(key)) {+ fields.push({+ path: key,+ wire_type: "json",+ value: "[REDACTED]",+ inferred: false,+ });+ } else {+ fields.push({+ path: key,+ wire_type: "json",+ value: value as never,+ inferred: false,+ });+ }+ }+ }++ return { decoded_fields: fields, truncated: false };+}++/** Decode a MessagePack message (simplified: only checks basic structure). */+export function decodeMessagePack(rawPayload: Uint8Array): {+ decoded_fields: DecodedField[];+ truncated: boolean;+} {+ const fields: DecodedField[] = [];+ // MessagePack uses a single-byte type tag+ if (rawPayload.length === 0) {+ return { decoded_fields: [], truncated: false };+ }+ const tag = rawPayload[0]!;+ // Map type: 0x80-0x8f (fixmap), 0xde (map16), 0xdf (map32)+ if ((tag >= 0x80 && tag <= 0x8f) || tag === 0xde || tag === 0xdf) {+ fields.push({+ path: "type",+ wire_type: "msgpack",+ value: "map",+ inferred: false,+ });+ }+ // Array type: 0x90-0x9f (fixarray), 0xdc (array16), 0xdd (array32)+ else if ((tag >= 0x90 && tag <= 0x9f) || tag === 0xdc || tag === 0xdd) {+ fields.push({+ path: "type",+ wire_type: "msgpack",+ value: "array",+ inferred: false,+ });+ } else {+ fields.push({+ path: "type",+ wire_type: "msgpack",+ value: "unknown",+ inferred: true,+ });+ }+ return {+ decoded_fields: fields,+ truncated: rawPayload.length > 10_000,+ };+}++/** Decode a gRPC frame (5-byte header + protobuf payload). */+export function decodeGrpcFrame(rawPayload: Uint8Array): {+ decoded_fields: DecodedField[];+ truncated: boolean;+} {+ // gRPC uses a 5-byte prefix: 1 byte compressed flag + 4 byte length+ if (rawPayload.length < 5) {+ return {+ decoded_fields: [+ {+ path: "error",+ wire_type: "grpc",+ value: "frame too short",+ inferred: false,+ },+ ],+ truncated: false,+ };+ }+ const compressed = rawPayload[0] !== 0;+ const length =+ (rawPayload[1]! << 24) |+ (rawPayload[2]! << 16) |+ (rawPayload[3]! << 8) |+ rawPayload[4]!;+ return {+ decoded_fields: [+ {+ path: "compressed",+ wire_type: "grpc",+ value: compressed,+ inferred: false,+ },+ {+ path: "message_length",+ wire_type: "grpc",+ value: length,+ inferred: false,+ },+ ],+ truncated: length > 10_000,+ };+}++/** Infer schema hypotheses from a set of decoded messages. */+export function inferMessageSchema(+ messages: readonly ProtocolMessage[],+): SchemaHypothesis[] {+ const fieldCounts = new Map<string, { count: number; types: Set<string> }>();+ for (const msg of messages) {+ for (const field of msg.decoded_fields) {+ const existing = fieldCounts.get(field.path);+ if (existing) {+ existing.count++;+ if (field.wire_type) existing.types.add(field.wire_type);+ } else {+ fieldCounts.set(field.path, {+ count: 1,+ types: new Set(field.wire_type ?? "unknown"),+ });+ }+ }+ }+ const hypotheses: SchemaHypothesis[] = [];+ for (const [field_name, data] of fieldCounts) {+ hypotheses.push({+ field_name,+ inferred_type: [...data.types].join("|"),+ confidence: Math.min(1, data.count / messages.length || 0),+ evidence_count: data.count,+ });+ }+ return hypotheses;+}diff --git a/tests/protocolCapture.test.ts b/tests/protocolCapture.test.tsnew file mode 100644index 000000000..23dac0e3a--- /dev/null+++ b/tests/protocolCapture.test.ts@@ -0,0 +1,207 @@+import { describe, expect, it } from "vitest";++import {+ classifyProtocolFamily,+ decodeGrpcFrame,+ decodeJsonRpc,+ decodeMessagePack,+ inferMessageSchema,+ looksLikeCredential,+ protocolCaptureSchema,+ type ProtocolMessage,+ type ProtocolCapture,+} from "../src/domain/protocolCapture.js";++describe("protocol capture", () => {+ it("classifies gRPC from content type", () => {+ expect(classifyProtocolFamily("application/grpc+proto", null)).toBe("grpc");+ });++ it("classifies protobuf from content type", () => {+ expect(classifyProtocolFamily("application/x-protobuf", null)).toBe(+ "protobuf",+ );+ });++ it("classifies json-rpc from content type", () => {+ expect(classifyProtocolFamily("application/json-rpc", null)).toBe(+ "json-rpc",+ );+ });++ it("classifies msgpack from content type", () => {+ expect(classifyProtocolFamily("application/msgpack", null)).toBe(+ "messagepack",+ );+ });++ it("classifies from path when content type is null", () => {+ expect(classifyProtocolFamily(null, "/some/grpc/method")).toBe("grpc");+ expect(classifyProtocolFamily(null, "/api/json-rpc/2.0")).toBe("json-rpc");+ });++ it("returns null for unknown protocols", () => {+ expect(classifyProtocolFamily("text/plain", null)).toBeNull();+ expect(classifyProtocolFamily(null, null)).toBeNull();+ });++ it("detects credential-like field names", () => {+ expect(looksLikeCredential("password")).toBe(true);+ expect(looksLikeCredential("api_key")).toBe(true);+ expect(looksLikeCredential("authorization")).toBe(true);+ expect(looksLikeCredential("username")).toBe(false);+ });+});++describe("JSON-RPC decoding", () => {+ it("decodes a simple JSON-RPC request", () => {+ const payload = new TextEncoder().encode(+ JSON.stringify({+ jsonrpc: "2.0",+ method: "add",+ params: [1, 2],+ id: 1,+ }),+ );+ const result = decodeJsonRpc(payload);+ expect(result.decoded_fields.length).toBe(4);+ expect(result.truncated).toBe(false);+ });++ it("redacts credential-like fields", () => {+ const payload = new TextEncoder().encode(+ JSON.stringify({+ password: "secret123",+ username: "admin",+ }),+ );+ const result = decodeJsonRpc(payload);+ const passwordField = result.decoded_fields.find(+ (f) => f.path === "password",+ );+ expect(passwordField?.value).toBe("[REDACTED]");+ const usernameField = result.decoded_fields.find(+ (f) => f.path === "username",+ );+ expect(usernameField?.value).toBe("admin");+ });+});++describe("MessagePack decoding", () => {+ it("detects map type", () => {+ const payload = new Uint8Array([0x82]);+ const result = decodeMessagePack(payload);+ expect(result.decoded_fields[0]?.value).toBe("map");+ });++ it("detects array type", () => {+ const payload = new Uint8Array([0x92]);+ const result = decodeMessagePack(payload);+ expect(result.decoded_fields[0]?.value).toBe("array");+ });++ it("reports unknown type for other tags", () => {+ const payload = new Uint8Array([0xff]);+ const result = decodeMessagePack(payload);+ expect(result.decoded_fields[0]?.value).toBe("unknown");+ expect(result.decoded_fields[0]?.inferred).toBe(true);+ });+});++describe("gRPC frame decoding", () => {+ it("decodes a valid gRPC frame header", () => {+ const payload = new Uint8Array([+ 0x00, 0x00, 0x00, 0x00, 0x05, 1, 2, 3, 4, 5,+ ]);+ const result = decodeGrpcFrame(payload);+ expect(result.decoded_fields).toHaveLength(2);+ expect(result.decoded_fields[0]?.path).toBe("compressed");+ expect(result.decoded_fields[0]?.value).toBe(false);+ expect(result.decoded_fields[1]?.path).toBe("message_length");+ expect(result.decoded_fields[1]?.value).toBe(5);+ });++ it("detects compressed flag", () => {+ const payload = new Uint8Array([0x01, 0x00, 0x00, 0x00, 0x0a]);+ const result = decodeGrpcFrame(payload);+ expect(result.decoded_fields[0]?.value).toBe(true);+ });++ it("handles too-short frames", () => {+ const payload = new Uint8Array([0x00, 0x01]);+ const result = decodeGrpcFrame(payload);+ expect(result.decoded_fields[0]?.path).toBe("error");+ });+});++describe("schema inference", () => {+ it("infers schema from repeated messages", () => {+ const messages: ProtocolMessage[] = [+ {+ sequence: 0,+ at_ms: 0,+ family: "json-rpc",+ direction: "request",+ endpoint: "/api",+ content_type: "application/json",+ raw_payload: null,+ decoded_fields: [+ { path: "method", wire_type: "json", value: "add", inferred: false },+ { path: "id", wire_type: "json", value: 1, inferred: false },+ ],+ schema_hypotheses: [],+ truncated: false,+ credentials_redacted: false,+ },+ {+ sequence: 1,+ at_ms: 10,+ family: "json-rpc",+ direction: "response",+ endpoint: "/api",+ content_type: "application/json",+ raw_payload: null,+ decoded_fields: [+ { path: "result", wire_type: "json", value: 3, inferred: false },+ { path: "id", wire_type: "json", value: 1, inferred: false },+ ],+ schema_hypotheses: [],+ truncated: false,+ credentials_redacted: false,+ },+ ];+ const hypotheses = inferMessageSchema(messages);+ expect(hypotheses).toHaveLength(3);+ const idHypothesis = hypotheses.find((h) => h.field_name === "id");+ expect(idHypothesis).toBeDefined();+ expect(idHypothesis!.evidence_count).toBe(2);+ expect(idHypothesis!.confidence).toBe(1);+ });+});++describe("uses ProtocolCapture type", () => {+ it("validates a well-formed capture typed value", () => {+ const capture: ProtocolCapture = {+ family: "grpc",+ messages: [],+ inferred_schema: [],+ has_truncated: false,+ credentials_detected: false,+ };+ expect(capture.family).toBe("grpc");+ });+});++describe("protocol capture schema validation", () => {+ it("validates a well-formed capture", () => {+ const capture = {+ family: "grpc" as const,+ messages: [],+ inferred_schema: [],+ has_truncated: false,+ credentials_detected: false,+ };+ const result = protocolCaptureSchema.safeParse(capture);+ expect(result.success).toBe(true);+ });+});