Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat(ingest): parse Protocol Buffers (.proto) (#596)
KageBinary0871b1af39e5
diff --git a/README.md b/README.mdindex c17c989b..90b7a19c 100644--- a/README.md+++ b/README.md@@ -303,7 +303,7 @@ CUDA (`.cu` / `.cuh`) is parsed with the C++ grammar; kernel-launch syntaxthe graph. Python stub files (`.pyi`) are parsed as Python.**Also recognized:**-YAML, JSON, TOML, SQL, Dockerfile, Markdown+YAML, JSON, TOML, SQL, Protocol Buffers, Dockerfile, Markdown## Quick Startdiff --git a/core-ingestion/src/__tests__/queries.proto.test.ts b/core-ingestion/src/__tests__/queries.proto.test.tsnew file mode 100644index 00000000..b20dfe49--- /dev/null+++ b/core-ingestion/src/__tests__/queries.proto.test.ts@@ -0,0 +1,160 @@+import { describe, expect, it } from 'vitest';++import { parseFile } from '../index.js';+import { SupportedLanguages } from '../languages.js';++const parse = (source: string, path = '/repo/api.proto') => parseFile(path, source)!;+const named = (source: string) =>+ parse(source).entities.map((entity) => `${entity.kind}:${entity.name}`);+const rels = (source: string, predicate: string) =>+ parse(source).relationships+ .filter((relationship) => relationship.predicate === predicate)+ .map((relationship) => `${relationship.srcName}->${relationship.dstName}`);++describe('Protocol Buffers', () => {+ it('detects the language and captures messages, enums and services', () => {+ const result = parse(`syntax = "proto3";+package ix.v1;++service Store {+ rpc Get(GetRequest) returns (GetResponse);+}++message GetRequest { string key = 1; }+enum Mode { FAST = 0; SLOW = 1; }+`);+ expect(result.language).toBe(SupportedLanguages.Proto);+ expect(result.entities.map((entity) => `${entity.kind}:${entity.name}`)).toEqual(+ expect.arrayContaining([+ 'module:ix.v1',+ 'interface:Store',+ 'method:Get',+ 'class:GetRequest',+ 'class:Mode',+ ]),+ );+ });++ it('links each rpc to its request and response messages', () => {+ const source = `service Store {+ rpc Get(GetRequest) returns (GetResponse);+ rpc Watch(WatchRequest) returns (stream Event);+}+`;+ expect(rels(source, 'REFERENCES')).toEqual([+ 'Get->GetRequest',+ 'Get->GetResponse',+ 'Watch->WatchRequest',+ 'Watch->Event',+ ]);+ expect(rels(source, 'CONTAINS')).toEqual(+ expect.arrayContaining(['Store->Get', 'Store->Watch']),+ );+ });++ it('links a message to the message types of its fields, skipping scalars', () => {+ const source = `message Response {+ bool found = 1;+ string key = 2;+ repeated int64 offsets = 3;+ SyncCursor cursor = 4;+ map<string, Attr> attrs = 5;+ .ix.v1.Outer qualified = 6;+}+`;+ // Scalars (bool/string/int64) contribute nothing; the map's value type and+ // the fully-qualified name resolve to their bare message names.+ expect(rels(source, 'REFERENCES')).toEqual([+ 'Response->SyncCursor',+ 'Response->Attr',+ 'Response->Outer',+ ]);+ });++ it('handles nested definitions and attributes them to their container', () => {+ const result = parse(`message Outer {+ message Inner { string a = 1; }+ enum Kind { A = 0; }+ Inner inner = 1;+}+`);+ expect(result.entities.map((entity) => entity.name)).toEqual(+ expect.arrayContaining(['Outer', 'Inner', 'Kind']),+ );+ expect(+ result.relationships+ .filter((relationship) => relationship.predicate === 'CONTAINS')+ .map((relationship) => `${relationship.srcName}->${relationship.dstName}`),+ ).toEqual(expect.arrayContaining(['Outer->Inner', 'Outer->Kind', 'api.proto->Outer']));++ const inner = result.chunks.find((chunk) => chunk.name === 'Inner');+ expect(inner?.container).toBe('Outer');+ });++ it('keeps a oneof from closing its enclosing message early', () => {+ // A oneof adds a brace level without being a definition; the enclosing+ // message must survive its closing brace.+ const result = parse(`message Envelope {+ oneof payload {+ Ping ping = 1;+ Pong pong = 2;+ }+ string id = 3;+}+`);+ const envelope = result.entities.find((entity) => entity.name === 'Envelope');+ expect(envelope!.lineEnd).toBe(7);+ expect(rels('message Envelope {\n oneof payload {\n Ping ping = 1;\n }\n}\n', 'REFERENCES'))+ .toEqual(['Envelope->Ping']);+ });++ it('ignores braces and keywords inside comments and string options', () => {+ const result = parse(`syntax = "proto3";+option go_package = "github.com/x/y{not_a_brace}";+// message Commented { }+/* service Blocked {+ rpc Nope(A) returns (B);+} */+message Real { string a = 1; }+`);+ expect(result.entities.map((entity) => entity.name)).not.toEqual(+ expect.arrayContaining(['Commented', 'Blocked', 'Nope']),+ );+ expect(result.entities.map((entity) => entity.name)).toContain('Real');+ });++ it('records imports by file name and keeps the raw specifier', () => {+ const result = parse(`import "google/protobuf/timestamp.proto";+import public "shared/common.proto";+`);+ const imports = result.relationships.filter((r) => r.predicate === 'IMPORTS');+ expect(imports.map((r) => r.dstName)).toEqual(['timestamp.proto', 'common.proto']);+ expect(imports[0].importRaw).toBe('google/protobuf/timestamp.proto');+ });++ it('handles single-line and empty messages', () => {+ expect(named('message Empty {}\nmessage OneLine { string a = 1; }\n')).toEqual(+ expect.arrayContaining(['class:Empty', 'class:OneLine']),+ );+ });++ it('closes an unterminated definition at end of file instead of dropping it', () => {+ const result = parse('message Truncated {\n string a = 1;\n');+ expect(result.entities.map((entity) => entity.name)).toContain('Truncated');+ });++ it('does not mistake a statement keyword for a field type', () => {+ // `option deprecated = 1;` matches the field shape exactly — type `option`,+ // name `deprecated`, tag `1` — and emitted a REFERENCES edge to a node+ // called "option" until statement keywords were excluded.+ const source = `message M {+ option deprecated = 1;+ reserved 2, 15;+ extensions 100 to 199;+ Attr attr = 3;+}+`;+ expect(rels(source, 'REFERENCES')).toEqual(['M->Attr']);+ });++});diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.tsindex e1d63f08..5cfd7cc2 100644--- a/core-ingestion/src/index.ts+++ b/core-ingestion/src/index.ts@@ -1942,6 +1942,212 @@ function parseSqlFile(filePath: string, source: string): FileParseResult {return { filePath, language, entities, chunks, relationships, fileRole };}+// ---------------------------------------------------------------------------+// Protocol Buffers+// ---------------------------------------------------------------------------++// proto3 scalar types, plus proto2 spellings. Field types outside this set name+// another message or enum, which is the edge worth recording.+const PROTO_SCALARS = new Set([+ 'double', 'float', 'int32', 'int64', 'uint32', 'uint64', 'sint32', 'sint64',+ 'fixed32', 'fixed64', 'sfixed32', 'sfixed64', 'bool', 'string', 'bytes',+ 'group',+]);++/** Blank comments and string bodies so braces and keywords inside them cannot+ * be mistaken for structure. Length and newlines are preserved so line numbers+ * and byte offsets stay accurate. */+function stripProtoNoise(source: string): string {+ let out = '';+ let i = 0;+ while (i < source.length) {+ const two = source.slice(i, i + 2);+ if (two === '//') {+ const nl = source.indexOf('\n', i);+ const end = nl === -1 ? source.length : nl;+ out += ' '.repeat(end - i);+ i = end;+ } else if (two === '/*') {+ const close = source.indexOf('*/', i + 2);+ const end = close === -1 ? source.length : close + 2;+ out += source.slice(i, end).replace(_blankNonNewline, ' ');+ i = end;+ } else if (source[i] === '"' || source[i] === "'") {+ const quote = source[i];+ let j = i + 1;+ while (j < source.length && source[j] !== quote) {+ if (source[j] === '\\') j += 1;+ j += 1;+ }+ const end = Math.min(j + 1, source.length);+ // Keep the quotes so `import "x.proto"` is still recognisable; blank the body.+ out += quote + source.slice(i + 1, end - 1).replace(_blankNonNewline, ' ') + (end > i + 1 ? quote : '');+ i = end;+ } else {+ out += source[i];+ i += 1;+ }+ }+ return out;+}++// Statement keywords that can begin a line shaped like a field declaration.+// `option deprecated = 1;` in particular matches PROTO_FIELD_RE exactly, and+// without this would emit a REFERENCES edge to a node called "option".+const PROTO_STATEMENT_KEYWORDS = new Set([+ 'option', 'reserved', 'extensions', 'extend', 'import', 'package', 'syntax',+ 'oneof', 'rpc', 'returns', 'message', 'enum', 'service', 'group', 'public',+ 'weak', 'stream',+]);++const PROTO_DEF_RE = /^\s*(message|enum|service)\s+([A-Za-z_]\w*)/;+const PROTO_RPC_RE =+ /^\s*rpc\s+([A-Za-z_]\w*)\s*\(\s*(?:stream\s+)?([\w.]+)\s*\)\s*returns\s*\(\s*(?:stream\s+)?([\w.]+)\s*\)/;+const PROTO_IMPORT_RE = /^\s*import\s+(?:public\s+|weak\s+)?"([^"]*)"/;+const PROTO_PACKAGE_RE = /^\s*package\s+([\w.]+)\s*;/;+const PROTO_FIELD_RE =+ /^\s*(?:repeated\s+|optional\s+|required\s+)?(map\s*<\s*[\w.]+\s*,\s*([\w.]+)\s*>|[\w.]+)\s+[A-Za-z_]\w*\s*=\s*\d+/;++function parseProtoFile(filePath: string, source: string): FileParseResult {+ const language = SupportedLanguages.Proto;+ const fileName = nodePath.basename(filePath);+ const sourceLineCount = countSourceLines(source);+ const fileRole = classifyFileRole(filePath);+ const entities: ParsedEntity[] = [+ { name: fileName, kind: 'file', lineStart: 1, lineEnd: sourceLineCount, language },+ ];+ const chunks: ParsedChunk[] = [];+ const relationships: ParsedRelationship[] = [];+ const lineStarts = computeLineStarts(source);+ const stripped = stripProtoNoise(source);+ const lines = stripped.split(/\r?\n/);+ const rawLines = source.split(/\r?\n/);++ interface OpenDef {+ name: string;+ kind: 'message' | 'enum' | 'service';+ lineStart: number;+ depth: number;+ container?: string;+ }+ const open: OpenDef[] = [];+ let depth = 0;++ const close = (def: OpenDef, lineEnd: number) => {+ const entityKind =+ def.kind === 'service' ? 'interface' : 'class';+ const chunkKind = def.kind === 'service' ? 'interface' : 'class';+ entities.push({ name: def.name, kind: entityKind, lineStart: def.lineStart, lineEnd, language });+ const startByte = lineStarts[def.lineStart - 1] ?? 0;+ const endByte =+ lineEnd < rawLines.length ? (lineStarts[lineEnd] ?? source.length) - 1 : source.length;+ chunks.push({+ name: def.name,+ chunkKind,+ lineStart: def.lineStart,+ lineEnd,+ startByte,+ endByte,+ contentHash: crypto+ .createHash('sha256')+ .update(rawLines.slice(def.lineStart - 1, lineEnd).join('\n'))+ .digest('hex'),+ language,+ ...(def.container ? { container: def.container } : {}),+ });+ relationships.push({+ srcName: def.container ?? fileName,+ dstName: def.name,+ predicate: 'CONTAINS',+ });+ };++ const addReference = (from: string | undefined, typeName: string) => {+ if (!from) return;+ const bare = typeName.replace(/^\./, '');+ const simple = bare.includes('.') ? bare.slice(bare.lastIndexOf('.') + 1) : bare;+ if (!simple || PROTO_SCALARS.has(bare) || simple === from) return;+ relationships.push({ srcName: from, dstName: simple, predicate: 'REFERENCES' });+ };++ for (let i = 0; i < lines.length; i++) {+ const line = lines[i];+ const enclosing = open.length ? open[open.length - 1] : undefined;++ const pkg = PROTO_PACKAGE_RE.exec(line);+ if (pkg) {+ entities.push({ name: pkg[1], kind: 'module', lineStart: i + 1, lineEnd: i + 1, language });+ relationships.push({ srcName: fileName, dstName: pkg[1], predicate: 'CONTAINS' });+ }++ const imp = PROTO_IMPORT_RE.exec(rawLines[i] ?? '');+ if (imp && imp[1]) {+ relationships.push({+ srcName: fileName,+ dstName: nodePath.basename(imp[1]),+ predicate: 'IMPORTS',+ importRaw: imp[1],+ });+ }++ const rpc = PROTO_RPC_RE.exec(line);+ if (rpc && enclosing) {+ entities.push({ name: rpc[1], kind: 'method', lineStart: i + 1, lineEnd: i + 1, language });+ relationships.push({ srcName: enclosing.name, dstName: rpc[1], predicate: 'CONTAINS' });+ addReference(rpc[1], rpc[2]);+ addReference(rpc[1], rpc[3]);+ } else if (enclosing && enclosing.kind === 'message' && !PROTO_DEF_RE.test(line)) {+ const field = PROTO_FIELD_RE.exec(line);+ if (field && !PROTO_STATEMENT_KEYWORDS.has(field[1])) {+ addReference(enclosing.name, field[2] ?? field[1]);+ }+ }++ const def = PROTO_DEF_RE.exec(line);+ if (def) {+ open.push({+ name: def[2],+ kind: def[1] as OpenDef['kind'],+ lineStart: i + 1,+ depth,+ container: enclosing?.name,+ });+ }++ for (const ch of line) {+ if (ch === '{') depth += 1;+ else if (ch === '}') {+ depth = Math.max(0, depth - 1);+ // A definition opened at this depth is now complete. Single-line forms+ // (`message Empty {}`) open and close on the same line, which this+ // handles because the push happens before the brace scan.+ while (open.length && open[open.length - 1].depth >= depth) {+ close(open.pop() as OpenDef, i + 1);+ }+ }+ }+ }++ // Unterminated definitions (truncated or malformed file): close at EOF rather+ // than dropping the symbols.+ while (open.length) close(open.pop() as OpenDef, sourceLineCount);++ if (chunks.length === 0) {+ chunks.push({+ name: null,+ chunkKind: 'file_body',+ lineStart: 1,+ lineEnd: Math.max(sourceLineCount, 1),+ startByte: 0,+ endByte: source.length,+ contentHash: crypto.createHash('sha256').update(source).digest('hex'),+ language,+ });+ }++ return { filePath, language, entities, chunks, relationships, fileRole };+}+// ---------------------------------------------------------------------------// Rust: cfg macro unwrapping// ---------------------------------------------------------------------------@@ -2148,6 +2354,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | nif (language === SupportedLanguages.TOML) return parseTomlFile(filePath, source);if (language === SupportedLanguages.Markdown) return parseMarkdownFile(filePath, source);if (language === SupportedLanguages.LaTeX) return parseLatexFile(filePath, source);+ if (language === SupportedLanguages.Proto) return parseProtoFile(filePath, source);// TypeScript TSX uses a separate grammarconst isTsx = filePath.endsWith('.tsx');diff --git a/core-ingestion/src/languages.ts b/core-ingestion/src/languages.tsindex 06314a4b..c388892c 100644--- a/core-ingestion/src/languages.ts+++ b/core-ingestion/src/languages.ts@@ -33,6 +33,7 @@ export enum SupportedLanguages {HCL = 'hcl',CSS = 'css',LaTeX = 'latex',+ Proto = 'proto',}const EXT_MAP: Record<string, SupportedLanguages> = {@@ -72,6 +73,7 @@ const EXT_MAP: Record<string, SupportedLanguages> = {'.yml': SupportedLanguages.YAML,'.dockerfile': SupportedLanguages.Dockerfile,'.sql': SupportedLanguages.SQL,+ '.proto': SupportedLanguages.Proto,'.json': SupportedLanguages.JSON,'.toml': SupportedLanguages.TOML,'.md': SupportedLanguages.Markdown,diff --git a/core-ingestion/src/queries.ts b/core-ingestion/src/queries.tsindex 9f1a56b5..b049b2d8 100644--- a/core-ingestion/src/queries.ts+++ b/core-ingestion/src/queries.ts@@ -1910,5 +1910,8 @@ export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {// LaTeX/TeX has no tree-sitter grammar in this stack; it is parsed by the// hand-rolled scanner in parseLatexFile (index.ts), like Markdown/YAML/etc.[SupportedLanguages.LaTeX]: '',+ // Protocol Buffers likewise: no grammar in this stack, parsed by+ // parseProtoFile's scanner in index.ts.+ [SupportedLanguages.Proto]: '',};diff --git a/ix-cli/src/cli/supported-extensions.ts b/ix-cli/src/cli/supported-extensions.tsindex 782f167a..feb91b16 100644--- a/ix-cli/src/cli/supported-extensions.ts+++ b/ix-cli/src/cli/supported-extensions.ts@@ -15,6 +15,7 @@ export const SUPPORTED_EXTENSIONS = new Set<string>([".yaml", ".yml",".dockerfile",".sql",+ ".proto",".json",".toml",".md", ".markdown",