Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat(ingest): parse Protocol Buffers (.proto) (#596)

KageBinary0871b1af39e5
diff --git a/README.md b/README.md
index c17c989b..90b7a19c 100644
--- a/README.md
+++ b/README.md
@@ -303,7 +303,7 @@ CUDA (`.cu` / `.cuh`) is parsed with the C++ grammar; kernel-launch syntax
the graph. Python stub files (`.pyi`) are parsed as Python.
**Also recognized:**
-YAML, JSON, TOML, SQL, Dockerfile, Markdown
+YAML, JSON, TOML, SQL, Protocol Buffers, Dockerfile, Markdown
## Quick Start
diff --git a/core-ingestion/src/__tests__/queries.proto.test.ts b/core-ingestion/src/__tests__/queries.proto.test.ts
new file mode 100644
index 00000000..b20dfe49
--- /dev/null
+++ b/core-ingestion/src/__tests__/queries.proto.test.ts
@@ -0,0 +1,160 @@
+import { describe, expect, it } from 'vitest';
+
+import { parseFile } from '../index.js';
+import { SupportedLanguages } from '../languages.js';
+
+const parse = (source: string, path = '/repo/api.proto') => parseFile(path, source)!;
+const named = (source: string) =>
+ parse(source).entities.map((entity) => `${entity.kind}:${entity.name}`);
+const rels = (source: string, predicate: string) =>
+ parse(source).relationships
+ .filter((relationship) => relationship.predicate === predicate)
+ .map((relationship) => `${relationship.srcName}->${relationship.dstName}`);
+
+describe('Protocol Buffers', () => {
+ it('detects the language and captures messages, enums and services', () => {
+ const result = parse(`syntax = "proto3";
+package ix.v1;
+
+service Store {
+ rpc Get(GetRequest) returns (GetResponse);
+}
+
+message GetRequest { string key = 1; }
+enum Mode { FAST = 0; SLOW = 1; }
+`);
+ expect(result.language).toBe(SupportedLanguages.Proto);
+ expect(result.entities.map((entity) => `${entity.kind}:${entity.name}`)).toEqual(
+ expect.arrayContaining([
+ 'module:ix.v1',
+ 'interface:Store',
+ 'method:Get',
+ 'class:GetRequest',
+ 'class:Mode',
+ ]),
+ );
+ });
+
+ it('links each rpc to its request and response messages', () => {
+ const source = `service Store {
+ rpc Get(GetRequest) returns (GetResponse);
+ rpc Watch(WatchRequest) returns (stream Event);
+}
+`;
+ expect(rels(source, 'REFERENCES')).toEqual([
+ 'Get->GetRequest',
+ 'Get->GetResponse',
+ 'Watch->WatchRequest',
+ 'Watch->Event',
+ ]);
+ expect(rels(source, 'CONTAINS')).toEqual(
+ expect.arrayContaining(['Store->Get', 'Store->Watch']),
+ );
+ });
+
+ it('links a message to the message types of its fields, skipping scalars', () => {
+ const source = `message Response {
+ bool found = 1;
+ string key = 2;
+ repeated int64 offsets = 3;
+ SyncCursor cursor = 4;
+ map<string, Attr> attrs = 5;
+ .ix.v1.Outer qualified = 6;
+}
+`;
+ // Scalars (bool/string/int64) contribute nothing; the map's value type and
+ // the fully-qualified name resolve to their bare message names.
+ expect(rels(source, 'REFERENCES')).toEqual([
+ 'Response->SyncCursor',
+ 'Response->Attr',
+ 'Response->Outer',
+ ]);
+ });
+
+ it('handles nested definitions and attributes them to their container', () => {
+ const result = parse(`message Outer {
+ message Inner { string a = 1; }
+ enum Kind { A = 0; }
+ Inner inner = 1;
+}
+`);
+ expect(result.entities.map((entity) => entity.name)).toEqual(
+ expect.arrayContaining(['Outer', 'Inner', 'Kind']),
+ );
+ expect(
+ result.relationships
+ .filter((relationship) => relationship.predicate === 'CONTAINS')
+ .map((relationship) => `${relationship.srcName}->${relationship.dstName}`),
+ ).toEqual(expect.arrayContaining(['Outer->Inner', 'Outer->Kind', 'api.proto->Outer']));
+
+ const inner = result.chunks.find((chunk) => chunk.name === 'Inner');
+ expect(inner?.container).toBe('Outer');
+ });
+
+ it('keeps a oneof from closing its enclosing message early', () => {
+ // A oneof adds a brace level without being a definition; the enclosing
+ // message must survive its closing brace.
+ const result = parse(`message Envelope {
+ oneof payload {
+ Ping ping = 1;
+ Pong pong = 2;
+ }
+ string id = 3;
+}
+`);
+ const envelope = result.entities.find((entity) => entity.name === 'Envelope');
+ expect(envelope!.lineEnd).toBe(7);
+ expect(rels('message Envelope {\n oneof payload {\n Ping ping = 1;\n }\n}\n', 'REFERENCES'))
+ .toEqual(['Envelope->Ping']);
+ });
+
+ it('ignores braces and keywords inside comments and string options', () => {
+ const result = parse(`syntax = "proto3";
+option go_package = "github.com/x/y{not_a_brace}";
+// message Commented { }
+/* service Blocked {
+ rpc Nope(A) returns (B);
+} */
+message Real { string a = 1; }
+`);
+ expect(result.entities.map((entity) => entity.name)).not.toEqual(
+ expect.arrayContaining(['Commented', 'Blocked', 'Nope']),
+ );
+ expect(result.entities.map((entity) => entity.name)).toContain('Real');
+ });
+
+ it('records imports by file name and keeps the raw specifier', () => {
+ const result = parse(`import "google/protobuf/timestamp.proto";
+import public "shared/common.proto";
+`);
+ const imports = result.relationships.filter((r) => r.predicate === 'IMPORTS');
+ expect(imports.map((r) => r.dstName)).toEqual(['timestamp.proto', 'common.proto']);
+ expect(imports[0].importRaw).toBe('google/protobuf/timestamp.proto');
+ });
+
+ it('handles single-line and empty messages', () => {
+ expect(named('message Empty {}\nmessage OneLine { string a = 1; }\n')).toEqual(
+ expect.arrayContaining(['class:Empty', 'class:OneLine']),
+ );
+ });
+
+ it('closes an unterminated definition at end of file instead of dropping it', () => {
+ const result = parse('message Truncated {\n string a = 1;\n');
+ expect(result.entities.map((entity) => entity.name)).toContain('Truncated');
+ });
+
+ it('does not mistake a statement keyword for a field type', () => {
+ // `option deprecated = 1;` matches the field shape exactly — type `option`,
+ // name `deprecated`, tag `1` — and emitted a REFERENCES edge to a node
+ // called "option" until statement keywords were excluded.
+ const source = `message M {
+ option deprecated = 1;
+ reserved 2, 15;
+ extensions 100 to 199;
+ Attr attr = 3;
+}
+`;
+ expect(rels(source, 'REFERENCES')).toEqual(['M->Attr']);
+ });
+
+});
diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.ts
index e1d63f08..5cfd7cc2 100644
--- a/core-ingestion/src/index.ts
+++ b/core-ingestion/src/index.ts
@@ -1942,6 +1942,212 @@ function parseSqlFile(filePath: string, source: string): FileParseResult {
return { filePath, language, entities, chunks, relationships, fileRole };
}
+// ---------------------------------------------------------------------------
+// Protocol Buffers
+// ---------------------------------------------------------------------------
+
+// proto3 scalar types, plus proto2 spellings. Field types outside this set name
+// another message or enum, which is the edge worth recording.
+const PROTO_SCALARS = new Set([
+ 'double', 'float', 'int32', 'int64', 'uint32', 'uint64', 'sint32', 'sint64',
+ 'fixed32', 'fixed64', 'sfixed32', 'sfixed64', 'bool', 'string', 'bytes',
+ 'group',
+]);
+
+/** Blank comments and string bodies so braces and keywords inside them cannot
+ * be mistaken for structure. Length and newlines are preserved so line numbers
+ * and byte offsets stay accurate. */
+function stripProtoNoise(source: string): string {
+ let out = '';
+ let i = 0;
+ while (i < source.length) {
+ const two = source.slice(i, i + 2);
+ if (two === '//') {
+ const nl = source.indexOf('\n', i);
+ const end = nl === -1 ? source.length : nl;
+ out += ' '.repeat(end - i);
+ i = end;
+ } else if (two === '/*') {
+ const close = source.indexOf('*/', i + 2);
+ const end = close === -1 ? source.length : close + 2;
+ out += source.slice(i, end).replace(_blankNonNewline, ' ');
+ i = end;
+ } else if (source[i] === '"' || source[i] === "'") {
+ const quote = source[i];
+ let j = i + 1;
+ while (j < source.length && source[j] !== quote) {
+ if (source[j] === '\\') j += 1;
+ j += 1;
+ }
+ const end = Math.min(j + 1, source.length);
+ // Keep the quotes so `import "x.proto"` is still recognisable; blank the body.
+ out += quote + source.slice(i + 1, end - 1).replace(_blankNonNewline, ' ') + (end > i + 1 ? quote : '');
+ i = end;
+ } else {
+ out += source[i];
+ i += 1;
+ }
+ }
+ return out;
+}
+
+// Statement keywords that can begin a line shaped like a field declaration.
+// `option deprecated = 1;` in particular matches PROTO_FIELD_RE exactly, and
+// without this would emit a REFERENCES edge to a node called "option".
+const PROTO_STATEMENT_KEYWORDS = new Set([
+ 'option', 'reserved', 'extensions', 'extend', 'import', 'package', 'syntax',
+ 'oneof', 'rpc', 'returns', 'message', 'enum', 'service', 'group', 'public',
+ 'weak', 'stream',
+]);
+
+const PROTO_DEF_RE = /^\s*(message|enum|service)\s+([A-Za-z_]\w*)/;
+const PROTO_RPC_RE =
+ /^\s*rpc\s+([A-Za-z_]\w*)\s*\(\s*(?:stream\s+)?([\w.]+)\s*\)\s*returns\s*\(\s*(?:stream\s+)?([\w.]+)\s*\)/;
+const PROTO_IMPORT_RE = /^\s*import\s+(?:public\s+|weak\s+)?"([^"]*)"/;
+const PROTO_PACKAGE_RE = /^\s*package\s+([\w.]+)\s*;/;
+const PROTO_FIELD_RE =
+ /^\s*(?:repeated\s+|optional\s+|required\s+)?(map\s*<\s*[\w.]+\s*,\s*([\w.]+)\s*>|[\w.]+)\s+[A-Za-z_]\w*\s*=\s*\d+/;
+
+function parseProtoFile(filePath: string, source: string): FileParseResult {
+ const language = SupportedLanguages.Proto;
+ const fileName = nodePath.basename(filePath);
+ const sourceLineCount = countSourceLines(source);
+ const fileRole = classifyFileRole(filePath);
+ const entities: ParsedEntity[] = [
+ { name: fileName, kind: 'file', lineStart: 1, lineEnd: sourceLineCount, language },
+ ];
+ const chunks: ParsedChunk[] = [];
+ const relationships: ParsedRelationship[] = [];
+ const lineStarts = computeLineStarts(source);
+ const stripped = stripProtoNoise(source);
+ const lines = stripped.split(/\r?\n/);
+ const rawLines = source.split(/\r?\n/);
+
+ interface OpenDef {
+ name: string;
+ kind: 'message' | 'enum' | 'service';
+ lineStart: number;
+ depth: number;
+ container?: string;
+ }
+ const open: OpenDef[] = [];
+ let depth = 0;
+
+ const close = (def: OpenDef, lineEnd: number) => {
+ const entityKind =
+ def.kind === 'service' ? 'interface' : 'class';
+ const chunkKind = def.kind === 'service' ? 'interface' : 'class';
+ entities.push({ name: def.name, kind: entityKind, lineStart: def.lineStart, lineEnd, language });
+ const startByte = lineStarts[def.lineStart - 1] ?? 0;
+ const endByte =
+ lineEnd < rawLines.length ? (lineStarts[lineEnd] ?? source.length) - 1 : source.length;
+ chunks.push({
+ name: def.name,
+ chunkKind,
+ lineStart: def.lineStart,
+ lineEnd,
+ startByte,
+ endByte,
+ contentHash: crypto
+ .createHash('sha256')
+ .update(rawLines.slice(def.lineStart - 1, lineEnd).join('\n'))
+ .digest('hex'),
+ language,
+ ...(def.container ? { container: def.container } : {}),
+ });
+ relationships.push({
+ srcName: def.container ?? fileName,
+ dstName: def.name,
+ predicate: 'CONTAINS',
+ });
+ };
+
+ const addReference = (from: string | undefined, typeName: string) => {
+ if (!from) return;
+ const bare = typeName.replace(/^\./, '');
+ const simple = bare.includes('.') ? bare.slice(bare.lastIndexOf('.') + 1) : bare;
+ if (!simple || PROTO_SCALARS.has(bare) || simple === from) return;
+ relationships.push({ srcName: from, dstName: simple, predicate: 'REFERENCES' });
+ };
+
+ for (let i = 0; i < lines.length; i++) {
+ const line = lines[i];
+ const enclosing = open.length ? open[open.length - 1] : undefined;
+
+ const pkg = PROTO_PACKAGE_RE.exec(line);
+ if (pkg) {
+ entities.push({ name: pkg[1], kind: 'module', lineStart: i + 1, lineEnd: i + 1, language });
+ relationships.push({ srcName: fileName, dstName: pkg[1], predicate: 'CONTAINS' });
+ }
+
+ const imp = PROTO_IMPORT_RE.exec(rawLines[i] ?? '');
+ if (imp && imp[1]) {
+ relationships.push({
+ srcName: fileName,
+ dstName: nodePath.basename(imp[1]),
+ predicate: 'IMPORTS',
+ importRaw: imp[1],
+ });
+ }
+
+ const rpc = PROTO_RPC_RE.exec(line);
+ if (rpc && enclosing) {
+ entities.push({ name: rpc[1], kind: 'method', lineStart: i + 1, lineEnd: i + 1, language });
+ relationships.push({ srcName: enclosing.name, dstName: rpc[1], predicate: 'CONTAINS' });
+ addReference(rpc[1], rpc[2]);
+ addReference(rpc[1], rpc[3]);
+ } else if (enclosing && enclosing.kind === 'message' && !PROTO_DEF_RE.test(line)) {
+ const field = PROTO_FIELD_RE.exec(line);
+ if (field && !PROTO_STATEMENT_KEYWORDS.has(field[1])) {
+ addReference(enclosing.name, field[2] ?? field[1]);
+ }
+ }
+
+ const def = PROTO_DEF_RE.exec(line);
+ if (def) {
+ open.push({
+ name: def[2],
+ kind: def[1] as OpenDef['kind'],
+ lineStart: i + 1,
+ depth,
+ container: enclosing?.name,
+ });
+ }
+
+ for (const ch of line) {
+ if (ch === '{') depth += 1;
+ else if (ch === '}') {
+ depth = Math.max(0, depth - 1);
+ // A definition opened at this depth is now complete. Single-line forms
+ // (`message Empty {}`) open and close on the same line, which this
+ // handles because the push happens before the brace scan.
+ while (open.length && open[open.length - 1].depth >= depth) {
+ close(open.pop() as OpenDef, i + 1);
+ }
+ }
+ }
+ }
+
+ // Unterminated definitions (truncated or malformed file): close at EOF rather
+ // than dropping the symbols.
+ while (open.length) close(open.pop() as OpenDef, sourceLineCount);
+
+ if (chunks.length === 0) {
+ chunks.push({
+ name: null,
+ chunkKind: 'file_body',
+ lineStart: 1,
+ lineEnd: Math.max(sourceLineCount, 1),
+ startByte: 0,
+ endByte: source.length,
+ contentHash: crypto.createHash('sha256').update(source).digest('hex'),
+ language,
+ });
+ }
+
+ return { filePath, language, entities, chunks, relationships, fileRole };
+}
+
// ---------------------------------------------------------------------------
// Rust: cfg macro unwrapping
// ---------------------------------------------------------------------------
@@ -2148,6 +2354,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
if (language === SupportedLanguages.TOML) return parseTomlFile(filePath, source);
if (language === SupportedLanguages.Markdown) return parseMarkdownFile(filePath, source);
if (language === SupportedLanguages.LaTeX) return parseLatexFile(filePath, source);
+ if (language === SupportedLanguages.Proto) return parseProtoFile(filePath, source);
// TypeScript TSX uses a separate grammar
const isTsx = filePath.endsWith('.tsx');
diff --git a/core-ingestion/src/languages.ts b/core-ingestion/src/languages.ts
index 06314a4b..c388892c 100644
--- a/core-ingestion/src/languages.ts
+++ b/core-ingestion/src/languages.ts
@@ -33,6 +33,7 @@ export enum SupportedLanguages {
HCL = 'hcl',
CSS = 'css',
LaTeX = 'latex',
+ Proto = 'proto',
}
const EXT_MAP: Record<string, SupportedLanguages> = {
@@ -72,6 +73,7 @@ const EXT_MAP: Record<string, SupportedLanguages> = {
'.yml': SupportedLanguages.YAML,
'.dockerfile': SupportedLanguages.Dockerfile,
'.sql': SupportedLanguages.SQL,
+ '.proto': SupportedLanguages.Proto,
'.json': SupportedLanguages.JSON,
'.toml': SupportedLanguages.TOML,
'.md': SupportedLanguages.Markdown,
diff --git a/core-ingestion/src/queries.ts b/core-ingestion/src/queries.ts
index 9f1a56b5..b049b2d8 100644
--- a/core-ingestion/src/queries.ts
+++ b/core-ingestion/src/queries.ts
@@ -1910,5 +1910,8 @@ export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {
// LaTeX/TeX has no tree-sitter grammar in this stack; it is parsed by the
// hand-rolled scanner in parseLatexFile (index.ts), like Markdown/YAML/etc.
[SupportedLanguages.LaTeX]: '',
+ // Protocol Buffers likewise: no grammar in this stack, parsed by
+ // parseProtoFile's scanner in index.ts.
+ [SupportedLanguages.Proto]: '',
};
diff --git a/ix-cli/src/cli/supported-extensions.ts b/ix-cli/src/cli/supported-extensions.ts
index 782f167a..feb91b16 100644
--- a/ix-cli/src/cli/supported-extensions.ts
+++ b/ix-cli/src/cli/supported-extensions.ts
@@ -15,6 +15,7 @@ export const SUPPORTED_EXTENSIONS = new Set<string>([
".yaml", ".yml",
".dockerfile",
".sql",
+ ".proto",
".json",
".toml",
".md", ".markdown",