Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat(ingest): parse CUDA (.cu/.cuh) and Python stubs (.pyi) (#593)
KageBinary701700b40807
diff --git a/README.md b/README.mdindex a3134a72..c96c3df8 100644--- a/README.md+++ b/README.md@@ -298,6 +298,10 @@ Ix parses and extracts symbols, calls, and imports across 26 languages, and reco**Languages:**JavaScript, TypeScript, Python, Java, C, C++, C#, Go, Ruby, Rust, PHP, Kotlin, Swift, Scala, R, SAS, Elixir, Haskell, Zig, Lua, Bash, HTML, XML, CSS, HCL / Terraform, Makefile+CUDA (`.cu` / `.cuh`) is parsed with the C++ grammar; kernel-launch syntax+(`kernel<<<grid, block>>>(args)`) is handled, so host-to-kernel calls appear in+the graph. Python stub files (`.pyi`) are parsed as Python.+**Also recognized:**YAML, JSON, TOML, SQL, Dockerfile, Markdowndiff --git a/core-ingestion/src/__tests__/extension-parity.test.ts b/core-ingestion/src/__tests__/extension-parity.test.tsnew file mode 100644index 00000000..24933000--- /dev/null+++ b/core-ingestion/src/__tests__/extension-parity.test.ts@@ -0,0 +1,48 @@+import { readFileSync } from 'node:fs';+import { fileURLToPath } from 'node:url';++import { describe, expect, it } from 'vitest';++import { languageFromPath } from '../languages.js';++// supported-extensions.ts documents that it "MUST stay in sync" with EXT_MAP,+// but nothing enforced it, so every new parser was one forgotten line away from+// ingesting nothing: a file type the walker never yields is silently absent from+// the graph rather than failing loudly.+//+// Both files are read as text so this stays independent of the ix-cli build.+const read = (relative: string) =>+ readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8');++const parsedExtensions = (() => {+ const source = read('../languages.ts');+ const map = source.slice(source.indexOf('const EXT_MAP'));+ return new Set([...map.matchAll(/'(\.[a-z0-9]+)'\s*:/gi)].map((match) => match[1]));+})();++const walkedExtensions = new Set(+ [...read('../../../ix-cli/src/cli/supported-extensions.ts')+ .matchAll(/"(\.[a-z0-9]+)"/gi)].map((match) => match[1]),+);++describe('extension parity between EXT_MAP and SUPPORTED_EXTENSIONS', () => {+ it('reads a non-empty set from both files', () => {+ // Guards against a regex that silently matches nothing after a refactor,+ // which would make every assertion below vacuously true.+ expect(parsedExtensions.size).toBeGreaterThan(50);+ expect(walkedExtensions.size).toBeGreaterThan(50);+ });++ it('walks every extension a parser handles', () => {+ expect([...parsedExtensions].filter((ext) => !walkedExtensions.has(ext))).toEqual([]);+ });++ it('has a parser for every extension it walks', () => {+ expect([...walkedExtensions].filter((ext) => !parsedExtensions.has(ext))).toEqual([]);+ });++ it('resolves each walked extension to a language', () => {+ const unresolved = [...walkedExtensions].filter((ext) => !languageFromPath(`file${ext}`));+ expect(unresolved).toEqual([]);+ });+});diff --git a/core-ingestion/src/__tests__/queries.cuda.test.ts b/core-ingestion/src/__tests__/queries.cuda.test.tsnew file mode 100644index 00000000..430516ef--- /dev/null+++ b/core-ingestion/src/__tests__/queries.cuda.test.ts@@ -0,0 +1,111 @@+import { describe, expect, it } from 'vitest';++import { parseFile } from '../index.js';++const KERNEL_SOURCE = `+#include <cuda_runtime.h>++__global__ void fused_attn(const float* q, float* out, int n) {+ int i = blockIdx.x * blockDim.x + threadIdx.x;+ __shared__ float tile[128];+ if (i < n) out[i] = q[i];+}++void launch_attn(const float* q, float* out, int n) {+ fused_attn<<<(n + 127) / 128, 128>>>(q, out, n);+}+`;++const callTargets = (result: ReturnType<typeof parseFile>) =>+ result!.relationships+ .filter((relationship) => relationship.predicate === 'CALLS')+ .map((relationship) => relationship.dstName);++describe('CUDA', () => {+ it('extracts kernels and host functions from a .cu file', () => {+ const result = parseFile('/repo/kernels/attn.cu', KERNEL_SOURCE);++ expect(result).not.toBeNull();+ expect(result!.entities.map((entity) => entity.name)).toEqual(+ expect.arrayContaining(['fused_attn', 'launch_attn']),+ );+ });++ it('keeps the host -> kernel edge across the launch config', () => {+ // The whole point: tree-sitter-cpp drops the call entirely when it chokes+ // on `<<<...>>>`, so without the blanking pass this edge does not exist.+ expect(callTargets(parseFile('/repo/kernels/attn.cu', KERNEL_SOURCE)))+ .toEqual(expect.arrayContaining(['fused_attn']));+ });++ it('handles a launch config spanning several lines without shifting line numbers', () => {+ const source = `void launch(float* x) {+ my_kernel<<<+ dim3(1, 2, 3),+ 256+ >>>(x);+}++__global__ void trailing_marker(int a) {}+`;+ const result = parseFile('/repo/kernels/multiline.cu', source);++ expect(callTargets(result)).toEqual(expect.arrayContaining(['my_kernel']));++ // `trailing_marker` sits on line 8; a space-only blank would have collapsed+ // the newlines inside the launch config and dragged it upwards.+ const marker = result!.entities.find((entity) => entity.name === 'trailing_marker');+ expect(marker).toBeDefined();+ expect(marker!.lineStart).toBe(8);+ });++ it('parses .cuh headers', () => {+ const result = parseFile(+ '/repo/kernels/attn.cuh',+ '__global__ void fused_attn(const float* q, float* out, int n);\n',+ );++ expect(result).not.toBeNull();+ expect(result!.entities.map((entity) => entity.name)).toEqual(+ expect.arrayContaining(['fused_attn']),+ );+ });++ it('leaves ordinary C++ untouched', () => {+ // Nested templates, shift operators and arrows inside a string literal all+ // contain the `<<<` / `>>>` character runs; none is a launch config.+ const source = `#include <vector>+void f(std::vector<std::vector<std::vector<int>>>& v, int a, int b) {+ int x = a << b;+ log("<<<HEAD>>>");+ consume(v, x);+}+`;+ const cpp = parseFile('/repo/src/plain.cpp', source);+ const cu = parseFile('/repo/src/plain.cu', source);++ expect(callTargets(cu)).toEqual(callTargets(cpp));+ expect(callTargets(cu)).toEqual(expect.arrayContaining(['consume']));+ });++ it('stays linear on adversarial input', () => {+ // Ingest input is an arbitrary repository, so the blanking pass is+ // attacker-controlled. The obvious regex (`/<<<[^;]*?>>>(?=\\s*\\()/g`)+ // is quadratic here: 16k `<` took 67ms and grew 4x per doubling.+ const timeFor = (n: number) => {+ const evil = `void f() { ${'<'.repeat(n)}>>>x }`;+ const started = performance.now();+ parseFile('/repo/evil.cu', evil);+ return performance.now() - started;+ };++ timeFor(1_000); // warm the parser so the comparison is not first-call cost++ const small = timeFor(8_000);+ const large = timeFor(32_000);++ // 4x the input. Quadratic would be ~16x; linear scanning stays well under.+ expect(large).toBeLessThan(Math.max(small, 1) * 8);+ });++});diff --git a/core-ingestion/src/__tests__/queries.python.test.ts b/core-ingestion/src/__tests__/queries.python.test.tsindex 2c0980f7..dee516bf 100644--- a/core-ingestion/src/__tests__/queries.python.test.ts+++ b/core-ingestion/src/__tests__/queries.python.test.ts@@ -177,4 +177,28 @@ def list_items():predicate: 'CALLS',});});++ it('extracts signatures from a .pyi stub', () => {+ // Stubs are the only place a compiled extension's API exists in Python, so+ // without them a call into a native op dead-ends at the import.+ const result = parseFile(+ '/repo/torch_ext/_C.pyi',+ `from typing import overload++import torch++def fused_attn(q: torch.Tensor, k: torch.Tensor, scale: float = 1.0) -> torch.Tensor: ...++class KernelHandle:+ device: int+ def launch(self, n: int) -> None: ...+`,+ );++ expect(result).not.toBeNull();+ expect(result!.entities.map((entity) => entity.name)).toEqual(+ expect.arrayContaining(['fused_attn', 'KernelHandle', 'launch']),+ );+ });+});diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.tsindex d3c71491..8f1698ca 100644--- a/core-ingestion/src/index.ts+++ b/core-ingestion/src/index.ts@@ -1989,6 +1989,79 @@ function unwrapRustCfgMacros(source: string): string {return parts.join('');}+/**+ * Blanks out CUDA kernel-launch configurations (`<<<grid, block>>>`) so that+ * tree-sitter-cpp sees an ordinary call expression.+ *+ * The launch syntax is not valid C++: the grammar fails on it and, worse,+ * recovers by dropping the whole call — so the edge from host code to the+ * kernel it launches is lost, which is the one edge worth having in a CUDA+ * file. Blanking the config in-place (preserving every character position and+ * newline, as with the Rust macro unwrapper above) turns `k<<<a,b>>>(x)` into+ * `k (x)`, which parses cleanly and yields the call.+ *+ * Hand-scanned rather than pattern-matched. The natural regex+ * (`/<<<[^;]*?>>>(?=\s*\()/g`) is quadratic on `<`-heavy input — a file of+ * 16k `<` took 67ms and grew 4x per doubling — and ingest input is an+ * arbitrary repository, so that is a denial-of-service vector, not a+ * micro-optimisation. This scanner never revisits a character: each failed+ * search advances the cursor past the region it just rejected.+ *+ * A launch config is always followed immediately by the argument list, so+ * requiring `(` after the closing `>>>` is what keeps this off ordinary C++ —+ * nested templates (`vector<vector<vector<int>>>`), shift operators and+ * `"<<<HEAD>>>"` in a string literal are all left alone. Stopping the search+ * at `;` keeps a stray `<<<` in a comment from reaching a later `>>>`.+ */+const _isSpace = (ch: string) => ch === ' ' || ch === '\n' || ch === '\t' || ch === '\r';++function blankCudaLaunchConfigs(source: string): string {+ if (!source.includes('<<<')) return source;++ const out: string[] = [];+ let copiedTo = 0;+ let cursor = 0;++ while (cursor < source.length) {+ const open = source.indexOf('<<<', cursor);+ if (open === -1) break;++ // Find the closing `>>>`, giving up at `;` — a launch config never spans a+ // statement boundary.+ let scan = open + 3;+ while (scan < source.length && source[scan] !== ';' && !source.startsWith('>>>', scan)) {+ scan += 1;+ }++ if (!source.startsWith('>>>', scan)) {+ // Hit `;` or end of file. No `<<<` inside the region just scanned can+ // reach a closing `>>>` either, so skip the whole thing.+ cursor = scan + 1;+ continue;+ }++ const close = scan + 3;+ let afterConfig = close;+ while (afterConfig < source.length && _isSpace(source[afterConfig])) afterConfig += 1;++ if (source[afterConfig] !== '(') {+ // A `>>>` that is not a launch — e.g. closing nested templates. Resume+ // after it; the region behind us cannot match.+ cursor = close;+ continue;+ }++ out.push(source.slice(copiedTo, open));+ out.push(source.slice(open, close).replace(_blankNonNewline, ' '));+ copiedTo = close;+ cursor = close;+ }++ if (copiedTo === 0) return source;+ out.push(source.slice(copiedTo));+ return out.join('');+}+// ---------------------------------------------------------------------------// Main parse function// ---------------------------------------------------------------------------@@ -2102,6 +2175,13 @@ export function parseFile(filePath: string, source: string): FileParseResult | nif (language === SupportedLanguages.Rust) {parseSource = unwrapRustCfgMacros(parseSource);}++ // CUDA: neutralise kernel-launch configs so the host -> kernel call survives.+ // Gated on the extension rather than the language so .cpp/.hpp parsing stays+ // byte-identical.+ if (filePath.endsWith('.cu') || filePath.endsWith('.cuh')) {+ parseSource = blankCudaLaunchConfigs(parseSource);+ }const tree = parser.parse(parseSource, undefined, { bufferSize: parseSource.length + 1 });const cacheKey = isTsx ? 'tsx' as const : language;const query = getCachedQuery(cacheKey, grammar, queries);diff --git a/core-ingestion/src/languages.ts b/core-ingestion/src/languages.tsindex e34d1382..8d411004 100644--- a/core-ingestion/src/languages.ts+++ b/core-ingestion/src/languages.ts@@ -42,6 +42,7 @@ const EXT_MAP: Record<string, SupportedLanguages> = {'.mjs': SupportedLanguages.JavaScript,'.cjs': SupportedLanguages.JavaScript,'.py': SupportedLanguages.Python,+ '.pyi': SupportedLanguages.Python,'.java': SupportedLanguages.Java,'.c': SupportedLanguages.C,'.h': SupportedLanguages.C,@@ -49,6 +50,10 @@ const EXT_MAP: Record<string, SupportedLanguages> = {'.cc': SupportedLanguages.CPlusPlus,'.cxx': SupportedLanguages.CPlusPlus,'.hpp': SupportedLanguages.CPlusPlus,+ // CUDA is C++ plus GPU qualifiers; kernel-launch syntax is neutralised at+ // parse time (see blankCudaLaunchConfigs in index.ts).+ '.cu': SupportedLanguages.CPlusPlus,+ '.cuh': SupportedLanguages.CPlusPlus,'.cs': SupportedLanguages.CSharp,'.go': SupportedLanguages.Go,'.rb': SupportedLanguages.Ruby,diff --git a/ix-cli/src/cli/supported-extensions.ts b/ix-cli/src/cli/supported-extensions.tsindex 61142cc3..83ee7af9 100644--- a/ix-cli/src/cli/supported-extensions.ts+++ b/ix-cli/src/cli/supported-extensions.ts@@ -8,7 +8,8 @@// name in the individual commands, not here.export const SUPPORTED_EXTENSIONS = new Set<string>([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",- ".py", ".java", ".c", ".h", ".cpp", ".cc", ".cxx", ".hpp",+ ".py", ".pyi", ".java", ".c", ".h", ".cpp", ".cc", ".cxx", ".hpp",+ ".cu", ".cuh",".cs", ".go", ".rb", ".rs", ".php", ".kt", ".kts", ".swift",".scala", ".sc",".yaml", ".yml",