Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat(ingest): parse CUDA (.cu/.cuh) and Python stubs (.pyi) (#593)

KageBinary701700b40807
diff --git a/README.md b/README.md
index a3134a72..c96c3df8 100644
--- a/README.md
+++ b/README.md
@@ -298,6 +298,10 @@ Ix parses and extracts symbols, calls, and imports across 26 languages, and reco
**Languages:**
JavaScript, TypeScript, Python, Java, C, C++, C#, Go, Ruby, Rust, PHP, Kotlin, Swift, Scala, R, SAS, Elixir, Haskell, Zig, Lua, Bash, HTML, XML, CSS, HCL / Terraform, Makefile
+CUDA (`.cu` / `.cuh`) is parsed with the C++ grammar; kernel-launch syntax
+(`kernel<<<grid, block>>>(args)`) is handled, so host-to-kernel calls appear in
+the graph. Python stub files (`.pyi`) are parsed as Python.
+
**Also recognized:**
YAML, JSON, TOML, SQL, Dockerfile, Markdown
diff --git a/core-ingestion/src/__tests__/extension-parity.test.ts b/core-ingestion/src/__tests__/extension-parity.test.ts
new file mode 100644
index 00000000..24933000
--- /dev/null
+++ b/core-ingestion/src/__tests__/extension-parity.test.ts
@@ -0,0 +1,48 @@
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+import { describe, expect, it } from 'vitest';
+
+import { languageFromPath } from '../languages.js';
+
+// supported-extensions.ts documents that it "MUST stay in sync" with EXT_MAP,
+// but nothing enforced it, so every new parser was one forgotten line away from
+// ingesting nothing: a file type the walker never yields is silently absent from
+// the graph rather than failing loudly.
+//
+// Both files are read as text so this stays independent of the ix-cli build.
+const read = (relative: string) =>
+ readFileSync(fileURLToPath(new URL(relative, import.meta.url)), 'utf8');
+
+const parsedExtensions = (() => {
+ const source = read('../languages.ts');
+ const map = source.slice(source.indexOf('const EXT_MAP'));
+ return new Set([...map.matchAll(/'(\.[a-z0-9]+)'\s*:/gi)].map((match) => match[1]));
+})();
+
+const walkedExtensions = new Set(
+ [...read('../../../ix-cli/src/cli/supported-extensions.ts')
+ .matchAll(/"(\.[a-z0-9]+)"/gi)].map((match) => match[1]),
+);
+
+describe('extension parity between EXT_MAP and SUPPORTED_EXTENSIONS', () => {
+ it('reads a non-empty set from both files', () => {
+ // Guards against a regex that silently matches nothing after a refactor,
+ // which would make every assertion below vacuously true.
+ expect(parsedExtensions.size).toBeGreaterThan(50);
+ expect(walkedExtensions.size).toBeGreaterThan(50);
+ });
+
+ it('walks every extension a parser handles', () => {
+ expect([...parsedExtensions].filter((ext) => !walkedExtensions.has(ext))).toEqual([]);
+ });
+
+ it('has a parser for every extension it walks', () => {
+ expect([...walkedExtensions].filter((ext) => !parsedExtensions.has(ext))).toEqual([]);
+ });
+
+ it('resolves each walked extension to a language', () => {
+ const unresolved = [...walkedExtensions].filter((ext) => !languageFromPath(`file${ext}`));
+ expect(unresolved).toEqual([]);
+ });
+});
diff --git a/core-ingestion/src/__tests__/queries.cuda.test.ts b/core-ingestion/src/__tests__/queries.cuda.test.ts
new file mode 100644
index 00000000..430516ef
--- /dev/null
+++ b/core-ingestion/src/__tests__/queries.cuda.test.ts
@@ -0,0 +1,111 @@
+import { describe, expect, it } from 'vitest';
+
+import { parseFile } from '../index.js';
+
+const KERNEL_SOURCE = `
+#include <cuda_runtime.h>
+
+__global__ void fused_attn(const float* q, float* out, int n) {
+ int i = blockIdx.x * blockDim.x + threadIdx.x;
+ __shared__ float tile[128];
+ if (i < n) out[i] = q[i];
+}
+
+void launch_attn(const float* q, float* out, int n) {
+ fused_attn<<<(n + 127) / 128, 128>>>(q, out, n);
+}
+`;
+
+const callTargets = (result: ReturnType<typeof parseFile>) =>
+ result!.relationships
+ .filter((relationship) => relationship.predicate === 'CALLS')
+ .map((relationship) => relationship.dstName);
+
+describe('CUDA', () => {
+ it('extracts kernels and host functions from a .cu file', () => {
+ const result = parseFile('/repo/kernels/attn.cu', KERNEL_SOURCE);
+
+ expect(result).not.toBeNull();
+ expect(result!.entities.map((entity) => entity.name)).toEqual(
+ expect.arrayContaining(['fused_attn', 'launch_attn']),
+ );
+ });
+
+ it('keeps the host -> kernel edge across the launch config', () => {
+ // The whole point: tree-sitter-cpp drops the call entirely when it chokes
+ // on `<<<...>>>`, so without the blanking pass this edge does not exist.
+ expect(callTargets(parseFile('/repo/kernels/attn.cu', KERNEL_SOURCE)))
+ .toEqual(expect.arrayContaining(['fused_attn']));
+ });
+
+ it('handles a launch config spanning several lines without shifting line numbers', () => {
+ const source = `void launch(float* x) {
+ my_kernel<<<
+ dim3(1, 2, 3),
+ 256
+ >>>(x);
+}
+
+__global__ void trailing_marker(int a) {}
+`;
+ const result = parseFile('/repo/kernels/multiline.cu', source);
+
+ expect(callTargets(result)).toEqual(expect.arrayContaining(['my_kernel']));
+
+ // `trailing_marker` sits on line 8; a space-only blank would have collapsed
+ // the newlines inside the launch config and dragged it upwards.
+ const marker = result!.entities.find((entity) => entity.name === 'trailing_marker');
+ expect(marker).toBeDefined();
+ expect(marker!.lineStart).toBe(8);
+ });
+
+ it('parses .cuh headers', () => {
+ const result = parseFile(
+ '/repo/kernels/attn.cuh',
+ '__global__ void fused_attn(const float* q, float* out, int n);\n',
+ );
+
+ expect(result).not.toBeNull();
+ expect(result!.entities.map((entity) => entity.name)).toEqual(
+ expect.arrayContaining(['fused_attn']),
+ );
+ });
+
+ it('leaves ordinary C++ untouched', () => {
+ // Nested templates, shift operators and arrows inside a string literal all
+ // contain the `<<<` / `>>>` character runs; none is a launch config.
+ const source = `#include <vector>
+void f(std::vector<std::vector<std::vector<int>>>& v, int a, int b) {
+ int x = a << b;
+ log("<<<HEAD>>>");
+ consume(v, x);
+}
+`;
+ const cpp = parseFile('/repo/src/plain.cpp', source);
+ const cu = parseFile('/repo/src/plain.cu', source);
+
+ expect(callTargets(cu)).toEqual(callTargets(cpp));
+ expect(callTargets(cu)).toEqual(expect.arrayContaining(['consume']));
+ });
+
+ it('stays linear on adversarial input', () => {
+ // Ingest input is an arbitrary repository, so the blanking pass is
+ // attacker-controlled. The obvious regex (`/<<<[^;]*?>>>(?=\\s*\\()/g`)
+ // is quadratic here: 16k `<` took 67ms and grew 4x per doubling.
+ const timeFor = (n: number) => {
+ const evil = `void f() { ${'<'.repeat(n)}>>>x }`;
+ const started = performance.now();
+ parseFile('/repo/evil.cu', evil);
+ return performance.now() - started;
+ };
+
+ timeFor(1_000); // warm the parser so the comparison is not first-call cost
+
+ const small = timeFor(8_000);
+ const large = timeFor(32_000);
+
+ // 4x the input. Quadratic would be ~16x; linear scanning stays well under.
+ expect(large).toBeLessThan(Math.max(small, 1) * 8);
+ });
+
+});
diff --git a/core-ingestion/src/__tests__/queries.python.test.ts b/core-ingestion/src/__tests__/queries.python.test.ts
index 2c0980f7..dee516bf 100644
--- a/core-ingestion/src/__tests__/queries.python.test.ts
+++ b/core-ingestion/src/__tests__/queries.python.test.ts
@@ -177,4 +177,28 @@ def list_items():
predicate: 'CALLS',
});
});
+
+ it('extracts signatures from a .pyi stub', () => {
+ // Stubs are the only place a compiled extension's API exists in Python, so
+ // without them a call into a native op dead-ends at the import.
+ const result = parseFile(
+ '/repo/torch_ext/_C.pyi',
+ `from typing import overload
+
+import torch
+
+def fused_attn(q: torch.Tensor, k: torch.Tensor, scale: float = 1.0) -> torch.Tensor: ...
+
+class KernelHandle:
+ device: int
+ def launch(self, n: int) -> None: ...
+`,
+ );
+
+ expect(result).not.toBeNull();
+ expect(result!.entities.map((entity) => entity.name)).toEqual(
+ expect.arrayContaining(['fused_attn', 'KernelHandle', 'launch']),
+ );
+ });
+
});
diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.ts
index d3c71491..8f1698ca 100644
--- a/core-ingestion/src/index.ts
+++ b/core-ingestion/src/index.ts
@@ -1989,6 +1989,79 @@ function unwrapRustCfgMacros(source: string): string {
return parts.join('');
}
+/**
+ * Blanks out CUDA kernel-launch configurations (`<<<grid, block>>>`) so that
+ * tree-sitter-cpp sees an ordinary call expression.
+ *
+ * The launch syntax is not valid C++: the grammar fails on it and, worse,
+ * recovers by dropping the whole call — so the edge from host code to the
+ * kernel it launches is lost, which is the one edge worth having in a CUDA
+ * file. Blanking the config in-place (preserving every character position and
+ * newline, as with the Rust macro unwrapper above) turns `k<<<a,b>>>(x)` into
+ * `k (x)`, which parses cleanly and yields the call.
+ *
+ * Hand-scanned rather than pattern-matched. The natural regex
+ * (`/<<<[^;]*?>>>(?=\s*\()/g`) is quadratic on `<`-heavy input — a file of
+ * 16k `<` took 67ms and grew 4x per doubling — and ingest input is an
+ * arbitrary repository, so that is a denial-of-service vector, not a
+ * micro-optimisation. This scanner never revisits a character: each failed
+ * search advances the cursor past the region it just rejected.
+ *
+ * A launch config is always followed immediately by the argument list, so
+ * requiring `(` after the closing `>>>` is what keeps this off ordinary C++ —
+ * nested templates (`vector<vector<vector<int>>>`), shift operators and
+ * `"<<<HEAD>>>"` in a string literal are all left alone. Stopping the search
+ * at `;` keeps a stray `<<<` in a comment from reaching a later `>>>`.
+ */
+const _isSpace = (ch: string) => ch === ' ' || ch === '\n' || ch === '\t' || ch === '\r';
+
+function blankCudaLaunchConfigs(source: string): string {
+ if (!source.includes('<<<')) return source;
+
+ const out: string[] = [];
+ let copiedTo = 0;
+ let cursor = 0;
+
+ while (cursor < source.length) {
+ const open = source.indexOf('<<<', cursor);
+ if (open === -1) break;
+
+ // Find the closing `>>>`, giving up at `;` — a launch config never spans a
+ // statement boundary.
+ let scan = open + 3;
+ while (scan < source.length && source[scan] !== ';' && !source.startsWith('>>>', scan)) {
+ scan += 1;
+ }
+
+ if (!source.startsWith('>>>', scan)) {
+ // Hit `;` or end of file. No `<<<` inside the region just scanned can
+ // reach a closing `>>>` either, so skip the whole thing.
+ cursor = scan + 1;
+ continue;
+ }
+
+ const close = scan + 3;
+ let afterConfig = close;
+ while (afterConfig < source.length && _isSpace(source[afterConfig])) afterConfig += 1;
+
+ if (source[afterConfig] !== '(') {
+ // A `>>>` that is not a launch — e.g. closing nested templates. Resume
+ // after it; the region behind us cannot match.
+ cursor = close;
+ continue;
+ }
+
+ out.push(source.slice(copiedTo, open));
+ out.push(source.slice(open, close).replace(_blankNonNewline, ' '));
+ copiedTo = close;
+ cursor = close;
+ }
+
+ if (copiedTo === 0) return source;
+ out.push(source.slice(copiedTo));
+ return out.join('');
+}
+
// ---------------------------------------------------------------------------
// Main parse function
// ---------------------------------------------------------------------------
@@ -2102,6 +2175,13 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
if (language === SupportedLanguages.Rust) {
parseSource = unwrapRustCfgMacros(parseSource);
}
+
+ // CUDA: neutralise kernel-launch configs so the host -> kernel call survives.
+ // Gated on the extension rather than the language so .cpp/.hpp parsing stays
+ // byte-identical.
+ if (filePath.endsWith('.cu') || filePath.endsWith('.cuh')) {
+ parseSource = blankCudaLaunchConfigs(parseSource);
+ }
const tree = parser.parse(parseSource, undefined, { bufferSize: parseSource.length + 1 });
const cacheKey = isTsx ? 'tsx' as const : language;
const query = getCachedQuery(cacheKey, grammar, queries);
diff --git a/core-ingestion/src/languages.ts b/core-ingestion/src/languages.ts
index e34d1382..8d411004 100644
--- a/core-ingestion/src/languages.ts
+++ b/core-ingestion/src/languages.ts
@@ -42,6 +42,7 @@ const EXT_MAP: Record<string, SupportedLanguages> = {
'.mjs': SupportedLanguages.JavaScript,
'.cjs': SupportedLanguages.JavaScript,
'.py': SupportedLanguages.Python,
+ '.pyi': SupportedLanguages.Python,
'.java': SupportedLanguages.Java,
'.c': SupportedLanguages.C,
'.h': SupportedLanguages.C,
@@ -49,6 +50,10 @@ const EXT_MAP: Record<string, SupportedLanguages> = {
'.cc': SupportedLanguages.CPlusPlus,
'.cxx': SupportedLanguages.CPlusPlus,
'.hpp': SupportedLanguages.CPlusPlus,
+ // CUDA is C++ plus GPU qualifiers; kernel-launch syntax is neutralised at
+ // parse time (see blankCudaLaunchConfigs in index.ts).
+ '.cu': SupportedLanguages.CPlusPlus,
+ '.cuh': SupportedLanguages.CPlusPlus,
'.cs': SupportedLanguages.CSharp,
'.go': SupportedLanguages.Go,
'.rb': SupportedLanguages.Ruby,
diff --git a/ix-cli/src/cli/supported-extensions.ts b/ix-cli/src/cli/supported-extensions.ts
index 61142cc3..83ee7af9 100644
--- a/ix-cli/src/cli/supported-extensions.ts
+++ b/ix-cli/src/cli/supported-extensions.ts
@@ -8,7 +8,8 @@
// name in the individual commands, not here.
export const SUPPORTED_EXTENSIONS = new Set<string>([
".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",
- ".py", ".java", ".c", ".h", ".cpp", ".cc", ".cxx", ".hpp",
+ ".py", ".pyi", ".java", ".c", ".h", ".cpp", ".cc", ".cxx", ".hpp",
+ ".cu", ".cuh",
".cs", ".go", ".rb", ".rs", ".php", ".kt", ".kts", ".swift",
".scala", ".sc",
".yaml", ".yml",