Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

perf: large repo map + ingest performance (#60)

Tanner Kocherbefde2c38786
diff --git a/CLAUDE.md b/CLAUDE.md
index 15f1f736..61258438 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -14,7 +14,7 @@ Use JSON when chaining command results.
2. AFTER every design or architecture decision → run `ix decide <title> --rationale <text>`.
3. When you notice contradictory information → run `ix conflicts` and present results to the user.
4. NEVER guess about codebase facts — if Ix has structured data, use it.
-5. IMMEDIATELY after modifying code → run `ix map` to re-ingest and update the graph.
+5. IMMEDIATELY after modifying code → run `ix map --silent` to re-ingest and update the graph.
6. When the user states a goal → run `ix truth add "<statement>"`.
## Ix CLI Command Routing
@@ -107,7 +107,7 @@ Workflows are staged command sequences (discover → implement → validate) att
```json
{
"discover": ["ix overview AuthService", "ix impact AuthService"],
- "implement": ["ix map"],
+ "implement": ["ix map --silent"],
"validate": ["ix smells --format json", "ix subsystems --format json"]
}
```
@@ -123,7 +123,7 @@ Workflows are staged command sequences (discover → implement → validate) att
### Ingestion & Health
| Goal | Command | Example |
|---|---|---|
-| Update graph + map | `ix map` | `ix map` |
+| Update graph + map | `ix map --silent` | `ix map --silent` |
| Ingest GitHub data | `ix ingest` | `ix ingest --github owner/repo --limit 50` |
| Backend health | `ix status` | `ix status` |
| Graph statistics | `ix stats` | `ix stats --format json` |
diff --git a/NOTICE b/NOTICE
index 28884be5..20d4cedc 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
Ix
-Copyright 2026 Ix Infrastructure
+Copyright 2026 Ix Infrastructure LLC
-This product includes software developed by Ix Infrastructure.
+This product includes software developed by Ix Infrastructure LLC.
diff --git a/assets/logo.png b/assets/logo.png
new file mode 100644
index 00000000..6eabd352
Binary files /dev/null and b/assets/logo.png differ
diff --git a/build.sbt b/build.sbt
index 4ccc51e6..1b129698 100644
--- a/build.sbt
+++ b/build.sbt
@@ -29,12 +29,6 @@ lazy val memoryLayer = (project in file("memory-layer"))
// ArangoDB
"com.arangodb" % "arangodb-java-driver" % "7.12.0",
- // Tree-sitter (bonede/tree-sitter-ng — ARM64 macOS compatible)
- "io.github.bonede" % "tree-sitter" % "0.26.6",
- "io.github.bonede" % "tree-sitter-scala" % "0.24.0",
- "io.github.bonede" % "tree-sitter-typescript" % "0.23.2",
- "io.github.bonede" % "tree-sitter-go" % "0.23.3",
-
// Logging
"org.typelevel" %% "log4cats-slf4j" % "2.7.0",
"ch.qos.logback" % "logback-classic" % "1.5.6",
diff --git a/core-ingestion/dist/index.d.ts b/core-ingestion/dist/index.d.ts
deleted file mode 100644
index d24611fc..00000000
--- a/core-ingestion/dist/index.d.ts
+++ /dev/null
@@ -1,104 +0,0 @@
-import { SupportedLanguages } from './languages.js';
-import type { RoleClassification } from './role-classifier.js';
-export interface ParsedEntity {
- name: string;
- kind: string;
- lineStart: number;
- lineEnd: number;
- language: string;
- /** Direct enclosing class/interface/trait, if any (undefined for file-level entities). */
- container?: string;
-}
-/** A semantic code span extracted from the AST. The primary LLM retrieval unit. */
-export interface ParsedChunk {
- /** Semantic name of the chunk (function/class/trait name), or null for file_body. */
- name: string | null;
- /** Chunk kind: "function" | "method" | "class" | "interface" | "trait" | "module_block" | "file_body" */
- chunkKind: string;
- lineStart: number;
- lineEnd: number;
- startByte: number;
- endByte: number;
- /** SHA-256 of the chunk source text for change detection and stable identity. */
- contentHash: string;
- language: string;
- /** Name of the directly enclosing class/trait/interface, if any. */
- container?: string;
-}
-export interface ParsedRelationship {
- srcName: string;
- dstName: string;
- predicate: string;
-}
-export interface FileParseResult {
- filePath: string;
- language: SupportedLanguages;
- entities: ParsedEntity[];
- chunks: ParsedChunk[];
- relationships: ParsedRelationship[];
- fileRole: RoleClassification;
-}
-/** Returns true if a grammar is installed for the given file's language. */
-export declare function isGrammarSupported(filePath: string): boolean;
-export declare function parseFile(filePath: string, source: string): FileParseResult | null;
-export interface ResolvedCallEdge {
- callerFilePath: string;
- callerName: string;
- calleeFilePath: string;
- calleeName: string;
- calleeQualifiedKey: string;
- confidence: number;
-}
-/** Generalised form of ResolvedCallEdge that covers any cross-file edge predicate. */
-export interface ResolvedEdge {
- srcFilePath: string;
- srcName: string;
- dstFilePath: string;
- dstName: string;
- dstQualifiedKey: string;
- predicate: string;
- confidence: number;
-}
-/**
- * Resolves CALLS and EXTENDS relationships to their cross-file targets by
- * building a symbol table and import map over the full batch of parsed files.
- *
- * Tiers (in priority order):
- * 0.9 — import-scoped: dst is in a file the src explicitly imports
- * 0.8 — transitive import-scoped: one re-export hop away
- * 0.5 — global fallback: dst exists in exactly one other file (unambiguous)
- *
- * Same-file edges are already handled correctly by buildPatch and are not
- * emitted here.
- *
- * Import resolution handles:
- * - Stem-based: './payments' → 'payments.ts'
- * - Path-aliased: '@/lib/payments' → 'payments.ts' (last segment)
- * - Directory index: './services' → 'services/index.ts'
- * - Bare aliases: '@components' → 'components.ts' (strip leading non-word chars)
- * - Dotted paths: 'ix.memory.model.Edge' → stem 'Edge' → 'Edge.scala'
- * (handles Scala/Java package imports where tree-sitter
- * emits each identifier separately)
- *
- * Qualified-key resolution:
- * - Module-level function/class: dstQualifiedKey === dstName
- * - Unambiguous class method: dstQualifiedKey === 'ClassName.method'
- * - Ambiguous (two entities share the same plain name): edge not emitted
- */
-/** Stats collected during resolution (exposed for debug instrumentation). */
-export interface ResolveStats {
- importLookups: number;
- transitiveLookups: number;
- globalFallbacks: number;
- globalCandidateTotal: number;
- resolvedImport: number;
- resolvedTransitive: number;
- resolvedGlobal: number;
- resolvedQualifier: number;
- skippedSameFile: number;
- skippedAmbiguous: number;
-}
-export declare function resolveEdges(results: FileParseResult[]): ResolvedEdge[];
-export declare function resolveEdges(results: FileParseResult[], statsOut: ResolveStats): ResolvedEdge[];
-/** @deprecated Use resolveEdges instead. */
-export declare function resolveCallEdges(results: FileParseResult[]): ResolvedCallEdge[];
diff --git a/core-ingestion/dist/index.js b/core-ingestion/dist/index.js
deleted file mode 100644
index 33d01b76..00000000
--- a/core-ingestion/dist/index.js
+++ /dev/null
@@ -1,1064 +0,0 @@
-import * as nodePath from 'node:path';
-import * as crypto from 'node:crypto';
-import { createRequire } from 'node:module';
-// @ts-ignore — tree-sitter has no bundled types
-import Parser from 'tree-sitter';
-// @ts-ignore
-import JavaScript from 'tree-sitter-javascript';
-// @ts-ignore
-import TypeScript from 'tree-sitter-typescript';
-// @ts-ignore
-import Python from 'tree-sitter-python';
-// @ts-ignore
-import Java from 'tree-sitter-java';
-// @ts-ignore
-import C from 'tree-sitter-c';
-// @ts-ignore
-import CPP from 'tree-sitter-cpp';
-// @ts-ignore
-import CSharp from 'tree-sitter-c-sharp';
-// @ts-ignore
-import Go from 'tree-sitter-go';
-// @ts-ignore
-import Rust from 'tree-sitter-rust';
-// @ts-ignore
-import Ruby from 'tree-sitter-ruby';
-// @ts-ignore
-import PHP from 'tree-sitter-php';
-// @ts-ignore
-import Scala from 'tree-sitter-scala';
-const _require = createRequire(import.meta.url);
-function tryLoadGrammar(pkg) {
- try {
- return _require(pkg);
- }
- catch {
- return null;
- }
-}
-const Kotlin = tryLoadGrammar('tree-sitter-kotlin');
-const Swift = tryLoadGrammar('tree-sitter-swift');
-import { SupportedLanguages, languageFromPath } from './languages.js';
-import { LANGUAGE_QUERIES } from './queries.js';
-import { classifyFileRole } from './role-classifier.js';
-// ---------------------------------------------------------------------------
-// Language → grammar map
-// ---------------------------------------------------------------------------
-const GRAMMAR_MAP = {
- [SupportedLanguages.JavaScript]: JavaScript,
- [SupportedLanguages.TypeScript]: TypeScript.typescript,
- [SupportedLanguages.Python]: Python,
- [SupportedLanguages.Java]: Java,
- [SupportedLanguages.C]: C,
- [SupportedLanguages.CPlusPlus]: CPP,
- [SupportedLanguages.CSharp]: CSharp,
- [SupportedLanguages.Go]: Go,
- [SupportedLanguages.Rust]: Rust,
- [SupportedLanguages.Ruby]: Ruby,
- [SupportedLanguages.PHP]: PHP.php_only,
- [SupportedLanguages.Scala]: Scala,
- ...(Kotlin ? { [SupportedLanguages.Kotlin]: Kotlin } : {}),
- ...(Swift ? { [SupportedLanguages.Swift]: Swift } : {}),
-};
-// Capture key prefix → NodeKind string
-const DEFINITION_KIND_MAP = {
- 'definition.class': 'class',
- 'definition.interface': 'interface',
- 'definition.function': 'function',
- 'definition.method': 'method',
- 'definition.struct': 'class',
- 'definition.enum': 'class',
- 'definition.trait': 'trait',
- 'definition.module': 'module',
- 'definition.namespace': 'module',
- 'definition.impl': 'class',
- 'definition.type': 'class',
- 'definition.property': 'function',
- 'definition.const': 'function',
- 'definition.static': 'function',
- 'definition.macro': 'function',
- 'definition.union': 'class',
- 'definition.typedef': 'class',
- 'definition.template': 'class',
- 'definition.record': 'class',
- 'definition.delegate': 'class',
- 'definition.annotation': 'class',
- 'definition.constructor': 'method',
-};
-// Primitive / built-in types to exclude from REFERENCES edges
-const TYPE_BUILTINS = new Set([
- // TypeScript / JavaScript
- 'string', 'number', 'boolean', 'void', 'null', 'undefined', 'any', 'never',
- 'unknown', 'object', 'bigint', 'symbol',
- // Java / Kotlin / Scala
- 'String', 'Integer', 'Long', 'Double', 'Float', 'Boolean', 'Byte', 'Short',
- 'Character', 'Object', 'Void', 'Int', 'Unit', 'Any', 'AnyVal', 'AnyRef',
- 'Nothing', 'Null', 'Char', 'Number',
- // C#
- 'decimal',
- // Python
- 'int', 'str', 'float', 'bool', 'bytes', 'Optional', 'Union', 'List', 'Dict',
- 'Set', 'Tuple', 'Type',
- // Rust
- 'i8', 'i16', 'i32', 'i64', 'i128', 'u8', 'u16', 'u32', 'u64', 'u128',
- 'f32', 'f64', 'usize', 'isize',
- // Go
- 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64',
- 'float32', 'float64', 'byte', 'rune', 'error',
- // Common stdlib collections / wrappers
- 'Array', 'Map', 'Seq', 'Vector', 'Option', 'Future', 'Promise', 'Result',
- 'Either', 'Try', 'IO', 'Observable', 'Iterator', 'Iterable',
- // C / C++
- 'size_t', 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int8_t',
- 'int16_t', 'int32_t', 'int64_t',
-]);
-// Builtins to exclude from CALLS edges — split by language family to avoid
-// suppressing valid method names in other languages (e.g. `filter` is a Python
-// builtin but also a common ORM method; `warn`/`error` are JS console methods
-// but also valid Python logging method names).
-// Keywords and pseudo-variables that are never valid call targets in any language.
-const SHARED_BUILTINS = new Set([
- 'if', 'for', 'while', 'return', 'new', 'this', 'self',
- 'undefined', 'null', 'true', 'false',
- 'println', // Scala/Java/Kotlin/Rust standard-output builtin — noise in any language
-]);
-// Python-specific builtins (bare function calls like `len(x)`, `range(n)`, etc.)
-const PYTHON_BUILTINS = new Set([
- ...SHARED_BUILTINS,
- 'print', 'len', 'range', 'int', 'str', 'float', 'list', 'dict',
- 'set', 'tuple', 'type', 'isinstance', 'super', 'property', 'enumerate',
- 'zip', 'map', 'filter', 'sorted', 'any', 'all', 'min', 'max', 'sum',
- 'open', 'repr', 'abs', 'round', 'hash', 'id', 'callable', 'iter', 'next',
- 'vars', 'dir', 'getattr', 'setattr', 'hasattr', 'delattr',
-]);
-// JavaScript/TypeScript-specific builtins
-const JS_BUILTINS = new Set([
- ...SHARED_BUILTINS,
- 'console', 'log', 'warn', 'error', 'debug', 'info',
- 'module', 'exports',
- 'Promise', 'Array', 'Object', 'String', 'Number', 'Boolean', 'JSON',
- 'Math', 'Date', 'Error', 'Map', 'Set', 'Symbol',
- 'setTimeout', 'setInterval', 'clearTimeout', 'clearInterval',
- 'process', 'Buffer', 'global', 'window', 'document',
- 'require', 'fetch', 'parseInt', 'parseFloat', 'isNaN', 'isFinite',
-]);
-// Per-language BUILTINS lookup — falls back to shared for languages without a
-// specific set (e.g. Java, Go, Rust) so they only skip obvious non-calls.
-function builtinsForLanguage(lang) {
- switch (lang) {
- case SupportedLanguages.Python:
- return PYTHON_BUILTINS;
- case SupportedLanguages.JavaScript:
- case SupportedLanguages.TypeScript:
- return JS_BUILTINS;
- default:
- return SHARED_BUILTINS;
- }
-}
-/** Returns true if a grammar is installed for the given file's language. */
-export function isGrammarSupported(filePath) {
- const language = languageFromPath(filePath);
- if (!language)
- return false;
- if (filePath.endsWith('.tsx'))
- return true; // TSX uses TypeScript.tsx, always available
- return GRAMMAR_MAP[language] !== undefined;
-}
-// ---------------------------------------------------------------------------
-// Parser instance (reused across calls)
-// ---------------------------------------------------------------------------
-let _parser = null;
-const _queryCache = new Map();
-function getParser() {
- if (!_parser)
- _parser = new Parser();
- return _parser;
-}
-function getCachedQuery(language, grammar, querySource) {
- const cached = _queryCache.get(language);
- if (cached && cached.grammar === grammar)
- return cached.query;
- const query = new Parser.Query(grammar, querySource);
- _queryCache.set(language, { grammar, query });
- return query;
-}
-// ---------------------------------------------------------------------------
-// Rust: cfg macro unwrapping
-// ---------------------------------------------------------------------------
-/**
- * Blanks out feature-gating macro wrappers in Rust source so that the items
- * inside become visible to tree-sitter as top-level declarations.
- *
- * Replaces `cfg_rt! { ... }` (and similar) with the body contents in-place,
- * preserving every character position and line number so that entity line
- * numbers remain accurate.
- */
-function unwrapRustCfgMacros(source) {
- const re = /\bcfg_(?:rt_multi_thread|not_rt|rt|io|time|sync|net|fs|process|signal)!\s*\{/g;
- const chars = source.split('');
- let m;
- while ((m = re.exec(source)) !== null) {
- const openBrace = m.index + m[0].length - 1; // position of the `{`
- // Find matching closing brace by counting depth
- let depth = 0;
- let closeBrace = -1;
- for (let i = openBrace; i < source.length; i++) {
- if (source[i] === '{')
- depth++;
- else if (source[i] === '}') {
- depth--;
- if (depth === 0) {
- closeBrace = i;
- break;
- }
- }
- }
- if (closeBrace === -1)
- continue;
- // Blank out everything from start of macro name up to and including `{`
- for (let i = m.index; i <= openBrace; i++) {
- if (chars[i] !== '\n')
- chars[i] = ' ';
- }
- // Blank out the matching closing `}`
- chars[closeBrace] = ' ';
- }
- return chars.join('');
-}
-// ---------------------------------------------------------------------------
-// Main parse function
-// ---------------------------------------------------------------------------
-export function parseFile(filePath, source) {
- const language = languageFromPath(filePath);
- if (!language)
- return null;
- // TypeScript TSX uses a separate grammar
- const isTsx = filePath.endsWith('.tsx');
- const grammar = isTsx ? TypeScript.tsx : GRAMMAR_MAP[language];
- if (!grammar)
- return null;
- const queries = LANGUAGE_QUERIES[language];
- if (!queries)
- return null;
- try {
- const parser = getParser();
- parser.setLanguage(grammar);
- // tree-sitter-java cannot parse array-type annotations on varargs params
- // (e.g. `@Nullable Object @Nullable ... args`). The second annotation causes
- // error recovery to truncate the enclosing class_declaration, orphaning all
- // subsequent methods. Strip any @Annotation immediately before `...` in-memory
- // so the class body parses correctly.
- let parseSource = language === SupportedLanguages.Java
- ? source.replace(/@\w+\s*(?=\.\.\.)/g, '')
- : source;
- // Rust: unwrap feature-gating macros (cfg_rt! { ... }, cfg_io! { ... }, etc.)
- // These macros are transparent pass-throughs — their bodies contain normal items
- // that tree-sitter cannot see because they are parsed as raw token_tree nodes.
- // We blank out the macro call and matching closing brace in-place (preserving
- // character positions and line numbers) so the inner items become top-level.
- if (language === SupportedLanguages.Rust) {
- parseSource = unwrapRustCfgMacros(parseSource);
- }
- const tree = parser.parse(parseSource, undefined, { bufferSize: parseSource.length + 1 });
- const cacheKey = isTsx ? 'tsx' : language;
- const query = getCachedQuery(cacheKey, grammar, queries);
- const matches = query.matches(tree.rootNode);
- const fileName = nodePath.basename(filePath);
- let sourceLineCount = 1;
- for (let i = 0; i < source.length; i++) {
- if (source.charCodeAt(i) === 10)
- sourceLineCount++;
- }
- const entities = [
- { name: fileName, kind: 'file', lineStart: 1, lineEnd: sourceLineCount, language },
- ];
- const chunks = [];
- const relationships = [];
- const pendingChunks = [];
- // Track class ranges for containment: [name, startLine, endLine]
- const classRanges = [];
- // Track seen calls per enclosing scope to avoid duplicate CALLS edges
- const seenCalls = new Map();
- // Track seen type references per enclosing class/file to avoid duplicate REFERENCES edges
- const seenRefs = new Map();
- // Python: map function name → (param name → declared type) for typed-parameter qualifier substitution
- const paramTypeMap = new Map();
- // Python: map function name → (variable name → assigned type) for untyped-param qualifier substitution
- const assignTypeMap = new Map();
- // --- First pass: collect definitions ---
- for (const match of matches) {
- // Definition captures: name + definition.*
- const defCapture = match.captures.find((c) => c.name.startsWith('definition.'));
- const nameCapture = match.captures.find((c) => c.name === 'name');
- if (defCapture) {
- const kind = DEFINITION_KIND_MAP[defCapture.name] ?? 'function';
- const name = nameCapture?.node.text
- ?? (defCapture.name === 'definition.constructor' ? 'init' : '');
- if (!name || name.length === 0)
- continue;
- const defNode = defCapture.node;
- const lineStart = defNode.startPosition.row + 1;
- const lineEnd = defNode.endPosition.row + 1;
- const startByte = defNode.startIndex;
- const endByte = defNode.endIndex;
- // Containment: file CONTAINS or class CONTAINS.
- // For Go methods the receiver type IS the container — methods are defined
- // outside the struct body so findEnclosing() always returns null for them.
- // The receiver.type capture (emitted by the Go method queries) overrides this.
- const enclosing = findEnclosing(classRanges, lineStart, name);
- const receiverCapture = match.captures.find((c) => c.name === 'receiver.type');
- // heritageClassCapture is set by the Go interface method query, where @heritage.class
- // captures the interface name and serves as the container (like receiver.type for structs).
- const heritageClassCapture = match.captures.find((c) => c.name === 'heritage.class');
- const effectiveContainer = (receiverCapture && kind === 'method')
- ? receiverCapture.node.text
- : (heritageClassCapture && kind === 'method')
- ? heritageClassCapture.node.text
- : (enclosing ?? undefined);
- entities.push({
- name,
- kind,
- lineStart,
- lineEnd,
- language,
- container: effectiveContainer,
- });
- pendingChunks.push({
- name,
- chunkKind: kind,
- lineStart,
- lineEnd,
- startByte,
- endByte,
- language,
- container: enclosing ?? undefined,
- });
- if (kind === 'class' || kind === 'interface' || kind === 'trait') {
- classRanges.push({ name, start: lineStart, end: lineEnd });
- }
- if (effectiveContainer) {
- relationships.push({ srcName: effectiveContainer, dstName: name, predicate: 'CONTAINS' });
- }
- else {
- relationships.push({ srcName: fileName, dstName: name, predicate: 'CONTAINS' });
- }
- continue;
- }
- // Heritage: EXTENDS
- const heritageClass = match.captures.find((c) => c.name === 'heritage.class');
- const heritageExtends = match.captures.find((c) => c.name === 'heritage.extends' || c.name === 'heritage.trait');
- if (heritageClass && heritageExtends) {
- relationships.push({
- srcName: heritageClass.node.text,
- dstName: heritageExtends.node.text,
- predicate: 'EXTENDS',
- });
- continue;
- }
- // Heritage: IMPLEMENTS (separate edge type, use EXTENDS for simplicity)
- const heritageImpl = match.captures.find((c) => c.name === 'heritage.implements');
- if (heritageClass && heritageImpl) {
- relationships.push({
- srcName: heritageClass.node.text,
- dstName: heritageImpl.node.text,
- predicate: 'EXTENDS',
- });
- continue;
- }
- // Python typed parameters: build paramTypeMap for qualifier substitution in the second pass.
- // Maps function name → (param name → declared type) so that e.g. `query: Query` lets us
- // rewrite `query.filter(...)` → `Query.filter` when building effectiveCallee.
- if (language === SupportedLanguages.Python) {
- const typedParamScope = match.captures.find((c) => c.name === '_typed_param_scope');
- const typedParamName = match.captures.find((c) => c.name === '_typed_param_name');
- const typedParamType = match.captures.find((c) => c.name === '_typed_param_type');
- if (typedParamScope && typedParamName && typedParamType) {
- const funcName = typedParamScope.node.childForFieldName?.('name')?.text;
- if (funcName) {
- if (!paramTypeMap.has(funcName))
- paramTypeMap.set(funcName, new Map());
- paramTypeMap.get(funcName).set(typedParamName.node.text, typedParamType.node.text);
- }
- continue;
- }
- // Assignment tracking: x = SomeClass() or x = Model.objects.method()
- const assignScope = match.captures.find((c) => c.name === '_assign_scope');
- const assignLhs = match.captures.find((c) => c.name === '_assign_lhs');
- const assignRhsType = match.captures.find((c) => c.name === '_assign_rhs_type');
- if (assignScope && assignLhs && assignRhsType) {
- // Only track PascalCase RHS names (constructors by convention).
- // Lowercase function calls like `select(...)` or `create_engine(...)` are skipped
- // to avoid cross-module false edges (e.g. orm.query → select.where in SQLAlchemy).
- const rhsName = assignRhsType.node.text;
- if (/^[A-Z]/.test(rhsName)) {
- const funcName = assignScope.node.childForFieldName?.('name')?.text;
- if (funcName) {
- if (!assignTypeMap.has(funcName))
- assignTypeMap.set(funcName, new Map());
- assignTypeMap.get(funcName).set(assignLhs.node.text, rhsName);
- }
- }
- continue;
- }
- }
- }
- // --- Second pass: calls and imports ---
- for (const match of matches) {
- // Full import statement captures (Scala: reconstructs dotted package paths)
- const importStmt = match.captures.find((c) => c.name === 'import.stmt');
- if (importStmt) {
- const raw = importStmt.node.text.replace(/^import\s+/, '').trim();
- const line = importStmt.node.startPosition.row + 1;
- if (raw.endsWith('._')) {
- // Wildcard: "ix.memory.model._" → package path "ix.memory.model"
- const pkgPath = raw.slice(0, -2);
- entities.push({ name: pkgPath, kind: 'module', lineStart: line, lineEnd: line, language });
- relationships.push({ srcName: fileName, dstName: pkgPath, predicate: 'IMPORTS' });
- }
- else {
- const braceIdx = raw.lastIndexOf('.{');
- if (braceIdx !== -1) {
- // Selective: "ix.memory.model.{NodeKind, GraphNode}"
- const prefix = raw.slice(0, braceIdx);
- const names = raw.slice(braceIdx + 2, -1).split(',').map((s) => s.trim()).filter(Boolean);
- for (const name of names) {
- const dstName = `${prefix}.${name}`;
- entities.push({ name: dstName, kind: 'module', lineStart: line, lineEnd: line, language });
- relationships.push({ srcName: fileName, dstName, predicate: 'IMPORTS' });
- }
- }
- else {
- // Simple: "ix.memory.model.NodeKind"
- entities.push({ name: raw, kind: 'module', lineStart: line, lineEnd: line, language });
- relationships.push({ srcName: fileName, dstName: raw, predicate: 'IMPORTS' });
- }
- }
- continue;
- }
- // Import captures (JS/TS/Python path-based)
- const importSource = match.captures.find((c) => c.name === 'import.source');
- if (importSource) {
- let importPath = importSource.node.text
- .replace(/^["'`]|["'`]$/g, '') // strip quotes
- .replace(/\\\\/g, '/'); // normalise backslashes
- if (importPath.length > 0 && importPath !== '*') {
- const rawMod = importPath.split('/').filter((s) => s !== '*').pop() ?? importPath;
- const modName = rawMod.replace(/^\.+/, ''); // strip leading dots: '.models' → 'models'
- if (!modName)
- continue; // skip bare '.' relative imports
- entities.push({ name: modName, kind: 'module', lineStart: importSource.node.startPosition.row + 1, lineEnd: importSource.node.startPosition.row + 1, language });
- relationships.push({ srcName: fileName, dstName: modName, predicate: 'IMPORTS' });
- }
- continue;
- }
- // Import name captures (e.g. from . import utils — the symbol name, not the module path)
- const importName = match.captures.find((c) => c.name === 'import.name');
- if (importName) {
- const name = importName.node.text;
- if (name && name !== '*' && name.length > 1) {
- entities.push({ name, kind: 'module', lineStart: importName.node.startPosition.row + 1, lineEnd: importName.node.startPosition.row + 1, language });
- relationships.push({ srcName: fileName, dstName: name, predicate: 'IMPORTS' });
- }
- continue;
- }
- // Call captures
- const callName = match.captures.find((c) => c.name === 'call.name');
- if (callName) {
- const callee = callName.node.text;
- if (!callee || callee.length <= 1)
- continue;
- // If a _qualifier capture is present (field_expression / stable_identifier
- // patterns like NodeKind.Decision, or Python attribute calls like
- // Session.execute), emit the fully qualified name so that resolution can use
- // the qualifier to break ties between same-named symbols in different files.
- // BUILTINS filtering is skipped for attribute calls: method names like
- // `filter` or `map` are Python builtins when called bare, but are valid
- // user-defined method calls when invoked as `query.filter(...)`.
- const qualifierCapture = match.captures.find((c) => c.name === '_qualifier');
- if (builtinsForLanguage(language).has(callee) && !qualifierCapture)
- continue;
- // Find enclosing function/method for the call; fall back to enclosing class
- // (e.g. calls in val/lazy val body at class level) before falling back to file.
- const callLine = callName.node.startPosition.row + 1;
- const caller = findEnclosingFunction(entities, callLine)
- ?? findEnclosing(classRanges, callLine, '')
- ?? fileName;
- const scope = caller;
- if (!seenCalls.has(scope))
- seenCalls.set(scope, new Set());
- const seen = seenCalls.get(scope);
- const effectiveCallee = (() => {
- if (!qualifierCapture)
- return callee;
- let qualifier = qualifierCapture.node.text;
- // Python: 'self'/'cls' always refers to the enclosing class — substitute it
- // so the edge reads 'Query.filter' instead of 'self.filter', enabling
- // same-file resolution without type inference.
- if (language === SupportedLanguages.Python && (qualifier === 'self' || qualifier === 'cls')) {
- const enclosingClass = findEnclosing(classRanges, callLine, '');
- if (enclosingClass)
- qualifier = enclosingClass;
- }
- else if (qualifier === 'this') {
- // JS/TS: 'this' inside a class method refers to the enclosing class.
- // Substitute so e.g. `this.save()` → `Document.save` instead of `this.save`.
- const enclosingClass = findEnclosing(classRanges, callLine, '');
- if (enclosingClass)
- qualifier = enclosingClass;
- }
- else if (language === SupportedLanguages.Python) {
- // Typed-parameter substitution: if the qualifier is a param with a declared type,
- // use the type name so e.g. `query.filter(...)` → `Query.filter`.
- // Also check assignTypeMap for variables assigned from constructor/ORM calls.
- const funcName = findEnclosingFunction(entities, callLine);
- if (funcName) {
- const typeForParam = paramTypeMap.get(funcName)?.get(qualifier);
- const typeForAssign = assignTypeMap.get(funcName)?.get(qualifier);
- if (typeForParam)
- qualifier = typeForParam;
- else if (typeForAssign)
- qualifier = typeForAssign;
- }
- }
- return `${qualifier}.${callee}`;
- })();
- if (!seen.has(effectiveCallee)) {
- seen.add(effectiveCallee);
- relationships.push({ srcName: caller, dstName: effectiveCallee, predicate: 'CALLS' });
- }
- continue;
- }
- // Type reference captures
- const refType = match.captures.find((c) => c.name === 'reference.type');
- if (refType) {
- const typeName = refType.node.text;
- if (!typeName || TYPE_BUILTINS.has(typeName) || typeName.length <= 1)
- continue;
- // Use the enclosing class as the src so the edge reads "ClassX REFERENCES TypeY".
- // Fall back to the file name when no enclosing class is found (top-level usage).
- const refLine = refType.node.startPosition.row + 1;
- const src = findEnclosing(classRanges, refLine, typeName) ?? fileName;
- if (!seenRefs.has(src))
- seenRefs.set(src, new Set());
- const seen = seenRefs.get(src);
- if (!seen.has(typeName)) {
- seen.add(typeName);
- relationships.push({ srcName: src, dstName: typeName, predicate: 'REFERENCES' });
- }
- continue;
- }
- }
- for (const pendingChunk of pendingChunks) {
- const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);
- const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);
- chunks.push({
- ...pendingChunk,
- contentHash,
- });
- }
- // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file
- if (chunks.length === 0) {
- const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);
- chunks.push({
- name: null,
- chunkKind: 'file_body',
- lineStart: 1,
- lineEnd: sourceLineCount,
- startByte: 0,
- endByte: source.length,
- contentHash,
- language,
- });
- }
- return { filePath, language, entities, chunks, relationships, fileRole: classifyFileRole(filePath, source) };
- }
- catch {
- return null;
- }
-}
-// ---------------------------------------------------------------------------
-// resolveCallEdges helpers
-// ---------------------------------------------------------------------------
-/** Build the qualified key for an entity (mirrors buildPatch's entityQKey logic). */
-function qualifiedKey(e) {
- return e.container ? `${e.container}.${e.name}` : e.name;
-}
-/**
- * Given a file's qualified-key map and a plain callee name, return the single
- * unambiguous qualified key, or null if the name maps to 0 or >1 entities.
- * Callers must treat null as "do not emit" to avoid dangling nodeIds.
- */
-function bestQKey(fileQKeys, filePath, plainName) {
- const qks = [...new Set(fileQKeys.get(filePath)?.get(plainName) ?? [])];
- return qks.length === 1 ? qks[0] : null;
-}
-/**
- * Resolves CALLS and EXTENDS relationships to their cross-file targets by
- * building a symbol table and import map over the full batch of parsed files.
- *
- * Tiers (in priority order):
- * 0.9 — import-scoped: dst is in a file the src explicitly imports
- * 0.8 — transitive import-scoped: one re-export hop away
- * 0.5 — global fallback: dst exists in exactly one other file (unambiguous)
- *
- * Same-file edges are already handled correctly by buildPatch and are not
- * emitted here.
- *
- * Import resolution handles:
- * - Stem-based: './payments' → 'payments.ts'
- * - Path-aliased: '@/lib/payments' → 'payments.ts' (last segment)
- * - Directory index: './services' → 'services/index.ts'
- * - Bare aliases: '@components' → 'components.ts' (strip leading non-word chars)
- * - Dotted paths: 'ix.memory.model.Edge' → stem 'Edge' → 'Edge.scala'
- * (handles Scala/Java package imports where tree-sitter
- * emits each identifier separately)
- *
- * Qualified-key resolution:
- * - Module-level function/class: dstQualifiedKey === dstName
- * - Unambiguous class method: dstQualifiedKey === 'ClassName.method'
- * - Ambiguous (two entities share the same plain name): edge not emitted
- */
-export function resolveEdges(results, stats) {
- // Provide a default no-op stats bag when caller passes none (backward compat).
- if (!stats)
- stats = {
- importLookups: 0, transitiveLookups: 0, globalFallbacks: 0,
- globalCandidateTotal: 0, resolvedImport: 0, resolvedTransitive: 0,
- resolvedGlobal: 0, resolvedQualifier: 0, skippedSameFile: 0, skippedAmbiguous: 0,
- };
- // fileQKeys: filePath → (plainName → qualifiedKeys[])
- // Mirrors the entityQKey computation in buildPatch so nodeIds match exactly.
- const fileQKeys = new Map();
- for (const r of results) {
- const qkMap = new Map();
- for (const e of r.entities) {
- if (e.kind === 'file' || e.kind === 'module')
- continue;
- const qk = qualifiedKey(e);
- const list = qkMap.get(e.name) ?? [];
- list.push(qk);
- qkMap.set(e.name, list);
- }
- fileQKeys.set(r.filePath, qkMap);
- }
- // fileHasSymbol: filePath → Set<plainName> (fast membership check)
- const fileHasSymbol = new Map();
- for (const [fp, qkMap] of fileQKeys) {
- fileHasSymbol.set(fp, new Set(qkMap.keys()));
- }
- // resultsByPath: O(1) lookup replacing results.find() in transitive import loop
- const resultsByPath = new Map(results.map(r => [r.filePath, r]));
- // symbolToFiles: plainName → filePath[] — replaces O(F) full-scan in tier-3 and qualifier fallback
- const symbolToFiles = new Map();
- for (const [fp, symbols] of fileHasSymbol) {
- for (const sym of symbols) {
- const list = symbolToFiles.get(sym) ?? [];
- list.push(fp);
- symbolToFiles.set(sym, list);
- }
- }
- // fileLanguage: filePath → SupportedLanguages (fast language lookup without re-calling languageFromPath)
- const fileLanguage = new Map();
- for (const r of results) {
- fileLanguage.set(r.filePath, r.language);
- }
- // stemToFiles: basename-without-extension → filePath[]
- // Matches the last path segment of an import to files in the batch.
- const stemToFiles = new Map();
- for (const r of results) {
- const stem = nodePath.basename(r.filePath, nodePath.extname(r.filePath));
- const list = stemToFiles.get(stem) ?? [];
- list.push(r.filePath);
- stemToFiles.set(stem, list);
- }
- // dirToIndexFiles: directory basename → filePath[] for 'index.*' files
- // Handles directory imports: './services' → 'services/index.ts'
- const dirToIndexFiles = new Map();
- for (const r of results) {
- const stem = nodePath.basename(r.filePath, nodePath.extname(r.filePath));
- if (stem === 'index') {
- const dirName = nodePath.basename(nodePath.dirname(r.filePath));
- const list = dirToIndexFiles.get(dirName) ?? [];
- list.push(r.filePath);
- dirToIndexFiles.set(dirName, list);
- }
- }
- // packageToFiles: dotted package path → filePath[] (Scala/Java wildcard imports)
- // e.g. "ix.memory.model" → all .scala files under .../ix/memory/model/
- // Builds all suffix keys so that both "model" and "ix.memory.model" resolve.
- const packageToFiles = new Map();
- for (const r of results) {
- const ext = nodePath.extname(r.filePath);
- if (ext !== '.scala' && ext !== '.java')
- continue;
- const dir = nodePath.dirname(r.filePath);
- const parts = dir.split(/[/\\]/);
- // Cap at 8 segments to avoid noise from very shallow path components like "src", "main".
- const maxDepth = Math.min(8, parts.length);
- for (let i = parts.length - 1; i >= parts.length - maxDepth; i--) {
- const pkg = parts.slice(i).join('.');
- const list = packageToFiles.get(pkg) ?? [];
- if (!list.includes(r.filePath))
- list.push(r.filePath);
- packageToFiles.set(pkg, list);
- }
- }
- // goPkgDirToFiles: Go package directory name → filePath[]
- // In Go, the last segment of an import path IS the package directory name.
- // e.g. import "go.etcd.io/etcd/server/etcdserver" is stripped to modName "etcdserver"
- // by the import processor; this index maps that name to all .go files in
- // any directory named "etcdserver" within the ingested batch.
- const goPkgDirToFiles = new Map();
- for (const r of results) {
- if (nodePath.extname(r.filePath) !== '.go')
- continue;
- const dirName = nodePath.basename(nodePath.dirname(r.filePath));
- const list = goPkgDirToFiles.get(dirName) ?? [];
- if (!list.includes(r.filePath))
- list.push(r.filePath);
- goPkgDirToFiles.set(dirName, list);
- }
- // ── Helpers ────────────────────────────────────────────────────────
- /** Resolve a module name to matching file paths in the batch. */
- function modNameToFiles(modName, excludeFp) {
- const fps = [];
- // Strip leading non-word chars for bare aliases: '@components' → 'components'
- const candidates = [modName];
- const stripped = modName.replace(/^[^a-zA-Z0-9_]+/, '');
- if (stripped && stripped !== modName)
- candidates.push(stripped);
- // Strip file extensions so "explain.js" resolves to the "explain" stem
- // (TS/JS ESM imports use .js extensions that map to .ts source files)
- const noExt = modName.replace(/\.(js|ts|mjs|cjs|jsx|tsx|py|scala|java|c|cc|cpp|cxx|h|hh|hpp|hxx)$/, '');
- if (noExt !== modName && noExt && !candidates.includes(noExt))
- candidates.push(noExt);
- // For dotted paths (Scala/Java: 'ix.memory.model.Edge'), also try last segment
- const lastDot = modName.lastIndexOf('.');
- if (lastDot !== -1) {
- const lastSegment = modName.slice(lastDot + 1);
- if (lastSegment && !candidates.includes(lastSegment))
- candidates.push(lastSegment);
- }
- for (const cand of candidates) {
- for (const fp of stemToFiles.get(cand) ?? []) {
- if (fp !== excludeFp)
- fps.push(fp);
- }
- for (const fp of dirToIndexFiles.get(cand) ?? []) {
- if (fp !== excludeFp)
- fps.push(fp);
- }
- }
- // Package-path wildcard resolution (Scala/Java): "ix.memory.model" → all files in that dir
- for (const fp of packageToFiles.get(modName) ?? []) {
- if (fp !== excludeFp && !fps.includes(fp))
- fps.push(fp);
- }
- // Go package directory resolution: "etcdserver" → all .go files in .../etcdserver/
- // The last segment of a Go import path is the package directory name, not a file stem.
- for (const fp of goPkgDirToFiles.get(modName) ?? []) {
- if (fp !== excludeFp && !fps.includes(fp))
- fps.push(fp);
- }
- return fps;
- }
- function tokenizeSymbolParts(value) {
- return value
- .replace(/([a-z\d])([A-Z])/g, '$1 $2')
- .replace(/[^a-zA-Z0-9]+/g, ' ')
- .toLowerCase()
- .split(/\s+/)
- .filter(Boolean);
- }
- const GENERIC_RESOLUTION_TOKENS = new Set([
- 'run', 'job', 'file', 'files', 'table', 'tables', 'output', 'outputs',
- 'input', 'inputs', 'background',
- ]);
- function candidateStem(filePath) {
- return nodePath.basename(filePath, nodePath.extname(filePath)).toLowerCase();
- }
- function pickCallerAlignedCandidate(matches, srcName, dstName) {
- if (matches.length === 0)
- return null;
- const srcTokens = new Set(tokenizeSymbolParts(srcName).filter(token => !GENERIC_RESOLUTION_TOKENS.has(token)));
- const overlapScores = matches.map(fp => {
- const qKeys = fileQKeys.get(fp)?.get(dstName) ?? [];
- const candidateTokens = new Set(tokenizeSymbolParts(candidateStem(fp)));
- for (const qKey of qKeys) {
- for (const token of tokenizeSymbolParts(qKey))
- candidateTokens.add(token);
- }
- let overlap = 0;
- for (const token of candidateTokens) {
- if (!GENERIC_RESOLUTION_TOKENS.has(token) && srcTokens.has(token))
- overlap++;
- }
- return { fp, overlap };
- });
- const maxOverlap = Math.max(...overlapScores.map(x => x.overlap));
- if (maxOverlap <= 0)
- return null;
- const bestMatches = overlapScores.filter(x => x.overlap === maxOverlap);
- return {
- chosen: bestMatches.length === 1 ? bestMatches[0].fp : null,
- best: bestMatches,
- };
- }
- function narrowCCandidates(matches, srcFilePath, srcLanguage, srcName, dstName) {
- if (matches.length <= 1)
- return matches;
- // C# partial class narrowing: multiple files may define the same class via
- // partial class (e.g. JsonReader.cs + JsonReader.Async.cs both define JsonReader).
- // Prefer the canonical file whose stem exactly matches the destination class name
- // over variant files that have additional dot-segments in the stem.
- if (srcLanguage === SupportedLanguages.CSharp) {
- const dstNameLower = dstName.toLowerCase();
- const canonicalMatches = matches.filter(fp => candidateStem(fp) === dstNameLower);
- if (canonicalMatches.length === 1)
- return canonicalMatches;
- }
- if (srcLanguage !== SupportedLanguages.C && srcLanguage !== SupportedLanguages.CPlusPlus)
- return matches;
- let narrowed = matches;
- const implExts = ['.c', '.cpp', '.cc', '.cxx'];
- const implMatches = narrowed.filter(fp => implExts.some(ext => fp.endsWith(ext)));
- if (implMatches.length === 1)
- narrowed = implMatches;
- if (narrowed.length <= 1)
- return narrowed;
- const srcParts = srcFilePath.replace(/\\/g, '/').split('/');
- const withProximity = narrowed.map(fp => {
- const fpParts = fp.replace(/\\/g, '/').split('/');
- let common = 0;
- while (common < srcParts.length && common < fpParts.length && srcParts[common] === fpParts[common]) {
- common++;
- }
- return { fp, common };
- });
- const maxCommon = Math.max(...withProximity.map(x => x.common));
- const proximityMatches = withProximity.filter(x => x.common === maxCommon).map(x => x.fp);
- if (proximityMatches.length === 1)
- return proximityMatches;
- const callerAligned = pickCallerAlignedCandidate(narrowed, srcName, dstName);
- if (callerAligned?.chosen)
- return [callerAligned.chosen];
- const callerAlignedCandidates = pickCallerAlignedCandidate(matches, srcName, dstName);
- if (callerAlignedCandidates?.best) {
- const implAlignedMatches = callerAlignedCandidates.best
- .map(match => match.fp)
- .filter(fp => ['.c', '.cpp', '.cc', '.cxx'].some(ext => fp.endsWith(ext)));
- if (implAlignedMatches.length === 1)
- return implAlignedMatches;
- }
- return narrowed;
- }
- // ── Main resolution loop ───────────────────────────────────────────
- const resolved = [];
- for (const result of results) {
- const srcFilePath = result.filePath;
- const srcLanguage = result.language;
- const srcSymbols = fileHasSymbol.get(srcFilePath);
- // Build the set of file paths this file explicitly imports.
- const importedFilePaths = new Set();
- for (const rel of result.relationships) {
- if (rel.predicate !== 'IMPORTS')
- continue;
- for (const fp of modNameToFiles(rel.dstName, srcFilePath)) {
- importedFilePaths.add(fp);
- }
- }
- // Build one-hop transitive import set.
- // Handles re-exports: baz.ts → index.ts (re-exports from bar.ts) → bar.ts
- const transitiveFilePaths = new Set();
- for (const fp of importedFilePaths) {
- const fpResult = resultsByPath.get(fp);
- if (!fpResult)
- continue;
- for (const rel of fpResult.relationships) {
- if (rel.predicate !== 'IMPORTS')
- continue;
- for (const transitiveFp of modNameToFiles(rel.dstName, srcFilePath)) {
- if (!importedFilePaths.has(transitiveFp))
- transitiveFilePaths.add(transitiveFp);
- }
- }
- }
- for (const rel of result.relationships) {
- if (rel.predicate !== 'CALLS' && rel.predicate !== 'EXTENDS' && rel.predicate !== 'REFERENCES' && rel.predicate !== 'IMPORTS')
- continue;
- // Resolve Scala/Java dotted class imports (e.g. "ix.memory.model.ClaimId")
- // to the actual indexed entity node in the defining file.
- // Uses package-path lookup: extract package ("ix.memory.model") and entity
- // name ("ClaimId"), find files registered under that package, then find
- // the one that defines the entity.
- if (rel.predicate === 'IMPORTS') {
- if (result.language !== SupportedLanguages.Scala && result.language !== SupportedLanguages.Java)
- continue;
- const dstName = rel.dstName;
- const lastDot = dstName.lastIndexOf('.');
- if (lastDot === -1)
- continue; // not a dotted class import
- const entityName = dstName.slice(lastDot + 1);
- if (!entityName || entityName === '_')
- continue; // wildcard import
- const pkgPath = dstName.slice(0, lastDot); // e.g. "ix.memory.model"
- // Find files in the declared package that define this entity
- const pkgFiles = packageToFiles.get(pkgPath) ?? [];
- const matchFiles = pkgFiles.filter(fp => fp !== srcFilePath && fileHasSymbol.get(fp)?.has(entityName));
- if (matchFiles.length !== 1)
- continue; // ambiguous or not found
- const fp = matchFiles[0];
- const dstQualifiedKey = bestQKey(fileQKeys, fp, entityName);
- if (dstQualifiedKey === null)
- continue;
- resolved.push({ srcFilePath, srcName: rel.srcName, dstFilePath: fp, dstName, dstQualifiedKey, predicate: 'IMPORTS', confidence: 0.9 });
- continue;
- }
- const dstName = rel.dstName;
- const srcName = rel.srcName;
- // Tier 1b: qualifier-assisted (confidence 0.9 / 0.7)
- // For dotted names like "NodeKind.Decision" (emitted by field_expression queries):
- // find the file defining the qualifier, then check it also defines the member.
- // This breaks ties where both NodeKind.Decision and SourceType.Decision exist.
- const qualDot = dstName.lastIndexOf('.');
- if (qualDot !== -1) {
- const qualifierPart = dstName.slice(0, qualDot);
- const memberPart = dstName.slice(qualDot + 1);
- if (memberPart && qualifierPart) {
- // Try import-scoped qualifier first
- const qualImportMatches = [];
- for (const fp of importedFilePaths) {
- if (fileHasSymbol.get(fp)?.has(qualifierPart))
- qualImportMatches.push(fp);
- }
- if (qualImportMatches.length === 1) {
- const qfp = qualImportMatches[0];
- if (fileHasSymbol.get(qfp)?.has(memberPart)) {
- const dstQualifiedKey = bestQKey(fileQKeys, qfp, memberPart);
- if (dstQualifiedKey !== null) {
- // dstName must match rel.dstName so buildPatchWithResolution can look it up
- resolved.push({ srcFilePath, srcName, dstFilePath: qfp, dstName, dstQualifiedKey, predicate: rel.predicate, confidence: 0.9 });
- }
- }
- continue;
- }
- // Global qualifier fallback — use symbolToFiles index instead of full scan
- const qualGlobalMatches = (symbolToFiles.get(qualifierPart) ?? []).filter(fp => fp !== srcFilePath);
- if (qualGlobalMatches.length === 1) {
- const qfp = qualGlobalMatches[0];
- if (fileHasSymbol.get(qfp)?.has(memberPart)) {
- const dstQualifiedKey = bestQKey(fileQKeys, qfp, memberPart);
- if (dstQualifiedKey !== null) {
- resolved.push({ srcFilePath, srcName, dstFilePath: qfp, dstName, dstQualifiedKey, predicate: rel.predicate, confidence: 0.7 });
- }
- }
- }
- continue; // qualified name exhausted — don't try bare-name tiers
- }
- }
- // Tier 1: same-file — already correct in buildPatch, skip here
- if (srcSymbols.has(dstName))
- continue;
- // Tier 2: import-scoped (confidence 0.9)
- const importMatches = [];
- for (const fp of importedFilePaths) {
- if (fileHasSymbol.get(fp)?.has(dstName))
- importMatches.push(fp);
- }
- const narrowedImportMatches = narrowCCandidates(importMatches, srcFilePath, srcLanguage, srcName, dstName);
- if (narrowedImportMatches.length === 1) {
- const fp = narrowedImportMatches[0];
- const dstQualifiedKey = bestQKey(fileQKeys, fp, dstName);
- if (dstQualifiedKey === null)
- continue; // ambiguous — do not emit bad nodeId
- resolved.push({ srcFilePath, srcName, dstFilePath: fp, dstName, dstQualifiedKey, predicate: rel.predicate, confidence: 0.9 });
- continue;
- }
- // Tier 2.5: transitive import-scoped (confidence 0.8) — one re-export hop away
- const transitiveMatches = [];
- for (const fp of transitiveFilePaths) {
- if (fileHasSymbol.get(fp)?.has(dstName))
- transitiveMatches.push(fp);
- }
- const narrowedTransitiveMatches = narrowCCandidates(transitiveMatches, srcFilePath, srcLanguage, srcName, dstName);
- if (narrowedTransitiveMatches.length === 1) {
- const fp = narrowedTransitiveMatches[0];
- const dstQualifiedKey = bestQKey(fileQKeys, fp, dstName);
- if (dstQualifiedKey === null)
- continue;
- resolved.push({ srcFilePath, srcName, dstFilePath: fp, dstName, dstQualifiedKey, predicate: rel.predicate, confidence: 0.8 });
- continue;
- }
- // Tier 3: global fallback (confidence 0.5) — uses inverted symbol index
- // instead of scanning all files.
- stats.globalFallbacks++;
- const candidates = symbolToFiles.get(dstName) ?? [];
- let globalMatches = candidates.filter(fp => fp !== srcFilePath && fileLanguage.get(fp) === srcLanguage);
- const importHint = pickCallerAlignedCandidate(importMatches, srcName, dstName)?.chosen
- ?? pickCallerAlignedCandidate(transitiveMatches, srcName, dstName)?.chosen;
- if (importHint) {
- const hintedStem = candidateStem(importHint);
- const stemMatches = globalMatches.filter(fp => candidateStem(fp) === hintedStem);
- if (stemMatches.length > 0)
- globalMatches = stemMatches;
- }
- stats.globalCandidateTotal += globalMatches.length;
- const resolvedMatches = narrowCCandidates(globalMatches, srcFilePath, srcLanguage, srcName, dstName);
- if (resolvedMatches.length === 1) {
- const fp = resolvedMatches[0];
- const dstQualifiedKey = bestQKey(fileQKeys, fp, dstName);
- if (dstQualifiedKey === null)
- continue; // ambiguous — do not emit bad nodeId
- resolved.push({ srcFilePath, srcName, dstFilePath: fp, dstName, dstQualifiedKey, predicate: rel.predicate, confidence: 0.5 });
- stats.resolvedGlobal++;
- continue;
- }
- if (narrowedImportMatches.length > 1 || narrowedTransitiveMatches.length > 1 || resolvedMatches.length > 1) {
- stats.skippedAmbiguous++;
- }
- // 0 or >1 matches after all tiers — leave as dangling edge, do not emit
- }
- }
- return resolved;
-}
-/** @deprecated Use resolveEdges instead. */
-export function resolveCallEdges(results) {
- return resolveEdges(results).map(e => ({
- callerFilePath: e.srcFilePath,
- callerName: e.srcName,
- calleeFilePath: e.dstFilePath,
- calleeName: e.dstName,
- calleeQualifiedKey: e.dstQualifiedKey,
- confidence: e.confidence,
- }));
-}
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-function findEnclosing(ranges, line, excludeName) {
- // Find the innermost class/interface that contains this line
- let best = null;
- for (const r of ranges) {
- if (r.name === excludeName)
- continue;
- if (line >= r.start && line <= r.end) {
- if (!best || (r.end - r.start) < (best.end - best.start)) {
- best = r;
- }
- }
- }
- return best?.name ?? null;
-}
-function findEnclosingFunction(entities, line) {
- let best = null;
- for (const e of entities) {
- if (e.kind !== 'function' && e.kind !== 'method')
- continue;
- if (line >= e.lineStart && line <= e.lineEnd) {
- if (!best || (e.lineEnd - e.lineStart) < (best.lineEnd - best.lineStart)) {
- best = e;
- }
- }
- }
- return best ? (best.container ? `${best.container}.${best.name}` : best.name) : null;
-}
diff --git a/core-ingestion/dist/languages.d.ts b/core-ingestion/dist/languages.d.ts
deleted file mode 100644
index 009cf3cd..00000000
--- a/core-ingestion/dist/languages.d.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-export declare enum SupportedLanguages {
- JavaScript = "javascript",
- TypeScript = "typescript",
- Python = "python",
- Java = "java",
- C = "c",
- CPlusPlus = "cpp",
- CSharp = "csharp",
- Go = "go",
- Ruby = "ruby",
- Rust = "rust",
- PHP = "php",
- Kotlin = "kotlin",
- Swift = "swift",
- Scala = "scala"
-}
-export declare function languageFromPath(filePath: string): SupportedLanguages | null;
diff --git a/core-ingestion/dist/languages.js b/core-ingestion/dist/languages.js
deleted file mode 100644
index ae45ad26..00000000
--- a/core-ingestion/dist/languages.js
+++ /dev/null
@@ -1,47 +0,0 @@
-export var SupportedLanguages;
-(function (SupportedLanguages) {
- SupportedLanguages["JavaScript"] = "javascript";
- SupportedLanguages["TypeScript"] = "typescript";
- SupportedLanguages["Python"] = "python";
- SupportedLanguages["Java"] = "java";
- SupportedLanguages["C"] = "c";
- SupportedLanguages["CPlusPlus"] = "cpp";
- SupportedLanguages["CSharp"] = "csharp";
- SupportedLanguages["Go"] = "go";
- SupportedLanguages["Ruby"] = "ruby";
- SupportedLanguages["Rust"] = "rust";
- SupportedLanguages["PHP"] = "php";
- SupportedLanguages["Kotlin"] = "kotlin";
- SupportedLanguages["Swift"] = "swift";
- SupportedLanguages["Scala"] = "scala";
-})(SupportedLanguages || (SupportedLanguages = {}));
-const EXT_MAP = {
- '.ts': SupportedLanguages.TypeScript,
- '.tsx': SupportedLanguages.TypeScript,
- '.js': SupportedLanguages.JavaScript,
- '.jsx': SupportedLanguages.JavaScript,
- '.mjs': SupportedLanguages.JavaScript,
- '.cjs': SupportedLanguages.JavaScript,
- '.py': SupportedLanguages.Python,
- '.java': SupportedLanguages.Java,
- '.c': SupportedLanguages.C,
- '.h': SupportedLanguages.C,
- '.cpp': SupportedLanguages.CPlusPlus,
- '.cc': SupportedLanguages.CPlusPlus,
- '.cxx': SupportedLanguages.CPlusPlus,
- '.hpp': SupportedLanguages.CPlusPlus,
- '.cs': SupportedLanguages.CSharp,
- '.go': SupportedLanguages.Go,
- '.rb': SupportedLanguages.Ruby,
- '.rs': SupportedLanguages.Rust,
- '.php': SupportedLanguages.PHP,
- '.kt': SupportedLanguages.Kotlin,
- '.kts': SupportedLanguages.Kotlin,
- '.swift': SupportedLanguages.Swift,
- '.scala': SupportedLanguages.Scala,
- '.sc': SupportedLanguages.Scala,
-};
-export function languageFromPath(filePath) {
- const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase();
- return EXT_MAP[ext] ?? null;
-}
diff --git a/core-ingestion/dist/patch-builder.d.ts b/core-ingestion/dist/patch-builder.d.ts
deleted file mode 100644
index 56c3ae9b..00000000
--- a/core-ingestion/dist/patch-builder.d.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import type { GraphPatchPayload } from './types.js';
-import type { FileParseResult, ResolvedEdge } from './index.js';
-export declare function extractorName(): string;
-/** Previous extractor versions — their patches are superseded when re-ingesting. */
-export declare const PREVIOUS_EXTRACTORS: string[];
-export declare function buildPatch(result: FileParseResult, sourceHash: string, previousSourceHash?: string): GraphPatchPayload;
-export declare function buildPatchWithResolution(result: FileParseResult, sourceHash: string, resolvedEdges: ResolvedEdge[], previousSourceHash?: string): GraphPatchPayload;
diff --git a/core-ingestion/dist/patch-builder.js b/core-ingestion/dist/patch-builder.js
deleted file mode 100644
index 026cba0f..00000000
--- a/core-ingestion/dist/patch-builder.js
+++ /dev/null
@@ -1,415 +0,0 @@
-import * as crypto from 'node:crypto';
-import * as nodePath from 'node:path';
-// ---------------------------------------------------------------------------
-// Deterministic UUID from a string (matches existing CLI convention)
-// ---------------------------------------------------------------------------
-function deterministicId(input) {
- const hash = crypto.createHash('sha256').update(input).digest('hex');
- return [
- hash.slice(0, 8),
- hash.slice(8, 12),
- hash.slice(12, 16),
- hash.slice(16, 20),
- hash.slice(20, 32),
- ].join('-');
-}
-function normalizePath(filePath) {
- return filePath.replace(/\\/g, '/').toLowerCase();
-}
-function nodeId(filePath, name) {
- return deterministicId(`${normalizePath(filePath)}:${name}`);
-}
-function edgeId(filePath, src, dst, predicate) {
- return deterministicId(`${normalizePath(filePath)}:${src}:${dst}:${predicate}`);
-}
-/** Deterministic ID for a chunk — keyed on file + kind + name + start line to survive minor edits. */
-function chunkId(filePath, chunkKind, name, startLine) {
- return deterministicId(`${normalizePath(filePath)}:chunk:${chunkKind}:${name ?? 'file_body'}:${startLine}`);
-}
-// ---------------------------------------------------------------------------
-// Source type from file extension
-// ---------------------------------------------------------------------------
-function sourceType(filePath) {
- const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase();
- if (['.json', '.yaml', '.yml', '.toml', '.ini', '.conf', '.env'].includes(ext))
- return 'config';
- if (['.md', '.mdx', '.rst', '.txt'].includes(ext))
- return 'doc';
- return 'code';
-}
-export function extractorName() {
- return `tree-sitter/1.18`;
-}
-/** Previous extractor versions — their patches are superseded when re-ingesting. */
-export const PREVIOUS_EXTRACTORS = ['tree-sitter/1.17', 'tree-sitter/1.16', 'tree-sitter/1.15', 'tree-sitter/1.14', 'tree-sitter/1.13', 'tree-sitter/1.12', 'tree-sitter/1.11', 'tree-sitter/1.10', 'tree-sitter/1.9', 'tree-sitter/1.8', 'tree-sitter/1.7', 'tree-sitter/1.6', 'tree-sitter/1.5', 'tree-sitter/1.4', 'tree-sitter/1.3', 'tree-sitter/1.2', 'tree-sitter/1.1'];
-/** Compute a patchId for a (filePath, sourceHash, extractorVersion) triple. */
-function computePatchId(filePath, sourceHash, extractor) {
- return deterministicId(`${normalizePath(filePath)}:${sourceHash}:${extractor}`);
-}
-/** Compute the legacy patchId (pre-1.1 scheme, no extractor suffix). */
-function legacyPatchId(filePath, sourceHash) {
- return deterministicId(`${filePath}:${sourceHash}`);
-}
-// ---------------------------------------------------------------------------
-// Build a GraphPatchPayload from a FileParseResult
-// ---------------------------------------------------------------------------
-export function buildPatch(result, sourceHash, previousSourceHash) {
- const { filePath, entities, chunks, relationships } = result;
- const ops = [];
- // Build a qualified-key map so that same-named entities in different
- // enclosing classes within the same file get distinct nodeIds.
- // e.g. ClassA.update vs ClassB.update instead of both being "update".
- const entityQKey = new Map();
- for (const e of entities) {
- entityQKey.set(e, e.container ? `${e.container}.${e.name}` : e.name);
- }
- // Reverse lookup: plain name → list of qualified keys (for edge resolution).
- const nameToQKeys = new Map();
- for (const [e, qk] of entityQKey) {
- const list = nameToQKeys.get(e.name) ?? [];
- list.push(qk);
- nameToQKeys.set(e.name, list);
- }
- // Resolve a relationship endpoint to the best qualified key.
- // For unambiguous names (appear once), returns the single qualified key.
- // For ambiguous names (appear multiple times), falls back to the plain name
- // so that the edge still points to *something* deterministic.
- function resolveKey(name, container) {
- const rawKeys = nameToQKeys.get(name);
- if (!rawKeys)
- return name;
- // Deduplicate: @overload in Python (and similar patterns in other languages)
- // produces multiple definitions with identical qualified keys. A Set collapses
- // these so we don't mistake three `Session.execute` overloads for ambiguity.
- const keys = [...new Set(rawKeys)];
- if (keys.length === 1)
- return keys[0];
- // More than one distinct entity with this name — try to pick by container
- if (container) {
- const qualified = `${container}.${name}`;
- if (keys.includes(qualified))
- return qualified;
- }
- // Ambiguous: return plain name so we don't silently drop the edge
- return name;
- }
- // UpsertNode for each entity (deduplicated by id — last occurrence wins)
- const seenNodeIds = new Set();
- for (const e of entities) {
- const qk = entityQKey.get(e);
- const id = nodeId(filePath, qk);
- if (!seenNodeIds.has(id)) {
- seenNodeIds.add(id);
- const roleAttrs = e.kind === 'file'
- ? { role: result.fileRole.role, role_confidence: result.fileRole.role_confidence, role_signals: result.fileRole.role_signals }
- : { role: result.fileRole.role, role_source: 'inherited_from_file' };
- ops.push({
- type: 'UpsertNode',
- id,
- kind: e.kind,
- name: e.name,
- attrs: {
- line_start: e.lineStart,
- line_end: e.lineEnd,
- language: e.language,
- ...roleAttrs,
- },
- });
- }
- }
- // UpsertNode + edges for each chunk
- const fileNodeId = nodeId(filePath, entities.find(e => e.kind === 'file')?.name ?? filePath);
- for (const chunk of chunks) {
- const cid = chunkId(filePath, chunk.chunkKind, chunk.name, chunk.lineStart);
- const chunkName = chunk.name ?? `file_body:${chunk.lineStart}`;
- ops.push({
- type: 'UpsertNode',
- id: cid,
- kind: 'chunk',
- name: chunkName,
- attrs: {
- file_uri: filePath,
- language: chunk.language,
- chunk_kind: chunk.chunkKind,
- start_line: chunk.lineStart,
- end_line: chunk.lineEnd,
- start_byte: chunk.startByte,
- end_byte: chunk.endByte,
- content_hash: chunk.contentHash,
- parser_version: extractorName(),
- },
- });
- // File -[CONTAINS]-> Chunk
- ops.push({
- type: 'UpsertEdge',
- id: edgeId(filePath, 'file', chunkName, 'CONTAINS_CHUNK'),
- src: fileNodeId,
- dst: cid,
- predicate: 'CONTAINS_CHUNK',
- attrs: {},
- });
- // Chunk -[DEFINES]-> Symbol (only for named chunks)
- if (chunk.name !== null) {
- const symbolKey = chunk.container ? `${chunk.container}.${chunk.name}` : chunk.name;
- const symbolNid = nodeId(filePath, symbolKey);
- ops.push({
- type: 'UpsertEdge',
- id: edgeId(filePath, chunkName, symbolKey, 'DEFINES'),
- src: cid,
- dst: symbolNid,
- predicate: 'DEFINES',
- attrs: {},
- });
- }
- }
- // NEXT edges for source-order chunk adjacency
- for (let i = 0; i + 1 < chunks.length; i++) {
- const a = chunks[i];
- const b = chunks[i + 1];
- // Only link top-level chunks (no container) to avoid intra-class ordering noise
- if (a.container == null && b.container == null) {
- const aid = chunkId(filePath, a.chunkKind, a.name, a.lineStart);
- const bid = chunkId(filePath, b.chunkKind, b.name, b.lineStart);
- const aName = a.name ?? `file_body:${a.lineStart}`;
- const bName = b.name ?? `file_body:${b.lineStart}`;
- ops.push({
- type: 'UpsertEdge',
- id: edgeId(filePath, aName, bName, 'NEXT'),
- src: aid,
- dst: bid,
- predicate: 'NEXT',
- attrs: {},
- });
- }
- }
- // UpsertEdge for each relationship
- for (const r of relationships) {
- // For CONTAINS edges, srcName is the container of dstName — use that to disambiguate.
- const srcKey = resolveKey(r.srcName);
- const dstKey = r.predicate === 'CONTAINS'
- ? resolveKey(r.dstName, r.srcName)
- : resolveKey(r.dstName);
- ops.push({
- type: 'UpsertEdge',
- id: edgeId(filePath, srcKey, dstKey, r.predicate),
- src: nodeId(filePath, srcKey),
- dst: nodeId(filePath, dstKey),
- predicate: r.predicate,
- attrs: {},
- });
- }
- // AssertClaim for each relationship (feeds the confidence/conflict engine)
- for (const r of relationships) {
- const srcKey = resolveKey(r.srcName);
- ops.push({
- type: 'AssertClaim',
- entityId: nodeId(filePath, srcKey),
- field: `${r.predicate.toLowerCase()}:${r.dstName}`,
- value: r.dstName,
- confidence: null,
- });
- }
- // patchId is deterministic: same file + same content + same extractor → same id.
- const extractor = extractorName();
- const patchId = computePatchId(filePath, sourceHash, extractor);
- // When re-ingesting with new extractor version, replace the old patch so the
- // server accepts the new ops rather than deduplicating on the old patchId.
- const previousPatchId = previousSourceHash
- ? computePatchId(filePath, previousSourceHash, extractor)
- : legacyPatchId(filePath, sourceHash);
- // Also supersede any patches created by previous extractor versions for the same file+content.
- const replaces = [previousPatchId, ...PREVIOUS_EXTRACTORS.map(prev => computePatchId(filePath, sourceHash, prev))];
- return {
- patchId,
- actor: 'ix/ingestion',
- timestamp: new Date().toISOString(),
- source: {
- uri: filePath,
- sourceHash,
- extractor,
- sourceType: sourceType(filePath),
- },
- baseRev: 0,
- ops,
- replaces,
- intent: `Parsed ${nodePath.basename(filePath)}`,
- };
-}
-// ---------------------------------------------------------------------------
-// buildPatchWithResolution — like buildPatch but fixes CALLS edge dst to point
-// to the actual defining file for cross-file calls resolved by resolveCallEdges.
-// ---------------------------------------------------------------------------
-export function buildPatchWithResolution(result, sourceHash, resolvedEdges, previousSourceHash) {
- // Build lookup: `${srcName}:${predicate}:${dstName}` → { dstFilePath, dstQualifiedKey }
- // Callers should pass only edges for this file (pre-grouped) for best performance,
- // but we still tolerate the full array for backward compatibility.
- const edgeResolution = new Map();
- for (const edge of resolvedEdges) {
- if (edge.srcFilePath !== result.filePath)
- continue;
- edgeResolution.set(`${edge.srcName}:${edge.predicate}:${edge.dstName}`, {
- dstFilePath: edge.dstFilePath,
- dstQualifiedKey: edge.dstQualifiedKey,
- });
- }
- const { filePath, entities, chunks, relationships } = result;
- const ops = [];
- const entityQKey = new Map();
- for (const e of entities) {
- entityQKey.set(e, e.container ? `${e.container}.${e.name}` : e.name);
- }
- const nameToQKeys = new Map();
- for (const [e, qk] of entityQKey) {
- const list = nameToQKeys.get(e.name) ?? [];
- list.push(qk);
- nameToQKeys.set(e.name, list);
- }
- function resolveKey(name, container) {
- const rawKeys = nameToQKeys.get(name);
- if (!rawKeys)
- return name;
- const keys = [...new Set(rawKeys)]; // deduplicate — @overload produces identical qks
- if (keys.length === 1)
- return keys[0];
- if (container) {
- const qualified = `${container}.${name}`;
- if (keys.includes(qualified))
- return qualified;
- }
- return name;
- }
- const seenNodeIds2 = new Set();
- for (const e of entities) {
- const qk = entityQKey.get(e);
- const id = nodeId(filePath, qk);
- if (!seenNodeIds2.has(id)) {
- seenNodeIds2.add(id);
- const roleAttrs = e.kind === 'file'
- ? { role: result.fileRole.role, role_confidence: result.fileRole.role_confidence, role_signals: result.fileRole.role_signals }
- : { role: result.fileRole.role, role_source: 'inherited_from_file' };
- ops.push({
- type: 'UpsertNode',
- id,
- kind: e.kind,
- name: e.name,
- attrs: { line_start: e.lineStart, line_end: e.lineEnd, language: e.language, ...roleAttrs },
- });
- }
- }
- // UpsertNode + edges for each chunk (same logic as buildPatch)
- const fileNodeId2 = nodeId(filePath, entities.find(e => e.kind === 'file')?.name ?? filePath);
- for (const chunk of chunks) {
- const cid = chunkId(filePath, chunk.chunkKind, chunk.name, chunk.lineStart);
- const chunkName = chunk.name ?? `file_body:${chunk.lineStart}`;
- ops.push({
- type: 'UpsertNode',
- id: cid,
- kind: 'chunk',
- name: chunkName,
- attrs: {
- file_uri: filePath,
- language: chunk.language,
- chunk_kind: chunk.chunkKind,
- start_line: chunk.lineStart,
- end_line: chunk.lineEnd,
- start_byte: chunk.startByte,
- end_byte: chunk.endByte,
- content_hash: chunk.contentHash,
- parser_version: extractorName(),
- },
- });
- ops.push({
- type: 'UpsertEdge',
- id: edgeId(filePath, 'file', chunkName, 'CONTAINS_CHUNK'),
- src: fileNodeId2,
- dst: cid,
- predicate: 'CONTAINS_CHUNK',
- attrs: {},
- });
- if (chunk.name !== null) {
- const symbolKey = chunk.container ? `${chunk.container}.${chunk.name}` : chunk.name;
- const symbolNid = nodeId(filePath, symbolKey);
- ops.push({
- type: 'UpsertEdge',
- id: edgeId(filePath, chunkName, symbolKey, 'DEFINES'),
- src: cid,
- dst: symbolNid,
- predicate: 'DEFINES',
- attrs: {},
- });
- }
- }
- for (let i = 0; i + 1 < chunks.length; i++) {
- const a = chunks[i];
- const b = chunks[i + 1];
- if (a.container == null && b.container == null) {
- const aid = chunkId(filePath, a.chunkKind, a.name, a.lineStart);
- const bid = chunkId(filePath, b.chunkKind, b.name, b.lineStart);
- const aName = a.name ?? `file_body:${a.lineStart}`;
- const bName = b.name ?? `file_body:${b.lineStart}`;
- ops.push({
- type: 'UpsertEdge',
- id: edgeId(filePath, aName, bName, 'NEXT'),
- src: aid,
- dst: bid,
- predicate: 'NEXT',
- attrs: {},
- });
- }
- }
- for (const r of relationships) {
- const srcKey = resolveKey(r.srcName);
- const dstKey = r.predicate === 'CONTAINS'
- ? resolveKey(r.dstName, r.srcName)
- : resolveKey(r.dstName);
- // For cross-file resolved edges (CALLS, EXTENDS), use the defining file's nodeId
- let dstNodeId;
- const resolutionKey = `${r.srcName}:${r.predicate}:${r.dstName}`;
- if (edgeResolution.has(resolutionKey)) {
- const { dstFilePath, dstQualifiedKey } = edgeResolution.get(resolutionKey);
- dstNodeId = nodeId(dstFilePath, dstQualifiedKey);
- }
- else {
- dstNodeId = nodeId(filePath, dstKey);
- }
- ops.push({
- type: 'UpsertEdge',
- id: edgeId(filePath, srcKey, dstKey, r.predicate),
- src: nodeId(filePath, srcKey),
- dst: dstNodeId,
- predicate: r.predicate,
- attrs: {},
- });
- }
- for (const r of relationships) {
- const srcKey = resolveKey(r.srcName);
- ops.push({
- type: 'AssertClaim',
- entityId: nodeId(filePath, srcKey),
- field: `${r.predicate.toLowerCase()}:${r.dstName}`,
- value: r.dstName,
- confidence: null,
- });
- }
- const extractor = extractorName();
- const patchId = computePatchId(filePath, sourceHash, extractor);
- const previousPatchId = previousSourceHash
- ? computePatchId(filePath, previousSourceHash, extractor)
- : legacyPatchId(filePath, sourceHash);
- const replaces = [previousPatchId, ...PREVIOUS_EXTRACTORS.map(prev => computePatchId(filePath, sourceHash, prev))];
- return {
- patchId,
- actor: 'ix/ingestion',
- timestamp: new Date().toISOString(),
- source: {
- uri: filePath,
- sourceHash,
- extractor,
- sourceType: sourceType(filePath),
- },
- baseRev: 0,
- ops,
- replaces,
- intent: `Parsed ${nodePath.basename(filePath)}`,
- };
-}
diff --git a/core-ingestion/dist/queries.d.ts b/core-ingestion/dist/queries.d.ts
deleted file mode 100644
index a6b93dd6..00000000
--- a/core-ingestion/dist/queries.d.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-import { SupportedLanguages } from './languages.js';
-export declare const TYPESCRIPT_QUERIES = "\n(class_declaration\n name: (type_identifier) @name) @definition.class\n\n(interface_declaration\n name: (type_identifier) @name) @definition.interface\n\n(function_declaration\n name: (identifier) @name) @definition.function\n\n(method_definition\n name: (property_identifier) @name) @definition.method\n\n(lexical_declaration\n (variable_declarator\n name: (identifier) @name\n value: (arrow_function))) @definition.function\n\n(lexical_declaration\n (variable_declarator\n name: (identifier) @name\n value: (function_expression))) @definition.function\n\n(export_statement\n declaration: (lexical_declaration\n (variable_declarator\n name: (identifier) @name\n value: (arrow_function)))) @definition.function\n\n(export_statement\n declaration: (lexical_declaration\n (variable_declarator\n name: (identifier) @name\n value: (function_expression)))) @definition.function\n\n(import_statement\n source: (string) @import.source) @import\n\n; Re-export statements: export { X } from './y'\n(export_statement\n source: (string) @import.source) @import\n\n; Dynamic import with string literal: import('./foo') or await import('./foo')\n(call_expression\n function: (import)\n arguments: (arguments (string) @import.source)) @import\n\n(call_expression\n function: (identifier) @call.name) @call\n\n(call_expression\n function: (member_expression\n property: (property_identifier) @call.name)) @call\n\n; this.method() \u2014 capture this as qualifier so index.ts can substitute the enclosing class.\n; Analogous to Python's self/cls substitution.\n(call_expression\n function: (member_expression\n object: (this) @_qualifier\n property: (property_identifier) @call.name)) @call\n\n; Constructor calls: new Foo()\n(new_expression\n constructor: (identifier) @call.name) @call\n\n; Heritage queries - class extends\n(class_declaration\n name: (type_identifier) @heritage.class\n (class_heritage\n (extends_clause\n value: (identifier) @heritage.extends))) @heritage\n\n; Heritage queries - class implements interface\n(class_declaration\n name: (type_identifier) @heritage.class\n (class_heritage\n (implements_clause\n (type_identifier) @heritage.implements))) @heritage.impl\n\n; Heritage queries - interface extends interface\n(interface_declaration\n name: (type_identifier) @heritage.class\n (extends_type_clause\n (type_identifier) @heritage.extends)) @heritage\n\n; Type references \u2014 captures types used in annotations/parameters/return types\n(type_annotation (type_identifier) @reference.type)\n";
-export declare const JAVASCRIPT_QUERIES = "\n(class_declaration\n name: (identifier) @name) @definition.class\n\n(function_declaration\n name: (identifier) @name) @definition.function\n\n(method_definition\n name: (property_identifier) @name) @definition.method\n\n(lexical_declaration\n (variable_declarator\n name: (identifier) @name\n value: (arrow_function))) @definition.function\n\n(lexical_declaration\n (variable_declarator\n name: (identifier) @name\n value: (function_expression))) @definition.function\n\n; var foo = function() {} (CommonJS var declarations)\n(variable_declaration\n (variable_declarator\n name: (identifier) @name\n value: (function_expression))) @definition.function\n\n; var foo = () => {} (CommonJS var arrow functions)\n(variable_declaration\n (variable_declarator\n name: (identifier) @name\n value: (arrow_function))) @definition.function\n\n; obj.method = function name() {} (prototype/mixin method assignments)\n(expression_statement\n (assignment_expression\n left: (member_expression\n property: (property_identifier) @name)\n right: (function_expression))) @definition.function\n\n(export_statement\n declaration: (lexical_declaration\n (variable_declarator\n name: (identifier) @name\n value: (arrow_function)))) @definition.function\n\n(export_statement\n declaration: (lexical_declaration\n (variable_declarator\n name: (identifier) @name\n value: (function_expression)))) @definition.function\n\n(import_statement\n source: (string) @import.source) @import\n\n; Re-export statements: export { X } from './y'\n(export_statement\n source: (string) @import.source) @import\n\n; Dynamic import with string literal: import('./foo') or await import('./foo')\n(call_expression\n function: (import)\n arguments: (arguments (string) @import.source)) @import\n\n; CommonJS require() \u2192 IMPORTS\n(call_expression\n function: (identifier) @_req\n arguments: (arguments (string) @import.source)\n (#eq? @_req \"require\")) @import\n\n(call_expression\n function: (identifier) @call.name) @call\n\n(call_expression\n function: (member_expression\n property: (property_identifier) @call.name)) @call\n\n; this.method() \u2014 capture this as qualifier so index.ts can substitute the enclosing class.\n; Analogous to Python's self/cls substitution.\n(call_expression\n function: (member_expression\n object: (this) @_qualifier\n property: (property_identifier) @call.name)) @call\n\n; Constructor calls: new Foo()\n(new_expression\n constructor: (identifier) @call.name) @call\n\n; Heritage queries - class extends (JavaScript uses different AST than TypeScript)\n; In tree-sitter-javascript, class_heritage directly contains the parent identifier\n(class_declaration\n name: (identifier) @heritage.class\n (class_heritage\n (identifier) @heritage.extends)) @heritage\n";
-export declare const PYTHON_QUERIES = "\n(class_definition\n name: (identifier) @name) @definition.class\n\n(function_definition\n name: (identifier) @name) @definition.function\n\n(import_statement\n name: (dotted_name) @import.source) @import\n\n(import_from_statement\n module_name: (dotted_name) @import.source) @import\n\n(import_from_statement\n module_name: (relative_import) @import.source) @import\n\n; from . import utils / from .models import User \u2014 also capture the imported symbol names\n(import_from_statement\n module_name: (relative_import)\n name: (dotted_name (identifier) @import.name)) @import.names\n\n(call\n function: (identifier) @call.name) @call\n\n(call\n function: (attribute\n attribute: (identifier) @call.name)) @call\n\n; Attribute calls with object captured as qualifier so Tier-1b resolution can use\n; class/module name to break ties (e.g. Session.execute \u2192 resolves to Session in\n; the importing file). Emitted alongside the bare-name pattern above so that\n; bare-name global fallback still fires when qualifier resolution fails.\n(call\n function: (attribute\n object: (identifier) @_qualifier\n attribute: (identifier) @call.name)) @call\n\n; Django ORM manager pattern: Model.objects.method(...)\n; Unwinds the intermediate accessor (objects/related_manager/etc.) and captures\n; the model class as qualifier, e.g. User.objects.filter(...) \u2192 CALLS User.filter.\n; This creates cross-file edges between view/form code and model definitions.\n(call\n function: (attribute\n object: (attribute\n object: (identifier) @_qualifier\n attribute: (identifier))\n attribute: (identifier) @call.name)) @call\n\n; Chained call pattern: qualifier.method1(...).method2(...)\n; Propagates the qualifier from the inner call to the outer, e.g.\n; User.filter(...).order_by(...) \u2192 CALLS User.order_by.\n(call\n function: (attribute\n object: (call\n function: (attribute\n object: (identifier) @_qualifier\n attribute: (identifier)))\n attribute: (identifier) @call.name)) @call\n\n; Heritage queries - Python class inheritance\n(class_definition\n name: (identifier) @heritage.class\n superclasses: (argument_list\n (identifier) @heritage.extends)) @heritage\n\n; Type references \u2014 captures types used in type annotations / hints\n(typed_parameter type: (type (identifier) @reference.type))\n(typed_default_parameter type: (type (identifier) @reference.type))\n\n; Typed parameters: capture name + type together so qualifier substitution can\n; map a local variable name to its declared class (e.g. query: Query \u2192 'query' \u2192 'Query').\n(function_definition\n parameters: (parameters\n (typed_parameter\n (identifier) @_typed_param_name\n type: (type (identifier) @_typed_param_type)))) @_typed_param_scope\n\n(function_definition\n parameters: (parameters\n (typed_default_parameter\n (identifier) @_typed_param_name\n type: (type (identifier) @_typed_param_type)))) @_typed_param_scope\n\n; Assignment tracking: x = SomeClass() or x = SomeClass(args)\n; Maps local variable name \u2192 constructor type for untyped-parameter qualifier substitution.\n; Only PascalCase RHS names are tracked in TypeScript (constructor convention).\n(function_definition\n body: (block\n (expression_statement\n (assignment\n left: (identifier) @_assign_lhs\n right: (call\n function: (identifier) @_assign_rhs_type))))) @_assign_scope\n\n; Assignment tracking: x = Model.objects.method()\n; Captures the outermost identifier (model class) for ORM manager patterns.\n(function_definition\n body: (block\n (expression_statement\n (assignment\n left: (identifier) @_assign_lhs\n right: (call\n function: (attribute\n object: (attribute\n object: (identifier) @_assign_rhs_type\n attribute: (identifier))\n attribute: (identifier))))))) @_assign_scope\n";
-export declare const JAVA_QUERIES = "\n; Classes, Interfaces, Enums, Annotations\n(class_declaration name: (identifier) @name) @definition.class\n(interface_declaration name: (identifier) @name) @definition.interface\n(enum_declaration name: (identifier) @name) @definition.enum\n(annotation_type_declaration name: (identifier) @name) @definition.annotation\n\n; Methods & Constructors\n(method_declaration name: (identifier) @name) @definition.method\n(constructor_declaration name: (identifier) @name) @definition.constructor\n\n; Imports - capture any import declaration child as source\n(import_declaration (_) @import.source) @import\n\n; Calls\n(method_invocation name: (identifier) @call.name) @call\n(method_invocation object: (_) name: (identifier) @call.name) @call\n\n; Constructor calls: new Foo()\n(object_creation_expression type: (type_identifier) @call.name) @call\n\n; Heritage - extends class (plain: extends Foo)\n(class_declaration name: (identifier) @heritage.class\n (superclass (type_identifier) @heritage.extends)) @heritage\n\n; Heritage - extends class with generics (extends Foo<T>)\n(class_declaration name: (identifier) @heritage.class\n (superclass (generic_type (type_identifier) @heritage.extends))) @heritage\n\n; Heritage - implements interfaces (plain: implements Foo)\n(class_declaration name: (identifier) @heritage.class\n (super_interfaces (type_list (type_identifier) @heritage.implements))) @heritage.impl\n\n; Heritage - implements interfaces with generics (implements Foo<T>)\n(class_declaration name: (identifier) @heritage.class\n (super_interfaces (type_list (generic_type (type_identifier) @heritage.implements)))) @heritage.impl\n\n; Heritage - interface extends interfaces (plain: extends Foo)\n(interface_declaration name: (identifier) @heritage.class\n (extends_interfaces (type_list (type_identifier) @heritage.extends))) @heritage\n\n; Heritage - interface extends interfaces with generics (extends Foo<T>)\n(interface_declaration name: (identifier) @heritage.class\n (extends_interfaces (type_list (generic_type (type_identifier) @heritage.extends)))) @heritage\n\n; Type references \u2014 captures types used in field/param/return-type positions\n(field_declaration type: (type_identifier) @reference.type)\n(formal_parameter type: (type_identifier) @reference.type)\n(local_variable_declaration type: (type_identifier) @reference.type)\n(method_declaration type: (type_identifier) @reference.type)\n";
-export declare const C_QUERIES = "\n; Functions (direct declarator)\n(function_definition declarator: (function_declarator declarator: (identifier) @name)) @definition.function\n(declaration declarator: (function_declarator declarator: (identifier) @name)) @definition.function\n\n; Functions returning pointers (pointer_declarator wraps function_declarator)\n(function_definition declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name))) @definition.function\n(declaration declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name))) @definition.function\n\n; Functions returning double pointers (nested pointer_declarator)\n(function_definition declarator: (pointer_declarator declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name)))) @definition.function\n\n; Structs, Unions, Enums, Typedefs\n; Only match struct/union with a body to avoid treating forward declarations as definitions\n(struct_specifier name: (type_identifier) @name body: (field_declaration_list)) @definition.struct\n(union_specifier name: (type_identifier) @name body: (field_declaration_list)) @definition.union\n(enum_specifier name: (type_identifier) @name) @definition.enum\n(type_definition declarator: (type_identifier) @name) @definition.typedef\n\n; Macros\n(preproc_function_def name: (identifier) @name) @definition.macro\n(preproc_def name: (identifier) @name) @definition.macro\n\n; Includes\n(preproc_include path: (_) @import.source) @import\n\n; Calls\n(call_expression function: (identifier) @call.name) @call\n(call_expression function: (field_expression field: (field_identifier) @call.name)) @call\n\n; Type references \u2014 captures types used in declaration/parameter positions\n(declaration type: (type_identifier) @reference.type)\n(parameter_declaration type: (type_identifier) @reference.type)\n; struct_specifier forms: e.g. \"const struct Curl_protocol foo = {...}\"\n(declaration type: (struct_specifier name: (type_identifier) @reference.type))\n(parameter_declaration type: (struct_specifier name: (type_identifier) @reference.type))\n";
-export declare const GO_QUERIES = "\n; Functions\n(function_declaration name: (identifier) @name) @definition.function\n\n; Methods \u2014 non-pointer receiver: func (r ReceiverType) Name(...)\n; The receiver.type capture lets the parser set container = ReceiverType so that\n; qualifiedKey becomes \"ReceiverType.Name\" instead of bare \"Name\".\n(method_declaration\n receiver: (parameter_list (parameter_declaration\n type: (type_identifier) @receiver.type))\n name: (field_identifier) @name) @definition.method\n\n; Methods \u2014 pointer receiver: func (r *ReceiverType) Name(...)\n(method_declaration\n receiver: (parameter_list (parameter_declaration\n type: (pointer_type (type_identifier) @receiver.type)))\n name: (field_identifier) @name) @definition.method\n\n; Types\n(type_declaration (type_spec name: (type_identifier) @name type: (struct_type))) @definition.struct\n(type_declaration (type_spec name: (type_identifier) @name type: (interface_type))) @definition.interface\n\n; Interface method signatures \u2014 emits each method inside an interface_type as a method entity.\n; @heritage.class = interface name (used as container in index.ts via heritageClassCapture).\n; node type is method_elem in tree-sitter-go v0.21+.\n(type_declaration\n (type_spec\n name: (type_identifier) @heritage.class\n type: (interface_type\n (method_elem\n name: (field_identifier) @name)))) @definition.method\n\n; Imports\n(import_declaration (import_spec path: (interpreted_string_literal) @import.source)) @import\n(import_declaration (import_spec_list (import_spec path: (interpreted_string_literal) @import.source))) @import\n\n; Struct embedding \u2014 value: type Foo struct { Bar }\n; NOTE: no @definition.* capture here \u2014 if it were present, defCapture fires first and\n; the heritage block is never reached (early-exit in the first pass).\n; The . anchors require the type_identifier to be the FIRST and ONLY named child of\n; field_declaration, distinguishing anonymous embedded fields from named fields like:\n; w Writer (named field \u2014 field_identifier_list precedes type, not matched)\n(type_declaration\n (type_spec\n name: (type_identifier) @heritage.class\n type: (struct_type\n (field_declaration_list\n (field_declaration .\n (type_identifier) @heritage.extends .)))))\n\n; Struct embedding \u2014 pointer: type Foo struct { *Bar }\n; The . anchor requires pointer_type to be the first named child (anonymous embedded).\n(type_declaration\n (type_spec\n name: (type_identifier) @heritage.class\n type: (struct_type\n (field_declaration_list\n (field_declaration .\n (pointer_type (type_identifier) @heritage.extends) .)))))\n\n; Calls\n(call_expression function: (identifier) @call.name) @call\n; Qualified calls: pkg.Func() or localVar.Method() where the operand is a simple\n; identifier \u2014 capture it as @_qualifier so cross-file resolution uses the fully\n; qualified name \"scrape.NewManager\" instead of bare \"NewManager\", preventing\n; ambiguous same-named symbols in different packages from being collapsed.\n(call_expression function: (selector_expression\n operand: (identifier) @_qualifier\n field: (field_identifier) @call.name)) @call\n; Chained/complex calls: a.b.Method(), call().Method(), index[i].Method() \u2014 the\n; operand is not a simple identifier, so emit without qualifier (bare method name).\n(call_expression function: (selector_expression\n operand: (selector_expression)\n field: (field_identifier) @call.name)) @call\n\n; Struct literal construction: User{Name: \"Alice\"}\n(composite_literal type: (type_identifier) @call.name) @call\n\n; Type references \u2014 captures types used in field/param/return-type positions\n(field_declaration type: (type_identifier) @reference.type)\n(parameter_declaration type: (type_identifier) @reference.type)\n; Return types: in tree-sitter-go, return type is a direct type_identifier child of function/method\n(function_declaration result: (type_identifier) @reference.type)\n(method_declaration result: (type_identifier) @reference.type)\n\n; Pointer-type variants: e.g. opts *Options, func(*Options) *Result\n(field_declaration type: (pointer_type (type_identifier) @reference.type))\n(parameter_declaration type: (pointer_type (type_identifier) @reference.type))\n(function_declaration result: (pointer_type (type_identifier) @reference.type))\n(method_declaration result: (pointer_type (type_identifier) @reference.type))\n";
-export declare const CPP_QUERIES = "\n; Classes, Structs, Namespaces\n(class_specifier name: (type_identifier) @name) @definition.class\n(struct_specifier name: (type_identifier) @name) @definition.struct\n(namespace_definition name: (namespace_identifier) @name) @definition.namespace\n(enum_specifier name: (type_identifier) @name) @definition.enum\n\n; Typedefs and unions (common in C-style headers and mixed C/C++ code)\n(type_definition declarator: (type_identifier) @name) @definition.typedef\n(union_specifier name: (type_identifier) @name) @definition.union\n\n; Macros\n(preproc_function_def name: (identifier) @name) @definition.macro\n(preproc_def name: (identifier) @name) @definition.macro\n\n; Functions & Methods (direct declarator)\n(function_definition declarator: (function_declarator declarator: (identifier) @name)) @definition.function\n(function_definition declarator: (function_declarator declarator: (qualified_identifier name: (identifier) @name))) @definition.method\n\n; Functions/methods returning pointers (pointer_declarator wraps function_declarator)\n(function_definition declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name))) @definition.function\n(function_definition declarator: (pointer_declarator declarator: (function_declarator declarator: (qualified_identifier name: (identifier) @name)))) @definition.method\n\n; Functions/methods returning double pointers (nested pointer_declarator)\n(function_definition declarator: (pointer_declarator declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name)))) @definition.function\n(function_definition declarator: (pointer_declarator declarator: (pointer_declarator declarator: (function_declarator declarator: (qualified_identifier name: (identifier) @name))))) @definition.method\n\n; Functions/methods returning references (reference_declarator wraps function_declarator)\n(function_definition declarator: (reference_declarator (function_declarator declarator: (identifier) @name))) @definition.function\n(function_definition declarator: (reference_declarator (function_declarator declarator: (qualified_identifier name: (identifier) @name)))) @definition.method\n\n; Destructors (destructor_name is distinct from identifier in tree-sitter-cpp)\n(function_definition declarator: (function_declarator declarator: (qualified_identifier name: (destructor_name) @name))) @definition.method\n\n; Function declarations / prototypes (common in headers)\n(declaration declarator: (function_declarator declarator: (identifier) @name)) @definition.function\n(declaration declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name))) @definition.function\n\n; Inline class method declarations (inside class body, no body: void Foo();)\n(field_declaration declarator: (function_declarator declarator: (identifier) @name)) @definition.method\n\n; Inline class method definitions (inside class body, with body: void Foo() { ... })\n(field_declaration_list\n (function_definition\n declarator: (function_declarator\n declarator: [(field_identifier) (identifier) (operator_name) (destructor_name)] @name)) @definition.method)\n\n; Templates\n(template_declaration (class_specifier name: (type_identifier) @name)) @definition.template\n(template_declaration (function_definition declarator: (function_declarator declarator: (identifier) @name))) @definition.template\n\n; Includes\n(preproc_include path: (_) @import.source) @import\n\n; Calls\n(call_expression function: (identifier) @call.name) @call\n(call_expression function: (field_expression field: (field_identifier) @call.name)) @call\n(call_expression function: (qualified_identifier name: (identifier) @call.name)) @call\n(call_expression function: (template_function name: (identifier) @call.name)) @call\n\n; Constructor calls: new User()\n(new_expression type: (type_identifier) @call.name) @call\n\n; Heritage\n(class_specifier name: (type_identifier) @heritage.class\n (base_class_clause (type_identifier) @heritage.extends)) @heritage\n(class_specifier name: (type_identifier) @heritage.class\n (base_class_clause (access_specifier) (type_identifier) @heritage.extends)) @heritage\n\n; Type references \u2014 captures types used in declaration/parameter positions\n(declaration type: (type_identifier) @reference.type)\n(parameter_declaration type: (type_identifier) @reference.type)\n";
-export declare const CSHARP_QUERIES = "\n; Types\n(class_declaration name: (identifier) @name) @definition.class\n(interface_declaration name: (identifier) @name) @definition.interface\n(struct_declaration name: (identifier) @name) @definition.struct\n(enum_declaration name: (identifier) @name) @definition.enum\n(record_declaration name: (identifier) @name) @definition.record\n(delegate_declaration name: (identifier) @name) @definition.delegate\n\n; Namespaces (block form and C# 10+ file-scoped form)\n(namespace_declaration name: (identifier) @name) @definition.namespace\n(namespace_declaration name: (qualified_name) @name) @definition.namespace\n(file_scoped_namespace_declaration name: (identifier) @name) @definition.namespace\n(file_scoped_namespace_declaration name: (qualified_name) @name) @definition.namespace\n\n; Methods & Properties\n(method_declaration name: (identifier) @name) @definition.method\n(local_function_statement name: (identifier) @name) @definition.function\n(constructor_declaration name: (identifier) @name) @definition.constructor\n(property_declaration name: (identifier) @name) @definition.property\n\n; Primary constructors (C# 12): class User(string name, int age) { }\n(class_declaration name: (identifier) @name (parameter_list) @definition.constructor)\n(record_declaration name: (identifier) @name (parameter_list) @definition.constructor)\n\n; Using\n(using_directive (qualified_name) @import.source) @import\n(using_directive (identifier) @import.source) @import\n\n; Calls\n(invocation_expression function: (identifier) @call.name) @call\n(invocation_expression function: (member_access_expression name: (identifier) @call.name)) @call\n\n; Null-conditional method calls: user?.Save()\n; Parses as: invocation_expression \u2192 conditional_access_expression \u2192 member_binding_expression \u2192 identifier\n(invocation_expression\n function: (conditional_access_expression\n (member_binding_expression\n (identifier) @call.name))) @call\n\n; Constructor calls: new Foo() and new Foo { Props }\n(object_creation_expression type: (identifier) @call.name) @call\n\n; Target-typed new (C# 9): User u = new(\"x\", 5)\n(variable_declaration type: (identifier) @call.name (variable_declarator (implicit_object_creation_expression) @call))\n\n; Heritage\n(class_declaration name: (identifier) @heritage.class\n (base_list (identifier) @heritage.extends)) @heritage\n(class_declaration name: (identifier) @heritage.class\n (base_list (generic_name (identifier) @heritage.extends))) @heritage\n\n; Type references \u2014 captures types used in field/param/return-type positions\n(parameter type: (identifier) @reference.type)\n(variable_declaration type: (identifier) @reference.type)\n(field_declaration (variable_declaration type: (identifier) @reference.type))\n(method_declaration returns: (identifier) @reference.type)\n";
-export declare const RUST_QUERIES = "\n; Functions & Items\n(function_item name: (identifier) @name) @definition.function\n(struct_item name: (type_identifier) @name) @definition.struct\n(enum_item name: (type_identifier) @name) @definition.enum\n(trait_item name: (type_identifier) @name) @definition.trait\n(impl_item type: (type_identifier) @name !trait) @definition.impl\n(impl_item type: (generic_type type: (type_identifier) @name) !trait) @definition.impl\n(mod_item name: (identifier) @name) @definition.module\n\n; Type aliases, const, static, macros\n(type_item name: (type_identifier) @name) @definition.type\n(const_item name: (identifier) @name) @definition.const\n(static_item name: (identifier) @name) @definition.static\n(macro_definition name: (identifier) @name) @definition.macro\n\n; Use statements\n(use_declaration argument: (_) @import.source) @import\n\n; Calls\n(call_expression function: (identifier) @call.name) @call\n(call_expression function: (field_expression field: (field_identifier) @call.name)) @call\n(call_expression function: (scoped_identifier\n path: (identifier) @_qualifier\n name: (identifier) @call.name)) @call\n(call_expression function: (generic_function function: (identifier) @call.name)) @call\n\n; Struct literal construction: User { name: value }\n(struct_expression name: (type_identifier) @call.name) @call\n\n; Type references \u2014 captures types used in field/param/return-type positions\n(field_declaration type: (type_identifier) @reference.type)\n(parameter pattern: (_) type: (type_identifier) @reference.type)\n(function_item return_type: (type_identifier) @reference.type)\n\n; Wrapped type references \u2014 &Type, &mut Type\n(field_declaration type: (reference_type (type_identifier) @reference.type))\n(parameter pattern: (_) type: (reference_type (type_identifier) @reference.type))\n(function_item return_type: (reference_type (type_identifier) @reference.type))\n\n; Generic wrapper types \u2014 Box<Type>, Arc<Type>, Vec<Type>, Option<Type>, Result<Type,_>, etc.\n; Capture the type argument (inner type), not the wrapper name.\n(field_declaration type: (generic_type type_arguments: (type_arguments (type_identifier) @reference.type)))\n(parameter pattern: (_) type: (generic_type type_arguments: (type_arguments (type_identifier) @reference.type)))\n(function_item return_type: (generic_type type_arguments: (type_arguments (type_identifier) @reference.type)))\n\n; impl Trait in parameter / return positions\n(parameter pattern: (_) type: (abstract_type (type_identifier) @reference.type))\n(function_item return_type: (abstract_type (type_identifier) @reference.type))\n\n; dyn Trait \u2014 trait objects (direct: fn foo(x: dyn Trait) or field: x: dyn Trait)\n(field_declaration type: (dynamic_type (type_identifier) @reference.type))\n(parameter pattern: (_) type: (dynamic_type (type_identifier) @reference.type))\n(function_item return_type: (dynamic_type (type_identifier) @reference.type))\n\n; dyn Trait nested inside generic wrapper \u2014 Box<dyn Trait>, Arc<dyn Trait>, etc.\n(field_declaration type: (generic_type type_arguments: (type_arguments (dynamic_type (type_identifier) @reference.type))))\n(parameter pattern: (_) type: (generic_type type_arguments: (type_arguments (dynamic_type (type_identifier) @reference.type))))\n(function_item return_type: (generic_type type_arguments: (type_arguments (dynamic_type (type_identifier) @reference.type))))\n\n; Heritage (trait implementation) \u2014 all combinations of concrete/generic trait \u00D7 concrete/generic type\n(impl_item trait: (type_identifier) @heritage.trait type: (type_identifier) @heritage.class) @heritage\n(impl_item trait: (generic_type type: (type_identifier) @heritage.trait) type: (type_identifier) @heritage.class) @heritage\n(impl_item trait: (type_identifier) @heritage.trait type: (generic_type type: (type_identifier) @heritage.class)) @heritage\n(impl_item trait: (generic_type type: (type_identifier) @heritage.trait) type: (generic_type type: (type_identifier) @heritage.class)) @heritage\n";
-export declare const PHP_QUERIES = "\n; \u2500\u2500 Namespace \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(namespace_definition\n name: (namespace_name) @name) @definition.namespace\n\n; \u2500\u2500 Classes \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(class_declaration\n name: (name) @name) @definition.class\n\n; \u2500\u2500 Interfaces \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(interface_declaration\n name: (name) @name) @definition.interface\n\n; \u2500\u2500 Traits \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(trait_declaration\n name: (name) @name) @definition.trait\n\n; \u2500\u2500 Enums (PHP 8.1) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(enum_declaration\n name: (name) @name) @definition.enum\n\n; \u2500\u2500 Top-level functions \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(function_definition\n name: (name) @name) @definition.function\n\n; \u2500\u2500 Methods (including constructors) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(method_declaration\n name: (name) @name) @definition.method\n\n; \u2500\u2500 Class properties (including Eloquent $fillable, $casts, etc.) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(property_declaration\n (property_element\n (variable_name\n (name) @name))) @definition.property\n\n; \u2500\u2500 Imports: use statements \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n; Simple: use App\\Models\\User;\n(namespace_use_declaration\n (namespace_use_clause\n (qualified_name) @import.source)) @import\n\n; \u2500\u2500 Function/method calls \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n; Regular function call: foo()\n(function_call_expression\n function: (name) @call.name) @call\n\n; Method call: $obj->method()\n(member_call_expression\n name: (name) @call.name) @call\n\n; Nullsafe method call: $obj?->method()\n(nullsafe_member_call_expression\n name: (name) @call.name) @call\n\n; Static call: Foo::bar() (php_only uses scoped_call_expression)\n(scoped_call_expression\n name: (name) @call.name) @call\n\n; Constructor call: new User()\n(object_creation_expression (name) @call.name) @call\n\n; \u2500\u2500 Heritage: extends \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(class_declaration\n name: (name) @heritage.class\n (base_clause\n [(name) (qualified_name)] @heritage.extends)) @heritage\n\n; \u2500\u2500 Heritage: implements \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(class_declaration\n name: (name) @heritage.class\n (class_interface_clause\n [(name) (qualified_name)] @heritage.implements)) @heritage.impl\n\n; \u2500\u2500 Heritage: use trait (must capture enclosing class name) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(class_declaration\n name: (name) @heritage.class\n body: (declaration_list\n (use_declaration\n [(name) (qualified_name)] @heritage.trait))) @heritage\n\n; \u2500\u2500 Type references \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(named_type (name) @reference.type)\n";
-export declare const RUBY_QUERIES = "\n; \u2500\u2500 Modules \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(module\n name: (constant) @name) @definition.module\n\n; \u2500\u2500 Classes \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(class\n name: (constant) @name) @definition.class\n\n; Scoped class name: class Module::ClassName\n(class\n name: (scope_resolution) @name) @definition.class\n\n; \u2500\u2500 Instance methods \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(method\n name: (identifier) @name) @definition.method\n\n; \u2500\u2500 Singleton (class-level) methods \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(singleton_method\n name: (identifier) @name) @definition.method\n\n; \u2500\u2500 Require / require_relative \u2192 IMPORTS \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n; Capture the string argument so the second pass emits an IMPORTS edge.\n(call\n method: (identifier) @_req\n arguments: (argument_list\n (string (string_content) @import.source))\n (#match? @_req \"^require\")) @import\n\n; \u2500\u2500 All other calls \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(call\n method: (identifier) @call.name) @call\n\n; \u2500\u2500 Bare calls without parens (identifiers at statement level are method calls) \u2500\n; NOTE: This may over-capture variable reads as calls (e.g. 'result' at\n; statement level). Ruby's grammar makes bare identifiers ambiguous \u2014 they\n; could be local variables or zero-arity method calls. Post-processing via\n; the BUILTINS set suppresses most false positives.\n(body_statement\n (identifier) @call.name @call)\n\n; \u2500\u2500 Heritage: class < SuperClass \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(class\n name: (constant) @heritage.class\n superclass: (superclass\n (constant) @heritage.extends)) @heritage\n\n; Scoped superclass: class Foo < Module::Class\n; Captures full \"Module::Class\" text to avoid ambiguity with bare \"Class\" names\n(class\n name: (constant) @heritage.class\n superclass: (superclass\n (scope_resolution) @heritage.extends)) @heritage\n\n; Scoped class name, bare superclass: class Module::Foo < Bar\n(class\n name: (scope_resolution) @heritage.class\n superclass: (superclass\n (constant) @heritage.extends)) @heritage\n\n; Scoped class name, scoped superclass: class Module::Foo < Module::Bar\n(class\n name: (scope_resolution) @heritage.class\n superclass: (superclass\n (scope_resolution) @heritage.extends)) @heritage\n";
-export declare const KOTLIN_QUERIES = "\n; \u2500\u2500 Interfaces \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n; tree-sitter-kotlin (fwcd) has no interface_declaration node type.\n; Interfaces are class_declaration nodes with an anonymous \"interface\" keyword child.\n(class_declaration\n \"interface\"\n (type_identifier) @name) @definition.interface\n\n; \u2500\u2500 Classes (regular, data, sealed, enum) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n; All have the anonymous \"class\" keyword child. enum class has both\n; \"enum\" and \"class\" children \u2014 the \"class\" child still matches.\n(class_declaration\n \"class\"\n (type_identifier) @name) @definition.class\n\n; \u2500\u2500 Object declarations (Kotlin singletons) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(object_declaration\n (type_identifier) @name) @definition.class\n\n; \u2500\u2500 Companion objects (named only) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(companion_object\n (type_identifier) @name) @definition.class\n\n; \u2500\u2500 Functions (top-level, member, extension) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(function_declaration\n (simple_identifier) @name) @definition.function\n\n; \u2500\u2500 Properties \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(property_declaration\n (variable_declaration\n (simple_identifier) @name)) @definition.property\n\n; \u2500\u2500 Enum entries \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(enum_entry\n (simple_identifier) @name) @definition.enum\n\n; \u2500\u2500 Type aliases \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(type_alias\n (type_identifier) @name) @definition.type\n\n; \u2500\u2500 Imports \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(import_header\n (identifier) @import.source) @import\n\n; \u2500\u2500 Function calls (direct) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(call_expression\n (simple_identifier) @call.name) @call\n\n; \u2500\u2500 Method calls (via navigation: obj.method()) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(call_expression\n (navigation_expression\n (navigation_suffix\n (simple_identifier) @call.name))) @call\n\n; \u2500\u2500 Constructor invocations \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(constructor_invocation\n (user_type\n (type_identifier) @call.name)) @call\n\n; \u2500\u2500 Infix function calls (e.g., a to b, x until y) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(infix_expression\n (simple_identifier) @call.name) @call\n\n; \u2500\u2500 Heritage: extends / implements via delegation_specifier \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n; Interface implementation (bare user_type): class Foo : Bar\n(class_declaration\n (type_identifier) @heritage.class\n (delegation_specifier\n (user_type (type_identifier) @heritage.extends))) @heritage\n\n; Class extension (constructor_invocation): class Foo : Bar()\n(class_declaration\n (type_identifier) @heritage.class\n (delegation_specifier\n (constructor_invocation\n (user_type (type_identifier) @heritage.extends)))) @heritage\n\n; \u2500\u2500 Type references \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(type_reference (user_type (type_identifier) @reference.type))\n";
-export declare const SWIFT_QUERIES = "\n; Classes\n(class_declaration \"class\" name: (type_identifier) @name) @definition.class\n\n; Structs\n(class_declaration \"struct\" name: (type_identifier) @name) @definition.struct\n\n; Enums\n(class_declaration \"enum\" name: (type_identifier) @name) @definition.enum\n\n; Extensions (mapped to class \u2014 no dedicated label in schema)\n(class_declaration \"extension\" name: (user_type (type_identifier) @name)) @definition.class\n\n; Actors\n(class_declaration \"actor\" name: (type_identifier) @name) @definition.class\n\n; Protocols (mapped to interface)\n(protocol_declaration name: (type_identifier) @name) @definition.interface\n\n; Type aliases\n(typealias_declaration name: (type_identifier) @name) @definition.type\n\n; Functions (top-level and methods)\n(function_declaration name: (simple_identifier) @name) @definition.function\n\n; Protocol method declarations\n(protocol_function_declaration name: (simple_identifier) @name) @definition.method\n\n; Initializers \u2014 \"init\" is the canonical name; line offset tracked via definition node\n(init_declaration \"init\" @name) @definition.constructor\n\n; Properties (stored and computed)\n(property_declaration (pattern (simple_identifier) @name)) @definition.property\n\n; Imports\n(import_declaration (identifier (simple_identifier) @import.source)) @import\n\n; Calls - direct function calls\n(call_expression (simple_identifier) @call.name) @call\n\n; Calls - member/navigation calls (obj.method())\n(call_expression (navigation_expression (navigation_suffix (simple_identifier) @call.name))) @call\n\n; Heritage - class/struct/enum inheritance and protocol conformance\n(class_declaration name: (type_identifier) @heritage.class\n (inheritance_specifier inherits_from: (user_type (type_identifier) @heritage.extends))) @heritage\n\n; Heritage - protocol inheritance\n(protocol_declaration name: (type_identifier) @heritage.class\n (inheritance_specifier inherits_from: (user_type (type_identifier) @heritage.extends))) @heritage\n\n; Heritage - extension protocol conformance (e.g. extension Foo: SomeProtocol)\n; Extensions wrap the name in user_type unlike class/struct/enum declarations\n(class_declaration \"extension\" name: (user_type (type_identifier) @heritage.class)\n (inheritance_specifier inherits_from: (user_type (type_identifier) @heritage.extends))) @heritage\n\n; Type references \u2014 captures types used in annotations/parameters/return types\n(type_annotation (user_type (type_identifier) @reference.type))\n";
-export declare const SCALA_QUERIES = "\n; \u2500\u2500 Classes (class, case class, abstract class, sealed class) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(class_definition\n name: (identifier) @name) @definition.class\n\n; \u2500\u2500 Traits (mapped to interface) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(trait_definition\n name: (identifier) @name) @definition.interface\n\n; \u2500\u2500 Object definitions (companion objects, case objects, singletons) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(object_definition\n name: (identifier) @name) @definition.class\n\n; \u2500\u2500 Function / method definitions \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(function_definition\n name: (identifier) @name) @definition.function\n\n; \u2500\u2500 Abstract method declarations (in traits / abstract classes) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(function_declaration\n name: (identifier) @name) @definition.method\n\n; \u2500\u2500 Type aliases \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(type_definition\n name: (type_identifier) @name) @definition.type\n\n; \u2500\u2500 Val/var definitions (class-level members only, not method-local vars) \u2500\u2500\u2500\u2500\u2500\u2500\n; Restricting to template_body ensures we only capture class/object-level\n; declarations (e.g. companion object implicits) and not local val bindings\n; inside method bodies, which would add noise entities to the graph.\n(template_body\n (val_definition (identifier) @name) @definition.const)\n\n(template_body\n (var_definition (identifier) @name) @definition.const)\n\n; \u2500\u2500 Imports: capture full declaration for dotted-path reconstruction \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n; Handles: import ix.memory.model._ (wildcard)\n; import ix.memory.model.{NodeKind, GraphNode} (selective)\n; import ix.memory.model.NodeKind (simple)\n(import_declaration) @import.stmt\n\n; \u2500\u2500 Calls: direct function call \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(call_expression\n function: (identifier) @call.name) @call\n\n; \u2500\u2500 Calls: method call (obj.method()) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(call_expression\n function: (field_expression\n field: (identifier) @call.name)) @call\n\n; \u2500\u2500 Constructor: new Foo() \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(instance_expression\n (type_identifier) @call.name) @call\n\n; \u2500\u2500 Heritage: class / object extends and with \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n(class_definition\n name: (identifier) @heritage.class\n extend: (extends_clause\n (type_identifier) @heritage.extends)) @heritage\n\n(trait_definition\n name: (identifier) @heritage.class\n extend: (extends_clause\n (type_identifier) @heritage.extends)) @heritage\n\n(object_definition\n name: (identifier) @heritage.class\n extend: (extends_clause\n (type_identifier) @heritage.extends)) @heritage\n\n; \u2500\u2500 Type references \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n; Case class parameter types: case class Foo(x: NodeKind)\n(class_parameter (type_identifier) @reference.type)\n; Method parameter types: def foo(x: NodeKind)\n(parameter (type_identifier) @reference.type)\n; Val/var type annotations: val x: NodeKind = ...\n(val_definition (type_identifier) @reference.type)\n(var_definition (type_identifier) @reference.type)\n; Return type annotations: def foo(): NodeKind = ...\n; (type_identifier is a direct child of function_definition, no named field)\n(function_definition (type_identifier) @reference.type)\n; Generic type arguments: Map[NodeId, NodeKind], Option[NodeKind], Set[NodeKind], etc.\n(generic_type (type_arguments (type_identifier) @reference.type))\n\n; \u2500\u2500 Qualified singleton / companion-object member access \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n; Two grammar shapes arise in practice:\n; expression context: NodeKind.Class \u2192 field_expression\n; pattern context: case Some(NodeKind.Class) => \u2026 \u2192 stable_identifier\n; Both shapes are captured below. The (#match? ^[A-Z]) predicates restrict to\n; UpperCamelCase identifiers (Scala convention for case objects / companion\n; object members), filtering out lowercase method / field names.\n\n; Expression: val x = NodeKind.Decision\n(field_expression\n value: (identifier) @_qualifier\n field: (identifier) @call.name\n (#match? @_qualifier \"^[A-Z]\")\n (#match? @call.name \"^[A-Z]\"))\n\n; Pattern: case Some(NodeKind.Class) | Some(NodeKind.Trait) => \u2026\n(stable_identifier\n (identifier) @_qualifier\n \".\"\n (identifier) @call.name\n (#match? @_qualifier \"^[A-Z]\")\n (#match? @call.name \"^[A-Z]\"))\n";
-export declare const LANGUAGE_QUERIES: Record<SupportedLanguages, string>;
diff --git a/core-ingestion/dist/queries.js b/core-ingestion/dist/queries.js
deleted file mode 100644
index 0b2b03c5..00000000
--- a/core-ingestion/dist/queries.js
+++ /dev/null
@@ -1,1100 +0,0 @@
-import { SupportedLanguages } from './languages.js';
-/*
- * Tree-sitter queries for extracting code definitions.
- *
- * Note: Different grammars (typescript vs tsx vs javascript) may have
- * slightly different node types. These queries are designed to be
- * compatible with the standard tree-sitter grammars.
- */
-// TypeScript queries - works with tree-sitter-typescript
-export const TYPESCRIPT_QUERIES = `
-(class_declaration
- name: (type_identifier) @name) @definition.class
-
-(interface_declaration
- name: (type_identifier) @name) @definition.interface
-
-(function_declaration
- name: (identifier) @name) @definition.function
-
-(method_definition
- name: (property_identifier) @name) @definition.method
-
-(lexical_declaration
- (variable_declarator
- name: (identifier) @name
- value: (arrow_function))) @definition.function
-
-(lexical_declaration
- (variable_declarator
- name: (identifier) @name
- value: (function_expression))) @definition.function
-
-(export_statement
- declaration: (lexical_declaration
- (variable_declarator
- name: (identifier) @name
- value: (arrow_function)))) @definition.function
-
-(export_statement
- declaration: (lexical_declaration
- (variable_declarator
- name: (identifier) @name
- value: (function_expression)))) @definition.function
-
-(import_statement
- source: (string) @import.source) @import
-
-; Re-export statements: export { X } from './y'
-(export_statement
- source: (string) @import.source) @import
-
-; Dynamic import with string literal: import('./foo') or await import('./foo')
-(call_expression
- function: (import)
- arguments: (arguments (string) @import.source)) @import
-
-(call_expression
- function: (identifier) @call.name) @call
-
-(call_expression
- function: (member_expression
- property: (property_identifier) @call.name)) @call
-
-; this.method() — capture this as qualifier so index.ts can substitute the enclosing class.
-; Analogous to Python's self/cls substitution.
-(call_expression
- function: (member_expression
- object: (this) @_qualifier
- property: (property_identifier) @call.name)) @call
-
-; Constructor calls: new Foo()
-(new_expression
- constructor: (identifier) @call.name) @call
-
-; Heritage queries - class extends
-(class_declaration
- name: (type_identifier) @heritage.class
- (class_heritage
- (extends_clause
- value: (identifier) @heritage.extends))) @heritage
-
-; Heritage queries - class implements interface
-(class_declaration
- name: (type_identifier) @heritage.class
- (class_heritage
- (implements_clause
- (type_identifier) @heritage.implements))) @heritage.impl
-
-; Heritage queries - interface extends interface
-(interface_declaration
- name: (type_identifier) @heritage.class
- (extends_type_clause
- (type_identifier) @heritage.extends)) @heritage
-
-; Type references — captures types used in annotations/parameters/return types
-(type_annotation (type_identifier) @reference.type)
-`;
-// JavaScript queries - works with tree-sitter-javascript
-export const JAVASCRIPT_QUERIES = `
-(class_declaration
- name: (identifier) @name) @definition.class
-
-(function_declaration
- name: (identifier) @name) @definition.function
-
-(method_definition
- name: (property_identifier) @name) @definition.method
-
-(lexical_declaration
- (variable_declarator
- name: (identifier) @name
- value: (arrow_function))) @definition.function
-
-(lexical_declaration
- (variable_declarator
- name: (identifier) @name
- value: (function_expression))) @definition.function
-
-; var foo = function() {} (CommonJS var declarations)
-(variable_declaration
- (variable_declarator
- name: (identifier) @name
- value: (function_expression))) @definition.function
-
-; var foo = () => {} (CommonJS var arrow functions)
-(variable_declaration
- (variable_declarator
- name: (identifier) @name
- value: (arrow_function))) @definition.function
-
-; obj.method = function name() {} (prototype/mixin method assignments)
-(expression_statement
- (assignment_expression
- left: (member_expression
- property: (property_identifier) @name)
- right: (function_expression))) @definition.function
-
-(export_statement
- declaration: (lexical_declaration
- (variable_declarator
- name: (identifier) @name
- value: (arrow_function)))) @definition.function
-
-(export_statement
- declaration: (lexical_declaration
- (variable_declarator
- name: (identifier) @name
- value: (function_expression)))) @definition.function
-
-(import_statement
- source: (string) @import.source) @import
-
-; Re-export statements: export { X } from './y'
-(export_statement
- source: (string) @import.source) @import
-
-; Dynamic import with string literal: import('./foo') or await import('./foo')
-(call_expression
- function: (import)
- arguments: (arguments (string) @import.source)) @import
-
-; CommonJS require() → IMPORTS
-(call_expression
- function: (identifier) @_req
- arguments: (arguments (string) @import.source)
- (#eq? @_req "require")) @import
-
-(call_expression
- function: (identifier) @call.name) @call
-
-(call_expression
- function: (member_expression
- property: (property_identifier) @call.name)) @call
-
-; this.method() — capture this as qualifier so index.ts can substitute the enclosing class.
-; Analogous to Python's self/cls substitution.
-(call_expression
- function: (member_expression
- object: (this) @_qualifier
- property: (property_identifier) @call.name)) @call
-
-; Constructor calls: new Foo()
-(new_expression
- constructor: (identifier) @call.name) @call
-
-; Heritage queries - class extends (JavaScript uses different AST than TypeScript)
-; In tree-sitter-javascript, class_heritage directly contains the parent identifier
-(class_declaration
- name: (identifier) @heritage.class
- (class_heritage
- (identifier) @heritage.extends)) @heritage
-`;
-// Python queries - works with tree-sitter-python
-export const PYTHON_QUERIES = `
-(class_definition
- name: (identifier) @name) @definition.class
-
-(function_definition
- name: (identifier) @name) @definition.function
-
-(import_statement
- name: (dotted_name) @import.source) @import
-
-(import_from_statement
- module_name: (dotted_name) @import.source) @import
-
-(import_from_statement
- module_name: (relative_import) @import.source) @import
-
-; from . import utils / from .models import User — also capture the imported symbol names
-(import_from_statement
- module_name: (relative_import)
- name: (dotted_name (identifier) @import.name)) @import.names
-
-(call
- function: (identifier) @call.name) @call
-
-(call
- function: (attribute
- attribute: (identifier) @call.name)) @call
-
-; Attribute calls with object captured as qualifier so Tier-1b resolution can use
-; class/module name to break ties (e.g. Session.execute → resolves to Session in
-; the importing file). Emitted alongside the bare-name pattern above so that
-; bare-name global fallback still fires when qualifier resolution fails.
-(call
- function: (attribute
- object: (identifier) @_qualifier
- attribute: (identifier) @call.name)) @call
-
-; Django ORM manager pattern: Model.objects.method(...)
-; Unwinds the intermediate accessor (objects/related_manager/etc.) and captures
-; the model class as qualifier, e.g. User.objects.filter(...) → CALLS User.filter.
-; This creates cross-file edges between view/form code and model definitions.
-(call
- function: (attribute
- object: (attribute
- object: (identifier) @_qualifier
- attribute: (identifier))
- attribute: (identifier) @call.name)) @call
-
-; Chained call pattern: qualifier.method1(...).method2(...)
-; Propagates the qualifier from the inner call to the outer, e.g.
-; User.filter(...).order_by(...) → CALLS User.order_by.
-(call
- function: (attribute
- object: (call
- function: (attribute
- object: (identifier) @_qualifier
- attribute: (identifier)))
- attribute: (identifier) @call.name)) @call
-
-; Heritage queries - Python class inheritance
-(class_definition
- name: (identifier) @heritage.class
- superclasses: (argument_list
- (identifier) @heritage.extends)) @heritage
-
-; Type references — captures types used in type annotations / hints
-(typed_parameter type: (type (identifier) @reference.type))
-(typed_default_parameter type: (type (identifier) @reference.type))
-
-; Typed parameters: capture name + type together so qualifier substitution can
-; map a local variable name to its declared class (e.g. query: Query → 'query' → 'Query').
-(function_definition
- parameters: (parameters
- (typed_parameter
- (identifier) @_typed_param_name
- type: (type (identifier) @_typed_param_type)))) @_typed_param_scope
-
-(function_definition
- parameters: (parameters
- (typed_default_parameter
- (identifier) @_typed_param_name
- type: (type (identifier) @_typed_param_type)))) @_typed_param_scope
-
-; Assignment tracking: x = SomeClass() or x = SomeClass(args)
-; Maps local variable name → constructor type for untyped-parameter qualifier substitution.
-; Only PascalCase RHS names are tracked in TypeScript (constructor convention).
-(function_definition
- body: (block
- (expression_statement
- (assignment
- left: (identifier) @_assign_lhs
- right: (call
- function: (identifier) @_assign_rhs_type))))) @_assign_scope
-
-; Assignment tracking: x = Model.objects.method()
-; Captures the outermost identifier (model class) for ORM manager patterns.
-(function_definition
- body: (block
- (expression_statement
- (assignment
- left: (identifier) @_assign_lhs
- right: (call
- function: (attribute
- object: (attribute
- object: (identifier) @_assign_rhs_type
- attribute: (identifier))
- attribute: (identifier))))))) @_assign_scope
-`;
-// Java queries - works with tree-sitter-java
-export const JAVA_QUERIES = `
-; Classes, Interfaces, Enums, Annotations
-(class_declaration name: (identifier) @name) @definition.class
-(interface_declaration name: (identifier) @name) @definition.interface
-(enum_declaration name: (identifier) @name) @definition.enum
-(annotation_type_declaration name: (identifier) @name) @definition.annotation
-
-; Methods & Constructors
-(method_declaration name: (identifier) @name) @definition.method
-(constructor_declaration name: (identifier) @name) @definition.constructor
-
-; Imports - capture any import declaration child as source
-(import_declaration (_) @import.source) @import
-
-; Calls
-(method_invocation name: (identifier) @call.name) @call
-(method_invocation object: (_) name: (identifier) @call.name) @call
-
-; Constructor calls: new Foo()
-(object_creation_expression type: (type_identifier) @call.name) @call
-
-; Heritage - extends class (plain: extends Foo)
-(class_declaration name: (identifier) @heritage.class
- (superclass (type_identifier) @heritage.extends)) @heritage
-
-; Heritage - extends class with generics (extends Foo<T>)
-(class_declaration name: (identifier) @heritage.class
- (superclass (generic_type (type_identifier) @heritage.extends))) @heritage
-
-; Heritage - implements interfaces (plain: implements Foo)
-(class_declaration name: (identifier) @heritage.class
- (super_interfaces (type_list (type_identifier) @heritage.implements))) @heritage.impl
-
-; Heritage - implements interfaces with generics (implements Foo<T>)
-(class_declaration name: (identifier) @heritage.class
- (super_interfaces (type_list (generic_type (type_identifier) @heritage.implements)))) @heritage.impl
-
-; Heritage - interface extends interfaces (plain: extends Foo)
-(interface_declaration name: (identifier) @heritage.class
- (extends_interfaces (type_list (type_identifier) @heritage.extends))) @heritage
-
-; Heritage - interface extends interfaces with generics (extends Foo<T>)
-(interface_declaration name: (identifier) @heritage.class
- (extends_interfaces (type_list (generic_type (type_identifier) @heritage.extends)))) @heritage
-
-; Type references — captures types used in field/param/return-type positions
-(field_declaration type: (type_identifier) @reference.type)
-(formal_parameter type: (type_identifier) @reference.type)
-(local_variable_declaration type: (type_identifier) @reference.type)
-(method_declaration type: (type_identifier) @reference.type)
-`;
-// C queries - works with tree-sitter-c
-export const C_QUERIES = `
-; Functions (direct declarator)
-(function_definition declarator: (function_declarator declarator: (identifier) @name)) @definition.function
-(declaration declarator: (function_declarator declarator: (identifier) @name)) @definition.function
-
-; Functions returning pointers (pointer_declarator wraps function_declarator)
-(function_definition declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name))) @definition.function
-(declaration declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name))) @definition.function
-
-; Functions returning double pointers (nested pointer_declarator)
-(function_definition declarator: (pointer_declarator declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name)))) @definition.function
-
-; Structs, Unions, Enums, Typedefs
-; Only match struct/union with a body to avoid treating forward declarations as definitions
-(struct_specifier name: (type_identifier) @name body: (field_declaration_list)) @definition.struct
-(union_specifier name: (type_identifier) @name body: (field_declaration_list)) @definition.union
-(enum_specifier name: (type_identifier) @name) @definition.enum
-(type_definition declarator: (type_identifier) @name) @definition.typedef
-
-; Macros
-(preproc_function_def name: (identifier) @name) @definition.macro
-(preproc_def name: (identifier) @name) @definition.macro
-
-; Includes
-(preproc_include path: (_) @import.source) @import
-
-; Calls
-(call_expression function: (identifier) @call.name) @call
-(call_expression function: (field_expression field: (field_identifier) @call.name)) @call
-
-; Type references — captures types used in declaration/parameter positions
-(declaration type: (type_identifier) @reference.type)
-(parameter_declaration type: (type_identifier) @reference.type)
-; struct_specifier forms: e.g. "const struct Curl_protocol foo = {...}"
-(declaration type: (struct_specifier name: (type_identifier) @reference.type))
-(parameter_declaration type: (struct_specifier name: (type_identifier) @reference.type))
-`;
-// Go queries - works with tree-sitter-go
-export const GO_QUERIES = `
-; Functions
-(function_declaration name: (identifier) @name) @definition.function
-
-; Methods — non-pointer receiver: func (r ReceiverType) Name(...)
-; The receiver.type capture lets the parser set container = ReceiverType so that
-; qualifiedKey becomes "ReceiverType.Name" instead of bare "Name".
-(method_declaration
- receiver: (parameter_list (parameter_declaration
- type: (type_identifier) @receiver.type))
- name: (field_identifier) @name) @definition.method
-
-; Methods — pointer receiver: func (r *ReceiverType) Name(...)
-(method_declaration
- receiver: (parameter_list (parameter_declaration
- type: (pointer_type (type_identifier) @receiver.type)))
- name: (field_identifier) @name) @definition.method
-
-; Types
-(type_declaration (type_spec name: (type_identifier) @name type: (struct_type))) @definition.struct
-(type_declaration (type_spec name: (type_identifier) @name type: (interface_type))) @definition.interface
-
-; Interface method signatures — emits each method inside an interface_type as a method entity.
-; @heritage.class = interface name (used as container in index.ts via heritageClassCapture).
-; node type is method_elem in tree-sitter-go v0.21+.
-(type_declaration
- (type_spec
- name: (type_identifier) @heritage.class
- type: (interface_type
- (method_elem
- name: (field_identifier) @name)))) @definition.method
-
-; Imports
-(import_declaration (import_spec path: (interpreted_string_literal) @import.source)) @import
-(import_declaration (import_spec_list (import_spec path: (interpreted_string_literal) @import.source))) @import
-
-; Struct embedding — value: type Foo struct { Bar }
-; NOTE: no @definition.* capture here — if it were present, defCapture fires first and
-; the heritage block is never reached (early-exit in the first pass).
-; The . anchors require the type_identifier to be the FIRST and ONLY named child of
-; field_declaration, distinguishing anonymous embedded fields from named fields like:
-; w Writer (named field — field_identifier_list precedes type, not matched)
-(type_declaration
- (type_spec
- name: (type_identifier) @heritage.class
- type: (struct_type
- (field_declaration_list
- (field_declaration .
- (type_identifier) @heritage.extends .)))))
-
-; Struct embedding — pointer: type Foo struct { *Bar }
-; The . anchor requires pointer_type to be the first named child (anonymous embedded).
-(type_declaration
- (type_spec
- name: (type_identifier) @heritage.class
- type: (struct_type
- (field_declaration_list
- (field_declaration .
- (pointer_type (type_identifier) @heritage.extends) .)))))
-
-; Calls
-(call_expression function: (identifier) @call.name) @call
-; Qualified calls: pkg.Func() or localVar.Method() where the operand is a simple
-; identifier — capture it as @_qualifier so cross-file resolution uses the fully
-; qualified name "scrape.NewManager" instead of bare "NewManager", preventing
-; ambiguous same-named symbols in different packages from being collapsed.
-(call_expression function: (selector_expression
- operand: (identifier) @_qualifier
- field: (field_identifier) @call.name)) @call
-; Chained/complex calls: a.b.Method(), call().Method(), index[i].Method() — the
-; operand is not a simple identifier, so emit without qualifier (bare method name).
-(call_expression function: (selector_expression
- operand: (selector_expression)
- field: (field_identifier) @call.name)) @call
-
-; Struct literal construction: User{Name: "Alice"}
-(composite_literal type: (type_identifier) @call.name) @call
-
-; Type references — captures types used in field/param/return-type positions
-(field_declaration type: (type_identifier) @reference.type)
-(parameter_declaration type: (type_identifier) @reference.type)
-; Return types: in tree-sitter-go, return type is a direct type_identifier child of function/method
-(function_declaration result: (type_identifier) @reference.type)
-(method_declaration result: (type_identifier) @reference.type)
-
-; Pointer-type variants: e.g. opts *Options, func(*Options) *Result
-(field_declaration type: (pointer_type (type_identifier) @reference.type))
-(parameter_declaration type: (pointer_type (type_identifier) @reference.type))
-(function_declaration result: (pointer_type (type_identifier) @reference.type))
-(method_declaration result: (pointer_type (type_identifier) @reference.type))
-`;
-// C++ queries - works with tree-sitter-cpp
-export const CPP_QUERIES = `
-; Classes, Structs, Namespaces
-(class_specifier name: (type_identifier) @name) @definition.class
-(struct_specifier name: (type_identifier) @name) @definition.struct
-(namespace_definition name: (namespace_identifier) @name) @definition.namespace
-(enum_specifier name: (type_identifier) @name) @definition.enum
-
-; Typedefs and unions (common in C-style headers and mixed C/C++ code)
-(type_definition declarator: (type_identifier) @name) @definition.typedef
-(union_specifier name: (type_identifier) @name) @definition.union
-
-; Macros
-(preproc_function_def name: (identifier) @name) @definition.macro
-(preproc_def name: (identifier) @name) @definition.macro
-
-; Functions & Methods (direct declarator)
-(function_definition declarator: (function_declarator declarator: (identifier) @name)) @definition.function
-(function_definition declarator: (function_declarator declarator: (qualified_identifier name: (identifier) @name))) @definition.method
-
-; Functions/methods returning pointers (pointer_declarator wraps function_declarator)
-(function_definition declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name))) @definition.function
-(function_definition declarator: (pointer_declarator declarator: (function_declarator declarator: (qualified_identifier name: (identifier) @name)))) @definition.method
-
-; Functions/methods returning double pointers (nested pointer_declarator)
-(function_definition declarator: (pointer_declarator declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name)))) @definition.function
-(function_definition declarator: (pointer_declarator declarator: (pointer_declarator declarator: (function_declarator declarator: (qualified_identifier name: (identifier) @name))))) @definition.method
-
-; Functions/methods returning references (reference_declarator wraps function_declarator)
-(function_definition declarator: (reference_declarator (function_declarator declarator: (identifier) @name))) @definition.function
-(function_definition declarator: (reference_declarator (function_declarator declarator: (qualified_identifier name: (identifier) @name)))) @definition.method
-
-; Destructors (destructor_name is distinct from identifier in tree-sitter-cpp)
-(function_definition declarator: (function_declarator declarator: (qualified_identifier name: (destructor_name) @name))) @definition.method
-
-; Function declarations / prototypes (common in headers)
-(declaration declarator: (function_declarator declarator: (identifier) @name)) @definition.function
-(declaration declarator: (pointer_declarator declarator: (function_declarator declarator: (identifier) @name))) @definition.function
-
-; Inline class method declarations (inside class body, no body: void Foo();)
-(field_declaration declarator: (function_declarator declarator: (identifier) @name)) @definition.method
-
-; Inline class method definitions (inside class body, with body: void Foo() { ... })
-(field_declaration_list
- (function_definition
- declarator: (function_declarator
- declarator: [(field_identifier) (identifier) (operator_name) (destructor_name)] @name)) @definition.method)
-
-; Templates
-(template_declaration (class_specifier name: (type_identifier) @name)) @definition.template
-(template_declaration (function_definition declarator: (function_declarator declarator: (identifier) @name))) @definition.template
-
-; Includes
-(preproc_include path: (_) @import.source) @import
-
-; Calls
-(call_expression function: (identifier) @call.name) @call
-(call_expression function: (field_expression field: (field_identifier) @call.name)) @call
-(call_expression function: (qualified_identifier name: (identifier) @call.name)) @call
-(call_expression function: (template_function name: (identifier) @call.name)) @call
-
-; Constructor calls: new User()
-(new_expression type: (type_identifier) @call.name) @call
-
-; Heritage
-(class_specifier name: (type_identifier) @heritage.class
- (base_class_clause (type_identifier) @heritage.extends)) @heritage
-(class_specifier name: (type_identifier) @heritage.class
- (base_class_clause (access_specifier) (type_identifier) @heritage.extends)) @heritage
-
-; Type references — captures types used in declaration/parameter positions
-(declaration type: (type_identifier) @reference.type)
-(parameter_declaration type: (type_identifier) @reference.type)
-`;
-// C# queries - works with tree-sitter-c-sharp
-export const CSHARP_QUERIES = `
-; Types
-(class_declaration name: (identifier) @name) @definition.class
-(interface_declaration name: (identifier) @name) @definition.interface
-(struct_declaration name: (identifier) @name) @definition.struct
-(enum_declaration name: (identifier) @name) @definition.enum
-(record_declaration name: (identifier) @name) @definition.record
-(delegate_declaration name: (identifier) @name) @definition.delegate
-
-; Namespaces (block form and C# 10+ file-scoped form)
-(namespace_declaration name: (identifier) @name) @definition.namespace
-(namespace_declaration name: (qualified_name) @name) @definition.namespace
-(file_scoped_namespace_declaration name: (identifier) @name) @definition.namespace
-(file_scoped_namespace_declaration name: (qualified_name) @name) @definition.namespace
-
-; Methods & Properties
-(method_declaration name: (identifier) @name) @definition.method
-(local_function_statement name: (identifier) @name) @definition.function
-(constructor_declaration name: (identifier) @name) @definition.constructor
-(property_declaration name: (identifier) @name) @definition.property
-
-; Primary constructors (C# 12): class User(string name, int age) { }
-(class_declaration name: (identifier) @name (parameter_list) @definition.constructor)
-(record_declaration name: (identifier) @name (parameter_list) @definition.constructor)
-
-; Using
-(using_directive (qualified_name) @import.source) @import
-(using_directive (identifier) @import.source) @import
-
-; Calls
-(invocation_expression function: (identifier) @call.name) @call
-(invocation_expression function: (member_access_expression name: (identifier) @call.name)) @call
-
-; Null-conditional method calls: user?.Save()
-; Parses as: invocation_expression → conditional_access_expression → member_binding_expression → identifier
-(invocation_expression
- function: (conditional_access_expression
- (member_binding_expression
- (identifier) @call.name))) @call
-
-; Constructor calls: new Foo() and new Foo { Props }
-(object_creation_expression type: (identifier) @call.name) @call
-
-; Target-typed new (C# 9): User u = new("x", 5)
-(variable_declaration type: (identifier) @call.name (variable_declarator (implicit_object_creation_expression) @call))
-
-; Heritage
-(class_declaration name: (identifier) @heritage.class
- (base_list (identifier) @heritage.extends)) @heritage
-(class_declaration name: (identifier) @heritage.class
- (base_list (generic_name (identifier) @heritage.extends))) @heritage
-
-; Type references — captures types used in field/param/return-type positions
-(parameter type: (identifier) @reference.type)
-(variable_declaration type: (identifier) @reference.type)
-(field_declaration (variable_declaration type: (identifier) @reference.type))
-(method_declaration returns: (identifier) @reference.type)
-`;
-// Rust queries - works with tree-sitter-rust
-export const RUST_QUERIES = `
-; Functions & Items
-(function_item name: (identifier) @name) @definition.function
-(struct_item name: (type_identifier) @name) @definition.struct
-(enum_item name: (type_identifier) @name) @definition.enum
-(trait_item name: (type_identifier) @name) @definition.trait
-(impl_item type: (type_identifier) @name !trait) @definition.impl
-(impl_item type: (generic_type type: (type_identifier) @name) !trait) @definition.impl
-(mod_item name: (identifier) @name) @definition.module
-
-; Type aliases, const, static, macros
-(type_item name: (type_identifier) @name) @definition.type
-(const_item name: (identifier) @name) @definition.const
-(static_item name: (identifier) @name) @definition.static
-(macro_definition name: (identifier) @name) @definition.macro
-
-; Use statements
-(use_declaration argument: (_) @import.source) @import
-
-; Calls
-(call_expression function: (identifier) @call.name) @call
-(call_expression function: (field_expression field: (field_identifier) @call.name)) @call
-(call_expression function: (scoped_identifier
- path: (identifier) @_qualifier
- name: (identifier) @call.name)) @call
-(call_expression function: (generic_function function: (identifier) @call.name)) @call
-
-; Struct literal construction: User { name: value }
-(struct_expression name: (type_identifier) @call.name) @call
-
-; Type references — captures types used in field/param/return-type positions
-(field_declaration type: (type_identifier) @reference.type)
-(parameter pattern: (_) type: (type_identifier) @reference.type)
-(function_item return_type: (type_identifier) @reference.type)
-
-; Wrapped type references — &Type, &mut Type
-(field_declaration type: (reference_type (type_identifier) @reference.type))
-(parameter pattern: (_) type: (reference_type (type_identifier) @reference.type))
-(function_item return_type: (reference_type (type_identifier) @reference.type))
-
-; Generic wrapper types — Box<Type>, Arc<Type>, Vec<Type>, Option<Type>, Result<Type,_>, etc.
-; Capture the type argument (inner type), not the wrapper name.
-(field_declaration type: (generic_type type_arguments: (type_arguments (type_identifier) @reference.type)))
-(parameter pattern: (_) type: (generic_type type_arguments: (type_arguments (type_identifier) @reference.type)))
-(function_item return_type: (generic_type type_arguments: (type_arguments (type_identifier) @reference.type)))
-
-; impl Trait in parameter / return positions
-(parameter pattern: (_) type: (abstract_type (type_identifier) @reference.type))
-(function_item return_type: (abstract_type (type_identifier) @reference.type))
-
-; dyn Trait — trait objects (direct: fn foo(x: dyn Trait) or field: x: dyn Trait)
-(field_declaration type: (dynamic_type (type_identifier) @reference.type))
-(parameter pattern: (_) type: (dynamic_type (type_identifier) @reference.type))
-(function_item return_type: (dynamic_type (type_identifier) @reference.type))
-
-; dyn Trait nested inside generic wrapper — Box<dyn Trait>, Arc<dyn Trait>, etc.
-(field_declaration type: (generic_type type_arguments: (type_arguments (dynamic_type (type_identifier) @reference.type))))
-(parameter pattern: (_) type: (generic_type type_arguments: (type_arguments (dynamic_type (type_identifier) @reference.type))))
-(function_item return_type: (generic_type type_arguments: (type_arguments (dynamic_type (type_identifier) @reference.type))))
-
-; Heritage (trait implementation) — all combinations of concrete/generic trait × concrete/generic type
-(impl_item trait: (type_identifier) @heritage.trait type: (type_identifier) @heritage.class) @heritage
-(impl_item trait: (generic_type type: (type_identifier) @heritage.trait) type: (type_identifier) @heritage.class) @heritage
-(impl_item trait: (type_identifier) @heritage.trait type: (generic_type type: (type_identifier) @heritage.class)) @heritage
-(impl_item trait: (generic_type type: (type_identifier) @heritage.trait) type: (generic_type type: (type_identifier) @heritage.class)) @heritage
-`;
-// PHP queries - works with tree-sitter-php (php_only grammar)
-export const PHP_QUERIES = `
-; ── Namespace ────────────────────────────────────────────────────────────────
-(namespace_definition
- name: (namespace_name) @name) @definition.namespace
-
-; ── Classes ──────────────────────────────────────────────────────────────────
-(class_declaration
- name: (name) @name) @definition.class
-
-; ── Interfaces ───────────────────────────────────────────────────────────────
-(interface_declaration
- name: (name) @name) @definition.interface
-
-; ── Traits ───────────────────────────────────────────────────────────────────
-(trait_declaration
- name: (name) @name) @definition.trait
-
-; ── Enums (PHP 8.1) ──────────────────────────────────────────────────────────
-(enum_declaration
- name: (name) @name) @definition.enum
-
-; ── Top-level functions ───────────────────────────────────────────────────────
-(function_definition
- name: (name) @name) @definition.function
-
-; ── Methods (including constructors) ─────────────────────────────────────────
-(method_declaration
- name: (name) @name) @definition.method
-
-; ── Class properties (including Eloquent $fillable, $casts, etc.) ────────────
-(property_declaration
- (property_element
- (variable_name
- (name) @name))) @definition.property
-
-; ── Imports: use statements ──────────────────────────────────────────────────
-; Simple: use App\\Models\\User;
-(namespace_use_declaration
- (namespace_use_clause
- (qualified_name) @import.source)) @import
-
-; ── Function/method calls ────────────────────────────────────────────────────
-; Regular function call: foo()
-(function_call_expression
- function: (name) @call.name) @call
-
-; Method call: $obj->method()
-(member_call_expression
- name: (name) @call.name) @call
-
-; Nullsafe method call: $obj?->method()
-(nullsafe_member_call_expression
- name: (name) @call.name) @call
-
-; Static call: Foo::bar() (php_only uses scoped_call_expression)
-(scoped_call_expression
- name: (name) @call.name) @call
-
-; Constructor call: new User()
-(object_creation_expression (name) @call.name) @call
-
-; ── Heritage: extends ────────────────────────────────────────────────────────
-(class_declaration
- name: (name) @heritage.class
- (base_clause
- [(name) (qualified_name)] @heritage.extends)) @heritage
-
-; ── Heritage: implements ─────────────────────────────────────────────────────
-(class_declaration
- name: (name) @heritage.class
- (class_interface_clause
- [(name) (qualified_name)] @heritage.implements)) @heritage.impl
-
-; ── Heritage: use trait (must capture enclosing class name) ──────────────────
-(class_declaration
- name: (name) @heritage.class
- body: (declaration_list
- (use_declaration
- [(name) (qualified_name)] @heritage.trait))) @heritage
-
-; ── Type references ───────────────────────────────────────────────────────────
-(named_type (name) @reference.type)
-`;
-// Ruby queries - works with tree-sitter-ruby
-export const RUBY_QUERIES = `
-; ── Modules ──────────────────────────────────────────────────────────────────
-(module
- name: (constant) @name) @definition.module
-
-; ── Classes ──────────────────────────────────────────────────────────────────
-(class
- name: (constant) @name) @definition.class
-
-; Scoped class name: class Module::ClassName
-(class
- name: (scope_resolution) @name) @definition.class
-
-; ── Instance methods ─────────────────────────────────────────────────────────
-(method
- name: (identifier) @name) @definition.method
-
-; ── Singleton (class-level) methods ──────────────────────────────────────────
-(singleton_method
- name: (identifier) @name) @definition.method
-
-; ── Require / require_relative → IMPORTS ─────────────────────────────────────
-; Capture the string argument so the second pass emits an IMPORTS edge.
-(call
- method: (identifier) @_req
- arguments: (argument_list
- (string (string_content) @import.source))
- (#match? @_req "^require")) @import
-
-; ── All other calls ───────────────────────────────────────────────────────────
-(call
- method: (identifier) @call.name) @call
-
-; ── Bare calls without parens (identifiers at statement level are method calls) ─
-; NOTE: This may over-capture variable reads as calls (e.g. 'result' at
-; statement level). Ruby's grammar makes bare identifiers ambiguous — they
-; could be local variables or zero-arity method calls. Post-processing via
-; the BUILTINS set suppresses most false positives.
-(body_statement
- (identifier) @call.name @call)
-
-; ── Heritage: class < SuperClass ─────────────────────────────────────────────
-(class
- name: (constant) @heritage.class
- superclass: (superclass
- (constant) @heritage.extends)) @heritage
-
-; Scoped superclass: class Foo < Module::Class
-; Captures full "Module::Class" text to avoid ambiguity with bare "Class" names
-(class
- name: (constant) @heritage.class
- superclass: (superclass
- (scope_resolution) @heritage.extends)) @heritage
-
-; Scoped class name, bare superclass: class Module::Foo < Bar
-(class
- name: (scope_resolution) @heritage.class
- superclass: (superclass
- (constant) @heritage.extends)) @heritage
-
-; Scoped class name, scoped superclass: class Module::Foo < Module::Bar
-(class
- name: (scope_resolution) @heritage.class
- superclass: (superclass
- (scope_resolution) @heritage.extends)) @heritage
-`;
-// Kotlin queries - works with tree-sitter-kotlin (fwcd/tree-sitter-kotlin)
-// Based on official tags.scm; functions use simple_identifier, classes use type_identifier
-export const KOTLIN_QUERIES = `
-; ── Interfaces ─────────────────────────────────────────────────────────────
-; tree-sitter-kotlin (fwcd) has no interface_declaration node type.
-; Interfaces are class_declaration nodes with an anonymous "interface" keyword child.
-(class_declaration
- "interface"
- (type_identifier) @name) @definition.interface
-
-; ── Classes (regular, data, sealed, enum) ────────────────────────────────
-; All have the anonymous "class" keyword child. enum class has both
-; "enum" and "class" children — the "class" child still matches.
-(class_declaration
- "class"
- (type_identifier) @name) @definition.class
-
-; ── Object declarations (Kotlin singletons) ──────────────────────────────
-(object_declaration
- (type_identifier) @name) @definition.class
-
-; ── Companion objects (named only) ───────────────────────────────────────
-(companion_object
- (type_identifier) @name) @definition.class
-
-; ── Functions (top-level, member, extension) ──────────────────────────────
-(function_declaration
- (simple_identifier) @name) @definition.function
-
-; ── Properties ───────────────────────────────────────────────────────────
-(property_declaration
- (variable_declaration
- (simple_identifier) @name)) @definition.property
-
-; ── Enum entries ─────────────────────────────────────────────────────────
-(enum_entry
- (simple_identifier) @name) @definition.enum
-
-; ── Type aliases ─────────────────────────────────────────────────────────
-(type_alias
- (type_identifier) @name) @definition.type
-
-; ── Imports ──────────────────────────────────────────────────────────────
-(import_header
- (identifier) @import.source) @import
-
-; ── Function calls (direct) ──────────────────────────────────────────────
-(call_expression
- (simple_identifier) @call.name) @call
-
-; ── Method calls (via navigation: obj.method()) ──────────────────────────
-(call_expression
- (navigation_expression
- (navigation_suffix
- (simple_identifier) @call.name))) @call
-
-; ── Constructor invocations ──────────────────────────────────────────────
-(constructor_invocation
- (user_type
- (type_identifier) @call.name)) @call
-
-; ── Infix function calls (e.g., a to b, x until y) ──────────────────────
-(infix_expression
- (simple_identifier) @call.name) @call
-
-; ── Heritage: extends / implements via delegation_specifier ──────────────
-; Interface implementation (bare user_type): class Foo : Bar
-(class_declaration
- (type_identifier) @heritage.class
- (delegation_specifier
- (user_type (type_identifier) @heritage.extends))) @heritage
-
-; Class extension (constructor_invocation): class Foo : Bar()
-(class_declaration
- (type_identifier) @heritage.class
- (delegation_specifier
- (constructor_invocation
- (user_type (type_identifier) @heritage.extends)))) @heritage
-
-; ── Type references ───────────────────────────────────────────────────────────
-(type_reference (user_type (type_identifier) @reference.type))
-`;
-// Swift queries - works with tree-sitter-swift
-export const SWIFT_QUERIES = `
-; Classes
-(class_declaration "class" name: (type_identifier) @name) @definition.class
-
-; Structs
-(class_declaration "struct" name: (type_identifier) @name) @definition.struct
-
-; Enums
-(class_declaration "enum" name: (type_identifier) @name) @definition.enum
-
-; Extensions (mapped to class — no dedicated label in schema)
-(class_declaration "extension" name: (user_type (type_identifier) @name)) @definition.class
-
-; Actors
-(class_declaration "actor" name: (type_identifier) @name) @definition.class
-
-; Protocols (mapped to interface)
-(protocol_declaration name: (type_identifier) @name) @definition.interface
-
-; Type aliases
-(typealias_declaration name: (type_identifier) @name) @definition.type
-
-; Functions (top-level and methods)
-(function_declaration name: (simple_identifier) @name) @definition.function
-
-; Protocol method declarations
-(protocol_function_declaration name: (simple_identifier) @name) @definition.method
-
-; Initializers — "init" is the canonical name; line offset tracked via definition node
-(init_declaration "init" @name) @definition.constructor
-
-; Properties (stored and computed)
-(property_declaration (pattern (simple_identifier) @name)) @definition.property
-
-; Imports
-(import_declaration (identifier (simple_identifier) @import.source)) @import
-
-; Calls - direct function calls
-(call_expression (simple_identifier) @call.name) @call
-
-; Calls - member/navigation calls (obj.method())
-(call_expression (navigation_expression (navigation_suffix (simple_identifier) @call.name))) @call
-
-; Heritage - class/struct/enum inheritance and protocol conformance
-(class_declaration name: (type_identifier) @heritage.class
- (inheritance_specifier inherits_from: (user_type (type_identifier) @heritage.extends))) @heritage
-
-; Heritage - protocol inheritance
-(protocol_declaration name: (type_identifier) @heritage.class
- (inheritance_specifier inherits_from: (user_type (type_identifier) @heritage.extends))) @heritage
-
-; Heritage - extension protocol conformance (e.g. extension Foo: SomeProtocol)
-; Extensions wrap the name in user_type unlike class/struct/enum declarations
-(class_declaration "extension" name: (user_type (type_identifier) @heritage.class)
- (inheritance_specifier inherits_from: (user_type (type_identifier) @heritage.extends))) @heritage
-
-; Type references — captures types used in annotations/parameters/return types
-(type_annotation (user_type (type_identifier) @reference.type))
-`;
-// Scala queries - works with tree-sitter-scala
-export const SCALA_QUERIES = `
-; ── Classes (class, case class, abstract class, sealed class) ────────────────
-(class_definition
- name: (identifier) @name) @definition.class
-
-; ── Traits (mapped to interface) ─────────────────────────────────────────────
-(trait_definition
- name: (identifier) @name) @definition.interface
-
-; ── Object definitions (companion objects, case objects, singletons) ─────────
-(object_definition
- name: (identifier) @name) @definition.class
-
-; ── Function / method definitions ────────────────────────────────────────────
-(function_definition
- name: (identifier) @name) @definition.function
-
-; ── Abstract method declarations (in traits / abstract classes) ──────────────
-(function_declaration
- name: (identifier) @name) @definition.method
-
-; ── Type aliases ──────────────────────────────────────────────────────────────
-(type_definition
- name: (type_identifier) @name) @definition.type
-
-; ── Val/var definitions (class-level members only, not method-local vars) ──────
-; Restricting to template_body ensures we only capture class/object-level
-; declarations (e.g. companion object implicits) and not local val bindings
-; inside method bodies, which would add noise entities to the graph.
-(template_body
- (val_definition (identifier) @name) @definition.const)
-
-(template_body
- (var_definition (identifier) @name) @definition.const)
-
-; ── Imports: capture full declaration for dotted-path reconstruction ─────────
-; Handles: import ix.memory.model._ (wildcard)
-; import ix.memory.model.{NodeKind, GraphNode} (selective)
-; import ix.memory.model.NodeKind (simple)
-(import_declaration) @import.stmt
-
-; ── Calls: direct function call ──────────────────────────────────────────────
-(call_expression
- function: (identifier) @call.name) @call
-
-; ── Calls: method call (obj.method()) ────────────────────────────────────────
-(call_expression
- function: (field_expression
- field: (identifier) @call.name)) @call
-
-; ── Constructor: new Foo() ────────────────────────────────────────────────────
-(instance_expression
- (type_identifier) @call.name) @call
-
-; ── Heritage: class / object extends and with ────────────────────────────────
-(class_definition
- name: (identifier) @heritage.class
- extend: (extends_clause
- (type_identifier) @heritage.extends)) @heritage
-
-(trait_definition
- name: (identifier) @heritage.class
- extend: (extends_clause
- (type_identifier) @heritage.extends)) @heritage
-
-(object_definition
- name: (identifier) @heritage.class
- extend: (extends_clause
- (type_identifier) @heritage.extends)) @heritage
-
-; ── Type references ───────────────────────────────────────────────────────────
-; Case class parameter types: case class Foo(x: NodeKind)
-(class_parameter (type_identifier) @reference.type)
-; Method parameter types: def foo(x: NodeKind)
-(parameter (type_identifier) @reference.type)
-; Val/var type annotations: val x: NodeKind = ...
-(val_definition (type_identifier) @reference.type)
-(var_definition (type_identifier) @reference.type)
-; Return type annotations: def foo(): NodeKind = ...
-; (type_identifier is a direct child of function_definition, no named field)
-(function_definition (type_identifier) @reference.type)
-; Generic type arguments: Map[NodeId, NodeKind], Option[NodeKind], Set[NodeKind], etc.
-(generic_type (type_arguments (type_identifier) @reference.type))
-
-; ── Qualified singleton / companion-object member access ──────────────────────
-; Two grammar shapes arise in practice:
-; expression context: NodeKind.Class → field_expression
-; pattern context: case Some(NodeKind.Class) => … → stable_identifier
-; Both shapes are captured below. The (#match? ^[A-Z]) predicates restrict to
-; UpperCamelCase identifiers (Scala convention for case objects / companion
-; object members), filtering out lowercase method / field names.
-
-; Expression: val x = NodeKind.Decision
-(field_expression
- value: (identifier) @_qualifier
- field: (identifier) @call.name
- (#match? @_qualifier "^[A-Z]")
- (#match? @call.name "^[A-Z]"))
-
-; Pattern: case Some(NodeKind.Class) | Some(NodeKind.Trait) => …
-(stable_identifier
- (identifier) @_qualifier
- "."
- (identifier) @call.name
- (#match? @_qualifier "^[A-Z]")
- (#match? @call.name "^[A-Z]"))
-`;
-export const LANGUAGE_QUERIES = {
- [SupportedLanguages.TypeScript]: TYPESCRIPT_QUERIES,
- [SupportedLanguages.JavaScript]: JAVASCRIPT_QUERIES,
- [SupportedLanguages.Python]: PYTHON_QUERIES,
- [SupportedLanguages.Java]: JAVA_QUERIES,
- [SupportedLanguages.C]: C_QUERIES,
- [SupportedLanguages.Go]: GO_QUERIES,
- [SupportedLanguages.CPlusPlus]: CPP_QUERIES,
- [SupportedLanguages.CSharp]: CSHARP_QUERIES,
- [SupportedLanguages.Ruby]: RUBY_QUERIES,
- [SupportedLanguages.Rust]: RUST_QUERIES,
- [SupportedLanguages.PHP]: PHP_QUERIES,
- [SupportedLanguages.Kotlin]: KOTLIN_QUERIES,
- [SupportedLanguages.Swift]: SWIFT_QUERIES,
- [SupportedLanguages.Scala]: SCALA_QUERIES,
-};
diff --git a/core-ingestion/dist/role-classifier.d.ts b/core-ingestion/dist/role-classifier.d.ts
deleted file mode 100644
index 3773ca06..00000000
--- a/core-ingestion/dist/role-classifier.d.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-export type EntityRole = 'production' | 'test' | 'fixture' | 'generated' | 'external' | 'tooling';
-export interface RoleClassification {
- role: EntityRole;
- role_confidence: number;
- role_signals: string[];
-}
-export declare function classifyFileRole(filePath: string, source?: string): RoleClassification;
diff --git a/core-ingestion/dist/role-classifier.js b/core-ingestion/dist/role-classifier.js
deleted file mode 100644
index d1671b21..00000000
--- a/core-ingestion/dist/role-classifier.js
+++ /dev/null
@@ -1,210 +0,0 @@
-import * as nodePath from 'node:path';
-// ---------------------------------------------------------------------------
-// Signal tables
-// ---------------------------------------------------------------------------
-const TEST_PATH_PATTERNS = [
- { pattern: /[/\\]__tests__[/\\]/i, signal: 'path:__tests__' },
- { pattern: /[/\\]tests?[/\\]/i, signal: 'path:tests/' },
- { pattern: /[/\\]spec[/\\]/i, signal: 'path:spec/' },
- { pattern: /[/\\]src[/\\]test[/\\]/i, signal: 'path:src/test/' },
- { pattern: /[/\\]test[/\\]unit[/\\]/i, signal: 'path:test/unit/' },
- { pattern: /[/\\]test[/\\]integration[/\\]/i, signal: 'path:test/integration/' },
- // C#/.NET: directories named Foo.Tests or Foo.Test (e.g. Newtonsoft.Json.Tests/)
- { pattern: /[/\\][^/\\]*\.tests?[/\\]/i, signal: 'path:.Tests/' },
-];
-const TEST_FILE_PATTERNS = [
- { pattern: /\.test\.[^.]+$/i, signal: 'filename:.test.' },
- { pattern: /\.spec\.[^.]+$/i, signal: 'filename:.spec.' },
- { pattern: /Tests?\.[^.]+$/, signal: 'filename:Test(s).' }, // matches Test.cs and Tests.cs
- { pattern: /Spec\.[^.]+$/, signal: 'filename:Spec.' },
- { pattern: /_test\.[^.]+$/i, signal: 'filename:_test.' },
- { pattern: /_spec\.[^.]+$/i, signal: 'filename:_spec.' },
-];
-const TEST_IMPORT_PATTERNS = [
- { pattern: /from ['"](?:jest|vitest|mocha|chai|sinon|jasmine|@testing-library)/, signal: 'import:js_test_framework' },
- { pattern: /require\(['"](?:jest|vitest|mocha|chai|sinon|jasmine)['"]\)/, signal: 'import:js_test_framework_require' },
- { pattern: /import\s+(?:unittest|pytest)/, signal: 'import:python_test_framework' },
- { pattern: /from\s+(?:unittest|pytest)/, signal: 'import:python_test_framework_from' },
- { pattern: /import\s+org\.junit/, signal: 'import:junit' },
- { pattern: /import\s+org\.testng/, signal: 'import:testng' },
- { pattern: /import\s+io\.mockk/, signal: 'import:mockk' },
- { pattern: /import\s+org\.mockito/, signal: 'import:mockito' },
- { pattern: /import\s+org\.scalatest/, signal: 'import:scalatest' },
- { pattern: /import\s+munit/, signal: 'import:munit' },
- { pattern: /import\s+zio\.test/, signal: 'import:zio_test' },
- { pattern: /"testing"/, signal: 'import:go_testing' },
- { pattern: /require\s+['"]rspec['"]/, signal: 'import:rspec' },
- { pattern: /require\s+['"]minitest['"]/, signal: 'import:minitest' },
- // C#/.NET test frameworks
- { pattern: /using\s+NUnit\.Framework/, signal: 'import:nunit' },
- { pattern: /using\s+Xunit/, signal: 'import:xunit' },
- { pattern: /using\s+Microsoft\.VisualStudio\.TestTools\.UnitTesting/, signal: 'import:mstest' },
-];
-const FIXTURE_PATH_PATTERNS = [
- { pattern: /[/\\]fixtures?[/\\]/i, signal: 'path:fixtures/' },
- { pattern: /[/\\]__fixtures__[/\\]/i, signal: 'path:__fixtures__/' },
- { pattern: /[/\\]test[/\\]resources[/\\]/i, signal: 'path:test/resources/' },
- { pattern: /[/\\]testdata[/\\]/i, signal: 'path:testdata/' },
-];
-const FIXTURE_FILE_PATTERNS = [
- { pattern: /fixture/i, signal: 'filename:fixture' },
- { pattern: /\.mock\.[^.]+$/i, signal: 'filename:.mock.' },
- { pattern: /stub/i, signal: 'filename:stub' },
- { pattern: /fake/i, signal: 'filename:fake' },
- { pattern: /sample/i, signal: 'filename:sample' },
- { pattern: /seed/i, signal: 'filename:seed' },
-];
-const GENERATED_PATH_PATTERNS = [
- { pattern: /[/\\]generated[/\\]/i, signal: 'path:generated/' },
- { pattern: /[/\\]gen[/\\]/i, signal: 'path:gen/' },
- { pattern: /[/\\]\.gen[/\\]/i, signal: 'path:.gen/' },
- { pattern: /\.pb\.[^.]+$/, signal: 'filename:.pb. (protobuf)' },
- { pattern: /\.gen\.[^.]+$/, signal: 'filename:.gen.' },
- { pattern: /_generated\.[^.]+$/, signal: 'filename:_generated.' },
-];
-const GENERATED_SOURCE_PATTERNS = [
- { pattern: /^\/\/ Code generated/m, signal: 'source_marker:code_generated' },
- { pattern: /^\/\/ DO NOT EDIT/m, signal: 'source_marker:do_not_edit' },
- { pattern: /^# Code generated/m, signal: 'source_marker:code_generated_hash' },
- { pattern: /^# DO NOT EDIT/m, signal: 'source_marker:do_not_edit_hash' },
- { pattern: /^\/\* Generated by/m, signal: 'source_marker:generated_by' },
- { pattern: /^\/\* AUTO-GENERATED/im, signal: 'source_marker:auto_generated' },
- { pattern: /^\/\/ AUTO-GENERATED/im, signal: 'source_marker:auto_generated_line' },
-];
-const EXTERNAL_PATH_PATTERNS = [
- { pattern: /[/\\]vendor[/\\]/i, signal: 'path:vendor/' },
- { pattern: /[/\\]third[-_]?party[/\\]/i, signal: 'path:third_party/' },
- { pattern: /[/\\]extern[/\\]/i, signal: 'path:extern/' },
-];
-const TOOLING_PATH_PATTERNS = [
- { pattern: /[/\\]scripts?[/\\]/i, signal: 'path:scripts/' },
- { pattern: /[/\\]tools?[/\\]/i, signal: 'path:tools/' },
- { pattern: /[/\\]ci[/\\]/i, signal: 'path:ci/' },
- { pattern: /[/\\]\.github[/\\]/i, signal: 'path:.github/' },
- { pattern: /[/\\]dev[/\\]/i, signal: 'path:dev/' },
-];
-const TOOLING_FILE_PATTERNS = [
- { pattern: /^Makefile$/, signal: 'filename:Makefile' },
- { pattern: /^Dockerfile[^/\\]*$/, signal: 'filename:Dockerfile' },
- { pattern: /^docker-compose[^/\\]*\.yml$/i, signal: 'filename:docker-compose' },
- { pattern: /\.config\.[^.]+$/i, signal: 'filename:.config.' },
- { pattern: /^vite\.config\./i, signal: 'filename:vite.config' },
- { pattern: /^webpack\.config\./i, signal: 'filename:webpack.config' },
- { pattern: /^jest\.config\./i, signal: 'filename:jest.config' },
- { pattern: /^babel\.config\./i, signal: 'filename:babel.config' },
- { pattern: /^tsconfig[^/\\]*\.json$/i, signal: 'filename:tsconfig.json' },
- { pattern: /^\.eslintrc/i, signal: 'filename:.eslintrc' },
- { pattern: /^rollup\.config\./i, signal: 'filename:rollup.config' },
- { pattern: /^build\.[^.]+$/i, signal: 'filename:build.' },
- { pattern: /migrate\.[^.]+$/i, signal: 'filename:migrate.' },
-];
-// ---------------------------------------------------------------------------
-// Main classifier
-// ---------------------------------------------------------------------------
-export function classifyFileRole(filePath, source) {
- const normalizedPath = filePath.replace(/\\/g, '/');
- const fileName = nodePath.basename(filePath);
- const signals = [];
- // --- Test ---
- let testScore = 0;
- for (const { pattern, signal } of TEST_PATH_PATTERNS) {
- if (pattern.test(normalizedPath)) {
- testScore += 0.8;
- signals.push(signal);
- break;
- }
- }
- for (const { pattern, signal } of TEST_FILE_PATTERNS) {
- if (pattern.test(fileName)) {
- testScore += 0.9;
- signals.push(signal);
- break;
- }
- }
- if (source) {
- for (const { pattern, signal } of TEST_IMPORT_PATTERNS) {
- if (pattern.test(source)) {
- testScore += 0.5;
- signals.push(signal);
- break;
- }
- }
- }
- // --- Fixture ---
- let fixtureScore = 0;
- for (const { pattern, signal } of FIXTURE_PATH_PATTERNS) {
- if (pattern.test(normalizedPath)) {
- fixtureScore += 0.8;
- signals.push(signal);
- break;
- }
- }
- for (const { pattern, signal } of FIXTURE_FILE_PATTERNS) {
- if (pattern.test(fileName)) {
- fixtureScore += 0.6;
- signals.push(signal);
- break;
- }
- }
- // --- Generated ---
- let generatedScore = 0;
- for (const { pattern, signal } of GENERATED_PATH_PATTERNS) {
- if (pattern.test(normalizedPath)) {
- generatedScore += 0.7;
- signals.push(signal);
- break;
- }
- }
- if (source) {
- for (const { pattern, signal } of GENERATED_SOURCE_PATTERNS) {
- if (pattern.test(source)) {
- generatedScore += 0.9;
- signals.push(signal);
- break;
- }
- }
- }
- // --- External ---
- let externalScore = 0;
- for (const { pattern, signal } of EXTERNAL_PATH_PATTERNS) {
- if (pattern.test(normalizedPath)) {
- externalScore += 0.9;
- signals.push(signal);
- break;
- }
- }
- // --- Tooling ---
- let toolingScore = 0;
- for (const { pattern, signal } of TOOLING_PATH_PATTERNS) {
- if (pattern.test(normalizedPath)) {
- toolingScore += 0.6;
- signals.push(signal);
- break;
- }
- }
- for (const { pattern, signal } of TOOLING_FILE_PATTERNS) {
- if (pattern.test(fileName)) {
- toolingScore += 0.7;
- signals.push(signal);
- break;
- }
- }
- // --- Pick winner ---
- const scores = [
- ['test', testScore],
- ['fixture', fixtureScore],
- ['generated', generatedScore],
- ['external', externalScore],
- ['tooling', toolingScore],
- ];
- scores.sort((a, b) => b[1] - a[1]);
- const [topRole, topScore] = scores[0];
- if (topScore >= 0.5) {
- return {
- role: topRole,
- role_confidence: parseFloat(Math.min(topScore, 1.0).toFixed(2)),
- role_signals: signals,
- };
- }
- return { role: 'production', role_confidence: 0.5, role_signals: [] };
-}
diff --git a/core-ingestion/dist/types.d.ts b/core-ingestion/dist/types.d.ts
deleted file mode 100644
index c0868c43..00000000
--- a/core-ingestion/dist/types.d.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-export interface PatchSource {
- uri: string;
- sourceHash?: string;
- extractor: string;
- sourceType: string;
-}
-export interface PatchOp {
- type: string;
- [key: string]: unknown;
-}
-export interface GraphPatchPayload {
- patchId: string;
- actor: string;
- timestamp: string;
- source: PatchSource;
- baseRev: number;
- ops: PatchOp[];
- replaces: string[];
- intent?: string;
-}
diff --git a/core-ingestion/dist/types.js b/core-ingestion/dist/types.js
deleted file mode 100644
index cb0ff5c3..00000000
--- a/core-ingestion/dist/types.js
+++ /dev/null
@@ -1 +0,0 @@
-export {};
diff --git a/core-ingestion/src/__tests__/patchBuilder.resolution.test.ts b/core-ingestion/src/__tests__/patchBuilder.resolution.test.ts
index 0a36cabb..1a4aae1e 100644
--- a/core-ingestion/src/__tests__/patchBuilder.resolution.test.ts
+++ b/core-ingestion/src/__tests__/patchBuilder.resolution.test.ts
@@ -2,7 +2,7 @@ import * as crypto from 'node:crypto';
import { describe, expect, it } from 'vitest';
-import type { FileParseResult, ParsedEntity, ParsedRelationship, ResolvedEdge } from '../index.js';
+import { parseFile, resolveEdges, type FileParseResult, type ParsedEntity, type ParsedRelationship, type ResolvedEdge } from '../index.js';
import { SupportedLanguages } from '../languages.js';
import { buildPatchWithResolution } from '../patch-builder.js';
@@ -108,4 +108,45 @@ describe('buildPatchWithResolution', () => {
dst: nodeId('/repo/compaction_job.cc', 'CompactionJob.Run'),
}));
});
+
+ it('rewrites resolved C++ imports to the imported file node', () => {
+ const importer = parseFile(
+ '/repo/include/KimeraRPGO/outlier/Pcm.h',
+ `
+#include "KimeraRPGO/utils/GraphUtils.h"
+
+class Pcm {};
+ `,
+ );
+ const imported = parseFile(
+ '/repo/include/KimeraRPGO/utils/GraphUtils.h',
+ `
+struct Trajectory {};
+ `,
+ );
+
+ expect(importer).not.toBeNull();
+ expect(imported).not.toBeNull();
+
+ const resolvedEdges = resolveEdges([importer!, imported!]);
+ expect(resolvedEdges).toContainEqual({
+ srcFilePath: '/repo/include/KimeraRPGO/outlier/Pcm.h',
+ srcName: 'Pcm.h',
+ dstFilePath: '/repo/include/KimeraRPGO/utils/GraphUtils.h',
+ dstName: 'KimeraRPGO/utils/GraphUtils.h',
+ dstQualifiedKey: 'GraphUtils.h',
+ predicate: 'IMPORTS',
+ confidence: 0.9,
+ });
+
+ const patch = buildPatchWithResolution(importer!, 'test-hash', resolvedEdges);
+ const importEdge = patch.ops.find(
+ (op) => op.type === 'UpsertEdge' && op.predicate === 'IMPORTS',
+ );
+
+ expect(importEdge).toEqual(expect.objectContaining({
+ src: nodeId('/repo/include/KimeraRPGO/outlier/Pcm.h', 'Pcm.h'),
+ dst: nodeId('/repo/include/KimeraRPGO/utils/GraphUtils.h', 'GraphUtils.h'),
+ }));
+ });
});
diff --git a/core-ingestion/src/__tests__/queries.c.test.ts b/core-ingestion/src/__tests__/queries.c.test.ts
index 3a81de31..dedd8ac4 100644
--- a/core-ingestion/src/__tests__/queries.c.test.ts
+++ b/core-ingestion/src/__tests__/queries.c.test.ts
@@ -106,4 +106,35 @@ static int smtp_connect(struct Curl_easy *data, struct connectdata *conn) {
.map(r => r.dstName);
expect(importTargets).toContain('curl.h');
});
+
+ // BUG-1: #define macros must be classified as 'macro', not 'function'
+ it('classifies #define macros as kind macro, not function', () => {
+ const result = parseFile(
+ '/repo/curl.h',
+ `
+#ifndef CURLINC_CURL_H
+#define CURLINC_CURL_H
+
+#define CURL_STRICTER
+#define CURL_EXTERN extern
+#define enquote(x) #x
+#define expand(x) enquote(x)
+
+void curl_global_init(void);
+ `,
+ );
+
+ expect(result).not.toBeNull();
+ const macroEntities = result!.entities.filter(e =>
+ ['CURLINC_CURL_H', 'CURL_STRICTER', 'CURL_EXTERN', 'enquote', 'expand'].includes(e.name),
+ );
+ expect(macroEntities.length).toBeGreaterThan(0);
+ for (const e of macroEntities) {
+ expect(e.kind).toBe('macro');
+ }
+
+ // Real function must still be 'function'
+ const fn = result!.entities.find(e => e.name === 'curl_global_init');
+ expect(fn?.kind).toBe('function');
+ });
});
diff --git a/core-ingestion/src/__tests__/queries.python.test.ts b/core-ingestion/src/__tests__/queries.python.test.ts
index 9e1dc0a1..a7114b38 100644
--- a/core-ingestion/src/__tests__/queries.python.test.ts
+++ b/core-ingestion/src/__tests__/queries.python.test.ts
@@ -95,6 +95,25 @@ def my_view(request):
});
});
+ it('emits REFERENCES edge for class passed as keyword argument value', () => {
+ const result = parseFile(
+ '/repo/admin/sites.py',
+ `
+class AdminSite:
+ def login(self, request):
+ return LoginView.as_view(authentication_form=AdminAuthenticationForm, extra_context=context)
+ `,
+ );
+
+ expect(result).not.toBeNull();
+ // AdminAuthenticationForm passed as kwarg value → CALLS edge from login
+ expect(result!.relationships).toContainEqual({
+ srcName: 'AdminSite.login',
+ dstName: 'AdminAuthenticationForm',
+ predicate: 'CALLS',
+ });
+ });
+
it('resolves qualifier from Model.objects.method() assignment (Pattern 2)', () => {
const result = parseFile(
'/repo/views.py',
diff --git a/core-ingestion/src/__tests__/resolveEdges.test.ts b/core-ingestion/src/__tests__/resolveEdges.test.ts
index cb1e3961..c46ad0fb 100644
--- a/core-ingestion/src/__tests__/resolveEdges.test.ts
+++ b/core-ingestion/src/__tests__/resolveEdges.test.ts
@@ -336,4 +336,32 @@ describe('resolveEdges', () => {
expect(resolveEdges([ambiguousGlobal, helperA, helperB])).toEqual([]);
});
+
+ // BUG-2: struct references in C that come from system headers (<net/if.h>)
+ // must not be linked to an in-repo definition of the same struct name via
+ // global tier-3 fallback.
+ it('does not create a false REFERENCES edge for a C struct from a system header', () => {
+ // CurlTests.c: includes system <net/if.h> (not in batch) and uses struct ifreq
+ const curlTests = fileResult(
+ '/repo/CMake/CurlTests.c',
+ SupportedLanguages.C,
+ [],
+ [
+ { srcName: 'CurlTests.c', dstName: 'net/if.h', predicate: 'IMPORTS' },
+ { srcName: 'CurlTests.c', dstName: 'ifreq', predicate: 'REFERENCES' },
+ ],
+ );
+
+ // if2ip.h: defines its own struct ifreq as a platform shim
+ const if2ip = fileResult(
+ '/repo/lib/if2ip.c',
+ SupportedLanguages.C,
+ [entity('ifreq', SupportedLanguages.C, 'class')],
+ [],
+ );
+
+ const resolved = resolveEdges([curlTests, if2ip]);
+ const refEdges = resolved.filter(e => e.predicate === 'REFERENCES');
+ expect(refEdges).toEqual([]);
+ });
});
diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.ts
index dd12532c..60dd05b8 100644
--- a/core-ingestion/src/index.ts
+++ b/core-ingestion/src/index.ts
@@ -122,7 +122,7 @@ const DEFINITION_KIND_MAP: Record<string, string> = {
'definition.property': 'function',
'definition.const': 'function',
'definition.static': 'function',
- 'definition.macro': 'function',
+ 'definition.macro': 'macro',
'definition.union': 'class',
'definition.typedef': 'class',
'definition.template': 'class',
@@ -221,6 +221,7 @@ export function isGrammarSupported(filePath: string): boolean {
// ---------------------------------------------------------------------------
let _parser: Parser | null = null;
+let _currentGrammar: any = null;
const _queryCache = new Map<SupportedLanguages | 'tsx', { grammar: any; query: any }>();
function getParser(): Parser {
@@ -240,6 +241,42 @@ function getCachedQuery(
return query;
}
+function normalizeCapturedImport(rawValue: string, language: SupportedLanguages): string {
+ const trimmed = rawValue.trim().replace(/\\\\/g, '/');
+ const unwrapped = trimmed
+ .replace(/^["'`<]/, '')
+ .replace(/[>"'`]$/, '');
+
+ if (language === SupportedLanguages.C || language === SupportedLanguages.CPlusPlus) {
+ return unwrapped.replace(/^\.\/+/, '');
+ }
+
+ const rawMod = unwrapped.split('/').filter((s: string) => s !== '*').pop() ?? unwrapped;
+ return rawMod.replace(/^\.+/, '');
+}
+
+function fileEntityName(filePath: string): string {
+ return nodePath.basename(filePath);
+}
+
+function looksLikeCppHeader(source: string): boolean {
+ return /\bclass\s+[A-Za-z_]\w*\b/.test(source)
+ || /\bnamespace\s+[A-Za-z_]\w*\b/.test(source)
+ || /\btemplate\s*</.test(source)
+ || /\busing\s+namespace\b/.test(source)
+ || /\b(public|private|protected)\s*:/.test(source)
+ || /\b(virtual|constexpr|typename|friend|operator)\b/.test(source)
+ || /\b[A-Za-z_]\w*::[A-Za-z_]\w*/.test(source);
+}
+
+function detectLanguageForSource(filePath: string, source: string): SupportedLanguages | null {
+ const language = languageFromPath(filePath);
+ if (language === SupportedLanguages.C && filePath.toLowerCase().endsWith('.h') && looksLikeCppHeader(source)) {
+ return SupportedLanguages.CPlusPlus;
+ }
+ return language;
+}
+
// ---------------------------------------------------------------------------
// Rust: cfg macro unwrapping
// ---------------------------------------------------------------------------
@@ -252,39 +289,52 @@ function getCachedQuery(
* preserving every character position and line number so that entity line
* numbers remain accurate.
*/
+// Pre-compiled for blanking macro headers (preserves newlines, replaces all else with space)
+const _blankNonNewline = /[^\n]/g;
+
function unwrapRustCfgMacros(source: string): string {
const re = /\bcfg_(?:rt_multi_thread|not_rt|rt|io|time|sync|net|fs|process|signal)!\s*\{/g;
- const chars = source.split('');
+ // Collect edit regions before mutating anything
+ const edits: Array<[number, number, number]> = []; // [macroStart, openBrace, closeBrace]
let m: RegExpExecArray | null;
while ((m = re.exec(source)) !== null) {
- const openBrace = m.index + m[0].length - 1; // position of the `{`
- // Find matching closing brace by counting depth
+ const openBrace = m.index + m[0].length - 1;
let depth = 0;
let closeBrace = -1;
for (let i = openBrace; i < source.length; i++) {
if (source[i] === '{') depth++;
else if (source[i] === '}') {
- depth--;
- if (depth === 0) { closeBrace = i; break; }
+ if (--depth === 0) { closeBrace = i; break; }
}
}
- if (closeBrace === -1) continue;
- // Blank out everything from start of macro name up to and including `{`
- for (let i = m.index; i <= openBrace; i++) {
- if (chars[i] !== '\n') chars[i] = ' ';
- }
- // Blank out the matching closing `}`
- chars[closeBrace] = ' ';
+ if (closeBrace !== -1) edits.push([m.index, openBrace, closeBrace]);
}
- return chars.join('');
+ if (edits.length === 0) return source;
+
+ // Build result from slices — avoids O(n) split('') allocation of N small string objects
+ const parts: string[] = [];
+ let pos = 0;
+ for (const [macroStart, openBrace, closeBrace] of edits) {
+ if (macroStart > pos) parts.push(source.slice(pos, macroStart));
+ // Blank macro name + opening brace (preserve newlines)
+ parts.push(source.slice(macroStart, openBrace + 1).replace(_blankNonNewline, ' '));
+ pos = openBrace + 1;
+ // Keep everything inside the braces unchanged
+ if (closeBrace > pos) parts.push(source.slice(pos, closeBrace));
+ // Blank closing brace
+ parts.push(' ');
+ pos = closeBrace + 1;
+ }
+ if (pos < source.length) parts.push(source.slice(pos));
+ return parts.join('');
}
// ---------------------------------------------------------------------------
// Main parse function
// ---------------------------------------------------------------------------
-export function parseFile(filePath: string, source: string): FileParseResult | null {
- const language = languageFromPath(filePath);
+export function parseFile(filePath: string, source: string, opts?: { emitChunks?: boolean }): FileParseResult | null {
+ const language = detectLanguageForSource(filePath, source);
if (!language) return null;
// TypeScript TSX uses a separate grammar
@@ -297,7 +347,10 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
try {
const parser = getParser();
- parser.setLanguage(grammar);
+ if (grammar !== _currentGrammar) {
+ parser.setLanguage(grammar);
+ _currentGrammar = grammar;
+ }
// tree-sitter-java cannot parse array-type annotations on varargs params
// (e.g. `@Nullable Object @Nullable ... args`). The second annotation causes
// error recovery to truncate the enclosing class_declaration, orphaning all
@@ -320,6 +373,26 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
const query = getCachedQuery(cacheKey, grammar, queries);
const matches = query.matches(tree.rootNode);
+ // Pre-partition matches so each pass only iterates its relevant subset.
+ // Pass 1: definitions, heritage, Python typed-param/assign annotations.
+ // Pass 2: imports, calls, type references.
+ // Without this, every definition match does 5 useless find() calls in pass 2,
+ // and every call match does 4 useless find() calls in pass 1.
+ const pass1Matches: any[] = [];
+ const pass2Matches: any[] = [];
+ for (const m of matches) {
+ let isPass1 = false;
+ for (const c of m.captures) {
+ const n = c.name;
+ if (n.startsWith('definition.') || n.startsWith('heritage.') ||
+ n === '_typed_param_scope' || n === '_assign_scope') {
+ isPass1 = true;
+ break;
+ }
+ }
+ (isPass1 ? pass1Matches : pass2Matches).push(m);
+ }
+
const fileName = nodePath.basename(filePath);
let sourceLineCount = 1;
for (let i = 0; i < source.length; i++) {
@@ -354,7 +427,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
const assignTypeMap = new Map<string, Map<string, string>>();
// --- First pass: collect definitions ---
- for (const match of matches) {
+ for (const match of pass1Matches) {
// Definition captures: name + definition.*
const defCapture = match.captures.find((c: any) =>
c.name.startsWith('definition.')
@@ -487,7 +560,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
}
// --- Second pass: calls and imports ---
- for (const match of matches) {
+ for (const match of pass2Matches) {
// Full import statement captures (Scala: reconstructs dotted package paths)
const importStmt = match.captures.find((c: any) => c.name === 'import.stmt');
if (importStmt) {
@@ -521,12 +594,8 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
// Import captures (JS/TS/Python path-based)
const importSource = match.captures.find((c: any) => c.name === 'import.source');
if (importSource) {
- let importPath = importSource.node.text
- .replace(/^["'`]|["'`]$/g, '') // strip quotes
- .replace(/\\\\/g, '/'); // normalise backslashes
- if (importPath.length > 0 && importPath !== '*') {
- const rawMod = importPath.split('/').filter((s: string) => s !== '*').pop() ?? importPath;
- const modName = rawMod.replace(/^\.+/, ''); // strip leading dots: '.models' → 'models'
+ const modName = normalizeCapturedImport(importSource.node.text, language);
+ if (modName.length > 0 && modName !== '*') {
if (!modName) continue; // skip bare '.' relative imports
entities.push({ name: modName, kind: 'module', lineStart: importSource.node.startPosition.row + 1, lineEnd: importSource.node.startPosition.row + 1, language });
relationships.push({ srcName: fileName, dstName: modName, predicate: 'IMPORTS' });
@@ -629,28 +698,30 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
}
}
- for (const pendingChunk of pendingChunks) {
- const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);
- const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);
- chunks.push({
- ...pendingChunk,
- contentHash,
- });
- }
+ if (opts?.emitChunks !== false) {
+ for (const pendingChunk of pendingChunks) {
+ const chunkText = source.slice(pendingChunk.startByte, pendingChunk.endByte);
+ const contentHash = crypto.createHash('sha256').update(chunkText).digest('hex').slice(0, 16);
+ chunks.push({
+ ...pendingChunk,
+ contentHash,
+ });
+ }
- // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file
- if (chunks.length === 0) {
- const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);
- chunks.push({
- name: null,
- chunkKind: 'file_body',
- lineStart: 1,
- lineEnd: sourceLineCount,
- startByte: 0,
- endByte: source.length,
- contentHash,
- language,
- });
+ // Fallback: if no semantic chunks found, emit one file_body chunk covering the whole file
+ if (chunks.length === 0) {
+ const contentHash = crypto.createHash('sha256').update(source).digest('hex').slice(0, 16);
+ chunks.push({
+ name: null,
+ chunkKind: 'file_body',
+ lineStart: 1,
+ lineEnd: sourceLineCount,
+ startByte: 0,
+ endByte: source.length,
+ contentHash,
+ language,
+ });
+ }
}
return { filePath, language, entities, chunks, relationships, fileRole: classifyFileRole(filePath, source) };
@@ -822,7 +893,7 @@ export function resolveEdges(results: FileParseResult[], stats?: {
for (let i = parts.length - 1; i >= parts.length - maxDepth; i--) {
const pkg = parts.slice(i).join('.');
const list = packageToFiles.get(pkg) ?? [];
- if (!list.includes(r.filePath)) list.push(r.filePath);
+ list.push(r.filePath);
packageToFiles.set(pkg, list);
}
}
@@ -837,7 +908,7 @@ export function resolveEdges(results: FileParseResult[], stats?: {
if (nodePath.extname(r.filePath) !== '.go') continue;
const dirName = nodePath.basename(nodePath.dirname(r.filePath));
const list = goPkgDirToFiles.get(dirName) ?? [];
- if (!list.includes(r.filePath)) list.push(r.filePath);
+ list.push(r.filePath);
goPkgDirToFiles.set(dirName, list);
}
@@ -854,6 +925,10 @@ export function resolveEdges(results: FileParseResult[], stats?: {
// (TS/JS ESM imports use .js extensions that map to .ts source files)
const noExt = modName.replace(/\.(js|ts|mjs|cjs|jsx|tsx|py|scala|java|c|cc|cpp|cxx|h|hh|hpp|hxx)$/, '');
if (noExt !== modName && noExt && !candidates.includes(noExt)) candidates.push(noExt);
+ const pathBasename = nodePath.posix.basename(modName);
+ if (pathBasename && !candidates.includes(pathBasename)) candidates.push(pathBasename);
+ const pathBasenameNoExt = nodePath.posix.basename(noExt);
+ if (pathBasenameNoExt && !candidates.includes(pathBasenameNoExt)) candidates.push(pathBasenameNoExt);
// For dotted paths (Scala/Java: 'ix.memory.model.Edge'), also try last segment
const lastDot = modName.lastIndexOf('.');
if (lastDot !== -1) {
@@ -947,6 +1022,21 @@ export function resolveEdges(results: FileParseResult[], stats?: {
if (canonicalMatches.length === 1) return canonicalMatches;
}
+ if (srcLanguage === SupportedLanguages.Java) {
+ const srcParts = srcFilePath.replace(/\\/g, '/').split('/');
+ const withProximity = matches.map(fp => {
+ const fpParts = fp.replace(/\\/g, '/').split('/');
+ let common = 0;
+ while (common < srcParts.length && common < fpParts.length && srcParts[common] === fpParts[common]) {
+ common++;
+ }
+ return { fp, common };
+ });
+ const maxCommon = Math.max(...withProximity.map(x => x.common));
+ const proximityMatches = withProximity.filter(x => x.common === maxCommon).map(x => x.fp);
+ if (proximityMatches.length === 1) return proximityMatches;
+ }
+
if (srcLanguage !== SupportedLanguages.C && srcLanguage !== SupportedLanguages.CPlusPlus) return matches;
let narrowed = matches;
@@ -1025,21 +1115,44 @@ export function resolveEdges(results: FileParseResult[], stats?: {
// name ("ClaimId"), find files registered under that package, then find
// the one that defines the entity.
if (rel.predicate === 'IMPORTS') {
- if (result.language !== SupportedLanguages.Scala && result.language !== SupportedLanguages.Java) continue;
- const dstName = rel.dstName;
- const lastDot = dstName.lastIndexOf('.');
- if (lastDot === -1) continue; // not a dotted class import
- const entityName = dstName.slice(lastDot + 1);
- if (!entityName || entityName === '_') continue; // wildcard import
- const pkgPath = dstName.slice(0, lastDot); // e.g. "ix.memory.model"
- // Find files in the declared package that define this entity
- const pkgFiles = packageToFiles.get(pkgPath) ?? [];
- const matchFiles = pkgFiles.filter(fp => fp !== srcFilePath && fileHasSymbol.get(fp)?.has(entityName));
- if (matchFiles.length !== 1) continue; // ambiguous or not found
- const fp = matchFiles[0];
- const dstQualifiedKey = bestQKey(fileQKeys, fp, entityName);
- if (dstQualifiedKey === null) continue;
- resolved.push({ srcFilePath, srcName: rel.srcName, dstFilePath: fp, dstName, dstQualifiedKey, predicate: 'IMPORTS', confidence: 0.9 });
+ if (result.language === SupportedLanguages.Scala || result.language === SupportedLanguages.Java) {
+ const dstName = rel.dstName;
+ const lastDot = dstName.lastIndexOf('.');
+ if (lastDot !== -1) {
+ const entityName = dstName.slice(lastDot + 1);
+ if (entityName && entityName !== '_') {
+ const pkgPath = dstName.slice(0, lastDot);
+ const pkgFiles = packageToFiles.get(pkgPath) ?? [];
+ const matchFiles = pkgFiles.filter(fp => fp !== srcFilePath && fileHasSymbol.get(fp)?.has(entityName));
+ if (matchFiles.length === 1) {
+ const fp = matchFiles[0];
+ const dstQualifiedKey = bestQKey(fileQKeys, fp, entityName);
+ if (dstQualifiedKey !== null) {
+ resolved.push({ srcFilePath, srcName: rel.srcName, dstFilePath: fp, dstName, dstQualifiedKey, predicate: 'IMPORTS', confidence: 0.9 });
+ stats.resolvedImport++;
+ continue;
+ }
+ }
+ }
+ }
+ }
+
+ const importMatches = modNameToFiles(rel.dstName, srcFilePath);
+ if (importMatches.length === 1) {
+ const fp = importMatches[0];
+ resolved.push({
+ srcFilePath,
+ srcName: rel.srcName,
+ dstFilePath: fp,
+ dstName: rel.dstName,
+ dstQualifiedKey: fileEntityName(fp),
+ predicate: 'IMPORTS',
+ confidence: 0.9,
+ });
+ stats.resolvedImport++;
+ } else if (importMatches.length > 1) {
+ stats.skippedAmbiguous++;
+ }
continue;
}
@@ -1119,6 +1232,13 @@ export function resolveEdges(results: FileParseResult[], stats?: {
}
// Tier 3: global fallback (confidence 0.5) — uses inverted symbol index
// instead of scanning all files.
+ // Skip for C/C++ REFERENCES: struct/type references that don't resolve via the
+ // import chain come from system headers (e.g. <net/if.h>) and must not be
+ // linked to an in-repo definition of the same name.
+ if (rel.predicate === 'REFERENCES' &&
+ (srcLanguage === SupportedLanguages.C || srcLanguage === SupportedLanguages.CPlusPlus)) {
+ continue;
+ }
stats.globalFallbacks++;
const candidates = symbolToFiles.get(dstName) ?? [];
let globalMatches = candidates.filter(fp => fp !== srcFilePath && fileLanguage.get(fp) === srcLanguage);
diff --git a/core-ingestion/src/patch-builder.ts b/core-ingestion/src/patch-builder.ts
index df704395..5500cb24 100644
--- a/core-ingestion/src/patch-builder.ts
+++ b/core-ingestion/src/patch-builder.ts
@@ -272,6 +272,7 @@ export function buildPatchWithResolution(
sourceHash: string,
resolvedEdges: ResolvedEdge[],
previousSourceHash?: string,
+ opts?: { emitClaims?: boolean },
): GraphPatchPayload {
// Build lookup: `${srcName}:${predicate}:${dstName}` → { dstFilePath, dstQualifiedKey }
// Callers should pass only edges for this file (pre-grouped) for best performance,
@@ -420,15 +421,17 @@ export function buildPatchWithResolution(
});
}
- for (const r of relationships) {
- const srcKey = resolveKey(r.srcName);
- ops.push({
- type: 'AssertClaim',
- entityId: nodeId(filePath, srcKey),
- field: `${r.predicate.toLowerCase()}:${r.dstName}`,
- value: r.dstName,
- confidence: null,
- });
+ if (opts?.emitClaims !== false) {
+ for (const r of relationships) {
+ const srcKey = resolveKey(r.srcName);
+ ops.push({
+ type: 'AssertClaim',
+ entityId: nodeId(filePath, srcKey),
+ field: `${r.predicate.toLowerCase()}:${r.dstName}`,
+ value: r.dstName,
+ confidence: null,
+ });
+ }
}
const extractor = extractorName();
diff --git a/core-ingestion/src/queries.ts b/core-ingestion/src/queries.ts
index 959bcf12..a65fe764 100644
--- a/core-ingestion/src/queries.ts
+++ b/core-ingestion/src/queries.ts
@@ -264,6 +264,14 @@ export const PYTHON_QUERIES = `
(typed_parameter type: (type (identifier) @reference.type))
(typed_default_parameter type: (type (identifier) @reference.type))
+; Class/function references passed as keyword argument values
+; e.g. LoginView.as_view(authentication_form=AdminAuthenticationForm, ...)
+; Emits a CALLS edge from the enclosing function to the referenced identifier.
+(call
+ arguments: (argument_list
+ (keyword_argument
+ value: (identifier) @call.name)))
+
; Typed parameters: capture name + type together so qualifier substitution can
; map a local variable name to its declared class (e.g. query: Query → 'query' → 'Query').
(function_definition
diff --git a/core-ingestion/tsconfig.json b/core-ingestion/tsconfig.json
index 77af2577..3c32c63f 100644
--- a/core-ingestion/tsconfig.json
+++ b/core-ingestion/tsconfig.json
@@ -11,5 +11,6 @@
"skipLibCheck": true,
"types": ["node"]
},
- "include": ["src"]
+ "include": ["src"],
+ "exclude": ["src/__tests__"]
}
diff --git a/docs/plans/2026-03-25-large-repo-ingest-perf.md b/docs/plans/2026-03-25-large-repo-ingest-perf.md
new file mode 100644
index 00000000..ce589a67
--- /dev/null
+++ b/docs/plans/2026-03-25-large-repo-ingest-perf.md
@@ -0,0 +1,288 @@
+# Large Repo Ingest Performance
+
+Date: 2026-03-25
+Branch: `feature/claude-plugin`
+
+---
+
+## Baseline
+
+Kubernetes repo (`files=16010`) before this work:
+
+| Phase | Time |
+|---|---|
+| `ix map` ingest | ~466s |
+| `ix map` map computation | ~53s |
+| **Total `ix map`** | **~519s** |
+
+Debug profile at baseline:
+```
+resolveEdgesMs=29581 buildPatchMs=10156 commitMs=386443 totalSaveMs=426834
+```
+
+The commit path dominated — 386s out of 427s total save time.
+
+---
+
+## What We Did (already merged)
+
+### 1. Compound DB indexes (`memory-layer/src/main/scala/ix/memory/db/ArangoSchema.scala`)
+
+Added compound indexes on `nodes(logical_id, deleted_rev)` and `claims(entity_id, deleted_rev)`. Tombstone + retire AQL queries filter both fields; without this, Arango returned all historical node versions then post-filtered. Now it does direct point lookups.
+
+### 2. Upfront batch tombstone + retire (`memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala`)
+
+In `commitBatchChunked`, when multiple DB chunks exist and `baseRev > 0`, now collects all entity IDs upfront and runs tombstone + retire **once in parallel** before any inserts (`skipTombstone = true` passed to each per-chunk call). Previously ran 320 AQL UPDATE passes for Kubernetes; now runs 2.
+
+Also parallelized tombstone + retire within `commitBatch` for the single-chunk case using `IO.both`.
+
+### 3. Generated file filtering (`ix-cli/src/cli/commands/ingest.ts`)
+
+Skip known generated files by filename before parsing: `*.pb.go`, `zz_generated*`, `*_deepcopy.go`, `*_mock.go`, `mock_*`, `*.pb.ts`, `*.pb.js`, `_pb.ts`, `_pb.js`. Applied in both `walkFiles` and `tryGitLsFiles`.
+
+### 4. Parallel HTTP save requests (`ix-cli/src/cli/commands/ingest.ts`)
+
+Replaced sequential `for` loop in `commitPreparedPatches` with a 4-worker pool (`COMMIT_CONCURRENCY = 4` at line ~416). Fixed progress bar to use `Math.max` so it doesn't jump backward when an out-of-order chunk completes.
+
+---
+
+## Current State (2026-03-25, pre-session baseline)
+
+| Phase | Before | After | Delta |
+|---|---|---|---|
+| Ingest | ~466s | ~200s | **-57%** |
+| Map computation | ~53s | ~87s | +64% (regression) |
+| **Total `ix map`** | **~519s** | **~287s** | **-45%** |
+
+Graph after full Kubernetes ingest:
+```
+nodes: 312,140 (113k chunk, 76k module, 43k function, 42k method, 16k class, 15k file)
+edges: 840,843 (319k CALLS, 107k CONTAINS_CHUNK, 105k CONTAINS, 95k NEXT, 76k IMPORTS)
+```
+
+---
+
+## Current State (2026-03-26, post-session)
+
+| Phase | Before | After | Delta |
+|---|---|---|---|
+| Ingest | ~200s | ~227s | +13% (conflict fix overhead) |
+| Map computation | ~87s | ~82s | -6% |
+| **Total `ix map`** | **~287s** | **~309s** | +8% |
+
+Ingest debug breakdown (15035 files, full fresh ingest):
+```
+resolveEdgesMs=26252 buildPatchMs=8731 commitMs=607952 totalSaveMs=188862
+```
+
+Ingest is currently slightly slower than the pre-session baseline. The chunk-skipping benefit (~30% fewer DB inserts) is real but offset by the COMMIT_CONCURRENCY revert to 4 (was already 4 at baseline, no gain there) and normal run variance. Ingest is clean — zero write-write conflict errors.
+
+Map regression (82s vs 53s pre-"What We Did") is **unresolved** — this is Task 3.
+
+---
+
+## Targets
+
+| Phase | Current | Target |
+|---|---|---|
+| Ingest | ~227s | ~90s |
+| Map computation | ~82s | ~20s |
+| **Total** | **~309s** | **~110s** |
+
+---
+
+## Task List
+
+Tasks are independent unless noted. Each is scoped to be completable by a single Claude instance with no context from the others.
+
+---
+
+### Task 1 — Bump `COMMIT_CONCURRENCY` 4 → 6 ⚠️ BLOCKED
+
+**Priority:** High — blocked by write-write conflict bug (see Task 1b below).
+
+Bumping to 6 caused systemic write-write conflicts in ArangoDB. Root cause was identified and fixed (see Task 1b), but after that fix concurrency was left at 4. Re-attempting the bump to 6 should now be safe — the `updateRevision` conflict is eliminated, and `retryOnConflict` handles any residual tombstone conflicts.
+
+**Acceptance:** Run `IX_DEBUG=1 ix map <kubernetes-repo>` with no `[commit error]` lines and confirm `commitMs` drops vs the 607952ms baseline.
+
+---
+
+### Task 1b — Fix `updateRevision` write-write conflict ✅ DONE
+
+**Root cause:** `updateRevision` used an AQL `UPSERT { _key: "current" }` on the `revisions` collection. Every HTTP commit request ends with this call. With multiple concurrent workers, all requests raced to update the same document → ArangoDB write-write conflict error 1200.
+
+**Fix (`memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala`):**
+Replaced the AQL UPSERT with two Document API inserts via the existing `bulkInsert(..., overwriteMode="replace")`. The Document API replace is atomic at the document level — concurrent replaces on the same `_key` don't conflict; last writer wins (acceptable since all concurrent requests write the same rev value).
+
+Also added `retryOnConflict` in the client (`ix-cli/src/cli/commands/ingest.ts`) with 10ms base jitter backoff as a safety net for any residual tombstone-phase conflicts.
+
+---
+
+### Task 2 — Skip chunk nodes/edges during map-triggered ingest ✅ DONE
+
+**Priority:** High — estimated ~30% fewer DB inserts.
+
+**Files:**
+- `ix-cli/src/cli/commands/ingest.ts` — add `skipChunks` flag, filter nodes/edges before patch submission
+- `ix-cli/src/client/api.ts` — pass flag through to request if needed
+- `ix-cli/src/client/types.ts` — add type if needed
+
+**Background:**
+Current insert volume per 200-file HTTP batch (estimated):
+- ~4,154 nodes, of which ~36% are `chunk` kind nodes
+- ~11,186 edges, of which ~24% are `CONTAINS_CHUNK` + `NEXT` edges
+
+Chunk nodes and their edges are LLM context artifacts (for `ix read`). The Louvain clustering in `MapService.computeAndPersistMap` runs on `IMPORTS` and `CALLS` edges between file nodes — it never touches chunks. Skipping them during `ix map` doesn't affect map quality.
+
+**What to do:**
+
+1. In `ingest.ts`, find where `PreparedPatch` objects are built (the `buildPatch` / `resolveEdges` path). Add a `mapMode: boolean` parameter that, when true, strips:
+ - Any node with `kind === "chunk"`
+ - Any edge with `kind === "CONTAINS_CHUNK"` or `kind === "NEXT"`
+
+2. The `ix map` code path in `ingest.ts` (around line ~544 and ~598, the `commitPreparedPatches` calls) should pass `mapMode: true`.
+
+3. The `ix ingest` code path (full ingest, used for `ix read`) must continue writing chunks — pass `mapMode: false` or omit the flag.
+
+4. Do not change the server side. The filtering should be purely client-side before the HTTP payload is built.
+
+**Acceptance:**
+- `IX_DEBUG=1 ix map <kubernetes-repo>` shows fewer nodes/edges inserted per batch (verify via backend `insertNodes`/`insertEdges` log lines)
+- `ix read <any-file>` after a full `ix ingest` still returns chunk content (chunks must still be written on full ingest)
+- No change to `ix map` output quality (regions/subsystems unchanged or better)
+
+---
+
+### Task 3 — Profile and fix map computation regression (82s → target 20s)
+
+**Priority:** High — map computation is the bottleneck for incremental `ix map` runs. Every file save triggers a re-map; keeping this fast matters more than ingest speed.
+
+**Note on terminology:** `ix ingest` is deprecated. `ix map` is the primary command. It has two phases: ingest phase (parse + commit files) and map computation phase (Louvain clustering + persist). Task 3 is about the map computation phase only.
+
+**Files:**
+- `memory-layer/src/main/scala/ix/memory/map/MapService.scala` — primary target
+- `memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala` — if `build()` is the bottleneck
+
+**Current timing:** map computation = ~82s on Kubernetes (15035 files, 828 regions). Pre-regression baseline was ~53s. Target: ≤20s (30s acceptable).
+
+**Background:**
+`computeAndPersistMap` has four steps:
+1. `builder.build()` — loads file graph from DB (IMPORTS/CALLS edges between file nodes)
+2. `detectCrosscut(rawGraph)` — pure CPU
+3. `LouvainClustering.cluster(...)` — pure CPU, `maxLevels=3`
+4. `persistMap(fresh, cached)` — writes region nodes to DB, compares old vs new regions
+
+The regression likely lives in `builder.build()` (more nodes/edges to load since ingest changes) or `persistMap` (`sameRegions` comparison may be slow at 828 regions).
+
+**What to do:**
+
+1. Read `MapService.scala` first to find `computeAndPersistMap` and understand the current structure before adding instrumentation.
+
+2. Add coarse `System.currentTimeMillis()` timing around each of the four steps and log at INFO level:
+ ```scala
+ val t0 = System.currentTimeMillis()
+ rawGraph <- builder.build()
+ val t1 = System.currentTimeMillis()
+ // ... etc
+ logger.info(s"map build=${t1-t0}ms crosscut=${t2-t1}ms cluster=${t3-t2}ms persist=${t4-t3}ms")
+ ```
+
+3. After adding instrumentation, rebuild and restart the backend:
+ ```bash
+ # from repo root (/home/ianhock/ix/IX-Memory)
+ ./rebuild.sh
+ ```
+
+4. Run to collect the timing breakdown:
+ ```bash
+ IX_DEBUG=1 ix map /home/ianhock/kubernetes 2>&1 | tee /tmp/ix-map-debug.log
+ grep "map build=" /tmp/ix-map-debug.log
+ ```
+
+5. Fix the dominant step:
+ - If `builder.build()` is slow: check `MapGraphBuilder` — it queries all IMPORTS/CALLS edges between file nodes. Look for missing indexes or N+1 queries. Check `ArangoSchema.scala` for relevant indexes.
+ - If `persistMap` is slow: check `sameRegions` — it sorts and compares full `Region` objects. At 828 regions this is O(n log n) with potentially high constant. Also check `submitMapPatch` / `loadOldRegionIds`.
+ - If `LouvainClustering.cluster` is slow: the graph got larger. Check how many file nodes are passed to the clusterer vs pre-regression.
+
+**Acceptance:**
+- Map computation ≤ 30s on Kubernetes (20s target, 30s acceptable for this task)
+- Timing log line visible in backend output during `IX_DEBUG=1 ix map`
+- No regression in `ix map` output (828 regions expected, same or similar structure)
+- Rebuild via `./rebuild.sh` from repo root before testing
+
+---
+
+### Task 4 — Vacuum old tombstoned node versions
+
+**Priority:** Medium — reduces DB size and disk I/O over time; not urgent for this sprint.
+
+**Files:**
+- `memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala` — add vacuum method
+- `memory-layer/src/main/scala/ix/memory/api/` — add a `/vacuum` or `/admin/vacuum` HTTP endpoint
+- `ix-cli/src/client/api.ts` and `ix-cli/src/cli/commands/` — add `ix vacuum` CLI command
+
+**Background:**
+Every ingest revision leaves behind tombstoned docs in the `nodes` collection (`deleted_rev IS NOT NULL`). After N ingests of Kubernetes (312k live nodes), the collection has ~N × 312k docs. These bloat ArangoDB's working set, spill indexes to disk, and account for the 12.4GB disk write profile.
+
+**What to do:**
+
+1. In `ArangoClient.scala`, add a method:
+ ```scala
+ def vacuumTombstoned(keepRevisions: Int): IO[Long]
+ ```
+ That runs:
+ ```aql
+ FOR n IN nodes
+ FILTER n.deleted_rev != null
+ AND n.deleted_rev < @cutoffRev
+ LIMIT 50000
+ REMOVE n IN nodes
+ RETURN 1
+ ```
+ Where `cutoffRev = latestRev - keepRevisions`. Use `LIMIT 50000` per pass and loop until 0 rows removed to avoid ArangoDB memory pressure on large deletes. Apply the same pattern to the `edges` and `claims` collections.
+
+2. Expose it via a backend endpoint (e.g. `POST /admin/vacuum?keepRevisions=5`).
+
+3. Add `ix vacuum [--keep-revisions N]` CLI command (default `--keep-revisions 5`).
+
+**Acceptance:**
+- After `ix vacuum` on Kubernetes graph, `nodes` collection count drops by at least 50% (or confirms there are minimal tombstoned docs if this is a fresh graph)
+- Subsequent `ix map` run shows reduced disk I/O in `IX_DEBUG=1` output
+- `ix vacuum` completes without OOM or ArangoDB timeout (batched deletes required)
+
+---
+
+### Task 5 — Dynamic concurrency auto-tuning (stretch goal)
+
+**Priority:** Low — only needed if fixed concurrency of 6 still leaves headroom.
+
+**File:** `ix-cli/src/cli/commands/ingest.ts`
+
+**What to do:**
+Replace the fixed `COMMIT_CONCURRENCY` constant with a simple AIMD (additive increase / multiplicative decrease) controller:
+- Track rolling P90 latency of the last 10 batch commits
+- If P90 < 8s, increment concurrency (up to max 12)
+- If P90 > 15s or a batch times out, decrement concurrency (min 2)
+- Log current concurrency level in the `[ingest-save]` debug line
+
+**Acceptance:**
+- On Kubernetes, concurrency auto-settles to a stable value within the first 10 batches
+- Total ingest time is ≤ fixed-concurrency-6 time (must not be slower)
+- No batch timeout errors introduced
+
+---
+
+## Profiling Instructions
+
+To collect the data needed for Task 3:
+
+```bash
+IX_DEBUG=1 ix map <kubernetes-repo-path> 2>&1 | tee /tmp/ix-map-debug.log
+```
+
+Look for:
+- Client: `[ingest-save]` lines — per-batch phase breakdown
+- Client: `[slow saving]` lines — which batch numbers are slow
+- Backend: `ix.bulk-write` log lines — `insertNodes`, `insertEdges`, `insertClaims`, `insertPatches` per-stage breakdown
+- Backend: `pre-tombstone` line — tombstone+retire time
+- Backend (after Task 3 instrumentation): `map build=Xms crosscut=Yms cluster=Zms persist=Wms`
diff --git a/docs/plans/2026-03-25-large-repo-map-followup.md b/docs/plans/2026-03-25-large-repo-map-followup.md
new file mode 100644
index 00000000..dc293c87
--- /dev/null
+++ b/docs/plans/2026-03-25-large-repo-map-followup.md
@@ -0,0 +1,154 @@
+# Large Repo `ix map` Follow-up
+
+Date: 2026-03-25
+
+## Current user workflow
+
+Use `ix map` only.
+
+Do not assume `ix ingest` is part of the normal workflow anymore. Any large-repo performance work needs to be evaluated through `ix map`, since `ix map` triggers ingest first and that is what the user experiences.
+
+## Main problem
+
+Large repos improved for map computation, but ingest/save regressed badly.
+
+Observed user report for Kubernetes:
+
+- `ix map` total ingest time on current branch: about `466s`
+- `ix map` total map time on current branch: about `53s`
+- Earlier target / expected behavior from dev: ingest closer to `90s`
+- Previous map-only improvement target: Kubernetes map around `20s`
+
+Observed debug sample from user:
+
+- `files=16010`
+- `resolveEdgesMs=29581`
+- `buildPatchMs=10156`
+- `commitMs=386443`
+- `totalSaveMs=426834`
+
+Conclusion: the dominant regression is still in the save/commit path, not parse, resolve, or map clustering.
+
+## Confirmed issues
+
+### 1. Save path is the bottleneck on large repos
+
+Client debug logs showed repeated slow commit chunks during saving, for example:
+
+- `commit 3401-3600 ... (15s)`
+- `commit 5601-5800 ... (17s)`
+- `commit 7801-8000 ... (25s)`
+
+This points at backend bulk commit behavior, especially chunk sizing and/or Arango bulk insert characteristics on large batches.
+
+### 2. `ix map` output had duplicate top-level regions
+
+The map output showed repeated roots like:
+
+- `MODULE Edges`
+- `SUBSYSTEM Edges`
+- `SYSTEM Edges`
+
+and similarly for `Builder`.
+
+Cause found:
+
+- the same file set could survive across multiple hierarchy levels
+- those duplicate regions shared the same deterministic region ID
+- they were all rendered as roots/orphans
+
+Status:
+
+- fixed in code by collapsing duplicate hierarchy levels to the coarsest retained region
+- regression test added
+- verified on a fresh backend
+
+Remaining note:
+
+- `Edges` and `Builder` may still appear once each as top-level systems because they are still weakly connected / isolated in the graph
+- that is a separate modeling issue from the duplicate-root bug
+
+## Changes already made
+
+### Map hierarchy fix
+
+Updated:
+
+- `memory-layer/src/main/scala/ix/memory/map/MapService.scala`
+- `memory-layer/src/test/scala/ix/memory/map/MapServiceScopeSpec.scala`
+
+What changed:
+
+- collapse duplicate same-member-set regions across module/subsystem/system levels
+- keep only the coarsest retained region
+- preserve children pointing at that surviving region
+
+### Ingest/save tuning already tried
+
+Updated:
+
+- `ix-cli/src/cli/commands/ingest.ts`
+- `memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala`
+
+What changed:
+
+- reduced client bulk HTTP batch size from `1000` files to `200`
+- reduced backend revision chunk size from `500` files to `100`
+- reduced backend payload cap from `16MB` to `4MB`
+- capped patch document insert batches instead of inserting arbitrarily large patch arrays
+
+## Measured result after tuning
+
+On a fresh isolated backend, full ingest of this repo improved from an earlier sample with:
+
+- `commitMs ~10899`
+
+to:
+
+- `commitMs 4021`
+- total ingest `5.48s` for `221` changed files
+
+This confirms the chunk-size rollback helped on the repo-sized benchmark.
+
+## What is still unresolved
+
+The large-repo `ix map` path for Kubernetes is still much too slow.
+
+Even after chunk-size reductions, the real user-facing test case remains:
+
+- `ix map <kubernetes repo>`
+
+and the unresolved part is:
+
+- save/commit time at large scale
+
+## What to collect next
+
+Since the user uses `ix map`, collect logs from `ix map`, not a standalone `ix ingest` run.
+
+Needed evidence next time:
+
+1. Client-side `ix map` debug output including:
+ - slow saving lines
+ - final `[ingest-save]` line
+ - final `ingest Xms · map Yms`
+2. Backend `memory-layer` logs at the same time, especially `ix.bulk-write` timings
+3. Identify which backend stage dominates:
+ - `insertNodes`
+ - `insertEdges`
+ - `insertClaims`
+ - `insertPatches`
+ - tombstoning / retiring old claims
+ - revision update
+
+## Likely next investigation targets
+
+1. Arango bulk insert performance for edges and patches on very large repos
+2. Whether generated files (`zz_generated`, `.pb.go`, deepcopy/defaults, etc.) should be skipped or downgraded during `ix map`
+3. Whether `ix map` should use a cheaper ingest mode than full graph ingest for very large repos
+4. Whether chunk sizing should adapt dynamically based on actual commit latency rather than fixed file counts
+5. Whether large repo map should avoid persisting some low-value entities/claims during the map-triggered ingest path
+
+## Operational note
+
+If a fix appears to work in code but not in normal `ix map` output, confirm the running backend has actually been restarted. During this session, the old `8090` backend continued serving stale behavior until a fresh backend process was started separately.
diff --git a/ix-cli/package-lock.json b/ix-cli/package-lock.json
index e92ca000..be5f49f1 100644
--- a/ix-cli/package-lock.json
+++ b/ix-cli/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@ix/cli",
- "version": "0.4.0",
+ "version": "0.4.4",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@ix/cli",
- "version": "0.3.0",
+ "version": "0.4.4",
"dependencies": {
"chalk": "^5.3.0",
"commander": "^12.0.0",
diff --git a/ix-cli/src/cli/__tests__/help-coverage.test.ts b/ix-cli/src/cli/__tests__/help-coverage.test.ts
index ac03a44a..9e3f9984 100644
--- a/ix-cli/src/cli/__tests__/help-coverage.test.ts
+++ b/ix-cli/src/cli/__tests__/help-coverage.test.ts
@@ -13,9 +13,9 @@ function stripAnsi(s: string): string {
const ossHelp = buildHelpText();
const REQUIRED_COMMANDS = [
- "init", "search", "locate", "explain", "impact", "overview", "watch",
- "read", "inventory", "rank", "history", "diff",
- "ingest", "status", "stats", "doctor", "docker",
+ "search", "locate", "explain", "impact", "overview", "watch",
+ "read", "inventory", "rank", "history", "diff", "smells", "subsystem",
+ "map", "trace", "status", "stats", "doctor", "docker",
];
describe("help coverage", () => {
@@ -34,7 +34,7 @@ describe("help coverage", () => {
});
it("OSS help uses new branding", () => {
- expect(stripAnsi(ossHelp)).toContain("ix — Code Memory CLI");
+ expect(stripAnsi(ossHelp)).toContain("ix — System Intelligence CLI");
expect(stripAnsi(ossHelp)).not.toContain("Persistent Memory for LLM Systems");
});
diff --git a/ix-cli/src/cli/__tests__/search-ranking.test.ts b/ix-cli/src/cli/__tests__/search-ranking.test.ts
index 19eac811..5cfc9444 100644
--- a/ix-cli/src/cli/__tests__/search-ranking.test.ts
+++ b/ix-cli/src/cli/__tests__/search-ranking.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect, vi } from "vitest";
-import { resolveEntityFull } from "../resolve.js";
+import { resolveEntityFull, scoreCandidate } from "../resolve.js";
/**
* Verify that search ranking correctly incorporates backend _search_weight.
@@ -8,8 +8,8 @@ import { resolveEntityFull } from "../resolve.js";
* without needing a running backend.
*/
-/** Minimal scoreCandidate replica for testing (exact name = 0, prefix = 15, fuzzy = 30). */
-function scoreCandidate(node: any, term: string, opts?: { kind?: string; path?: string }): number {
+/** Minimal scoreCandidate replica for rankScore tests (exact name = 0, prefix = 15, fuzzy = 30). */
+function scoreCandidateSimple(node: any, term: string, opts?: { kind?: string; path?: string }): number {
const name: string = (node.name || "").toLowerCase();
const kind: string = (node.kind || "").toLowerCase();
const symbolLower = term.toLowerCase();
@@ -32,7 +32,7 @@ function rankScore(
): { tier: number; score: number; matchSource: string } {
// Weight comes through attrs (embedded by AQL into attrs JSON)
const backendWeight: number = node.attrs?._search_weight ?? node._search_weight ?? 0;
- const resolverScore = scoreCandidate(node, term, { kind: requestedKind, path: pathFilter });
+ const resolverScore = scoreCandidateSimple(node, term, { kind: requestedKind, path: pathFilter });
if (backendWeight >= 100) {
if (resolverScore <= -3) return { tier: 0, score: -backendWeight + resolverScore, matchSource: "name_exact" };
@@ -116,6 +116,83 @@ describe("search ranking with backend weight", () => {
});
});
+describe("scoreCandidate: structural kinds outrank chunks", () => {
+ // BUG-1 / BUG-2 regression: for PascalCase names, chunks must never outscore
+ // function/method entities. Previously a +3 penalty on method/function for
+ // looksTypeLikeSymbol names let chunks (score=0) beat them (score=2).
+
+ it("method outscores chunk for PascalCase name (Apply)", () => {
+ const method = { name: "Apply", kind: "method", provenance: { sourceUri: "etcd/apply/backend.go" } };
+ const chunk = { name: "Apply", kind: "chunk", provenance: { sourceUri: "etcd/apply/backend.go" } };
+ expect(scoreCandidate(method, "Apply")).toBeLessThan(scoreCandidate(chunk, "Apply"));
+ });
+
+ it("function outscores chunk for PascalCase name (StartEtcd)", () => {
+ const fn = { name: "StartEtcd", kind: "function", provenance: { sourceUri: "etcd/embed/etcd.go" } };
+ const chunk = { name: "StartEtcd", kind: "chunk", provenance: { sourceUri: "etcd/embed/etcd.go" } };
+ expect(scoreCandidate(fn, "StartEtcd")).toBeLessThan(scoreCandidate(chunk, "StartEtcd"));
+ });
+
+ it("class still outscores function for PascalCase type-like name", () => {
+ const cls = { name: "EtcdServer", kind: "class", provenance: { sourceUri: "etcd/server.go" } };
+ const fn = { name: "EtcdServer", kind: "function", provenance: { sourceUri: "etcd/server.go" } };
+ expect(scoreCandidate(cls, "EtcdServer")).toBeLessThan(scoreCandidate(fn, "EtcdServer"));
+ });
+});
+
+describe("resolveEntityFull: method preferred over chunk for PascalCase names", () => {
+ // BUG-1 regression: resolving "Apply" should return the method entity, not a chunk.
+ it("resolves to method entity, not chunk, when both have the same name", async () => {
+ const method = {
+ id: "method-apply-id",
+ name: "Apply",
+ kind: "method",
+ provenance: { sourceUri: "etcd/apply/backend.go" },
+ };
+ const chunk = {
+ id: "chunk-apply-id",
+ name: "Apply",
+ kind: "chunk",
+ provenance: { sourceUri: "etcd/apply/backend.go" },
+ };
+ const mockClient = {
+ search: vi.fn().mockResolvedValue([chunk, method]),
+ } as any;
+
+ const result = await resolveEntityFull(mockClient, "Apply", ["method"], { path: "etcd" });
+ expect(result.resolved).toBe(true);
+ if (result.resolved) {
+ expect(result.entity.id).toBe("method-apply-id");
+ expect(result.entity.kind).toBe("method");
+ }
+ });
+
+ it("resolves StartEtcd to function entity, not chunk (BUG-2 regression)", async () => {
+ const fn = {
+ id: "fn-startetcd-id",
+ name: "StartEtcd",
+ kind: "function",
+ provenance: { sourceUri: "etcd/embed/etcd.go" },
+ };
+ const chunk = {
+ id: "chunk-startetcd-id",
+ name: "StartEtcd",
+ kind: "chunk",
+ provenance: { sourceUri: "etcd/embed/etcd.go" },
+ };
+ const mockClient = {
+ search: vi.fn().mockResolvedValue([chunk, fn]),
+ } as any;
+
+ const result = await resolveEntityFull(mockClient, "StartEtcd", ["function"], { path: "etcd" });
+ expect(result.resolved).toBe(true);
+ if (result.resolved) {
+ expect(result.entity.id).toBe("fn-startetcd-id");
+ expect(result.entity.kind).toBe("function");
+ }
+ });
+});
+
describe("resolveEntityFull --path hard exclusion", () => {
it("excludes out-of-scope candidates even on exact name match", async () => {
// Simulates Bug 1: etcd has 'Notifier' but prometheus does not.
diff --git a/ix-cli/src/cli/commands/explain.ts b/ix-cli/src/cli/commands/explain.ts
index f2663338..af5fbc0d 100644
--- a/ix-cli/src/cli/commands/explain.ts
+++ b/ix-cli/src/cli/commands/explain.ts
@@ -52,7 +52,7 @@ export function registerExplainCommand(program: Command): void {
console.log(JSON.stringify(output, null, 2));
} else {
if (facts.stale) {
- renderWarning("Source has changed since last ingest. Run ix ingest to update.");
+ renderWarning("Source has changed since last ingest. Run ix map to update.");
}
renderSection("Explanation");
@@ -213,7 +213,7 @@ async function rawExplain(
}
console.log(JSON.stringify(output, null, 2));
} else {
- if (stale) renderWarning("Source has changed since last ingest. Run ix ingest to update.");
+ if (stale) renderWarning("Source has changed since last ingest. Run ix map to update.");
formatExplain(result, "text");
}
}
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 8769f524..3f809146 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -2,6 +2,7 @@ import * as nodePath from 'node:path';
import * as fs from 'node:fs';
import * as crypto from 'node:crypto';
import * as os from 'node:os';
+import { spawnSync } from 'node:child_process';
import type { Command } from 'commander';
import chalk from 'chalk';
import { IxClient } from '../../client/api.js';
@@ -59,13 +60,57 @@ function parseLangs(raw: string): Set<string> {
}
const IGNORE_DIRS = new Set([
- 'node_modules', '.git', 'dist', 'build', 'target', '.next',
- '.cache', '__pycache__', '.ix', '.claude', '.gitnexus',
+ 'node_modules', '.git', '.hg', '.svn', 'dist', 'build', 'target', 'out', '.next',
+ '__pycache__', '.tox', '.venv', 'venv', '.mypy_cache', '.pytest_cache',
+ '.gradle', '.idea', '.vscode', '.settings', '.vs',
+ '.cache', '.parcel-cache', 'coverage', '.nyc_output',
+ 'vendor', 'Pods', '.dart_tool', '.pub-cache',
+ 'bin', 'obj', 'pkg',
+ '.ix', '.claude', '.gitnexus',
'test', 'tests', '__tests__', 'spec', 'specs', 'e2e',
'examples', 'fixtures', '__mocks__', '__fixtures__',
]);
+// Generated file patterns — skipped by filename to avoid ingesting low-value autogenerated code.
+// These files inflate parse time and graph size without contributing to architectural signal.
+const GENERATED_FILE_SUFFIXES = [
+ '.pb.go', // protobuf generated Go
+ '_deepcopy.go', // controller-gen deepcopy (Kubernetes ecosystem)
+ '_mock.go', // gomock generated mocks
+ '.pb.ts', // protobuf generated TypeScript
+ '.pb.js', // protobuf generated JavaScript
+ '_pb.ts', // alternative protobuf TypeScript naming
+ '_pb.js', // alternative protobuf JavaScript naming
+];
+
+const GENERATED_FILE_PREFIXES = [
+ 'zz_generated', // controller-gen convention (Kubernetes ecosystem)
+ 'mock_', // gomock convention
+];
+
+function isGeneratedFile(basename: string): boolean {
+ for (const suffix of GENERATED_FILE_SUFFIXES) {
+ if (basename.endsWith(suffix)) return true;
+ }
+ for (const prefix of GENERATED_FILE_PREFIXES) {
+ if (basename.startsWith(prefix)) return true;
+ }
+ return false;
+}
+
const MAX_FILE_BYTES = 1024 * 1024; // 1 MB
+const MINIFIED_BYTES_THRESHOLD = 200 * 1024;
+const MINIFIED_MAX_LINE_THRESHOLD = 20_000;
+const MINIFIED_AVG_LINE_THRESHOLD = 2_000;
+const SLOW_WORK_LOG_MS = 5_000;
+const SLOW_WORK_REPEAT_MS = 10_000;
+
+function parsePositiveIntEnv(name: string, fallback: number): number {
+ const raw = process.env[name];
+ if (!raw) return fallback;
+ const parsed = Number.parseInt(raw, 10);
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
function* walkFiles(
dir: string,
@@ -76,20 +121,115 @@ function* walkFiles(
catch { return; }
for (const entry of entries) {
+ if (entry.isDirectory() && entry.name.startsWith('.') && entry.name !== '.') continue;
if (IGNORE_DIRS.has(entry.name)) continue;
const full = nodePath.join(dir, entry.name);
if (entry.isDirectory()) {
if (recursive) yield* walkFiles(full, true);
} else if (entry.isFile()) {
- if (SUPPORTED_EXTENSIONS.has(nodePath.extname(entry.name))) yield full;
+ if (SUPPORTED_EXTENSIONS.has(nodePath.extname(entry.name)) && !isGeneratedFile(entry.name)) yield full;
}
}
}
+function tryGitLsFiles(dir: string, recursive: boolean): string[] | null {
+ try {
+ const result = spawnSync(
+ 'git',
+ ['ls-files', '--cached', '--others', '--exclude-standard'],
+ {
+ cwd: dir,
+ encoding: 'utf-8',
+ maxBuffer: 32 * 1024 * 1024,
+ }
+ );
+ if (result.status !== 0) return null;
+
+ const files: string[] = [];
+ for (const line of result.stdout.split(/\r?\n/)) {
+ if (!line) continue;
+ const fullPath = nodePath.resolve(dir, line);
+ if (!recursive && nodePath.dirname(fullPath) !== dir) continue;
+ if (!SUPPORTED_EXTENSIONS.has(nodePath.extname(fullPath))) continue;
+ if (isGeneratedFile(nodePath.basename(fullPath))) continue;
+ try {
+ if (fs.statSync(fullPath).isFile()) files.push(fullPath);
+ } catch {
+ // Ignore racy deletes and permission issues during discovery.
+ }
+ }
+ return files;
+ } catch {
+ return null;
+ }
+}
+
function sha256(content: Buffer): string {
return crypto.createHash('sha256').update(content).digest('hex');
}
+function isLikelyMinifiedSource(source: string): boolean {
+ if (source.length < MINIFIED_BYTES_THRESHOLD) return false;
+
+ let lineCount = 1;
+ let currentLineLength = 0;
+ let maxLineLength = 0;
+ for (let i = 0; i < source.length; i++) {
+ if (source.charCodeAt(i) === 10) {
+ lineCount++;
+ if (currentLineLength > maxLineLength) maxLineLength = currentLineLength;
+ currentLineLength = 0;
+ } else {
+ currentLineLength++;
+ }
+ }
+ if (currentLineLength > maxLineLength) maxLineLength = currentLineLength;
+
+ const averageLineLength = source.length / Math.max(lineCount, 1);
+ return maxLineLength >= MINIFIED_MAX_LINE_THRESHOLD || (lineCount <= 50 && averageLineLength >= MINIFIED_AVG_LINE_THRESHOLD);
+}
+
+function stripChunkOps(patch: GraphPatchPayload): GraphPatchPayload {
+ return {
+ ...patch,
+ ops: patch.ops.filter(op => {
+ if (op.type === 'UpsertNode' && op['kind'] === 'chunk') return false;
+ if (op.type === 'UpsertEdge' && (op['predicate'] === 'CONTAINS_CHUNK' || op['predicate'] === 'NEXT')) return false;
+ return true;
+ }),
+ };
+}
+
+const COMMIT_CONFLICT_RETRY_PATTERNS = [
+ 'write-write conflict',
+ 'timeout waiting to lock key',
+ 'error: 1200',
+ // Transport-level failures (k8s ingress reset, socket drop under load).
+ // Safe to retry because the server never received / committed the payload.
+ 'fetch failed',
+ 'econnreset',
+ 'econnrefused',
+];
+
+export function isRetryableCommitConflict(err: unknown): boolean {
+ const message = String(err).toLowerCase();
+ return COMMIT_CONFLICT_RETRY_PATTERNS.some(pattern => message.includes(pattern));
+}
+
+async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Promise<T> {
+ for (let attempt = 0; ; attempt++) {
+ try {
+ return await fn();
+ } catch (err) {
+ if (attempt >= maxRetries || !isRetryableCommitConflict(err)) throw err;
+ // ArangoDB lock conflicts usually clear as soon as the competing transaction
+ // commits. Transport errors may need a bit more breathing room under k8s load.
+ const delay = 10 * (1 << attempt) + Math.random() * 10;
+ await new Promise<void>(r => setTimeout(r, delay));
+ }
+ }
+}
+
// ---------------------------------------------------------------------------
// Mtime cache — skip readFileSync+sha256 for unchanged files
// ---------------------------------------------------------------------------
@@ -188,9 +328,10 @@ export function registerIngestCommand(program: Command): void {
export async function ingestFiles(
path: string,
- opts: { recursive?: boolean; force?: boolean; format: string; root?: string; debug?: boolean; printSummary?: boolean; lang?: string }
+ opts: { recursive?: boolean; force?: boolean; format: string; root?: string; debug?: boolean; printSummary?: boolean; lang?: string; mapMode?: boolean }
): Promise<void> {
const debug = opts.debug || process.env.IX_DEBUG === '1';
+ const mapMode = opts.mapMode === true;
const trueStart = performance.now();
const [{ parseFile, resolveEdges, isGrammarSupported }, { buildPatchWithResolution }, { languageFromPath }] = await loadIngestionModules();
@@ -210,14 +351,14 @@ export async function ingestFiles(
let progressStart = performance.now();
const interval = opts.format === 'text' ? setInterval(() => {
- const elapsed = performance.now() - progressStart;
- const tau = progressPhase === 'Saving' ? 3000 : 2000;
- let display = progressCurrent;
- if (progressTotal > 0 && progressCurrent < progressTotal) {
- const simulated = Math.floor((1 - Math.exp(-elapsed / tau)) * 0.88 * progressTotal);
- display = Math.max(display, simulated);
+ if (debug && currentWorkLabel) {
+ const workElapsed = performance.now() - currentWorkStart;
+ if (workElapsed >= SLOW_WORK_LOG_MS && (lastSlowWorkNotice === 0 || workElapsed - lastSlowWorkNotice >= SLOW_WORK_REPEAT_MS)) {
+ lastSlowWorkNotice = workElapsed;
+ process.stderr.write(`\n [slow ${progressPhase.toLowerCase()}] ${currentWorkLabel} (${Math.round(workElapsed / 1000)}s)\n`);
+ }
}
- process.stderr.write('\r' + renderProgressLine(progressPhase, display, progressTotal));
+ process.stderr.write('\r' + renderProgressLine(progressPhase, progressCurrent, progressTotal));
}, 80) : null;
let filesDiscovered = 0;
@@ -226,16 +367,57 @@ export async function ingestFiles(
let filesSkipped = 0;
let parseErrors = 0;
let tooLarge = 0;
+ let minifiedLikely = 0;
let latestRev = 0;
let entitiesParsed = 0;
+ let currentWorkLabel = '';
+ let currentWorkStart = performance.now();
+ let lastSlowWorkNotice = 0;
+
+ const setCurrentWork = (label: string): void => {
+ currentWorkLabel = label;
+ currentWorkStart = performance.now();
+ lastSlowWorkNotice = 0;
+ };
// Phase timing marks
- const timings = { moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0, resolveMs: 0, commitMs: 0 };
+ const timings = {
+ moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0,
+ parseOnlyMs: 0, // sum of just parseFile() call durations
+ resolveMs: 0, buildPatchMs: 0, commitMs: 0,
+ stripChunkOpsMs: 0, // sum of stripChunkOps() durations (mapMode only)
+ bulkCommitMs: 0, // time in successful commitPatchBulk() calls
+ fallbackCommitMs: 0, // time in per-file fallback commitPatch() calls
+ };
const resolveStats = {
importLookups: 0, transitiveLookups: 0, globalFallbacks: 0,
globalCandidateTotal: 0, resolvedImport: 0, resolvedTransitive: 0,
resolvedGlobal: 0, resolvedQualifier: 0, skippedSameFile: 0, skippedAmbiguous: 0,
};
+ // Per-language parse timing: language → { ms, files }
+ const langParseMs = new Map<string, { ms: number; files: number }>();
+ // Total graph op counts across all committed batches
+ let totalNodesEmitted = 0;
+ let totalEdgesEmitted = 0;
+ let totalClaimsEmitted = 0;
+
+ const logSaveTiming = (
+ files: number,
+ resolveEdgesMs: number,
+ buildPatchMs: number,
+ stripMs: number,
+ commitMs: number,
+ totalSaveMs: number,
+ nodes: number,
+ edges: number,
+ claims: number,
+ ): void => {
+ process.stderr.write(
+ `\n[ingest-save] files=${files} resolveEdgesMs=${resolveEdgesMs} buildPatchMs=${buildPatchMs}` +
+ (mapMode ? ` stripChunkOpsMs=${stripMs}` : '') +
+ ` commitMs=${commitMs} totalSaveMs=${totalSaveMs} nodes=${nodes} edges=${edges} claims=${claims}\n`
+ );
+ };
try {
// Phase: discover files
@@ -250,7 +432,7 @@ export async function ingestFiles(
const stat = fs.statSync(resolvedPath);
const filePaths: string[] = stat.isFile()
? (SUPPORTED_EXTENSIONS.has(nodePath.extname(resolvedPath)) ? [resolvedPath] : [])
- : Array.from(walkFiles(resolvedPath, opts.recursive ?? true));
+ : (tryGitLsFiles(resolvedPath, opts.recursive ?? true) ?? Array.from(walkFiles(resolvedPath, opts.recursive ?? true)));
filesDiscovered = filePaths.length;
const discovered = performance.now();
@@ -284,17 +466,19 @@ export async function ingestFiles(
for (const filePath of filePaths) {
try {
const st = fs.statSync(filePath);
- if (st.size === 0) { filesSkipped++; continue; }
- if (st.size > MAX_FILE_BYTES) { tooLarge++; continue; }
+ if (st.size === 0) { filesSkipped++; progressCurrent++; continue; }
+ if (st.size > MAX_FILE_BYTES) { tooLarge++; progressCurrent++; continue; }
const mtime = st.mtimeMs;
currentMtimes.set(filePath, mtime);
if (!opts.force && mtimeCache.get(filePath) === mtime) {
- filesSkipped++; // mtime clean — assume unchanged
+ filesSkipped++;
+ progressCurrent++; // mtime clean — assume unchanged
} else {
mtimeChangedPaths.push(filePath);
}
} catch (err) {
parseErrors++;
+ progressCurrent++;
process.stderr.write(`\n [stat error] ${filePath}: ${err}\n`);
}
}
@@ -330,55 +514,252 @@ export async function ingestFiles(
let resolveEdgesFn: Function | null = null;
let buildPatchFn: Function | null = null;
- const PARSE_STREAM_CHUNK = 1000; // resolve + commit after this many parsed files
- const COMMIT_CHUNK_SIZE = 200; // files per HTTP batch to server
- const YIELD_EVERY = 100; // yield event loop every N files during parse
+ const PARSE_STREAM_CHUNK = 1000; // streaming batch size for large repos
+ const SMALL_REPO_THRESHOLD = 5_000; // parse-all-first below this; streaming above
+ const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 200); // files per HTTP request to the backend
+ const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', mapMode ? 2 : 4); // parallel HTTP save requests
+ const COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflicts
+ const YIELD_EVERY = 100; // yield event loop every N files during parse
+
+ if (debug) {
+ process.stderr.write(
+ `\n Save config: httpBatch=${COMMIT_HTTP_MAX_FILES} concurrency=${COMMIT_CONCURRENCY}\n`
+ );
+ }
const emptyEdges: any[] = [];
+ type PreparedPatch = {
+ patch: GraphPatchPayload;
+ fileNumber: number;
+ filePath: string;
+ };
+
+ const makePreparedPatch = (
+ patch: GraphPatchPayload,
+ fileNumber: number,
+ filePath: string,
+ ): PreparedPatch => ({ patch, fileNumber, filePath });
+
+ // Commit HTTP batches in parallel (COMMIT_CONCURRENCY workers). Each worker pulls
+ // the next unsent chunk from a shared queue. JS is single-threaded so the queue
+ // index increment is atomic across await boundaries — no locking needed.
+ const commitPreparedPatches = async (
+ preparedPatches: PreparedPatch[],
+ totalFiles: number,
+ opts?: { updateProgress?: boolean },
+ ): Promise<number> => {
+ if (preparedPatches.length === 0) return 0;
+
+ const chunks: PreparedPatch[][] = [];
+ for (let i = 0; i < preparedPatches.length; i += COMMIT_HTTP_MAX_FILES) {
+ chunks.push(preparedPatches.slice(i, i + COMMIT_HTTP_MAX_FILES));
+ }
+
+ const commitMsPerChunk = new Array<number>(chunks.length).fill(0);
+ let nextChunk = 0;
+ // Fallback mutex: serializes per-file commits across all workers so that
+ // concurrent fallback loops don't race on revisions.current (Error 1200).
+ let fallbackTail: Promise<void> = Promise.resolve();
+
+ const runChunk = async (ci: number): Promise<void> => {
+ const chunk = chunks[ci];
+ const startFile = chunk[0].fileNumber;
+ const endFile = chunk[chunk.length - 1].fileNumber;
+ const endingPath = nodePath.basename(chunk[chunk.length - 1].filePath);
+ const patches = chunk.map(item => item.patch);
+
+ try {
+ setCurrentWork(`commit ${startFile}-${endFile} of ${totalFiles} ending ${endingPath}`);
+ const commitStart = performance.now();
+ const result = await retryOnConflict(() => client.commitPatchBulk(patches), COMMIT_CONFLICT_RETRIES);
+ const chunkMs = Math.round(performance.now() - commitStart);
+ commitMsPerChunk[ci] = chunkMs;
+ timings.bulkCommitMs += chunkMs;
+ if (result.rev > latestRev) latestRev = result.rev;
+ patchesApplied += patches.length;
+ } catch (err) {
+ if (debug) {
+ process.stderr.write(
+ `\n [bulk failed, falling back to per-file] files ${startFile}-${endFile} (${patches.length} patches): ${err}\n`
+ );
+ }
+ // Chain this chunk's fallback work onto the shared tail so that only one
+ // fallback loop runs at a time. This prevents concurrent commitPatch
+ // transactions from racing on revisions.current (write-write conflict).
+ const prev = fallbackTail;
+ let resolveThis!: () => void;
+ fallbackTail = new Promise<void>(r => { resolveThis = r; });
+ await prev;
+ try {
+ for (const item of chunk) {
+ try {
+ const commitStart = performance.now();
+ const result = await retryOnConflict(() => client.commitPatch(item.patch), COMMIT_CONFLICT_RETRIES);
+ const chunkMs = Math.round(performance.now() - commitStart);
+ commitMsPerChunk[ci] += chunkMs;
+ timings.fallbackCommitMs += chunkMs;
+ if (result.rev > latestRev) latestRev = result.rev;
+ patchesApplied++;
+ } catch (commitErr) {
+ parseErrors++;
+ process.stderr.write(`\n [commit error] ${item.patch.source?.uri}: ${commitErr}\n`);
+ }
+ }
+ } finally {
+ resolveThis();
+ }
+ }
+
+ if (opts?.updateProgress) {
+ progressCurrent = Math.max(progressCurrent, endFile);
+ }
+ };
+
+ const worker = async (): Promise<void> => {
+ while (true) {
+ const ci = nextChunk++;
+ if (ci >= chunks.length) break;
+ await runChunk(ci);
+ }
+ };
+
+ await Promise.all(
+ Array.from({ length: Math.min(COMMIT_CONCURRENCY, chunks.length) }, () => worker())
+ );
+
+ return commitMsPerChunk.reduce((a, b) => a + b, 0);
+ };
+
/** Resolve edges within a batch, build patches, and commit in sub-chunks. */
const flushBatch = async (batch: ParsedFile[]): Promise<void> => {
if (batch.length === 0) return;
filesChanged += batch.length;
+ const saveStart = performance.now();
+ let resolveEdgesMs = 0;
+ let buildPatchMs = 0;
+ let localStripMs = 0;
+ let commitMs = 0;
+ let batchNodes = 0;
+ let batchEdges = 0;
+ let batchClaims = 0;
+ progressPhase = 'Saving';
+ try {
+ setCurrentWork(`resolve+commit batch ending ${nodePath.basename(batch[batch.length - 1].filePath)}`);
+
+ const resolveStart = performance.now();
+ const resolvedEdges = resolveEdgesFn!(batch.map(f => f.parsed), resolveStats);
+ resolveEdgesMs = Math.round(performance.now() - resolveStart);
+ const batchEdgesByFile = new Map<string, any[]>();
+ for (const edge of resolvedEdges) {
+ let arr = batchEdgesByFile.get(edge.srcFilePath);
+ if (!arr) { arr = []; batchEdgesByFile.set(edge.srcFilePath, arr); }
+ arr.push(edge);
+ }
- const resolvedEdges = resolveEdgesFn!(batch.map(f => f.parsed), resolveStats);
- const batchEdgesByFile = new Map<string, any[]>();
- for (const edge of resolvedEdges) {
- let arr = batchEdgesByFile.get(edge.srcFilePath);
- if (!arr) { arr = []; batchEdgesByFile.set(edge.srcFilePath, arr); }
- arr.push(edge);
+ setCurrentWork(`build commit payloads ending ${nodePath.basename(batch[batch.length - 1].filePath)}`);
+ const buildStart = performance.now();
+ const preparedPatches: PreparedPatch[] = [];
+ for (let j = 0; j < batch.length; j++) {
+ const { parsed: p, hash, previousHash } = batch[j];
+ try {
+ const patch = buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash, mapMode ? { emitClaims: false } : undefined);
+ preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));
+ } catch (err) {
+ parseErrors++;
+ process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);
+ }
+ }
+ buildPatchMs += Math.round(performance.now() - buildStart);
+ for (const pp of preparedPatches) {
+ for (const op of pp.patch.ops) {
+ if (op.type === 'UpsertNode') batchNodes++;
+ else if (op.type === 'UpsertEdge') batchEdges++;
+ else if (op.type === 'AssertClaim') batchClaims++;
+ }
+ }
+ if (preparedPatches.length === 0) return;
+ commitMs += await commitPreparedPatches(preparedPatches, batch.length);
+ } finally {
+ const totalSaveMs = Math.round(performance.now() - saveStart);
+ timings.resolveMs += resolveEdgesMs;
+ timings.buildPatchMs += buildPatchMs;
+ timings.stripChunkOpsMs += localStripMs;
+ timings.commitMs += commitMs;
+ totalNodesEmitted += batchNodes;
+ totalEdgesEmitted += batchEdges;
+ totalClaimsEmitted += batchClaims;
+ logSaveTiming(batch.length, resolveEdgesMs, buildPatchMs, localStripMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);
+ progressPhase = 'Parsing';
+ progressStart = performance.now();
}
+ };
- for (let i = 0; i < batch.length; i += COMMIT_CHUNK_SIZE) {
- const end = Math.min(i + COMMIT_CHUNK_SIZE, batch.length);
- const patchChunk: GraphPatchPayload[] = [];
- for (let j = i; j < end; j++) {
- const { parsed: p, hash, previousHash } = batch[j];
+ /** Parse-all-first commit: resolve edges over full set, then commit in HTTP-sized batches. */
+ const flushAll = async (allParsed: ParsedFile[]): Promise<void> => {
+ if (allParsed.length === 0) return;
+ filesChanged += allParsed.length;
+ const previousPhase = progressPhase;
+ const saveStart = performance.now();
+ let resolveEdgesMs = 0;
+ let buildPatchMs = 0;
+ let localStripMs = 0;
+ let commitMs = 0;
+ let batchNodes = 0;
+ let batchEdges = 0;
+ let batchClaims = 0;
+ progressPhase = 'Saving';
+ progressTotal = allParsed.length;
+ progressCurrent = 0;
+ try {
+ setCurrentWork(`resolve ${allParsed.length} files`);
+ const resolveStart = performance.now();
+ const resolvedEdges = resolveEdgesFn!(allParsed.map(f => f.parsed), resolveStats);
+ resolveEdgesMs = Math.round(performance.now() - resolveStart);
+ const edgesByFile = new Map<string, any[]>();
+ for (const edge of resolvedEdges) {
+ let arr = edgesByFile.get(edge.srcFilePath);
+ if (!arr) { arr = []; edgesByFile.set(edge.srcFilePath, arr); }
+ arr.push(edge);
+ }
+ setCurrentWork(`build commit payloads ending ${nodePath.basename(allParsed[allParsed.length - 1].filePath)}`);
+ const buildStart = performance.now();
+ const preparedPatches: PreparedPatch[] = [];
+ for (let j = 0; j < allParsed.length; j++) {
+ const { parsed: p, hash, previousHash } = allParsed[j];
try {
- patchChunk.push(buildPatchFn!(p, hash, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash));
+ const patch = buildPatchFn!(p, hash, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, mapMode ? { emitClaims: false } : undefined);
+ preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));
} catch (err) {
parseErrors++;
process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);
}
}
- if (patchChunk.length === 0) continue;
- try {
- const result = await client.commitPatchBulk(patchChunk);
- if (result.rev > latestRev) latestRev = result.rev;
- patchesApplied += patchChunk.length;
- } catch (err) {
- if (debug) process.stderr.write(`\n [bulk chunk failed, falling back to per-file] ${err}\n`);
- for (const patch of patchChunk) {
- try {
- const result = await client.commitPatch(patch);
- if (result.rev > latestRev) latestRev = result.rev;
- patchesApplied++;
- } catch (commitErr) {
- parseErrors++;
- process.stderr.write(`\n [commit error] ${patch.source?.uri}: ${commitErr}\n`);
- }
+ buildPatchMs += Math.round(performance.now() - buildStart);
+ for (const pp of preparedPatches) {
+ for (const op of pp.patch.ops) {
+ if (op.type === 'UpsertNode') batchNodes++;
+ else if (op.type === 'UpsertEdge') batchEdges++;
+ else if (op.type === 'AssertClaim') batchClaims++;
}
}
+ if (preparedPatches.length === 0) {
+ progressCurrent = allParsed.length;
+ return;
+ }
+ commitMs += await commitPreparedPatches(preparedPatches, allParsed.length, { updateProgress: true });
+ } finally {
+ const totalSaveMs = Math.round(performance.now() - saveStart);
+ timings.resolveMs += resolveEdgesMs;
+ timings.buildPatchMs += buildPatchMs;
+ timings.stripChunkOpsMs += localStripMs;
+ timings.commitMs += commitMs;
+ totalNodesEmitted += batchNodes;
+ totalEdgesEmitted += batchEdges;
+ totalClaimsEmitted += batchClaims;
+ logSaveTiming(allParsed.length, resolveEdgesMs, buildPatchMs, localStripMs, commitMs, totalSaveMs, batchNodes, batchEdges, batchClaims);
+ progressPhase = previousPhase;
+ progressStart = performance.now();
}
};
@@ -390,11 +771,12 @@ export async function ingestFiles(
try {
const bytes = fs.readFileSync(filePath);
const hash = sha256(bytes);
- if (knownHashes.get(filePath) === hash) { filesSkipped++; continue; }
+ if (knownHashes.get(filePath) === hash) { filesSkipped++; progressCurrent++; continue; }
const previousHash = knownHashes.get(filePath);
changedPaths.push({ filePath, bytes, hash, previousHash: previousHash !== hash ? previousHash : undefined });
} catch (err) {
parseErrors++;
+ progressCurrent++;
process.stderr.write(`\n [read error] ${filePath}: ${err}\n`);
}
}
@@ -406,14 +788,29 @@ export async function ingestFiles(
resolveEdgesFn = ingestion.resolveEdges as Function;
buildPatchFn = patchBuilder.buildPatchWithResolution as Function;
- let batch: ParsedFile[] = [];
+ const allParsed: ParsedFile[] = [];
for (let i = 0; i < changedPaths.length; i++) {
const { filePath, bytes, hash, previousHash } = changedPaths[i];
+ setCurrentWork(`parse ${filePath}`);
try {
- const parsed = (ingestion.parseFile as Function)(filePath, bytes.toString('utf-8'));
+ const sourceText = bytes.toString('utf-8');
+ if (isLikelyMinifiedSource(sourceText)) {
+ minifiedLikely++;
+ filesSkipped++;
+ progressCurrent++;
+ if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
+ continue;
+ }
+ const p0 = performance.now();
+ const parsed = (ingestion.parseFile as Function)(filePath, sourceText, { emitChunks: !mapMode });
+ const pfMs = performance.now() - p0;
if (!parsed) { filesSkipped++; progressCurrent++; continue; }
+ timings.parseOnlyMs += pfMs;
+ const lang: string = (parsed as any).language ?? 'unknown';
+ const ls = langParseMs.get(lang) ?? { ms: 0, files: 0 };
+ ls.ms += pfMs; ls.files++; langParseMs.set(lang, ls);
entitiesParsed += parsed.entities.length;
- batch.push({ filePath, parsed, hash, previousHash });
+ allParsed.push({ filePath, parsed, hash, previousHash });
progressCurrent++;
} catch (err) {
parseErrors++;
@@ -421,9 +818,8 @@ export async function ingestFiles(
process.stderr.write(`\n [parse error] ${filePath}: ${err}\n`);
}
if ((i + 1) % YIELD_EVERY === 0) await new Promise<void>(resolve => setImmediate(resolve));
- if (batch.length >= PARSE_STREAM_CHUNK) { await flushBatch(batch); batch = []; }
}
- await flushBatch(batch);
+ await flushAll(allParsed);
}
} else {
// Path B: no baseline (first ingest) or --force → load modules, then stream parse + commit.
@@ -433,9 +829,12 @@ export async function ingestFiles(
resolveEdgesFn = ingestion.resolveEdges as Function;
buildPatchFn = patchBuilder.buildPatchWithResolution as Function;
- let batch: ParsedFile[] = [];
+ const useParseAllFirst = filePaths.length <= SMALL_REPO_THRESHOLD;
+ const allParsedB: ParsedFile[] = useParseAllFirst ? [] : (null as any);
+ let batch: ParsedFile[] = useParseAllFirst ? (null as any) : [];
for (let i = 0; i < filePaths.length; i++) {
const filePath = filePaths[i];
+ setCurrentWork(`parse ${filePath}`);
try {
const fileSize = fs.statSync(filePath).size;
if (fileSize === 0) { filesSkipped++; progressCurrent++; continue; }
@@ -444,10 +843,25 @@ export async function ingestFiles(
const hash = sha256(bytes);
if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; progressCurrent++; continue; }
const previousHash = knownHashes.get(filePath);
- const parsed = (ingestion.parseFile as Function)(filePath, bytes.toString('utf-8'));
+ const sourceText = bytes.toString('utf-8');
+ if (isLikelyMinifiedSource(sourceText)) {
+ minifiedLikely++;
+ filesSkipped++;
+ progressCurrent++;
+ if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
+ continue;
+ }
+ const p0 = performance.now();
+ const parsed = (ingestion.parseFile as Function)(filePath, sourceText, { emitChunks: !mapMode });
+ const pfMs = performance.now() - p0;
if (!parsed) { filesSkipped++; progressCurrent++; continue; }
+ timings.parseOnlyMs += pfMs;
+ const lang: string = (parsed as any).language ?? 'unknown';
+ const ls = langParseMs.get(lang) ?? { ms: 0, files: 0 };
+ ls.ms += pfMs; ls.files++; langParseMs.set(lang, ls);
entitiesParsed += parsed.entities.length;
- batch.push({ filePath, parsed, hash, previousHash: previousHash !== hash ? previousHash : undefined });
+ const entry: ParsedFile = { filePath, parsed, hash, previousHash: previousHash !== hash ? previousHash : undefined };
+ if (useParseAllFirst) { allParsedB.push(entry); } else { batch.push(entry); }
progressCurrent++;
} catch (err) {
parseErrors++;
@@ -455,16 +869,17 @@ export async function ingestFiles(
process.stderr.write(`\n [parse error] ${filePath}: ${err}\n`);
}
if ((i + 1) % YIELD_EVERY === 0) await new Promise<void>(resolve => setImmediate(resolve));
- if (batch.length >= PARSE_STREAM_CHUNK) { await flushBatch(batch); batch = []; }
+ if (!useParseAllFirst && batch.length >= PARSE_STREAM_CHUNK) { await flushBatch(batch); batch = []; }
+ }
+ if (useParseAllFirst) {
+ await flushAll(allParsedB);
+ } else {
+ await flushBatch(batch);
}
- await flushBatch(batch);
}
const parsed = performance.now();
timings.parseMs = Math.round(parsed - hashed);
- timings.resolveMs = 0; // now interleaved with parse
- timings.commitMs = 0; // now interleaved with parse
-
// Clear the parse bar
if (interval) process.stderr.write('\r' + ' '.repeat(PROG_LINE_WIDTH) + '\r');
@@ -497,9 +912,17 @@ export async function ingestFiles(
filesSkipped,
entitiesParsed,
latestRev,
- skipReasons: { unchanged: filesSkipped, emptyFile: 0, parseError: parseErrors, tooLarge },
+ skipReasons: { unchanged: filesSkipped, emptyFile: 0, parseError: parseErrors, tooLarge, minifiedLikely },
elapsedSeconds: parseFloat(elapsed),
- timings,
+ timings: {
+ ...timings,
+ parseOnlyMs: Math.round(timings.parseOnlyMs),
+ stripChunkOpsMs: Math.round(timings.stripChunkOpsMs),
+ },
+ emitted: { nodes: totalNodesEmitted, edges: totalEdgesEmitted, claims: totalClaimsEmitted },
+ langParseMs: Object.fromEntries(
+ [...langParseMs.entries()].map(([lang, s]) => [lang, { ms: Math.round(s.ms), files: s.files }])
+ ),
resolveStats,
}, null, 2));
} else if (opts.printSummary !== false) {
@@ -510,6 +933,7 @@ export async function ingestFiles(
if (filesSkipped > 0) console.log(` ${chalk.dim('skipped unchanged:')} ${filesSkipped}`);
if (parseErrors > 0) console.log(` ${chalk.red('parse errors:')} ${parseErrors}`);
if (tooLarge > 0) console.log(` ${chalk.dim('skipped too large:')} ${tooLarge}`);
+ if (minifiedLikely > 0) console.log(` ${chalk.dim('skipped minified:')} ${minifiedLikely}`);
console.log(` rev: ${latestRev}`);
if (patchesApplied === 0 && filesDiscovered === 0) {
@@ -520,12 +944,30 @@ export async function ingestFiles(
if (debug) {
console.log(chalk.dim(`\n Phase timings:`));
- console.log(chalk.dim(` modules: ${timings.moduleLoadMs}ms`));
- console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));
- console.log(chalk.dim(` hash: ${timings.hashMs}ms`));
- console.log(chalk.dim(` parse: ${timings.parseMs}ms`));
- console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));
- console.log(chalk.dim(` commit: ${timings.commitMs}ms`));
+ console.log(chalk.dim(` modules: ${timings.moduleLoadMs}ms`));
+ console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));
+ console.log(chalk.dim(` hash: ${timings.hashMs}ms`));
+ console.log(chalk.dim(` parse total: ${timings.parseMs}ms`));
+ console.log(chalk.dim(` parse only: ${Math.round(timings.parseOnlyMs)}ms`));
+ console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));
+ console.log(chalk.dim(` build: ${timings.buildPatchMs}ms`));
+ if (mapMode) {
+ console.log(chalk.dim(` stripChunk: ${Math.round(timings.stripChunkOpsMs)}ms`));
+ }
+ console.log(chalk.dim(` commit: ${timings.commitMs}ms`));
+ console.log(chalk.dim(` bulk: ${timings.bulkCommitMs}ms`));
+ console.log(chalk.dim(` fallback: ${timings.fallbackCommitMs}ms`));
+ console.log(chalk.dim(`\n Emitted ops:`));
+ console.log(chalk.dim(` nodes: ${totalNodesEmitted}`));
+ console.log(chalk.dim(` edges: ${totalEdgesEmitted}`));
+ console.log(chalk.dim(` claims: ${totalClaimsEmitted}`));
+ if (langParseMs.size > 0) {
+ console.log(chalk.dim(`\n Per-language parse times:`));
+ const sorted = [...langParseMs.entries()].sort((a, b) => b[1].ms - a[1].ms);
+ for (const [lang, stat] of sorted) {
+ console.log(chalk.dim(` ${lang.padEnd(12)} ${Math.round(stat.ms)}ms (${stat.files} files)`));
+ }
+ }
console.log(chalk.dim(`\n Resolve stats:`));
console.log(chalk.dim(` import lookups: ${resolveStats.importLookups}`));
console.log(chalk.dim(` transitive lookups: ${resolveStats.transitiveLookups}`));
diff --git a/ix-cli/src/cli/commands/locate.ts b/ix-cli/src/cli/commands/locate.ts
index 0842a8f4..11096451 100644
--- a/ix-cli/src/cli/commands/locate.ts
+++ b/ix-cli/src/cli/commands/locate.ts
@@ -219,7 +219,7 @@ function outputLocate(output: LocateOutput, symbol: string, format: string): voi
return;
}
- if (output.stale) renderWarning("Some results may be stale. Run ix ingest to update.");
+ if (output.stale) renderWarning("Some results may be stale. Run ix map to update.");
if (!output.resolvedTarget) {
stderr(`No graph entity found for "${symbol}".`);
diff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.ts
index 3388ba71..8118ff04 100644
--- a/ix-cli/src/cli/commands/map.ts
+++ b/ix-cli/src/cli/commands/map.ts
@@ -91,6 +91,7 @@ export function registerMapCommand(program: Command): void {
.option("--list", "Render the ranked list view instead of the default graph/tree view")
.option("--full", "Force full local map, bypassing automatic safety limits (advanced/testing)")
.option("--verbose", "Show raw confidence scores, crosscut scores, boundary ratios, and signals")
+ .option("--silent", "Suppress all output except a one-line summary (useful for LLM hooks)")
.addHelpText(
"after",
`
@@ -107,10 +108,14 @@ Advanced:
--full Override automatic local safety limits and force the full local map
path. Bypasses automatic downgrade to fast mode and the persistence
safety guardrail. Intended for testing and performance diagnosis.
+ --silent Skip the full map rendering. Prints one summary line to stderr and
+ exits. Ideal for LLM hooks and automated workflows where the full
+ output would waste context tokens.
Examples:
ix map .
ix map --format json
+ ix map --silent
ix map --level 2
ix map --min-confidence 0.5
ix map --max-items 10
@@ -121,7 +126,7 @@ Examples:
ix map . --full
ix --debug map . --full`
)
- .action(async (pathArg: string | undefined, opts: { format: string; level?: string; minConfidence: string; maxItems: string; allItems?: boolean; sort: string; graph?: boolean; list?: boolean; full?: boolean; verbose?: boolean }) => {
+ .action(async (pathArg: string | undefined, opts: { format: string; level?: string; minConfidence: string; maxItems: string; allItems?: boolean; sort: string; graph?: boolean; list?: boolean; full?: boolean; verbose?: boolean; silent?: boolean }) => {
const cwd = pathArg ? resolve(pathArg) : process.cwd();
try {
@@ -132,8 +137,10 @@ Examples:
return;
}
+ const silent = opts.silent === true || opts.format === "silent";
+
// Print warning when --full override is active
- if (opts.full && opts.format !== "json") {
+ if (opts.full && opts.format !== "json" && !silent) {
console.log(chalk.yellow("\nWarning"));
console.log(chalk.yellow(" Full local map override enabled.\n"));
console.log(" Ix will ignore automatic local safety limits and attempt full local mapping.");
@@ -142,14 +149,14 @@ Examples:
// Ingest the path before mapping so the graph is up to date
const ingestStart = performance.now();
- await ingestFiles(cwd, { recursive: true, format: opts.format === "json" ? "json" : "text", printSummary: false });
+ await ingestFiles(cwd, { recursive: true, format: (opts.format === "json" || silent) ? "json" : "text", printSummary: false, mapMode: true });
const ingestMs = Math.round(performance.now() - ingestStart);
const client = new IxClient(getEndpoint());
const mapBarWidth = 25;
const mapStart = performance.now();
- const mapInterval = opts.format !== "json" ? setInterval(() => {
+ const mapInterval = (opts.format !== "json" && !silent) ? setInterval(() => {
const elapsed = performance.now() - mapStart;
const pct = 1 - Math.exp(-elapsed / 4000);
const filled = Math.round(pct * mapBarWidth);
@@ -169,8 +176,20 @@ Examples:
}
if (mapInterval) { clearInterval(mapInterval); process.stderr.write('\r' + ' '.repeat(60) + '\r'); }
const mapMs = Math.round(performance.now() - mapStart);
+
+ if (silent) {
+ const systems = result.regions.filter(r => r.level === 3).length;
+ const subsystems = result.regions.filter(r => r.level === 2).length;
+ const modules = result.regions.filter(r => r.level === 1).length;
+ process.stderr.write(
+ `map: ${result.file_count} files · ${systems}s/${subsystems}ss/${modules}m regions · ${mapMs}ms\n`
+ );
+ return;
+ }
+
if (opts.format !== "json") {
- process.stderr.write(chalk.dim(` ingest ${ingestMs}ms · map ${mapMs}ms\n`));
+ const mapSec = (mapMs / 1000).toFixed(1);
+ process.stderr.write(chalk.dim(` Mapped in ${mapSec}s\n`));
}
const minConf = parseFloat(opts.minConfidence ?? "0");
diff --git a/ix-cli/src/cli/commands/read.ts b/ix-cli/src/cli/commands/read.ts
index 88c8b46a..d67f5864 100644
--- a/ix-cli/src/cli/commands/read.ts
+++ b/ix-cli/src/cli/commands/read.ts
@@ -60,7 +60,7 @@ function outputResult(result: ReadResult, format: string): void {
if (format === "json") {
console.log(JSON.stringify(result, null, 2));
} else {
- if (result.stale) stderr(chalk.yellow("⚠ File has changed since last ingest. Run ix ingest to update.\n"));
+ if (result.stale) stderr(chalk.yellow("⚠ File has changed since last ingest. Run ix map to update.\n"));
if (result.targetType === "symbol" || result.targetType === "filename-match") {
stderr(chalk.dim(` ${result.path}:${result.lineStart}-${result.lineEnd}\n`));
}
diff --git a/ix-cli/src/cli/commands/reset.ts b/ix-cli/src/cli/commands/reset.ts
index 696752f6..897547fb 100644
--- a/ix-cli/src/cli/commands/reset.ts
+++ b/ix-cli/src/cli/commands/reset.ts
@@ -49,19 +49,31 @@ export function registerResetCommand(program: Command): void {
}
const client = new IxClient(getEndpoint());
+ const label = opts.code ? "Wiping code graph..." : "Wiping graph...";
+ const spinnerFrames = ["⠋","⠙","⠹","⠸","⠼","⠴","⠦","⠧","⠇","⠏"];
+ let spinIdx = 0;
+ const interval = setInterval(() => {
+ process.stderr.write(`\r${chalk.cyan(spinnerFrames[spinIdx++ % spinnerFrames.length])} ${chalk.dim(label)}`);
+ }, 80);
try {
if (opts.code) {
await client.resetCode();
+ clearInterval(interval);
+ process.stderr.write("\r\x1b[K");
// Clear the mtime cache so the next ix map re-ingests all files
clearMtimeCache(process.cwd());
console.log(chalk.green("✓") + " Code graph wiped. Planning artifacts preserved.");
console.log(chalk.dim(" Run `ix map` to rebuild the code graph."));
} else {
await client.reset();
+ clearInterval(interval);
+ process.stderr.write("\r\x1b[K");
clearMtimeCache(process.cwd());
console.log(chalk.green("✓") + " Graph wiped.");
}
} catch (err: any) {
+ clearInterval(interval);
+ process.stderr.write("\r\x1b[K");
console.error(chalk.red("Error:"), err.message);
process.exitCode = 1;
return;
diff --git a/ix-cli/src/cli/commands/status.ts b/ix-cli/src/cli/commands/status.ts
index 5fd9490c..824419e2 100644
--- a/ix-cli/src/cli/commands/status.ts
+++ b/ix-cli/src/cli/commands/status.ts
@@ -51,7 +51,7 @@ export function registerStatusCommand(program: Command): void {
if (staleInfo.staleFiles > staleInfo.sampleChangedFiles.length) {
renderNote(`... and ${staleInfo.staleFiles - staleInfo.sampleChangedFiles.length} more`);
}
- renderNote("Run ix ingest to update.");
+ renderNote("Run ix map to update.");
} else {
renderSuccess("Graph is up to date.");
}
diff --git a/ix-cli/src/cli/commands/trace.ts b/ix-cli/src/cli/commands/trace.ts
index 0a79d258..75d5625a 100644
--- a/ix-cli/src/cli/commands/trace.ts
+++ b/ix-cli/src/cli/commands/trace.ts
@@ -117,7 +117,7 @@ async function buildTraceTree(
// ── Path search (BFS) ────────────────────────────────────────────────
-async function findPath(
+export async function findPath(
client: IxClient,
fromId: string,
toId: string,
@@ -134,9 +134,12 @@ async function findPath(
const { id, path } = entry;
if (path.length >= maxDepth) continue;
- const result = await client.expand(id, { direction: "out", predicates, hops: 1 });
+ const [outResult, inResult] = await Promise.all([
+ client.expand(id, { direction: "out", predicates, hops: 1 }),
+ client.expand(id, { direction: "in", predicates, hops: 1 }),
+ ]);
- for (const n of result.nodes) {
+ for (const n of [...outResult.nodes, ...inResult.nodes]) {
const name = n.name || n.attrs?.name || n.id.slice(0, 8);
if (!nodeMap.has(n.id)) {
nodeMap.set(n.id, { name, kind: n.kind ?? "unknown" });
@@ -441,7 +444,7 @@ export function registerTraceCommand(program: Command): void {
for (const child of children) {
nodes.push({ id: child.id, name: child.name, kind: child.kind });
// downstream = in edges (child → parent), upstream = out edges (parent → child)
- edges.push({ from: doUpstream ? parentId : child.id, to: doUpstream ? child.id : parentId, kind: relKind });
+ edges.push({ from: doUpstream ? child.id : parentId, to: doUpstream ? parentId : child.id, kind: relKind });
collectJson(child.id, child.children);
}
}
diff --git a/ix-cli/src/cli/commands/upgrade.ts b/ix-cli/src/cli/commands/upgrade.ts
index 0b6ad72f..ead6cf63 100644
--- a/ix-cli/src/cli/commands/upgrade.ts
+++ b/ix-cli/src/cli/commands/upgrade.ts
@@ -108,6 +108,7 @@ export async function checkForUpdate(): Promise<void> {
}
function printUpdateNotice(current: string, latest: string): void {
+ process.stderr.write('\r' + ' '.repeat(80) + '\r');
console.error("");
console.error(
chalk.yellow(` Update available: ${current} → ${latest}`)
diff --git a/ix-cli/src/cli/explain/render.ts b/ix-cli/src/cli/explain/render.ts
index 3a9c2bed..52f6fd0c 100644
--- a/ix-cli/src/cli/explain/render.ts
+++ b/ix-cli/src/cli/explain/render.ts
@@ -524,7 +524,7 @@ function renderNotes(facts: EntityFacts): string[] {
for (const d of facts.diagnostics) {
switch (d.code) {
case "unresolved_call_target":
- notes.push("Some downstream calls could not be resolved to named entities. Run `ix ingest` to improve coverage.");
+ notes.push("Some downstream calls could not be resolved to named entities. Run `ix map` to improve coverage.");
break;
case "stale_source":
notes.push("Source file has changed since last ingest — results may be incomplete.");
diff --git a/ix-cli/src/cli/explain/role-inference.ts b/ix-cli/src/cli/explain/role-inference.ts
index 90ff4b5b..45eaf0d6 100644
--- a/ix-cli/src/cli/explain/role-inference.ts
+++ b/ix-cli/src/cli/explain/role-inference.ts
@@ -68,7 +68,8 @@ export function inferRole(facts: EntityFacts): RoleInference {
// ── 1. Test ──────────────────────────────────────────────────────────
if (
- nameOrPathContains(facts, "test", "spec", "__tests__") ||
+ /[/\\](tests?|spec|__tests__)[/\\]/i.test(facts.path ?? "") ||
+ /\.(test|spec)\.[a-z]+$/i.test(facts.path ?? "") ||
tokens.includes("test") ||
tokens.includes("spec")
) {
diff --git a/ix-cli/src/cli/explain/subsystem.ts b/ix-cli/src/cli/explain/subsystem.ts
index 8fe19052..f7d9cd74 100644
--- a/ix-cli/src/cli/explain/subsystem.ts
+++ b/ix-cli/src/cli/explain/subsystem.ts
@@ -311,7 +311,7 @@ function densityNote(chunkDensity: number): string {
return " High chunk density indicates thorough code coverage in the knowledge graph.";
}
if (chunkDensity < 2.0) {
- return " Low chunk density - run 'ix ingest' to improve coverage.";
+ return " Low chunk density - run 'ix ingest' to enable ix read / ix explain on this subsystem.";
}
return "";
}
diff --git a/ix-cli/src/cli/explain/system-meaning.ts b/ix-cli/src/cli/explain/system-meaning.ts
index 6e135788..1802d6b4 100644
--- a/ix-cli/src/cli/explain/system-meaning.ts
+++ b/ix-cli/src/cli/explain/system-meaning.ts
@@ -27,7 +27,7 @@ function domainNounFromName(name: string): string {
.toLowerCase()
.split(/\s+/);
// Drop common suffixes that aren't the domain noun
- const drop = new Set(["kind", "type", "api", "result", "config", "service", "interface"]);
+ const drop = new Set(["kind", "type", "api", "result", "config", "service", "interface", "i"]);
const domainWords = words.filter((w) => !drop.has(w));
return domainWords.length > 0 ? domainWords[0] : words[0];
}
diff --git a/ix-cli/src/cli/help-text.ts b/ix-cli/src/cli/help-text.ts
index 52c7fd81..5d352395 100644
--- a/ix-cli/src/cli/help-text.ts
+++ b/ix-cli/src/cli/help-text.ts
@@ -1,40 +1,41 @@
import chalk from "chalk";
const OSS_HELP = [
- `${chalk.cyan("ix")} — Code Memory CLI`,
+ `${chalk.cyan("ix")} — System Intelligence CLI`,
``,
chalk.bold(`Start:`),
- ` map [path] Map the architectural hierarchy of a codebase`,
- ` watch Watch files and auto-ingest on change`,
- ` config Show or update configuration`,
+ ` map [path] Map system`,
+ ` watch Watch changes`,
+ ` config Update config`,
``,
chalk.bold(`Understand:`),
- ` search <term> Search the knowledge graph by term`,
- ` locate <symbol> Resolve symbol to definition with context`,
- ` explain <symbol> Explain an entity with history`,
- ` impact <target> Blast-radius / dependency analysis`,
- ` overview <target> One-shot structural summary`,
- ` read <target> Read file content or symbol source code`,
+ ` search <term> Search system`,
+ ` locate <symbol> Find definition`,
+ ` explain <symbol> Explain behavior`,
+ ` impact <target> Analyze impact`,
+ ` overview <target> Summarize structure`,
+ ` read <target> Read source`,
+ ` smells Detect issues`,
``,
chalk.bold(`Explore:`),
- ` trace <symbol> Follow how a symbol connects upstream and downstream`,
- ` inventory List entities by kind`,
- ` rank Hotspot discovery by metric`,
- ` history <entityId> Show entity provenance chain`,
- ` diff <from> <to> Show diff between revisions`,
+ ` trace <symbol> Trace flow`,
+ ` subsystem Explore structure`,
+ ` inventory List components`,
+ ` rank Rank importance`,
+ ` history <target> Show history`,
+ ` diff <from> <to> Compare changes`,
``,
chalk.bold(`System:`),
- ` status Show backend health`,
- ` stats Show graph statistics`,
- ` doctor Check system health`,
- ` docker <action> Manage the IX backend containers`,
- ` reset Wipe all graph data for a clean re-ingest`,
- ` upgrade Upgrade ix CLI and backend to the latest version`,
+ ` status Check status`,
+ ` stats Show stats`,
+ ` doctor Diagnose issues`,
+ ` docker <action> Manage backend`,
+ ` reset Reset map`,
+ ` upgrade Update ix`,
``,
].join("\n");
-const FOOTER = `Use "ix help advanced" for low-level graph commands.
-Use "ix <command> --help" for details on any command.
+const FOOTER = `Use "ix <command> --help" for details.
`;
export function buildHelpText(
diff --git a/ix-cli/src/cli/resolve.ts b/ix-cli/src/cli/resolve.ts
index 3582fff0..c656676f 100644
--- a/ix-cli/src/cli/resolve.ts
+++ b/ix-cli/src/cli/resolve.ts
@@ -35,6 +35,10 @@ const STRUCTURAL_KINDS = new Set([
...CONTAINER_KINDS, "function", "method",
]);
+function looksTypeLikeSymbol(symbol: string): boolean {
+ return /^[A-Z][A-Za-z0-9_]*$/.test(symbol);
+}
+
// ── Scoring ───────────────────────────────────────────────────────────────
/**
@@ -77,9 +81,20 @@ export function scoreCandidate(
score -= 3; // containers are high-value resolution targets
} else if (STRUCTURAL_KINDS.has(kind)) {
score -= 1; // methods/functions are useful but lower than containers
+ } else if (kind === "chunk") {
+ score += 5; // chunks are retrieval units, not useful as trace starting points
}
// non-structural kinds (config_entry, doc, decision, etc.) get no boost
+ if (looksTypeLikeSymbol(symbol) && name === symbolLower) {
+ if (kind === "class" || kind === "interface" || kind === "trait" || kind === "object") {
+ score -= 4;
+ }
+ // No penalty for method/function: the +3 that was here caused chunks (score=0)
+ // to outscore structural entities (function/method net score=2) for PascalCase
+ // names like "Apply", "StartEtcd", "Range". Chunks are now penalised explicitly above.
+ }
+
// ── Path match ──────────────────────────────────────────────────────
if (opts?.path) {
const pathLower = opts.path.toLowerCase();
@@ -129,7 +144,11 @@ export async function resolveEntityFull(
opts?: { kind?: string; path?: string; pick?: number; includeTests?: boolean; testsOnly?: boolean; searchLimit?: number }
): Promise<ResolveResult> {
const kindFilter = opts?.kind;
- const nodes = await client.search(symbol, { limit: opts?.searchLimit ?? 20, kind: kindFilter, nameOnly: true });
+ const nodes = await client.search(symbol, {
+ limit: opts?.searchLimit ?? (looksTypeLikeSymbol(symbol) ? 50 : 20),
+ kind: kindFilter,
+ nameOnly: true,
+ });
if (nodes.length === 0) {
stderr(`No entity found matching "${symbol}".`);
@@ -202,8 +221,6 @@ function pickBest(
preferredKinds: string[],
opts?: { kind?: string; path?: string }
): ResolveResult | null {
- // Here is a test
- // LEt's see if it worked
// Score all candidates
const scored = candidates.map(n => ({
node: n,
@@ -351,7 +368,6 @@ function nodeToResolved(node: any, symbol: string, mode: ResolutionMode): Resolv
}
function buildAmbiguous(nodes: any[], scores?: number[]): AmbiguousResult {
- //this is a test
const seen = new Set<string>();
const candidates: AmbiguousResult["candidates"] = [];
let rank = 0;
diff --git a/ix-cli/src/client/api.ts b/ix-cli/src/client/api.ts
index c52fd7ab..81936304 100644
--- a/ix-cli/src/client/api.ts
+++ b/ix-cli/src/client/api.ts
@@ -171,7 +171,17 @@ export class IxClient {
}
async commitPatch(patch: GraphPatchPayload): Promise<PatchCommitResult> {
- return this.post("/v1/patch", patch);
+ const resp = await fetch(`${this.endpoint}/v1/patch`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(patch),
+ signal: AbortSignal.timeout(5 * 60 * 1000), // 5 min — matches commitPatchBulk
+ });
+ if (!resp.ok) {
+ const text = await resp.text();
+ throw new Error(`${resp.status}: ${text}`);
+ }
+ return resp.json() as Promise<PatchCommitResult>;
}
async hasIngestBaseline(): Promise<boolean> {
@@ -189,13 +199,31 @@ export class IxClient {
}
async map(opts?: { full?: boolean }): Promise<any> {
- return this.post("/v1/map", opts ?? {});
+ const resp = await fetch(`${this.endpoint}/v1/map`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(opts ?? {}),
+ signal: AbortSignal.timeout(30 * 60 * 1000), // 30 minute timeout
+ });
+ if (!resp.ok) {
+ const text = await resp.text();
+ throw new Error(`${resp.status}: ${text}`);
+ }
+ return resp.json();
}
- async commitPatchBulk(patches: GraphPatchPayload[]): Promise<{ rev: number }> {
- const results = await this.commitPatchBatch(patches);
- const rev = results.reduce((max, r) => Math.max(max, r.rev ?? 0), 0);
- return { rev };
+ async commitPatchBulk(patches: GraphPatchPayload[]): Promise<PatchCommitResult> {
+ const resp = await fetch(`${this.endpoint}/v1/patches/bulk`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ patches }),
+ signal: AbortSignal.timeout(5 * 60 * 1000), // 5 min — prevents hang when k8s ingress closes idle connections
+ });
+ if (!resp.ok) {
+ const text = await resp.text();
+ throw new Error(`${resp.status}: ${text}`);
+ }
+ return resp.json() as Promise<PatchCommitResult>;
}
async runSmells(opts?: {
@@ -234,11 +262,31 @@ export class IxClient {
}
async reset(): Promise<{ ok: boolean; message: string }> {
- return this.post("/v1/reset", {});
+ const resp = await fetch(`${this.endpoint}/v1/reset`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({}),
+ signal: AbortSignal.timeout(10 * 60 * 1000), // 10 minutes
+ });
+ if (!resp.ok) {
+ const text = await resp.text();
+ throw new Error(`${resp.status}: ${text}`);
+ }
+ return resp.json() as Promise<{ ok: boolean; message: string }>;
}
async resetCode(): Promise<{ ok: boolean; message: string }> {
- return this.post("/v1/reset/code", {});
+ const resp = await fetch(`${this.endpoint}/v1/reset/code`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({}),
+ signal: AbortSignal.timeout(10 * 60 * 1000), // 10 minutes
+ });
+ if (!resp.ok) {
+ const text = await resp.text();
+ throw new Error(`${resp.status}: ${text}`);
+ }
+ return resp.json() as Promise<{ ok: boolean; message: string }>;
}
async stats(): Promise<any> {
diff --git a/ix-cli/src/client/types.ts b/ix-cli/src/client/types.ts
index 5603dc34..a41fb0f5 100644
--- a/ix-cli/src/client/types.ts
+++ b/ix-cli/src/client/types.ts
@@ -136,6 +136,7 @@ export interface SkipReasons {
emptyFile: number;
parseError: number;
tooLarge: number;
+ minifiedLikely?: number;
}
export interface IngestResult {
diff --git a/ix-plugin/.claude-plugin/plugin.json b/ix-plugin/.claude-plugin/plugin.json
index 65f37d2d..27b0dcf3 100644
--- a/ix-plugin/.claude-plugin/plugin.json
+++ b/ix-plugin/.claude-plugin/plugin.json
@@ -1,7 +1,7 @@
{
"name": "ix-memory",
- "version": "1.0.0",
- "description": "Ix Memory integration — auto-ingests edited files and front-runs Grep/Glob/Read with graph-aware ix searches for token-efficient codebase understanding.",
+ "version": "2.0.0",
+ "description": "Ix Memory integration — intercepts all searches and file reads with graph-aware ix queries, auto-ingests edits, and refreshes the full architectural map after each session.",
"author": {
"name": "Ix Memory"
},
diff --git a/ix-plugin/README.md b/ix-plugin/README.md
index c17231a6..648bdd9a 100644
--- a/ix-plugin/README.md
+++ b/ix-plugin/README.md
@@ -1,46 +1,51 @@
# Ix Memory — Claude Code Plugin
-A Claude Code plugin that makes Claude use Ix for codebase searches instead of
-native `Grep`/`Glob`, and keeps the Ix graph current as Claude edits files.
+A Claude Code plugin that makes Claude always use Ix for codebase understanding —
+injecting session context on every prompt, intercepting all searches and file reads
+with graph-aware queries, and keeping the Ix graph current as Claude edits.
## What It Does
| Trigger | Hook | Effect |
|---------|------|--------|
-| Claude runs `Grep` or `Glob` | `PreToolUse` → `ix-intercept.sh` | Runs `ix text` + `ix locate` (or `ix inventory`) first and injects results into Claude's context before the native tool fires |
-| Claude runs `Write`, `Edit`, or `MultiEdit` | `PostToolUse` → `ix-ingest.sh` (async) | Ingests the changed file into the Ix graph silently in the background |
+| User sends any prompt | `UserPromptSubmit` → `ix-briefing.sh` | Injects compact session briefing (goals, bugs, decisions, tasks) into every prompt |
+| Claude runs `Grep` or `Glob` | `PreToolUse` → `ix-intercept.sh` | Runs `ix text` + `ix locate` (or `ix inventory`) first; injects graph-aware results |
+| Claude runs `Read` | `PreToolUse` → `ix-read.sh` | Runs `ix inventory` + `ix overview` for the file; injects entity context before Claude sees raw source |
+| Claude runs `Bash` (grep/rg) | `PreToolUse` → `ix-bash.sh` | Detects grep/rg patterns and front-runs with `ix text` + `ix locate` |
+| Claude runs `Write`, `Edit`, `MultiEdit`, or `NotebookEdit` | `PostToolUse` → `ix-ingest.sh` (async) | Ingests the changed file into the Ix graph silently in the background |
+| Claude finishes responding | `Stop` → `ix-map.sh` (async) | Runs `ix map` to refresh the full architectural graph for the next session |
-Both hooks bail silently if `ix` is not in PATH or the Ix backend is unreachable.
+All hooks bail silently if `ix` is not in PATH or the Ix backend is unreachable.
## Why Hooks Instead of MCP
- **No user approval prompt** — hooks fire silently and automatically
-- **Guaranteed execution** — every Grep/Glob gets front-run, every edit gets ingested
+- **Guaranteed execution** — every prompt, search, read, and edit is covered
- **Token-efficient** — Ix returns structured, bounded results vs. raw file content
+- **Graph always current** — edits ingested immediately; full map refreshes at session end
## Installation
-**1. Add the Ix marketplace (one-time):**
+### From the repo (local)
-```
-/plugin marketplace add ix-infrastructure/Ix
+```bash
+bash ix-plugin/install.sh
```
-**2. Install the plugin:**
+### Via curl
-```
-/plugin install ix-memory
+```bash
+curl -fsSL https://raw.githubusercontent.com/ix-infrastructure/IX-Memory/main/ix-plugin/install.sh | bash
```
-**3. Restart Claude Code.** Verify the hooks loaded with `/hooks`.
+Restart Claude Code after installing. Verify hooks loaded with `/hooks`.
## Requirements
- `ix` CLI in PATH and Ix backend running (`ix status` should return ok)
-- `jq` in PATH (used to parse hook JSON input)
-- `ripgrep` (`rg`) in PATH — required by `ix text` for file-level text search
+- `jq` in PATH
+- `ripgrep` (`rg`) in PATH — required by `ix text`
-Install missing deps:
```bash
# Ubuntu/Debian
sudo apt install jq ripgrep
@@ -57,33 +62,65 @@ ix-plugin/
plugin.json # manifest
hooks/
hooks.json # hook event registration
- ix-intercept.sh # PreToolUse: Grep|Glob → ix text/locate/inventory
- ix-ingest.sh # PostToolUse: Write|Edit|MultiEdit → ix ingest
+ ix-briefing.sh # UserPromptSubmit → ix briefing (session context)
+ ix-intercept.sh # PreToolUse: Grep|Glob → ix text + ix locate/inventory
+ ix-read.sh # PreToolUse: Read → ix inventory + ix overview
+ ix-bash.sh # PreToolUse: Bash (grep/rg) → ix text + ix locate
+ ix-ingest.sh # PostToolUse: Write|Edit|MultiEdit|NotebookEdit → ix ingest
+ ix-map.sh # Stop → ix map (full graph refresh)
+ install.sh
+ uninstall.sh
README.md
```
-## How the Intercept Hook Works
+## Hook Details
+
+### ix-briefing.sh (UserPromptSubmit)
+
+On every user prompt:
+1. Runs `ix briefing --format json` (60s health cache to avoid per-prompt overhead)
+2. Injects active goals, open bugs, recent decisions, and pending tasks as `additionalContext`
+3. Claude always has project context without needing an explicit `ix briefing` call
+
+### ix-intercept.sh (PreToolUse: Grep|Glob)
When Claude runs `Grep "verify_token"`:
+1. Runs `ix text "verify_token" --limit 20 --format json` + `ix locate "verify_token" --limit 10 --format json` in parallel
+2. Injects results as `additionalContext` — Claude uses them before Grep output arrives
-1. `ix-intercept.sh` fires with the tool input as JSON on stdin
-2. Script runs `ix text "verify_token" --limit 20 --format json`
-3. Script also runs `ix locate "verify_token" --limit 10 --format json`
-4. Results are returned as `additionalContext` — injected into Claude's next context window
-5. Claude sees the Ix results and typically uses them instead of waiting for Grep output
+For `Glob`, runs `ix inventory` (optionally filtered by path) instead.
-For `Glob`, the script runs `ix inventory` (optionally filtered by path) instead.
+### ix-read.sh (PreToolUse: Read)
-## How the Ingest Hook Works
+When Claude reads a file:
+1. Runs `ix inventory --path <file>` (all entities defined in the file)
+2. Runs `ix overview <name>` (structural summary of the module)
+3. Injects both as `additionalContext` — Claude understands the file's structure before seeing raw source
-When Claude edits a file:
+### ix-bash.sh (PreToolUse: Bash)
-1. `ix-ingest.sh` fires asynchronously after the edit succeeds
-2. Reads `tool_input.file_path` from stdin
-3. Runs `ix ingest <file_path>` silently
-4. Returns a short `additionalContext` confirmation: `[ix] Graph updated — ingested: <path>`
+When Claude runs a `grep` or `rg` command:
+1. Extracts the search pattern from the command string
+2. Runs `ix text <pattern> --limit 20` + `ix locate <pattern> --limit 10` in parallel
+3. Injects results as `additionalContext`
+
+### ix-ingest.sh (PostToolUse: Write|Edit|MultiEdit|NotebookEdit)
+
+After each file edit:
+1. Reads `tool_input.file_path` from stdin
+2. Runs `ix ingest <file>` silently in the background
+3. Returns `additionalContext`: `[ix] Graph updated — ingested: <path>`
+
+### ix-map.sh (Stop)
+
+After Claude finishes each response:
+1. Runs `ix map` in the background (detached, non-blocking)
+2. Keeps the full architectural graph current for the next session
## Uninstall
-Remove the `hooks` block from `~/.claude/settings.json` (or `.claude/settings.json` for
-project-scoped installs), then restart Claude Code.
+```bash
+bash ix-plugin/uninstall.sh
+# or:
+curl -fsSL https://raw.githubusercontent.com/ix-infrastructure/IX-Memory/main/ix-plugin/uninstall.sh | bash
+```
diff --git a/ix-plugin/hooks/hooks.json b/ix-plugin/hooks/hooks.json
index ba178bf8..72b68508 100644
--- a/ix-plugin/hooks/hooks.json
+++ b/ix-plugin/hooks/hooks.json
@@ -1,5 +1,16 @@
{
"hooks": {
+ "UserPromptSubmit": [
+ {
+ "hooks": [
+ {
+ "type": "command",
+ "command": "${CLAUDE_PLUGIN_ROOT}/hooks/ix-briefing.sh",
+ "timeout": 10
+ }
+ ]
+ }
+ ],
"PreToolUse": [
{
"matcher": "Grep|Glob",
@@ -10,11 +21,31 @@
"timeout": 10
}
]
+ },
+ {
+ "matcher": "Read",
+ "hooks": [
+ {
+ "type": "command",
+ "command": "${CLAUDE_PLUGIN_ROOT}/hooks/ix-read.sh",
+ "timeout": 8
+ }
+ ]
+ },
+ {
+ "matcher": "Bash",
+ "hooks": [
+ {
+ "type": "command",
+ "command": "${CLAUDE_PLUGIN_ROOT}/hooks/ix-bash.sh",
+ "timeout": 10
+ }
+ ]
}
],
"PostToolUse": [
{
- "matcher": "Write|Edit|MultiEdit",
+ "matcher": "Write|Edit|MultiEdit|NotebookEdit",
"hooks": [
{
"type": "command",
@@ -24,6 +55,18 @@
}
]
}
+ ],
+ "Stop": [
+ {
+ "hooks": [
+ {
+ "type": "command",
+ "command": "${CLAUDE_PLUGIN_ROOT}/hooks/ix-map.sh",
+ "timeout": 60,
+ "async": true
+ }
+ ]
+ }
]
}
}
diff --git a/ix-plugin/hooks/ix-bash.sh b/ix-plugin/hooks/ix-bash.sh
new file mode 100755
index 00000000..2044c82e
--- /dev/null
+++ b/ix-plugin/hooks/ix-bash.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# ix-bash.sh — PreToolUse hook for Bash
+#
+# Fires before Claude runs a Bash command. Detects grep/rg search patterns and
+# front-runs them with ix text + ix locate for graph-aware, token-efficient
+# results before the raw shell command executes.
+#
+# Exit 0 + JSON stdout → injects additionalContext, Bash still runs
+# Exit 0 + no stdout → no-op, Bash runs normally
+
+set -euo pipefail
+
+INPUT=$(cat)
+COMMAND=$(echo "$INPUT" | jq -r '.tool_input.command // empty')
+[ -z "$COMMAND" ] && exit 0
+
+# Only intercept grep/rg invocations
+echo "$COMMAND" | grep -qE '^\s*(grep|rg)\s' || exit 0
+
+# Bail silently if ix is not in PATH
+command -v ix >/dev/null 2>&1 || exit 0
+
+# ── Health check with 30s TTL cache ─────────────────────────────────────────
+IX_HEALTH_CACHE="${TMPDIR:-/tmp}/ix-healthy"
+_now=$(date +%s)
+_cache_ok=0
+if [ -f "$IX_HEALTH_CACHE" ]; then
+ _cached=$(cat "$IX_HEALTH_CACHE" 2>/dev/null || echo 0)
+ (( (_now - _cached) < 30 )) && _cache_ok=1
+fi
+if [ "$_cache_ok" -eq 0 ]; then
+ ix status >/dev/null 2>&1 || exit 0
+ echo "$_now" > "$IX_HEALTH_CACHE"
+fi
+
+# ── Extract search pattern ────────────────────────────────────────────────────
+# Try double-quoted pattern first, then single-quoted, then unquoted
+PATTERN=""
+PATTERN=$(echo "$COMMAND" | sed -E 's/.*\s"([^"]+)".*/\1/' 2>/dev/null) || PATTERN=""
+if [ -z "$PATTERN" ] || [ "$PATTERN" = "$COMMAND" ]; then
+ PATTERN=$(echo "$COMMAND" | sed -E "s/.*\s'([^']+)'.*/\1/" 2>/dev/null) || PATTERN=""
+fi
+if [ -z "$PATTERN" ] || [ "$PATTERN" = "$COMMAND" ]; then
+ # Unquoted: grab first non-flag token after command name
+ PATTERN=$(echo "$COMMAND" | sed -E 's/^\s*(grep|rg)\s+(-[a-zA-Z0-9]+\s+|--[a-zA-Z-]+=\S+\s+)*([^-][^ ]*).*/\3/' 2>/dev/null) || PATTERN=""
+fi
+
+[ -z "$PATTERN" ] && exit 0
+[ ${#PATTERN} -lt 2 ] && exit 0
+
+# ── Run ix text + ix locate in parallel ─────────────────────────────────────
+_text_tmp=$(mktemp)
+_locate_tmp=$(mktemp)
+trap 'rm -f "$_text_tmp" "$_locate_tmp"' EXIT
+
+ix text "$PATTERN" --limit 20 --format json > "$_text_tmp" 2>/dev/null &
+_text_pid=$!
+ix locate "$PATTERN" --limit 10 --format json > "$_locate_tmp" 2>/dev/null &
+_locate_pid=$!
+
+wait "$_text_pid" || true
+wait "$_locate_pid" || true
+
+TEXT_RESULT=$(cat "$_text_tmp")
+LOCATE_RESULT=$(cat "$_locate_tmp")
+
+[ -z "$TEXT_RESULT" ] && [ -z "$LOCATE_RESULT" ] && exit 0
+
+CONTEXT="[ix] Pre-bash search results for pattern: '${PATTERN}'"
+[ -n "$TEXT_RESULT" ] && CONTEXT="${CONTEXT}\n\n--- ix text ---\n${TEXT_RESULT}"
+[ -n "$LOCATE_RESULT" ] && CONTEXT="${CONTEXT}\n\n--- ix locate ---\n${LOCATE_RESULT}"
+
+jq -n --arg ctx "$CONTEXT" '{"additionalContext": $ctx}'
+exit 0
diff --git a/ix-plugin/hooks/ix-briefing.sh b/ix-plugin/hooks/ix-briefing.sh
new file mode 100755
index 00000000..bfd1e220
--- /dev/null
+++ b/ix-plugin/hooks/ix-briefing.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# ix-briefing.sh — UserPromptSubmit hook
+#
+# Fires at the start of each user prompt. Injects a compact ix session briefing
+# once per 10 minutes — simulating "on session open" behaviour. Subsequent
+# prompts within the window are no-ops (no tokens consumed).
+#
+# Exit 0 + JSON stdout → injects additionalContext into the prompt
+# Exit 0 + no stdout → no-op
+
+set -euo pipefail
+
+BRIEFING_TTL=600 # 10 minutes
+
+# Bail silently if ix is not in PATH
+command -v ix >/dev/null 2>&1 || exit 0
+
+IX_BRIEFING_CACHE="${TMPDIR:-/tmp}/ix-briefing-cache"
+_now=$(date +%s)
+
+# If cache exists and is fresh, stay silent (briefing already injected this session)
+if [ -f "$IX_BRIEFING_CACHE" ]; then
+ _cached_time=$(head -1 "$IX_BRIEFING_CACHE" 2>/dev/null || echo 0)
+ if (( (_now - _cached_time) < BRIEFING_TTL )); then
+ exit 0
+ fi
+fi
+
+# Cache is stale — check backend health before calling ix
+ix status >/dev/null 2>&1 || exit 0
+
+BRIEFING=$(ix briefing --format json 2>/dev/null) || exit 0
+[ -z "$BRIEFING" ] && exit 0
+
+# Write timestamp + briefing to cache
+{ echo "$_now"; echo "$BRIEFING"; } > "$IX_BRIEFING_CACHE"
+
+jq -n --arg b "$BRIEFING" \
+ '{"additionalContext": ("[ix] Session briefing:\n" + $b)}'
+exit 0
diff --git a/ix-plugin/hooks/ix-ingest.sh b/ix-plugin/hooks/ix-ingest.sh
index dd070c48..92f5ad8e 100755
--- a/ix-plugin/hooks/ix-ingest.sh
+++ b/ix-plugin/hooks/ix-ingest.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# ix-ingest.sh — PostToolUse hook for Write, Edit, MultiEdit
+# ix-ingest.sh — PostToolUse hook for Write, Edit, MultiEdit, NotebookEdit
#
# Fires after Claude modifies a file. Automatically ingests the changed file
# into the Ix graph so the next query reflects the current code state.
diff --git a/ix-plugin/hooks/ix-map.sh b/ix-plugin/hooks/ix-map.sh
new file mode 100755
index 00000000..ae23f617
--- /dev/null
+++ b/ix-plugin/hooks/ix-map.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# ix-map.sh — Stop hook
+#
+# Fires after Claude finishes each response. Runs ix map asynchronously to
+# keep the architectural graph current so the next session starts fresh.
+#
+# Runs async (does not block Claude's response or session end).
+
+set -euo pipefail
+
+# Bail silently if ix is not in PATH
+command -v ix >/dev/null 2>&1 || exit 0
+
+# Bail if backend is not reachable
+ix status >/dev/null 2>&1 || exit 0
+
+# Run ix map in background — detach from current process so it survives
+nohup ix map >/dev/null 2>&1 &
+disown
+
+exit 0
diff --git a/ix-plugin/hooks/ix-read.sh b/ix-plugin/hooks/ix-read.sh
new file mode 100755
index 00000000..1f761833
--- /dev/null
+++ b/ix-plugin/hooks/ix-read.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# ix-read.sh — PreToolUse hook for Read
+#
+# Fires before Claude reads a file. Runs ix inventory (all entities in the file)
+# and ix overview (structural summary) in parallel and injects results as
+# additionalContext so Claude has graph-aware context before reading raw source.
+#
+# Exit 0 + JSON stdout → injects additionalContext, native Read still runs
+# Exit 0 + no stdout → no-op, Read runs normally
+
+set -euo pipefail
+
+INPUT=$(cat)
+FILE_PATH=$(echo "$INPUT" | jq -r '.tool_input.file_path // empty')
+[ -z "$FILE_PATH" ] && exit 0
+
+# Skip binary/non-code files
+case "$FILE_PATH" in
+ *.png|*.jpg|*.jpeg|*.gif|*.ico|*.pdf|*.zip|*.tar|*.gz|*.bin|*.exe) exit 0 ;;
+esac
+
+# Bail silently if ix is not in PATH
+command -v ix >/dev/null 2>&1 || exit 0
+
+# ── Health check with 30s TTL cache ─────────────────────────────────────────
+IX_HEALTH_CACHE="${TMPDIR:-/tmp}/ix-healthy"
+_now=$(date +%s)
+_cache_ok=0
+if [ -f "$IX_HEALTH_CACHE" ]; then
+ _cached=$(cat "$IX_HEALTH_CACHE" 2>/dev/null || echo 0)
+ (( (_now - _cached) < 30 )) && _cache_ok=1
+fi
+if [ "$_cache_ok" -eq 0 ]; then
+ ix status >/dev/null 2>&1 || exit 0
+ echo "$_now" > "$IX_HEALTH_CACHE"
+fi
+
+# ── Run ix inventory + ix overview in parallel ───────────────────────────────
+_inv_tmp=$(mktemp)
+_ov_tmp=$(mktemp)
+trap 'rm -f "$_inv_tmp" "$_ov_tmp"' EXIT
+
+ix inventory --path "$FILE_PATH" --format json > "$_inv_tmp" 2>/dev/null &
+_inv_pid=$!
+
+# Strip extension for a cleaner symbol match
+_name=$(basename "$FILE_PATH")
+_name="${_name%.*}"
+ix overview "$_name" --format json > "$_ov_tmp" 2>/dev/null &
+_ov_pid=$!
+
+wait "$_inv_pid" || true
+wait "$_ov_pid" || true
+
+INV_RESULT=$(cat "$_inv_tmp")
+OV_RESULT=$(cat "$_ov_tmp")
+
+[ -z "$INV_RESULT" ] && [ -z "$OV_RESULT" ] && exit 0
+
+CONTEXT="[ix] Pre-read context for: ${FILE_PATH}"
+[ -n "$INV_RESULT" ] && CONTEXT="${CONTEXT}\n\n--- ix inventory (entities in file) ---\n${INV_RESULT}"
+[ -n "$OV_RESULT" ] && CONTEXT="${CONTEXT}\n\n--- ix overview ---\n${OV_RESULT}"
+
+jq -n --arg ctx "$CONTEXT" '{"additionalContext": $ctx}'
+exit 0
diff --git a/ix-plugin/install.sh b/ix-plugin/install.sh
index ce7e14d8..e597828e 100755
--- a/ix-plugin/install.sh
+++ b/ix-plugin/install.sh
@@ -2,13 +2,13 @@
# ix-plugin/install.sh — Standalone Claude Code plugin installer for Ix Memory
#
# Usage (curl):
-# curl -fsSL https://raw.githubusercontent.com/ix-infrastructure/Ix/main/ix-plugin/install.sh | bash
+# curl -fsSL https://raw.githubusercontent.com/ix-infrastructure/IX-Memory/main/ix-plugin/install.sh | bash
#
# Usage (local, from the Ix repo):
# bash ix-plugin/install.sh
#
# What it does:
-# 1. Downloads ix-intercept.sh and ix-ingest.sh to ~/.local/share/ix/plugin/hooks/
+# 1. Copies/downloads all hook scripts to ~/.local/share/ix/plugin/hooks/
# 2. Wires them into ~/.claude/settings.json
#
# Prerequisites: ix CLI must already be installed and in PATH.
@@ -17,10 +17,19 @@ set -euo pipefail
# ── Config ────────────────────────────────────────────────────────────────────
-GITHUB_RAW="https://raw.githubusercontent.com/ix-infrastructure/Ix/main/ix-plugin/hooks"
+GITHUB_RAW="https://raw.githubusercontent.com/ix-infrastructure/IX-Memory/main/ix-plugin/hooks"
INSTALL_DIR="${IX_PLUGIN_DIR:-$HOME/.local/share/ix/plugin/hooks}"
SETTINGS="$HOME/.claude/settings.json"
+HOOKS=(
+ "ix-briefing.sh"
+ "ix-intercept.sh"
+ "ix-read.sh"
+ "ix-bash.sh"
+ "ix-ingest.sh"
+ "ix-map.sh"
+)
+
# ── Helpers ───────────────────────────────────────────────────────────────────
info() { echo " [ok] $*"; }
@@ -43,13 +52,13 @@ info "curl, jq"
if ! command -v ix >/dev/null 2>&1; then
warn "'ix' not found in PATH — hooks will be installed but won't activate until ix is installed."
- warn "Install ix first: see https://github.com/ix-infrastructure/Ix"
+ warn "Install ix first: see https://github.com/ix-infrastructure/IX-Memory"
fi
-# ── Download hooks ────────────────────────────────────────────────────────────
+# ── Download / copy hooks ─────────────────────────────────────────────────────
echo ""
-echo "── Downloading hooks ──"
+echo "── Installing hooks ──"
mkdir -p "$INSTALL_DIR"
@@ -57,39 +66,37 @@ mkdir -p "$INSTALL_DIR"
_repo_hooks=""
if [ -n "${BASH_SOURCE[0]:-}" ] && [ "${BASH_SOURCE[0]}" != "bash" ]; then
_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)"
- # Hooks live in hooks/ subdir relative to this script
if [ -f "$_script_dir/hooks/ix-intercept.sh" ]; then
_repo_hooks="$_script_dir/hooks"
fi
fi
-if [ -n "$_repo_hooks" ]; then
- # Local install — copy from repo
- cp "$_repo_hooks/ix-intercept.sh" "$INSTALL_DIR/ix-intercept.sh"
- cp "$_repo_hooks/ix-ingest.sh" "$INSTALL_DIR/ix-ingest.sh"
- info "Copied hooks from local repo → $INSTALL_DIR"
-else
- # Remote install — download from GitHub
- curl -fsSL "$GITHUB_RAW/ix-intercept.sh" -o "$INSTALL_DIR/ix-intercept.sh"
- curl -fsSL "$GITHUB_RAW/ix-ingest.sh" -o "$INSTALL_DIR/ix-ingest.sh"
- info "Downloaded hooks → $INSTALL_DIR"
-fi
-
-chmod +x "$INSTALL_DIR/ix-intercept.sh" "$INSTALL_DIR/ix-ingest.sh"
-info "Permissions set"
+for hook in "${HOOKS[@]}"; do
+ if [ -n "$_repo_hooks" ]; then
+ cp "$_repo_hooks/$hook" "$INSTALL_DIR/$hook"
+ else
+ curl -fsSL "$GITHUB_RAW/$hook" -o "$INSTALL_DIR/$hook"
+ fi
+ chmod +x "$INSTALL_DIR/$hook"
+ info "$hook → $INSTALL_DIR"
+done
# ── Wire Claude Code settings ─────────────────────────────────────────────────
echo ""
echo "── Configuring Claude Code ──"
+BRIEFING="$INSTALL_DIR/ix-briefing.sh"
INTERCEPT="$INSTALL_DIR/ix-intercept.sh"
+READ_HOOK="$INSTALL_DIR/ix-read.sh"
+BASH_HOOK="$INSTALL_DIR/ix-bash.sh"
INGEST="$INSTALL_DIR/ix-ingest.sh"
+MAP_HOOK="$INSTALL_DIR/ix-map.sh"
mkdir -p "$HOME/.claude"
[ -f "$SETTINGS" ] || echo "{}" > "$SETTINGS"
-# Idempotent: skip if already wired
+# Idempotent: skip if already wired (check for intercept hook as sentinel)
already=$(jq --arg cmd "$INTERCEPT" \
'[.hooks?.PreToolUse[]?.hooks[]?.command? // empty] | map(select(. == $cmd)) | length' \
"$SETTINGS" 2>/dev/null || echo "0")
@@ -98,20 +105,54 @@ if [ "$already" -gt 0 ]; then
info "Hooks already registered in ~/.claude/settings.json — skipping"
else
tmp=$(mktemp)
- jq --arg intercept "$INTERCEPT" --arg ingest "$INGEST" '
+ jq \
+ --arg briefing "$BRIEFING" \
+ --arg intercept "$INTERCEPT" \
+ --arg read_hook "$READ_HOOK" \
+ --arg bash_hook "$BASH_HOOK" \
+ --arg ingest "$INGEST" \
+ --arg map_hook "$MAP_HOOK" '
.hooks |= (. // {}) |
+
+ .hooks.UserPromptSubmit |= (. // []) |
+ .hooks.UserPromptSubmit += [
+ {
+ "hooks": [{ "type": "command", "command": $briefing, "timeout": 10 }]
+ }
+ ] |
+
.hooks.PreToolUse |= (. // []) |
- .hooks.PreToolUse += [{
- "matcher": "Grep|Glob",
- "hooks": [{ "type": "command", "command": $intercept, "timeout": 10 }]
- }] |
+ .hooks.PreToolUse += [
+ {
+ "matcher": "Grep|Glob",
+ "hooks": [{ "type": "command", "command": $intercept, "timeout": 10 }]
+ },
+ {
+ "matcher": "Read",
+ "hooks": [{ "type": "command", "command": $read_hook, "timeout": 8 }]
+ },
+ {
+ "matcher": "Bash",
+ "hooks": [{ "type": "command", "command": $bash_hook, "timeout": 10 }]
+ }
+ ] |
+
.hooks.PostToolUse |= (. // []) |
- .hooks.PostToolUse += [{
- "matcher": "Write|Edit|MultiEdit",
- "hooks": [{ "type": "command", "command": $ingest, "timeout": 30 }]
- }]
+ .hooks.PostToolUse += [
+ {
+ "matcher": "Write|Edit|MultiEdit|NotebookEdit",
+ "hooks": [{ "type": "command", "command": $ingest, "timeout": 30, "async": true }]
+ }
+ ] |
+
+ .hooks.Stop |= (. // []) |
+ .hooks.Stop += [
+ {
+ "hooks": [{ "type": "command", "command": $map_hook, "timeout": 60, "async": true }]
+ }
+ ]
' "$SETTINGS" > "$tmp" && mv "$tmp" "$SETTINGS"
- info "Registered hooks → ~/.claude/settings.json"
+ info "Registered all hooks → ~/.claude/settings.json"
fi
# ── Done ──────────────────────────────────────────────────────────────────────
@@ -123,8 +164,18 @@ echo "╚═══════════════════════
echo ""
echo " Hooks installed to: $INSTALL_DIR"
echo ""
+echo " Active hooks:"
+echo " UserPromptSubmit → ix briefing (session context on every prompt)"
+echo " PreToolUse Grep|Glob → ix text + ix locate (graph-aware search)"
+echo " PreToolUse Read → ix inventory + ix overview (file context)"
+echo " PreToolUse Bash (grep/rg) → ix text + ix locate (shell search intercept)"
+echo " PostToolUse Write|Edit|... → ix ingest (auto-update graph after edits)"
+echo " Stop → ix map (full graph refresh after session)"
+echo ""
echo " Restart Claude Code to activate."
echo ""
echo " To uninstall:"
-echo " curl -fsSL https://raw.githubusercontent.com/ix-infrastructure/Ix/main/ix-plugin/uninstall.sh | bash"
+echo " bash ix-plugin/uninstall.sh"
+echo " # or:"
+echo " curl -fsSL https://raw.githubusercontent.com/ix-infrastructure/IX-Memory/main/ix-plugin/uninstall.sh | bash"
echo ""
diff --git a/ix-plugin/uninstall.sh b/ix-plugin/uninstall.sh
index 2c19d5f9..c4decb4a 100755
--- a/ix-plugin/uninstall.sh
+++ b/ix-plugin/uninstall.sh
@@ -1,11 +1,5 @@
#!/usr/bin/env bash
# ix-plugin/uninstall.sh — Remove the Ix Memory Claude Code plugin
-#
-# Usage (curl):
-# curl -fsSL https://raw.githubusercontent.com/ix-infrastructure/Ix/main/ix-plugin/uninstall.sh | bash
-#
-# Usage (local):
-# bash ix-plugin/uninstall.sh
set -euo pipefail
@@ -19,7 +13,13 @@ echo "── Removing Ix Claude Code plugin ──"
# Remove hook files
if [ -d "$INSTALL_DIR" ]; then
- rm -f "$INSTALL_DIR/ix-intercept.sh" "$INSTALL_DIR/ix-ingest.sh"
+ rm -f \
+ "$INSTALL_DIR/ix-briefing.sh" \
+ "$INSTALL_DIR/ix-intercept.sh" \
+ "$INSTALL_DIR/ix-read.sh" \
+ "$INSTALL_DIR/ix-bash.sh" \
+ "$INSTALL_DIR/ix-ingest.sh" \
+ "$INSTALL_DIR/ix-map.sh"
rmdir "$INSTALL_DIR" 2>/dev/null || true
info "Removed $INSTALL_DIR"
else
@@ -28,25 +28,31 @@ fi
# Remove hook entries from ~/.claude/settings.json
if [ -f "$SETTINGS" ] && command -v jq >/dev/null 2>&1; then
+ BRIEFING="$INSTALL_DIR/ix-briefing.sh"
INTERCEPT="$INSTALL_DIR/ix-intercept.sh"
+ READ_HOOK="$INSTALL_DIR/ix-read.sh"
+ BASH_HOOK="$INSTALL_DIR/ix-bash.sh"
INGEST="$INSTALL_DIR/ix-ingest.sh"
+ MAP_HOOK="$INSTALL_DIR/ix-map.sh"
+
+ _ix_cmds=$(jq -n \
+ --arg a "$BRIEFING" --arg b "$INTERCEPT" \
+ --arg c "$READ_HOOK" --arg d "$BASH_HOOK" \
+ --arg e "$INGEST" --arg f "$MAP_HOOK" \
+ '[$a,$b,$c,$d,$e,$f]')
tmp=$(mktemp)
- jq --arg intercept "$INTERCEPT" --arg ingest "$INGEST" '
- if .hooks.PreToolUse then
- .hooks.PreToolUse |= map(select(
- (.hooks // []) | map(.command) | any(. == $intercept) | not
- ))
- else . end |
- if .hooks.PostToolUse then
- .hooks.PostToolUse |= map(select(
- (.hooks // []) | map(.command) | any(. == $ingest) | not
- ))
- else . end
+ jq --argjson ix_cmds "$_ix_cmds" '
+ def is_ix_hook: (.hooks // []) | map(.command) | any(. as $c | $ix_cmds | any(. == $c));
+
+ if .hooks.UserPromptSubmit then .hooks.UserPromptSubmit |= map(select(is_ix_hook | not)) else . end |
+ if .hooks.PreToolUse then .hooks.PreToolUse |= map(select(is_ix_hook | not)) else . end |
+ if .hooks.PostToolUse then .hooks.PostToolUse |= map(select(is_ix_hook | not)) else . end |
+ if .hooks.Stop then .hooks.Stop |= map(select(is_ix_hook | not)) else . end
' "$SETTINGS" > "$tmp" && mv "$tmp" "$SETTINGS"
info "Removed hooks from ~/.claude/settings.json"
else
- echo " (settings.json not found — nothing to remove)"
+ echo " (settings.json not found or jq unavailable — nothing to remove)"
fi
echo ""
diff --git a/memory-layer/src/main/scala/ix/memory/Main.scala b/memory-layer/src/main/scala/ix/memory/Main.scala
index 9ce6d3db..6020f93a 100644
--- a/memory-layer/src/main/scala/ix/memory/Main.scala
+++ b/memory-layer/src/main/scala/ix/memory/Main.scala
@@ -9,7 +9,6 @@ import ix.memory.api.Routes
import ix.memory.conflict.ConflictService
import ix.memory.context._
import ix.memory.db._
-import ix.memory.ingestion._
import ix.memory.map.MapService
import ix.memory.smell.SmellService
import ix.memory.subsystem.SubsystemScoringService
@@ -33,10 +32,7 @@ object Main extends IOApp.Simple {
queryApi = new ArangoGraphQueryApi(client)
// 3. Services
- parserRouter = new ParserRouter()
- ingestionService = new IngestionService(parserRouter, writeApi, queryApi)
bulkWriteApi = new BulkWriteApi(client)
- bulkIngestionService = new BulkIngestionService(parserRouter, bulkWriteApi, queryApi)
seeder = new GraphSeeder(queryApi)
expander = new GraphExpander(queryApi)
claimCollector = new ClaimCollector(queryApi)
@@ -50,7 +46,7 @@ object Main extends IOApp.Simple {
subsystemService = new SubsystemScoringService(client, writeApi, mapService)
// 4. HTTP routes
- routes = Routes.all(contextService, ingestionService, bulkIngestionService, queryApi, writeApi, conflictService, client, mapService, bulkWriteApi, smellService, subsystemService)
+ routes = Routes.all(contextService, queryApi, writeApi, conflictService, client, mapService, bulkWriteApi, smellService, subsystemService)
// 5. Server
server <- EmberServerBuilder
diff --git a/memory-layer/src/main/scala/ix/memory/api/BulkPatchCommitRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/BulkPatchCommitRoutes.scala
index 3e1812e2..b20a1981 100644
--- a/memory-layer/src/main/scala/ix/memory/api/BulkPatchCommitRoutes.scala
+++ b/memory-layer/src/main/scala/ix/memory/api/BulkPatchCommitRoutes.scala
@@ -16,8 +16,6 @@ class BulkPatchCommitRoutes(bulkWriteApi: BulkWriteApi, queryApi: GraphQueryApi)
private case class BulkPatchRequest(patches: List[GraphPatch])
private implicit val reqDecoder: Decoder[BulkPatchRequest] = deriveDecoder
- private val mapper = new com.fasterxml.jackson.databind.ObjectMapper()
-
val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {
case req @ POST -> Root / "v1" / "patches" / "bulk" =>
(for {
@@ -43,13 +41,12 @@ class BulkPatchCommitRoutes(bulkWriteApi: BulkWriteApi, queryApi: GraphQueryApi)
}
private def buildProvenanceMap(patch: GraphPatch): java.util.Map[String, AnyRef] = {
- val json = Json.obj(
- "source_uri" -> Json.fromString(patch.source.uri),
- "source_hash" -> patch.source.sourceHash.fold(Json.Null)(Json.fromString),
- "extractor" -> Json.fromString(patch.source.extractor),
- "source_type" -> Json.fromString(patch.source.sourceType.asJson.asString.getOrElse("code")),
- "observed_at" -> Json.fromString(patch.timestamp.toString)
- )
- mapper.readValue(json.noSpaces, classOf[java.util.Map[String, AnyRef]])
+ val result = new java.util.HashMap[String, AnyRef](5)
+ result.put("source_uri", patch.source.uri)
+ result.put("source_hash", patch.source.sourceHash.orNull)
+ result.put("extractor", patch.source.extractor)
+ result.put("source_type", patch.source.sourceType.asJson.asString.getOrElse("code"))
+ result.put("observed_at", patch.timestamp.toString)
+ result
}
}
diff --git a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
deleted file mode 100644
index d393c47d..00000000
--- a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-package ix.memory.api
-
-import java.nio.file.Paths
-
-import cats.effect.IO
-import io.circe.{Decoder, Encoder}
-import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
-import org.http4s.HttpRoutes
-import org.http4s.circe.CirceEntityCodec._
-import org.http4s.dsl.io._
-
-import ix.memory.ingestion.{BulkIngestionService, IngestionProgress, IngestionService}
-import org.typelevel.log4cats.slf4j.Slf4jLogger
-
-case class IngestRequest(path: String, language: Option[String], recursive: Option[Boolean], force: Option[Boolean])
-
-object IngestRequest {
- implicit val decoder: Decoder[IngestRequest] = deriveDecoder[IngestRequest]
- implicit val encoder: Encoder[IngestRequest] = deriveEncoder[IngestRequest]
-}
-
-case class SkipReasonsResponse(unchanged: Int, emptyFile: Int, parseError: Int, tooLarge: Int)
-
-object SkipReasonsResponse {
- implicit val encoder: Encoder[SkipReasonsResponse] = deriveEncoder[SkipReasonsResponse]
- implicit val decoder: Decoder[SkipReasonsResponse] = deriveDecoder[SkipReasonsResponse]
-}
-
-case class IngestResponse(filesProcessed: Int, patchesApplied: Int, filesSkipped: Int,
- entitiesCreated: Int, latestRev: Long, skipReasons: SkipReasonsResponse)
-
-object IngestResponse {
- implicit val encoder: Encoder[IngestResponse] = deriveEncoder[IngestResponse]
- implicit val decoder: Decoder[IngestResponse] = deriveDecoder[IngestResponse]
-}
-
-class IngestionRoutes(ingestionService: IngestionService, bulkIngestionService: BulkIngestionService) {
-
- private val logger = Slf4jLogger.getLoggerFromName[IO]("ix.ingest.progress")
-
- private val progressLog: IngestionProgress => IO[Unit] = { p =>
- logger.info(s"discovered=${p.filesDiscovered} parsed=${p.filesParsed} chunks=${p.chunksWritten}/${p.totalChunks}")
- }
-
- val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {
- case req @ POST -> Root / "v1" / "ingest" =>
- (for {
- body <- req.as[IngestRequest]
- _ <- IO.raiseWhen(body.path.contains(".."))(
- new IllegalArgumentException("Path traversal not allowed")
- )
- path = Paths.get(body.path)
- result <- bulkIngestionService.ingestPath(path, body.language, body.recursive.getOrElse(false), progressLog, body.force.getOrElse(false))
- resp <- Ok(IngestResponse(
- filesProcessed = result.filesProcessed,
- patchesApplied = result.patchesApplied,
- filesSkipped = result.filesSkipped,
- entitiesCreated = result.entitiesCreated,
- latestRev = result.latestRev.value,
- skipReasons = SkipReasonsResponse(
- result.skipReasons.unchanged,
- result.skipReasons.emptyFile,
- result.skipReasons.parseError,
- result.skipReasons.tooLarge
- )
- ))
- } yield resp).handleErrorWith(ErrorHandler.handle(_))
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/api/MapEncoder.scala b/memory-layer/src/main/scala/ix/memory/api/MapEncoder.scala
index 14a2b333..fb0daee1 100644
--- a/memory-layer/src/main/scala/ix/memory/api/MapEncoder.scala
+++ b/memory-layer/src/main/scala/ix/memory/api/MapEncoder.scala
@@ -3,7 +3,7 @@ package ix.memory.api
import io.circe.Json
import io.circe.syntax._
-import ix.memory.map.{ArchitectureEdge, ArchitectureMap, Region, RegionMatch, ScopedSubsystemView}
+import ix.memory.map.{ArchitectureMap, MapPreflightResult, PersistenceEstimate, Region, RegionMatch, ScopedSubsystemView}
import ix.memory.model.NodeId
/** Shared JSON encoding for ArchitectureMap, used by MapRoutes and SubsystemRoutes. */
@@ -11,15 +11,26 @@ object MapEncoder {
def encodeMap(m: ArchitectureMap): Json = {
val regionsByLevel = m.regions.groupBy(_.level).toVector.sortBy(_._1)
- Json.obj(
+ val base = Json.obj(
"file_count" -> m.fileCount.asJson,
"region_count" -> m.regions.size.asJson,
"levels" -> regionsByLevel.size.asJson,
"map_rev" -> m.mapRev.asJson,
+ "outcome" -> m.outcome.label.asJson,
"regions" -> m.regions.sortBy(r => (r.level, r.label)).map(encodeRegion).asJson,
- "edges" -> m.edges.sortBy(e => (e.level, e.src.value.toString, e.dst.value.toString)).map(encodeEdge).asJson,
"hierarchy" -> buildHierarchyTree(m).asJson
)
+
+ val withPreflight =
+ m.preflight match {
+ case Some(value) => base.deepMerge(Json.obj("preflight" -> encodePreflight(value)))
+ case None => base
+ }
+
+ m.persistenceEstimate match {
+ case Some(value) => withPreflight.deepMerge(Json.obj("persistence" -> encodePersistence(value)))
+ case None => withPreflight
+ }
}
def encodeRegion(r: Region): Json =
@@ -40,17 +51,6 @@ object MapEncoder {
"interface_node_count" -> r.interfaceNodeCount.asJson
)
- def encodeEdge(edge: ArchitectureEdge): Json =
- Json.obj(
- "id" -> edge.id.asJson,
- "src" -> edge.src.value.toString.asJson,
- "dst" -> edge.dst.value.toString.asJson,
- "level" -> edge.level.asJson,
- "weight" -> edge.weight.asJson,
- "dominant_signal" -> edge.dominantSignal.asJson,
- "predicate_counts" -> edge.predicateCounts.asJson
- )
-
def encodeScopedView(view: ScopedSubsystemView, isCrossCutting: Region => Boolean): Json =
Json.obj(
"target" -> encodeScopedRegion(view.target, isCrossCutting).asJson,
@@ -140,4 +140,35 @@ object MapEncoder {
renderNode(view.target, Set.empty)
}
+
+ private def encodePreflight(preflight: MapPreflightResult): Json =
+ Json.obj(
+ "cost" -> Json.obj(
+ "file_count" -> preflight.cost.fileCount.asJson,
+ "directory_count" -> preflight.cost.directoryCount.asJson,
+ "directory_quadratic" -> preflight.cost.directoryQuadratic.asJson,
+ "symbol_estimate" -> preflight.cost.symbolEstimate.asJson,
+ "edge_estimate" -> preflight.cost.edgeEstimate.asJson
+ ),
+ "capacity" -> Json.obj(
+ "cpu_cores" -> preflight.capacity.cpuCores.asJson,
+ "heap_max_bytes" -> preflight.capacity.heapMaxBytes.asJson,
+ "heap_free_bytes" -> preflight.capacity.heapFreeBytes.asJson,
+ "container_memory" -> preflight.capacity.containerMemory.asJson,
+ "disk_free_bytes" -> preflight.capacity.diskFreeBytes.asJson
+ ),
+ "risk" -> preflight.risk.label.asJson,
+ "mode" -> preflight.mode.label.asJson,
+ "warnings" -> preflight.warnings.asJson,
+ "duration_ms" -> preflight.durationMs.asJson
+ )
+
+ private def encodePersistence(persistence: PersistenceEstimate): Json =
+ Json.obj(
+ "region_nodes" -> persistence.regionNodes.asJson,
+ "file_edges" -> persistence.fileEdges.asJson,
+ "region_edges" -> persistence.regionEdges.asJson,
+ "delete_ops" -> persistence.deleteOps.asJson,
+ "total_ops" -> persistence.totalOps.asJson
+ )
}
diff --git a/memory-layer/src/main/scala/ix/memory/api/MapRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/MapRoutes.scala
index 3189be0d..1acadb4e 100644
--- a/memory-layer/src/main/scala/ix/memory/api/MapRoutes.scala
+++ b/memory-layer/src/main/scala/ix/memory/api/MapRoutes.scala
@@ -16,8 +16,8 @@ class MapRoutes(mapService: MapService) {
case req @ POST -> Root / "v1" / "map" =>
(for {
body <- req.as[Json].handleError(_ => Json.obj())
- force = body.hcursor.get[Boolean]("full").toOption.getOrElse(false)
- archMap <- mapService.buildMap(forceRecompute = force)
+ forceFull = body.hcursor.get[Boolean]("full").toOption.getOrElse(false)
+ archMap <- mapService.buildMap(forceRecompute = forceFull, forceFull = forceFull)
resp <- Ok(MapEncoder.encodeMap(archMap))
} yield resp).handleErrorWith(ErrorHandler.handle(_))
}
diff --git a/memory-layer/src/main/scala/ix/memory/api/PatchCommitRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/PatchCommitRoutes.scala
index 19d7e7e9..def177bb 100644
--- a/memory-layer/src/main/scala/ix/memory/api/PatchCommitRoutes.scala
+++ b/memory-layer/src/main/scala/ix/memory/api/PatchCommitRoutes.scala
@@ -13,8 +13,6 @@ import ix.memory.model.{GraphPatch, Rev}
class PatchCommitRoutes(writeApi: GraphWriteApi, bulkWriteApi: BulkWriteApi, queryApi: GraphQueryApi) {
- private val mapper = new com.fasterxml.jackson.databind.ObjectMapper()
-
val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {
case req @ POST -> Root / "v1" / "patch" =>
(for {
@@ -52,13 +50,12 @@ class PatchCommitRoutes(writeApi: GraphWriteApi, bulkWriteApi: BulkWriteApi, que
}
private def buildProvenanceMap(patch: GraphPatch): java.util.Map[String, AnyRef] = {
- val json = Json.obj(
- "source_uri" -> Json.fromString(patch.source.uri),
- "source_hash" -> patch.source.sourceHash.fold(Json.Null)(Json.fromString),
- "extractor" -> Json.fromString(patch.source.extractor),
- "source_type" -> Json.fromString(patch.source.sourceType.asJson.asString.getOrElse("code")),
- "observed_at" -> Json.fromString(patch.timestamp.toString)
- )
- mapper.readValue(json.noSpaces, classOf[java.util.Map[String, AnyRef]])
+ val result = new java.util.HashMap[String, AnyRef](5)
+ result.put("source_uri", patch.source.uri)
+ result.put("source_hash", patch.source.sourceHash.orNull)
+ result.put("extractor", patch.source.extractor)
+ result.put("source_type", patch.source.sourceType.asJson.asString.getOrElse("code"))
+ result.put("observed_at", patch.timestamp.toString)
+ result
}
}
diff --git a/memory-layer/src/main/scala/ix/memory/api/Routes.scala b/memory-layer/src/main/scala/ix/memory/api/Routes.scala
index 376a16c9..d8dd22dd 100644
--- a/memory-layer/src/main/scala/ix/memory/api/Routes.scala
+++ b/memory-layer/src/main/scala/ix/memory/api/Routes.scala
@@ -11,7 +11,6 @@ import org.http4s.dsl.io._
import ix.memory.conflict.ConflictService
import ix.memory.context.ContextService
import ix.memory.db.{ArangoClient, BulkWriteApi, GraphQueryApi, GraphWriteApi}
-import ix.memory.ingestion.{BulkIngestionService, IngestionService}
import ix.memory.map.MapService
import ix.memory.smell.SmellService
import ix.memory.subsystem.SubsystemScoringService
@@ -20,8 +19,6 @@ object Routes {
def all(
contextService: ContextService,
- ingestionService: IngestionService,
- bulkIngestionService: BulkIngestionService,
queryApi: GraphQueryApi,
writeApi: GraphWriteApi,
conflictService: ConflictService,
@@ -38,7 +35,6 @@ object Routes {
}
val contextRoutes = new ContextRoutes(contextService).routes
- val ingestionRoutes = new IngestionRoutes(ingestionService, bulkIngestionService).routes
val entityRoutes = new EntityRoutes(queryApi).routes
val diffRoutes = new DiffRoutes(queryApi).routes
val conflictRoutes = new ConflictRoutes(conflictService).routes
@@ -59,7 +55,7 @@ object Routes {
val smellRoutes = new SmellRoutes(smellService).routes
val subsystemRoutes = new SubsystemRoutes(subsystemService, mapService).routes
- health <+> contextRoutes <+> ingestionRoutes <+> entityRoutes <+>
+ health <+> contextRoutes <+> entityRoutes <+>
diffRoutes <+> conflictRoutes <+> decideRoutes <+> searchRoutes <+>
truthRoutes <+> patchRoutes <+> decisionListRoutes <+> expandRoutes <+>
statsRoutes <+> patchCommitRoutes <+> listRoutes <+> goalRoutes <+>
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
index 7ddc575e..2e8d7451 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
@@ -1,5 +1,7 @@
package ix.memory.db
+import scala.concurrent.duration._
+
import cats.effect.{IO, Resource}
import cats.syntax.traverse._
import com.arangodb.{ArangoDB, ArangoDatabase}
@@ -56,6 +58,15 @@ class ArangoClient private (db: ArangoDatabase) {
()
}
+ def queryLong(
+ aql: String,
+ bindVars: Map[String, AnyRef] = Map.empty,
+ txId: Option[String] = None
+ ): IO[Long] =
+ queryOne(aql, bindVars, txId).map {
+ _.flatMap(_.asNumber.flatMap(_.toLong)).getOrElse(0L)
+ }
+
/** Bulk insert/replace documents into a collection. Uses Document API (not AQL). */
def bulkInsert(
collection: String,
@@ -67,6 +78,7 @@ class ArangoClient private (db: ArangoDatabase) {
val opts = new DocumentCreateOptions()
.overwriteMode(OverwriteMode.valueOf(overwriteMode))
.waitForSync(false)
+ .silent(true)
val docs = new java.util.ArrayList[java.util.Map[String, AnyRef]](documents.size)
documents.foreach(docs.add)
val result = db.collection(collection).insertDocuments(docs, opts)
@@ -123,8 +135,14 @@ class ArangoClient private (db: ArangoDatabase) {
/**
* Delete only code-derived nodes and edges, preserving planning artifacts
* (goals/intents, plans, tasks, decisions, bugs and their linking edges).
+ *
+ * Uses larger batched REMOVE passes and avoids materializing `RETURN 1`
+ * result sets for every deleted document. We first count the matching docs,
+ * then execute enough bounded delete passes to cover that cardinality, with
+ * a final recount only if needed. Edges are removed before nodes.
*/
def truncateCodeGraph(): IO[Unit] = {
+ val batchSize = 5000
val codeKinds = Array(
"file", "function", "class", "method", "interface",
"module", "chunk", "region"
@@ -133,19 +151,73 @@ class ArangoClient private (db: ArangoDatabase) {
"CALLS", "IMPORTS", "EXTENDS", "IMPLEMENTS",
"CONTAINS", "DEFINES", "CONTAINS_CHUNK", "REFERENCES", "NEXT", "IN_REGION"
)
+
+ def removeAll(
+ countAql: String,
+ removeAql: String,
+ bindVars: Map[String, AnyRef]
+ ): IO[Unit] = {
+ val varsWithBatch = bindVars + ("batchSize" -> Int.box(batchSize))
+
+ def runPasses(remaining: Long): IO[Unit] = {
+ val passes = math.ceil(remaining.toDouble / batchSize.toDouble).toInt
+ (0 until passes).foldLeft(IO.unit) { (acc, _) =>
+ acc *> execute(removeAql, varsWithBatch)
+ }
+ }
+
+ def loop: IO[Unit] =
+ queryLong(countAql, bindVars).flatMap { remaining =>
+ if (remaining <= 0) IO.unit
+ else runPasses(remaining) *> queryLong(countAql, bindVars).flatMap { tail =>
+ if (tail <= 0) IO.unit else loop
+ }
+ }
+
+ loop
+ }
+
for {
- _ <- execute(
- "FOR n IN nodes FILTER n.kind IN @kinds REMOVE n IN nodes",
+ // Edges first — avoids locking nodes that are about to be removed
+ _ <- removeAll(
+ "RETURN LENGTH(FOR e IN edges FILTER e.predicate IN @predicates RETURN 1)",
+ "FOR e IN edges FILTER e.predicate IN @predicates LIMIT @batchSize REMOVE e IN edges",
+ Map("predicates" -> codePredicates.asInstanceOf[AnyRef])
+ )
+ // Claims next — code ingests append versioned/retracted claims forever, so
+ // reset-code must clear them before removing the corresponding nodes.
+ _ <- removeAll(
+ """RETURN LENGTH(
+ | FOR c IN claims
+ | FILTER LENGTH(
+ | FOR n IN nodes
+ | FILTER n.logical_id == c.entity_id
+ | AND n.kind IN @kinds
+ | LIMIT 1
+ | RETURN 1
+ | ) > 0
+ | RETURN 1
+ |)""".stripMargin,
+ """FOR c IN claims
+ | FILTER LENGTH(
+ | FOR n IN nodes
+ | FILTER n.logical_id == c.entity_id
+ | AND n.kind IN @kinds
+ | LIMIT 1
+ | RETURN 1
+ | ) > 0
+ | LIMIT @batchSize
+ | REMOVE c IN claims""".stripMargin,
Map("kinds" -> codeKinds.asInstanceOf[AnyRef])
)
- _ <- execute(
- "FOR e IN edges FILTER e.predicate IN @predicates REMOVE e IN edges",
- Map("predicates" -> codePredicates.asInstanceOf[AnyRef])
+ _ <- removeAll(
+ "RETURN LENGTH(FOR n IN nodes FILTER n.kind IN @kinds RETURN 1)",
+ "FOR n IN nodes FILTER n.kind IN @kinds LIMIT @batchSize REMOVE n IN nodes",
+ Map("kinds" -> codeKinds.asInstanceOf[AnyRef])
)
- // Also remove code-ingest patches so getSourceHashes returns empty after reset,
- // preventing the ingest client from treating all files as unchanged.
- _ <- execute(
- "FOR p IN patches FILTER STARTS_WITH(p.data.source.extractor, 'tree-sitter') REMOVE p IN patches",
+ _ <- removeAll(
+ "RETURN LENGTH(FOR p IN patches FILTER STARTS_WITH(p.data.source.extractor, 'tree-sitter') RETURN 1)",
+ "FOR p IN patches FILTER STARTS_WITH(p.data.source.extractor, 'tree-sitter') LIMIT @batchSize REMOVE p IN patches",
Map.empty
)
} yield ()
@@ -166,6 +238,7 @@ object ArangoClient {
.host(host, port)
.user(user)
.password(if (password.isEmpty) null else password)
+ .maxConnections(16)
.build()
val existingDbs = arango.getDatabases.asScala.toSet
if (!existingDbs.contains(database)) {
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
index aa06be2a..9d2b9b84 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoGraphQueryApi.scala
@@ -343,7 +343,8 @@ class ArangoGraphQueryApi(client: ArangoClient) extends GraphQueryApi {
val nodeIdStr = entityId.value.toString
client.query(
"""FOR p IN patches
- | FILTER LENGTH(
+ | FILTER @nodeId IN (IS_ARRAY(p.data.entityIds) ? p.data.entityIds : [])
+ | OR LENGTH(
| FOR op IN (IS_ARRAY(p.data.ops) ? p.data.ops : [])
| FILTER op.id == @nodeId OR op.entityId == @nodeId
| RETURN 1
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoSchema.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoSchema.scala
index bec90b28..d4319f2c 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoSchema.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoSchema.scala
@@ -33,6 +33,13 @@ object ArangoSchema {
java.util.Arrays.asList("logical_id"),
new PersistentIndexOptions()
)
+ // Compound index for tombstone query: FILTER logical_id IN @ids AND deleted_rev == null
+ // Without this, the single-field logical_id index returns all historical versions of each
+ // node and applies deleted_rev as a post-filter — gets slower as the collection grows.
+ nodes.ensurePersistentIndex(
+ java.util.Arrays.asList("logical_id", "deleted_rev"),
+ new PersistentIndexOptions()
+ )
// Indexes — edges
val edges = db.collection("edges")
@@ -61,6 +68,12 @@ object ArangoSchema {
java.util.Arrays.asList("entity_id"),
new PersistentIndexOptions()
)
+ // Compound index for retire query: FILTER entity_id IN @ids AND deleted_rev == null
+ // Mirrors the nodes compound index — avoids post-filtering all historical claim versions.
+ claims.ensurePersistentIndex(
+ java.util.Arrays.asList("entity_id", "deleted_rev"),
+ new PersistentIndexOptions()
+ )
claims.ensurePersistentIndex(
java.util.Arrays.asList("status"),
new PersistentIndexOptions()
diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
index d0c51ae0..7c2e64ba 100644
--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
@@ -8,6 +8,8 @@ import cats.effect.IO
import cats.syntax.traverse._
import io.circe.Json
import io.circe.syntax._
+import org.typelevel.log4cats.Logger
+import org.typelevel.log4cats.slf4j.Slf4jLogger
import scala.concurrent.duration._
@@ -18,8 +20,18 @@ import ix.memory.model._
* instead of individual AQL queries.
*/
class BulkWriteApi(client: ArangoClient) {
+ private type BulkDoc = java.util.Map[String, AnyRef]
private val CurrentRevisionKey = "current"
private val SourceGraphRevisionKey = "source-graph"
+ private val DefaultChunkSize = 100
+ private val DefaultChunkPayloadBytes = 4 * 1024 * 1024
+ private val MaxNodeDocs = 10000
+ private val MaxEdgeDocs = 20000
+ private val MaxClaimDocs = 10000
+ private val MaxPatchDocs = 250
+ private val SlowChunkMs = 2000L
+ private val debugBulkWrites = sys.env.get("IX_DEBUG").exists(v => v.nonEmpty && v != "0" && !v.equalsIgnoreCase("false"))
+ private val logger: Logger[IO] = Slf4jLogger.getLoggerFromName[IO]("ix.bulk-write")
private def withRetry[A](action: IO[A], maxAttempts: Int = 3, delay: FiniteDuration = 500.millis): IO[A] = {
action.handleErrorWith { err =>
@@ -28,30 +40,87 @@ class BulkWriteApi(client: ArangoClient) {
}
}
+ private def chunkDocs[T](docs: Seq[T], maxDocs: Int): Seq[Seq[T]] =
+ docs.grouped(maxDocs).toVector
+
+ private def insertCollectionChunked(
+ collection: String,
+ docs: Seq[BulkDoc],
+ maxDocs: Int
+ )(insert: Seq[BulkDoc] => IO[Int]): IO[Long] =
+ if (docs.isEmpty) IO.pure(0L)
+ else {
+ val batches = chunkDocs(docs, maxDocs)
+ batches.foldLeft(IO.pure(0L)) { case (accIO, batch) =>
+ for {
+ acc <- accIO
+ ms <- timedUnit(withRetry(insert(batch)))
+ _ <- if (debugBulkWrites)
+ logger.info(s"[bulk-write] collection=$collection docs=${batch.size} insertMs=$ms")
+ else IO.unit
+ } yield acc + ms
+ }
+ }
+
def commitBatch(
fileBatches: Vector[FileBatch],
- baseRev: Long
+ baseRev: Long,
+ skipTombstone: Boolean = false
): IO[CommitResult] = {
val newRev = baseRev + 1L
- val allNodes = fileBatches.flatMap(_.nodeDocuments(newRev))
- val allEdges = fileBatches.flatMap(_.edgeDocuments(newRev))
- val allClaims = fileBatches.flatMap(_.claimDocuments(newRev))
- val allPatches = fileBatches.map(_.patchDocument(newRev))
-
- // Collect all logical_ids being upserted for tombstoning
- val logicalIds = allNodes.flatMap { doc =>
- Option(doc.get("logical_id")).map(_.toString)
- }
-
for {
- _ <- tombstoneExistingNodes(logicalIds, newRev)
- _ <- retireOldClaims(fileBatches, newRev)
- _ <- withRetry(client.bulkInsert("nodes", allNodes))
- _ <- withRetry(client.bulkInsertEdges("edges", allEdges))
- _ <- withRetry(client.bulkInsert("claims", allClaims))
- _ <- withRetry(client.bulkInsert("patches", allPatches))
- _ <- updateRevision(newRev)
+ // Build all document maps in a single IO.blocking pass — one traversal instead of four.
+ docsAndBuildMs <- timed {
+ IO.blocking {
+ val nodesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
+ val edgesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
+ val claimsBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
+ val patchesBuf = Vector.newBuilder[java.util.Map[String, AnyRef]]
+ fileBatches.foreach { fb =>
+ nodesBuf ++= fb.nodeDocuments(newRev)
+ edgesBuf ++= fb.edgeDocuments(newRev)
+ claimsBuf ++= fb.claimDocuments(newRev)
+ patchesBuf += fb.patchDocument(newRev)
+ }
+ (nodesBuf.result(), edgesBuf.result(), claimsBuf.result(), patchesBuf.result())
+ }
+ }
+ (docs, buildDocsMs) = docsAndBuildMs
+ (allNodes, allEdges, allClaims, allPatches) = docs
+ logicalIds = allNodes.flatMap(doc => Option(doc.get("logical_id")).map(_.toString))
+ chunkSummary = s"files=${fileBatches.size} nodes=${allNodes.size} edges=${allEdges.size} claims=${allClaims.size} patches=${allPatches.size}"
+ // Skip tombstone/retire on first ingest — DB is empty, no existing docs to update.
+ tombstoneAndRetireMs <-
+ if (skipTombstone || baseRev == 0) IO.pure((0L, 0L))
+ else timedUnit(tombstoneExistingNodes(logicalIds, newRev))
+ .both(timedUnit(retireOldClaims(fileBatches, newRev)))
+ (tombstoneMs: Long, retireClaimsMs: Long) = tombstoneAndRetireMs
+ // Run all four collection inserts concurrently — they write to independent collections
+ // with no referential-integrity enforcement between them. Wall time drops from the
+ // sum of all four to the max of any one (typically nodes or edges).
+ allInsertMs <- insertCollectionChunked("nodes", allNodes, MaxNodeDocs)(docs => client.bulkInsert("nodes", docs))
+ .both(insertCollectionChunked("edges", allEdges, MaxEdgeDocs)(docs => client.bulkInsertEdges("edges", docs)))
+ .both(insertCollectionChunked("claims", allClaims, MaxClaimDocs)(docs => client.bulkInsert("claims", docs)))
+ .both(insertCollectionChunked("patches", allPatches, MaxPatchDocs)(docs => client.bulkInsert("patches", docs)))
+ .map { case (((n, e), c), p) => (n, e, c, p) }
+ (insertNodesMs, insertEdgesMs, insertClaimsMs, insertPatchesMs) = allInsertMs
+ updateRevMs <- timedUnit(updateRevision(newRev))
+ totalMs = buildDocsMs + tombstoneMs + retireClaimsMs + insertNodesMs + insertEdgesMs + insertClaimsMs + insertPatchesMs + updateRevMs
+ _ <- logChunkTiming(
+ s"rev=$newRev $chunkSummary",
+ totalMs,
+ List(
+ "buildDocs" -> buildDocsMs,
+ "tombstoneNodes" -> tombstoneMs,
+ "retireClaims" -> retireClaimsMs,
+ "insertNodes" -> insertNodesMs,
+ "insertEdges" -> insertEdgesMs,
+ "insertClaims" -> insertClaimsMs,
+ "insertPatches" -> insertPatchesMs,
+ "updateRevision" -> updateRevMs,
+ )
+ )
} yield CommitResult(Rev(newRev), CommitStatus.Ok)
}
@@ -74,10 +143,7 @@ class BulkWriteApi(client: ArangoClient) {
}
}
- private def retireOldClaims(fileBatches: Vector[FileBatch], newRev: Long): IO[Unit] = {
- val entityIds = fileBatches.flatMap(_.patch.ops.collect {
- case PatchOp.UpsertNode(id, _, _, _) => id.value.toString
- }).distinct
+ private def retireOldClaimsByEntityIds(entityIds: Seq[String], newRev: Long): IO[Unit] =
if (entityIds.isEmpty) IO.unit
else {
val idList = new java.util.ArrayList[String](entityIds.size)
@@ -93,41 +159,124 @@ class BulkWriteApi(client: ArangoClient) {
)
)
}
- }
+
+ private def retireOldClaims(fileBatches: Vector[FileBatch], newRev: Long): IO[Unit] =
+ retireOldClaimsByEntityIds(
+ fileBatches.flatMap(_.patch.ops.collect {
+ case PatchOp.UpsertNode(id, _, _, _) => id.value.toString
+ }).distinct,
+ newRev
+ )
/**
* Commit file batches in chunks. Each chunk gets its own revision.
* If a chunk fails, prior chunks are preserved (partial progress).
+ *
+ * When multiple chunks exist and baseRev > 0 (incremental update), tombstone+retire
+ * runs once upfront for all files in parallel rather than once per chunk sequentially.
+ * This reduces 2*N AQL UPDATE scans to 2 — e.g. 160 → 2 for Kubernetes.
*/
def commitBatchChunked(
fileBatches: Vector[FileBatch],
baseRev: Long,
- chunkSize: Int = 100
+ chunkSize: Int = DefaultChunkSize,
+ maxPayloadBytes: Int = DefaultChunkPayloadBytes
): IO[CommitResult] = {
if (fileBatches.isEmpty)
return IO.pure(CommitResult(Rev(baseRev), CommitStatus.Ok))
- val chunks = fileBatches.grouped(chunkSize).toVector
+ val chunks = chunkForCommit(fileBatches, chunkSize, maxPayloadBytes)
+ val multiChunk = chunks.size > 1 && baseRev > 0
- chunks.foldLeft(IO.pure(baseRev)) { case (revIO, chunk) =>
- revIO.flatMap { currentRev =>
- commitBatch(chunk, currentRev).map(_.newRev.value)
+ // Pre-tombstone: collect all entity IDs upfront and retire/tombstone in one parallel pass.
+ // Uses baseRev+1 as the tombstone rev, which is the same rev chunk 1 will use for its inserts.
+ val preTombstoneIO: IO[Unit] =
+ if (!multiChunk) IO.unit
+ else {
+ val allIds = fileBatches.flatMap(_.patch.ops.collect {
+ case PatchOp.UpsertNode(id, _, _, _) => id.value.toString
+ }).distinct
+ val preTombRev = baseRev + 1L
+ timedUnit(tombstoneExistingNodes(allIds.toVector, preTombRev))
+ .both(timedUnit(retireOldClaimsByEntityIds(allIds, preTombRev)))
+ .flatMap { case (tombMs, retireMs) =>
+ if (debugBulkWrites)
+ logger.info(s"[bulk-write] pre-tombstone files=${fileBatches.size} tombstoneMs=$tombMs retireMs=$retireMs")
+ else IO.unit
+ }
+ }
+
+ for {
+ _ <- preTombstoneIO
+ finalRev <- chunks.foldLeft(IO.pure(baseRev)) { case (revIO, chunk) =>
+ revIO.flatMap { currentRev =>
+ commitBatch(chunk, currentRev, skipTombstone = multiChunk).map(_.newRev.value)
+ }
}
- }.map(finalRev => CommitResult(Rev(finalRev), CommitStatus.Ok))
+ } yield CommitResult(Rev(finalRev), CommitStatus.Ok)
}
- private def updateRevision(newRev: Long): IO[Unit] =
- client.execute(
- """FOR key IN @keys
- | UPSERT { _key: key }
- | INSERT { _key: key, rev: @rev }
- | UPDATE { rev: @rev }
- | IN revisions""".stripMargin,
- Map(
- "keys" -> Array(CurrentRevisionKey, SourceGraphRevisionKey).asInstanceOf[AnyRef],
- "rev" -> Long.box(newRev).asInstanceOf[AnyRef]
- )
- )
+ private def updateRevision(newRev: Long): IO[Unit] = {
+ // Use Document API (replace mode) instead of AQL UPSERT.
+ // AQL UPSERT produces write-write conflicts when multiple concurrent HTTP
+ // requests all try to update the same "current" revision key simultaneously.
+ // The Document API replace is atomic at the document level — no conflict possible.
+ val doc1 = new java.util.HashMap[String, AnyRef]()
+ doc1.put("_key", CurrentRevisionKey)
+ doc1.put("rev", Long.box(newRev))
+ val doc2 = new java.util.HashMap[String, AnyRef]()
+ doc2.put("_key", SourceGraphRevisionKey)
+ doc2.put("rev", Long.box(newRev))
+ client.bulkInsert("revisions", Seq(doc1, doc2)).void
+ }
+
+ private def chunkForCommit(
+ fileBatches: Vector[FileBatch],
+ maxFiles: Int,
+ maxPayloadBytes: Int
+ ): Vector[Vector[FileBatch]] = {
+ val chunks = Vector.newBuilder[Vector[FileBatch]]
+ var current = Vector.empty[FileBatch]
+ var currentBytes = 0
+
+ fileBatches.foreach { batch =>
+ val batchBytes = batch.estimatedPayloadBytes
+ val exceedsPayload = current.nonEmpty && currentBytes + batchBytes > maxPayloadBytes
+ val exceedsCount = current.size >= maxFiles
+ if (exceedsPayload || exceedsCount) {
+ chunks += current
+ current = Vector.empty
+ currentBytes = 0
+ }
+ current = current :+ batch
+ currentBytes += batchBytes
+ }
+
+ if (current.nonEmpty) {
+ chunks += current
+ }
+
+ chunks.result()
+ }
+
+ private def timed[A](ioa: IO[A]): IO[(A, Long)] =
+ for {
+ start <- IO.monotonic
+ result <- ioa
+ end <- IO.monotonic
+ } yield (result, (end - start).toMillis)
+
+ private def timedUnit[A](ioa: IO[A]): IO[Long] =
+ timed(ioa).map(_._2)
+
+ private def logChunkTiming(summary: String, totalMs: Long, stages: List[(String, Long)]): IO[Unit] = {
+ val shouldLog = debugBulkWrites || totalMs >= SlowChunkMs
+ if (!shouldLog) IO.unit
+ else {
+ val stageSummary = stages.map { case (name, ms) => s"$name=${ms}ms" }.mkString(" ")
+ logger.info(s"$summary total=${totalMs}ms $stageSummary")
+ }
+ }
}
/**
@@ -140,32 +289,47 @@ case class FileBatch(
patch: GraphPatch,
provenance: java.util.Map[String, AnyRef]
) {
- private val mapper = new com.fasterxml.jackson.databind.ObjectMapper()
-
- private def parseToJavaMap(jsonStr: String): java.util.Map[String, AnyRef] =
- mapper.readValue(jsonStr, classOf[java.util.Map[String, AnyRef]])
+ // Direct Circe-to-Java conversion — no string serialization round-trip.
+ private def jsonToJava(json: Json): AnyRef = json.fold(
+ jsonNull = null.asInstanceOf[AnyRef],
+ jsonBoolean = b => java.lang.Boolean.valueOf(b),
+ jsonNumber = n => java.lang.Double.valueOf(n.toDouble),
+ jsonString = s => s,
+ jsonArray = arr => {
+ val list = new java.util.ArrayList[AnyRef](arr.size)
+ arr.foreach(j => list.add(jsonToJava(j)))
+ list
+ },
+ jsonObject = obj => {
+ val map = new java.util.LinkedHashMap[String, AnyRef](obj.size)
+ obj.toMap.foreach { case (k, v) => map.put(k, jsonToJava(v)) }
+ map
+ }
+ )
- private def jsonToJava(json: Json): AnyRef =
- mapper.readValue(json.noSpaces, classOf[AnyRef])
+ private def attrsToJavaMap(attrs: Map[String, Json]): java.util.Map[String, AnyRef] = {
+ val result = new java.util.HashMap[String, AnyRef](attrs.size)
+ attrs.foreach { case (k, v) => result.put(k, jsonToJava(v)) }
+ result
+ }
private def nodeKindToString(nk: NodeKind): String =
nk.asJson.asString.getOrElse(nk.toString)
lazy val estimatedPayloadBytes: Int =
- (patch.asJson.noSpaces.getBytes(StandardCharsets.UTF_8).length * 4) + 1024
+ patch.asJson.noSpaces.getBytes(StandardCharsets.UTF_8).length + 1024
def nodeDocuments(rev: Long): Vector[java.util.Map[String, AnyRef]] = {
val now = Instant.now().toString
patch.ops.collect { case PatchOp.UpsertNode(id, kind, name, attrs) =>
val logicalId = id.value.toString
- val attrsJson = Json.obj(attrs.toSeq.map { case (k, v) => k -> v }: _*).noSpaces
val doc = new java.util.HashMap[String, AnyRef]()
doc.put("_key", s"${logicalId}_${rev}")
doc.put("logical_id", logicalId)
doc.put("id", logicalId)
doc.put("kind", nodeKindToString(kind))
doc.put("name", name)
- doc.put("attrs", parseToJavaMap(attrsJson))
+ doc.put("attrs", attrsToJavaMap(attrs))
doc.put("provenance", provenance)
doc.put("created_rev", Long.box(rev))
doc.put("deleted_rev", null)
@@ -178,7 +342,6 @@ case class FileBatch(
def edgeDocuments(rev: Long): Vector[java.util.Map[String, AnyRef]] = {
val now = Instant.now().toString
patch.ops.collect { case PatchOp.UpsertEdge(id, src, dst, predicate, attrs) =>
- val attrsJson = Json.obj(attrs.toSeq.map { case (k, v) => k -> v }: _*).noSpaces
val doc = new java.util.HashMap[String, AnyRef]()
doc.put("_key", id.value.toString)
doc.put("_from", s"nodes/${src.value}")
@@ -187,7 +350,7 @@ case class FileBatch(
doc.put("src", src.value.toString)
doc.put("dst", dst.value.toString)
doc.put("predicate", predicate.value)
- doc.put("attrs", parseToJavaMap(attrsJson))
+ doc.put("attrs", attrsToJavaMap(attrs))
doc.put("created_rev", Long.box(rev))
doc.put("deleted_rev", null)
doc.put("provenance", provenance)
@@ -214,12 +377,68 @@ case class FileBatch(
}
def patchDocument(rev: Long): java.util.Map[String, AnyRef] = {
- val patchJson = patch.asJson.noSpaces
+ val source = new java.util.HashMap[String, AnyRef]()
+ source.put("uri", patch.source.uri)
+ source.put("sourceHash", patch.source.sourceHash.orNull)
+ source.put("extractor", patch.source.extractor)
+ source.put("sourceType", patch.source.sourceType.asJson.asString.getOrElse(patch.source.sourceType.toString))
+
+ val replaces = new java.util.ArrayList[String](patch.replaces.size)
+ patch.replaces.foreach(id => replaces.add(id.value.toString))
+
+ val entityIds = new java.util.LinkedHashSet[String]()
+ var nodeOpCount = 0
+ var edgeOpCount = 0
+ var claimOpCount = 0
+ patch.ops.foreach {
+ case PatchOp.UpsertNode(id, _, _, _) =>
+ nodeOpCount += 1
+ entityIds.add(id.value.toString)
+ case PatchOp.DeleteNode(id) =>
+ nodeOpCount += 1
+ entityIds.add(id.value.toString)
+ case PatchOp.UpsertEdge(id, _, _, _, _) =>
+ edgeOpCount += 1
+ entityIds.add(id.value.toString)
+ case PatchOp.DeleteEdge(id) =>
+ edgeOpCount += 1
+ entityIds.add(id.value.toString)
+ case PatchOp.AssertClaim(entityId, _, _, _, _) =>
+ claimOpCount += 1
+ entityIds.add(entityId.value.toString)
+ case PatchOp.RetractClaim(claimId) =>
+ claimOpCount += 1
+ entityIds.add(claimId.value.toString)
+ }
+
+ val entityIdList = new java.util.ArrayList[String](entityIds.size)
+ val entityIdIter = entityIds.iterator()
+ while (entityIdIter.hasNext) {
+ entityIdList.add(entityIdIter.next())
+ }
+
+ val data = new java.util.HashMap[String, AnyRef]()
+ data.put("patchId", patch.patchId.value.toString)
+ data.put("actor", patch.actor)
+ data.put("timestamp", patch.timestamp.toString)
+ data.put("source", source)
+ data.put("baseRev", Long.box(patch.baseRev.value))
+ data.put("replaces", replaces)
+ data.put("intent", patch.intent.orNull)
+ data.put("entityIds", entityIdList)
+ data.put("opCount", Int.box(patch.ops.size))
+ data.put("nodeOpCount", Int.box(nodeOpCount))
+ data.put("edgeOpCount", Int.box(edgeOpCount))
+ data.put("claimOpCount", Int.box(claimOpCount))
+
val doc = new java.util.HashMap[String, AnyRef]()
doc.put("_key", patch.patchId.value.toString)
doc.put("patch_id", patch.patchId.value.toString)
doc.put("rev", Long.box(rev))
- doc.put("data", mapper.readValue(patchJson, classOf[java.util.Map[String, AnyRef]]))
+ // Bulk ingest only needs patch metadata plus touched-entity IDs for provenance
+ // and source-hash lookups. Storing full ops here duplicates the graph payload
+ // and significantly inflates Kubernetes save latency.
+ doc.put("data", data)
doc: java.util.Map[String, AnyRef]
}
}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
deleted file mode 100644
index d94e39fe..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
+++ /dev/null
@@ -1,245 +0,0 @@
-package ix.memory.ingestion
-
-import java.nio.file.{Files, Path}
-
-import cats.effect.IO
-import cats.syntax.all._
-import cats.effect.implicits._
-import io.circe.Json
-import io.circe.syntax._
-import org.typelevel.log4cats.slf4j.Slf4jLogger
-import org.typelevel.log4cats.Logger
-
-import ix.memory.db.{BulkWriteApi, FileBatch, GraphQueryApi}
-import ix.memory.model._
-
-private[ingestion] sealed trait ParseOutcome
-private[ingestion] case class Parsed(batch: FileBatch) extends ParseOutcome
-private[ingestion] case class Skipped(reason: String) extends ParseOutcome
-
-/**
- * High-performance ingestion service that:
- * 1. Discovers files (with optional language filter)
- * 2. Parses files in parallel (bounded by CPU cores)
- * 3. Batches parsed results
- * 4. Bulk-inserts via Document API
- */
-class BulkIngestionService(
- parserRouter: ParserRouter,
- bulkWriteApi: BulkWriteApi,
- queryApi: GraphQueryApi
-) {
-
- private val parallelism = Runtime.getRuntime.availableProcessors.max(2)
- private val MaxFileBytes: Long = 1024 * 1024 // 1 MB
- private val ParseBatchSize = 100
- private val MaxCommitFiles = 100
- private val MaxCommitPayloadBytes = 1024 * 1024
- private val mapper = new com.fasterxml.jackson.databind.ObjectMapper()
- private val logger: Logger[IO] = Slf4jLogger.getLoggerFromName[IO]("ix.ingestion")
-
- /**
- * Ingest files under a path using parallel parsing and bulk writes.
- */
- def ingestPath(
- path: Path,
- language: Option[String],
- recursive: Boolean,
- onProgress: IngestionProgress => IO[Unit] = _ => IO.unit,
- force: Boolean = false
- ): IO[IngestionResult] = {
- for {
- files <- discoverFiles(path, language, recursive)
- _ <- onProgress(IngestionProgress(files.size, 0, 0, 0))
- latestRev <- queryApi.getLatestRev
- state <- files.grouped(ParseBatchSize).toVector.foldLeft(IO.pure(BulkIngestionState.empty(latestRev.value))) {
- case (stateIO, batch) =>
- stateIO.flatMap(processBatch(batch, force, files.size, onProgress, _))
- }
- } yield IngestionResult(
- filesProcessed = files.size,
- patchesApplied = state.patchesApplied,
- filesSkipped = state.skipped.total,
- entitiesCreated = state.entitiesCreated,
- latestRev = Rev(state.currentRev),
- skipReasons = state.skipped
- )
- }
-
- private def processBatch(
- files: List[Path],
- force: Boolean,
- discovered: Int,
- onProgress: IngestionProgress => IO[Unit],
- state: BulkIngestionState
- ): IO[BulkIngestionState] = {
- for {
- hashMap <- if (force) IO.pure(Map.empty[String, String]) else loadExistingHashes(files)
- outcomes <- files.parTraverseN(parallelism) { f =>
- parseFile(f, hashMap).handleErrorWith { err =>
- logger.warn(s"Parse error: ${f.toString} - ${err.getMessage}") *>
- IO.pure(Skipped("parseError"): ParseOutcome)
- }
- }
- validBatches = outcomes.collect { case Parsed(batch) => batch }.toVector
- batchSkipped = SkipReasons(
- unchanged = outcomes.count { case Skipped("unchanged") => true; case _ => false },
- emptyFile = outcomes.count { case Skipped("emptyFile") => true; case _ => false },
- parseError = outcomes.count { case Skipped("parseError") => true; case _ => false },
- tooLarge = outcomes.count { case Skipped("tooLarge") => true; case _ => false }
- )
- parsedFiles = state.filesParsed + outcomes.size
- commitChunks = chunkForCommit(validBatches)
- totalChunks = state.totalChunksPlanned + commitChunks.size
- _ <- onProgress(IngestionProgress(discovered, parsedFiles, state.chunksWritten, totalChunks))
- committed <- commitChunks.foldLeft(IO.pure((state.currentRev, state.chunksWritten))) {
- case (revIO, chunk) =>
- revIO.flatMap { case (currentRev, chunksWritten) =>
- bulkWriteApi.commitBatch(chunk, currentRev).flatMap { result =>
- val updatedChunks = chunksWritten + 1
- onProgress(IngestionProgress(discovered, parsedFiles, updatedChunks, totalChunks)) *>
- IO.pure((result.newRev.value, updatedChunks))
- }
- }
- }
- entitiesCreated = validBatches.flatMap(_.patch.ops.collect { case _: PatchOp.UpsertNode => 1 }).size
- } yield state.copy(
- currentRev = committed._1,
- filesParsed = parsedFiles,
- patchesApplied = state.patchesApplied + validBatches.size,
- chunksWritten = committed._2,
- totalChunksPlanned = totalChunks,
- entitiesCreated = state.entitiesCreated + entitiesCreated,
- skipped = combineSkipReasons(state.skipped, batchSkipped)
- )
- }
-
- /**
- * Parse a single file into a FileBatch (or None if unchanged).
- */
- private def parseFile(filePath: Path, hashMap: Map[String, String]): IO[ParseOutcome] = {
- for {
- fileSize <- IO.blocking(Files.size(filePath))
- result <- if (fileSize > MaxFileBytes) IO.pure(Skipped("tooLarge"): ParseOutcome)
- else if (fileSize == 0L) IO.pure(Skipped("emptyFile"): ParseOutcome)
- else {
- for {
- bytes <- IO.blocking(Files.readAllBytes(filePath))
- hash = sha256Bytes(bytes)
- outcome <- hashMap.get(filePath.toString) match {
- case Some(existingHash) if existingHash == hash =>
- IO.pure(Skipped("unchanged"): ParseOutcome)
- case _ =>
- for {
- source <- IO.blocking {
- try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)
- catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }
- }
- parserOpt = parserRouter.parserFor(filePath.toString)
- parseResult = parserOpt match {
- case Some(p) => p.parse(filePath.getFileName.toString, source)
- case None => GenericTextFallback.parse(filePath.getFileName.toString, source)
- }
- patch = GraphPatchBuilder.build(filePath.toString, Some(hash), parseResult)
- _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))
- provenance = buildProvenanceMap(patch)
- } yield Parsed(FileBatch(filePath.toString, Some(hash), patch, provenance)): ParseOutcome
- }
- } yield outcome
- }
- } yield result
- }
-
- private def loadExistingHashes(files: Seq[Path]): IO[Map[String, String]] = {
- val paths = files.map(_.toString)
- queryApi.getSourceHashes(paths)
- }
-
- private def buildProvenanceMap(patch: GraphPatch): java.util.Map[String, AnyRef] = {
- val json = Json.obj(
- "source_uri" -> Json.fromString(patch.source.uri),
- "source_hash" -> patch.source.sourceHash.fold(Json.Null)(Json.fromString),
- "extractor" -> Json.fromString(patch.source.extractor),
- "source_type" -> Json.fromString(patch.source.sourceType.asJson.asString.getOrElse("code")),
- "observed_at" -> Json.fromString(patch.timestamp.toString)
- )
- mapper.readValue(json.noSpaces, classOf[java.util.Map[String, AnyRef]])
- }
-
- private def discoverFiles(
- path: Path,
- language: Option[String],
- recursive: Boolean
- ): IO[List[Path]] = IO.blocking {
- val extensions = FileDiscovery.extensionsFor(language)
- if (Files.isRegularFile(path)) {
- if (language.isEmpty || FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty
- } else if (Files.isDirectory(path)) {
- // Try git ls-files first (respects .gitignore), fall back to manual walk
- FileDiscovery.tryGitLsFiles(path, recursive, extensions)
- .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))
- } else List.empty
- }
-
- private def chunkForCommit(batches: Vector[FileBatch]): Vector[Vector[FileBatch]] = {
- val chunks = Vector.newBuilder[Vector[FileBatch]]
- var current = Vector.empty[FileBatch]
- var currentBytes = 0
-
- batches.foreach { batch =>
- val batchBytes = batch.estimatedPayloadBytes
- val exceedsPayload = current.nonEmpty && currentBytes + batchBytes > MaxCommitPayloadBytes
- val exceedsCount = current.size >= MaxCommitFiles
- if (exceedsPayload || exceedsCount) {
- chunks += current
- current = Vector.empty
- currentBytes = 0
- }
- current = current :+ batch
- currentBytes += batchBytes
- }
-
- if (current.nonEmpty) {
- chunks += current
- }
-
- chunks.result()
- }
-
- private def combineSkipReasons(left: SkipReasons, right: SkipReasons): SkipReasons =
- SkipReasons(
- unchanged = left.unchanged + right.unchanged,
- emptyFile = left.emptyFile + right.emptyFile,
- parseError = left.parseError + right.parseError,
- tooLarge = left.tooLarge + right.tooLarge
- )
-
- private def sha256Bytes(bytes: Array[Byte]): String = {
- val md = java.security.MessageDigest.getInstance("SHA-256")
- md.digest(bytes).map("%02x".format(_)).mkString
- }
-
-}
-
-private[ingestion] final case class BulkIngestionState(
- currentRev: Long,
- filesParsed: Int,
- patchesApplied: Int,
- chunksWritten: Int,
- totalChunksPlanned: Int,
- entitiesCreated: Int,
- skipped: SkipReasons
-)
-
-private[ingestion] object BulkIngestionState {
- def empty(baseRev: Long): BulkIngestionState =
- BulkIngestionState(
- currentRev = baseRev,
- filesParsed = 0,
- patchesApplied = 0,
- chunksWritten = 0,
- totalChunksPlanned = 0,
- entitiesCreated = 0,
- skipped = SkipReasons()
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala b/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala
deleted file mode 100644
index 26e1d212..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala
+++ /dev/null
@@ -1,120 +0,0 @@
-package ix.memory.ingestion
-
-import java.nio.file.{Files, Path}
-
-/**
- * Discovers source files for ingestion.
- *
- * Primary strategy: `git ls-files` — respects .gitignore, fast, works for any repo.
- * Fallback: manual file walk with hardcoded ignore list (for non-git directories).
- */
-object FileDiscovery {
-
- private val defaultExtensions = Set(
- ".py", ".ts", ".tsx",
- ".scala", ".sc",
- ".json", ".yaml", ".yml", ".toml",
- ".md"
- )
-
- def extensionsFor(language: Option[String]): Set[String] = language match {
- case Some("python") => Set(".py")
- case Some("typescript") => Set(".ts", ".tsx")
- case Some("javascript") => Set(".js", ".jsx", ".mjs", ".cjs")
- case Some("scala") => Set(".scala", ".sc")
- case Some("java") => Set(".java")
- case Some("go") => Set(".go")
- case Some("rust") => Set(".rs")
- case Some("ruby") => Set(".rb")
- case Some("config") => Set(".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".properties", ".xml")
- case Some("markdown") => Set(".md", ".mdx")
- case _ => defaultExtensions
- }
-
- def matchesFilter(file: Path, extensions: Set[String]): Boolean = {
- val name = file.getFileName.toString
- extensions.exists(name.endsWith)
- }
-
- /**
- * Use `git ls-files` to discover tracked files. Returns None if not a git repo
- * or git is not available.
- */
- def tryGitLsFiles(dir: Path, recursive: Boolean, extensions: Set[String]): Option[List[Path]] = {
- try {
- val pb = new ProcessBuilder("git", "ls-files", "--cached", "--others", "--exclude-standard")
- pb.directory(dir.toFile)
- pb.redirectErrorStream(true)
- val proc = pb.start()
- val reader = new java.io.BufferedReader(
- new java.io.InputStreamReader(proc.getInputStream, java.nio.charset.StandardCharsets.UTF_8)
- )
- val result = scala.collection.mutable.ListBuffer.empty[Path]
-
- try {
- var line = reader.readLine()
- while (line != null) {
- if (line.nonEmpty) {
- val p = dir.resolve(line).normalize()
- val isUnder = if (recursive) true else p.getParent == dir
- if (isUnder && Files.isRegularFile(p) && matchesFilter(p, extensions)) {
- result += p
- }
- }
- line = reader.readLine()
- }
- } finally {
- reader.close()
- }
-
- val exitCode = proc.waitFor()
- if (exitCode != 0) None else Some(result.toList)
- } catch {
- case _: Throwable => None
- }
- }
-
- /**
- * Fallback: walk the directory tree, skipping common non-source directories.
- */
- def walkFiles(dir: Path, recursive: Boolean, extensions: Set[String]): List[Path] = {
- import scala.jdk.CollectionConverters._
-
- if (!recursive) {
- val stream = Files.list(dir)
- try {
- stream.iterator().asScala
- .filter(Files.isRegularFile(_))
- .filter(matchesFilter(_, extensions))
- .toList
- } finally stream.close()
- } else {
- // Directories to skip when not using git
- val ignoredDirs = Set(
- "node_modules", ".git", ".hg", ".svn",
- "target", "dist", "build", "out", ".next",
- "__pycache__", ".tox", ".venv", "venv", ".mypy_cache", ".pytest_cache",
- ".gradle", ".idea", ".vscode", ".settings", ".vs",
- ".cache", ".parcel-cache", "coverage", ".nyc_output",
- "vendor", "Pods", ".dart_tool", ".pub-cache",
- "bin", "obj", "pkg"
- )
- val result = scala.collection.mutable.ListBuffer.empty[Path]
- Files.walkFileTree(dir, new java.nio.file.SimpleFileVisitor[Path] {
- override def preVisitDirectory(d: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
- val name = d.getFileName.toString
- if (d != dir && (ignoredDirs.contains(name) || name.startsWith(".")))
- java.nio.file.FileVisitResult.SKIP_SUBTREE
- else
- java.nio.file.FileVisitResult.CONTINUE
- }
- override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
- if (matchesFilter(file, extensions))
- result += file
- java.nio.file.FileVisitResult.CONTINUE
- }
- })
- result.toList
- }
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala b/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala
deleted file mode 100644
index 8a7c4baf..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-package ix.memory.ingestion
-
-import java.security.MessageDigest
-
-object Fingerprint {
-
- /** Compute a normalized content fingerprint from signature + body.
- * Returns a 32-char hex string (first 16 bytes of SHA-256).
- */
- def compute(signature: String, body: String): String = {
- val normalized = (signature.trim + "\n" + body.trim)
- .replaceAll("\\s+", " ")
- .trim
- val digest = MessageDigest.getInstance("SHA-256")
- digest.digest(normalized.getBytes("UTF-8"))
- .take(16)
- .map("%02x".format(_))
- .mkString
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/GenericTextFallback.scala b/memory-layer/src/main/scala/ix/memory/ingestion/GenericTextFallback.scala
deleted file mode 100644
index 29d670ea..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/GenericTextFallback.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-package ix.memory.ingestion
-
-import io.circe.Json
-import ix.memory.model.NodeKind
-
-object GenericTextFallback {
- private val PreviewLimit = 240
-
- def parse(fileName: String, source: String): ParseResult = {
- val lines = if (source.isEmpty) 1 else source.count(_ == '\n') + 1
- val preview = source.linesIterator.find(_.trim.nonEmpty).map(_.trim.take(PreviewLimit))
-
- val attrs = Map(
- "file_name" -> Json.fromString(fileName),
- "line_count" -> Json.fromInt(lines)
- ) ++ preview.filter(_.nonEmpty).map(value => "preview" -> Json.fromString(value))
-
- ParseResult(
- entities = Vector(
- ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = attrs,
- lineStart = 1,
- lineEnd = lines,
- contentFingerprint = Some(Fingerprint.compute(fileName, source))
- )
- ),
- relationships = Vector.empty
- )
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
deleted file mode 100644
index 3a903acd..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
+++ /dev/null
@@ -1,100 +0,0 @@
-package ix.memory.ingestion
-
-import java.time.Instant
-import java.util.UUID
-
-import io.circe.Json
-
-import ix.memory.model._
-
-object GraphPatchBuilder {
-
- /**
- * Build a GraphPatch from a ParseResult and file metadata.
- *
- * Uses deterministic UUID generation (UUID v3 (MD5-based nameUUID)) so that
- * re-parsing the same file produces the same node/edge IDs, enabling
- * idempotent upserts.
- */
- def build(
- filePath: String,
- sourceHash: Option[String],
- parseResult: ParseResult
- ): GraphPatch = {
-
- def nodeIdFor(name: String): NodeId =
- NodeId(UUID.nameUUIDFromBytes(s"$filePath:$name".getBytes("UTF-8")))
-
- def edgeIdFor(src: String, dst: String, predicate: String): EdgeId =
- EdgeId(UUID.nameUUIDFromBytes(s"$filePath:$src:$dst:$predicate".getBytes("UTF-8")))
-
- val extractor = if (filePath.endsWith(".py")) "tree-sitter-python/1.0"
- else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) "typescript-parser/1.0"
- else if (filePath.endsWith(".scala") || filePath.endsWith(".sc")) "scala-parser/1.0"
- else if (filePath.endsWith(".go")) "tree-sitter-go/1.0"
- else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) "config-parser/1.0"
- else if (filePath.endsWith(".md")) "markdown-parser/1.0"
- else "unknown-parser/1.0"
-
- val nodeOps = parseResult.entities.map { e =>
- val baseAttrs = e.attrs +
- ("line_start" -> Json.fromInt(e.lineStart)) +
- ("line_end" -> Json.fromInt(e.lineEnd))
- val attrsWithFp = e.contentFingerprint match {
- case Some(fp) => baseAttrs + ("content_fingerprint" -> Json.fromString(fp))
- case None => baseAttrs
- }
- PatchOp.UpsertNode(nodeIdFor(e.name), e.kind, e.name, attrsWithFp)
- }
-
- val edgeOps = parseResult.relationships.map { r =>
- PatchOp.UpsertEdge(
- edgeIdFor(r.srcName, r.dstName, r.predicate),
- nodeIdFor(r.srcName),
- nodeIdFor(r.dstName),
- EdgePredicate(r.predicate),
- Map.empty
- )
- }
-
- // Generate claims so the confidence/conflict engine has data to operate on.
- val relationshipClaims = parseResult.relationships.map { r =>
- PatchOp.AssertClaim(
- entityId = nodeIdFor(r.srcName),
- field = s"${r.predicate.toLowerCase}:${r.dstName}",
- value = Json.fromString(r.dstName),
- confidence = None // will be computed at query time
- )
- }
-
- val attributeClaims = parseResult.entities.flatMap { e =>
- e.attrs.flatMap { case (key, value) =>
- Vector(PatchOp.AssertClaim(
- entityId = nodeIdFor(e.name),
- field = key,
- value = value,
- confidence = None
- ))
- }
- }
-
- val claimOps = relationshipClaims ++ attributeClaims
- val allOps = nodeOps ++ edgeOps ++ claimOps
-
- val sourceType = if (filePath.endsWith(".py") || filePath.endsWith(".ts") || filePath.endsWith(".tsx") || filePath.endsWith(".scala") || filePath.endsWith(".sc") || filePath.endsWith(".go")) SourceType.Code
- else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) SourceType.Config
- else if (filePath.endsWith(".md")) SourceType.Doc
- else SourceType.Code
-
- GraphPatch(
- patchId = PatchId(UUID.randomUUID()),
- actor = "ix/ingestion",
- timestamp = Instant.now(),
- source = PatchSource(filePath, sourceHash, extractor, sourceType),
- baseRev = Rev(0L),
- ops = allOps,
- replaces = Vector.empty,
- intent = Some(s"Parsed $filePath")
- )
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
deleted file mode 100644
index 363fbc0d..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
+++ /dev/null
@@ -1,256 +0,0 @@
-package ix.memory.ingestion
-
-import java.nio.file.{Files, Path}
-import java.time.Instant
-import java.util.UUID
-
-import cats.effect.IO
-import cats.syntax.all._
-import ix.memory.db.{CommitResult, GraphQueryApi, GraphWriteApi}
-import ix.memory.model._
-/**
- * Orchestrates file ingestion: reads source files, parses them,
- * builds graph patches, and commits them through the write API.
- */
-class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, queryApi: GraphQueryApi) {
-
- /**
- * Ingest a single file: read, parse, build patch, commit.
- */
- def ingestFile(filePath: Path): IO[CommitResult] = {
- for {
- bytes <- IO.blocking(Files.readAllBytes(filePath))
- _ <- if (bytes.isEmpty) IO.raiseError(new RuntimeException(s"Skipping empty file: $filePath"))
- else IO.unit
- // Hash raw bytes so revisions are stable across encodings
- hash = Some(sha256Bytes(bytes))
- // Decode text with UTF-8, fallback to ISO-8859-1 to avoid hard failures
- source <- IO.blocking {
- try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)
- catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }
- }
- // Use the language-specific parser when available; otherwise fall back to generic text
- parserOpt = parserRouter.parserFor(filePath.toString)
- result = parserOpt match {
- case Some(p) => p.parse(filePath.getFileName.toString, source)
- case None => GenericTextFallback.parse(filePath.getFileName.toString, source)
- }
- val fp = filePath.toString
- extractor = if (fp.endsWith(".py")) "tree-sitter-python/1.0"
- else if (fp.endsWith(".ts") || fp.endsWith(".tsx")) "typescript-parser/1.0"
- else if (fp.endsWith(".scala") || fp.endsWith(".sc")) "scala-parser/1.0"
- else if (fp.endsWith(".go")) "tree-sitter-go/1.0"
- else if (fp.endsWith(".json") || fp.endsWith(".yaml") || fp.endsWith(".yml") || fp.endsWith(".toml") || fp.endsWith(".ini") || fp.endsWith(".conf") || fp.endsWith(".properties") || fp.endsWith(".env")) "config-parser/1.0"
- else if (fp.endsWith(".md") || fp.endsWith(".mdx") || fp.endsWith(".rst") || fp.endsWith(".txt")) "markdown-parser/1.0"
- else "text-parser/1.0"
- existing <- queryApi.getPatchesBySource(filePath.toString, extractor)
- prevPatch = existing.headOption
- prevHash = prevPatch.flatMap(_.hcursor.downField("data").downField("source").get[String]("sourceHash").toOption)
- // If hash unchanged, skip (idempotent)
- _ <- if (prevHash == hash) IO.raiseError(new RuntimeException("Idempotent — file unchanged"))
- else IO.unit
- prevPatchId = prevPatch.flatMap(_.hcursor.downField("data").get[String]("patchId").toOption)
- .flatMap(s => scala.util.Try(java.util.UUID.fromString(s)).toOption)
- .map(PatchId(_))
- replaces = prevPatchId.toVector
- patch = GraphPatchBuilder.build(filePath.toString, hash, result).copy(replaces = replaces)
- _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))
- commit <- writeApi.commitPatch(patch)
- // Resolve cross-file import edges after main patch is committed
- _ <- resolveImportEdges(filePath, result, hash).attempt.void
- // Resolve named import symbols to their actual source entities
- _ <- resolveNamedImports(filePath, result, hash).attempt.void
- } yield commit
- }
-
- /**
- * Ingest all matching files under a path.
- *
- * @param path root directory or single file
- * @param language optional language filter (e.g. "python")
- * @param recursive whether to recurse into subdirectories
- * @return aggregated ingestion result
- */
- def ingestPath(
- path: Path,
- language: Option[String],
- recursive: Boolean
- ): IO[IngestionResult] = {
- for {
- files <- discoverFiles(path, language, recursive)
- results <- files.traverse(f => ingestFile(f).attempt)
- successes = results.collect { case Right(cr) => cr }
- idempotentCount = results.count {
- case Left(e: RuntimeException) if e.getMessage.startsWith("Idempotent") => true
- case _ => false
- }
- latestRev = successes.lastOption.map(_.newRev).getOrElse(Rev(0L))
- } yield IngestionResult(
- filesProcessed = files.size,
- patchesApplied = successes.size,
- filesSkipped = idempotentCount,
- entitiesCreated = 0, // Would need to count from parse results
- latestRev = latestRev
- )
- }
-
- private def discoverFiles(
- path: Path,
- language: Option[String],
- recursive: Boolean
- ): IO[List[Path]] = IO.blocking {
- val extensions = FileDiscovery.extensionsFor(language)
- if (Files.isRegularFile(path)) {
- if (language.isEmpty || FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty
- } else if (Files.isDirectory(path)) {
- FileDiscovery.tryGitLsFiles(path, recursive, extensions)
- .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))
- } else {
- List.empty
- }
- }
-
- private def resolveImportEdges(
- filePath: Path,
- parseResult: ParseResult,
- sourceHash: Option[String]
- ): IO[Unit] = {
- val importRels = parseResult.relationships.filter(_.predicate == "IMPORTS")
- importRels.traverse_ { rel =>
- val targetName = rel.dstName
- // Try to resolve relative imports (./config, ../utils)
- val resolvedPath = resolveRelativePath(filePath, targetName)
- resolvedPath match {
- case Some(resolved) =>
- queryApi.searchNodes(resolved.getFileName.toString, limit = 1).flatMap {
- case nodes if nodes.nonEmpty =>
- val srcId = NodeId(UUID.nameUUIDFromBytes(s"${filePath}:${filePath.getFileName}".getBytes("UTF-8")))
- val dstId = nodes.head.id
- val edgeId = EdgeId(UUID.nameUUIDFromBytes(s"${srcId.value}:${dstId.value}:IMPORTS".getBytes("UTF-8")))
- val patch = GraphPatch(
- patchId = PatchId(UUID.randomUUID()),
- actor = "ix/cross-file",
- timestamp = Instant.now(),
- source = PatchSource(filePath.toString, sourceHash, "cross-file-resolver/1.0", SourceType.Code),
- baseRev = Rev(0L),
- ops = Vector(PatchOp.UpsertEdge(edgeId, srcId, dstId, EdgePredicate("IMPORTS"), Map.empty)),
- replaces = Vector.empty,
- intent = Some(s"Cross-file import: ${filePath.getFileName} -> ${resolved.getFileName}")
- )
- writeApi.commitPatch(patch).void
- case _ => IO.unit
- }
- case None => IO.unit
- }
- }
- }
-
- private def resolveNamedImports(
- filePath: Path,
- parseResult: ParseResult,
- sourceHash: Option[String]
- ): IO[Unit] = {
- val namedImports = parseResult.relationships.filter { rel =>
- rel.predicate == "IMPORTS" && !rel.dstName.contains("/") && !rel.dstName.contains(".")
- }
- namedImports.traverse_ { rel =>
- queryApi.searchNodes(rel.dstName, limit = 5, nameOnly = true).flatMap { nodes =>
- val external = nodes.filter { n =>
- n.name == rel.dstName &&
- n.provenance.sourceUri != filePath.toString
- }
- external.headOption match {
- case Some(target) =>
- val srcId = NodeId(UUID.nameUUIDFromBytes(
- s"${filePath}:${filePath.getFileName}".getBytes("UTF-8")))
- val edgeId = EdgeId(UUID.nameUUIDFromBytes(
- s"${srcId.value}:${target.id.value}:IMPORTS".getBytes("UTF-8")))
- val patch = GraphPatch(
- patchId = PatchId(UUID.randomUUID()),
- actor = "ix/cross-file",
- timestamp = Instant.now(),
- source = PatchSource(filePath.toString, sourceHash,
- "cross-file-resolver/1.0", SourceType.Code),
- baseRev = Rev(0L),
- ops = Vector(PatchOp.UpsertEdge(
- edgeId, srcId, target.id, EdgePredicate("IMPORTS"), Map.empty)),
- replaces = Vector.empty,
- intent = Some(s"Named import: ${filePath.getFileName} -> ${rel.dstName}")
- )
- writeApi.commitPatch(patch).void
- case None => IO.unit
- }
- }
- }
- }
-
- private def resolveRelativePath(from: Path, importTarget: String): Option[Path] = {
- if (importTarget.startsWith("./") || importTarget.startsWith("../")) {
- val base = from.getParent
- if (base == null) return None
-
- // 1. Try the import target as-is and with source extensions appended
- val directExtensions = List("", ".ts", ".tsx", ".scala", ".py", ".js")
- val directMatch = directExtensions.collectFirst {
- case ext =>
- val candidate = base.resolve(importTarget + ext).normalize()
- if (Files.exists(candidate)) candidate else null
- }.flatMap(Option(_))
- if (directMatch.isDefined) return directMatch
-
- // 2. Strip compiled extensions (.js/.mjs/.cjs) and try source extensions
- val strippable = List(".js", ".mjs", ".cjs")
- val stripped = strippable.collectFirst {
- case ext if importTarget.endsWith(ext) =>
- importTarget.stripSuffix(ext)
- }
- val strippedMatch = stripped.flatMap { baseName =>
- val sourceExtensions = List(".ts", ".tsx")
- sourceExtensions.collectFirst {
- case ext =>
- val candidate = base.resolve(baseName + ext).normalize()
- if (Files.exists(candidate)) candidate else null
- }.flatMap(Option(_))
- }
- if (strippedMatch.isDefined) return strippedMatch
-
- // 3. Try directory index files (e.g., ./utils -> ./utils/index.ts)
- val indexExtensions = List("index.ts", "index.tsx", "index.js")
- indexExtensions.collectFirst {
- case idx =>
- val candidate = base.resolve(importTarget).resolve(idx).normalize()
- if (Files.exists(candidate)) candidate else null
- }.flatMap(Option(_))
- } else None
- }
-
- private def sha256Bytes(bytes: Array[Byte]): String = {
- val md = java.security.MessageDigest.getInstance("SHA-256")
- md.digest(bytes).map("%02x".format(_)).mkString
- }
-}
-
-case class IngestionProgress(
- filesDiscovered: Int,
- filesParsed: Int,
- chunksWritten: Int,
- totalChunks: Int
-)
-
-case class SkipReasons(
- unchanged: Int = 0,
- emptyFile: Int = 0,
- parseError: Int = 0,
- tooLarge: Int = 0
-) {
- def total: Int = unchanged + emptyFile + parseError + tooLarge
-}
-
-case class IngestionResult(
- filesProcessed: Int,
- patchesApplied: Int,
- filesSkipped: Int,
- entitiesCreated: Int,
- latestRev: Rev,
- skipReasons: SkipReasons = SkipReasons()
-)
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala
deleted file mode 100644
index 43ba1269..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package ix.memory.ingestion
-
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-case class ParsedEntity(
- name: String,
- kind: NodeKind,
- attrs: Map[String, Json],
- lineStart: Int,
- lineEnd: Int,
- contentFingerprint: Option[String] = None
-)
-
-case class ParsedRelationship(
- srcName: String,
- dstName: String,
- predicate: String
-)
-
-case class ParseResult(
- entities: Vector[ParsedEntity],
- relationships: Vector[ParsedRelationship]
-)
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala
deleted file mode 100644
index cec94982..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala
+++ /dev/null
@@ -1,9 +0,0 @@
-package ix.memory.ingestion
-
-/**
- * Common trait for language-specific source code parsers.
- * Both TreeSitterPythonParser and TypeScriptParser implement this.
- */
-trait Parser {
- def parse(fileName: String, source: String): ParseResult
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
deleted file mode 100644
index 74639975..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-package ix.memory.ingestion
-
-import ix.memory.ingestion.parsers.{ConfigParser, MarkdownParser, ScalaParser, TreeSitterGoParser, TreeSitterPythonParser, TypeScriptParser}
-
-/**
- * Routes file paths to the appropriate language parser.
- * Supports Python (.py), TypeScript (.ts/.tsx), config files (.json/.yaml/.yml/.toml),
- * and Markdown (.md).
- */
-class ParserRouter {
- private val pythonParser = new TreeSitterPythonParser()
- private val tsParser = new TypeScriptParser()
- private val configParser = new ConfigParser()
- private val markdownParser = new MarkdownParser()
- private val scalaParser = new ScalaParser()
- private val goParser = new TreeSitterGoParser()
-
- def parserFor(filePath: String): Option[Parser] = {
- if (filePath.endsWith(".py")) Some(pythonParser)
- else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) Some(tsParser)
- else if (filePath.endsWith(".scala") || filePath.endsWith(".sc")) Some(scalaParser)
- else if (filePath.endsWith(".go")) Some(goParser)
- else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) Some(configParser)
- else if (filePath.endsWith(".md")) Some(markdownParser)
- else None
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala b/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala
deleted file mode 100644
index f2823309..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala
+++ /dev/null
@@ -1,16 +0,0 @@
-package ix.memory.ingestion
-
-import ix.memory.model.GraphPatch
-
-/**
- * Simple validation for graph patches before committing.
- */
-object PatchValidator {
-
- def validate(patch: GraphPatch): Either[String, GraphPatch] = {
- if (patch.ops.isEmpty)
- Left("Patch has no operations")
- else
- Right(patch)
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
deleted file mode 100644
index bd396ed7..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
+++ /dev/null
@@ -1,190 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-/**
- * Parser for configuration files (JSON, YAML, TOML).
- *
- * Flattens nested structures into dot-path entries so that each leaf value
- * becomes a ConfigEntry node with a meaningful value (never "{" or "[").
- */
-class ConfigParser extends Parser {
-
- private val JsonExtensions = Set(".json")
- private val YamlExtensions = Set(".yaml", ".yml")
- private val TomlExtensions = Set(".toml")
-
- def canParse(fileName: String): Boolean = {
- val lower = fileName.toLowerCase
- (JsonExtensions ++ YamlExtensions ++ TomlExtensions).exists(ext => lower.endsWith(ext))
- }
-
- def parse(fileName: String, source: String): ParseResult = {
- val lower = fileName.toLowerCase
-
- val entries: Vector[(String, String, Int)] =
- if (JsonExtensions.exists(ext => lower.endsWith(ext))) parseJson(source)
- else if (YamlExtensions.exists(ext => lower.endsWith(ext))) parseYaml(source)
- else if (TomlExtensions.exists(ext => lower.endsWith(ext))) parseToml(source)
- else Vector.empty
-
- buildResult(fileName, source, entries)
- }
-
- // ---------------------------------------------------------------------------
- // Result builder
- // ---------------------------------------------------------------------------
-
- private def buildResult(
- fileName: String,
- source: String,
- entries: Vector[(String, String, Int)]
- ): ParseResult = {
- val lines = source.split("\n", -1)
-
- val configEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.Config,
- attrs = Map("format" -> Json.fromString(detectFormat(fileName))),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- val entryEntities = entries.map { case (key, value, line) =>
- ParsedEntity(
- name = key,
- kind = NodeKind.ConfigEntry,
- attrs = Map("value" -> Json.fromString(value)),
- lineStart = line,
- lineEnd = line
- )
- }
-
- val relationships = entries.map { case (key, _, _) =>
- ParsedRelationship(fileName, key, "CONTAINS")
- }
-
- ParseResult(
- entities = configEntity +: entryEntities,
- relationships = relationships
- )
- }
-
- private def detectFormat(fileName: String): String = {
- val lower = fileName.toLowerCase
- if (JsonExtensions.exists(ext => lower.endsWith(ext))) "json"
- else if (YamlExtensions.exists(ext => lower.endsWith(ext))) "yaml"
- else if (TomlExtensions.exists(ext => lower.endsWith(ext))) "toml"
- else "unknown"
- }
-
- // ---------------------------------------------------------------------------
- // JSON parsing — Circe-based with flattening
- // ---------------------------------------------------------------------------
-
- private def parseJson(source: String): Vector[(String, String, Int)] = {
- io.circe.parser.parse(source) match {
- case Right(json) => flattenJson("", json, 1)
- case Left(_) => regexParseJson(source)
- }
- }
-
- private def flattenJson(prefix: String, json: io.circe.Json, line: Int): Vector[(String, String, Int)] = {
- json.fold(
- jsonNull = Vector.empty,
- jsonBoolean = b => Vector((prefix, b.toString, line)),
- jsonNumber = n => Vector((prefix, n.toString, line)),
- jsonString = s => Vector((prefix, s, line)),
- jsonArray = arr => arr.zipWithIndex.flatMap { case (v, i) =>
- val key = if (prefix.isEmpty) s"[$i]" else s"$prefix[$i]"
- flattenJson(key, v, line)
- }.toVector,
- jsonObject = obj => obj.toVector.flatMap { case (k, v) =>
- val key = if (prefix.isEmpty) k else s"$prefix.$k"
- v.fold(
- jsonNull = Vector((key, "null", line)),
- jsonBoolean = b => Vector((key, b.toString, line)),
- jsonNumber = n => Vector((key, n.toString, line)),
- jsonString = s => Vector((key, s, line)),
- jsonArray = _ => flattenJson(key, v, line),
- jsonObject = _ => flattenJson(key, v, line)
- )
- }
- )
- }
-
- /** Regex fallback for malformed JSON */
- private def regexParseJson(source: String): Vector[(String, String, Int)] = {
- val KvPattern = """"([^"]+)"\s*:\s*"([^"]*)"""".r
- val lines = source.split("\n", -1)
- lines.zipWithIndex.flatMap { case (line, idx) =>
- KvPattern.findAllMatchIn(line).map { m =>
- (m.group(1), m.group(2), idx + 1)
- }
- }.toVector
- }
-
- // ---------------------------------------------------------------------------
- // YAML parsing — indentation-aware
- // ---------------------------------------------------------------------------
-
- private def parseYaml(source: String): Vector[(String, String, Int)] = {
- val lines = source.split("\n", -1)
- val result = Vector.newBuilder[(String, String, Int)]
- val stack = scala.collection.mutable.Stack.empty[(Int, String)] // (indent, prefix)
-
- val KvWithValue = """^([a-zA-Z_][\w.-]*)\s*:\s*(.+)$""".r
- val KvNoValue = """^([a-zA-Z_][\w.-]*)\s*:\s*$""".r
-
- lines.zipWithIndex.foreach { case (line, idx) =>
- val trimmed = line.trim
- if (!trimmed.startsWith("#") && trimmed.nonEmpty) {
- val indent = line.takeWhile(_ == ' ').length
- trimmed match {
- case KvWithValue(key, value) =>
- while (stack.nonEmpty && stack.top._1 >= indent) stack.pop()
- val fullKey = if (stack.nonEmpty) s"${stack.top._2}.$key" else key
- result += ((fullKey, value.trim, idx + 1))
- case KvNoValue(key) =>
- while (stack.nonEmpty && stack.top._1 >= indent) stack.pop()
- val fullKey = if (stack.nonEmpty) s"${stack.top._2}.$key" else key
- stack.push((indent, fullKey))
- case _ =>
- // skip list items, comments, etc.
- }
- }
- }
- result.result()
- }
-
- // ---------------------------------------------------------------------------
- // TOML parsing — section-aware
- // ---------------------------------------------------------------------------
-
- private def parseToml(source: String): Vector[(String, String, Int)] = {
- val lines = source.split("\n", -1)
- val result = Vector.newBuilder[(String, String, Int)]
- var currentSection = ""
-
- val SectionPattern = """^\[([^\]]+)\]""".r
- val KvPattern = """^([a-zA-Z_][\w.-]*)\s*=\s*(.+)$""".r
-
- lines.zipWithIndex.foreach { case (line, idx) =>
- val trimmed = line.trim
- if (!trimmed.startsWith("#") && trimmed.nonEmpty) {
- trimmed match {
- case SectionPattern(section) =>
- currentSection = section
- case KvPattern(key, value) =>
- val fullKey = if (currentSection.isEmpty) key else s"$currentSection.$key"
- val cleanValue = value.trim.stripPrefix("\"").stripSuffix("\"")
- result += ((fullKey, cleanValue, idx + 1))
- case _ =>
- }
- }
- }
- result.result()
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/GoParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/GoParser.scala
deleted file mode 100644
index 2abd19ae..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/GoParser.scala
+++ /dev/null
@@ -1,485 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-import scala.util.matching.Regex
-
-/**
- * Regex-based Go source parser (fallback for when tree-sitter is unavailable).
- * Extracts: functions, methods, structs, interfaces, type declarations,
- * imports, const/var declarations, and basic call edges.
- */
-class GoParser extends Parser {
-
- private val PackagePattern: Regex = """^\s*package\s+(\w+)""".r
- private val FuncPattern: Regex = """^\s*func\s+(\w+)\s*(?:\[.*?\])?\s*\(""".r
- private val MethodPattern: Regex = """^\s*func\s+\(\s*\w+\s+\*?(\w+)\s*\)\s+(\w+)\s*(?:\[.*?\])?\s*\(""".r
- private val StructPattern: Regex = """^\s*type\s+(\w+)\s+struct\s*\{""".r
- private val InterfacePattern: Regex = """^\s*type\s+(\w+)\s+interface\s*\{""".r
- private val TypeAliasPattern: Regex = """^\s*type\s+(\w+)\s*=\s*(.+)""".r
- private val NamedTypePattern: Regex = """^\s*type\s+(\w+)\s+(\w+)""".r
- private val ImportSinglePattern: Regex = """^\s*import\s+"([^"]+)"""".r
- private val ImportGroupStart: Regex = """^\s*import\s*\(""".r
- private val ImportSpecPattern: Regex = """^\s*(?:(\w+|\.)\s+)?"([^"]+)"""".r
- private val ConstPattern: Regex = """^\s*const\s+(\w+)\s""".r
- private val VarPattern: Regex = """^\s*var\s+(\w+)\s""".r
- private val ConstGroupStart: Regex = """^\s*const\s*\(""".r
- private val VarGroupStart: Regex = """^\s*var\s*\(""".r
- private val ConstSpecPattern: Regex = """^\s*(\w+)""".r
- private val CallPattern: Regex = """\b(\w+)\s*\(""".r
-
- private val GoBuiltins: Set[String] = Set(
- "make", "len", "cap", "append", "copy", "close", "delete",
- "complex", "real", "imag", "new", "panic", "recover",
- "print", "println", "error", "string", "int", "float64",
- "bool", "byte", "rune", "nil", "true", "false", "iota",
- "int8", "int16", "int32", "int64",
- "uint", "uint8", "uint16", "uint32", "uint64", "uintptr",
- "float32", "complex64", "complex128", "any", "comparable"
- )
-
- private val GoKeywords: Set[String] = Set(
- "break", "case", "chan", "const", "continue", "default", "defer",
- "else", "fallthrough", "for", "func", "go", "goto", "if",
- "import", "interface", "map", "package", "range", "return",
- "select", "struct", "switch", "type", "var"
- )
-
- def parse(fileName: String, source: String): ParseResult = {
- val lines = source.split("\n", -1)
-
- var fileAttrs: Map[String, Json] = Map("language" -> Json.fromString("go"))
- var entities = Vector.empty[ParsedEntity]
- var relationships = Vector.empty[ParsedRelationship]
-
- // Track type ranges for containment
- var typeRanges = Vector.empty[(String, Int, Int)]
-
- var i = 0
- while (i < lines.length) {
- val line = lines(i)
- val lineNum = i + 1
- val trimmed = line.trim
-
- // Skip comments
- if (trimmed.startsWith("//") || trimmed.startsWith("/*") || trimmed.startsWith("*")) {
- i += 1
- } else {
- var handled = false
-
- // Package
- if (!handled) {
- PackagePattern.findFirstMatchIn(line).foreach { m =>
- fileAttrs = fileAttrs + ("package" -> Json.fromString(m.group(1)))
- handled = true
- }
- }
-
- // Method (must check before function)
- if (!handled) {
- MethodPattern.findFirstMatchIn(line).foreach { m =>
- val receiverType = m.group(1)
- val methodName = m.group(2)
- val endLine = findBraceBlockEnd(lines, i).getOrElse(lineNum)
- val sig = trimmed.replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = lines.slice(i, endLine).mkString("\n")
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = methodName.nonEmpty && methodName.head.isUpper
-
- entities = entities :+ ParsedEntity(
- name = methodName,
- kind = NodeKind.Method,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "summary" -> Json.fromString(sig),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported),
- "receiver" -> Json.fromString(receiverType)
- ),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(receiverType, methodName, "CONTAINS")
-
- // Extract calls
- relationships = relationships ++ extractCalls(lines, i, endLine, methodName)
-
- i = endLine
- handled = true
- }
- }
-
- // Function
- if (!handled) {
- FuncPattern.findFirstMatchIn(line).foreach { m =>
- val funcName = m.group(1)
- val endLine = findBraceBlockEnd(lines, i).getOrElse(lineNum)
- val sig = trimmed.replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = lines.slice(i, endLine).mkString("\n")
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = funcName.nonEmpty && funcName.head.isUpper
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "summary" -> Json.fromString(sig),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
-
- // Extract calls
- relationships = relationships ++ extractCalls(lines, i, endLine, funcName)
-
- i = endLine
- handled = true
- }
- }
-
- // Struct
- if (!handled) {
- StructPattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, i).getOrElse(lineNum)
- val sig = trimmed.replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = lines.slice(i, endLine).mkString("\n")
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = name.nonEmpty && name.head.isUpper
-
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("struct"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
-
- i = endLine
- handled = true
- }
- }
-
- // Interface
- if (!handled) {
- InterfacePattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, i).getOrElse(lineNum)
- val sig = trimmed.replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = lines.slice(i, endLine).mkString("\n")
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = name.nonEmpty && name.head.isUpper
-
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Interface,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("interface"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
-
- i = endLine
- handled = true
- }
- }
-
- // Type alias
- if (!handled) {
- TypeAliasPattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- val exported = name.nonEmpty && name.head.isUpper
- val sig = trimmed.take(120)
- val fp = Fingerprint.compute(sig, sig)
-
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("type_alias"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = lineNum,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- handled = true
- }
- }
-
- // Named type (must be after struct/interface/alias checks)
- if (!handled) {
- NamedTypePattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- val baseType = m.group(2)
- if (baseType != "struct" && baseType != "interface") {
- val exported = name.nonEmpty && name.head.isUpper
- val sig = trimmed.take(120)
- val fp = Fingerprint.compute(sig, sig)
-
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("named_type"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = lineNum,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- handled = true
- }
- }
- }
-
- // Import group
- if (!handled) {
- ImportGroupStart.findFirstMatchIn(line).foreach { _ =>
- i += 1
- while (i < lines.length && !lines(i).trim.startsWith(")")) {
- ImportSpecPattern.findFirstMatchIn(lines(i)).foreach { m =>
- val importPath = m.group(2)
- if (!entities.exists(e => e.name == importPath && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = importPath,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = i + 1,
- lineEnd = i + 1
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, importPath, "IMPORTS")
- }
- i += 1
- }
- handled = true
- }
- }
-
- // Single import
- if (!handled) {
- ImportSinglePattern.findFirstMatchIn(line).foreach { m =>
- val importPath = m.group(1)
- if (!entities.exists(e => e.name == importPath && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = importPath,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, importPath, "IMPORTS")
- handled = true
- }
- }
-
- // Const (single)
- if (!handled) {
- ConstGroupStart.findFirstMatchIn(line).foreach { _ =>
- i += 1
- while (i < lines.length && !lines(i).trim.startsWith(")")) {
- ConstSpecPattern.findFirstMatchIn(lines(i).trim).foreach { m =>
- val name = m.group(1)
- if (name != "//" && name != "/*" && name != "_") {
- val exported = name.nonEmpty && name.head.isUpper
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Variable,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("const"),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = i + 1,
- lineEnd = i + 1
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- }
- }
- i += 1
- }
- handled = true
- }
- }
-
- if (!handled) {
- ConstPattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- if (name != "(" && name != "_") {
- val exported = name.nonEmpty && name.head.isUpper
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Variable,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("const"),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = lineNum
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- handled = true
- }
- }
- }
-
- // Var group
- if (!handled) {
- VarGroupStart.findFirstMatchIn(line).foreach { _ =>
- i += 1
- while (i < lines.length && !lines(i).trim.startsWith(")")) {
- val varLine = lines(i).trim
- if (varLine.nonEmpty && !varLine.startsWith("//") && !varLine.startsWith("/*")) {
- varLine.split("\\s+").headOption.foreach { name =>
- if (name != "_" && name.matches("\\w+")) {
- val exported = name.nonEmpty && name.head.isUpper
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Variable,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("var"),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = i + 1,
- lineEnd = i + 1
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- }
- }
- }
- i += 1
- }
- handled = true
- }
- }
-
- if (!handled) {
- VarPattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- if (name != "(" && name != "_") {
- val exported = name.nonEmpty && name.head.isUpper
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Variable,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("var"),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = lineNum
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- handled = true
- }
- }
- }
-
- if (!handled) i += 1
- else if (i == lineNum - 1) i += 1 // advance if we didn't already
- }
- }
-
- val fileEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = fileAttrs,
- lineStart = 1,
- lineEnd = lines.length
- )
-
- ParseResult(fileEntity +: entities, relationships)
- }
-
- private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Option[Int] = {
- var braceCount = 0
- var foundOpen = false
- var i = startIdx
-
- while (i < lines.length) {
- val line = lines(i)
- for (ch <- line) {
- ch match {
- case '{' =>
- braceCount += 1
- foundOpen = true
- case '}' =>
- braceCount -= 1
- if (foundOpen && braceCount == 0) {
- return Some(i + 1)
- }
- case _ =>
- }
- }
- i += 1
- }
-
- if (foundOpen) Some(lines.length) else None
- }
-
- private def extractCalls(lines: Array[String], startIdx: Int, endIdx: Int, callerName: String): Vector[ParsedRelationship] = {
- var calls = Vector.empty[ParsedRelationship]
- val seen = scala.collection.mutable.Set.empty[String]
- for (i <- (startIdx + 1) until endIdx.min(lines.length)) {
- val line = lines(i).trim
- if (!line.startsWith("//") && !line.startsWith("/*")) {
- CallPattern.findAllMatchIn(line).foreach { m =>
- val callee = m.group(1)
- if (callee != callerName && !GoBuiltins.contains(callee) &&
- !GoKeywords.contains(callee) && !seen.contains(callee)) {
- seen += callee
- calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
- }
- }
- }
- }
- calls
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
deleted file mode 100644
index 7d99504c..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-import scala.util.matching.Regex
-
-/**
- * Markdown parser that extracts structural entities from `.md` files.
- *
- * Each heading becomes a Doc entity with attrs for `level`, `title`, and `content`.
- * Content captures all non-header lines in the section, up to 2000 characters.
- * A top-level File entity is also emitted, with CONTAINS edges to each section.
- */
-class MarkdownParser extends Parser {
-
- private val HeadingPattern: Regex = """^(#{1,6})\s+(.+)$""".r
-
- def parse(fileName: String, source: String): ParseResult = {
- val lines = source.split("\n", -1)
-
- // -- File entity --
- val fileEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("markdown")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- // Collect headings with their line indices
- val headings: Vector[(Int, Int, String)] = lines.zipWithIndex.flatMap { case (line, idx) =>
- HeadingPattern.findFirstMatchIn(line.trim).map { m =>
- val level = m.group(1).length
- val title = m.group(2).trim
- (idx, level, title)
- }
- }.toVector
-
- var entities = Vector(fileEntity)
- var relationships = Vector.empty[ParsedRelationship]
-
- for (i <- headings.indices) {
- val (startIdx, level, title) = headings(i)
- val endIdx = if (i + 1 < headings.length) headings(i + 1)._1 else lines.length
-
- val content = extractContent(lines, startIdx, endIdx)
-
- val entity = ParsedEntity(
- name = title,
- kind = NodeKind.Doc,
- attrs = Map(
- "level" -> Json.fromInt(level),
- "title" -> Json.fromString(title),
- "content" -> Json.fromString(content)
- ),
- lineStart = startIdx + 1,
- lineEnd = endIdx
- )
-
- entities = entities :+ entity
- relationships = relationships :+ ParsedRelationship(fileName, title, "CONTAINS")
- }
-
- ParseResult(entities, relationships)
- }
-
- /**
- * Extract content from all non-header lines in the section, up to 2000 characters.
- */
- private def extractContent(lines: Array[String], startIdx: Int, endIdx: Int): String = {
- val contentLines = lines.slice(startIdx + 1, endIdx).filterNot { line =>
- HeadingPattern.findFirstMatchIn(line.trim).isDefined
- }
- contentLines.mkString("\n").trim.take(2000)
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala
deleted file mode 100644
index deac202c..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala
+++ /dev/null
@@ -1,357 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-import scala.util.matching.Regex
-
-/**
- * Regex-based Scala source parser that extracts structural entities and relationships.
- *
- * Extracts: traits, classes, case classes, objects, methods (def), and imports.
- * Uses brace counting for block boundary detection.
- */
-class ScalaParser extends Parser {
-
- // Patterns for Scala constructs
- private val TraitPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*trait\s+(\w+)""".r
- private val ClassPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*class\s+(\w+)""".r
- private val CaseClassPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*case\s+class\s+(\w+)""".r
- private val ObjectPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*(?:case\s+)?object\s+(\w+)""".r
- private val DefPattern: Regex = """^\s*(?:override\s+)?(?:private(?:\[\w+\])?\s+|protected(?:\[\w+\])?\s+)?(?:final\s+)?def\s+(\w+)""".r
- private val ImportPattern: Regex = """^\s*import\s+(.+)$""".r
- private val CallPattern: Regex = """(?:(?:this|super)\.)?\b(\w+)\s*(?:\[.*?\])?\s*\(""".r
- private val ExtendsPattern: Regex = """extends\s+(\w+)""".r
- private val TypeRefPattern: Regex = """\b([A-Z]\w+)""".r
- private val QualifiedRefPattern: Regex = """\b([A-Z]\w+)\.(\w+)""".r
-
- private val ScalaBuiltinTypes: Set[String] = Set(
- "String", "Int", "Long", "Double", "Float", "Boolean", "Unit", "Any", "AnyRef", "AnyVal",
- "Nothing", "Null", "Option", "List", "Map", "Set", "Vector", "Seq", "Array", "Future",
- "IO", "Either", "Try", "Some", "None", "Right", "Left", "Success", "Failure"
- )
-
- private val ScalaBuiltins: Set[String] = Set(
- "println", "print", "require", "assert", "assume",
- "Some", "None", "Option", "List", "Map", "Set", "Vector",
- "Right", "Left", "Try", "Success", "Failure",
- "Future", "IO", "pure", "flatMap", "map", "filter",
- "foreach", "fold", "foldLeft", "foldRight",
- "toString", "hashCode", "equals", "getClass",
- "apply", "unapply", "copy", "andThen", "compose",
- "getOrElse", "orElse", "contains", "exists", "forall",
- "collect", "traverse", "sequence", "attempt", "handleErrorWith",
- "raiseError", "blocking", "delay", "suspend", "async",
- "mkString", "toList", "toVector", "toSet", "toMap",
- "head", "tail", "size", "length", "isEmpty", "nonEmpty",
- "take", "drop", "slice", "zip", "zipWithIndex",
- "sortBy", "groupBy", "distinct", "flatten"
- )
-
- def parse(fileName: String, source: String): ParseResult = {
- val lines = source.split("\n", -1)
-
- val fileEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("scala")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- var entities = Vector(fileEntity)
- var relationships = Vector.empty[ParsedRelationship]
-
- // Track type ranges for containment: (name, startLine, endLine)
- var typeRanges = Vector.empty[(String, Int, Int)]
-
- for ((line, idx) <- lines.zipWithIndex) {
- val lineNum = idx + 1
- val trimmed = line.trim
-
- // Skip comment lines
- if (trimmed.startsWith("//") || trimmed.startsWith("*") || trimmed.startsWith("/*")) {
- // no-op
- } else {
-
- // Case class must be checked before class (since "case class" contains "class")
- val handledAsType = CaseClassPattern.findFirstMatchIn(line) match {
- case Some(m) =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
- val bodyText = lines.slice(idx, endLine).mkString("\n")
- val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Class,
- attrs = Map("scala_kind" -> Json.fromString("case_class"), "language" -> Json.fromString("scala"),
- "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
- val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
- ExtendsPattern.findFirstMatchIn(line).foreach { em =>
- relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
- }
- true
-
- case None =>
- // Check for trait
- TraitPattern.findFirstMatchIn(line) match {
- case Some(m) =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
- val bodyText = lines.slice(idx, endLine).mkString("\n")
- val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Trait,
- attrs = Map("scala_kind" -> Json.fromString("trait"), "language" -> Json.fromString("scala"),
- "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
- val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
- ExtendsPattern.findFirstMatchIn(line).foreach { em =>
- relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
- }
- true
-
- case None =>
- // Check for object (before class, since object keyword won't match class pattern)
- ObjectPattern.findFirstMatchIn(line) match {
- case Some(m) =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
- val bodyText = lines.slice(idx, endLine).mkString("\n")
- val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Object,
- attrs = Map("scala_kind" -> Json.fromString("object"), "language" -> Json.fromString("scala"),
- "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
- val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
- ExtendsPattern.findFirstMatchIn(line).foreach { em =>
- relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
- }
- true
-
- case None =>
- // Check for class
- ClassPattern.findFirstMatchIn(line) match {
- case Some(m) =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
- val bodyText = lines.slice(idx, endLine).mkString("\n")
- val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Class,
- attrs = Map("scala_kind" -> Json.fromString("class"), "language" -> Json.fromString("scala"),
- "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
- val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
- ExtendsPattern.findFirstMatchIn(line).foreach { em =>
- relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
- }
- true
- case None => false
- }
- }
- }
- }
-
- // Check for def (only if we didn't match a type on this line)
- if (!handledAsType) {
- DefPattern.findFirstMatchIn(line).foreach { m =>
- val funcName = m.group(1)
- val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
-
- val signature = extractSignature(trimmed)
- val visibility = extractVisibility(trimmed)
-
- // Determine enclosing type
- val enclosing = typeRanges.find { case (_, start, end) =>
- lineNum > start && lineNum <= end
- }
-
- val (kind, edgePredicate, edgeSrc) = enclosing match {
- case Some((typeName, _, _)) => (NodeKind.Method, "CONTAINS", typeName)
- case None => (NodeKind.Function, "CONTAINS", fileName)
- }
-
- val bodyText = lines.slice(idx, endLine).mkString("\n")
- val fp = Fingerprint.compute(signature, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = kind,
- attrs = Map(
- "language" -> Json.fromString("scala"),
- "summary" -> Json.fromString(signature),
- "signature" -> Json.fromString(signature),
- "visibility" -> Json.fromString(visibility)
- ),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
-
- relationships = relationships :+ ParsedRelationship(edgeSrc, funcName, edgePredicate)
-
- // Extract CALLS and REFERENCES from method/function body
- val callsInFunc = extractCalls(lines, idx, endLine, funcName)
- relationships = relationships ++ callsInFunc
-
- // Extract REFERENCES from method signature type annotations
- val enclosingTypeName = enclosing.map(_._1).getOrElse("")
- val sigRefs = TypeRefPattern.findAllMatchIn(signature).map(_.group(1)).toSet
- for (ref <- sigRefs) {
- if (!ScalaBuiltinTypes.contains(ref) && ref != funcName && ref != enclosingTypeName && ref != fileName) {
- relationships = relationships :+ ParsedRelationship(funcName, ref, "REFERENCES")
- }
- }
- }
- }
-
- // Check for imports
- ImportPattern.findFirstMatchIn(line).foreach { m =>
- val importPath = m.group(1).trim
- // Parse import path: "ix.memory.model.{Claim, GraphNode}" -> module "ix.memory.model"
- // "cats.effect.IO" -> module "cats.effect.IO"
- val moduleName = if (importPath.contains("{")) {
- // Grouped import: take the prefix before the braces
- importPath.substring(0, importPath.indexOf('{')).stripSuffix(".").trim
- } else {
- importPath
- }
-
- // Avoid duplicate module entities
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
-
- } // end else (non-comment)
- }
-
- ParseResult(entities, relationships)
- }
-
- /**
- * Find the end of a brace-delimited block starting at `startIdx`.
- * Counts opening and closing braces. Returns None if no opening brace is found
- * (e.g. abstract method in a trait with no body).
- */
- private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Option[Int] = {
- var braceCount = 0
- var foundOpen = false
- var i = startIdx
-
- while (i < lines.length) {
- val line = lines(i)
- for (ch <- line) {
- ch match {
- case '{' =>
- braceCount += 1
- foundOpen = true
- case '}' =>
- braceCount -= 1
- if (foundOpen && braceCount == 0) {
- return Some(i + 1) // 1-indexed
- }
- case _ => // skip
- }
- }
- // If we found an opening brace and closed it, we'd have returned above
- // If line ends and no brace found yet, keep going
- i += 1
- }
-
- if (foundOpen) Some(lines.length) else None
- }
-
- /** Extract CALLS and qualified REFERENCES relationships from a method/function body. */
- private def extractCalls(lines: Array[String], startIdx: Int, endIdx: Int, callerName: String): Vector[ParsedRelationship] = {
- var calls = Vector.empty[ParsedRelationship]
- val seen = scala.collection.mutable.Set.empty[String]
- val refSeen = scala.collection.mutable.Set.empty[String]
- for (i <- (startIdx + 1) until endIdx.min(lines.length)) {
- val line = lines(i).trim
- if (!line.startsWith("//") && !line.startsWith("*")) {
- CallPattern.findAllMatchIn(line).foreach { m =>
- val callee = m.group(1)
- // Only include lowercase-starting identifiers (methods/functions, not types)
- if (callee != callerName && !ScalaBuiltins.contains(callee) && !seen.contains(callee) && callee.head.isLower) {
- seen += callee
- calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
- }
- }
- // Extract qualified REFERENCES (e.g., NodeKind.File, SourceType.Code)
- QualifiedRefPattern.findAllMatchIn(line).foreach { m =>
- val qualifier = m.group(1)
- if (!ScalaBuiltinTypes.contains(qualifier) && !ScalaBuiltins.contains(qualifier) &&
- !refSeen.contains(qualifier) && qualifier != callerName) {
- refSeen += qualifier
- calls = calls :+ ParsedRelationship(callerName, qualifier, "REFERENCES")
- }
- }
- }
- }
- calls
- }
-
- /** Extract a type signature (trait/class/object), stripping body braces. */
- private def extractTypeSignature(trimmedLine: String): String = {
- val cleaned = trimmedLine
- .replaceAll("""\s*\{\s*$""", "")
- .trim
- cleaned.take(120)
- }
-
- /** Extract a clean signature from a trimmed line, stripping trailing body markers. */
- private def extractSignature(trimmedLine: String): String = {
- val cleaned = trimmedLine
- .replaceAll("""\s*=\s*\{?\s*$""", "")
- .replaceAll("""\s*\{\s*$""", "")
- .trim
- cleaned.take(120)
- }
-
- /** Detect visibility from a trimmed line (after stripping `override`). */
- private def extractVisibility(trimmedLine: String): String = {
- val withoutOverride = trimmedLine.replaceFirst("""^\s*override\s+""", "").trim
- if (withoutOverride.startsWith("private")) "private"
- else if (withoutOverride.startsWith("protected")) "protected"
- else "public"
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterGoParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterGoParser.scala
deleted file mode 100644
index 39b94b57..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterGoParser.scala
+++ /dev/null
@@ -1,523 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-import org.treesitter.{TSNode, TreeSitterGo}
-
-/**
- * Tree-sitter AST-based Go parser with regex fallback.
- * Extracts packages, functions, methods, structs, interfaces, type aliases,
- * const/var declarations, imports, call edges, and struct embedding (EXTENDS).
- */
-class TreeSitterGoParser extends Parser {
-
- private val regexFallback = new GoParser()
-
- def parse(fileName: String, source: String): ParseResult = {
- try treeSitterParse(fileName, source)
- catch { case _: Throwable => regexFallback.parse(fileName, source) }
- }
-
- private def treeSitterParse(fileName: String, source: String): ParseResult = {
- TreeSitterUtils.withParse(new TreeSitterGo(), source) { root =>
- val ctx = new GoExtractionContext(fileName, source, root)
- ctx.result()
- }
- }
-
- // -------------------------------------------------------------------------
- // Extraction context
- // -------------------------------------------------------------------------
-
- private class GoExtractionContext(
- fileName: String,
- source: String,
- root: TSNode
- ) {
- import TreeSitterUtils._
-
- var entities = Vector.empty[ParsedEntity]
- var relationships = Vector.empty[ParsedRelationship]
-
- private val lines = source.split("\n", -1)
-
- // Initialize with file entity
- entities = entities :+ ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("go")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- // Walk the program
- walkProgram(root)
-
- def result(): ParseResult = ParseResult(entities, relationships)
-
- private def walkProgram(node: TSNode): Unit = {
- for (child <- namedChildren(node)) {
- child.getType() match {
- case "package_clause" =>
- handlePackageClause(child)
-
- case "import_declaration" =>
- handleImportDeclaration(child)
-
- case "function_declaration" =>
- handleFunctionDeclaration(child)
-
- case "method_declaration" =>
- handleMethodDeclaration(child)
-
- case "type_declaration" =>
- handleTypeDeclaration(child)
-
- case "const_declaration" =>
- handleConstDeclaration(child)
-
- case "var_declaration" =>
- handleVarDeclaration(child)
-
- case _ =>
- // Ignore other top-level nodes (comments, etc.)
- }
- }
- }
-
- // --- Package handling ---
-
- private def handlePackageClause(node: TSNode): Unit = {
- fieldChild(node, "name").orElse(findChild(node, "package_identifier")).foreach { nameNode =>
- val pkgName = nodeText(nameNode, source)
- // Set package as attribute on the File entity
- entities = entities.map { e =>
- if (e.name == fileName && e.kind == NodeKind.File) {
- e.copy(attrs = e.attrs + ("package" -> Json.fromString(pkgName)))
- } else e
- }
- }
- }
-
- // --- Import handling ---
-
- private def handleImportDeclaration(node: TSNode): Unit = {
- // Single import: import "fmt"
- findChild(node, "import_spec").foreach(handleImportSpec)
-
- // Grouped import: import ( "fmt"; "os" )
- findChild(node, "import_spec_list").foreach { specList =>
- for (spec <- childrenOfType(specList, "import_spec")) {
- handleImportSpec(spec)
- }
- }
- }
-
- private def handleImportSpec(spec: TSNode): Unit = {
- // import_spec has optional name (alias) and path (interpreted_string_literal)
- val pathOpt = fieldChild(spec, "path")
- .orElse(findChild(spec, "interpreted_string_literal"))
- .map(n => nodeText(n, source).stripPrefix("\"").stripSuffix("\""))
-
- pathOpt.foreach { importPath =>
- val aliasOpt = fieldChild(spec, "name")
- .map(n => nodeText(n, source))
-
- val entityName = aliasOpt match {
- case Some(".") => importPath // dot import
- case Some("_") => importPath // side-effect import
- case Some(alias) => alias // named import
- case None => importPath // bare import
- }
-
- if (!entities.exists(e => e.name == importPath && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = importPath,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineStart(spec),
- lineEnd = lineEnd(spec)
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, importPath, "IMPORTS")
- }
- }
-
- // --- Function handling ---
-
- private def handleFunctionDeclaration(node: TSNode): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { funcName =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val sig = extractFirstLine(node).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = funcName.nonEmpty && funcName.head.isUpper
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "summary" -> Json.fromString(sig),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
-
- // Extract calls from function body
- fieldChild(node, "body").foreach { body =>
- extractCallsFromBody(body, funcName)
- }
- }
- }
-
- // --- Method handling ---
-
- private def handleMethodDeclaration(node: TSNode): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { methodName =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val sig = extractFirstLine(node).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = methodName.nonEmpty && methodName.head.isUpper
-
- // Extract receiver type name
- val receiverType = fieldChild(node, "receiver")
- .orElse(findChild(node, "parameter_list"))
- .flatMap(extractReceiverType)
-
- val attrs = Map(
- "language" -> Json.fromString("go"),
- "summary" -> Json.fromString(sig),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ) ++ receiverType.map(r => "receiver" -> Json.fromString(r))
-
- entities = entities :+ ParsedEntity(
- name = methodName,
- kind = NodeKind.Method,
- attrs = attrs,
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
-
- // Method belongs to receiver type, not file
- val containsSrc = receiverType.getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, methodName, "CONTAINS")
-
- // Extract calls from method body
- fieldChild(node, "body").foreach { body =>
- extractCallsFromBody(body, methodName)
- }
- }
- }
-
- private def extractReceiverType(paramList: TSNode): Option[String] = {
- // The receiver parameter list contains a single parameter_declaration
- // e.g., (r *Receiver) or (r Receiver)
- val params = namedChildren(paramList)
- params.headOption.flatMap { param =>
- // Look for type_identifier or pointer_type → type_identifier
- val typeNode = fieldChild(param, "type").getOrElse(param)
- extractTypeName(typeNode)
- }
- }
-
- private def extractTypeName(node: TSNode): Option[String] = {
- node.getType() match {
- case "type_identifier" =>
- Some(nodeText(node, source))
- case "pointer_type" =>
- // *Type — recurse to get the type name
- namedChildren(node).headOption.flatMap(extractTypeName)
- case "parameter_declaration" =>
- // Get the type field
- fieldChild(node, "type").flatMap(extractTypeName)
- case "generic_type" =>
- // Type[T] — extract the base type
- fieldChild(node, "type").flatMap(extractTypeName)
- case _ =>
- // Fallback: try to find a type_identifier descendant
- findChild(node, "type_identifier").map(n => nodeText(n, source))
- }
- }
-
- // --- Type declaration handling ---
-
- private def handleTypeDeclaration(node: TSNode): Unit = {
- // Single type: type Foo struct { ... }
- findChild(node, "type_spec").foreach(handleTypeSpec)
-
- // Grouped type: type ( Foo struct {}; Bar interface {} )
- for (child <- namedChildren(node)) {
- if (child.getType() == "type_spec") {
- handleTypeSpec(child)
- }
- }
- }
-
- private def handleTypeSpec(spec: TSNode): Unit = {
- val nameOpt = extractName(spec, source)
- nameOpt.foreach { typeName =>
- val start = lineStart(spec)
- val end = lineEnd(spec)
- val exported = typeName.nonEmpty && typeName.head.isUpper
- val bodyText = nodeText(spec, source)
-
- // Determine the type kind from the type expression
- val typeNode = fieldChild(spec, "type")
- val typeKind = typeNode.map(_.getType()).getOrElse("")
-
- typeKind match {
- case "struct_type" =>
- val sig = extractFirstLine(spec).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val fp = Fingerprint.compute(sig, bodyText)
- entities = entities :+ ParsedEntity(
- name = typeName,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("struct"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
-
- // Extract struct embedding (EXTENDS)
- typeNode.foreach(extractStructEmbedding(_, typeName))
-
- case "interface_type" =>
- val sig = extractFirstLine(spec).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val fp = Fingerprint.compute(sig, bodyText)
- entities = entities :+ ParsedEntity(
- name = typeName,
- kind = NodeKind.Interface,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("interface"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
-
- // Extract interface embedding
- typeNode.foreach(extractInterfaceEmbedding(_, typeName))
-
- case _ =>
- // Type alias (type Foo = Bar) or named type (type Foo int)
- val isAlias = allChildren(spec).exists(c => nodeText(c, source) == "=")
- val goKind = if (isAlias) "type_alias" else "named_type"
- val sig = nodeText(spec, source).trim.take(120)
- val fp = Fingerprint.compute(sig, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = typeName,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString(goKind),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
- }
- }
- }
-
- private def extractStructEmbedding(structType: TSNode, typeName: String): Unit = {
- // Walk field_declaration_list for embedded fields (no field name)
- findChild(structType, "field_declaration_list").foreach { fieldList =>
- for (field <- childrenOfType(fieldList, "field_declaration")) {
- // An embedded field has a type but no name field
- val hasName = fieldChild(field, "name").isDefined
- if (!hasName) {
- // The type is the embedded type
- fieldChild(field, "type").foreach { typeNode =>
- extractTypeName(typeNode).foreach { embeddedName =>
- if (embeddedName != typeName && !GoBuiltins.contains(embeddedName)) {
- relationships = relationships :+ ParsedRelationship(typeName, embeddedName, "EXTENDS")
- }
- }
- }
- }
- }
- }
- }
-
- private def extractInterfaceEmbedding(ifaceType: TSNode, typeName: String): Unit = {
- // Walk interface body for bare type identifiers (embedded interfaces)
- for (child <- namedChildren(ifaceType)) {
- child.getType() match {
- case "type_identifier" =>
- val embeddedName = nodeText(child, source)
- if (embeddedName != typeName && !GoBuiltins.contains(embeddedName)) {
- relationships = relationships :+ ParsedRelationship(typeName, embeddedName, "EXTENDS")
- }
- case "qualified_type" =>
- // pkg.Type embedded
- fieldChild(child, "name")
- .orElse(namedChildren(child).lastOption)
- .foreach { nameNode =>
- val embeddedName = nodeText(nameNode, source)
- if (embeddedName != typeName) {
- relationships = relationships :+ ParsedRelationship(typeName, embeddedName, "EXTENDS")
- }
- }
- case _ => // method specs, etc. — skip
- }
- }
- }
-
- // --- Const/Var handling ---
-
- private def handleConstDeclaration(node: TSNode): Unit = {
- // Single const or grouped const ( ... )
- val specs = childrenOfType(node, "const_spec")
- if (specs.isEmpty) {
- // May have a single const_spec as direct structure
- handleConstOrVarSpecs(namedChildren(node).filter(_.getType() == "const_spec"), "const")
- } else {
- handleConstOrVarSpecs(specs, "const")
- }
- }
-
- private def handleVarDeclaration(node: TSNode): Unit = {
- val specs = childrenOfType(node, "var_spec")
- if (specs.isEmpty) {
- handleConstOrVarSpecs(namedChildren(node).filter(_.getType() == "var_spec"), "var")
- } else {
- handleConstOrVarSpecs(specs, "var")
- }
- }
-
- private def handleConstOrVarSpecs(specs: Vector[TSNode], declKind: String): Unit = {
- for (spec <- specs) {
- // Each spec may have multiple names: const a, b = 1, 2
- val names = fieldChild(spec, "name")
- .map(Vector(_))
- .getOrElse(childrenOfType(spec, "identifier"))
-
- for (nameNode <- names) {
- val varName = nodeText(nameNode, source)
- if (varName.nonEmpty && varName != "_") {
- val start = lineStart(spec)
- val end = lineEnd(spec)
- val exported = varName.head.isUpper
- val sig = nodeText(spec, source).trim.take(120)
- val fp = Fingerprint.compute(sig, sig)
-
- entities = entities :+ ParsedEntity(
- name = varName,
- kind = NodeKind.Variable,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString(declKind),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, varName, "CONTAINS")
- }
- }
- }
- }
-
- // --- CALLS extraction ---
-
- private def extractCallsFromBody(bodyNode: TSNode, callerName: String): Unit = {
- val seen = scala.collection.mutable.Set.empty[String]
- val callExprs = collectDescendants(bodyNode, _.getType() == "call_expression")
-
- for (callExpr <- callExprs) {
- fieldChild(callExpr, "function").foreach { funcNode =>
- val callee = resolveCalleeName(funcNode)
- callee.foreach { name =>
- if (name != callerName && !GoBuiltins.contains(name) &&
- !GoKeywords.contains(name) && !seen.contains(name) && name != fileName) {
- seen += name
- relationships = relationships :+ ParsedRelationship(callerName, name, "CALLS")
- }
- }
- }
- }
- }
-
- private def resolveCalleeName(funcNode: TSNode): Option[String] = {
- funcNode.getType() match {
- case "identifier" =>
- Some(nodeText(funcNode, source))
- case "selector_expression" =>
- // e.g., pkg.Func() or obj.Method() — extract the field (rightmost)
- fieldChild(funcNode, "field").map(n => nodeText(n, source))
- case "call_expression" =>
- // Chained calls — recurse
- fieldChild(funcNode, "function").flatMap(resolveCalleeName)
- case "parenthesized_expression" =>
- // (func)() — recurse into the inner expression
- namedChildren(funcNode).headOption.flatMap(resolveCalleeName)
- case _ =>
- None
- }
- }
-
- // --- Helpers ---
-
- private def extractFirstLine(node: TSNode): String = {
- val text = nodeText(node, source)
- text.split("\n").head
- }
- }
-
- // --- Shared constants ---
-
- private val GoKeywords: Set[String] = Set(
- "break", "case", "chan", "const", "continue", "default", "defer",
- "else", "fallthrough", "for", "func", "go", "goto", "if",
- "import", "interface", "map", "package", "range", "return",
- "select", "struct", "switch", "type", "var"
- )
-
- private val GoBuiltins: Set[String] = Set(
- "make", "len", "cap", "append", "copy", "close", "delete",
- "complex", "real", "imag", "new", "panic", "recover",
- "print", "println", "error", "string", "int", "float64",
- "bool", "byte", "rune", "nil", "true", "false", "iota",
- "int8", "int16", "int32", "int64",
- "uint", "uint8", "uint16", "uint32", "uint64", "uintptr",
- "float32", "complex64", "complex128", "any", "comparable"
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala
deleted file mode 100644
index deb06bdd..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala
+++ /dev/null
@@ -1,250 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-import scala.util.matching.Regex
-
-/**
- * Python source parser that extracts structural entities and relationships.
- *
- * Attempts to use tree-sitter JNI first; falls back to regex-based extraction
- * when the native library is unavailable (common on some platforms).
- * The regex fallback is sufficient for Phase 1 since the goal is the pipeline
- * structure, not production-grade parsing.
- */
-class TreeSitterPythonParser extends Parser {
-
- /**
- * Parse a Python source file and extract entities and relationships.
- *
- * @param fileName the file name (e.g. "billing_service.py")
- * @param source the full source code as a string
- * @return ParseResult with entities and relationships
- */
- def parse(fileName: String, source: String): ParseResult = {
- // Try tree-sitter first, fall back to regex
- tryTreeSitter(fileName, source).getOrElse(regexParse(fileName, source))
- }
-
- // ---------------------------------------------------------------------------
- // Tree-sitter JNI attempt
- // ---------------------------------------------------------------------------
-
- private def tryTreeSitter(fileName: String, source: String): Option[ParseResult] = {
- try {
- val langClass = Class.forName("ch.usi.si.seart.treesitter.Language")
- val pythonLang = langClass.getMethod("valueOf", classOf[String]).invoke(null, "PYTHON")
-
- val parserClass = Class.forName("ch.usi.si.seart.treesitter.Parser")
- val parser = parserClass.getMethod("getFor", langClass).invoke(null, pythonLang)
-
- val tree = parserClass.getMethod("parse", classOf[String]).invoke(parser, source)
- val treeClass = Class.forName("ch.usi.si.seart.treesitter.Tree")
- val root = treeClass.getMethod("getRootNode").invoke(tree)
-
- // If we get here, tree-sitter loaded successfully
- // We would walk the AST here, but for now this path is not reached
- // on platforms where the JNI library is unavailable.
- parserClass.getMethod("close").invoke(parser)
- treeClass.getMethod("close").invoke(tree)
- None // TODO: implement full tree-sitter AST walking when JNI works
- } catch {
- case _: Throwable => None // Catches both Exception and Error (e.g. UnsatisfiedLinkError)
- }
- }
-
- // ---------------------------------------------------------------------------
- // Regex-based fallback parser
- // ---------------------------------------------------------------------------
-
- private val ClassPattern: Regex = """^class\s+(\w+)""".r
- private val FuncPattern: Regex = """^(\s*)def\s+(\w+)""".r
- private val ImportPattern: Regex = """^import\s+(\w+)""".r
- private val FromImportPattern: Regex = """^from\s+(\w+)\s+import""".r
- private val CallPattern: Regex = """(?:self\.)?(\w+)\(""".r
-
- private def regexParse(fileName: String, source: String): ParseResult = {
- val lines = source.split("\n", -1)
-
- // -- File entity --
- val fileEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("python")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- // -- Classes and functions --
- var entities = Vector(fileEntity)
- var relationships = Vector.empty[ParsedRelationship]
-
- // Track class ranges: (className, lineStart, lineEnd) for containment checks
- var classRanges = Vector.empty[(String, Int, Int)]
-
- for ((line, idx) <- lines.zipWithIndex) {
- val lineNum = idx + 1
-
- // Skip comment lines — prevents matching patterns inside comments
- if (line.trim.startsWith("#")) {
- // no-op: skip this line entirely
- } else {
-
- // Check for class definition
- ClassPattern.findFirstMatchIn(line).foreach { m =>
- val className = m.group(1)
- val classEnd = findBlockEnd(lines, idx)
- val bodyText = lines.slice(idx, classEnd).mkString("\n")
- val fp = Fingerprint.compute(line.trim.take(120), bodyText)
- entities = entities :+ ParsedEntity(
- name = className,
- kind = NodeKind.Class,
- attrs = Map("language" -> Json.fromString("python")),
- lineStart = lineNum,
- lineEnd = classEnd,
- contentFingerprint = Some(fp)
- )
- // File CONTAINS class
- relationships = relationships :+ ParsedRelationship(fileName, className, "CONTAINS")
- classRanges = classRanges :+ (className, lineNum, classEnd)
- }
-
- // Check for function/method definition
- FuncPattern.findFirstMatchIn(line).foreach { m =>
- val funcName = m.group(2)
- val funcEnd = findBlockEnd(lines, idx)
- val bodyText = lines.slice(idx, funcEnd).mkString("\n")
- val fp = Fingerprint.compute(line.trim.take(120), bodyText)
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map("language" -> Json.fromString("python")),
- lineStart = lineNum,
- lineEnd = funcEnd,
- contentFingerprint = Some(fp)
- )
-
- // Check if this function's line falls within any class range
- val enclosingClass = classRanges.find { case (_, start, end) =>
- lineNum > start && lineNum <= end
- }
- enclosingClass match {
- case Some((className, _, _)) =>
- // Method inside a class
- relationships = relationships :+ ParsedRelationship(className, funcName, "CONTAINS")
- case None =>
- // Top-level function — file defines it
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
- }
-
- // Extract function calls within the function body
- val callsInFunc = extractCalls(lines, idx, funcName)
- relationships = relationships ++ callsInFunc
- }
-
- // Check for imports
- ImportPattern.findFirstMatchIn(line).foreach { m =>
- val moduleName = m.group(1)
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
-
- FromImportPattern.findFirstMatchIn(line).foreach { m =>
- val moduleName = m.group(1)
- // Avoid duplicate module entities
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
-
- } // end else (non-comment line)
- }
-
- ParseResult(entities, relationships)
- }
-
- /**
- * Find the end line of a block starting at `startIdx` using indentation.
- * Looks for the last line that has greater indentation than the definition line.
- */
- private def findBlockEnd(lines: Array[String], startIdx: Int): Int = {
- val baseIndent = lines(startIdx).takeWhile(_ == ' ').length
- var endIdx = startIdx
- var i = startIdx + 1
- while (i < lines.length) {
- val line = lines(i)
- if (line.trim.nonEmpty) {
- val indent = line.takeWhile(_ == ' ').length
- if (indent > baseIndent) {
- endIdx = i
- } else {
- // Block ended
- return endIdx + 1
- }
- }
- i += 1
- }
- // If we reach end of file, the block extends to the end
- endIdx + 1
- }
-
- /**
- * Extract CALLS relationships from the body of a function starting at `funcIdx`.
- * Uses a simple heuristic: look for `identifier(` patterns in subsequent indented lines.
- */
- private def extractCalls(lines: Array[String], funcIdx: Int, callerName: String): Vector[ParsedRelationship] = {
- val baseIndent = lines(funcIdx).takeWhile(_ == ' ').length
- var calls = Vector.empty[ParsedRelationship]
- val seen = scala.collection.mutable.Set.empty[String]
- var i = funcIdx + 1
-
- while (i < lines.length) {
- val line = lines(i)
- if (line.trim.nonEmpty) {
- val indent = line.takeWhile(_ == ' ').length
- if (indent <= baseIndent) {
- // Exited the function body
- return calls
- }
- // Find calls in this line
- CallPattern.findAllMatchIn(line).foreach { m =>
- val callee = m.group(1)
- // Filter out Python builtins and self-references, and skip 'self' prefix calls
- // that match the method name itself
- if (callee != callerName && !PythonBuiltins.contains(callee) && !seen.contains(callee)) {
- seen += callee
- calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
- }
- }
- }
- i += 1
- }
- calls
- }
-
- private val PythonBuiltins: Set[String] = Set(
- "print", "len", "range", "int", "str", "float", "list", "dict",
- "set", "tuple", "type", "isinstance", "issubclass", "hasattr",
- "getattr", "setattr", "delattr", "super", "property", "staticmethod",
- "classmethod", "enumerate", "zip", "map", "filter", "sorted",
- "reversed", "any", "all", "min", "max", "sum", "abs", "round",
- "input", "open", "format", "repr", "id", "hash", "callable",
- "iter", "next", "not", "raise"
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterScalaParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterScalaParser.scala
deleted file mode 100644
index 09693c7a..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterScalaParser.scala
+++ /dev/null
@@ -1,350 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-import org.treesitter.{TSNode, TreeSitterScala}
-
-import scala.util.matching.Regex
-
-/**
- * Tree-sitter AST-based Scala parser with regex fallback.
- * Produces the same ParseResult shape as the original regex ScalaParser.
- */
-class TreeSitterScalaParser extends Parser {
-
- private val regexFallback = new ScalaParser()
-
- def parse(fileName: String, source: String): ParseResult = {
- try treeSitterParse(fileName, source)
- catch { case _: Throwable => regexFallback.parse(fileName, source) }
- }
-
- private def treeSitterParse(fileName: String, source: String): ParseResult = {
- TreeSitterUtils.withParse(new TreeSitterScala(), source) { root =>
- val ctx = new ScalaExtractionContext(fileName, source, root)
- ctx.result()
- }
- }
-
- // -------------------------------------------------------------------------
- // Extraction context — accumulates entities and relationships
- // -------------------------------------------------------------------------
-
- private class ScalaExtractionContext(
- fileName: String,
- source: String,
- root: TSNode
- ) {
- import TreeSitterUtils._
-
- var entities = Vector.empty[ParsedEntity]
- var relationships = Vector.empty[ParsedRelationship]
-
- // Initialize with file entity
- entities = entities :+ ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("scala")),
- lineStart = 1,
- lineEnd = source.split("\n", -1).length
- )
-
- // Walk the root
- walkChildren(root, enclosing = None)
-
- def result(): ParseResult = ParseResult(entities, relationships)
-
- private def walkChildren(node: TSNode, enclosing: Option[String]): Unit = {
- for (child <- namedChildren(node)) {
- walkNode(child, enclosing)
- }
- }
-
- private def walkNode(node: TSNode, enclosing: Option[String]): Unit = {
- node.getType() match {
- case "object_definition" =>
- handleTypeDefinition(node, enclosing, NodeKind.Object, "object")
- case "class_definition" =>
- val isCaseClass = hasCaseModifier(node)
- val scalaKind = if (isCaseClass) "case_class" else "class"
- handleTypeDefinition(node, enclosing, NodeKind.Class, scalaKind)
- case "trait_definition" =>
- handleTypeDefinition(node, enclosing, NodeKind.Trait, "trait")
- case "function_definition" =>
- handleFunctionDefinition(node, enclosing)
- case "import_declaration" =>
- handleImport(node)
- case "template_body" =>
- walkChildren(node, enclosing)
- case "package_clause" =>
- // recurse into package body
- walkChildren(node, enclosing)
- case _ =>
- // Recurse into other nodes to catch nested definitions
- walkChildren(node, enclosing)
- }
- }
-
- private def handleTypeDefinition(
- node: TSNode,
- enclosing: Option[String],
- kind: NodeKind,
- scalaKind: String
- ): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { name =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val sig = extractTypeSignature(node)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
- val vis = extractVisibility(node)
-
- entities = entities :+ ParsedEntity(
- name = name,
- kind = kind,
- attrs = Map(
- "scala_kind" -> Json.fromString(scalaKind),
- "language" -> Json.fromString("scala"),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(vis)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
-
- // CONTAINS edge
- val containsSrc = enclosing.getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
-
- // EXTENDS edge
- findChild(node, "extends_clause").foreach { extendsNode =>
- // First type in extends clause is the parent
- val types = collectDescendants(extendsNode, n =>
- n.getType() == "type_identifier" || n.getType() == "identifier"
- )
- types.headOption.foreach { typeNode =>
- val parentName = nodeText(typeNode, source)
- if (parentName.nonEmpty && parentName.head.isUpper) {
- relationships = relationships :+ ParsedRelationship(name, parentName, "EXTENDS")
- }
- }
- }
-
- // Recurse into template_body
- findChild(node, "template_body").orElse(findChild(node, "body")).foreach { body =>
- walkChildren(body, Some(name))
- }
- }
- }
-
- private def handleFunctionDefinition(node: TSNode, enclosing: Option[String]): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { funcName =>
- val start = lineStart(node)
- val end = lineEnd(node)
-
- val (kind, containsSrc) = enclosing match {
- case Some(typeName) => (NodeKind.Method, typeName)
- case None => (NodeKind.Function, fileName)
- }
-
- val sig = extractFunctionSignature(node)
- val vis = extractVisibility(node)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = kind,
- attrs = Map(
- "language" -> Json.fromString("scala"),
- "summary" -> Json.fromString(sig),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(vis)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
-
- relationships = relationships :+ ParsedRelationship(containsSrc, funcName, "CONTAINS")
-
- // Extract CALLS from body
- fieldChild(node, "body").foreach { body =>
- extractCalls(body, funcName, enclosing.getOrElse(""))
- }
-
- // Extract REFERENCES from signature types
- extractSignatureRefs(node, funcName, enclosing.getOrElse(""))
- }
- }
-
- private def extractCalls(bodyNode: TSNode, callerName: String, enclosingTypeName: String): Unit = {
- val seen = scala.collection.mutable.Set.empty[String]
- val refSeen = scala.collection.mutable.Set.empty[String]
-
- val callExprs = collectDescendants(bodyNode, _.getType() == "call_expression")
- for (callExpr <- callExprs) {
- // Scala grammar: call_expression first named child is the function
- val funcNodeOpt = fieldChild(callExpr, "function")
- .orElse(if (callExpr.getNamedChildCount() > 0) Some(callExpr.getNamedChild(0)) else None)
-
- funcNodeOpt.foreach { funcNode =>
- val callee = resolveScalaCallee(funcNode)
- callee.foreach { name =>
- if (name != callerName && !ScalaBuiltins.contains(name) &&
- !seen.contains(name) && name.nonEmpty && name.head.isLower) {
- seen += name
- relationships = relationships :+ ParsedRelationship(callerName, name, "CALLS")
- }
- }
- }
- }
-
- // Extract qualified REFERENCES (e.g., NodeKind.File, SourceType.Code)
- // Scala tree-sitter grammar: field_expression has plain identifier children
- // (first child = object, third child = field, with "." in between)
- val fieldExprs = collectDescendants(bodyNode, _.getType() == "field_expression")
- for (fe <- fieldExprs) {
- // Try named field first, then fall back to first named child as qualifier
- val objNode = fieldChild(fe, "object")
- .orElse(fieldChild(fe, "qualifier"))
- .orElse {
- val named = namedChildren(fe)
- named.headOption.filter(_.getType() == "identifier")
- }
- objNode.foreach { on =>
- if (on.getType() == "identifier") {
- val qualifier = nodeText(on, source)
- if (qualifier.nonEmpty && qualifier.head.isUpper &&
- !ScalaBuiltinTypes.contains(qualifier) && !ScalaBuiltins.contains(qualifier) &&
- !refSeen.contains(qualifier) && qualifier != callerName &&
- qualifier != enclosingTypeName) {
- refSeen += qualifier
- relationships = relationships :+ ParsedRelationship(callerName, qualifier, "REFERENCES")
- }
- }
- }
- }
- }
-
- private def resolveScalaCallee(funcNode: TSNode): Option[String] = {
- funcNode.getType() match {
- case "identifier" =>
- Some(nodeText(funcNode, source))
- case "field_expression" =>
- // Scala field_expression: children are [object, ".", field]
- // Last named child is the field/method name
- val named = namedChildren(funcNode)
- named.lastOption.map(n => nodeText(n, source))
- case "call_expression" =>
- // Chained calls — recurse
- val inner = if (funcNode.getNamedChildCount() > 0) Some(funcNode.getNamedChild(0)) else None
- inner.flatMap(resolveScalaCallee)
- case _ => None
- }
- }
-
- private def extractSignatureRefs(node: TSNode, funcName: String, enclosingTypeName: String): Unit = {
- // Use regex on the signature text for type references — pragmatic hybrid approach
- val sig = extractFunctionSignature(node)
- val TypeRefPattern: Regex = """\b([A-Z]\w+)""".r
- val refs = TypeRefPattern.findAllMatchIn(sig).map(_.group(1)).toSet
- for (ref <- refs) {
- if (!ScalaBuiltinTypes.contains(ref) && ref != funcName &&
- ref != enclosingTypeName && ref != fileName) {
- relationships = relationships :+ ParsedRelationship(funcName, ref, "REFERENCES")
- }
- }
- }
-
- private def handleImport(node: TSNode): Unit = {
- val importText = nodeText(node, source)
- .stripPrefix("import ")
- .trim
-
- val moduleName = if (importText.contains("{")) {
- importText.substring(0, importText.indexOf('{')).stripSuffix(".").trim
- } else {
- importText
- }
-
- if (moduleName.nonEmpty && !entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineStart(node),
- lineEnd = lineEnd(node)
- )
- }
- if (moduleName.nonEmpty) {
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
- }
-
- // --- Helpers ---
-
- private def hasCaseModifier(node: TSNode): Boolean = {
- // Check for "case" modifier in the node text before the keyword
- val text = nodeText(node, source)
- text.trim.startsWith("case ") || text.contains(" case class") ||
- // Check modifiers children
- allChildren(node).exists { child =>
- child.getType() == "case" ||
- (child.getType() == "modifiers" && nodeText(child, source).contains("case"))
- }
- }
-
- private def extractTypeSignature(node: TSNode): String = {
- val text = nodeText(node, source)
- val firstLine = text.split("\n").head
- firstLine.replaceAll("""\s*\{\s*$""", "").trim.take(120)
- }
-
- private def extractFunctionSignature(node: TSNode): String = {
- val text = nodeText(node, source)
- val sig = text.split("\n").head
- .replaceAll("""\s*=\s*\{?\s*$""", "")
- .replaceAll("""\s*\{\s*$""", "")
- .trim
- sig.take(120)
- }
-
- private def extractVisibility(node: TSNode): String = {
- val text = nodeText(node, source).trim
- val withoutOverride = text.replaceFirst("""^\s*override\s+""", "").trim
- if (withoutOverride.startsWith("private")) "private"
- else if (withoutOverride.startsWith("protected")) "protected"
- else "public"
- }
- }
-
- // --- Shared constants (reused from ScalaParser) ---
-
- private val ScalaBuiltinTypes: Set[String] = Set(
- "String", "Int", "Long", "Double", "Float", "Boolean", "Unit", "Any", "AnyRef", "AnyVal",
- "Nothing", "Null", "Option", "List", "Map", "Set", "Vector", "Seq", "Array", "Future",
- "IO", "Either", "Try", "Some", "None", "Right", "Left", "Success", "Failure"
- )
-
- private val ScalaBuiltins: Set[String] = Set(
- "println", "print", "require", "assert", "assume",
- "Some", "None", "Option", "List", "Map", "Set", "Vector",
- "Right", "Left", "Try", "Success", "Failure",
- "Future", "IO", "pure", "flatMap", "map", "filter",
- "foreach", "fold", "foldLeft", "foldRight",
- "toString", "hashCode", "equals", "getClass",
- "apply", "unapply", "copy", "andThen", "compose",
- "getOrElse", "orElse", "contains", "exists", "forall",
- "collect", "traverse", "sequence", "attempt", "handleErrorWith",
- "raiseError", "blocking", "delay", "suspend", "async",
- "mkString", "toList", "toVector", "toSet", "toMap",
- "head", "tail", "size", "length", "isEmpty", "nonEmpty",
- "take", "drop", "slice", "zip", "zipWithIndex",
- "sortBy", "groupBy", "distinct", "flatten"
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterTypeScriptParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterTypeScriptParser.scala
deleted file mode 100644
index 69cdac18..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterTypeScriptParser.scala
+++ /dev/null
@@ -1,531 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-import org.treesitter.{TSNode, TreeSitterTypescript}
-
-/**
- * Tree-sitter AST-based TypeScript parser with regex fallback.
- * Produces the same ParseResult shape as the original regex TypeScriptParser.
- */
-class TreeSitterTypeScriptParser extends Parser {
-
- private val regexFallback = new TypeScriptParser()
-
- def parse(fileName: String, source: String): ParseResult = {
- try treeSitterParse(fileName, source)
- catch { case _: Throwable => regexFallback.parse(fileName, source) }
- }
-
- private def treeSitterParse(fileName: String, source: String): ParseResult = {
- TreeSitterUtils.withParse(new TreeSitterTypescript(), source) { root =>
- val ctx = new TSExtractionContext(fileName, source, root)
- ctx.result()
- }
- }
-
- // -------------------------------------------------------------------------
- // Extraction context
- // -------------------------------------------------------------------------
-
- private class TSExtractionContext(
- fileName: String,
- source: String,
- root: TSNode
- ) {
- import TreeSitterUtils._
-
- var entities = Vector.empty[ParsedEntity]
- var relationships = Vector.empty[ParsedRelationship]
-
- private val lines = source.split("\n", -1)
-
- // Initialize with file entity
- entities = entities :+ ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("typescript")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- // Walk the program
- walkProgram(root)
-
- def result(): ParseResult = ParseResult(entities, relationships)
-
- private def walkProgram(node: TSNode): Unit = {
- for (child <- namedChildren(node)) {
- walkTopLevel(child, exported = false)
- }
- }
-
- private def walkTopLevel(node: TSNode, exported: Boolean): Unit = {
- node.getType() match {
- case "export_statement" =>
- // The actual declaration is a child
- for (child <- namedChildren(node)) {
- walkTopLevel(child, exported = true)
- }
-
- case "class_declaration" | "abstract_class_declaration" =>
- handleClassDeclaration(node, exported)
-
- case "function_declaration" =>
- handleFunctionDeclaration(node, exported)
-
- case "lexical_declaration" | "variable_declaration" =>
- handleVariableDeclaration(node, exported)
-
- case "interface_declaration" =>
- handleInterfaceDeclaration(node, exported)
-
- case "type_alias_declaration" =>
- handleTypeAliasDeclaration(node, exported)
-
- case "import_statement" =>
- handleImport(node)
-
- case "expression_statement" =>
- // File-level code — extract CALLS
- extractFileLevelCalls(node)
-
- case "enum_declaration" =>
- handleEnumDeclaration(node, exported)
-
- case _ =>
- // Ignore other top-level nodes
- }
- }
-
- // --- Class handling ---
-
- private def handleClassDeclaration(node: TSNode, exported: Boolean): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { className =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val sig = extractFirstLine(node).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = className,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, className, "CONTAINS")
-
- // Extract methods from class body
- findChild(node, "class_body").foreach { body =>
- extractMethods(body, className)
- }
- }
- }
-
- private def extractMethods(classBody: TSNode, className: String): Unit = {
- for (child <- namedChildren(classBody)) {
- child.getType() match {
- case "method_definition" =>
- handleMethodDefinition(child, className)
- case "public_field_definition" | "property_definition" =>
- // Skip property definitions
- case _ =>
- // Skip other class body members
- }
- }
- }
-
- private def handleMethodDefinition(node: TSNode, className: String): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { methodName =>
- // Skip constructor
- if (methodName != "constructor" && !TypeScriptKeywords.contains(methodName)) {
- val start = lineStart(node)
- val end = lineEnd(node)
- val summary = extractFirstLine(node).trim.take(120)
- val vis = extractVisibility(node)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(summary, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = methodName,
- kind = NodeKind.Method,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString(vis)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(className, methodName, "CONTAINS")
-
- // Extract calls from method body
- fieldChild(node, "body").foreach { body =>
- extractCallsFromBody(body, methodName)
- }
- }
- }
- }
-
- // --- Function handling ---
-
- private def handleFunctionDeclaration(node: TSNode, exported: Boolean): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { funcName =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val summary = extractFirstLine(node).trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(summary, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString("public"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
-
- // Extract calls from function body
- fieldChild(node, "body").foreach { body =>
- extractCallsFromBody(body, funcName)
- }
- }
- }
-
- // --- Variable/arrow function handling ---
-
- private def handleVariableDeclaration(node: TSNode, exported: Boolean): Unit = {
- for (declarator <- childrenOfType(node, "variable_declarator")) {
- val nameOpt = extractName(declarator, source)
- nameOpt.foreach { varName =>
- // Check if the value is an arrow function or function expression
- fieldChild(declarator, "value").foreach { value =>
- val isFunc = value.getType() == "arrow_function" ||
- value.getType() == "function" ||
- value.getType() == "function_expression"
- if (isFunc && !entities.exists(e => e.name == varName && e.kind == NodeKind.Function)) {
- val start = lineStart(node)
- val end = lineEnd(node)
- val summary = extractFirstLine(node).trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(summary, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = varName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString("public"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, varName, "CONTAINS")
-
- // Extract calls from function body
- fieldChild(value, "body").foreach { body =>
- extractCallsFromBody(body, varName)
- }
- }
- }
- }
- }
- }
-
- // --- Interface handling ---
-
- private def handleInterfaceDeclaration(node: TSNode, exported: Boolean): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { ifaceName =>
- if (!entities.exists(e => e.name == ifaceName && e.kind == NodeKind.Interface)) {
- val start = lineStart(node)
- val end = lineEnd(node)
- val sig = extractFirstLine(node).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = ifaceName,
- kind = NodeKind.Interface,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("interface"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, ifaceName, "CONTAINS")
- }
- }
- }
-
- // --- Type alias handling ---
-
- private def handleTypeAliasDeclaration(node: TSNode, exported: Boolean): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { typeName =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val text = nodeText(node, source).trim
- val fp = Fingerprint.compute(text.take(120), text)
-
- entities = entities :+ ParsedEntity(
- name = typeName,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("type_alias"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
- }
- }
-
- // --- Enum handling ---
-
- private def handleEnumDeclaration(node: TSNode, exported: Boolean): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { enumName =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(extractFirstLine(node).trim.take(120), bodyText)
-
- entities = entities :+ ParsedEntity(
- name = enumName,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("enum"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, enumName, "CONTAINS")
- }
- }
-
- // --- Import handling ---
-
- private def handleImport(node: TSNode): Unit = {
- // Extract module path from the source/string child
- val moduleOpt = findChild(node, "string").map(n => nodeText(n, source))
- .map(_.stripPrefix("\"").stripSuffix("\"").stripPrefix("'").stripSuffix("'"))
-
- moduleOpt.foreach { moduleName =>
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineStart(node),
- lineEnd = lineEnd(node)
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
-
- // Extract named imports
- findChild(node, "import_clause").foreach { clause =>
- // Default import: import Foo from "mod"
- findChild(clause, "identifier").foreach { id =>
- val name = nodeText(id, source)
- if (name.nonEmpty && !relationships.exists(r =>
- r.srcName == fileName && r.dstName == name && r.predicate == "IMPORTS")) {
- relationships = relationships :+ ParsedRelationship(fileName, name, "IMPORTS")
- }
- }
-
- // Named imports: import { a, b as c } from "mod"
- findChild(clause, "named_imports").foreach { namedImports =>
- for (spec <- childrenOfType(namedImports, "import_specifier")) {
- // Use alias if present, otherwise use name
- val alias = fieldChild(spec, "alias").map(n => nodeText(n, source))
- val importName = alias.orElse(extractName(spec, source))
- importName.foreach { name =>
- if (!relationships.exists(r =>
- r.srcName == fileName && r.dstName == name && r.predicate == "IMPORTS")) {
- relationships = relationships :+ ParsedRelationship(fileName, name, "IMPORTS")
- }
- }
- }
- }
-
- // Namespace import: import * as foo from "mod"
- findChild(clause, "namespace_import").foreach { nsImport =>
- findChild(nsImport, "identifier").foreach { id =>
- val name = nodeText(id, source)
- if (name.nonEmpty && !relationships.exists(r =>
- r.srcName == fileName && r.dstName == name && r.predicate == "IMPORTS")) {
- relationships = relationships :+ ParsedRelationship(fileName, name, "IMPORTS")
- }
- }
- }
- }
- }
-
- // Handle side-effect imports (no import clause, just `import "mod"`)
- if (findChild(node, "import_clause").isEmpty) {
- findChild(node, "string").foreach { strNode =>
- val moduleName = nodeText(strNode, source)
- .stripPrefix("\"").stripSuffix("\"")
- .stripPrefix("'").stripSuffix("'")
- if (moduleName.nonEmpty) {
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineStart(node),
- lineEnd = lineEnd(node)
- )
- }
- if (!relationships.exists(r => r.dstName == moduleName && r.predicate == "IMPORTS")) {
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
- }
- }
- }
- }
-
- // --- CALLS extraction ---
-
- private def extractCallsFromBody(bodyNode: TSNode, callerName: String): Unit = {
- val seen = scala.collection.mutable.Set.empty[String]
- val callExprs = collectDescendants(bodyNode, _.getType() == "call_expression")
-
- for (callExpr <- callExprs) {
- fieldChild(callExpr, "function").foreach { funcNode =>
- val callee = resolveCalleeName(funcNode)
- callee.foreach { name =>
- if (name != callerName && !TypeScriptBuiltins.contains(name) &&
- !TypeScriptKeywords.contains(name) && !seen.contains(name) && name != fileName) {
- seen += name
- relationships = relationships :+ ParsedRelationship(callerName, name, "CALLS")
- }
- }
- }
- }
- }
-
- private def extractFileLevelCalls(exprStatement: TSNode): Unit = {
- val callExprs = collectDescendants(exprStatement, _.getType() == "call_expression")
- for (callExpr <- callExprs) {
- fieldChild(callExpr, "function").foreach { funcNode =>
- val callee = resolveCalleeName(funcNode)
- callee.foreach { name =>
- if (!TypeScriptBuiltins.contains(name) && !TypeScriptKeywords.contains(name) &&
- name != fileName && !relationships.exists(r =>
- r.srcName == fileName && r.dstName == name && r.predicate == "CALLS")) {
- relationships = relationships :+ ParsedRelationship(fileName, name, "CALLS")
- }
- }
- }
- }
- }
-
- private def resolveCalleeName(funcNode: TSNode): Option[String] = {
- funcNode.getType() match {
- case "identifier" =>
- Some(nodeText(funcNode, source))
- case "member_expression" =>
- // e.g., this.method(), obj.method() — extract the property name
- fieldChild(funcNode, "property").map(n => nodeText(n, source))
- case "call_expression" =>
- // Chained: foo.bar().baz() — take the outermost function
- fieldChild(funcNode, "function").flatMap(resolveCalleeName)
- case _ =>
- None
- }
- }
-
- // --- Helpers ---
-
- private def extractFirstLine(node: TSNode): String = {
- val text = nodeText(node, source)
- text.split("\n").head
- }
-
- private def extractVisibility(node: TSNode): String = {
- // Check for accessibility_modifier child
- val modifier = allChildren(node).find(c =>
- c.getType() == "accessibility_modifier" ||
- c.getType() == "private" || c.getType() == "public" || c.getType() == "protected"
- )
- modifier match {
- case Some(m) =>
- val text = nodeText(m, source).trim
- if (text == "private") "private"
- else if (text == "protected") "protected"
- else "public"
- case None =>
- // Also check the first line text
- val firstLine = extractFirstLine(node).trim
- if (firstLine.startsWith("private ") || firstLine.startsWith("private\t")) "private"
- else if (firstLine.startsWith("protected ") || firstLine.startsWith("protected\t")) "protected"
- else "public"
- }
- }
- }
-
- // --- Shared constants (reused from TypeScriptParser) ---
-
- private val TypeScriptKeywords: Set[String] = Set(
- "if", "else", "for", "while", "do", "switch", "case", "break",
- "continue", "return", "throw", "try", "catch", "finally",
- "new", "delete", "typeof", "instanceof", "void", "in", "of",
- "class", "extends", "implements", "interface", "enum", "type",
- "const", "let", "var", "function", "async", "await",
- "import", "export", "default", "from", "as",
- "this", "super", "constructor", "static", "private", "public", "protected"
- )
-
- private val TypeScriptBuiltins: Set[String] = Set(
- "console", "log", "error", "warn", "parseInt", "parseFloat",
- "String", "Number", "Boolean", "Array", "Object", "Promise",
- "JSON", "Math", "Date", "RegExp", "Error", "Map", "Set",
- "WeakMap", "WeakSet", "Symbol",
- "setTimeout", "setInterval", "clearTimeout", "clearInterval",
- "require", "module", "exports",
- "fetch", "Response", "Request", "URL", "Buffer", "process",
- "map", "filter", "reduce", "forEach", "sort", "find", "some", "every",
- "includes", "indexOf", "slice", "splice", "push", "pop", "shift", "unshift",
- "concat", "join", "flat", "flatMap", "fill", "from",
- "split", "replace", "trim", "toUpperCase", "toLowerCase", "substring",
- "charAt", "startsWith", "endsWith", "padStart", "padEnd", "match", "test",
- "keys", "values", "entries", "assign", "create", "freeze",
- "defineProperty", "getOwnPropertyNames", "hasOwnProperty",
- "then", "catch", "finally", "all", "race", "resolve", "reject"
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterUtils.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterUtils.scala
deleted file mode 100644
index 7c69a61c..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterUtils.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import org.treesitter.{TSNode, TSParser, TSTree, TSLanguage}
-
-object TreeSitterUtils {
-
- /** 1-based line start */
- def lineStart(node: TSNode): Int = node.getStartPoint().getRow() + 1
-
- /** 1-based line end */
- def lineEnd(node: TSNode): Int = node.getEndPoint().getRow() + 1
-
- /** Extract source text for a node */
- def nodeText(node: TSNode, source: String): String =
- source.substring(node.getStartByte(), node.getEndByte())
-
- /** Find named child by field name, returning None if null */
- def fieldChild(node: TSNode, field: String): Option[TSNode] = {
- val child = node.getChildByFieldName(field)
- if (child == null || child.isNull()) None else Some(child)
- }
-
- /** Find first named child with given type */
- def findChild(node: TSNode, nodeType: String): Option[TSNode] =
- namedChildren(node).find(_.getType() == nodeType)
-
- /** All named children */
- def namedChildren(node: TSNode): Vector[TSNode] =
- (0 until node.getNamedChildCount()).map(node.getNamedChild).toVector
-
- /** All named children of a given type */
- def childrenOfType(node: TSNode, nodeType: String): Vector[TSNode] =
- namedChildren(node).filter(_.getType() == nodeType)
-
- /** All children (named and anonymous) */
- def allChildren(node: TSNode): Vector[TSNode] =
- (0 until node.getChildCount()).map(node.getChild).toVector
-
- /** Recursively collect all descendants matching a predicate */
- def collectDescendants(node: TSNode, pred: TSNode => Boolean): Vector[TSNode] = {
- val buf = Vector.newBuilder[TSNode]
- def walk(n: TSNode): Unit = {
- if (pred(n)) buf += n
- for (i <- 0 until n.getChildCount()) walk(n.getChild(i))
- }
- walk(node)
- buf.result()
- }
-
- /** Extract identifier name from a node's "name" field */
- def extractName(node: TSNode, source: String): Option[String] =
- fieldChild(node, "name").map(n => nodeText(n, source))
-
- /** Loan pattern: parse source, run body, close parser+tree */
- def withParse[A](lang: TSLanguage, source: String)(body: TSNode => A): A = {
- val parser = new TSParser()
- parser.setLanguage(lang)
- val tree = parser.parseString(null, source)
- try body(tree.getRootNode())
- finally { tree.close(); parser.close() }
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala
deleted file mode 100644
index fc2670a2..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala
+++ /dev/null
@@ -1,422 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-import scala.util.matching.Regex
-
-/**
- * TypeScript/JavaScript source parser that extracts structural entities
- * and relationships using regex-based extraction.
- */
-class TypeScriptParser extends Parser {
-
- /**
- * Parse a TypeScript source file and extract entities and relationships.
- *
- * @param fileName the file name (e.g. "api.ts")
- * @param source the full source code as a string
- * @return ParseResult with entities and relationships
- */
- def parse(fileName: String, source: String): ParseResult = {
- val lines = source.split("\n", -1)
-
- // -- File entity (no raw content stored) --
- val fileEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("typescript")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- var entities = Vector(fileEntity)
- var relationships = Vector.empty[ParsedRelationship]
-
- // Track class ranges for containment
- var classRanges = Vector.empty[(String, Int, Int)]
-
- for ((line, idx) <- lines.zipWithIndex) {
- val lineNum = idx + 1
- val trimmed = line.trim
-
- // Skip comment lines
- if (trimmed.startsWith("//") || trimmed.startsWith("*") || trimmed.startsWith("/*")) {
- // no-op
- } else {
-
- // Check for class definition
- ClassPattern.findFirstMatchIn(line).foreach { m =>
- val className = m.group(1)
- val classEnd = findBraceBlockEnd(lines, idx)
- val bodyText = lines.slice(idx, classEnd).mkString("\n")
- val fp = Fingerprint.compute(trimmed.replaceAll("""\s*\{\s*$""", "").take(120), bodyText)
- val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
- entities = entities :+ ParsedEntity(
- name = className,
- kind = NodeKind.Class,
- attrs = Map("language" -> Json.fromString("typescript"), "exported" -> Json.fromBoolean(exported)),
- lineStart = lineNum,
- lineEnd = classEnd,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, className, "CONTAINS")
- classRanges = classRanges :+ (className, lineNum, classEnd)
-
- // Extract methods from this class
- val methods = extractMethods(lines, idx, classEnd, className)
- entities = entities ++ methods._1
- relationships = relationships ++ methods._2
- }
-
- // Check for top-level function (not inside a class)
- if (!insideClass(lineNum, classRanges)) {
- FuncPattern.findFirstMatchIn(line).foreach { m =>
- val funcName = m.group(1)
- val funcEnd = findBraceBlockEnd(lines, idx)
- val summary = line.trim.take(120)
- val bodyText = lines.slice(idx, funcEnd).mkString("\n")
- val fp = Fingerprint.compute(summary, bodyText)
- val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString("public"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = funcEnd,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
-
- // Extract calls within this function
- val calls = extractCalls(lines, idx, funcEnd, funcName)
- relationships = relationships ++ calls
- }
-
- // Check for arrow function / const function
- val arrowMatch = ArrowFuncPattern.findFirstMatchIn(line)
- val constFuncMatch = ConstFuncPattern.findFirstMatchIn(line)
- val funcMatch = arrowMatch.orElse(constFuncMatch)
- funcMatch.foreach { m =>
- val funcName = m.group(1)
- // Avoid duplicates if already captured by FuncPattern
- if (!entities.exists(e => e.name == funcName && e.kind == NodeKind.Function)) {
- val funcEnd = findBraceBlockEnd(lines, idx)
- val summary = line.trim.take(120)
- val bodyText = lines.slice(idx, funcEnd).mkString("\n")
- val fp = Fingerprint.compute(summary, bodyText)
- val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString("public"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = funcEnd,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
-
- val callRels = extractCalls(lines, idx, funcEnd, funcName)
- relationships = relationships ++ callRels
- }
- }
- }
-
- // Check for imports
- ImportPattern.findFirstMatchIn(line).foreach { m =>
- val moduleName = m.group(1)
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
-
- // Extract named import bindings: import { foo, bar } from "mod"
- NamedImportPattern.findFirstMatchIn(line).foreach { m =>
- val names = m.group(1).split(",").map(_.trim).filter(_.nonEmpty)
- for (rawName <- names) {
- val symbolName = if (rawName.contains(" as ")) rawName.split("\\s+as\\s+").last.trim else rawName
- if (symbolName.nonEmpty && !relationships.exists(r => r.srcName == fileName && r.dstName == symbolName && r.predicate == "IMPORTS")) {
- relationships = relationships :+ ParsedRelationship(fileName, symbolName, "IMPORTS")
- }
- }
- }
-
- SideEffectImportPattern.findFirstMatchIn(line).foreach { m =>
- val moduleName = m.group(1)
- if (!relationships.exists(r => r.dstName == moduleName && r.predicate == "IMPORTS")) {
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
- }
-
- // Check for interface definition
- InterfacePattern.findFirstMatchIn(line).foreach { m =>
- val ifaceName = m.group(1)
- if (!entities.exists(e => e.name == ifaceName && e.kind == NodeKind.Interface)) {
- val ifaceEnd = findBraceBlockEnd(lines, idx)
- val bodyText = lines.slice(idx, ifaceEnd).mkString("\n")
- val fp = Fingerprint.compute(trimmed.replaceAll("""\s*\{\s*$""", "").take(120), bodyText)
- val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
- entities = entities :+ ParsedEntity(
- name = ifaceName,
- kind = NodeKind.Interface,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("interface"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = ifaceEnd,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, ifaceName, "CONTAINS")
- }
- }
-
- // Check for type alias
- TypeAliasPattern.findFirstMatchIn(line).foreach { m =>
- val typeName = m.group(1)
- if (!trimmed.startsWith("import")) {
- val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
- val fp = Fingerprint.compute(trimmed.take(120), trimmed)
- entities = entities :+ ParsedEntity(
- name = typeName,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("type_alias"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = lineNum,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
- }
- }
-
- } // end else (non-comment)
- }
-
- // Extract file-level calls from code outside named functions/methods/classes
- val coveredRanges: Vector[(Int, Int)] = entities
- .filter(e => e.kind == NodeKind.Function || e.kind == NodeKind.Method || e.kind == NodeKind.Class)
- .map(e => (e.lineStart, e.lineEnd))
-
- val fileCallsSeen = scala.collection.mutable.Set.empty[String]
- for ((line, idx) <- lines.zipWithIndex) {
- val lineNum = idx + 1
- val inNamedBody = coveredRanges.exists { case (s, e) => lineNum >= s && lineNum <= e }
- if (!inNamedBody && !line.trim.startsWith("//") && !line.trim.startsWith("*")) {
- CallPattern.findAllMatchIn(line).foreach { m =>
- val callee = m.group(1)
- if (!TypeScriptBuiltins.contains(callee) && !TypeScriptKeywords.contains(callee)
- && !fileCallsSeen.contains(callee) && callee != fileName) {
- fileCallsSeen += callee
- relationships = relationships :+ ParsedRelationship(fileName, callee, "CALLS")
- }
- }
- }
- }
-
- ParseResult(entities, relationships)
- }
-
- // ---------------------------------------------------------------------------
- // Patterns
- // ---------------------------------------------------------------------------
-
- private val ClassPattern: Regex = """(?:export\s+)?(?:abstract\s+)?class\s+(\w+)""".r
- private val FuncPattern: Regex = """(?:export\s+)?(?:async\s+)?function\s+(\w+)""".r
- private val ArrowFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[a-zA-Z_]\w*)\s*(?::\s*[^=]+)?\s*=>""".r
- private val ConstFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?function""".r
- private val MethodPattern: Regex = """^\s+(?:async\s+)?(?:private\s+|public\s+|protected\s+|static\s+|readonly\s+)*(\w+)\s*(?:<[^>]+>)?\s*\(""".r
- private val ImportPattern: Regex = """import\s+.*?from\s+['"]([^'"]+)['"]""".r
- private val NamedImportPattern: Regex = """import\s+\{([^}]+)\}\s+from\s+['"]([^'"]+)['"]""".r
- private val SideEffectImportPattern: Regex = """^import\s+['"]([^'"]+)['"]""".r
- private val InterfacePattern: Regex = """(?:export\s+)?interface\s+(\w+)""".r
- private val TypeAliasPattern: Regex = """(?:export\s+)?type\s+(\w+)\s*(?:<[^>]+>)?\s*=""".r
- private val CallPattern: Regex = """(?:(?:this|self|await)\.)?\b(\w+)\s*\(""".r
-
- // ---------------------------------------------------------------------------
- // Helpers
- // ---------------------------------------------------------------------------
-
- private def insideClass(lineNum: Int, classRanges: Vector[(String, Int, Int)]): Boolean =
- classRanges.exists { case (_, start, end) => lineNum > start && lineNum <= end }
-
- private def extractVisibility(trimmedLine: String): String = {
- if (trimmedLine.startsWith("private ") || trimmedLine.startsWith("private\t")) "private"
- else if (trimmedLine.startsWith("protected ") || trimmedLine.startsWith("protected\t")) "protected"
- else "public"
- }
-
- /**
- * Extract methods from a class body.
- */
- private def extractMethods(
- lines: Array[String],
- classStartIdx: Int,
- classEndLine: Int,
- className: String
- ): (Vector[ParsedEntity], Vector[ParsedRelationship]) = {
- var entities = Vector.empty[ParsedEntity]
- var relationships = Vector.empty[ParsedRelationship]
-
- var i = classStartIdx + 1
- while (i < lines.length && (i + 1) <= classEndLine) {
- val line = lines(i)
- val lineNum = i + 1
-
- if (!line.trim.startsWith("//") && !line.trim.startsWith("*")) {
- MethodPattern.findFirstMatchIn(line).foreach { m =>
- val methodName = m.group(1)
- // Skip keywords that look like methods
- if (!TypeScriptKeywords.contains(methodName)) {
- val methodEnd = findBraceBlockEnd(lines, i)
- val summary = line.trim.take(120)
- val visibility = extractVisibility(line.trim)
- val bodyText = lines.slice(i, methodEnd).mkString("\n")
- val fp = Fingerprint.compute(summary, bodyText)
- entities = entities :+ ParsedEntity(
- name = methodName,
- kind = NodeKind.Method,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString(visibility)
- ),
- lineStart = lineNum,
- lineEnd = methodEnd,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(className, methodName, "CONTAINS")
-
- // Extract calls within this method
- val calls = extractCalls(lines, i, methodEnd, methodName)
- relationships = relationships ++ calls
- }
- }
- }
- i += 1
- }
-
- (entities, relationships)
- }
-
- /**
- * Find end of a brace-delimited block starting near `startIdx`.
- *
- * Uses end-of-line brace depth to detect block entry. This correctly
- * skips inline TypeScript type annotations like `{ kind?: string }`
- * that balance to zero on a single line, which would otherwise cause
- * the parser to think the function body ended on the parameter line.
- */
- private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Int = {
- var braceCount = 0
- var entered = false
- var i = startIdx
- while (i < lines.length) {
- for (ch <- lines(i)) {
- if (ch == '{') braceCount += 1
- if (ch == '}') braceCount -= 1
- }
- // Only mark as entered when braceCount > 0 at end of line.
- // This skips inline type annotations like { a: string; b: number }
- // that balance to 0 within the same line.
- if (!entered && braceCount > 0) entered = true
- if (entered && braceCount <= 0) return i + 1
- i += 1
- }
- lines.length
- }
-
- /**
- * Extract CALLS relationships from a function/method body.
- */
- private def extractCalls(
- lines: Array[String],
- funcStartIdx: Int,
- funcEndLine: Int,
- callerName: String
- ): Vector[ParsedRelationship] = {
- var calls = Vector.empty[ParsedRelationship]
- val seen = scala.collection.mutable.Set.empty[String]
- var i = funcStartIdx + 1
-
- while (i < lines.length && (i + 1) <= funcEndLine) {
- val line = lines(i)
- CallPattern.findAllMatchIn(line).foreach { m =>
- val callee = m.group(1)
- if (callee != callerName && !TypeScriptBuiltins.contains(callee) &&
- !TypeScriptKeywords.contains(callee) && !seen.contains(callee)) {
- seen += callee
- calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
- }
- }
- i += 1
- }
- calls
- }
-
- private val TypeScriptKeywords: Set[String] = Set(
- "if", "else", "for", "while", "do", "switch", "case", "break",
- "continue", "return", "throw", "try", "catch", "finally",
- "new", "delete", "typeof", "instanceof", "void", "in", "of",
- "class", "extends", "implements", "interface", "enum", "type",
- "const", "let", "var", "function", "async", "await",
- "import", "export", "default", "from", "as",
- "this", "super", "constructor", "static", "private", "public", "protected"
- )
-
- private val TypeScriptBuiltins: Set[String] = Set(
- "console", "log", "error", "warn", "parseInt", "parseFloat",
- "String", "Number", "Boolean", "Array", "Object", "Promise",
- "JSON", "Math", "Date", "RegExp", "Error", "Map", "Set",
- "WeakMap", "WeakSet", "Symbol",
- "setTimeout", "setInterval", "clearTimeout", "clearInterval",
- "require", "module", "exports",
- "fetch", "Response", "Request", "URL", "Buffer", "process",
- // Array/iterable methods
- "map", "filter", "reduce", "forEach", "sort", "find", "some", "every",
- "includes", "indexOf", "slice", "splice", "push", "pop", "shift", "unshift",
- "concat", "join", "flat", "flatMap", "fill", "from",
- // String methods
- "split", "replace", "trim", "toUpperCase", "toLowerCase", "substring",
- "charAt", "startsWith", "endsWith", "padStart", "padEnd", "match", "test",
- // Object methods
- "keys", "values", "entries", "assign", "create", "freeze",
- "defineProperty", "getOwnPropertyNames", "hasOwnProperty",
- // Promise methods
- "then", "catch", "finally", "all", "race", "resolve", "reject"
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala b/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala
index 1784f991..9cde2da4 100644
--- a/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala
+++ b/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala
@@ -1,6 +1,7 @@
package ix.memory.map
import cats.effect.IO
+import org.typelevel.log4cats.slf4j.Slf4jLogger
import ix.memory.db.ArangoClient
import ix.memory.model.{NodeId, Rev}
@@ -21,6 +22,7 @@ import ix.memory.model.{NodeId, Rev}
* ε = 0.2 path proximity (weak structural prior)
*/
class MapGraphBuilder(client: ArangoClient) {
+ private val logger = Slf4jLogger.getLoggerFromName[IO]("ix.map.build")
private val Alpha = 1.0
private val Beta = 0.5
@@ -32,7 +34,8 @@ class MapGraphBuilder(client: ArangoClient) {
private val SourceGraphRevisionKey = "source-graph"
private val IgnoredNodeKinds = Vector(
"module", "config", "config_entry", "doc", "decision", "intent",
- "bug", "plan", "task", "goal", "region"
+ "bug", "plan", "task", "goal", "region",
+ "chunk" // chunks are code-text segments, never CALLS/IMPORTS/EXTENDS/IMPLEMENTS endpoints
)
private val CouplingPredicates = Vector("CALLS", "IMPORTS", "EXTENDS", "IMPLEMENTS")
@@ -40,8 +43,14 @@ class MapGraphBuilder(client: ArangoClient) {
def buildGraph(files: Vector[FileVertex]): IO[WeightedFileGraph] =
for {
+ t0 <- IO(System.currentTimeMillis())
rawPairs <- fetchCouplingByUri()
+ t1 <- IO(System.currentTimeMillis())
graph = computeGraph(files, rawPairs)
+ t2 <- IO(System.currentTimeMillis())
+ _ <- logger.info(
+ s"map graph files=${files.size} raw_pairs=${rawPairs.size} fetch=${t1-t0}ms compute=${t2-t1}ms total=${t2-t0}ms"
+ )
} yield graph
def build(): IO[WeightedFileGraph] =
@@ -226,9 +235,7 @@ class MapGraphBuilder(client: ArangoClient) {
scala.collection.mutable.Map[(NodeId, NodeId), Map[String, Int]]()
var edgeCount = 0
- for (((srcId, dstId), predicateCounts) <- pairAcc.toVector.sortBy { case ((a, b), _) =>
- (a.value.toString, b.value.toString)
- }) {
+ for (((srcId, dstId), predicateCounts) <- pairAcc.iterator) {
val sv = byId(srcId)
val dv = byId(dstId)
val srcSegments = pathSegmentsById(srcId)
diff --git a/memory-layer/src/main/scala/ix/memory/map/MapService.scala b/memory-layer/src/main/scala/ix/memory/map/MapService.scala
index def439fb..a12622cb 100644
--- a/memory-layer/src/main/scala/ix/memory/map/MapService.scala
+++ b/memory-layer/src/main/scala/ix/memory/map/MapService.scala
@@ -4,8 +4,10 @@ import java.time.Instant
import java.util.UUID
import cats.effect.IO
+import com.fasterxml.jackson.databind.ObjectMapper
import io.circe.Json
import io.circe.syntax._
+import org.typelevel.log4cats.slf4j.Slf4jLogger
import ix.memory.db.{ArangoClient, GraphQueryApi, GraphWriteApi}
import ix.memory.model._
@@ -26,6 +28,8 @@ class MapService(
queryApi: GraphQueryApi,
writeApi: GraphWriteApi
) {
+ private val mapper = new ObjectMapper()
+ private val logger = Slf4jLogger.getLoggerFromName[IO]("ix.map")
private val CrosscutThreshold = 0.10
// Cross-cut penalty multiplier (reduces edge weights involving crosscut files)
@@ -46,7 +50,8 @@ class MapService(
"PATH" -> 0.2
)
- private val builder = new MapGraphBuilder(client)
+ private val fullBuilder = new MapGraphBuilder(client)
+ private val preflight = new MapPreflight()
private final class RetainedRegion(val level: Int, val communityIndex: Int, val region: Region)
// Label kinds by level
@@ -56,20 +61,20 @@ class MapService(
else if (level == 1) "module"
else "subsystem"
- def buildMap(forceRecompute: Boolean = false): IO[ArchitectureMap] =
- getOrBuildMap(forceRecompute)
+ def buildMap(forceRecompute: Boolean = false, forceFull: Boolean = false): IO[ArchitectureMap] =
+ getOrBuildMap(forceRecompute, forceFull)
- def getOrBuildMap(forceRecompute: Boolean = false): IO[ArchitectureMap] =
+ def getOrBuildMap(forceRecompute: Boolean = false, forceFull: Boolean = false): IO[ArchitectureMap] =
for {
- inputRev <- builder.currentInputRev()
+ inputRev <- fullBuilder.currentInputRev()
cached <- loadCachedMap(inputRev)
archMap <- if (cached.nonEmpty && !forceRecompute) IO.pure(cached.get)
- else computeAndPersistMap(inputRev, cached)
+ else computeAndPersistMap(inputRev, cached, forceFull)
} yield archMap
def loadPersistedMapForCurrentGraph(): IO[Option[ArchitectureMap]] =
for {
- inputRev <- builder.currentInputRev()
+ inputRev <- fullBuilder.currentInputRev()
cached <- loadCachedMap(inputRev)
} yield cached
@@ -263,14 +268,15 @@ class MapService(
val withChildren = withParents.map { r =>
r.copy(childRegionIds = childrenByParent.getOrElse(r.id, Set.empty))
}
+ val dedupedHierarchy = collapseDuplicateHierarchyLevels(withChildren)
// Disambiguate duplicate labels within each level.
// For each group of same-label regions, find a path segment unique to each.
- val labelGroups = withChildren.groupBy(r => (r.level, r.label))
+ val labelGroups = dedupedHierarchy.groupBy(r => (r.level, r.label))
val needsDisambig = labelGroups.filter(_._2.size > 1)
- if (needsDisambig.isEmpty) withChildren
- else withChildren.map { r =>
+ if (needsDisambig.isEmpty) dedupedHierarchy
+ else dedupedHierarchy.map { r =>
needsDisambig.get((r.level, r.label)) match {
case Some(group) =>
val myPaths = r.memberFiles.flatMap(id => fileVertexById.get(id).map(_.path)).toSet
@@ -299,6 +305,14 @@ class MapService(
}
}
+ private[map] def collapseDuplicateHierarchyLevels(regions: Vector[Region]): Vector[Region] = {
+ val keepById = regions.groupBy(_.id).view.mapValues { duplicates =>
+ duplicates.maxBy(region => (region.level, region.confidence, region.label.length))
+ }.toMap
+
+ regions.filter(region => keepById.get(region.id).contains(region))
+ }
+
private val LabelNoiseTokens = Set(
"main", "scala", "java", "src", "test", "tests", "__tests__", "kotlin",
"python", "js", "ts", "tsx", "jsx", "com", "org", "io", "net",
@@ -594,98 +608,6 @@ class MapService(
case other => other.toLowerCase
}
- private def computeArchitectureEdges(
- regions: Vector[Region],
- graph: WeightedFileGraph
- ): Vector[ArchitectureEdge] = {
- if (regions.isEmpty || graph.predicatePairs.isEmpty) return Vector.empty
-
- def buildEdge(
- idSeed: String,
- src: NodeId,
- dst: NodeId,
- level: Int,
- predicateCounts: Map[String, Int]
- ): Option[ArchitectureEdge] = {
- val signalScores = predicateCounts.toVector
- .map { case (predicate, count) =>
- signalCategory(predicate) -> (SignalWeights.getOrElse(predicate, 0.0) * math.log1p(count.toDouble))
- }
- .groupBy(_._1)
- .view
- .mapValues(_.map(_._2).sum)
- .toMap
-
- val weight = signalScores.values.sum
- if (weight <= 0.0) None
- else {
- val dominantSignal = signalScores.toVector
- .sortBy { case (signal, score) => (-score, signal) }
- .headOption
- .map(_._1)
- .getOrElse("path")
-
- Some(ArchitectureEdge(
- id = UUID.nameUUIDFromBytes(idSeed.getBytes("UTF-8")).toString,
- src = src,
- dst = dst,
- level = level,
- weight = weight,
- dominantSignal = dominantSignal,
- predicateCounts = predicateCounts,
- ))
- }
- }
-
- val fileEdges = graph.predicatePairs.toVector.flatMap { case ((src, dst), predicateCounts) =>
- buildEdge(
- idSeed = s"map:file-edge:${src.value}:${dst.value}",
- src = src,
- dst = dst,
- level = 0,
- predicateCounts = predicateCounts.filter(_._2 > 0),
- )
- }
-
- val fileToRegionByLevel: Map[Int, Map[NodeId, Region]] = regions.groupBy(_.level).view.mapValues { levelRegions =>
- levelRegions.toVector.flatMap(region =>
- region.memberFiles.toVector.map(fileId => fileId -> region)
- ).toMap
- }.toMap
-
- val aggregated = scala.collection.mutable.Map.empty[(Int, NodeId, NodeId), scala.collection.mutable.Map[String, Int]]
-
- for {
- ((srcFile, dstFile), predicateCounts) <- graph.predicatePairs
- (level, byFile) <- fileToRegionByLevel
- srcRegion <- byFile.get(srcFile)
- dstRegion <- byFile.get(dstFile)
- if srcRegion.id != dstRegion.id
- } {
- val (srcRegionId, dstRegionId) = canonicalRegionPair(srcRegion.id, dstRegion.id)
- val bucket = aggregated.getOrElseUpdate(
- (level, srcRegionId, dstRegionId),
- scala.collection.mutable.Map.empty[String, Int],
- )
- predicateCounts.foreach { case (predicate, count) =>
- bucket.update(predicate, bucket.getOrElse(predicate, 0) + count)
- }
- }
-
- val regionEdges = aggregated.toVector.flatMap { case ((level, src, dst), predicateCountsMut) =>
- val predicateCounts = predicateCountsMut.toMap.filter(_._2 > 0)
- buildEdge(
- idSeed = s"map:region-edge:$level:${src.value}:${dst.value}",
- src = src,
- dst = dst,
- level = level,
- predicateCounts = predicateCounts,
- )
- }
-
- (regionEdges ++ fileEdges).sortBy(edge => (edge.level, edge.src.value.toString, edge.dst.value.toString))
- }
-
// ── Confidence scoring ──────────────────────────────────────────────
private def computeConfidence(
@@ -704,16 +626,36 @@ class MapService(
// ── Persistence ─────────────────────────────────────────────────────
- private def computeAndPersistMap(inputRev: Rev, cached: Option[ArchitectureMap]): IO[ArchitectureMap] =
+ private def computeAndPersistMap(
+ inputRev: Rev,
+ cached: Option[ArchitectureMap],
+ forceFull: Boolean
+ ): IO[ArchitectureMap] =
for {
- rawGraph <- builder.build()
- crosscutScores = detectCrosscut(rawGraph)
- graph = applyPenalty(rawGraph, crosscutScores)
- levels = LouvainClustering.cluster(graph, maxLevels = 3, minCommunitySize = 3)
- regions = buildRegions(graph, levels, crosscutScores, inputRev)
- edges = computeArchitectureEdges(regions, rawGraph)
- fresh = ArchitectureMap(regions, edges, rawGraph.vertices.size, inputRev.value)
- _ <- persistMap(fresh, cached)
+ files <- fullBuilder.discoverFiles()
+ preflightEval <- preflight.evaluate(files)
+ t0 <- IO(System.currentTimeMillis())
+ rawGraph <- fullBuilder.buildGraph(files)
+ t1 <- IO(System.currentTimeMillis())
+ crosscutScores <- IO.blocking(detectCrosscut(rawGraph))
+ graph <- IO.blocking(applyPenalty(rawGraph, crosscutScores))
+ t2 <- IO(System.currentTimeMillis())
+ levels <- IO.blocking(LouvainClustering.cluster(graph, maxLevels = 3, minCommunitySize = 3))
+ t3 <- IO(System.currentTimeMillis())
+ regions <- IO.blocking(buildRegions(graph, levels, crosscutScores, inputRev))
+ fresh = ArchitectureMap(
+ regions = regions,
+ fileCount = rawGraph.vertices.size,
+ mapRev = inputRev.value,
+ preflight = Some(preflightEval),
+ outcome = MapOutcome.FullLocalCompleted
+ )
+ t4 <- IO(System.currentTimeMillis())
+ _ <- persistMap(fresh, cached)
+ t5 <- IO(System.currentTimeMillis())
+ _ <- logger.info(
+ s"map mode=${fresh.outcome.label} files=${files.size} build=${t1-t0}ms crosscut=${t2-t1}ms cluster=${t3-t2}ms regions=${t4-t3}ms persist=${t5-t4}ms total=${t5-t0}ms"
+ )
} yield fresh
private def loadCachedMap(inputRev: Rev): IO[Option[ArchitectureMap]] =
@@ -734,18 +676,10 @@ class MapService(
)
decoded = rows.flatMap(decodePersistedRegion(_, inputRev)).toVector
regions = rehydrateRelationships(decoded)
- rawGraphOpt <- if (regions.isEmpty) IO.pure(None) else builder.build().map(Some(_))
+ fileCount <- if (regions.isEmpty) IO.pure(0) else fullBuilder.liveFileCount()
} yield {
if (regions.isEmpty) None
- else {
- val rawGraph = rawGraphOpt.get
- Some(ArchitectureMap(
- regions,
- computeArchitectureEdges(regions, rawGraph),
- rawGraph.vertices.size,
- inputRev.value,
- ))
- }
+ else Some(ArchitectureMap(regions, fileCount, inputRev.value))
}
private def decodePersistedRegion(json: Json, inputRev: Rev): Option[Region] = {
@@ -813,9 +747,8 @@ class MapService(
if (cached.exists(existing => sameRegions(existing.regions, fresh.regions))) IO.unit
else {
for {
- oldRegions <- queryApi.findNodesByKind(NodeKind.Region, limit = 5000)
- oldEdgeIds <- loadActiveRegionEdgeIds(oldRegions.map(_.id))
- _ <- submitMapPatch(oldRegions, oldEdgeIds, fresh.regions, Rev(fresh.mapRev))
+ oldLogicalIds <- loadOldRegionIds()
+ _ <- submitMapPatch(oldLogicalIds, fresh.regions, Rev(fresh.mapRev))
} yield ()
}
}
@@ -823,103 +756,189 @@ class MapService(
private def sameRegions(left: Vector[Region], right: Vector[Region]): Boolean =
left.sortBy(_.id.value.toString) == right.sortBy(_.id.value.toString)
- private def loadActiveRegionEdgeIds(regionIds: Vector[NodeId]): IO[Vector[EdgeId]] =
- if (regionIds.isEmpty) IO.pure(Vector.empty)
- else {
- val ids = regionIds.map(_.value.toString).toArray
- client.query(
- """FOR e IN edges
- | FILTER e.deleted_rev == null
- | AND e.predicate == "IN_REGION"
- | AND (e.src IN @regionIds OR e.dst IN @regionIds)
- | RETURN e._key""".stripMargin,
- Map("regionIds" -> ids.asInstanceOf[AnyRef])
- ).map(_.flatMap(_.as[String].toOption).flatMap(id => scala.util.Try(UUID.fromString(id)).toOption.map(EdgeId(_))).toVector)
- }
+ /** Fetch only the logical_ids of current live region nodes — cheaper than full GraphNode. */
+ private def loadOldRegionIds(): IO[Vector[String]] =
+ client.query(
+ """FOR n IN nodes
+ | FILTER n.kind == "region"
+ | AND n.deleted_rev == null
+ | LIMIT 5000
+ | RETURN n.logical_id""".stripMargin,
+ Map.empty
+ ).map(_.flatMap(_.as[String].toOption).toVector)
+ /**
+ * Persist map regions using direct bulk operations instead of the per-op transaction path.
+ *
+ * This replaces the old commitPatch approach (which made N+1 DB round-trips inside
+ * a transaction, causing timeouts on large repos) with five bulk operations:
+ * 1. Batch AQL tombstone for old region nodes
+ * 2. Batch AQL tombstone for old region edges
+ * 3. bulkInsert for new region nodes
+ * 4. bulkInsertEdges for IN_REGION edges
+ * 5. Revision update
+ */
private def submitMapPatch(
- oldRegions: Vector[GraphNode],
- oldEdgeIds: Vector[EdgeId],
- regions: Vector[Region],
- rev: Rev
+ oldRegionIds: Vector[String],
+ regions: Vector[Region],
+ rev: Rev
): IO[Unit] = {
- val deleteOps: Vector[PatchOp] = oldRegions.map(n => PatchOp.DeleteNode(n.id))
- val deleteEdgeOps: Vector[PatchOp] = oldEdgeIds.map(PatchOp.DeleteEdge)
-
- val nodeOps: Vector[PatchOp] = regions.map { r =>
- PatchOp.UpsertNode(
- id = r.id,
- kind = NodeKind.Region,
- name = r.label,
- attrs = Map(
- "label" -> r.label.asJson,
- "map_generated" -> Json.True,
- "level" -> r.level.asJson,
- "label_kind" -> r.labelKind.asJson,
- "file_count" -> r.memberFiles.size.asJson,
- "member_files" -> r.memberFiles.map(_.value.toString).asJson,
- "parent_id" -> r.parentId.map(_.value.toString).asJson,
- "child_region_ids" -> r.childRegionIds.toVector.map(_.value.toString).sorted.asJson,
- "cohesion" -> r.cohesion.asJson,
- "external_coupling" -> r.externalCoupling.asJson,
- "boundary_ratio" -> r.boundaryRatio.asJson,
- "confidence" -> r.confidence.asJson,
- "crosscut_score" -> r.crosscutScore.asJson,
- "dominant_signals" -> r.dominantSignals.asJson,
- "interface_node_count" -> r.interfaceNodeCount.asJson,
- "map_rev" -> r.mapRev.asJson
- )
- )
+ if (oldRegionIds.isEmpty && regions.isEmpty) return IO.unit
+
+ val now = Instant.now().toString
+ val provMap = {
+ val m = new java.util.HashMap[String, AnyRef]()
+ m.put("source_uri", "ix:map")
+ m.put("source_hash", null)
+ m.put("extractor", "map-service")
+ m.put("source_type", "inferred")
+ m.put("observed_at", now)
+ m
}
- // IN_REGION edges: region → file (level-1 regions only, top-down)
- val fileEdgeOps: Vector[PatchOp] = regions
- .filter(_.level == 1)
- .flatMap { r =>
- r.memberFiles.toVector.map { fileId =>
- val edgeId = EdgeId(
- UUID.nameUUIDFromBytes(s"in_region:${r.id.value}:${fileId.value}".getBytes("UTF-8"))
- )
- PatchOp.UpsertEdge(
- id = edgeId,
- src = r.id,
- dst = fileId,
- predicate = EdgePredicate("IN_REGION"),
- attrs = Map.empty
- )
- }
- }
+ for {
+ latestRev <- queryApi.getLatestRev
+ newRev = latestRev.value + 1L
+
+ // 1. Tombstone old region nodes
+ _ <- if (oldRegionIds.isEmpty) IO.unit
+ else {
+ val ids = new java.util.ArrayList[String]()
+ oldRegionIds.foreach(ids.add)
+ client.execute(
+ """FOR n IN nodes
+ | FILTER n.logical_id IN @ids AND n.deleted_rev == null
+ | UPDATE n WITH { deleted_rev: @rev, updated_at: @now } IN nodes""".stripMargin,
+ Map(
+ "ids" -> ids.asInstanceOf[AnyRef],
+ "rev" -> Long.box(newRev).asInstanceOf[AnyRef],
+ "now" -> now.asInstanceOf[AnyRef]
+ )
+ )
+ }
+
+ // 2. Tombstone all active IN_REGION edges in one indexed query.
+ // Replaces the old loadActiveRegionEdgeIds approach (which did a full
+ // collection scan with a large src/dst IN clause — the main freeze point).
+ _ <- client.execute(
+ """FOR e IN edges
+ | FILTER e.predicate == "IN_REGION"
+ | AND e.deleted_rev == null
+ | UPDATE e WITH { deleted_rev: @rev, updated_at: @now } IN edges""".stripMargin,
+ Map(
+ "rev" -> Long.box(newRev).asInstanceOf[AnyRef],
+ "now" -> now.asInstanceOf[AnyRef]
+ )
+ )
+
+ // 3. Bulk-insert new region nodes
+ nodeDocs = regions.map(r => buildRegionNodeDoc(r, newRev, now, provMap))
+ _ <- if (nodeDocs.isEmpty) IO.unit
+ else client.bulkInsert("nodes", nodeDocs).void
+
+ // 4. Bulk-insert IN_REGION edges
+ edgeDocs = buildRegionEdgeDocs(regions, newRev, now, provMap)
+ _ <- if (edgeDocs.isEmpty) IO.unit
+ else client.bulkInsertEdges("edges", edgeDocs).void
+
+ // 5. Update current revision
+ _ <- client.execute(
+ """UPSERT { _key: "current" }
+ | INSERT { _key: "current", rev: @rev }
+ | UPDATE { rev: @rev }
+ | IN revisions""".stripMargin,
+ Map("rev" -> Long.box(newRev).asInstanceOf[AnyRef])
+ )
+ } yield ()
+ }
- // IN_REGION edges: parent region → child region (top-down)
- val regionEdgeOps: Vector[PatchOp] = regions.flatMap { r =>
+ private def buildRegionNodeDoc(
+ r: Region,
+ newRev: Long,
+ now: String,
+ provMap: java.util.Map[String, AnyRef]
+ ): java.util.Map[String, AnyRef] = {
+ val logicalId = r.id.value.toString
+ val attrsJson = Json.obj(
+ "label" -> r.label.asJson,
+ "map_generated" -> Json.True,
+ "level" -> r.level.asJson,
+ "label_kind" -> r.labelKind.asJson,
+ "file_count" -> r.memberFiles.size.asJson,
+ "member_files" -> r.memberFiles.map(_.value.toString).asJson,
+ "parent_id" -> r.parentId.map(_.value.toString).asJson,
+ "child_region_ids" -> r.childRegionIds.toVector.map(_.value.toString).sorted.asJson,
+ "cohesion" -> r.cohesion.asJson,
+ "external_coupling" -> r.externalCoupling.asJson,
+ "boundary_ratio" -> r.boundaryRatio.asJson,
+ "confidence" -> r.confidence.asJson,
+ "crosscut_score" -> r.crosscutScore.asJson,
+ "dominant_signals" -> r.dominantSignals.asJson,
+ "interface_node_count" -> r.interfaceNodeCount.asJson,
+ "map_rev" -> r.mapRev.asJson
+ ).noSpaces
+ val doc = new java.util.HashMap[String, AnyRef]()
+ doc.put("_key", s"${logicalId}_${newRev}")
+ doc.put("logical_id", logicalId)
+ doc.put("id", logicalId)
+ doc.put("kind", "region")
+ doc.put("name", r.label)
+ doc.put("attrs", mapper.readValue(attrsJson, classOf[java.util.Map[String, AnyRef]]))
+ doc.put("provenance", provMap)
+ doc.put("created_rev", Long.box(newRev))
+ doc.put("deleted_rev", null)
+ doc.put("created_at", now)
+ doc.put("updated_at", now)
+ doc
+ }
+
+ private def buildRegionEdgeDocs(
+ regions: Vector[Region],
+ newRev: Long,
+ now: String,
+ provMap: java.util.Map[String, AnyRef]
+ ): Vector[java.util.Map[String, AnyRef]] = {
+ val fileEdges = regions.filter(_.level == 1).flatMap { r =>
+ r.memberFiles.toVector.map { fileId =>
+ val edgeId = UUID.nameUUIDFromBytes(s"in_region:${r.id.value}:${fileId.value}".getBytes("UTF-8"))
+ buildRegionEdgeDoc(edgeId.toString, s"nodes/${r.id.value}", s"nodes/${fileId.value}",
+ r.id.value.toString, fileId.value.toString, newRev, now, provMap)
+ }
+ }
+ val regionEdges = regions.flatMap { r =>
r.parentId.map { pid =>
- val edgeId = EdgeId(
- UUID.nameUUIDFromBytes(s"in_region:${pid.value}:${r.id.value}".getBytes("UTF-8"))
- )
- PatchOp.UpsertEdge(
- id = edgeId,
- src = pid,
- dst = r.id,
- predicate = EdgePredicate("IN_REGION"),
- attrs = Map.empty
- )
+ val edgeId = UUID.nameUUIDFromBytes(s"in_region:${pid.value}:${r.id.value}".getBytes("UTF-8"))
+ buildRegionEdgeDoc(edgeId.toString, s"nodes/${pid.value}", s"nodes/${r.id.value}",
+ pid.value.toString, r.id.value.toString, newRev, now, provMap)
}
}
+ fileEdges ++ regionEdges
+ }
- val allOps = deleteEdgeOps ++ deleteOps ++ nodeOps ++ fileEdgeOps ++ regionEdgeOps
- if (allOps.isEmpty) return IO.unit
-
- val patch = GraphPatch(
- patchId = PatchId(UUID.nameUUIDFromBytes(s"map:${rev.value}".getBytes("UTF-8"))),
- actor = "map-service",
- timestamp = Instant.now(),
- source = PatchSource("ix:map", None, "map-service", SourceType.Inferred),
- baseRev = rev,
- ops = allOps,
- replaces = Vector.empty,
- intent = Some(s"Architectural map computed at rev ${rev.value}")
- )
-
- writeApi.commitPatch(patch).void
+ private def buildRegionEdgeDoc(
+ key: String,
+ from: String,
+ to: String,
+ src: String,
+ dst: String,
+ newRev: Long,
+ now: String,
+ provMap: java.util.Map[String, AnyRef]
+ ): java.util.Map[String, AnyRef] = {
+ val doc = new java.util.HashMap[String, AnyRef]()
+ doc.put("_key", key)
+ doc.put("_from", from)
+ doc.put("_to", to)
+ doc.put("id", key)
+ doc.put("src", src)
+ doc.put("dst", dst)
+ doc.put("predicate", "IN_REGION")
+ doc.put("attrs", new java.util.HashMap[String, AnyRef]())
+ doc.put("created_rev", Long.box(newRev))
+ doc.put("deleted_rev", null)
+ doc.put("provenance", provMap)
+ doc.put("created_at", now)
+ doc.put("updated_at", now)
+ doc
}
}
diff --git a/memory-layer/src/main/scala/ix/memory/map/MapTypes.scala b/memory-layer/src/main/scala/ix/memory/map/MapTypes.scala
index b416fc2f..ac56de8b 100644
--- a/memory-layer/src/main/scala/ix/memory/map/MapTypes.scala
+++ b/memory-layer/src/main/scala/ix/memory/map/MapTypes.scala
@@ -139,7 +139,6 @@ class MapCapacityException(
/** The full multi-level architecture map returned by MapService. */
final case class ArchitectureMap(
regions: Vector[Region],
- edges: Vector[ArchitectureEdge],
fileCount: Int,
mapRev: Long,
preflight: Option[MapPreflightResult] = None,
diff --git a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
index 35ec4613..b6f67e6a 100644
--- a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
@@ -17,7 +17,6 @@ import ix.memory.TestDbHelper
import ix.memory.conflict.ConflictService
import ix.memory.context._
import ix.memory.db._
-import ix.memory.ingestion._
import ix.memory.map.MapService
import ix.memory.model._
import ix.memory.smell.SmellService
@@ -65,18 +64,13 @@ class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestD
new ConfidenceScorerImpl(),
new ConflictDetectorImpl()
)
- val parserRouter = new ParserRouter()
- val ingestionService = new IngestionService(parserRouter, writeApi, queryApi)
val bulkWriteApi = new BulkWriteApi(client)
- val bulkIngestionService = new BulkIngestionService(parserRouter, bulkWriteApi, queryApi)
val mapService = new MapService(client, queryApi, writeApi)
val smellService = new SmellService(client, writeApi)
val subsystemService = new SubsystemScoringService(client, writeApi, mapService)
Routes.all(
contextService,
- ingestionService,
- bulkIngestionService,
queryApi,
writeApi,
conflictService,
@@ -221,6 +215,38 @@ class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestD
}
}
+ it should "return provenance chain for entity after bulk commit" in {
+ clientResource.use { client =>
+ val writeApi = new ArangoGraphWriteApi(client)
+ val queryApi = new ArangoGraphQueryApi(client)
+ val bulkWriteApi = new BulkWriteApi(client)
+ val app = buildRoutes(client, writeApi, queryApi)
+ val nodeId = NodeId(UUID.randomUUID())
+ val patch = makePatch(ops = Vector(PatchOp.UpsertNode(nodeId, NodeKind.Function, "bulk_prov_func", Map.empty[String, Json])))
+ val provenance = new java.util.HashMap[String, AnyRef]()
+ provenance.put("source_uri", patch.source.uri)
+ provenance.put("source_hash", patch.source.sourceHash.orNull)
+ provenance.put("extractor", patch.source.extractor)
+ provenance.put("source_type", patch.source.sourceType.asJson.asString.getOrElse("code"))
+ provenance.put("observed_at", patch.timestamp.toString)
+
+ for {
+ _ <- client.ensureSchema()
+ _ <- cleanDatabase(client)
+ _ <- bulkWriteApi.commitBatchChunked(
+ Vector(FileBatch(patch.source.uri, patch.source.sourceHash, patch, provenance)),
+ baseRev = 0L
+ )
+ resp <- app.run(Request[IO](Method.POST, Uri.unsafeFromString(s"/v1/provenance/${nodeId.value}")))
+ body <- resp.as[Json]
+ } yield {
+ resp.status shouldBe Status.Ok
+ body.hcursor.downField("entityId").as[String] shouldBe Right(nodeId.value.toString)
+ body.hcursor.downField("chain").focus.flatMap(_.asArray).exists(_.nonEmpty) shouldBe true
+ }
+ }
+ }
+
it should "create a decision node via POST /v1/decide" in {
clientResource.use { client =>
val app = buildRoutes(client, new ArangoGraphWriteApi(client), new ArangoGraphQueryApi(client))
diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/FileDiscoverySpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/FileDiscoverySpec.scala
deleted file mode 100644
index 806e275c..00000000
--- a/memory-layer/src/test/scala/ix/memory/ingestion/FileDiscoverySpec.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-package ix.memory.ingestion
-
-import java.nio.file.Paths
-
-import org.scalatest.flatspec.AnyFlatSpec
-import org.scalatest.matchers.should.Matchers
-
-class FileDiscoverySpec extends AnyFlatSpec with Matchers {
-
- "extensionsFor(None)" should "default to parser-supported file types" in {
- val extensions = FileDiscovery.extensionsFor(None)
-
- extensions should contain allOf (".py", ".ts", ".tsx", ".scala", ".sc", ".json", ".yaml", ".yml", ".toml", ".md")
- extensions should not contain ".txt"
- extensions should not contain ".ini"
- extensions should not contain ".sbt"
- }
-
- "matchesFilter" should "exclude unsupported generic-text files by default" in {
- val extensions = FileDiscovery.extensionsFor(None)
-
- FileDiscovery.matchesFilter(Paths.get("README.md"), extensions) shouldBe true
- FileDiscovery.matchesFilter(Paths.get("Dockerfile"), extensions) shouldBe false
- FileDiscovery.matchesFilter(Paths.get("notes.txt"), extensions) shouldBe false
- }
-}
diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/GenericTextFallbackSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/GenericTextFallbackSpec.scala
deleted file mode 100644
index 14fca5c8..00000000
--- a/memory-layer/src/test/scala/ix/memory/ingestion/GenericTextFallbackSpec.scala
+++ /dev/null
@@ -1,17 +0,0 @@
-package ix.memory.ingestion
-
-import org.scalatest.flatspec.AnyFlatSpec
-import org.scalatest.matchers.should.Matchers
-
-class GenericTextFallbackSpec extends AnyFlatSpec with Matchers {
-
- "GenericTextFallback" should "store compact metadata instead of raw content" in {
- val parsed = GenericTextFallback.parse("notes.txt", "first line\nsecond line\n")
- val entity = parsed.entities.head
-
- entity.attrs.keySet should contain allOf ("file_name", "line_count", "preview")
- entity.attrs.keySet should not contain "content"
- entity.attrs("preview").asString shouldBe Some("first line")
- entity.contentFingerprint should not be empty
- }
-}
diff --git a/memory-layer/src/test/scala/ix/memory/map/MapServiceScopeSpec.scala b/memory-layer/src/test/scala/ix/memory/map/MapServiceScopeSpec.scala
index 4d4f9176..6b3848e4 100644
--- a/memory-layer/src/test/scala/ix/memory/map/MapServiceScopeSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/map/MapServiceScopeSpec.scala
@@ -19,7 +19,7 @@ class MapServiceScopeSpec extends AnyFlatSpec with Matchers {
val map = ArchitectureMap(
regions = Vector(context, apiSubsystem, apiModule, apiClient),
- edges = Vector.empty,
+
fileCount = 257,
mapRev = 42L
)
@@ -39,7 +39,7 @@ class MapServiceScopeSpec extends AnyFlatSpec with Matchers {
val map = ArchitectureMap(
regions = Vector(system, subsystem, moduleA, moduleB, moduleC),
- edges = Vector.empty,
+
fileCount = 140,
mapRev = 42L
)
@@ -54,6 +54,24 @@ class MapServiceScopeSpec extends AnyFlatSpec with Matchers {
view.summary.crossCutting shouldBe 1
}
+ "collapseDuplicateHierarchyLevels" should "keep only the coarsest region for repeated member sets" in {
+ val sharedId = NodeId(UUID.randomUUID())
+ val sharedFiles = (1 to 10).map(_ => NodeId(UUID.randomUUID())).toSet
+ val child = region("Edges / Resolver", "module", 1, files = 4, parentId = Some(sharedId))
+ val module = regionWithMembers("Edges", "module", 1, sharedId, sharedFiles)
+ val subsystem = regionWithMembers("Edges", "subsystem", 2, sharedId, sharedFiles)
+ val system = regionWithMembers("Edges", "system", 3, sharedId, sharedFiles)
+
+ val collapsed = service.collapseDuplicateHierarchyLevels(Vector(child, module, subsystem, system))
+
+ collapsed.map(region => region.label -> region.labelKind) shouldBe Vector(
+ "Edges / Resolver" -> "module",
+ "Edges" -> "system"
+ )
+ collapsed.map(_.id) should contain (sharedId)
+ collapsed.find(_.label == "Edges / Resolver").flatMap(_.parentId) shouldBe Some(sharedId)
+ }
+
private def region(
label: String,
labelKind: String,
@@ -62,9 +80,21 @@ class MapServiceScopeSpec extends AnyFlatSpec with Matchers {
confidence: Double = 0.80,
crosscut: Double = 0.0,
parent: Option[Region] = None,
+ parentId: Option[NodeId] = None,
+ ): Region = {
+ regionWithMembers(label, labelKind, level, NodeId(UUID.randomUUID()), (1 to files).map(_ => NodeId(UUID.randomUUID())).toSet, confidence, crosscut, parent.map(_.id).orElse(parentId))
+ }
+
+ private def regionWithMembers(
+ label: String,
+ labelKind: String,
+ level: Int,
+ id: NodeId,
+ memberFiles: Set[NodeId],
+ confidence: Double = 0.80,
+ crosscut: Double = 0.0,
+ parentId: Option[NodeId] = None,
): Region = {
- val id = NodeId(UUID.randomUUID())
- val memberFiles = (1 to files).map(_ => NodeId(UUID.randomUUID())).toSet
Region(
id = id,
label = label,
@@ -72,7 +102,7 @@ class MapServiceScopeSpec extends AnyFlatSpec with Matchers {
level = level,
memberFiles = memberFiles,
childRegionIds = Set.empty,
- parentId = parent.map(_.id),
+ parentId = parentId,
cohesion = 0.5,
externalCoupling = 0.2,
boundaryRatio = 1.5,