Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

Patch/parse refinement (#96)

josephismikhaild3c62090c1bc
diff --git a/core-ingestion/src/__tests__/queries.go.test.ts b/core-ingestion/src/__tests__/queries.go.test.ts
index 273cdcb3..d9253afa 100644
--- a/core-ingestion/src/__tests__/queries.go.test.ts
+++ b/core-ingestion/src/__tests__/queries.go.test.ts
@@ -146,6 +146,60 @@ func run(s *Service) {
});
});
+ it('preserves full Go import paths and captures qualified type references in function signatures', () => {
+ const result = parseFile(
+ '/repo/server.go',
+ `
+package app
+
+import scheduler "k8s.io/kubernetes/pkg/scheduler"
+
+func Run(sched *scheduler.Scheduler) error {
+ return nil
+}
+ `,
+ );
+
+ expect(result).not.toBeNull();
+ expect(result!.relationships).toContainEqual({
+ srcName: 'server.go',
+ dstName: 'k8s.io/kubernetes/pkg/scheduler',
+ predicate: 'IMPORTS',
+ });
+ expect(result!.relationships).toContainEqual({
+ srcName: 'Run',
+ dstName: 'Scheduler',
+ predicate: 'REFERENCES',
+ });
+ });
+
+ it('records explicit Go import aliases so alias-qualified package calls can be resolved later', () => {
+ const result = parseFile(
+ '/repo/server.go',
+ `
+package app
+
+import (
+ controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
+)
+
+func CreateServerChain() {
+ controlplaneapiserver.CreateAggregatorServer()
+}
+ `,
+ );
+
+ expect(result).not.toBeNull();
+ expect(result!.importAliases).toEqual({
+ controlplaneapiserver: 'k8s.io/kubernetes/pkg/controlplane/apiserver',
+ });
+ expect(result!.relationships).toContainEqual({
+ srcName: 'CreateServerChain',
+ dstName: 'controlplaneapiserver.CreateAggregatorServer',
+ predicate: 'CALLS',
+ });
+ });
+
it('emits all qualified CALLS when multiple packages share a function name', () => {
// Regression: before Bug 3 fix, only the first NewManager call was emitted
// because seenCalls deduped bare "NewManager" across all packages.
diff --git a/core-ingestion/src/__tests__/queries.yaml.test.ts b/core-ingestion/src/__tests__/queries.yaml.test.ts
new file mode 100644
index 00000000..15871da2
--- /dev/null
+++ b/core-ingestion/src/__tests__/queries.yaml.test.ts
@@ -0,0 +1,132 @@
+import { describe, expect, it } from 'vitest';
+
+import { parseFile } from '../index.js';
+import { languageFromPath, SupportedLanguages } from '../languages.js';
+
+describe('YAML parsing', () => {
+ it('recognizes .yaml and .yml as YAML', () => {
+ expect(languageFromPath('/repo/docker-compose.yaml')).toBe(SupportedLanguages.YAML);
+ expect(languageFromPath('/repo/docker-compose.yml')).toBe(SupportedLanguages.YAML);
+ });
+
+ it('parses nested config keys from .yaml files', () => {
+ const result = parseFile(
+ '/repo/docker-compose.yaml',
+ [
+ 'services:',
+ ' api:',
+ ' image: ix/api:latest',
+ ' environment:',
+ ' PORT: 8090',
+ ].join('\n'),
+ );
+
+ expect(result).not.toBeNull();
+ expect(result!.language).toBe(SupportedLanguages.YAML);
+ expect(result!.entities).toContainEqual(expect.objectContaining({
+ name: 'services',
+ kind: 'config_entry',
+ language: SupportedLanguages.YAML,
+ }));
+ expect(result!.entities).toContainEqual(expect.objectContaining({
+ name: 'api',
+ kind: 'config_entry',
+ container: 'services',
+ }));
+ expect(result!.entities).toContainEqual(expect.objectContaining({
+ name: 'PORT',
+ kind: 'config_entry',
+ container: 'environment',
+ }));
+ expect(result!.relationships).toContainEqual({
+ srcName: 'docker-compose.yaml',
+ dstName: 'services',
+ predicate: 'CONTAINS',
+ });
+ expect(result!.relationships).toContainEqual({
+ srcName: 'services',
+ dstName: 'api',
+ predicate: 'CONTAINS',
+ });
+ expect(result!.relationships).toContainEqual({
+ srcName: 'environment',
+ dstName: 'PORT',
+ predicate: 'CONTAINS',
+ });
+ });
+
+ it('parses list item mappings from .yml files', () => {
+ const result = parseFile(
+ '/repo/pipeline.yml',
+ [
+ 'jobs:',
+ ' - name: build',
+ ' image: node:20',
+ ].join('\n'),
+ );
+
+ expect(result).not.toBeNull();
+ expect(result!.entities).toContainEqual(expect.objectContaining({
+ name: 'name',
+ kind: 'config_entry',
+ container: 'jobs',
+ }));
+ expect(result!.entities).toContainEqual(expect.objectContaining({
+ name: 'image',
+ kind: 'config_entry',
+ container: 'jobs',
+ }));
+ });
+
+ it('recognizes Dockerfile names and parses stages plus stage imports', () => {
+ expect(languageFromPath('/repo/Dockerfile')).toBe(SupportedLanguages.Dockerfile);
+ expect(languageFromPath('/repo/docker/prod.dockerfile')).toBe(SupportedLanguages.Dockerfile);
+
+ const result = parseFile(
+ '/repo/Dockerfile',
+ [
+ 'FROM node:20 AS build',
+ 'WORKDIR /app',
+ 'COPY package.json ./',
+ 'RUN npm ci \\',
+ ' --omit=dev',
+ 'FROM nginx:alpine',
+ 'COPY --from=build /app/dist /usr/share/nginx/html',
+ ].join('\n'),
+ );
+
+ expect(result).not.toBeNull();
+ expect(result!.language).toBe(SupportedLanguages.Dockerfile);
+ expect(result!.entities).toContainEqual(expect.objectContaining({
+ name: 'build',
+ kind: 'config',
+ language: SupportedLanguages.Dockerfile,
+ }));
+ expect(result!.entities).toContainEqual(expect.objectContaining({
+ name: 'node:20',
+ kind: 'module',
+ container: 'build',
+ }));
+ expect(result!.entities).toContainEqual(expect.objectContaining({
+ name: 'copy:--from=build',
+ kind: 'config_entry',
+ container: 'stage-1',
+ }));
+ expect(result!.relationships).toContainEqual({
+ srcName: 'build',
+ dstName: 'node:20',
+ predicate: 'IMPORTS',
+ });
+ expect(result!.relationships).toContainEqual({
+ srcName: 'stage-1',
+ dstName: 'build',
+ predicate: 'IMPORTS',
+ });
+ expect(result!.chunks).toContainEqual(expect.objectContaining({
+ name: 'run',
+ chunkKind: 'docker_instruction',
+ lineStart: 4,
+ lineEnd: 5,
+ }));
+ });
+});
diff --git a/core-ingestion/src/__tests__/resolveEdges.test.ts b/core-ingestion/src/__tests__/resolveEdges.test.ts
index 58f2adac..9693ef94 100644
--- a/core-ingestion/src/__tests__/resolveEdges.test.ts
+++ b/core-ingestion/src/__tests__/resolveEdges.test.ts
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest';
-import { resolveEdges, type FileParseResult, type ParsedEntity, type ParsedRelationship } from '../index.js';
+import { buildGlobalResolutionIndex, resolveEdges, type FileParseResult, type ParsedEntity, type ParsedRelationship } from '../index.js';
import { SupportedLanguages } from '../languages.js';
function entity(
@@ -132,6 +132,93 @@ describe('resolveEdges', () => {
});
});
+ it('resolves Go alias-qualified package calls through the aliased import path', () => {
+ const caller = fileResult(
+ '/repo/cmd/kube-apiserver/app/server.go',
+ SupportedLanguages.Go,
+ [entity('CreateServerChain', SupportedLanguages.Go)],
+ [
+ { srcName: 'server.go', dstName: 'k8s.io/kubernetes/pkg/controlplane/apiserver', predicate: 'IMPORTS' },
+ { srcName: 'CreateServerChain', dstName: 'controlplaneapiserver.CreateAggregatorServer', predicate: 'CALLS' },
+ ],
+ );
+ caller.importAliases = {
+ controlplaneapiserver: 'k8s.io/kubernetes/pkg/controlplane/apiserver',
+ };
+ const callee = fileResult(
+ '/repo/pkg/controlplane/apiserver/server.go',
+ SupportedLanguages.Go,
+ [entity('CreateAggregatorServer', SupportedLanguages.Go)],
+ );
+
+ expect(resolveEdges([caller, callee])).toContainEqual({
+ srcFilePath: '/repo/cmd/kube-apiserver/app/server.go',
+ srcName: 'CreateServerChain',
+ dstFilePath: '/repo/pkg/controlplane/apiserver/server.go',
+ dstName: 'controlplaneapiserver.CreateAggregatorServer',
+ dstQualifiedKey: 'CreateAggregatorServer',
+ predicate: 'CALLS',
+ confidence: 0.9,
+ });
+ });
+
+ it('resolves Go alias-qualified package calls when using the global index anchor path', () => {
+ const caller = fileResult(
+ '/repo/cmd/kube-apiserver/app/server.go',
+ SupportedLanguages.Go,
+ [entity('CreateServerChain', SupportedLanguages.Go)],
+ [
+ { srcName: 'server.go', dstName: 'k8s.io/kubernetes/pkg/controlplane/apiserver', predicate: 'IMPORTS' },
+ { srcName: 'CreateServerChain', dstName: 'controlplaneapiserver.CreateAggregatorServer', predicate: 'CALLS' },
+ ],
+ );
+ caller.importAliases = {
+ controlplaneapiserver: 'k8s.io/kubernetes/pkg/controlplane/apiserver',
+ };
+
+ const apiserverDoc = fileResult(
+ '/repo/pkg/controlplane/apiserver/apiserver.go',
+ SupportedLanguages.Go,
+ [entity('Config', SupportedLanguages.Go, 'class')],
+ );
+ const aggregator = fileResult(
+ '/repo/pkg/controlplane/apiserver/aggregator.go',
+ SupportedLanguages.Go,
+ [entity('CreateAggregatorServer', SupportedLanguages.Go)],
+ );
+
+ const sources = new Map<string, string>([
+ ['/repo/pkg/controlplane/apiserver/apiserver.go', 'package apiserver\ntype Config struct{}\n'],
+ ['/repo/pkg/controlplane/apiserver/aggregator.go', 'package apiserver\nfunc CreateAggregatorServer() {}\n'],
+ ]);
+ const globalIndex = buildGlobalResolutionIndex(
+ ['/repo/pkg/controlplane/apiserver/apiserver.go', '/repo/pkg/controlplane/apiserver/aggregator.go'],
+ sources,
+ );
+
+ expect(resolveEdges([caller], undefined, globalIndex)).toContainEqual({
+ srcFilePath: '/repo/cmd/kube-apiserver/app/server.go',
+ srcName: 'CreateServerChain',
+ dstFilePath: '/repo/pkg/controlplane/apiserver/aggregator.go',
+ dstName: 'controlplaneapiserver.CreateAggregatorServer',
+ dstQualifiedKey: 'CreateAggregatorServer',
+ predicate: 'CALLS',
+ confidence: 0.9,
+ });
+
+ // Keep the extra parsed files here to mirror production more closely and ensure
+ // the same answer still wins when batch data and global index are both present.
+ expect(resolveEdges([caller, apiserverDoc, aggregator], undefined, globalIndex)).toContainEqual({
+ srcFilePath: '/repo/cmd/kube-apiserver/app/server.go',
+ srcName: 'CreateServerChain',
+ dstFilePath: '/repo/pkg/controlplane/apiserver/aggregator.go',
+ dstName: 'controlplaneapiserver.CreateAggregatorServer',
+ dstQualifiedKey: 'CreateAggregatorServer',
+ predicate: 'CALLS',
+ confidence: 0.9,
+ });
+ });
+
it('does not resolve qualifier-assisted edges when the member is missing or the qualifier is ambiguous', () => {
const caller = fileResult(
'/repo/consumer.scala',
@@ -209,7 +296,8 @@ describe('resolveEdges', () => {
[entity('helperFn', SupportedLanguages.TypeScript)],
);
- expect(resolveEdges([ambiguousCaller, imported, baz])).toEqual([]);
+ const resolved = resolveEdges([ambiguousCaller, imported, baz]);
+ expect(resolved.filter(edge => edge.predicate === 'CALLS')).toEqual([]);
});
it('resolves tier-2.5 transitive imports', () => {
@@ -275,7 +363,8 @@ describe('resolveEdges', () => {
[entity('helperFn', SupportedLanguages.TypeScript)],
);
- expect(resolveEdges([caller, index, helperA, helperB])).toEqual([]);
+ const resolved = resolveEdges([caller, index, helperA, helperB]);
+ expect(resolved.filter(edge => edge.predicate === 'CALLS')).toEqual([]);
});
it('keeps tier-3 same-language fallback working for TypeScript and Scala and rejects ambiguous globals', () => {
@@ -391,4 +480,149 @@ describe('resolveEdges', () => {
confidence: 0.7,
});
});
+
+ it('resolves Go package imports to the package anchor and uses them for cross-file type references', () => {
+ const caller = fileResult(
+ '/repo/cmd/kube-scheduler/app/server.go',
+ SupportedLanguages.Go,
+ [entity('Run', SupportedLanguages.Go)],
+ [
+ { srcName: 'server.go', dstName: 'k8s.io/kubernetes/pkg/scheduler', predicate: 'IMPORTS' },
+ { srcName: 'Run', dstName: 'Scheduler', predicate: 'REFERENCES' },
+ ],
+ );
+ const nearbyFalseMatch = fileResult(
+ '/repo/cmd/kube-scheduler/app/scheduler.go',
+ SupportedLanguages.Go,
+ [entity('LocalHelper', SupportedLanguages.Go)],
+ );
+ const scheduler = fileResult(
+ '/repo/pkg/scheduler/scheduler.go',
+ SupportedLanguages.Go,
+ [entity('Scheduler', SupportedLanguages.Go, 'class')],
+ Array.from({ length: 10 }, (_, i) => ({
+ srcName: 'scheduler.go',
+ dstName: `dep${i}`,
+ predicate: 'IMPORTS',
+ })),
+ );
+ const eventhandlers = fileResult(
+ '/repo/pkg/scheduler/eventhandlers.go',
+ SupportedLanguages.Go,
+ [entity('registerHandlers', SupportedLanguages.Go)],
+ [{ srcName: 'eventhandlers.go', dstName: 'dep', predicate: 'IMPORTS' }],
+ );
+
+ expect(resolveEdges([caller, nearbyFalseMatch, scheduler, eventhandlers])).toEqual(
+ expect.arrayContaining([
+ {
+ srcFilePath: '/repo/cmd/kube-scheduler/app/server.go',
+ srcName: 'server.go',
+ dstFilePath: '/repo/pkg/scheduler/scheduler.go',
+ dstName: 'k8s.io/kubernetes/pkg/scheduler',
+ dstQualifiedKey: 'scheduler.go',
+ predicate: 'IMPORTS',
+ confidence: 0.9,
+ },
+ {
+ srcFilePath: '/repo/cmd/kube-scheduler/app/server.go',
+ srcName: 'Run',
+ dstFilePath: '/repo/pkg/scheduler/scheduler.go',
+ dstName: 'Scheduler',
+ dstQualifiedKey: 'Scheduler',
+ predicate: 'REFERENCES',
+ confidence: 0.9,
+ },
+ ]),
+ );
+ });
+
+ it('chooses the highest-signal Go package anchor when a package directory has multiple files', () => {
+ const caller = fileResult(
+ '/repo/cmd/kube-apiserver/app/server.go',
+ SupportedLanguages.Go,
+ [entity('Run', SupportedLanguages.Go)],
+ [{ srcName: 'server.go', dstName: 'k8s.io/kubernetes/pkg/controlplane', predicate: 'IMPORTS' }],
+ );
+ const doc = fileResult('/repo/pkg/controlplane/doc.go', SupportedLanguages.Go, []);
+ const versions = fileResult(
+ '/repo/pkg/controlplane/import_known_versions.go',
+ SupportedLanguages.Go,
+ [entity('KnownVersions', SupportedLanguages.Go)],
+ Array.from({ length: 4 }, (_, i) => ({
+ srcName: 'import_known_versions.go',
+ dstName: `dep${i}`,
+ predicate: 'IMPORTS',
+ })),
+ );
+ const instance = fileResult(
+ '/repo/pkg/controlplane/instance.go',
+ SupportedLanguages.Go,
+ [entity('Config', SupportedLanguages.Go, 'class')],
+ Array.from({ length: 8 }, (_, i) => ({
+ srcName: 'instance.go',
+ dstName: `dep${i}`,
+ predicate: 'IMPORTS',
+ })),
+ );
+
+ expect(resolveEdges([caller, doc, versions, instance])).toContainEqual({
+ srcFilePath: '/repo/cmd/kube-apiserver/app/server.go',
+ srcName: 'server.go',
+ dstFilePath: '/repo/pkg/controlplane/instance.go',
+ dstName: 'k8s.io/kubernetes/pkg/controlplane',
+ dstQualifiedKey: 'instance.go',
+ predicate: 'IMPORTS',
+ confidence: 0.9,
+ });
+ });
+
+ it('Python: from X import ClassName resolves IMPORTS edge to the class node via Tier 2', () => {
+ // `from models import Column` produces two IMPORTS edges: one for the module
+ // (resolves to models.py as a file) and one for the symbol (dstName='Column',
+ // no file match). The PascalCase fallthrough should bind Column to the class node.
+ const consumer = fileResult(
+ '/repo/consumer.py',
+ SupportedLanguages.Python,
+ [entity('use_column', SupportedLanguages.Python)],
+ [
+ { srcName: 'consumer.py', dstName: 'models', predicate: 'IMPORTS' },
+ { srcName: 'consumer.py', dstName: 'Column', predicate: 'IMPORTS' },
+ ],
+ );
+ const models = fileResult(
+ '/repo/models.py',
+ SupportedLanguages.Python,
+ [entity('Column', SupportedLanguages.Python, 'class')],
+ );
+
+ expect(resolveEdges([consumer, models])).toContainEqual({
+ srcFilePath: '/repo/consumer.py',
+ srcName: 'consumer.py',
+ dstFilePath: '/repo/models.py',
+ dstName: 'Column',
+ dstQualifiedKey: 'Column',
+ predicate: 'IMPORTS',
+ confidence: 0.9,
+ });
+ });
+
+ it('Go: IMPORTS edge with PascalCase name and zero importMatches does not fall through to symbol resolution', () => {
+ // An unresolvable external package (no matching file) with a PascalCase name
+ // should not leak into Tier 2/3 and bind to an in-repo Go symbol of the same name.
+ const consumer = fileResult(
+ '/repo/main.go',
+ SupportedLanguages.Go,
+ [entity('main', SupportedLanguages.Go, 'function')],
+ [{ srcName: 'main.go', dstName: 'HttpClient', predicate: 'IMPORTS' }],
+ );
+ const lib = fileResult(
+ '/repo/lib/http.go',
+ SupportedLanguages.Go,
+ [entity('HttpClient', SupportedLanguages.Go, 'class')],
+ );
+
+ const resolved = resolveEdges([consumer, lib]);
+ expect(resolved.filter(e => e.predicate === 'IMPORTS' && e.dstName === 'HttpClient')).toEqual([]);
+ });
});
diff --git a/core-ingestion/src/index.ts b/core-ingestion/src/index.ts
index d0a141b0..e8f73cdd 100644
--- a/core-ingestion/src/index.ts
+++ b/core-ingestion/src/index.ts
@@ -82,6 +82,7 @@ export interface FileParseResult {
entities: ParsedEntity[];
chunks: ParsedChunk[];
relationships: ParsedRelationship[];
+ importAliases?: Record<string, string>;
fileRole: RoleClassification;
}
@@ -212,6 +213,7 @@ function builtinsForLanguage(lang: SupportedLanguages): Set<string> {
export function isGrammarSupported(filePath: string): boolean {
const language = languageFromPath(filePath);
if (!language) return false;
+ if (language === SupportedLanguages.YAML || language === SupportedLanguages.Dockerfile) return true;
if (filePath.endsWith('.tsx')) return true; // TSX uses TypeScript.tsx, always available
return GRAMMAR_MAP[language] !== undefined;
}
@@ -251,6 +253,10 @@ function normalizeCapturedImport(rawValue: string, language: SupportedLanguages)
return unwrapped.replace(/^\.\/+/, '');
}
+ if (language === SupportedLanguages.Go) {
+ return unwrapped.replace(/^\.+/, '');
+ }
+
const rawMod = unwrapped.split('/').filter((s: string) => s !== '*').pop() ?? unwrapped;
return rawMod.replace(/^\.+/, '');
}
@@ -277,6 +283,268 @@ function detectLanguageForSource(filePath: string, source: string): SupportedLan
return language;
}
+function countSourceLines(source: string): number {
+ let lineCount = 1;
+ for (let i = 0; i < source.length; i++) {
+ if (source.charCodeAt(i) === 10) lineCount++;
+ }
+ return lineCount;
+}
+
+function computeLineStarts(source: string): number[] {
+ const lineStarts = [0];
+ for (let i = 0; i < source.length; i++) {
+ if (source.charCodeAt(i) === 10) lineStarts.push(i + 1);
+ }
+ return lineStarts;
+}
+
+function parseYamlFile(filePath: string, source: string): FileParseResult {
+ const language = SupportedLanguages.YAML;
+ const fileName = nodePath.basename(filePath);
+ const sourceLineCount = countSourceLines(source);
+ const fileRole = classifyFileRole(filePath);
+ const entities: ParsedEntity[] = [
+ { name: fileName, kind: 'file', lineStart: 1, lineEnd: sourceLineCount, language },
+ ];
+ const chunks: ParsedChunk[] = [];
+ const relationships: ParsedRelationship[] = [];
+ const lineStarts = computeLineStarts(source);
+ const lines = source.split(/\r?\n/);
+ const stack: Array<{ indent: number; key: string }> = [];
+ const keyLinePattern = /^(\s*)(?:-\s+)?([A-Za-z0-9_.-]+)\s*:(?:\s*.*)?$/;
+
+ for (let index = 0; index < lines.length; index++) {
+ const line = lines[index];
+ const match = keyLinePattern.exec(line);
+ if (!match) continue;
+
+ const indent = match[1].replace(/\t/g, ' ').length;
+ const listPrefixWidth = line.trimStart().startsWith('- ') ? 2 : 0;
+ const effectiveIndent = indent + listPrefixWidth;
+ const key = match[2];
+ while (stack.length > 0 && effectiveIndent <= stack[stack.length - 1].indent) {
+ stack.pop();
+ }
+
+ const parent = stack[stack.length - 1];
+ const lineNumber = index + 1;
+ const lineStartOffset = lineStarts[index] ?? 0;
+ const startByte = lineStartOffset + match[1].length + listPrefixWidth;
+ const endByte = Math.max(startByte + key.length, lineStartOffset + line.length);
+
+ entities.push({
+ name: key,
+ kind: 'config_entry',
+ lineStart: lineNumber,
+ lineEnd: lineNumber,
+ language,
+ container: parent?.key,
+ });
+
+ chunks.push({
+ name: key,
+ chunkKind: 'config_key',
+ lineStart: lineNumber,
+ lineEnd: lineNumber,
+ startByte,
+ endByte,
+ contentHash: crypto.createHash('sha256').update(line).digest('hex'),
+ language,
+ container: parent?.key,
+ });
+
+ relationships.push({
+ srcName: parent?.key ?? fileName,
+ dstName: key,
+ predicate: 'CONTAINS',
+ });
+
+ if (listPrefixWidth === 0) {
+ stack.push({ indent: effectiveIndent, key });
+ }
+ }
+
+ if (chunks.length === 0) {
+ chunks.push({
+ name: null,
+ chunkKind: 'file_body',
+ lineStart: 1,
+ lineEnd: Math.max(sourceLineCount, 1),
+ startByte: 0,
+ endByte: source.length,
+ contentHash: crypto.createHash('sha256').update(source).digest('hex'),
+ language,
+ });
+ }
+
+ return {
+ filePath,
+ language,
+ entities,
+ chunks,
+ relationships,
+ fileRole,
+ };
+}
+
+function parseDockerfileFile(filePath: string, source: string): FileParseResult {
+ const language = SupportedLanguages.Dockerfile;
+ const fileName = nodePath.basename(filePath);
+ const sourceLineCount = countSourceLines(source);
+ const fileRole = classifyFileRole(filePath);
+ const entities: ParsedEntity[] = [
+ { name: fileName, kind: 'file', lineStart: 1, lineEnd: sourceLineCount, language },
+ ];
+ const chunks: ParsedChunk[] = [];
+ const relationships: ParsedRelationship[] = [];
+ const lines = source.split(/\r?\n/);
+ const lineStarts = computeLineStarts(source);
+ const stageNames: string[] = [];
+ let pending = '';
+ let logicalStartLine = 1;
+
+ const pushChunk = (
+ name: string | null,
+ chunkKind: string,
+ lineStart: number,
+ lineEnd: number,
+ startByte: number,
+ endByte: number,
+ content: string,
+ container?: string,
+ ) => {
+ chunks.push({
+ name,
+ chunkKind,
+ lineStart,
+ lineEnd,
+ startByte,
+ endByte,
+ contentHash: crypto.createHash('sha256').update(content).digest('hex'),
+ language,
+ container,
+ });
+ };
+
+ const addInstruction = (instructionLine: string, lineStart: number, lineEnd: number) => {
+ const trimmed = instructionLine.trim();
+ if (!trimmed || trimmed.startsWith('#')) return;
+
+ const instructionMatch = /^([A-Z]+)\b(?:\s+([\s\S]*))?$/i.exec(trimmed);
+ if (!instructionMatch) return;
+
+ const keyword = instructionMatch[1].toUpperCase();
+ const argument = (instructionMatch[2] ?? '').trim();
+ const startByte = lineStarts[lineStart - 1] ?? 0;
+ const endByte = lineEnd < lines.length
+ ? (lineStarts[lineEnd] ?? source.length) - 1
+ : source.length;
+
+ if (keyword === 'FROM') {
+ const fromMatch = /^([^\s]+)(?:\s+AS\s+([A-Za-z0-9._-]+))?/i.exec(argument);
+ const baseImage = fromMatch?.[1] ?? argument;
+ const stageName = fromMatch?.[2] ?? `stage-${stageNames.length}`;
+ stageNames.push(stageName);
+
+ entities.push({
+ name: stageName,
+ kind: 'config',
+ lineStart,
+ lineEnd,
+ language,
+ });
+ relationships.push({
+ srcName: fileName,
+ dstName: stageName,
+ predicate: 'CONTAINS',
+ });
+ pushChunk(stageName, 'build_stage', lineStart, lineEnd, startByte, endByte, instructionLine);
+
+ if (baseImage) {
+ entities.push({
+ name: baseImage,
+ kind: 'module',
+ lineStart,
+ lineEnd,
+ language,
+ container: stageName,
+ });
+ relationships.push({
+ srcName: stageName,
+ dstName: baseImage,
+ predicate: 'IMPORTS',
+ });
+ }
+ return;
+ }
+
+ const scope = stageNames[stageNames.length - 1];
+ const entityName =
+ keyword === 'COPY' || keyword === 'ADD'
+ ? `${keyword.toLowerCase()}:${argument.split(/\s+/).find(token => token.length > 0) ?? keyword.toLowerCase()}`
+ : keyword.toLowerCase();
+
+ entities.push({
+ name: entityName,
+ kind: 'config_entry',
+ lineStart,
+ lineEnd,
+ language,
+ container: scope,
+ });
+ relationships.push({
+ srcName: scope ?? fileName,
+ dstName: entityName,
+ predicate: 'CONTAINS',
+ });
+ pushChunk(entityName, 'docker_instruction', lineStart, lineEnd, startByte, endByte, instructionLine, scope);
+
+ if (keyword === 'COPY') {
+ const fromFlag = /--from=([^\s]+)/i.exec(argument)?.[1];
+ if (fromFlag) {
+ relationships.push({
+ srcName: scope ?? fileName,
+ dstName: fromFlag,
+ predicate: 'IMPORTS',
+ });
+ }
+ }
+ };
+
+ for (let index = 0; index < lines.length; index++) {
+ const line = lines[index];
+ const trimmedEnd = line.replace(/\s+$/, '');
+ const lineContent = pending ? `${pending}\n${line}` : line;
+ if (!pending) logicalStartLine = index + 1;
+
+ if (trimmedEnd.endsWith('\\')) {
+ pending = lineContent.slice(0, lineContent.lastIndexOf('\\')).trimEnd();
+ continue;
+ }
+
+ addInstruction(lineContent, logicalStartLine, index + 1);
+ pending = '';
+ }
+
+ if (pending) {
+ addInstruction(pending, logicalStartLine, lines.length);
+ }
+
+ if (chunks.length === 0) {
+ pushChunk(null, 'file_body', 1, Math.max(sourceLineCount, 1), 0, source.length, source);
+ }
+
+ return {
+ filePath,
+ language,
+ entities,
+ chunks,
+ relationships,
+ fileRole,
+ };
+}
+
// ---------------------------------------------------------------------------
// Rust: cfg macro unwrapping
// ---------------------------------------------------------------------------
@@ -336,6 +604,8 @@ function unwrapRustCfgMacros(source: string): string {
export function parseFile(filePath: string, source: string): FileParseResult | null {
const language = detectLanguageForSource(filePath, source);
if (!language) return null;
+ if (language === SupportedLanguages.YAML) return parseYamlFile(filePath, source);
+ if (language === SupportedLanguages.Dockerfile) return parseDockerfileFile(filePath, source);
// TypeScript TSX uses a separate grammar
const isTsx = filePath.endsWith('.tsx');
@@ -394,10 +664,7 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
}
const fileName = nodePath.basename(filePath);
- let sourceLineCount = 1;
- for (let i = 0; i < source.length; i++) {
- if (source.charCodeAt(i) === 10) sourceLineCount++;
- }
+ const sourceLineCount = countSourceLines(source);
const entities: ParsedEntity[] = [
{ name: fileName, kind: 'file', lineStart: 1, lineEnd: sourceLineCount, language },
@@ -427,6 +694,8 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
const assignTypeMap = new Map<string, Map<string, string>>();
// C/C++: map enclosing scope → (variable/member name → declared type)
const declaredTypeMap = new Map<string, Map<string, string>>();
+ // Import aliases keyed by the in-file alias name (Go explicit aliases, etc.).
+ const importAliases: Record<string, string> = {};
// --- First pass: collect definitions ---
for (const match of pass1Matches) {
@@ -615,6 +884,10 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
const importSource = match.captures.find((c: any) => c.name === 'import.source');
if (importSource) {
const modName = normalizeCapturedImport(importSource.node.text, language);
+ const importAlias = match.captures.find((c: any) => c.name === 'import.alias')?.node.text;
+ if (importAlias && modName && importAlias !== '.' && importAlias !== '_') {
+ importAliases[importAlias] = modName;
+ }
if (modName.length > 0 && modName !== '*') {
if (!modName) continue; // skip bare '.' relative imports
entities.push({ name: modName, kind: 'module', lineStart: importSource.node.startPosition.row + 1, lineEnd: importSource.node.startPosition.row + 1, language });
@@ -650,6 +923,22 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
const qualifierCapture = match.captures.find((c: any) => c.name === '_qualifier');
if (builtinsForLanguage(language).has(callee) && !qualifierCapture) continue;
+ // For Python: skip calls that are decorator applications.
+ // When a method is decorated (e.g. @util.deprecated(...)), the call sits at the
+ // class-body line before the def, so findEnclosingFunction misses the method and
+ // the caller falls back to the enclosing class — producing false edges like
+ // Table → deprecated. Decorator application is not an architectural CALLS edge.
+ if (language === SupportedLanguages.Python) {
+ let isDecoratorCall = false;
+ let anc = callName.node.parent;
+ while (anc) {
+ if (anc.type === 'decorator') { isDecoratorCall = true; break; }
+ if (anc.type === 'module' || anc.type === 'function_definition' || anc.type === 'class_definition') break;
+ anc = anc.parent;
+ }
+ if (isDecoratorCall) continue;
+ }
+
// Find enclosing function/method for the call; fall back to enclosing class
// (e.g. calls in val/lazy val body at class level) before falling back to file.
const callLine = callName.node.startPosition.row + 1;
@@ -662,7 +951,18 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
const seen = seenCalls.get(scope)!;
const effectiveCallee = (() => {
- if (!qualifierCapture) return callee;
+ if (!qualifierCapture) {
+ // Python: if the callee is a local alias for a class, substitute the class name.
+ // e.g. engineclass = base.Engine; engineclass(pool, ...) → Engine(pool, ...)
+ if (language === SupportedLanguages.Python) {
+ const funcName = findEnclosingFunction(entities, callLine);
+ if (funcName) {
+ const typeForAssign = assignTypeMap.get(funcName)?.get(callee);
+ if (typeForAssign) return typeForAssign;
+ }
+ }
+ return callee;
+ }
let qualifier = qualifierCapture.node.text;
// Python: 'self'/'cls' always refers to the enclosing class — substitute it
// so the edge reads 'Query.filter' instead of 'self.filter', enabling
@@ -712,10 +1012,12 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
const typeName = refType.node.text;
if (!typeName || TYPE_BUILTINS.has(typeName) || typeName.length <= 1) continue;
- // Use the enclosing class as the src so the edge reads "ClassX REFERENCES TypeY".
- // Fall back to the file name when no enclosing class is found (top-level usage).
+ // Prefer the enclosing function/method so parameter and return-type references
+ // point back to the actual caller. Fall back to class, then file.
const refLine = refType.node.startPosition.row + 1;
- const src = findEnclosing(classRanges, refLine, typeName) ?? fileName;
+ const src = findEnclosingFunction(entities, refLine)
+ ?? findEnclosing(classRanges, refLine, typeName)
+ ?? fileName;
if (!seenRefs.has(src)) seenRefs.set(src, new Set());
const seen = seenRefs.get(src)!;
@@ -751,7 +1053,15 @@ export function parseFile(filePath: string, source: string): FileParseResult | n
});
}
- return { filePath, language, entities, chunks, relationships, fileRole: classifyFileRole(filePath, source) };
+ return {
+ filePath,
+ language,
+ entities,
+ chunks,
+ relationships,
+ importAliases: Object.keys(importAliases).length > 0 ? importAliases : undefined,
+ fileRole: classifyFileRole(filePath, source),
+ };
} catch {
return null;
}
@@ -808,6 +1118,156 @@ function bestQKey(
return qks.length === 1 ? qks[0] : null;
}
+// ---------------------------------------------------------------------------
+// Global resolution index — pre-scan for cross-batch edge resolution
+// ---------------------------------------------------------------------------
+
+export interface GlobalResolutionIndex {
+ fileHasSymbol: Map<string, Set<string>>;
+ fileQKeys: Map<string, Map<string, string[]>>;
+ symbolToFiles: Map<string, string[]>;
+ stemToFiles: Map<string, string[]>;
+ dirToIndexFiles: Map<string, string[]>;
+ packageToFiles: Map<string, string[]>;
+ goPkgDirToFiles: Map<string, string[]>;
+ goPkgPathToFiles: Map<string, string[]>;
+}
+
+/**
+ * Build a GlobalResolutionIndex from a list of file paths (and optional source
+ * texts for entity extraction). Covers the entire repository so that each
+ * streaming batch can resolve edges pointing to files outside the batch.
+ */
+export function buildGlobalResolutionIndex(
+ filePaths: string[],
+ sources?: Map<string, string>,
+): GlobalResolutionIndex {
+ const stemToFiles = new Map<string, string[]>();
+ const dirToIndexFiles = new Map<string, string[]>();
+ const packageToFiles = new Map<string, string[]>();
+ const goPkgDirToFiles = new Map<string, string[]>();
+ const goPkgPathToFiles = new Map<string, string[]>();
+
+ for (const fp of filePaths) {
+ const ext = nodePath.extname(fp);
+ const stem = nodePath.basename(fp, ext);
+
+ // stemToFiles
+ const stemList = stemToFiles.get(stem) ?? [];
+ stemList.push(fp);
+ stemToFiles.set(stem, stemList);
+
+ // dirToIndexFiles
+ if (stem === 'index') {
+ const dirName = nodePath.basename(nodePath.dirname(fp));
+ const dirList = dirToIndexFiles.get(dirName) ?? [];
+ dirList.push(fp);
+ dirToIndexFiles.set(dirName, dirList);
+ }
+
+ // packageToFiles (Scala/Java)
+ if (ext === '.scala' || ext === '.java') {
+ const dir = nodePath.dirname(fp);
+ const parts = dir.split(/[/\\]/);
+ const maxDepth = Math.min(8, parts.length);
+ for (let i = parts.length - 1; i >= parts.length - maxDepth; i--) {
+ const pkg = parts.slice(i).join('.');
+ const list = packageToFiles.get(pkg) ?? [];
+ list.push(fp);
+ packageToFiles.set(pkg, list);
+ }
+ }
+
+ // goPkgDirToFiles / goPkgPathToFiles (Go)
+ if (ext === '.go') {
+ const dirName = nodePath.basename(nodePath.dirname(fp));
+ const goDirList = goPkgDirToFiles.get(dirName) ?? [];
+ goDirList.push(fp);
+ goPkgDirToFiles.set(dirName, goDirList);
+
+ const parts = nodePath.dirname(fp).replace(/\\/g, '/').split('/').filter(Boolean);
+ const maxDepth = Math.min(8, parts.length);
+ for (let i = parts.length - 1; i >= parts.length - maxDepth; i--) {
+ const pkgPath = parts.slice(i).join('/');
+ const pkgList = goPkgPathToFiles.get(pkgPath) ?? [];
+ pkgList.push(fp);
+ goPkgPathToFiles.set(pkgPath, pkgList);
+ }
+ }
+ }
+
+ // Entity maps (built from sources if provided, via fast regex scan)
+ const fileQKeys = new Map<string, Map<string, string[]>>();
+ const fileHasSymbol = new Map<string, Set<string>>();
+ const symbolToFiles = new Map<string, string[]>();
+
+ if (sources) {
+ const pkgRe = /^package\s+(\w+)/m;
+ const typeRe = /^type\s+([A-Z]\w*)\s+/gm;
+ const funcRe = /^func\s+([A-Z]\w*)\s*[(\[]/gm;
+ const methodRe = /^func\s+\([^)]+\)\s+([A-Z]\w*)\s*[(\[]/gm;
+
+ for (const [fp, src] of sources) {
+ if (nodePath.extname(fp) !== '.go') continue;
+
+ const qkMap = new Map<string, string[]>();
+
+ // Package name — critical for qualifier-assisted REFERENCES resolution
+ const pkgMatch = pkgRe.exec(src);
+ if (pkgMatch) {
+ const name = pkgMatch[1];
+ qkMap.set(name, [name]);
+ }
+
+ // Exported types (struct/interface/aliases/type defs)
+ typeRe.lastIndex = 0;
+ let m: RegExpExecArray | null;
+ while ((m = typeRe.exec(src)) !== null) {
+ const name = m[1];
+ if (!qkMap.has(name)) qkMap.set(name, [name]);
+ }
+
+ // Exported top-level functions
+ funcRe.lastIndex = 0;
+ while ((m = funcRe.exec(src)) !== null) {
+ const name = m[1];
+ if (!qkMap.has(name)) qkMap.set(name, [name]);
+ }
+
+ // Exported methods (receiver form)
+ methodRe.lastIndex = 0;
+ while ((m = methodRe.exec(src)) !== null) {
+ const name = m[1];
+ if (!qkMap.has(name)) qkMap.set(name, [name]);
+ }
+
+ if (qkMap.size > 0) {
+ fileQKeys.set(fp, qkMap);
+ fileHasSymbol.set(fp, new Set(qkMap.keys()));
+ }
+ }
+
+ for (const [fp, symbols] of fileHasSymbol) {
+ for (const sym of symbols) {
+ const list = symbolToFiles.get(sym) ?? [];
+ list.push(fp);
+ symbolToFiles.set(sym, list);
+ }
+ }
+ }
+
+ return {
+ fileHasSymbol,
+ fileQKeys,
+ symbolToFiles,
+ stemToFiles,
+ dirToIndexFiles,
+ packageToFiles,
+ goPkgDirToFiles,
+ goPkgPathToFiles,
+ };
+}
+
/**
* Resolves CALLS and EXTENDS relationships to their cross-file targets by
* building a symbol table and import map over the full batch of parsed files.
@@ -834,20 +1294,26 @@ function bestQKey(
* - Unambiguous class method: dstQualifiedKey === 'ClassName.method'
* - Ambiguous (two entities share the same plain name): edge not emitted
*/
-export function resolveEdges(results: FileParseResult[], stats?: {
- importLookups: number; transitiveLookups: number; globalFallbacks: number;
- globalCandidateTotal: number; resolvedImport: number; resolvedTransitive: number;
- resolvedGlobal: number; resolvedQualifier: number; skippedSameFile: number; skippedAmbiguous: number;
-}): ResolvedEdge[] {
+export function resolveEdges(
+ results: FileParseResult[],
+ stats?: {
+ importLookups: number; transitiveLookups: number; globalFallbacks: number;
+ globalCandidateTotal: number; resolvedImport: number; resolvedTransitive: number;
+ resolvedGlobal: number; resolvedQualifier: number; skippedSameFile: number; skippedAmbiguous: number;
+ },
+ globalIndex?: GlobalResolutionIndex,
+): ResolvedEdge[] {
// Provide a default no-op stats bag when caller passes none (backward compat).
if (!stats) stats = {
importLookups: 0, transitiveLookups: 0, globalFallbacks: 0,
globalCandidateTotal: 0, resolvedImport: 0, resolvedTransitive: 0,
resolvedGlobal: 0, resolvedQualifier: 0, skippedSameFile: 0, skippedAmbiguous: 0,
};
- // fileQKeys: filePath → (plainName → qualifiedKeys[])
+ // fileQKeys: seed from global index (cross-batch files), then batch entries override.
// Mirrors the entityQKey computation in buildPatch so nodeIds match exactly.
- const fileQKeys = new Map<string, Map<string, string[]>>();
+ const fileQKeys = globalIndex
+ ? new Map<string, Map<string, string[]>>(globalIndex.fileQKeys)
+ : new Map<string, Map<string, string[]>>();
for (const r of results) {
const qkMap = new Map<string, string[]>();
for (const e of r.entities) {
@@ -857,10 +1323,10 @@ export function resolveEdges(results: FileParseResult[], stats?: {
list.push(qk);
qkMap.set(e.name, list);
}
- fileQKeys.set(r.filePath, qkMap);
+ fileQKeys.set(r.filePath, qkMap); // overrides global entry if present
}
- // fileHasSymbol: filePath → Set<plainName> (fast membership check)
+ // fileHasSymbol: rebuilt from merged fileQKeys so per-batch overrides take effect.
const fileHasSymbol = new Map<string, Set<string>>();
for (const [fp, qkMap] of fileQKeys) {
fileHasSymbol.set(fp, new Set(qkMap.keys()));
@@ -869,7 +1335,8 @@ export function resolveEdges(results: FileParseResult[], stats?: {
// resultsByPath: O(1) lookup replacing results.find() in transitive import loop
const resultsByPath = new Map<string, FileParseResult>(results.map(r => [r.filePath, r]));
- // symbolToFiles: plainName → filePath[] — replaces O(F) full-scan in tier-3 and qualifier fallback
+ // symbolToFiles: rebuilt from merged fileHasSymbol (not seeded directly — per-batch
+ // overrides can change which files define a symbol).
const symbolToFiles = new Map<string, string[]>();
for (const [fp, symbols] of fileHasSymbol) {
for (const sym of symbols) {
@@ -885,83 +1352,102 @@ export function resolveEdges(results: FileParseResult[], stats?: {
fileLanguage.set(r.filePath, r.language);
}
- // stemToFiles: basename-without-extension → filePath[]
- // Matches the last path segment of an import to files in the batch.
- const stemToFiles = new Map<string, string[]>();
+ // stemToFiles: seed from global index, then add batch entries.
+ const stemToFiles = globalIndex
+ ? new Map<string, string[]>(globalIndex.stemToFiles)
+ : new Map<string, string[]>();
for (const r of results) {
const stem = nodePath.basename(r.filePath, nodePath.extname(r.filePath));
const list = stemToFiles.get(stem) ?? [];
- list.push(r.filePath);
+ if (!list.includes(r.filePath)) list.push(r.filePath);
stemToFiles.set(stem, list);
}
- // dirToIndexFiles: directory basename → filePath[] for 'index.*' files
- // Handles directory imports: './services' → 'services/index.ts'
- const dirToIndexFiles = new Map<string, string[]>();
+ // dirToIndexFiles: seed from global index, then add batch entries.
+ const dirToIndexFiles = globalIndex
+ ? new Map<string, string[]>(globalIndex.dirToIndexFiles)
+ : new Map<string, string[]>();
for (const r of results) {
const stem = nodePath.basename(r.filePath, nodePath.extname(r.filePath));
if (stem === 'index') {
const dirName = nodePath.basename(nodePath.dirname(r.filePath));
const list = dirToIndexFiles.get(dirName) ?? [];
- list.push(r.filePath);
+ if (!list.includes(r.filePath)) list.push(r.filePath);
dirToIndexFiles.set(dirName, list);
}
}
- // packageToFiles: dotted package path → filePath[] (Scala/Java wildcard imports)
+ // packageToFiles: seed from global index, then add batch entries.
// e.g. "ix.memory.model" → all .scala files under .../ix/memory/model/
- // Builds all suffix keys so that both "model" and "ix.memory.model" resolve.
- const packageToFiles = new Map<string, string[]>();
+ const packageToFiles = globalIndex
+ ? new Map<string, string[]>(globalIndex.packageToFiles)
+ : new Map<string, string[]>();
for (const r of results) {
const ext = nodePath.extname(r.filePath);
if (ext !== '.scala' && ext !== '.java') continue;
const dir = nodePath.dirname(r.filePath);
const parts = dir.split(/[/\\]/);
- // Cap at 8 segments to avoid noise from very shallow path components like "src", "main".
const maxDepth = Math.min(8, parts.length);
for (let i = parts.length - 1; i >= parts.length - maxDepth; i--) {
const pkg = parts.slice(i).join('.');
const list = packageToFiles.get(pkg) ?? [];
- list.push(r.filePath);
+ if (!list.includes(r.filePath)) list.push(r.filePath);
packageToFiles.set(pkg, list);
}
}
- // goPkgDirToFiles: Go package directory name → filePath[]
- // In Go, the last segment of an import path IS the package directory name.
- // e.g. import "go.etcd.io/etcd/server/etcdserver" is stripped to modName "etcdserver"
- // by the import processor; this index maps that name to all .go files in
- // any directory named "etcdserver" within the ingested batch.
- const goPkgDirToFiles = new Map<string, string[]>();
+ // goPkgDirToFiles / goPkgPathToFiles: seed from global index, then add batch entries.
+ const goPkgDirToFiles = globalIndex
+ ? new Map<string, string[]>(globalIndex.goPkgDirToFiles)
+ : new Map<string, string[]>();
+ const goPkgPathToFiles = globalIndex
+ ? new Map<string, string[]>(globalIndex.goPkgPathToFiles)
+ : new Map<string, string[]>();
for (const r of results) {
if (nodePath.extname(r.filePath) !== '.go') continue;
const dirName = nodePath.basename(nodePath.dirname(r.filePath));
const list = goPkgDirToFiles.get(dirName) ?? [];
- list.push(r.filePath);
+ if (!list.includes(r.filePath)) list.push(r.filePath);
goPkgDirToFiles.set(dirName, list);
+ const parts = nodePath.dirname(r.filePath).replace(/\\/g, '/').split('/').filter(Boolean);
+ const maxDepth = Math.min(8, parts.length);
+ for (let i = parts.length - 1; i >= parts.length - maxDepth; i--) {
+ const pkgPath = parts.slice(i).join('/');
+ const pkgList = goPkgPathToFiles.get(pkgPath) ?? [];
+ if (!pkgList.includes(r.filePath)) pkgList.push(r.filePath);
+ goPkgPathToFiles.set(pkgPath, pkgList);
+ }
}
// ── Helpers ────────────────────────────────────────────────────────
+ function normalizeImportTarget(modName: unknown): string {
+ if (typeof modName === 'string') return modName;
+ if (modName == null) return '';
+ return String(modName);
+ }
+
/** Resolve a module name to matching file paths in the batch. */
- function modNameToFiles(modName: string, excludeFp: string): string[] {
+ function modNameToFiles(modName: unknown, excludeFp: string): string[] {
+ const normalizedModName = normalizeImportTarget(modName);
+ if (!normalizedModName) return [];
const fps: string[] = [];
// Strip leading non-word chars for bare aliases: '@components' → 'components'
- const candidates = [modName];
- const stripped = modName.replace(/^[^a-zA-Z0-9_]+/, '');
- if (stripped && stripped !== modName) candidates.push(stripped);
+ const candidates = [normalizedModName];
+ const stripped = normalizedModName.replace(/^[^a-zA-Z0-9_]+/, '');
+ if (stripped && stripped !== normalizedModName) candidates.push(stripped);
// Strip file extensions so "explain.js" resolves to the "explain" stem
// (TS/JS ESM imports use .js extensions that map to .ts source files)
- const noExt = modName.replace(/\.(js|ts|mjs|cjs|jsx|tsx|py|scala|java|c|cc|cpp|cxx|h|hh|hpp|hxx)$/, '');
- if (noExt !== modName && noExt && !candidates.includes(noExt)) candidates.push(noExt);
- const pathBasename = nodePath.posix.basename(modName);
+ const noExt = normalizedModName.replace(/\.(js|ts|mjs|cjs|jsx|tsx|py|scala|java|c|cc|cpp|cxx|h|hh|hpp|hxx)$/, '');
+ if (noExt !== normalizedModName && noExt && !candidates.includes(noExt)) candidates.push(noExt);
+ const pathBasename = nodePath.posix.basename(normalizedModName);
if (pathBasename && !candidates.includes(pathBasename)) candidates.push(pathBasename);
const pathBasenameNoExt = nodePath.posix.basename(noExt);
if (pathBasenameNoExt && !candidates.includes(pathBasenameNoExt)) candidates.push(pathBasenameNoExt);
// For dotted paths (Scala/Java: 'ix.memory.model.Edge'), also try last segment
- const lastDot = modName.lastIndexOf('.');
+ const lastDot = normalizedModName.lastIndexOf('.');
if (lastDot !== -1) {
- const lastSegment = modName.slice(lastDot + 1);
+ const lastSegment = normalizedModName.slice(lastDot + 1);
if (lastSegment && !candidates.includes(lastSegment)) candidates.push(lastSegment);
}
for (const cand of candidates) {
@@ -973,17 +1459,116 @@ export function resolveEdges(results: FileParseResult[], stats?: {
}
}
// Package-path wildcard resolution (Scala/Java): "ix.memory.model" → all files in that dir
- for (const fp of packageToFiles.get(modName) ?? []) {
+ for (const fp of packageToFiles.get(normalizedModName) ?? []) {
if (fp !== excludeFp && !fps.includes(fp)) fps.push(fp);
}
// Go package directory resolution: "etcdserver" → all .go files in .../etcdserver/
// The last segment of a Go import path is the package directory name, not a file stem.
- for (const fp of goPkgDirToFiles.get(modName) ?? []) {
+ for (const fp of goPkgDirToFiles.get(normalizedModName) ?? []) {
if (fp !== excludeFp && !fps.includes(fp)) fps.push(fp);
}
+ const slashParts = normalizedModName.replace(/\\/g, '/').split('/').filter(Boolean);
+ const maxSlashDepth = Math.min(8, slashParts.length);
+ for (let i = 0; i < maxSlashDepth; i++) {
+ const suffix = slashParts.slice(slashParts.length - 1 - i).join('/');
+ for (const fp of goPkgPathToFiles.get(suffix) ?? []) {
+ if (fp !== excludeFp && !fps.includes(fp)) fps.push(fp);
+ }
+ }
return fps;
}
+ function goImportSuffixes(modName: unknown): string[] {
+ const normalizedModName = normalizeImportTarget(modName);
+ if (!normalizedModName) return [];
+ const parts = normalizedModName.replace(/\\/g, '/').split('/').filter(Boolean);
+ const suffixes: string[] = [];
+ const maxDepth = Math.min(8, parts.length);
+ for (let i = 0; i < maxDepth; i++) {
+ suffixes.push(parts.slice(parts.length - 1 - i).join('/'));
+ }
+ return suffixes;
+ }
+
+ function pickGoPackageAnchor(files: string[]): string | null {
+ if (files.length === 0) return null;
+ const nonTestFiles = files.filter(fp => !fp.endsWith('_test.go'));
+ const pool = nonTestFiles.length > 0 ? nonTestFiles : files;
+ const dirName = nodePath.basename(nodePath.dirname(pool[0])).toLowerCase();
+
+ const exactStemMatch = pool.filter(fp => candidateStem(fp) === dirName);
+ if (exactStemMatch.length === 1) return exactStemMatch[0];
+
+ const ranked = [...pool].sort((a, b) => {
+ const aResult = resultsByPath.get(a);
+ const bResult = resultsByPath.get(b);
+ const aImports = aResult?.relationships.filter(rel => rel.predicate === 'IMPORTS').length ?? 0;
+ const bImports = bResult?.relationships.filter(rel => rel.predicate === 'IMPORTS').length ?? 0;
+ if (aImports !== bImports) return bImports - aImports;
+
+ // Fall back to global index entity count for cross-batch files (not in resultsByPath).
+ // fileQKeys is seeded from buildGlobalResolutionIndex which pre-scans exported
+ // types/functions via regex, so substantive files like instance.go score higher
+ // than package-doc stubs like doc.go (which only has the package name).
+ const aEntities = aResult?.entities.length ?? fileQKeys.get(a)?.size ?? 0;
+ const bEntities = bResult?.entities.length ?? fileQKeys.get(b)?.size ?? 0;
+ if (aEntities !== bEntities) return bEntities - aEntities;
+
+ return a.localeCompare(b);
+ });
+ return ranked[0] ?? null;
+ }
+
+ function narrowGoImportCandidates(
+ srcFilePath: string,
+ modName: unknown,
+ importMatches: string[],
+ pickFromSingleDir: (files: string[]) => string[],
+ ): string[] {
+ const suffixes = goImportSuffixes(modName);
+ const scored = importMatches.map(fp => {
+ const dirPath = nodePath.dirname(fp).replace(/\\/g, '/');
+ let bestSuffixLen = -1;
+ for (const suffix of suffixes) {
+ if (dirPath === suffix || dirPath.endsWith(`/${suffix}`)) {
+ if (suffix.length > bestSuffixLen) bestSuffixLen = suffix.length;
+ }
+ }
+ return { fp, bestSuffixLen };
+ });
+
+ const bestSuffixLen = Math.max(...scored.map(item => item.bestSuffixLen));
+ const narrowed = bestSuffixLen >= 0
+ ? scored.filter(item => item.bestSuffixLen === bestSuffixLen).map(item => item.fp)
+ : importMatches;
+
+ const dirGroups = new Map<string, string[]>();
+ for (const fp of narrowed) {
+ const key = nodePath.dirname(fp).replace(/\\/g, '/');
+ const list = dirGroups.get(key) ?? [];
+ list.push(fp);
+ dirGroups.set(key, list);
+ }
+
+ if (dirGroups.size === 1) return pickFromSingleDir([...dirGroups.values()][0]);
+ return narrowed;
+ }
+
+ function resolveImportTargets(srcFilePath: string, srcLanguage: SupportedLanguages, modName: unknown): string[] {
+ const importMatches = modNameToFiles(modName, srcFilePath);
+ if (srcLanguage !== SupportedLanguages.Go || importMatches.length <= 1) return importMatches;
+ return narrowGoImportCandidates(srcFilePath, modName, importMatches, files => {
+ const anchor = pickGoPackageAnchor(files);
+ return anchor ? [anchor] : [];
+ });
+ }
+
+ function resolveImportQualifierTargets(srcFilePath: string, srcLanguage: SupportedLanguages, modName: unknown): string[] {
+ const importMatches = modNameToFiles(modName, srcFilePath);
+ if (srcLanguage !== SupportedLanguages.Go || importMatches.length <= 1) return importMatches;
+ return narrowGoImportCandidates(srcFilePath, modName, importMatches, files => files);
+ }
+
function tokenizeSymbolParts(value: string): string[] {
return value
.replace(/([a-z\d])([A-Z])/g, '$1 $2')
@@ -1121,7 +1706,7 @@ export function resolveEdges(results: FileParseResult[], stats?: {
const importedFilePaths = new Set<string>();
for (const rel of result.relationships) {
if (rel.predicate !== 'IMPORTS') continue;
- for (const fp of modNameToFiles(rel.dstName, srcFilePath)) {
+ for (const fp of resolveImportTargets(srcFilePath, srcLanguage, rel.dstName)) {
importedFilePaths.add(fp);
}
}
@@ -1134,7 +1719,7 @@ export function resolveEdges(results: FileParseResult[], stats?: {
if (!fpResult) continue;
for (const rel of fpResult.relationships) {
if (rel.predicate !== 'IMPORTS') continue;
- for (const transitiveFp of modNameToFiles(rel.dstName, srcFilePath)) {
+ for (const transitiveFp of resolveImportTargets(fp, fpResult.language, rel.dstName)) {
if (!importedFilePaths.has(transitiveFp)) transitiveFilePaths.add(transitiveFp);
}
}
@@ -1171,7 +1756,7 @@ export function resolveEdges(results: FileParseResult[], stats?: {
}
}
- const importMatches = modNameToFiles(rel.dstName, srcFilePath);
+ const importMatches = resolveImportTargets(srcFilePath, result.language, rel.dstName);
if (importMatches.length === 1) {
const fp = importMatches[0];
resolved.push({
@@ -1184,10 +1769,16 @@ export function resolveEdges(results: FileParseResult[], stats?: {
confidence: 0.9,
});
stats.resolvedImport++;
+ continue;
} else if (importMatches.length > 1) {
stats.skippedAmbiguous++;
+ continue;
}
- continue;
+ // importMatches.length === 0: if dstName is a PascalCase symbol (class/function name
+ // captured via import.name from "from X import ClassName"), fall through to Tier 2/3
+ // symbol resolution so the edge connects to the actual class node rather than a file.
+ if (srcLanguage !== SupportedLanguages.Python || !/^[A-Z]/.test(rel.dstName)) continue;
+ // fall through to Tier 1b → Tier 2 → Tier 3 below
}
const dstName = rel.dstName;
@@ -1202,17 +1793,30 @@ export function resolveEdges(results: FileParseResult[], stats?: {
const qualifierPart = dstName.slice(0, qualDot);
const memberPart = dstName.slice(qualDot + 1);
if (memberPart && qualifierPart) {
+ const aliasedImportMatches =
+ srcLanguage === SupportedLanguages.Go
+ ? resolveImportQualifierTargets(srcFilePath, srcLanguage, result.importAliases?.[qualifierPart] ?? '')
+ : [];
// Try import-scoped qualifier first
+ const qualifierSearchPool = aliasedImportMatches.length > 0
+ ? aliasedImportMatches
+ : [...importedFilePaths];
const qualImportMatches: string[] = [];
- for (const fp of importedFilePaths) {
+ for (const fp of qualifierSearchPool) {
if (fileDefinesQualifiedMember(fp, qualifierPart, memberPart) || fileHasSymbol.get(fp)?.has(qualifierPart)) {
qualImportMatches.push(fp);
}
}
+ if (qualImportMatches.length === 0 && aliasedImportMatches.length > 0) {
+ for (const fp of aliasedImportMatches) {
+ if (fileHasSymbol.get(fp)?.has(memberPart)) qualImportMatches.push(fp);
+ }
+ }
if (qualImportMatches.length === 1) {
const qfp = qualImportMatches[0];
+ const preferredQKey = aliasedImportMatches.length > 0 ? undefined : `${qualifierPart}.${memberPart}`;
if (fileHasSymbol.get(qfp)?.has(memberPart)) {
- const dstQualifiedKey = bestQKey(fileQKeys, qfp, memberPart, `${qualifierPart}.${memberPart}`);
+ const dstQualifiedKey = bestQKey(fileQKeys, qfp, memberPart, preferredQKey);
if (dstQualifiedKey !== null) {
// dstName must match rel.dstName so buildPatchWithResolution can look it up
resolved.push({ srcFilePath, srcName, dstFilePath: qfp, dstName, dstQualifiedKey, predicate: rel.predicate, confidence: 0.9 });
diff --git a/core-ingestion/src/languages.ts b/core-ingestion/src/languages.ts
index 48b81165..0d7a6822 100644
--- a/core-ingestion/src/languages.ts
+++ b/core-ingestion/src/languages.ts
@@ -13,6 +13,8 @@ export enum SupportedLanguages {
Kotlin = 'kotlin',
Swift = 'swift',
Scala = 'scala',
+ YAML = 'yaml',
+ Dockerfile = 'dockerfile',
}
const EXT_MAP: Record<string, SupportedLanguages> = {
@@ -40,9 +42,20 @@ const EXT_MAP: Record<string, SupportedLanguages> = {
'.swift':SupportedLanguages.Swift,
'.scala':SupportedLanguages.Scala,
'.sc': SupportedLanguages.Scala,
+ '.yaml': SupportedLanguages.YAML,
+ '.yml': SupportedLanguages.YAML,
+ '.dockerfile': SupportedLanguages.Dockerfile,
};
export function languageFromPath(filePath: string): SupportedLanguages | null {
- const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase();
+ const normalized = filePath.replace(/\\/g, '/');
+ const fileName = normalized.slice(normalized.lastIndexOf('/') + 1);
+ const lowerFileName = fileName.toLowerCase();
+ if (lowerFileName === 'dockerfile' || lowerFileName.endsWith('.dockerfile')) {
+ return SupportedLanguages.Dockerfile;
+ }
+ const dotIndex = lowerFileName.lastIndexOf('.');
+ if (dotIndex === -1) return null;
+ const ext = lowerFileName.slice(dotIndex);
return EXT_MAP[ext] ?? null;
}
diff --git a/core-ingestion/src/patch-builder.ts b/core-ingestion/src/patch-builder.ts
index 74f674bd..8b2ac80b 100644
--- a/core-ingestion/src/patch-builder.ts
+++ b/core-ingestion/src/patch-builder.ts
@@ -40,18 +40,22 @@ function chunkId(filePath: string, chunkKind: string, name: string | null, start
// ---------------------------------------------------------------------------
function sourceType(filePath: string): string {
- const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase();
+ const normalized = filePath.replace(/\\/g, '/');
+ const fileName = normalized.slice(normalized.lastIndexOf('/') + 1).toLowerCase();
+ if (fileName === 'dockerfile' || fileName.endsWith('.dockerfile')) return 'config';
+ const dotIndex = fileName.lastIndexOf('.');
+ const ext = dotIndex === -1 ? '' : fileName.slice(dotIndex);
if (['.json', '.yaml', '.yml', '.toml', '.ini', '.conf', '.env'].includes(ext)) return 'config';
if (['.md', '.mdx', '.rst', '.txt'].includes(ext)) return 'doc';
return 'code';
}
export function extractorName(): string {
- return `tree-sitter/1.19`;
+ return `tree-sitter/1.20`;
}
/** Previous extractor versions — their patches are superseded when re-ingesting. */
-export const PREVIOUS_EXTRACTORS = ['tree-sitter/1.18', 'tree-sitter/1.17', 'tree-sitter/1.16', 'tree-sitter/1.15', 'tree-sitter/1.14', 'tree-sitter/1.13', 'tree-sitter/1.12', 'tree-sitter/1.11', 'tree-sitter/1.10', 'tree-sitter/1.9', 'tree-sitter/1.8', 'tree-sitter/1.7', 'tree-sitter/1.6', 'tree-sitter/1.5', 'tree-sitter/1.4', 'tree-sitter/1.3', 'tree-sitter/1.2', 'tree-sitter/1.1'];
+export const PREVIOUS_EXTRACTORS = ['tree-sitter/1.19', 'tree-sitter/1.18', 'tree-sitter/1.17', 'tree-sitter/1.16', 'tree-sitter/1.15', 'tree-sitter/1.14', 'tree-sitter/1.13', 'tree-sitter/1.12', 'tree-sitter/1.11', 'tree-sitter/1.10', 'tree-sitter/1.9', 'tree-sitter/1.8', 'tree-sitter/1.7', 'tree-sitter/1.6', 'tree-sitter/1.5', 'tree-sitter/1.4', 'tree-sitter/1.3', 'tree-sitter/1.2', 'tree-sitter/1.1'];
/** Compute a patchId for a (filePath, sourceHash, extractorVersion) triple. */
function computePatchId(filePath: string, sourceHash: string, extractor: string): string {
diff --git a/core-ingestion/src/queries.ts b/core-ingestion/src/queries.ts
index ef7c2d6e..afd0fd55 100644
--- a/core-ingestion/src/queries.ts
+++ b/core-ingestion/src/queries.ts
@@ -211,11 +211,17 @@ export const PYTHON_QUERIES = `
(import_from_statement
module_name: (relative_import) @import.source) @import
-; from . import utils / from .models import User — also capture the imported symbol names
+; from . import utils / from .models import User — capture imported symbol names (relative imports)
(import_from_statement
module_name: (relative_import)
name: (dotted_name (identifier) @import.name)) @import.names
+; from module import Class — capture imported symbol names (absolute imports)
+; e.g. from sqlalchemy.sql.schema import Column → import.name = Column
+(import_from_statement
+ module_name: (dotted_name)
+ name: (dotted_name (identifier) @import.name)) @import.names
+
(call
function: (identifier) @call.name) @call
@@ -310,6 +316,17 @@ export const PYTHON_QUERIES = `
object: (identifier) @_assign_rhs_type
attribute: (identifier))
attribute: (identifier))))))) @_assign_scope
+
+; Assignment tracking: x = module.Class (direct attribute reference, no call)
+; Handles variable aliasing like: engineclass = base.Engine
+; Combined with direct-call resolution in index.ts, this lets engineclass(...) → Engine(...)
+(function_definition
+ body: (block
+ (expression_statement
+ (assignment
+ left: (identifier) @_assign_lhs
+ right: (attribute
+ attribute: (identifier) @_assign_rhs_type))))) @_assign_scope
`;
// Java queries - works with tree-sitter-java
@@ -438,8 +455,10 @@ export const GO_QUERIES = `
name: (field_identifier) @name)))) @definition.method
; Imports
+(import_declaration (import_spec name: (package_identifier) @import.alias path: (interpreted_string_literal) @import.source)) @import
(import_declaration (import_spec path: (interpreted_string_literal) @import.source)) @import
(import_declaration (import_spec_list (import_spec path: (interpreted_string_literal) @import.source))) @import
+(import_declaration (import_spec_list (import_spec name: (package_identifier) @import.alias path: (interpreted_string_literal) @import.source))) @import
; Struct embedding — value: type Foo struct { Bar }
; NOTE: no @definition.* capture here — if it were present, defCapture fires first and
@@ -495,6 +514,15 @@ export const GO_QUERIES = `
(parameter_declaration type: (pointer_type (type_identifier) @reference.type))
(function_declaration result: (pointer_type (type_identifier) @reference.type))
(method_declaration result: (pointer_type (type_identifier) @reference.type))
+; Imported/package-qualified type refs: *scheduler.Scheduler, scheduler.Scheduler
+(field_declaration type: (qualified_type (type_identifier) @reference.type))
+(parameter_declaration type: (qualified_type (type_identifier) @reference.type))
+(function_declaration result: (qualified_type (type_identifier) @reference.type))
+(method_declaration result: (qualified_type (type_identifier) @reference.type))
+(field_declaration type: (pointer_type (qualified_type (type_identifier) @reference.type)))
+(parameter_declaration type: (pointer_type (qualified_type (type_identifier) @reference.type)))
+(function_declaration result: (pointer_type (qualified_type (type_identifier) @reference.type)))
+(method_declaration result: (pointer_type (qualified_type (type_identifier) @reference.type)))
`;
// C++ queries - works with tree-sitter-cpp
@@ -1151,5 +1179,7 @@ export const LANGUAGE_QUERIES: Record<SupportedLanguages, string> = {
[SupportedLanguages.Kotlin]: KOTLIN_QUERIES,
[SupportedLanguages.Swift]: SWIFT_QUERIES,
[SupportedLanguages.Scala]: SCALA_QUERIES,
+ [SupportedLanguages.YAML]: '',
+ [SupportedLanguages.Dockerfile]: '',
};
diff --git a/ix-cli/src/cli/__tests__/ingest-discovery.test.ts b/ix-cli/src/cli/__tests__/ingest-discovery.test.ts
new file mode 100644
index 00000000..83867140
--- /dev/null
+++ b/ix-cli/src/cli/__tests__/ingest-discovery.test.ts
@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest';
+
+import { dedupeDiscoveredFilePaths, isSupportedSourceFile } from '../commands/ingest.js';
+
+describe('dedupeDiscoveredFilePaths', () => {
+ it('collapses alternate discovered paths that point at the same canonical file', () => {
+ const deduped = dedupeDiscoveredFilePaths(
+ [
+ 'C:/repo/staging/src/k8s.io/apiserver/pkg/server/config.go',
+ 'C:/repo/vendor/k8s.io/apiserver/pkg/server/config.go',
+ 'C:/repo/cmd/kube-apiserver/app/server.go',
+ ],
+ (filePath) => {
+ if (filePath.includes('/vendor/')) {
+ return 'C:/repo/staging/src/k8s.io/apiserver/pkg/server/config.go';
+ }
+ return filePath;
+ },
+ );
+
+ expect(deduped).toEqual([
+ 'C:/repo/staging/src/k8s.io/apiserver/pkg/server/config.go',
+ 'C:/repo/cmd/kube-apiserver/app/server.go',
+ ]);
+ });
+
+ it('treats Dockerfile variants as supported source inputs', () => {
+ expect(isSupportedSourceFile('Dockerfile')).toBe(true);
+ expect(isSupportedSourceFile('deploy/prod.dockerfile')).toBe(true);
+ expect(isSupportedSourceFile('Makefile')).toBe(false);
+ });
+});
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index d2673921..bdc48d9f 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -32,8 +32,17 @@ const SUPPORTED_EXTENSIONS = new Set([
'.py', '.java', '.c', '.h', '.cpp', '.cc', '.cxx', '.hpp',
'.cs', '.go', '.rb', '.rs', '.php', '.kt', '.kts', '.swift',
'.scala', '.sc',
+ '.yaml', '.yml',
+ '.dockerfile',
]);
+export function isSupportedSourceFile(filePath: string): boolean {
+ const fileName = nodePath.basename(filePath).toLowerCase();
+ return fileName === 'dockerfile'
+ || fileName.endsWith('.dockerfile')
+ || SUPPORTED_EXTENSIONS.has(nodePath.extname(filePath).toLowerCase());
+}
+
// ---------------------------------------------------------------------------
// Language filter helpers
// ---------------------------------------------------------------------------
@@ -129,11 +138,34 @@ function* walkFiles(
if (entry.isDirectory()) {
if (recursive) yield* walkFiles(full, true);
} else if (entry.isFile()) {
- if (SUPPORTED_EXTENSIONS.has(nodePath.extname(entry.name)) && !isGeneratedFile(entry.name)) yield full;
+ if (isSupportedSourceFile(entry.name) && !isGeneratedFile(entry.name)) yield full;
}
}
}
+function canonicalizeDiscoveredFilePath(filePath: string): string {
+ try {
+ return fs.realpathSync.native(filePath);
+ } catch {
+ return nodePath.resolve(filePath);
+ }
+}
+
+export function dedupeDiscoveredFilePaths(
+ filePaths: string[],
+ canonicalize: (filePath: string) => string = canonicalizeDiscoveredFilePath,
+): string[] {
+ const deduped: string[] = [];
+ const seenCanonical = new Set<string>();
+ for (const filePath of filePaths) {
+ const canonicalPath = canonicalize(filePath);
+ if (seenCanonical.has(canonicalPath)) continue;
+ seenCanonical.add(canonicalPath);
+ deduped.push(canonicalPath);
+ }
+ return deduped;
+}
+
function tryGitLsFiles(dir: string, recursive: boolean): string[] | null {
try {
const result = spawnSync(
@@ -148,14 +180,19 @@ function tryGitLsFiles(dir: string, recursive: boolean): string[] | null {
if (result.status !== 0) return null;
const files: string[] = [];
+ const seenCanonical = new Set<string>();
for (const line of result.stdout.split(/\r?\n/)) {
if (!line) continue;
const fullPath = nodePath.resolve(dir, line);
if (!recursive && nodePath.dirname(fullPath) !== dir) continue;
- if (!SUPPORTED_EXTENSIONS.has(nodePath.extname(fullPath))) continue;
+ if (!isSupportedSourceFile(fullPath)) continue;
if (isGeneratedFile(nodePath.basename(fullPath))) continue;
try {
- if (fs.statSync(fullPath).isFile()) files.push(fullPath);
+ if (!fs.statSync(fullPath).isFile()) continue;
+ const canonicalPath = canonicalizeDiscoveredFilePath(fullPath);
+ if (seenCanonical.has(canonicalPath)) continue;
+ seenCanonical.add(canonicalPath);
+ files.push(canonicalPath);
} catch {
// Ignore racy deletes and permission issues during discovery.
}
@@ -343,7 +380,7 @@ export function registerIngestCommand(program: Command): void {
export async function ingestFiles(
path: string,
- opts: { recursive?: boolean; force?: boolean; format: string; root?: string; debug?: boolean; printSummary?: boolean; lang?: string; mapMode?: boolean }
+ opts: { recursive?: boolean; force?: boolean; format: string; root?: string; debug?: boolean; printSummary?: boolean; suppressOutput?: boolean; lang?: string; mapMode?: boolean }
): Promise<void> {
const debug = opts.debug || process.env.IX_DEBUG === '1';
const mapMode = opts.mapMode === true;
@@ -365,8 +402,13 @@ export async function ingestFiles(
nodePath.dirname(fileURLToPath(import.meta.url)),
'../../../../core-ingestion/dist/parse-worker.js',
);
- const pool = new ParsePool(workerPath, Math.max(1, os.cpus().length - 1));
- pool.init();
+ let pool: ParsePool | null = null;
+ const ensureParsePool = (): ParsePool => {
+ if (pool) return pool;
+ pool = new ParsePool(workerPath, Math.max(1, os.cpus().length - 1));
+ pool.init();
+ return pool;
+ };
let progressPhase = 'Scanning';
let progressCurrent = 0;
@@ -453,10 +495,15 @@ export async function ingestFiles(
return lang !== null && langFilter.has(lang);
};
+ if (!fs.existsSync(resolvedPath)) {
+ throw new Error(`Path not found: ${resolvedPath}`);
+ }
const stat = fs.statSync(resolvedPath);
- const filePaths: string[] = stat.isFile()
- ? (SUPPORTED_EXTENSIONS.has(nodePath.extname(resolvedPath)) ? [resolvedPath] : [])
- : (tryGitLsFiles(resolvedPath, opts.recursive ?? true) ?? Array.from(walkFiles(resolvedPath, opts.recursive ?? true)));
+ const filePaths: string[] = dedupeDiscoveredFilePaths(
+ stat.isFile()
+ ? (isSupportedSourceFile(resolvedPath) ? [resolvedPath] : [])
+ : (tryGitLsFiles(resolvedPath, opts.recursive ?? true) ?? Array.from(walkFiles(resolvedPath, opts.recursive ?? true))),
+ );
filesDiscovered = filePaths.length;
const discovered = performance.now();
@@ -537,6 +584,7 @@ export async function ingestFiles(
type ParsedFile = { filePath: string; parsed: any; hash: string; previousHash: string | undefined };
let resolveEdgesFn: Function | null = null;
let buildPatchFn: Function | null = null;
+ let globalIndex: any = undefined;
const PARSE_STREAM_CHUNK = 1000; // streaming batch size for large repos
const SMALL_REPO_THRESHOLD = 5_000; // parse-all-first below this; streaming above
@@ -676,7 +724,7 @@ export async function ingestFiles(
setCurrentWork(`resolve+commit batch ending ${nodePath.basename(batch[batch.length - 1].filePath)}`);
const resolveStart = performance.now();
- const resolvedEdges = resolveEdgesFn!(batch.map(f => f.parsed), resolveStats);
+ const resolvedEdges = resolveEdgesFn!(batch.map(f => f.parsed), resolveStats, globalIndex);
resolveEdgesMs = Math.round(performance.now() - resolveStart);
const batchEdgesByFile = new Map<string, any[]>();
for (const edge of resolvedEdges) {
@@ -743,7 +791,7 @@ export async function ingestFiles(
try {
setCurrentWork(`resolve ${allParsed.length} files`);
const resolveStart = performance.now();
- const resolvedEdges = resolveEdgesFn!(allParsed.map(f => f.parsed), resolveStats);
+ const resolvedEdges = resolveEdgesFn!(allParsed.map(f => f.parsed), resolveStats, globalIndex);
resolveEdgesMs = Math.round(performance.now() - resolveStart);
const edgesByFile = new Map<string, any[]>();
for (const edge of resolvedEdges) {
@@ -836,13 +884,35 @@ export async function ingestFiles(
progressTotal = parseable.length;
progressCurrent = 0;
- // Pipeline overlap: parse next chunk while committing current chunk
+ // Build global resolution index from all repo file paths (not just changed files)
+ // so cross-batch imports resolve correctly even in streaming per-chunk mode.
+ // Reuse bytes already read into changedPaths (on first install = all files, zero extra reads).
+ // Only async-read Go files that were mtime-clean and therefore not in changedPaths.
+ {
+ const sources = new Map<string, string>();
+ const changedSet = new Set(changedPaths.map(c => c.filePath));
+ for (const { filePath, bytes } of changedPaths) {
+ if (nodePath.extname(filePath) === '.go') sources.set(filePath, bytes.toString('utf-8'));
+ }
+ const remainingGoFiles = filePaths.filter(fp => nodePath.extname(fp) === '.go' && !changedSet.has(fp));
+ const GO_READ_BATCH = 500;
+ for (let i = 0; i < remainingGoFiles.length; i += GO_READ_BATCH) {
+ const batch = remainingGoFiles.slice(i, i + GO_READ_BATCH);
+ const texts = await Promise.all(batch.map(fp => fs.promises.readFile(fp, 'utf-8').catch(() => null)));
+ for (let j = 0; j < batch.length; j++) {
+ if (texts[j] != null) sources.set(batch[j], texts[j]!);
+ }
+ }
+ globalIndex = (ingestion.buildGlobalResolutionIndex as Function)(filePaths, sources);
+ sources.clear();
+ }
+
let pendingFlush: Promise<void> = Promise.resolve();
for (let i = 0; i < parseable.length; i += PARSE_STREAM_CHUNK) {
const chunk = parseable.slice(i, i + PARSE_STREAM_CHUNK);
const parseResults = await Promise.all(
chunk.map(f =>
- pool.parse(f.filePath, f.source).then(r => { progressCurrent++; return r; }),
+ ensureParsePool().parse(f.filePath, f.source).then(r => { progressCurrent++; return r; }),
),
);
const batch: ParsedFile[] = [];
@@ -865,6 +935,24 @@ export async function ingestFiles(
resolveEdgesFn = ingestion.resolveEdges as Function;
buildPatchFn = patchBuilder.buildPatchWithResolution as Function;
+ // Build global resolution index from all repo file paths before the parse loop
+ // so cross-batch imports resolve correctly even in streaming per-chunk mode.
+ // Use async batched reads (same as Path A) to avoid blocking the event loop.
+ {
+ const sources = new Map<string, string>();
+ const goFiles = filePaths.filter(fp => nodePath.extname(fp) === '.go');
+ const GO_READ_BATCH = 500;
+ for (let i = 0; i < goFiles.length; i += GO_READ_BATCH) {
+ const batch = goFiles.slice(i, i + GO_READ_BATCH);
+ const texts = await Promise.all(batch.map(fp => fs.promises.readFile(fp, 'utf-8').catch(() => null)));
+ for (let j = 0; j < batch.length; j++) {
+ if (texts[j] != null) sources.set(batch[j], texts[j]!);
+ }
+ }
+ globalIndex = ingestion.buildGlobalResolutionIndex(filePaths, sources);
+ sources.clear();
+ }
+
progressPhase = 'Parsing';
progressTotal = filePaths.length;
progressCurrent = 0;
@@ -873,7 +961,7 @@ export async function ingestFiles(
// Each file is dispatched to the parse pool immediately after it is read so workers
// start processing as soon as the first file in the chunk is ready rather than
// waiting for all reads in the chunk to complete.
- let pendingFlush: Promise<void> = Promise.resolve();
+ let pendingFlushB: Promise<void> = Promise.resolve();
for (let i = 0; i < filePaths.length; i += PARSE_STREAM_CHUNK) {
const chunk = filePaths.slice(i, i + PARSE_STREAM_CHUNK);
type FileData = { filePath: string; source: string; hash: string; previousHash: string | undefined } | null;
@@ -899,7 +987,7 @@ export async function ingestFiles(
}
const fd: NonNullable<FileData> = { filePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined };
fileData.push(fd);
- parsePromises.push(pool.parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; }));
+ parsePromises.push(ensureParsePool().parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; }));
} catch (err) {
parseErrors++;
fileData.push(null);
@@ -917,10 +1005,10 @@ export async function ingestFiles(
entitiesParsed += parsed.entities.length;
batch.push({ filePath: f.filePath, parsed, hash: f.hash, previousHash: f.previousHash });
}
- await pendingFlush;
- pendingFlush = flushBatch(batch);
+ await pendingFlushB;
+ pendingFlushB = flushBatch(batch);
}
- await pendingFlush;
+ await pendingFlushB;
}
const parsed = performance.now();
@@ -936,7 +1024,9 @@ export async function ingestFiles(
saveMtimeCache(projectRoot, currentMtimes);
}
} finally {
- await pool.destroy();
+ if (pool) {
+ await pool.destroy();
+ }
if (interval) {
clearInterval(interval);
if (progressTotal > 0) {
@@ -950,6 +1040,10 @@ export async function ingestFiles(
const elapsed = ((performance.now() - start) / 1000).toFixed(2);
+ if (opts.suppressOutput === true) {
+ return;
+ }
+
if (opts.format === 'json') {
console.log(JSON.stringify({
filesProcessed: filesDiscovered,
diff --git a/ix-cli/src/cli/commands/ingestion-loader.ts b/ix-cli/src/cli/commands/ingestion-loader.ts
index 80a5c290..a02eaf58 100644
--- a/ix-cli/src/cli/commands/ingestion-loader.ts
+++ b/ix-cli/src/cli/commands/ingestion-loader.ts
@@ -3,8 +3,9 @@ import { fileURLToPath, pathToFileURL } from "node:url";
type IngestionModule = {
parseFile: (filePath: string, source: string) => any;
- resolveEdges: (results: any[]) => any[];
+ resolveEdges: (results: any[], stats?: any, globalIndex?: any) => any[];
isGrammarSupported: (filePath: string) => boolean;
+ buildGlobalResolutionIndex: (filePaths: string[], sources?: Map<string, string>) => any;
};
type PatchBuilderModule = {
diff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.ts
index dbbc9488..1649d454 100644
--- a/ix-cli/src/cli/commands/map.ts
+++ b/ix-cli/src/cli/commands/map.ts
@@ -150,7 +150,19 @@ Examples:
// Ingest the path before mapping so the graph is up to date
const ingestStart = performance.now();
- await ingestFiles(cwd, { recursive: true, format: (opts.format === "json" || silent) ? "json" : "text", printSummary: false, mapMode: true });
+ try {
+ await ingestFiles(cwd, {
+ recursive: true,
+ format: (opts.format === "json" || silent) ? "json" : "text",
+ printSummary: false,
+ suppressOutput: true,
+ mapMode: true,
+ });
+ } catch (err: any) {
+ console.error(chalk.red("Error:"), err.message ?? err);
+ process.exitCode = 1;
+ return;
+ }
const ingestMs = Math.round(performance.now() - ingestStart);
const client = new IxClient(getEndpoint());
diff --git a/ix-cli/src/cli/commands/trace.ts b/ix-cli/src/cli/commands/trace.ts
index f49511c6..6ff8427a 100644
--- a/ix-cli/src/cli/commands/trace.ts
+++ b/ix-cli/src/cli/commands/trace.ts
@@ -1,491 +1,525 @@
-import type { Command } from "commander";
-import chalk from "chalk";
-import { IxClient } from "../../client/api.js";
-import { getEndpoint } from "../config.js";
-import { resolveFileOrEntity, isRawId } from "../resolve.js";
-import { stderr } from "../stderr.js";
-import { buildDependencyTree } from "./depends.js";
-import { renderSection, renderKeyValue, renderResolvedHeader, colorizeKind } from "../ui.js";
-import { compactTreeNode, relativePath } from "../format.js";
-
-// ── Types ────────────────────────────────────────────────────────────
-
-interface TraceNode {
- id: string;
- name: string;
- kind: string;
- resolved: boolean;
- path?: string;
- children: TraceNode[];
- cycle?: boolean;
-}
-
-interface PathNode {
- id: string;
- name: string;
- kind: string;
-}
-
-// ── Constants ────────────────────────────────────────────────────────
-
-const DEFAULT_MAX_DEPTH = Infinity;
-const MAX_NODES = Infinity;
-
-// ── Helpers ──────────────────────────────────────────────────────────
-
-const ALL_PREDICATES = ["CALLS", "IMPORTS", "REFERENCES", "EXTENDS", "IMPLEMENTS"];
-
-/** Map --kind flag to API predicates. If no kind specified, use all predicates (same as depends). */
-function kindToPredicates(kind?: string): string[] {
- if (!kind) return ALL_PREDICATES;
- switch (kind.toLowerCase()) {
- case "calls": return ["CALLS", "REFERENCES"];
- case "imports": return ["IMPORTS"];
- case "depends": return ["IMPORTS"];
- case "contains": return ["CONTAINS"];
- default: return ALL_PREDICATES;
- }
-}
-
-// ── Tree traversal ───────────────────────────────────────────────────
-
-async function buildTraceTree(
- client: IxClient,
- rootId: string,
- opts: {
- direction: "in" | "out";
- predicates: string[];
- maxDepth: number;
- maxNodes: number;
- },
-): Promise<{ tree: TraceNode[]; truncated: boolean; nodesVisited: number; maxDepthReached: number }> {
- const { direction, predicates, maxDepth, maxNodes } = opts;
- const visited = new Set<string>([rootId]);
- let nodesVisited = 0;
- let truncated = false;
- let maxDepthReached = 0;
-
- async function expand(nodeId: string, depth: number): Promise<TraceNode[]> {
- if (depth > maxDepth) { truncated = true; return []; }
- if (nodesVisited >= maxNodes) { truncated = true; return []; }
- maxDepthReached = Math.max(maxDepthReached, depth);
-
- // One call per predicate (matches depends.ts behaviour — order and dedup are stable)
- const results = await Promise.all(
- predicates.map((p) => client.expand(nodeId, { direction, predicates: [p], hops: 1 })),
- );
-
- const children: TraceNode[] = [];
- const levelSeen = new Set<string>();
-
- for (const result of results) {
- for (const n of result.nodes) {
- if (nodesVisited >= maxNodes) { truncated = true; break; }
- if (levelSeen.has(n.id)) continue;
-
- const name = n.name || n.attrs?.name || "";
- const resolved = !!name && !isRawId(name);
- const isCycle = visited.has(n.id);
-
- nodesVisited++;
- levelSeen.add(n.id);
- visited.add(n.id);
-
- const child: TraceNode = {
- id: n.id,
- name: resolved ? name : n.id.slice(0, 8),
- kind: n.kind ?? "unknown",
- resolved,
- path: n.provenance?.sourceUri ?? n.provenance?.source_uri ?? undefined,
- children: [],
- ...(isCycle ? { cycle: true } : {}),
- };
-
- if (!isCycle && resolved) {
- child.children = await expand(n.id, depth + 1);
- }
-
- children.push(child);
- }
- }
-
- return children;
- }
-
- const tree = await expand(rootId, 1);
- return { tree, truncated, nodesVisited, maxDepthReached };
-}
-
-// ── Path search (BFS) ────────────────────────────────────────────────
-
-export async function findPath(
- client: IxClient,
- fromId: string,
- toId: string,
- predicates: string[],
- maxDepth: number = 10,
-): Promise<PathNode[] | null> {
- const nodeMap = new Map<string, { name: string; kind: string }>();
-
- const queue: Array<{ id: string; path: string[] }> = [{ id: fromId, path: [fromId] }];
- const visited = new Set<string>([fromId]);
-
- while (queue.length > 0) {
- const entry = queue.shift()!;
- const { id, path } = entry;
- if (path.length >= maxDepth) continue;
-
- const [outResult, inResult] = await Promise.all([
- client.expand(id, { direction: "out", predicates, hops: 1 }),
- client.expand(id, { direction: "in", predicates, hops: 1 }),
- ]);
-
- for (const n of [...outResult.nodes, ...inResult.nodes]) {
- const name = n.name || n.attrs?.name || n.id.slice(0, 8);
- if (!nodeMap.has(n.id)) {
- nodeMap.set(n.id, { name, kind: n.kind ?? "unknown" });
- }
-
- if (n.id === toId) {
- const fullPath = [...path, n.id];
- return fullPath.map((nodeId) => {
- if (nodeId === fromId) return { id: nodeId, name: "", kind: "" }; // filled below
- const meta = nodeMap.get(nodeId) ?? { name: nodeId.slice(0, 8), kind: "unknown" };
- return { id: nodeId, ...meta };
- });
- }
-
- if (visited.has(n.id)) continue;
- visited.add(n.id);
- queue.push({ id: n.id, path: [...path, n.id] });
- }
- }
-
- return null;
-}
-
-// ── Text rendering ───────────────────────────────────────────────────
-
-function renderTraceTree(children: Array<{ name: string; kind?: string; children: any[]; cycle?: boolean }>, isLast: boolean[]): string[] {
- const lines: string[] = [];
-
- for (let i = 0; i < children.length; i++) {
- const child = children[i];
- const last = i === children.length - 1;
- const connector = last ? "└─ " : "├─ ";
-
- let indent = "";
- for (const il of isLast) {
- indent += il ? " " : "│ ";
- }
-
- const kindStr = child.cycle
- ? chalk.dim((child.kind ?? "").padEnd(10))
- : colorizeKind(child.kind ?? "");
- const nameStr = child.cycle
- ? chalk.dim(child.name) + chalk.yellow(" ↺")
- : child.name;
-
- lines.push(`${indent}${connector}${kindStr} ${nameStr}`);
-
- if (child.children.length > 0) {
- lines.push(...renderTraceTree(child.children, [...isLast, last]));
- }
- }
-
- return lines;
-}
-
-
-const cap = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
-
-
-// ── Command ──────────────────────────────────────────────────────────
-
-export function registerTraceCommand(program: Command): void {
- program
- .command("trace <symbol>")
- .description("Follow how it connects")
- .option("--to <target>", "Find path to target symbol")
- .option("--upstream", "Show who calls/imports this (same as depends)")
- .option("--downstream", "Show what this calls/imports (outward flow)")
- .option("--kind <kind>", "Relationship kind: calls|imports|depends|contains")
- .option("--depth <n>", "Cap traversal depth")
- .option("--cap <n>", "Cap number of nodes visited, per direction")
- .option("--pick <n>", "Pick Nth candidate from ambiguous results (1-based)")
- .option("--path <path>", "Prefer symbols from files matching this path substring")
- .option("--format <fmt>", "Output format (text|json)", "text")
- .option("--include-tests", "Include test and fixture entities")
- .option("--tests-only", "Show only test and fixture entities")
- .addHelpText(
- "after",
- `\nExamples:
- ix trace IxClient
- ix trace IxClient --downstream
- ix trace resolve --upstream
- ix trace registerImpactCommand --to IxClient
- ix trace api.ts --kind imports
- ix trace IxClient --depth 3 --format json`,
- )
- .action(
- async (
- symbol: string,
- opts: {
- to?: string;
- upstream?: boolean;
- downstream?: boolean;
- kind?: string;
- depth?: string;
- cap?: string;
- pick?: string;
- path?: string;
- format: string;
- includeTests?: boolean;
- testsOnly?: boolean;
- },
- ) => {
- const client = new IxClient(getEndpoint());
-
- // Validate --pick
- if (opts.pick !== undefined) {
- const pickVal = parseInt(opts.pick, 10);
- if (isNaN(pickVal) || pickVal < 1) {
- stderr("Invalid value for --pick: must be a positive integer.");
- return;
- }
- }
-
- const resolveOpts = {
- path: opts.path,
- pick: opts.pick ? parseInt(opts.pick, 10) : undefined,
- includeTests: opts.includeTests,
- testsOnly: opts.testsOnly,
- };
-
- const maxDepth = opts.depth ? parseInt(opts.depth, 10) : DEFAULT_MAX_DEPTH;
- const maxNodes = opts.cap ? parseInt(opts.cap, 10) : MAX_NODES;
-
- // ── Path mode (--to) ────────────────────────────────────────
- if (opts.to) {
- const toResolveOpts = {
- ...resolveOpts,
- path: undefined,
- };
- const [fromTarget, toTarget] = await Promise.all([
- resolveFileOrEntity(client, symbol, resolveOpts),
- resolveFileOrEntity(client, opts.to, toResolveOpts),
- ]);
- if (!fromTarget || !toTarget) return;
-
- const relKind = opts.kind ?? "mixed";
- const predicates = kindToPredicates(opts.kind);
-
- const rawPath = await findPath(client, fromTarget.id, toTarget.id, predicates, maxDepth + 7);
-
- // Fill in the from-node name (was left blank above)
- const pathNodes: PathNode[] = rawPath
- ? rawPath.map((n, i) =>
- i === 0 ? { id: n.id, name: fromTarget.name, kind: fromTarget.kind } : n,
- )
- : [];
-
- // ── JSON output ────────────────────────────────────────
- if (opts.format === "json") {
- const output: Record<string, unknown> = {
- mode: "path",
- from: { name: fromTarget.name, kind: fromTarget.kind, path: relativePath(fromTarget.path) },
- to: { name: toTarget.name, kind: toTarget.kind, path: relativePath(toTarget.path) },
- kind: relKind,
- };
- if (pathNodes.length > 0) {
- const mapped = pathNodes.map((n) => ({ name: n.name, kind: n.kind }));
- output.path = mapped;
- output.summary = { path_length: pathNodes.length };
- } else {
- output.path = null;
- output.diagnostics = [
- {
- code: "no_path",
- message: `No route found from ${fromTarget.name} to ${toTarget.name}.`,
- },
- ];
- }
- console.log(JSON.stringify(output, null, 2));
- return;
- }
-
- // ── Text output ────────────────────────────────────────
- renderResolvedHeader(fromTarget.kind, fromTarget.name);
- renderSection("Trace");
- renderKeyValue("From", fromTarget.name);
- renderKeyValue("To", toTarget.name);
- renderKeyValue("Kind", cap(relKind));
-
- if (pathNodes.length === 0) {
- console.log(`\nNo route found from ${chalk.bold(fromTarget.name)} to ${chalk.bold(toTarget.name)}.`);
- return;
- }
-
- renderSection("Route");
- for (let i = 0; i < pathNodes.length; i++) {
- const n = pathNodes[i];
- const kindStr = colorizeKind(n.kind ?? "");
- if (i === 0) {
- console.log(` ${kindStr} ${n.name}`);
- } else {
- console.log(` ${chalk.dim("→")} ${kindStr} ${n.name}`);
- }
- }
-
- renderSection("Summary");
- renderKeyValue("Path length", String(pathNodes.length));
- return;
- }
-
- // ── Directional mode ────────────────────────────────────────
- const target = await resolveFileOrEntity(client, symbol, resolveOpts);
- if (!target) return;
-
- const predicates = kindToPredicates(opts.kind);
- const relKind = opts.kind ?? "mixed";
-
- // Determine direction(s)
- const doUpstream = opts.upstream === true;
- const doDownstream = opts.downstream === true;
- const doBoth = !doUpstream && !doDownstream;
- const direction = doBoth ? "both" : doUpstream ? "upstream" : "downstream";
-
- // upstream = "in" (who depends on this — same as depends)
- // downstream = "out" (what this depends on — inverse of depends)
-
- // ── Both: run up + down in parallel ────────────────────────
- if (doBoth) {
- const [upResult, downResult] = await Promise.all([
- buildDependencyTree(client, target.id, { maxDepth, maxNodes, predicates }),
- buildTraceTree(client, target.id, {
- direction: "out",
- predicates,
- maxDepth,
- maxNodes,
- }),
- ]);
-
- // ── JSON ──────────────────────────────────────────────
- if (opts.format === "json") {
- console.log(
- JSON.stringify(
- {
- mode: "directional",
- target: { name: target.name, kind: target.kind, path: relativePath(target.path) },
- direction: "both",
- kind: relKind,
- depth: maxDepth,
- upstream: {
- tree: upResult.tree.map(compactTreeNode),
- summary: { nodes_visited: upResult.nodesVisited, max_depth: upResult.maxDepthReached },
- },
- downstream: {
- tree: downResult.tree.map(compactTreeNode),
- summary: { nodes_visited: downResult.nodesVisited, max_depth: downResult.maxDepthReached },
- },
- },
- null,
- 2,
- ),
- );
- return;
- }
-
- // ── Text ──────────────────────────────────────────────
- renderResolvedHeader(target.kind, target.name);
- renderSection("Trace");
- renderKeyValue("Direction", "Both");
- renderKeyValue("Kind", cap(relKind));
-
- // Upstream section (in edges — who depends on this)
- renderSection("Upstream");
- console.log(` ${target.name}`);
- if (upResult.tree.length === 0) {
- console.log(chalk.dim(" (none)"));
- } else {
- for (const line of renderTraceTree(upResult.tree, [])) {
- console.log(` ${line}`);
- }
- }
-
- // Downstream section (out edges — what this depends on)
- renderSection("Downstream");
- console.log(` ${target.name}`);
- if (downResult.tree.length === 0) {
- console.log(chalk.dim(" (none)"));
- } else {
- for (const line of renderTraceTree(downResult.tree, [])) {
- console.log(` ${line}`);
- }
- }
-
- const totalNodes = upResult.nodesVisited + downResult.nodesVisited;
- const maxD = Math.max(upResult.maxDepthReached, downResult.maxDepthReached);
- renderSection("Summary");
- renderKeyValue("Nodes visited", String(totalNodes));
- renderKeyValue("Max depth", String(maxD));
- return;
- }
-
- // ── Single direction ────────────────────────────────────────
- const { tree, truncated, nodesVisited, maxDepthReached } = doUpstream
- ? await buildDependencyTree(client, target.id, { maxDepth, maxNodes, predicates })
- : await buildTraceTree(client, target.id, { direction: "out", predicates, maxDepth, maxNodes });
-
- // ── JSON ────────────────────────────────────────────────────
- if (opts.format === "json") {
- const output: Record<string, unknown> = {
- mode: "directional",
- target: { name: target.name, kind: target.kind, path: relativePath(target.path) },
- direction,
- kind: relKind,
- depth: maxDepth,
- tree: tree.map(compactTreeNode),
- summary: { nodes_visited: nodesVisited, max_depth: maxDepthReached },
- };
-
- if (tree.length === 0) {
- output.diagnostics = [
- { code: "no_edges", message: `No ${direction} ${relKind} found for ${target.name}.` },
- ];
- }
- if (truncated) {
- const diags = (output.diagnostics as unknown[]) ?? [];
- (diags as unknown[]).push({ code: "truncated", message: `Traversal truncated (depth: ${maxDepth}, node cap: ${maxNodes}).` });
- output.diagnostics = diags;
- }
-
- console.log(JSON.stringify(output, null, 2));
- return;
- }
-
- // ── Text ────────────────────────────────────────────────────
- renderResolvedHeader(target.kind, target.name);
- renderSection("Trace");
- renderKeyValue("Direction", cap(direction));
- renderKeyValue("Kind", cap(relKind));
-
- if (tree.length === 0) {
- const relDir = doUpstream ? "upstream" : "downstream";
- console.log(`\nNo ${relDir} ${relKind} found for ${chalk.bold(target.name)}.`);
- return;
- }
-
- renderSection("Path");
- console.log(` ${target.name}`);
- for (const line of renderTraceTree(tree, [])) {
- console.log(` ${line}`);
- }
-
-
- renderSection("Summary");
- renderKeyValue("Nodes visited", String(nodesVisited));
- renderKeyValue("Max depth", String(maxDepthReached));
- },
- );
-}
+import type { Command } from "commander";
+import chalk from "chalk";
+import { IxClient } from "../../client/api.js";
+import { getEndpoint } from "../config.js";
+import { resolveFileOrEntity, isRawId } from "../resolve.js";
+import { stderr } from "../stderr.js";
+import { buildDependencyTree } from "./depends.js";
+import { renderSection, renderKeyValue, renderResolvedHeader, colorizeKind } from "../ui.js";
+import { compactTreeNode, relativePath } from "../format.js";
+
+// ── Types ────────────────────────────────────────────────────────────
+
+interface TraceNode {
+ id: string;
+ name: string;
+ kind: string;
+ resolved: boolean;
+ path?: string;
+ children: TraceNode[];
+ cycle?: boolean;
+}
+
+interface PathNode {
+ id: string;
+ name: string;
+ kind: string;
+}
+
+// ── Constants ────────────────────────────────────────────────────────
+
+const DEFAULT_MAX_DEPTH = Infinity;
+const MAX_NODES = Infinity;
+
+// ── Helpers ──────────────────────────────────────────────────────────
+
+const ALL_PREDICATES = ["CALLS", "IMPORTS", "REFERENCES", "EXTENDS", "IMPLEMENTS"];
+
+/** Map --kind flag to API predicates. If no kind specified, use all predicates (same as depends). */
+function kindToPredicates(kind?: string): string[] {
+ if (!kind) return ALL_PREDICATES;
+ switch (kind.toLowerCase()) {
+ case "calls": return ["CALLS", "REFERENCES"];
+ case "imports": return ["IMPORTS"];
+ case "depends": return ["IMPORTS"];
+ case "contains": return ["CONTAINS"];
+ default: return ALL_PREDICATES;
+ }
+}
+
+// ── Tree traversal ───────────────────────────────────────────────────
+
+async function buildTraceTree(
+ client: IxClient,
+ rootId: string,
+ opts: {
+ direction: "in" | "out";
+ predicates: string[];
+ maxDepth: number;
+ maxNodes: number;
+ },
+): Promise<{ tree: TraceNode[]; truncated: boolean; nodesVisited: number; maxDepthReached: number }> {
+ const { direction, predicates, maxDepth, maxNodes } = opts;
+ const visited = new Set<string>([rootId]);
+ let nodesVisited = 0;
+ let truncated = false;
+ let maxDepthReached = 0;
+
+ async function expand(nodeId: string, depth: number): Promise<TraceNode[]> {
+ if (depth > maxDepth) { truncated = true; return []; }
+ if (nodesVisited >= maxNodes) { truncated = true; return []; }
+ maxDepthReached = Math.max(maxDepthReached, depth);
+
+ // One call per predicate (matches depends.ts behaviour — order and dedup are stable)
+ const results = await Promise.all(
+ predicates.map((p) => client.expand(nodeId, { direction, predicates: [p], hops: 1 })),
+ );
+
+ const children: TraceNode[] = [];
+ const levelSeen = new Set<string>();
+
+ for (const result of results) {
+ for (const n of result.nodes) {
+ if (nodesVisited >= maxNodes) { truncated = true; break; }
+ if (levelSeen.has(n.id)) continue;
+
+ const name = n.name || n.attrs?.name || "";
+ const resolved = !!name && !isRawId(name);
+ const isCycle = visited.has(n.id);
+
+ nodesVisited++;
+ levelSeen.add(n.id);
+ visited.add(n.id);
+
+ const child: TraceNode = {
+ id: n.id,
+ name: resolved ? name : n.id.slice(0, 8),
+ kind: n.kind ?? "unknown",
+ resolved,
+ path: n.provenance?.sourceUri ?? n.provenance?.source_uri ?? undefined,
+ children: [],
+ ...(isCycle ? { cycle: true } : {}),
+ };
+
+ if (!isCycle && resolved) {
+ child.children = await expand(n.id, depth + 1);
+ }
+
+ children.push(child);
+ }
+ }
+
+ return children;
+ }
+
+ const tree = await expand(rootId, 1);
+ return { tree, truncated, nodesVisited, maxDepthReached };
+}
+
+// ── Path search (BFS) ────────────────────────────────────────────────
+
+export async function findPath(
+ client: IxClient,
+ fromId: string,
+ toId: string,
+ predicates: string[],
+ maxDepth: number = 10,
+): Promise<PathNode[] | null> {
+ const nodeMap = new Map<string, { name: string; kind: string }>();
+
+ const queue: Array<{ id: string; path: string[] }> = [{ id: fromId, path: [fromId] }];
+ const visited = new Set<string>([fromId]);
+
+ while (queue.length > 0) {
+ const entry = queue.shift()!;
+ const { id, path } = entry;
+ if (path.length >= maxDepth) continue;
+
+ const [outResult, inResult] = await Promise.all([
+ client.expand(id, { direction: "out", predicates, hops: 1 }),
+ client.expand(id, { direction: "in", predicates, hops: 1 }),
+ ]);
+
+ for (const n of [...outResult.nodes, ...inResult.nodes]) {
+ const name = n.name || n.attrs?.name || n.id.slice(0, 8);
+ if (!nodeMap.has(n.id)) {
+ nodeMap.set(n.id, { name, kind: n.kind ?? "unknown" });
+ }
+
+ if (n.id === toId) {
+ const fullPath = [...path, n.id];
+ return fullPath.map((nodeId) => {
+ if (nodeId === fromId) return { id: nodeId, name: "", kind: "" }; // filled below
+ const meta = nodeMap.get(nodeId) ?? { name: nodeId.slice(0, 8), kind: "unknown" };
+ return { id: nodeId, ...meta };
+ });
+ }
+
+ if (visited.has(n.id)) continue;
+ visited.add(n.id);
+ queue.push({ id: n.id, path: [...path, n.id] });
+ }
+ }
+
+ return null;
+}
+
+// ── Text rendering ───────────────────────────────────────────────────
+
+/** Return last 3 path components from a file URI, for compact display. */
+function shortPath(uri: string | undefined): string {
+ if (!uri) return "";
+ const p = uri.replace(/^file:\/\//, "").replace(/\\/g, "/");
+ const parts = p.split("/").filter(Boolean);
+ return parts.slice(-3).join("/");
+}
+
+/** Walk a tree and count how many times each node name appears. */
+function collectAllNames(children: Array<{ name: string; children: any[] }>, out: Map<string, number>): void {
+ for (const c of children) {
+ out.set(c.name, (out.get(c.name) ?? 0) + 1);
+ collectAllNames(c.children, out);
+ }
+}
+
+function renderTraceTree(
+ children: Array<{ name: string; kind?: string; path?: string; children: any[]; cycle?: boolean }>,
+ isLast: boolean[],
+ disambiguate?: Set<string>,
+): string[] {
+ const lines: string[] = [];
+
+ for (let i = 0; i < children.length; i++) {
+ const child = children[i];
+ const last = i === children.length - 1;
+ const connector = last ? "└─ " : "├─ ";
+
+ let indent = "";
+ for (const il of isLast) {
+ indent += il ? " " : "│ ";
+ }
+
+ const kindStr = child.cycle
+ ? chalk.dim((child.kind ?? "").padEnd(10))
+ : colorizeKind(child.kind ?? "");
+
+ let nameStr = child.cycle
+ ? chalk.dim(child.name) + chalk.yellow(" ↺")
+ : child.name;
+
+ if (!child.cycle && disambiguate?.has(child.name) && child.path) {
+ nameStr += chalk.dim(` (${shortPath(child.path)})`);
+ }
+
+ lines.push(`${indent}${connector}${kindStr} ${nameStr}`);
+
+ if (child.children.length > 0) {
+ lines.push(...renderTraceTree(child.children, [...isLast, last], disambiguate));
+ }
+ }
+
+ return lines;
+}
+
+
+const cap = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
+
+
+// ── Command ──────────────────────────────────────────────────────────
+
+export function registerTraceCommand(program: Command): void {
+ program
+ .command("trace <symbol>")
+ .description("Follow how it connects")
+ .option("--to <target>", "Find path to target symbol")
+ .option("--upstream", "Show who calls/imports this (same as depends)")
+ .option("--downstream", "Show what this calls/imports (outward flow)")
+ .option("--kind <kind>", "Relationship kind: calls|imports|depends|contains")
+ .option("--depth <n>", "Cap traversal depth")
+ .option("--cap <n>", "Cap number of nodes visited, per direction")
+ .option("--pick <n>", "Pick Nth candidate from ambiguous results (1-based)")
+ .option("--path <path>", "Prefer symbols from files matching this path substring")
+ .option("--format <fmt>", "Output format (text|json)", "text")
+ .option("--include-tests", "Include test and fixture entities")
+ .option("--tests-only", "Show only test and fixture entities")
+ .addHelpText(
+ "after",
+ `\nExamples:
+ ix trace IxClient
+ ix trace IxClient --downstream
+ ix trace resolve --upstream
+ ix trace registerImpactCommand --to IxClient
+ ix trace api.ts --kind imports
+ ix trace IxClient --depth 3 --format json`,
+ )
+ .action(
+ async (
+ symbol: string,
+ opts: {
+ to?: string;
+ upstream?: boolean;
+ downstream?: boolean;
+ kind?: string;
+ depth?: string;
+ cap?: string;
+ pick?: string;
+ path?: string;
+ format: string;
+ includeTests?: boolean;
+ testsOnly?: boolean;
+ },
+ ) => {
+ const client = new IxClient(getEndpoint());
+
+ // Validate --pick
+ if (opts.pick !== undefined) {
+ const pickVal = parseInt(opts.pick, 10);
+ if (isNaN(pickVal) || pickVal < 1) {
+ stderr("Invalid value for --pick: must be a positive integer.");
+ return;
+ }
+ }
+
+ const resolveOpts = {
+ path: opts.path,
+ pick: opts.pick ? parseInt(opts.pick, 10) : undefined,
+ includeTests: opts.includeTests,
+ testsOnly: opts.testsOnly,
+ };
+
+ const maxDepth = opts.depth ? parseInt(opts.depth, 10) : DEFAULT_MAX_DEPTH;
+ const maxNodes = opts.cap ? parseInt(opts.cap, 10) : MAX_NODES;
+
+ // ── Path mode (--to) ────────────────────────────────────────
+ if (opts.to) {
+ const toResolveOpts = {
+ ...resolveOpts,
+ path: undefined,
+ };
+ const [fromTarget, toTarget] = await Promise.all([
+ resolveFileOrEntity(client, symbol, resolveOpts),
+ resolveFileOrEntity(client, opts.to, toResolveOpts),
+ ]);
+ if (!fromTarget || !toTarget) return;
+
+ const relKind = opts.kind ?? "mixed";
+ const predicates = kindToPredicates(opts.kind);
+
+ const rawPath = await findPath(client, fromTarget.id, toTarget.id, predicates, maxDepth + 7);
+
+ // Fill in the from-node name (was left blank above)
+ const pathNodes: PathNode[] = rawPath
+ ? rawPath.map((n, i) =>
+ i === 0 ? { id: n.id, name: fromTarget.name, kind: fromTarget.kind } : n,
+ )
+ : [];
+
+ // ── JSON output ────────────────────────────────────────
+ if (opts.format === "json") {
+ const output: Record<string, unknown> = {
+ mode: "path",
+ from: { name: fromTarget.name, kind: fromTarget.kind, path: relativePath(fromTarget.path) },
+ to: { name: toTarget.name, kind: toTarget.kind, path: relativePath(toTarget.path) },
+ kind: relKind,
+ };
+ if (pathNodes.length > 0) {
+ const mapped = pathNodes.map((n) => ({ name: n.name, kind: n.kind }));
+ output.path = mapped;
+ output.summary = { path_length: pathNodes.length };
+ } else {
+ output.path = null;
+ output.diagnostics = [
+ {
+ code: "no_path",
+ message: `No route found from ${fromTarget.name} to ${toTarget.name}.`,
+ },
+ ];
+ }
+ console.log(JSON.stringify(output, null, 2));
+ return;
+ }
+
+ // ── Text output ────────────────────────────────────────
+ renderResolvedHeader(fromTarget.kind, fromTarget.name);
+ renderSection("Trace");
+ renderKeyValue("From", fromTarget.name);
+ renderKeyValue("To", toTarget.name);
+ renderKeyValue("Kind", cap(relKind));
+
+ if (pathNodes.length === 0) {
+ console.log(`\nNo route found from ${chalk.bold(fromTarget.name)} to ${chalk.bold(toTarget.name)}.`);
+ return;
+ }
+
+ renderSection("Route");
+ for (let i = 0; i < pathNodes.length; i++) {
+ const n = pathNodes[i];
+ const kindStr = colorizeKind(n.kind ?? "");
+ if (i === 0) {
+ console.log(` ${kindStr} ${n.name}`);
+ } else {
+ console.log(` ${chalk.dim("→")} ${kindStr} ${n.name}`);
+ }
+ }
+
+ renderSection("Summary");
+ renderKeyValue("Path length", String(pathNodes.length));
+ return;
+ }
+
+ // ── Directional mode ────────────────────────────────────────
+ const target = await resolveFileOrEntity(client, symbol, resolveOpts);
+ if (!target) return;
+
+ const predicates = kindToPredicates(opts.kind);
+ const relKind = opts.kind ?? "mixed";
+
+ // Determine direction(s)
+ const doUpstream = opts.upstream === true;
+ const doDownstream = opts.downstream === true;
+ const doBoth = !doUpstream && !doDownstream;
+ const direction = doBoth ? "both" : doUpstream ? "upstream" : "downstream";
+
+ // upstream = "in" (who depends on this — same as depends)
+ // downstream = "out" (what this depends on — inverse of depends)
+
+ // ── Both: run up + down in parallel ────────────────────────
+ if (doBoth) {
+ const [upResult, downResult] = await Promise.all([
+ buildDependencyTree(client, target.id, { maxDepth, maxNodes, predicates }),
+ buildTraceTree(client, target.id, {
+ direction: "out",
+ predicates,
+ maxDepth,
+ maxNodes,
+ }),
+ ]);
+
+ // ── JSON ──────────────────────────────────────────────
+ if (opts.format === "json") {
+ console.log(
+ JSON.stringify(
+ {
+ mode: "directional",
+ target: { name: target.name, kind: target.kind, path: relativePath(target.path) },
+ direction: "both",
+ kind: relKind,
+ depth: maxDepth,
+ upstream: {
+ tree: upResult.tree.map(compactTreeNode),
+ summary: { nodes_visited: upResult.nodesVisited, max_depth: upResult.maxDepthReached },
+ },
+ downstream: {
+ tree: downResult.tree.map(compactTreeNode),
+ summary: { nodes_visited: downResult.nodesVisited, max_depth: downResult.maxDepthReached },
+ },
+ },
+ null,
+ 2,
+ ),
+ );
+ return;
+ }
+
+ // ── Text ──────────────────────────────────────────────
+ renderResolvedHeader(target.kind, target.name);
+ renderSection("Trace");
+ renderKeyValue("Direction", "Both");
+ renderKeyValue("Kind", cap(relKind));
+
+ // Upstream section (in edges — who depends on this)
+ renderSection("Upstream");
+ console.log(` ${target.name}`);
+ const upNameCounts = new Map<string, number>();
+ collectAllNames(upResult.tree, upNameCounts);
+ const upDisambiguate = new Set([...upNameCounts.entries()].filter(([, n]) => n > 1).map(([k]) => k));
+ if (upResult.tree.length === 0) {
+ console.log(chalk.dim(" (none)"));
+ } else {
+ for (const line of renderTraceTree(upResult.tree, [], upDisambiguate)) {
+ console.log(` ${line}`);
+ }
+ }
+
+ // Downstream section (out edges — what this depends on)
+ renderSection("Downstream");
+ console.log(` ${target.name}`);
+ const downNameCounts = new Map<string, number>();
+ collectAllNames(downResult.tree, downNameCounts);
+ const downDisambiguate = new Set([...downNameCounts.entries()].filter(([, n]) => n > 1).map(([k]) => k));
+ if (downResult.tree.length === 0) {
+ console.log(chalk.dim(" (none)"));
+ } else {
+ for (const line of renderTraceTree(downResult.tree, [], downDisambiguate)) {
+ console.log(` ${line}`);
+ }
+ }
+
+ const totalNodes = upResult.nodesVisited + downResult.nodesVisited;
+ const maxD = Math.max(upResult.maxDepthReached, downResult.maxDepthReached);
+ renderSection("Summary");
+ renderKeyValue("Nodes visited", String(totalNodes));
+ renderKeyValue("Max depth", String(maxD));
+ return;
+ }
+
+ // ── Single direction ────────────────────────────────────────
+ const { tree, truncated, nodesVisited, maxDepthReached } = doUpstream
+ ? await buildDependencyTree(client, target.id, { maxDepth, maxNodes, predicates })
+ : await buildTraceTree(client, target.id, { direction: "out", predicates, maxDepth, maxNodes });
+
+ // ── JSON ────────────────────────────────────────────────────
+ if (opts.format === "json") {
+ const output: Record<string, unknown> = {
+ mode: "directional",
+ target: { name: target.name, kind: target.kind, path: relativePath(target.path) },
+ direction,
+ kind: relKind,
+ depth: maxDepth,
+ tree: tree.map(compactTreeNode),
+ summary: { nodes_visited: nodesVisited, max_depth: maxDepthReached },
+ };
+
+ if (tree.length === 0) {
+ output.diagnostics = [
+ { code: "no_edges", message: `No ${direction} ${relKind} found for ${target.name}.` },
+ ];
+ }
+ if (truncated) {
+ const diags = (output.diagnostics as unknown[]) ?? [];
+ (diags as unknown[]).push({ code: "truncated", message: `Traversal truncated (depth: ${maxDepth}, node cap: ${maxNodes}).` });
+ output.diagnostics = diags;
+ }
+
+ console.log(JSON.stringify(output, null, 2));
+ return;
+ }
+
+ // ── Text ────────────────────────────────────────────────────
+ renderResolvedHeader(target.kind, target.name);
+ renderSection("Trace");
+ renderKeyValue("Direction", cap(direction));
+ renderKeyValue("Kind", cap(relKind));
+
+ if (tree.length === 0) {
+ const relDir = doUpstream ? "upstream" : "downstream";
+ console.log(`\nNo ${relDir} ${relKind} found for ${chalk.bold(target.name)}.`);
+ return;
+ }
+
+ renderSection("Path");
+ console.log(` ${target.name}`);
+ const singleNameCounts = new Map<string, number>();
+ collectAllNames(tree, singleNameCounts);
+ const singleDisambiguate = new Set([...singleNameCounts.entries()].filter(([, n]) => n > 1).map(([k]) => k));
+ for (const line of renderTraceTree(tree, [], singleDisambiguate)) {
+ console.log(` ${line}`);
+ }
+
+
+ renderSection("Summary");
+ renderKeyValue("Nodes visited", String(nodesVisited));
+ renderKeyValue("Max depth", String(maxDepthReached));
+ },
+ );
+}
diff --git a/ix-cli/src/cli/commands/watch.ts b/ix-cli/src/cli/commands/watch.ts
index 562189ed..ed96733b 100644
--- a/ix-cli/src/cli/commands/watch.ts
+++ b/ix-cli/src/cli/commands/watch.ts
@@ -13,6 +13,7 @@ const SUPPORTED_EXTENSIONS = new Set([
".scala", ".sc", ".java",
".py", ".rb", ".go", ".rs", ".kt", ".kts", ".cs", ".php", ".swift",
".c", ".h", ".cpp", ".cc", ".cxx", ".hpp",
+ ".yaml", ".yml",
]);
const SUPPORTED_NAMES = new Set([
diff --git a/ix-cli/test/parser.stress.mjs b/ix-cli/test/parser.stress.mjs
index 451970e4..824f8f0c 100644
--- a/ix-cli/test/parser.stress.mjs
+++ b/ix-cli/test/parser.stress.mjs
@@ -64,6 +64,7 @@ const extExpectations = [
['.php', 'php'],
['.kt', 'kotlin'], ['.kts', 'kotlin'],
['.swift', 'swift'],
+ ['.dockerfile', 'dockerfile'],
];
for (const [ext, expected] of extExpectations) {
@@ -73,6 +74,7 @@ for (const [ext, expected] of extExpectations) {
assert(languageFromPath('file.txt') === null, 'languageFromPath(.txt) === null');
assert(languageFromPath('Makefile') === null, 'languageFromPath(no-ext) === null');
+assert(languageFromPath('Dockerfile') === 'dockerfile', 'languageFromPath(Dockerfile) === dockerfile');
// ---------------------------------------------------------------------------
// § 2 Core entity + relationship extraction (TypeScript)
diff --git a/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala b/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala
index 9cde2da4..53441ca8 100644
--- a/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala
+++ b/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala
@@ -116,8 +116,9 @@ class MapGraphBuilder(client: ArangoClient) {
| FILTER n.kind == "file"
| AND n.deleted_rev == null
| AND n.provenance.source_uri != null
- | LET ext = LOWER(LAST(SPLIT(n.provenance.source_uri, ".")))
- | FILTER ext IN @extensions
+ | LET base = LOWER(n.name)
+ | LET ext = CONTAINS(base, ".") ? LOWER(LAST(SPLIT(base, "."))) : null
+ | FILTER ext IN @extensions OR base == "dockerfile" OR LIKE(base, "%.dockerfile")
| COLLECT WITH COUNT INTO cnt
| RETURN cnt""".stripMargin,
Map("extensions" -> SourceExtensions.toArray.asInstanceOf[AnyRef])
@@ -140,8 +141,9 @@ class MapGraphBuilder(client: ArangoClient) {
| FILTER n.kind == "file"
| AND n.deleted_rev == null
| AND n.provenance.source_uri != null
- | LET ext = LOWER(LAST(SPLIT(n.provenance.source_uri, ".")))
- | FILTER ext IN @extensions
+ | LET base = LOWER(n.name)
+ | LET ext = CONTAINS(base, ".") ? LOWER(LAST(SPLIT(base, "."))) : null
+ | FILTER ext IN @extensions OR base == "dockerfile" OR LIKE(base, "%.dockerfile")
| RETURN {id: n.logical_id, path: n.provenance.source_uri}""".stripMargin,
Map("extensions" -> SourceExtensions.toArray.asInstanceOf[AnyRef])
).map { rows =>
diff --git a/memory-layer/src/main/scala/ix/memory/map/MapService.scala b/memory-layer/src/main/scala/ix/memory/map/MapService.scala
index 183ce5fe..b0c1d5fb 100644
--- a/memory-layer/src/main/scala/ix/memory/map/MapService.scala
+++ b/memory-layer/src/main/scala/ix/memory/map/MapService.scala
@@ -270,14 +270,24 @@ class MapService(
r.copy(childRegionIds = childrenByParent.getOrElse(r.id, Set.empty))
}
val dedupedHierarchy = collapseDuplicateHierarchyLevels(withChildren)
+ val withSingletonFallbacks = appendSingletonRegionsForUncoveredFiles(
+ dedupedHierarchy,
+ graph,
+ fileVertexById,
+ crosscutScores,
+ signalAdjacency,
+ rev,
+ commonPfxLen,
+ globalDirTokenCounts
+ )
// Disambiguate duplicate labels within each level.
// For each group of same-label regions, find a path segment unique to each.
- val labelGroups = dedupedHierarchy.groupBy(r => (r.level, r.label))
+ val labelGroups = withSingletonFallbacks.groupBy(r => (r.level, r.label))
val needsDisambig = labelGroups.filter(_._2.size > 1)
- if (needsDisambig.isEmpty) dedupedHierarchy
- else dedupedHierarchy.map { r =>
+ if (needsDisambig.isEmpty) withSingletonFallbacks
+ else withSingletonFallbacks.map { r =>
needsDisambig.get((r.level, r.label)) match {
case Some(group) =>
val myPaths = r.memberFiles.flatMap(id => fileVertexById.get(id).map(_.path)).toSet
@@ -306,6 +316,38 @@ class MapService(
}
}
+ private[map] def appendSingletonRegionsForUncoveredFiles(
+ regions: Vector[Region],
+ graph: WeightedFileGraph,
+ fileVertexById: Map[NodeId, FileVertex],
+ crosscutScores: Map[NodeId, Double],
+ signalAdjacency: Map[NodeId, Vector[(NodeId, Map[String, Int])]],
+ rev: Rev,
+ commonPfxLen: Int = 0,
+ globalDirTokenCounts: Map[String, Int] = Map.empty
+ ): Vector[Region] = {
+ val coveredFileIds = regions.iterator.flatMap(_.memberFiles.iterator).toSet
+ val uncoveredFileIds = graph.vertices.iterator.map(_.id).filterNot(coveredFileIds.contains).toVector
+ if (uncoveredFileIds.isEmpty) return regions
+
+ val singletonRegions = uncoveredFileIds.map { fileId =>
+ buildOneRegion(
+ memberFiles = Set(fileId),
+ level = 1,
+ kind = "module",
+ graph = graph,
+ fileVertexById = fileVertexById,
+ crosscutScores = crosscutScores,
+ signalAdjacency = signalAdjacency,
+ rev = rev,
+ commonPfxLen = commonPfxLen,
+ globalDirTokenCounts = globalDirTokenCounts
+ )
+ }
+
+ regions ++ singletonRegions
+ }
+
private[map] def collapseDuplicateHierarchyLevels(regions: Vector[Region]): Vector[Region] = {
val keepById = regions.groupBy(_.id).view.mapValues { duplicates =>
duplicates.maxBy(region => (region.level, region.confidence, region.label.length))
diff --git a/memory-layer/src/test/scala/ix/memory/map/MapServiceScopeSpec.scala b/memory-layer/src/test/scala/ix/memory/map/MapServiceScopeSpec.scala
index 6b3848e4..5a054b31 100644
--- a/memory-layer/src/test/scala/ix/memory/map/MapServiceScopeSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/map/MapServiceScopeSpec.scala
@@ -72,6 +72,34 @@ class MapServiceScopeSpec extends AnyFlatSpec with Matchers {
collapsed.find(_.label == "Edges / Resolver").flatMap(_.parentId) shouldBe Some(sharedId)
}
+ "appendSingletonRegionsForUncoveredFiles" should "create module regions for files left out by clustering" in {
+ val fileA = fileVertex("C:/repo/src/core/service.ts")
+ val fileB = fileVertex("C:/repo/infra/Dockerfile")
+ val graph = WeightedFileGraph(
+ vertices = Vector(fileA, fileB),
+ adjMatrix = Map(fileA.id -> Map.empty, fileB.id -> Map.empty),
+ degrees = Map(fileA.id -> 0.0, fileB.id -> 0.0),
+ totalWeight = 0.0,
+ predicatePairs = Map.empty
+ )
+ val existing = regionWithMembers("Core", "module", 1, NodeId(UUID.randomUUID()), Set(fileA.id))
+
+ val completed = service.appendSingletonRegionsForUncoveredFiles(
+ regions = Vector(existing),
+ graph = graph,
+ fileVertexById = graph.vertices.map(v => v.id -> v).toMap,
+ crosscutScores = Map.empty,
+ signalAdjacency = Map.empty,
+ rev = ix.memory.model.Rev(42L)
+ )
+
+ completed.map(_.memberFiles) should contain (Set(fileB.id))
+ completed.find(_.memberFiles == Set(fileB.id)).map(_.labelKind) shouldBe Some("module")
+ }
+
+ private def fileVertex(path: String): FileVertex =
+ FileVertex(NodeId(UUID.randomUUID()), path)
+
private def region(
label: String,
labelKind: String,