Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

Adaptive map hierarchy + ingestion perf + reset fix (#119)

Tanner Kocher3996214f130e
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index e35ef430..19a77b2f 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -6,6 +6,7 @@ import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import type { Command } from 'commander';
import { ParsePool } from './parse-pool.js';
+// import { ResolveWorker } from './resolve-pool.js';
import chalk from 'chalk';
import { IxClient } from '../../client/api.js';
import type { GraphPatchPayload } from '../../client/types.js';
@@ -411,6 +412,7 @@ export async function ingestFiles(
return pool;
};
+
let progressPhase = 'Scanning';
let progressCurrent = 0;
let progressTotal = 0;
@@ -449,6 +451,7 @@ export async function ingestFiles(
// Phase timing marks
const timings = {
moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0,
+ goIndexMs: 0, // time to build Go global resolution index
parseOnlyMs: 0, // sum of just parseFile() call durations
resolveMs: 0, buildPatchMs: 0, commitMs: 0,
stripChunkOpsMs: 0, // sum of stripChunkOps() durations (mapMode only)
@@ -587,9 +590,12 @@ export async function ingestFiles(
let buildPatchFn: Function | null = null;
let globalIndex: any = undefined;
- const PARSE_STREAM_CHUNK = 1000; // streaming batch size for large repos
- const SMALL_REPO_THRESHOLD = 5_000; // parse-all-first below this; streaming above
- const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 400); // files per HTTP request to the backend
+ // Larger chunks for big repos: fewer resolve+commit cycles, better edge resolution per batch.
+ // Smaller chunks pipeline better: resolve(N) overlaps with commit(N-1) + parse(N+1).
+ // With 2000-file chunks, resolve takes 68s and dominates. With 500, resolve ~17s
+ // and overlaps well with commit (~5s) + parse (~5s).
+ const PARSE_STREAM_CHUNK = filePaths.length > 10_000 ? 500 : 500;
+ const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 1000); // files per HTTP request to the backend
const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 8); // parallel HTTP save requests
const COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflicts
const YIELD_EVERY = 100; // yield event loop every N files during parse
@@ -896,9 +902,9 @@ export async function ingestFiles(
if (nodePath.extname(filePath) === '.go') sources.set(filePath, bytes.toString('utf-8'));
}
const remainingGoFiles = filePaths.filter(fp => nodePath.extname(fp) === '.go' && !changedSet.has(fp));
- const GO_READ_BATCH = 500;
- for (let i = 0; i < remainingGoFiles.length; i += GO_READ_BATCH) {
- const batch = remainingGoFiles.slice(i, i + GO_READ_BATCH);
+ const GO_READ_CONCURRENCY = 2000;
+ for (let i = 0; i < remainingGoFiles.length; i += GO_READ_CONCURRENCY) {
+ const batch = remainingGoFiles.slice(i, i + GO_READ_CONCURRENCY);
const texts = await Promise.all(batch.map(fp => fs.promises.readFile(fp, 'utf-8').catch(() => null)));
for (let j = 0; j < batch.length; j++) {
if (texts[j] != null) sources.set(batch[j], texts[j]!);
@@ -938,13 +944,14 @@ export async function ingestFiles(
// Build global resolution index from all repo file paths before the parse loop
// so cross-batch imports resolve correctly even in streaming per-chunk mode.
- // Use async batched reads (same as Path A) to avoid blocking the event loop.
+ // Read all Go files in parallel to maximize I/O throughput.
{
+ const goIndexStart = performance.now();
const sources = new Map<string, string>();
const goFiles = filePaths.filter(fp => nodePath.extname(fp) === '.go');
- const GO_READ_BATCH = 500;
- for (let i = 0; i < goFiles.length; i += GO_READ_BATCH) {
- const batch = goFiles.slice(i, i + GO_READ_BATCH);
+ const GO_READ_CONCURRENCY = 2000;
+ for (let i = 0; i < goFiles.length; i += GO_READ_CONCURRENCY) {
+ const batch = goFiles.slice(i, i + GO_READ_CONCURRENCY);
const texts = await Promise.all(batch.map(fp => fs.promises.readFile(fp, 'utf-8').catch(() => null)));
for (let j = 0; j < batch.length; j++) {
if (texts[j] != null) sources.set(batch[j], texts[j]!);
@@ -952,51 +959,52 @@ export async function ingestFiles(
}
globalIndex = ingestion.buildGlobalResolutionIndex(filePaths, sources);
sources.clear();
+ timings.goIndexMs = Math.round(performance.now() - goIndexStart);
}
progressPhase = 'Parsing';
progressTotal = filePaths.length;
progressCurrent = 0;
- // Pipeline overlap: parse next chunk in worker pool while committing current chunk.
- // Each file is dispatched to the parse pool immediately after it is read so workers
- // start processing as soon as the first file in the chunk is ready rather than
- // waiting for all reads in the chunk to complete.
+ // Pipeline overlap: read files asynchronously and dispatch to parse pool
+ // immediately so workers start processing while more files are still being read.
+ // Commit previous chunk's results while parsing the next chunk.
let pendingFlushB: Promise<void> = Promise.resolve();
for (let i = 0; i < filePaths.length; i += PARSE_STREAM_CHUNK) {
const chunk = filePaths.slice(i, i + PARSE_STREAM_CHUNK);
type FileData = { filePath: string; source: string; hash: string; previousHash: string | undefined } | null;
- const fileData: FileData[] = [];
- const parsePromises: Promise<unknown>[] = [];
- for (const filePath of chunk) {
+ const fileData: FileData[] = new Array(chunk.length).fill(null);
+ const parsePromises: Promise<unknown>[] = new Array(chunk.length).fill(Promise.resolve(null));
+
+ // Read files asynchronously and dispatch to parse pool as each file completes.
+ // This keeps workers busy while I/O is still in flight for other files.
+ const readAndDispatch = chunk.map(async (filePath, idx) => {
try {
- const fileSize = fs.statSync(filePath).size;
- if (fileSize === 0) { filesSkipped++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }
- if (fileSize > MAX_FILE_BYTES) { tooLarge++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }
- const bytes = fs.readFileSync(filePath);
+ const st = await fs.promises.stat(filePath);
+ if (st.size === 0) { filesSkipped++; return; }
+ if (st.size > MAX_FILE_BYTES) { tooLarge++; return; }
+ const bytes = await fs.promises.readFile(filePath);
const hash = sha256(bytes);
- if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }
+ if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; return; }
const previousHash = knownHashes.get(filePath);
const sourceText = bytes.toString('utf-8');
if (isLikelyMinifiedSource(sourceText)) {
minifiedLikely++;
filesSkipped++;
if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);
- fileData.push(null);
- parsePromises.push(Promise.resolve(null));
- continue;
+ return;
}
const fd: NonNullable<FileData> = { filePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined };
- fileData.push(fd);
- parsePromises.push(ensureParsePool().parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; }));
+ fileData[idx] = fd;
+ parsePromises[idx] = ensureParsePool().parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; });
} catch (err) {
parseErrors++;
- fileData.push(null);
- parsePromises.push(Promise.resolve(null));
process.stderr.write(`\n [read error] ${filePath}: ${err}\n`);
}
- }
+ });
+ await Promise.all(readAndDispatch);
const parseResults = await Promise.all(parsePromises);
+
const batch: ParsedFile[] = [];
for (let j = 0; j < fileData.length; j++) {
const f = fileData[j];
@@ -1088,6 +1096,7 @@ export async function ingestFiles(
console.log(chalk.dim(` modules: ${timings.moduleLoadMs}ms`));
console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));
console.log(chalk.dim(` hash: ${timings.hashMs}ms`));
+ console.log(chalk.dim(` go index: ${timings.goIndexMs}ms`));
console.log(chalk.dim(` parse total: ${timings.parseMs}ms`));
console.log(chalk.dim(` parse only: ${Math.round(timings.parseOnlyMs)}ms`));
console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));
diff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.ts
index 1649d454..bc2f648b 100644
--- a/ix-cli/src/cli/commands/map.ts
+++ b/ix-cli/src/cli/commands/map.ts
@@ -191,9 +191,9 @@ Examples:
const mapMs = Math.round(performance.now() - mapStart);
if (silent) {
- const systems = result.regions.filter(r => r.level === 3).length;
- const subsystems = result.regions.filter(r => r.level === 2).length;
- const modules = result.regions.filter(r => r.level === 1).length;
+ const systems = result.regions.filter(r => r.label_kind === "system").length;
+ const subsystems = result.regions.filter(r => r.label_kind === "subsystem").length;
+ const modules = result.regions.filter(r => r.label_kind === "module").length;
process.stderr.write(
`map: ${result.file_count} files · ${systems}s/${subsystems}ss/${modules}m regions · ${mapMs}ms\n`
);
@@ -271,9 +271,9 @@ export function renderMapText(result: MapResult, cwd: string, opts: MapTextRende
const regionById = new Map(result.regions.map(r => [r.id, r]));
const CROSSCUT_THRESHOLD = 0.10;
- const systemsCount = regions.filter(r => r.level === 3).length;
- const subsystemsCount = regions.filter(r => r.level === 2).length;
- const modulesCount = regions.filter(r => r.level === 1).length;
+ const systemsCount = regions.filter(r => r.label_kind === "system").length;
+ const subsystemsCount = regions.filter(r => r.label_kind === "subsystem").length;
+ const modulesCount = regions.filter(r => r.label_kind === "module").length;
const wellDefined = regions.filter(r => r.confidence >= 0.75).length;
const moderate = regions.filter(r => r.confidence >= 0.50 && r.confidence < 0.75).length;
const fuzzy = regions.filter(r => r.confidence < 0.50).length;
@@ -329,7 +329,7 @@ function confidenceLabel(conf: number): string {
function pickTopSystemName(regions: MapRegion[], cwd: string): string {
const systems = regions
- .filter(r => r.level === 3 || r.label_kind === "system")
+ .filter(r => r.label_kind === "system")
.slice()
.sort((a, b) => b.file_count - a.file_count || b.confidence - a.confidence);
@@ -392,7 +392,7 @@ function renderMapTree(
}
const roots = sortRegions(
- regions.filter(region => region.level === 3 || region.parent_id === null || !regionById.has(region.parent_id)),
+ regions.filter(region => region.label_kind === "system" || region.parent_id === null || !regionById.has(region.parent_id)),
sortMode,
);
const shownRoots = allItems ? roots : roots.slice(0, maxItems);
@@ -457,11 +457,11 @@ function renderRankedList(
verbose: boolean,
sortMode: MapSortMode,
): void {
- const subsystems = sortRegions(regions.filter(r => r.level === 2), sortMode);
+ const subsystems = sortRegions(regions.filter(r => r.label_kind === "subsystem"), sortMode);
const shownSubsystems = allItems ? subsystems : subsystems.slice(0, maxItems);
const shownSubsystemIds = new Set(shownSubsystems.map(region => region.id));
const candidateModules = sortRegions(
- regions.filter(r => r.level === 1 && (
+ regions.filter(r => r.label_kind === "module" && (
shownSubsystemIds.size === 0 ||
(r.parent_id !== null && shownSubsystemIds.has(r.parent_id))
)),
@@ -578,7 +578,7 @@ function formatRegionLine(region: MapRegion, verbose: boolean, depth = 0): strin
const clarityColor = region.confidence >= 0.75 ? chalk.green : region.confidence >= 0.50 ? chalk.yellow : chalk.red;
const confPct = Math.round(region.confidence * 100);
const crosscut = region.crosscut_score > 0.10 ? chalk.yellow(" shared") : "";
- const levelTag = region.level === 3 ? "system" : region.level === 2 ? "subsystem" : "module";
+ const levelTag = region.label_kind || (region.level === 3 ? "system" : region.level === 2 ? "subsystem" : "module");
const badge = chalk.bgBlackBright.white(` ${levelTag.toUpperCase()} `);
const signals = region.dominant_signals.slice(0, 2).join(" · ");
diff --git a/ix-cli/src/cli/commands/subsystems.ts b/ix-cli/src/cli/commands/subsystems.ts
index 2149894e..3be9241e 100644
--- a/ix-cli/src/cli/commands/subsystems.ts
+++ b/ix-cli/src/cli/commands/subsystems.ts
@@ -214,7 +214,13 @@ function printScores(scores: SubsystemScore[]): void {
byLevel.get(s.level)!.push(s);
}
- const levelLabels: Record<number, string> = { 1: "Modules", 2: "Subsystems", 3: "Systems" };
+ const maxLevel = scores.length > 0 ? Math.max(...scores.map(s => s.level)) : 3;
+ const levelLabels: Record<number, string> = {};
+ for (let l = 1; l <= maxLevel; l++) {
+ if (l === maxLevel) levelLabels[l] = "Systems";
+ else if (l === maxLevel - 1) levelLabels[l] = "Subsystems";
+ else levelLabels[l] = "Modules";
+ }
const sorted = [...byLevel.entries()].sort((a, b) => b[0] - a[0]);
for (const [level, group] of sorted) {
diff --git a/ix-cli/src/cli/explain/subsystem.ts b/ix-cli/src/cli/explain/subsystem.ts
index f7d9cd74..6d09261a 100644
--- a/ix-cli/src/cli/explain/subsystem.ts
+++ b/ix-cli/src/cli/explain/subsystem.ts
@@ -250,7 +250,7 @@ function renderWhyItMatters(
);
}
- if (score && score.health_score >= 0.8 && region.level === 3) {
+ if (score && score.health_score >= 0.8 && region.label_kind === "system") {
return (
`${region.label} is a stable, well-defined system boundary. Changes within it are usually ` +
`well-contained. Run \`ix impact ${JSON.stringify(region.label)}\` to review cross-system dependencies.`
diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
index b583c0c1..0f12b9e1 100644
--- a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala
@@ -85,11 +85,13 @@ class ArangoClient private (db: ArangoDatabase) {
documents.size - result.getErrors.size
}
- /** Bulk insert edge documents. Same as bulkInsert but for edge collections. */
+ /** Bulk insert edge documents. Uses 'update' overwrite mode by default —
+ * edge documents are keyed by a deterministic ID so update-merge is safe
+ * and faster than full replace (avoids read-delete-write per document). */
def bulkInsertEdges(
collection: String,
documents: Seq[java.util.Map[String, AnyRef]],
- overwriteMode: String = "replace"
+ overwriteMode: String = "update"
): IO[Int] = bulkInsert(collection, documents, overwriteMode)
// ── Stream Transaction lifecycle ──────────────────────────────────
@@ -119,9 +121,19 @@ class ArangoClient private (db: ArangoDatabase) {
def ensureSchema(): IO[Unit] = ArangoSchema.ensure(db)
+ /** Abort all active stream transactions so truncation can acquire locks. */
+ private def abortAllTransactions(): IO[Unit] =
+ IO.blocking {
+ val txns = db.getStreamTransactions().asScala.toList
+ txns.filter(_.getState.toString == "running").foreach { tx =>
+ try db.abortStreamTransaction(tx.getId)
+ catch { case _: Exception => () }
+ }
+ }
+
/** Truncate all graph state. Destructive — dev use only. */
def truncateGraph(): IO[Unit] =
- IO.blocking {
+ abortAllTransactions() *> IO.blocking {
db.collection("nodes").truncate()
db.collection("edges").truncate()
db.collection("patches").truncate()
@@ -141,7 +153,7 @@ class ArangoClient private (db: ArangoDatabase) {
* then execute enough bounded delete passes to cover that cardinality, with
* a final recount only if needed. Edges are removed before nodes.
*/
- def truncateCodeGraph(): IO[Unit] = {
+ def truncateCodeGraph(): IO[Unit] = abortAllTransactions() *> {
val batchSize = 5000
val codeKinds = Array(
"file", "function", "class", "method", "interface",
diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
index 3a2b8aa8..353c08df 100644
--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala
@@ -27,7 +27,7 @@ class BulkWriteApi(client: ArangoClient) {
private val DefaultChunkSize = 500
private val DefaultChunkPayloadBytes = 16 * 1024 * 1024
private val MaxNodeDocs = 10000
- private val MaxEdgeDocs = 20000
+ private val MaxEdgeDocs = 50000
private val MaxClaimDocs = 10000
private val MaxPatchDocs = 500
private val SlowChunkMs = 2000L
diff --git a/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala b/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala
index 60e7693d..3ae6331d 100644
--- a/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala
+++ b/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala
@@ -44,35 +44,71 @@ object LouvainClustering {
*
* Returns a vector of LevelPartitions, finest first (index 0 = most granular).
* Each partition covers ALL original file nodes.
+ *
+ * Adaptive re-clustering: when the coarsest level exceeds
+ * MaxTopLevelCommunities, the algorithm halves the resolution and
+ * retries up to MaxAdaptiveRetries times to produce coarser groupings.
+ *
+ * Singleton absorption: after each Louvain pass, singleton communities
+ * (size 1) that have edges to other communities are merged into the
+ * community they have the strongest total edge weight to.
*/
+ private val MaxTopLevelCommunities = 60
+ private val MaxAdaptiveRetries = 3
+ /** When consecutive levels jump by more than this ratio, insert intermediate levels. */
+ private val MaxLevelJumpRatio = 15
+ private val MaxGapFillAttempts = 3
+
def cluster(
graph: WeightedFileGraph,
- maxLevels: Int = 4,
+ maxLevels: Int = 5,
resolution: Double = -1.0,
minCommunitySize: Int = 2,
seed: Long = 42L
): Vector[LevelPartition] = {
if (graph.vertices.isEmpty || graph.totalWeight == 0.0) return Vector.empty
- val effectiveResolution =
+ val baseResolution =
if (resolution > 0.0) resolution else adaptiveResolution(graph.vertices.size)
- val numSeeds = if (graph.vertices.size < 30) 1L else 3L
- val runs = (0L until numSeeds).map { offset =>
- val levels = clusterOnce(
- graph = graph,
- maxLevels = maxLevels,
- resolution = effectiveResolution,
- minCommunitySize = minCommunitySize,
- seed = seed + offset
+ var effectiveResolution = baseResolution
+ var bestLevels: Vector[LevelPartition] = Vector.empty
+ var retries = 0
+
+ while (retries <= MaxAdaptiveRetries) {
+ val numSeeds = if (graph.vertices.size < 30) 1L else 3L
+ val runs = (0L until numSeeds).map { offset =>
+ val levels = clusterOnce(
+ graph = graph,
+ maxLevels = maxLevels,
+ resolution = effectiveResolution,
+ minCommunitySize = minCommunitySize,
+ seed = seed + offset
+ )
+ val score = levels.headOption
+ .map(level => modularity(graph, level.assignment, effectiveResolution))
+ .getOrElse(Double.NegativeInfinity)
+ levels -> score
+ }
+
+ bestLevels = fillHierarchyGaps(
+ runs.maxBy(_._2)._1,
+ graph, minCommunitySize, seed
)
- val score = levels.headOption
- .map(level => modularity(graph, level.assignment, effectiveResolution))
- .getOrElse(Double.NegativeInfinity)
- levels -> score
+
+ val coarsestCount = bestLevels.lastOption
+ .map(_.communities.count(_.size >= minCommunitySize))
+ .getOrElse(0)
+
+ if (coarsestCount > MaxTopLevelCommunities && retries < MaxAdaptiveRetries) {
+ effectiveResolution *= 0.5
+ retries += 1
+ } else {
+ retries = MaxAdaptiveRetries + 1 // exit loop
+ }
}
- runs.maxBy(_._2)._1
+ bestLevels
}
private def clusterOnce(
@@ -92,7 +128,7 @@ object LouvainClustering {
var level = 0
while (level < maxLevels && currentGraph.vertices.size > 1) {
- val rawAssignment = louvainPass(currentGraph, resolution, rng)
+ val rawAssignment = absorbSingletons(louvainPass(currentGraph, resolution, rng), currentGraph)
val projected = projectPartition(rawAssignment, nodeExpansion)
val meaningful = projected.communities.count(_.size >= minCommunitySize)
@@ -115,10 +151,151 @@ object LouvainClustering {
levels.toVector
}
- private def adaptiveResolution(fileCount: Int): Double =
- if (fileCount < 50) 1.2
- else if (fileCount <= 500) 1.0
- else 0.8
+ /**
+ * When consecutive levels jump from N communities to M (ratio N/M > MaxLevelJumpRatio),
+ * insert intermediate levels by sub-clustering the finer level's communities at
+ * progressively lower resolutions to bridge the gap.
+ *
+ * Example: if level 1 has 847 communities and level 2 has 7, this inserts
+ * intermediate levels (e.g., ~60-80 communities) between them.
+ */
+ private def fillHierarchyGaps(
+ levels: Vector[LevelPartition],
+ graph: WeightedFileGraph,
+ minCommunitySize: Int,
+ seed: Long
+ ): Vector[LevelPartition] = {
+ if (levels.size < 2) return levels
+
+ val result = scala.collection.mutable.ArrayBuffer[LevelPartition]()
+ result += levels.head
+
+ for (i <- 1 until levels.size) {
+ val finer = levels(i - 1)
+ val coarser = levels(i)
+ val finerCount = finer.communities.count(_.size >= minCommunitySize)
+ val coarserCount = coarser.communities.count(_.size >= minCommunitySize)
+
+ val ratio = if (coarserCount > 0) finerCount.toDouble / coarserCount else 0.0
+ if (finerCount > 0 && coarserCount > 0 && ratio > MaxLevelJumpRatio) {
+ // Build a weighted graph of the finer-level communities
+ val interLevels = bridgeLevels(finer, coarser, graph, minCommunitySize, seed)
+ result ++= interLevels
+ }
+
+ result += coarser
+ }
+
+ result.toVector
+ }
+
+ /**
+ * Create intermediate partitions between a fine level and a coarse level.
+ * Builds a community graph from the fine level and runs Louvain at varying
+ * resolutions to produce partitions with intermediate community counts.
+ */
+ private def bridgeLevels(
+ finer: LevelPartition,
+ coarser: LevelPartition,
+ graph: WeightedFileGraph,
+ minCommunitySize: Int,
+ seed: Long
+ ): Vector[LevelPartition] = {
+ val finerCount = finer.communities.count(_.size >= minCommunitySize)
+ val coarserCount = coarser.communities.count(_.size >= minCommunitySize)
+
+ // Build a supernode graph where each node = one fine-level community
+ val commToId: Map[Int, NodeId] = finer.communities.indices.map { ci =>
+ val key = "map:bridge:" + ci
+ ci -> NodeId(java.util.UUID.nameUUIDFromBytes(key.getBytes("UTF-8")))
+ }.toMap
+
+ val superVertices = commToId.map { case (ci, nid) =>
+ FileVertex(nid, s"comm_$ci")
+ }.toVector
+
+ // Aggregate inter-community edge weights
+ val edgeWeights = scala.collection.mutable.Map[(NodeId, NodeId), Double]()
+ for ((src, neighbors) <- graph.adjMatrix) {
+ val ci = finer.assignment.getOrElse(src, -1)
+ if (ci >= 0) {
+ val si = commToId(ci)
+ for ((dst, w) <- neighbors) {
+ val cj = finer.assignment.getOrElse(dst, -1)
+ if (cj >= 0 && ci != cj) {
+ val sj = commToId(cj)
+ val key = if (si.value.compareTo(sj.value) < 0) (si, sj) else (sj, si)
+ edgeWeights(key) = edgeWeights.getOrElse(key, 0.0) + w / 2.0
+ }
+ }
+ }
+ }
+
+ val newAdj = scala.collection.mutable.Map[NodeId, scala.collection.mutable.Map[NodeId, Double]]()
+ for (((si, sj), w) <- edgeWeights) {
+ newAdj.getOrElseUpdate(si, scala.collection.mutable.Map())(sj) = w
+ newAdj.getOrElseUpdate(sj, scala.collection.mutable.Map())(si) = w
+ }
+ val adjFinal = newAdj.map { case (k, m) => k -> m.toMap }.toMap
+ val degrees = adjFinal.map { case (k, m) => k -> m.values.sum }
+ val totalWeight = edgeWeights.values.sum
+
+ val bridgeGraph = WeightedFileGraph(superVertices, adjFinal, degrees.toMap, totalWeight, Map.empty)
+
+ // Try different resolutions to find intermediate community counts.
+ // Higher resolution → more communities (finer); lower → fewer (coarser).
+ val rng = new Random(seed + 999)
+ val targetMin = coarserCount * 2
+ val targetMax = finerCount / 2
+ val inserted = scala.collection.mutable.ArrayBuffer[LevelPartition]()
+
+ // Sweep resolutions: start very high (fine) and decrease toward coarse.
+ // This explores intermediate granularities between the two existing levels.
+ val resolutions = Vector(4.0, 2.0, 1.0, 0.5, 0.25, 0.1)
+ for (res <- resolutions) {
+ val assignment = absorbSingletons(louvainPass(bridgeGraph, res, rng), bridgeGraph)
+ val projected = projectBridgePartition(assignment, commToId, finer)
+ val count = projected.communities.count(_.size >= minCommunitySize)
+
+ if (count > coarserCount && count < finerCount) {
+ val isDuplicate = inserted.exists { existing =>
+ val ec = existing.communities.count(_.size >= minCommunitySize)
+ ec > 0 && (count.toDouble / ec > 0.7 && count.toDouble / ec < 1.4)
+ }
+ if (!isDuplicate) inserted += projected
+ }
+ }
+
+ // Sort by community count descending (finest first) to maintain level ordering
+ inserted.sortBy(p => -p.communities.count(_.size >= minCommunitySize)).toVector
+ }
+
+ /** Project a bridge-graph partition back to original file IDs. */
+ private def projectBridgePartition(
+ bridgeAssignment: Map[NodeId, Int],
+ commToId: Map[Int, NodeId],
+ finer: LevelPartition
+ ): LevelPartition = {
+ val idToComm = commToId.map(_.swap)
+ // For each bridge community, collect all original file IDs from the fine-level communities it contains
+ val bridgeComms = scala.collection.mutable.Map[Int, scala.collection.mutable.Set[NodeId]]()
+ for ((nid, bridgeIdx) <- bridgeAssignment) {
+ idToComm.get(nid).foreach { fineCommIdx =>
+ val files = finer.communities(fineCommIdx)
+ bridgeComms.getOrElseUpdate(bridgeIdx, scala.collection.mutable.Set()) ++= files
+ }
+ }
+ val commVec = bridgeComms.toVector.sortBy(_._1).map(_._2.toSet)
+ val reverseAssign = (for ((members, idx) <- commVec.zipWithIndex; f <- members) yield f -> idx).toMap
+ LevelPartition(commVec, reverseAssign)
+ }
+
+ private[map] def adaptiveResolution(fileCount: Int): Double =
+ if (fileCount < 50) 1.2
+ else if (fileCount <= 500) 1.0
+ else if (fileCount <= 2000) 0.8
+ else if (fileCount <= 5000) 0.6
+ else 0.4
// ── Single Louvain pass ────────────────────────────────────────────
@@ -242,6 +419,50 @@ object LouvainClustering {
}
}
+ // ── Singleton absorption ───────────────────────────────────────────
+
+ /**
+ * After a Louvain pass, merge singleton communities into the neighbor
+ * community they have the strongest total edge weight to. Nodes with
+ * zero edges to any other community stay as singletons (truly isolated).
+ */
+ private def absorbSingletons(
+ assignment: Map[NodeId, Int],
+ graph: WeightedFileGraph
+ ): Map[NodeId, Int] = {
+ // Identify singleton communities
+ val commMembers = assignment.groupBy(_._2).map { case (c, m) => c -> m.keySet }
+ val singletonComms = commMembers.filter(_._2.size == 1).keySet
+
+ if (singletonComms.isEmpty) return assignment
+
+ val result = scala.collection.mutable.Map(assignment.toSeq: _*)
+
+ for (comm <- singletonComms) {
+ val node = commMembers(comm).head
+ val neighbors = graph.adjMatrix.getOrElse(node, Map.empty)
+ if (neighbors.nonEmpty) {
+ // Sum edge weight to each neighboring community (excluding our own singleton)
+ val weightByComm = scala.collection.mutable.Map.empty[Int, Double]
+ for ((neighbor, w) <- neighbors) {
+ val nc = result(neighbor)
+ if (nc != comm) {
+ weightByComm(nc) = weightByComm.getOrElse(nc, 0.0) + w
+ }
+ }
+ // Merge into the community with the strongest total weight
+ if (weightByComm.nonEmpty) {
+ result(node) = weightByComm.maxBy(_._2)._1
+ }
+ }
+ }
+
+ // Re-index to keep dense community indices
+ val usedComms = result.values.toVector.distinct.sorted
+ val reindex = usedComms.zipWithIndex.toMap
+ result.map { case (n, c) => n -> reindex(c) }.toMap
+ }
+
// ── Projection ─────────────────────────────────────────────────────
private def projectPartition(
diff --git a/memory-layer/src/main/scala/ix/memory/map/MapService.scala b/memory-layer/src/main/scala/ix/memory/map/MapService.scala
index b0c1d5fb..a0638a65 100644
--- a/memory-layer/src/main/scala/ix/memory/map/MapService.scala
+++ b/memory-layer/src/main/scala/ix/memory/map/MapService.scala
@@ -33,8 +33,8 @@ class MapService(
private val logger = Slf4jLogger.getLoggerFromName[IO]("ix.map")
private val CrosscutThreshold = 0.10
- // Cross-cut penalty multiplier (reduces edge weights involving crosscut files)
- private val CrosscutLambda = 0.5
+ // Cross-cut penalty base multiplier (reduces edge weights involving crosscut files)
+ private val CrosscutLambdaBase = 0.5
// High-degree threshold: files in the top N% by degree are crosscut candidates
private val CrosscutDegreeTop = 0.10
// Path tokens that suggest cross-cutting utilities
@@ -55,12 +55,13 @@ class MapService(
private val preflight = new MapPreflight()
private final class RetainedRegion(val level: Int, val communityIndex: Int, val region: Region)
- // Label kinds by level
+ // Label kinds by level — supports arbitrary depth.
+ // Top level = "system", bottom level = "module", everything in between = "subsystem".
private def labelKind(level: Int, maxLevel: Int): String =
- if (maxLevel <= 1) "system"
+ if (maxLevel <= 1) "system"
else if (level == maxLevel) "system"
- else if (level == 1) "module"
- else "subsystem"
+ else if (level == 1) "module"
+ else "subsystem"
def buildMap(forceRecompute: Boolean = false, forceFull: Boolean = false): IO[ArchitectureMap] =
getOrBuildMap(forceRecompute, forceFull)
@@ -184,15 +185,22 @@ class MapService(
}
}
+ private def adaptiveCrosscutLambda(fileCount: Int): Double =
+ if (fileCount <= 500) CrosscutLambdaBase // 0.5
+ else if (fileCount <= 2000) CrosscutLambdaBase * 0.6 // 0.3
+ else if (fileCount <= 5000) CrosscutLambdaBase * 0.4 // 0.2
+ else CrosscutLambdaBase * 0.3 // 0.15
+
private def applyPenalty(
graph: WeightedFileGraph,
scores: Map[NodeId, Double]
): WeightedFileGraph = {
+ val lambda = adaptiveCrosscutLambda(graph.vertices.size)
val newAdj = graph.adjMatrix.map { case (src, neighbors) =>
val sCross = scores.getOrElse(src, 0.0)
val adjusted = neighbors.map { case (dst, w) =>
val dCross = scores.getOrElse(dst, 0.0)
- val penalty = (sCross + dCross) / 2.0 * CrosscutLambda
+ val penalty = (sCross + dCross) / 2.0 * lambda
dst -> w * (1.0 - penalty)
}
src -> adjusted
@@ -779,7 +787,7 @@ class MapService(
crosscutScores <- IO.blocking(detectCrosscut(rawGraph))
graph <- IO.blocking(applyPenalty(rawGraph, crosscutScores))
t2 <- IO(System.currentTimeMillis())
- levels <- IO.blocking(LouvainClustering.cluster(graph, maxLevels = 3, minCommunitySize = 3))
+ levels <- IO.blocking(LouvainClustering.cluster(graph, maxLevels = 5, minCommunitySize = 3))
t3 <- IO(System.currentTimeMillis())
regions <- IO.blocking(buildRegions(graph, levels, crosscutScores, inputRev))
edges <- IO.blocking(computeArchitectureEdges(regions, rawGraph))