Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
Adaptive map hierarchy + ingestion perf + reset fix (#119)
Tanner Kocher3996214f130e
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex e35ef430..19a77b2f 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -6,6 +6,7 @@ import { spawnSync } from 'node:child_process';import { fileURLToPath } from 'node:url';import type { Command } from 'commander';import { ParsePool } from './parse-pool.js';+// import { ResolveWorker } from './resolve-pool.js';import chalk from 'chalk';import { IxClient } from '../../client/api.js';import type { GraphPatchPayload } from '../../client/types.js';@@ -411,6 +412,7 @@ export async function ingestFiles(return pool;};+let progressPhase = 'Scanning';let progressCurrent = 0;let progressTotal = 0;@@ -449,6 +451,7 @@ export async function ingestFiles(// Phase timing marksconst timings = {moduleLoadMs: 0, discoverMs: 0, hashMs: 0, parseMs: 0,+ goIndexMs: 0, // time to build Go global resolution indexparseOnlyMs: 0, // sum of just parseFile() call durationsresolveMs: 0, buildPatchMs: 0, commitMs: 0,stripChunkOpsMs: 0, // sum of stripChunkOps() durations (mapMode only)@@ -587,9 +590,12 @@ export async function ingestFiles(let buildPatchFn: Function | null = null;let globalIndex: any = undefined;- const PARSE_STREAM_CHUNK = 1000; // streaming batch size for large repos- const SMALL_REPO_THRESHOLD = 5_000; // parse-all-first below this; streaming above- const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 400); // files per HTTP request to the backend+ // Larger chunks for big repos: fewer resolve+commit cycles, better edge resolution per batch.+ // Smaller chunks pipeline better: resolve(N) overlaps with commit(N-1) + parse(N+1).+ // With 2000-file chunks, resolve takes 68s and dominates. With 500, resolve ~17s+ // and overlaps well with commit (~5s) + parse (~5s).+ const PARSE_STREAM_CHUNK = filePaths.length > 10_000 ? 500 : 500;+ const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 1000); // files per HTTP request to the backendconst COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 8); // parallel HTTP save requestsconst COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflictsconst YIELD_EVERY = 100; // yield event loop every N files during parse@@ -896,9 +902,9 @@ export async function ingestFiles(if (nodePath.extname(filePath) === '.go') sources.set(filePath, bytes.toString('utf-8'));}const remainingGoFiles = filePaths.filter(fp => nodePath.extname(fp) === '.go' && !changedSet.has(fp));- const GO_READ_BATCH = 500;- for (let i = 0; i < remainingGoFiles.length; i += GO_READ_BATCH) {- const batch = remainingGoFiles.slice(i, i + GO_READ_BATCH);+ const GO_READ_CONCURRENCY = 2000;+ for (let i = 0; i < remainingGoFiles.length; i += GO_READ_CONCURRENCY) {+ const batch = remainingGoFiles.slice(i, i + GO_READ_CONCURRENCY);const texts = await Promise.all(batch.map(fp => fs.promises.readFile(fp, 'utf-8').catch(() => null)));for (let j = 0; j < batch.length; j++) {if (texts[j] != null) sources.set(batch[j], texts[j]!);@@ -938,13 +944,14 @@ export async function ingestFiles(// Build global resolution index from all repo file paths before the parse loop// so cross-batch imports resolve correctly even in streaming per-chunk mode.- // Use async batched reads (same as Path A) to avoid blocking the event loop.+ // Read all Go files in parallel to maximize I/O throughput.{+ const goIndexStart = performance.now();const sources = new Map<string, string>();const goFiles = filePaths.filter(fp => nodePath.extname(fp) === '.go');- const GO_READ_BATCH = 500;- for (let i = 0; i < goFiles.length; i += GO_READ_BATCH) {- const batch = goFiles.slice(i, i + GO_READ_BATCH);+ const GO_READ_CONCURRENCY = 2000;+ for (let i = 0; i < goFiles.length; i += GO_READ_CONCURRENCY) {+ const batch = goFiles.slice(i, i + GO_READ_CONCURRENCY);const texts = await Promise.all(batch.map(fp => fs.promises.readFile(fp, 'utf-8').catch(() => null)));for (let j = 0; j < batch.length; j++) {if (texts[j] != null) sources.set(batch[j], texts[j]!);@@ -952,51 +959,52 @@ export async function ingestFiles(}globalIndex = ingestion.buildGlobalResolutionIndex(filePaths, sources);sources.clear();+ timings.goIndexMs = Math.round(performance.now() - goIndexStart);}progressPhase = 'Parsing';progressTotal = filePaths.length;progressCurrent = 0;- // Pipeline overlap: parse next chunk in worker pool while committing current chunk.- // Each file is dispatched to the parse pool immediately after it is read so workers- // start processing as soon as the first file in the chunk is ready rather than- // waiting for all reads in the chunk to complete.+ // Pipeline overlap: read files asynchronously and dispatch to parse pool+ // immediately so workers start processing while more files are still being read.+ // Commit previous chunk's results while parsing the next chunk.let pendingFlushB: Promise<void> = Promise.resolve();for (let i = 0; i < filePaths.length; i += PARSE_STREAM_CHUNK) {const chunk = filePaths.slice(i, i + PARSE_STREAM_CHUNK);type FileData = { filePath: string; source: string; hash: string; previousHash: string | undefined } | null;- const fileData: FileData[] = [];- const parsePromises: Promise<unknown>[] = [];- for (const filePath of chunk) {+ const fileData: FileData[] = new Array(chunk.length).fill(null);+ const parsePromises: Promise<unknown>[] = new Array(chunk.length).fill(Promise.resolve(null));++ // Read files asynchronously and dispatch to parse pool as each file completes.+ // This keeps workers busy while I/O is still in flight for other files.+ const readAndDispatch = chunk.map(async (filePath, idx) => {try {- const fileSize = fs.statSync(filePath).size;- if (fileSize === 0) { filesSkipped++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }- if (fileSize > MAX_FILE_BYTES) { tooLarge++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }- const bytes = fs.readFileSync(filePath);+ const st = await fs.promises.stat(filePath);+ if (st.size === 0) { filesSkipped++; return; }+ if (st.size > MAX_FILE_BYTES) { tooLarge++; return; }+ const bytes = await fs.promises.readFile(filePath);const hash = sha256(bytes);- if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; fileData.push(null); parsePromises.push(Promise.resolve(null)); continue; }+ if (!opts.force && knownHashes.get(filePath) === hash) { filesSkipped++; return; }const previousHash = knownHashes.get(filePath);const sourceText = bytes.toString('utf-8');if (isLikelyMinifiedSource(sourceText)) {minifiedLikely++;filesSkipped++;if (debug) process.stderr.write(`\n [skip minified-likely] ${filePath}\n`);- fileData.push(null);- parsePromises.push(Promise.resolve(null));- continue;+ return;}const fd: NonNullable<FileData> = { filePath, source: sourceText, hash, previousHash: previousHash !== hash ? previousHash : undefined };- fileData.push(fd);- parsePromises.push(ensureParsePool().parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; }));+ fileData[idx] = fd;+ parsePromises[idx] = ensureParsePool().parse(fd.filePath, fd.source).then(r => { progressCurrent++; return r; });} catch (err) {parseErrors++;- fileData.push(null);- parsePromises.push(Promise.resolve(null));process.stderr.write(`\n [read error] ${filePath}: ${err}\n`);}- }+ });+ await Promise.all(readAndDispatch);const parseResults = await Promise.all(parsePromises);+const batch: ParsedFile[] = [];for (let j = 0; j < fileData.length; j++) {const f = fileData[j];@@ -1088,6 +1096,7 @@ export async function ingestFiles(console.log(chalk.dim(` modules: ${timings.moduleLoadMs}ms`));console.log(chalk.dim(` discover: ${timings.discoverMs}ms`));console.log(chalk.dim(` hash: ${timings.hashMs}ms`));+ console.log(chalk.dim(` go index: ${timings.goIndexMs}ms`));console.log(chalk.dim(` parse total: ${timings.parseMs}ms`));console.log(chalk.dim(` parse only: ${Math.round(timings.parseOnlyMs)}ms`));console.log(chalk.dim(` resolve: ${timings.resolveMs}ms`));diff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.tsindex 1649d454..bc2f648b 100644--- a/ix-cli/src/cli/commands/map.ts+++ b/ix-cli/src/cli/commands/map.ts@@ -191,9 +191,9 @@ Examples:const mapMs = Math.round(performance.now() - mapStart);if (silent) {- const systems = result.regions.filter(r => r.level === 3).length;- const subsystems = result.regions.filter(r => r.level === 2).length;- const modules = result.regions.filter(r => r.level === 1).length;+ const systems = result.regions.filter(r => r.label_kind === "system").length;+ const subsystems = result.regions.filter(r => r.label_kind === "subsystem").length;+ const modules = result.regions.filter(r => r.label_kind === "module").length;process.stderr.write(`map: ${result.file_count} files · ${systems}s/${subsystems}ss/${modules}m regions · ${mapMs}ms\n`);@@ -271,9 +271,9 @@ export function renderMapText(result: MapResult, cwd: string, opts: MapTextRendeconst regionById = new Map(result.regions.map(r => [r.id, r]));const CROSSCUT_THRESHOLD = 0.10;- const systemsCount = regions.filter(r => r.level === 3).length;- const subsystemsCount = regions.filter(r => r.level === 2).length;- const modulesCount = regions.filter(r => r.level === 1).length;+ const systemsCount = regions.filter(r => r.label_kind === "system").length;+ const subsystemsCount = regions.filter(r => r.label_kind === "subsystem").length;+ const modulesCount = regions.filter(r => r.label_kind === "module").length;const wellDefined = regions.filter(r => r.confidence >= 0.75).length;const moderate = regions.filter(r => r.confidence >= 0.50 && r.confidence < 0.75).length;const fuzzy = regions.filter(r => r.confidence < 0.50).length;@@ -329,7 +329,7 @@ function confidenceLabel(conf: number): string {function pickTopSystemName(regions: MapRegion[], cwd: string): string {const systems = regions- .filter(r => r.level === 3 || r.label_kind === "system")+ .filter(r => r.label_kind === "system").slice().sort((a, b) => b.file_count - a.file_count || b.confidence - a.confidence);@@ -392,7 +392,7 @@ function renderMapTree(}const roots = sortRegions(- regions.filter(region => region.level === 3 || region.parent_id === null || !regionById.has(region.parent_id)),+ regions.filter(region => region.label_kind === "system" || region.parent_id === null || !regionById.has(region.parent_id)),sortMode,);const shownRoots = allItems ? roots : roots.slice(0, maxItems);@@ -457,11 +457,11 @@ function renderRankedList(verbose: boolean,sortMode: MapSortMode,): void {- const subsystems = sortRegions(regions.filter(r => r.level === 2), sortMode);+ const subsystems = sortRegions(regions.filter(r => r.label_kind === "subsystem"), sortMode);const shownSubsystems = allItems ? subsystems : subsystems.slice(0, maxItems);const shownSubsystemIds = new Set(shownSubsystems.map(region => region.id));const candidateModules = sortRegions(- regions.filter(r => r.level === 1 && (+ regions.filter(r => r.label_kind === "module" && (shownSubsystemIds.size === 0 ||(r.parent_id !== null && shownSubsystemIds.has(r.parent_id)))),@@ -578,7 +578,7 @@ function formatRegionLine(region: MapRegion, verbose: boolean, depth = 0): strinconst clarityColor = region.confidence >= 0.75 ? chalk.green : region.confidence >= 0.50 ? chalk.yellow : chalk.red;const confPct = Math.round(region.confidence * 100);const crosscut = region.crosscut_score > 0.10 ? chalk.yellow(" shared") : "";- const levelTag = region.level === 3 ? "system" : region.level === 2 ? "subsystem" : "module";+ const levelTag = region.label_kind || (region.level === 3 ? "system" : region.level === 2 ? "subsystem" : "module");const badge = chalk.bgBlackBright.white(` ${levelTag.toUpperCase()} `);const signals = region.dominant_signals.slice(0, 2).join(" · ");diff --git a/ix-cli/src/cli/commands/subsystems.ts b/ix-cli/src/cli/commands/subsystems.tsindex 2149894e..3be9241e 100644--- a/ix-cli/src/cli/commands/subsystems.ts+++ b/ix-cli/src/cli/commands/subsystems.ts@@ -214,7 +214,13 @@ function printScores(scores: SubsystemScore[]): void {byLevel.get(s.level)!.push(s);}- const levelLabels: Record<number, string> = { 1: "Modules", 2: "Subsystems", 3: "Systems" };+ const maxLevel = scores.length > 0 ? Math.max(...scores.map(s => s.level)) : 3;+ const levelLabels: Record<number, string> = {};+ for (let l = 1; l <= maxLevel; l++) {+ if (l === maxLevel) levelLabels[l] = "Systems";+ else if (l === maxLevel - 1) levelLabels[l] = "Subsystems";+ else levelLabels[l] = "Modules";+ }const sorted = [...byLevel.entries()].sort((a, b) => b[0] - a[0]);for (const [level, group] of sorted) {diff --git a/ix-cli/src/cli/explain/subsystem.ts b/ix-cli/src/cli/explain/subsystem.tsindex f7d9cd74..6d09261a 100644--- a/ix-cli/src/cli/explain/subsystem.ts+++ b/ix-cli/src/cli/explain/subsystem.ts@@ -250,7 +250,7 @@ function renderWhyItMatters();}- if (score && score.health_score >= 0.8 && region.level === 3) {+ if (score && score.health_score >= 0.8 && region.label_kind === "system") {return (`${region.label} is a stable, well-defined system boundary. Changes within it are usually ` +`well-contained. Run \`ix impact ${JSON.stringify(region.label)}\` to review cross-system dependencies.`diff --git a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scalaindex b583c0c1..0f12b9e1 100644--- a/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala+++ b/memory-layer/src/main/scala/ix/memory/db/ArangoClient.scala@@ -85,11 +85,13 @@ class ArangoClient private (db: ArangoDatabase) {documents.size - result.getErrors.size}- /** Bulk insert edge documents. Same as bulkInsert but for edge collections. */+ /** Bulk insert edge documents. Uses 'update' overwrite mode by default —+ * edge documents are keyed by a deterministic ID so update-merge is safe+ * and faster than full replace (avoids read-delete-write per document). */def bulkInsertEdges(collection: String,documents: Seq[java.util.Map[String, AnyRef]],- overwriteMode: String = "replace"+ overwriteMode: String = "update"): IO[Int] = bulkInsert(collection, documents, overwriteMode)// ── Stream Transaction lifecycle ──────────────────────────────────@@ -119,9 +121,19 @@ class ArangoClient private (db: ArangoDatabase) {def ensureSchema(): IO[Unit] = ArangoSchema.ensure(db)+ /** Abort all active stream transactions so truncation can acquire locks. */+ private def abortAllTransactions(): IO[Unit] =+ IO.blocking {+ val txns = db.getStreamTransactions().asScala.toList+ txns.filter(_.getState.toString == "running").foreach { tx =>+ try db.abortStreamTransaction(tx.getId)+ catch { case _: Exception => () }+ }+ }+/** Truncate all graph state. Destructive — dev use only. */def truncateGraph(): IO[Unit] =- IO.blocking {+ abortAllTransactions() *> IO.blocking {db.collection("nodes").truncate()db.collection("edges").truncate()db.collection("patches").truncate()@@ -141,7 +153,7 @@ class ArangoClient private (db: ArangoDatabase) {* then execute enough bounded delete passes to cover that cardinality, with* a final recount only if needed. Edges are removed before nodes.*/- def truncateCodeGraph(): IO[Unit] = {+ def truncateCodeGraph(): IO[Unit] = abortAllTransactions() *> {val batchSize = 5000val codeKinds = Array("file", "function", "class", "method", "interface",diff --git a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scalaindex 3a2b8aa8..353c08df 100644--- a/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala+++ b/memory-layer/src/main/scala/ix/memory/db/BulkWriteApi.scala@@ -27,7 +27,7 @@ class BulkWriteApi(client: ArangoClient) {private val DefaultChunkSize = 500private val DefaultChunkPayloadBytes = 16 * 1024 * 1024private val MaxNodeDocs = 10000- private val MaxEdgeDocs = 20000+ private val MaxEdgeDocs = 50000private val MaxClaimDocs = 10000private val MaxPatchDocs = 500private val SlowChunkMs = 2000Ldiff --git a/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala b/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scalaindex 60e7693d..3ae6331d 100644--- a/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala+++ b/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala@@ -44,35 +44,71 @@ object LouvainClustering {** Returns a vector of LevelPartitions, finest first (index 0 = most granular).* Each partition covers ALL original file nodes.+ *+ * Adaptive re-clustering: when the coarsest level exceeds+ * MaxTopLevelCommunities, the algorithm halves the resolution and+ * retries up to MaxAdaptiveRetries times to produce coarser groupings.+ *+ * Singleton absorption: after each Louvain pass, singleton communities+ * (size 1) that have edges to other communities are merged into the+ * community they have the strongest total edge weight to.*/+ private val MaxTopLevelCommunities = 60+ private val MaxAdaptiveRetries = 3+ /** When consecutive levels jump by more than this ratio, insert intermediate levels. */+ private val MaxLevelJumpRatio = 15+ private val MaxGapFillAttempts = 3+def cluster(graph: WeightedFileGraph,- maxLevels: Int = 4,+ maxLevels: Int = 5,resolution: Double = -1.0,minCommunitySize: Int = 2,seed: Long = 42L): Vector[LevelPartition] = {if (graph.vertices.isEmpty || graph.totalWeight == 0.0) return Vector.empty- val effectiveResolution =+ val baseResolution =if (resolution > 0.0) resolution else adaptiveResolution(graph.vertices.size)- val numSeeds = if (graph.vertices.size < 30) 1L else 3L- val runs = (0L until numSeeds).map { offset =>- val levels = clusterOnce(- graph = graph,- maxLevels = maxLevels,- resolution = effectiveResolution,- minCommunitySize = minCommunitySize,- seed = seed + offset+ var effectiveResolution = baseResolution+ var bestLevels: Vector[LevelPartition] = Vector.empty+ var retries = 0++ while (retries <= MaxAdaptiveRetries) {+ val numSeeds = if (graph.vertices.size < 30) 1L else 3L+ val runs = (0L until numSeeds).map { offset =>+ val levels = clusterOnce(+ graph = graph,+ maxLevels = maxLevels,+ resolution = effectiveResolution,+ minCommunitySize = minCommunitySize,+ seed = seed + offset+ )+ val score = levels.headOption+ .map(level => modularity(graph, level.assignment, effectiveResolution))+ .getOrElse(Double.NegativeInfinity)+ levels -> score+ }++ bestLevels = fillHierarchyGaps(+ runs.maxBy(_._2)._1,+ graph, minCommunitySize, seed)- val score = levels.headOption- .map(level => modularity(graph, level.assignment, effectiveResolution))- .getOrElse(Double.NegativeInfinity)- levels -> score++ val coarsestCount = bestLevels.lastOption+ .map(_.communities.count(_.size >= minCommunitySize))+ .getOrElse(0)++ if (coarsestCount > MaxTopLevelCommunities && retries < MaxAdaptiveRetries) {+ effectiveResolution *= 0.5+ retries += 1+ } else {+ retries = MaxAdaptiveRetries + 1 // exit loop+ }}- runs.maxBy(_._2)._1+ bestLevels}private def clusterOnce(@@ -92,7 +128,7 @@ object LouvainClustering {var level = 0while (level < maxLevels && currentGraph.vertices.size > 1) {- val rawAssignment = louvainPass(currentGraph, resolution, rng)+ val rawAssignment = absorbSingletons(louvainPass(currentGraph, resolution, rng), currentGraph)val projected = projectPartition(rawAssignment, nodeExpansion)val meaningful = projected.communities.count(_.size >= minCommunitySize)@@ -115,10 +151,151 @@ object LouvainClustering {levels.toVector}- private def adaptiveResolution(fileCount: Int): Double =- if (fileCount < 50) 1.2- else if (fileCount <= 500) 1.0- else 0.8+ /**+ * When consecutive levels jump from N communities to M (ratio N/M > MaxLevelJumpRatio),+ * insert intermediate levels by sub-clustering the finer level's communities at+ * progressively lower resolutions to bridge the gap.+ *+ * Example: if level 1 has 847 communities and level 2 has 7, this inserts+ * intermediate levels (e.g., ~60-80 communities) between them.+ */+ private def fillHierarchyGaps(+ levels: Vector[LevelPartition],+ graph: WeightedFileGraph,+ minCommunitySize: Int,+ seed: Long+ ): Vector[LevelPartition] = {+ if (levels.size < 2) return levels++ val result = scala.collection.mutable.ArrayBuffer[LevelPartition]()+ result += levels.head++ for (i <- 1 until levels.size) {+ val finer = levels(i - 1)+ val coarser = levels(i)+ val finerCount = finer.communities.count(_.size >= minCommunitySize)+ val coarserCount = coarser.communities.count(_.size >= minCommunitySize)++ val ratio = if (coarserCount > 0) finerCount.toDouble / coarserCount else 0.0+ if (finerCount > 0 && coarserCount > 0 && ratio > MaxLevelJumpRatio) {+ // Build a weighted graph of the finer-level communities+ val interLevels = bridgeLevels(finer, coarser, graph, minCommunitySize, seed)+ result ++= interLevels+ }++ result += coarser+ }++ result.toVector+ }++ /**+ * Create intermediate partitions between a fine level and a coarse level.+ * Builds a community graph from the fine level and runs Louvain at varying+ * resolutions to produce partitions with intermediate community counts.+ */+ private def bridgeLevels(+ finer: LevelPartition,+ coarser: LevelPartition,+ graph: WeightedFileGraph,+ minCommunitySize: Int,+ seed: Long+ ): Vector[LevelPartition] = {+ val finerCount = finer.communities.count(_.size >= minCommunitySize)+ val coarserCount = coarser.communities.count(_.size >= minCommunitySize)++ // Build a supernode graph where each node = one fine-level community+ val commToId: Map[Int, NodeId] = finer.communities.indices.map { ci =>+ val key = "map:bridge:" + ci+ ci -> NodeId(java.util.UUID.nameUUIDFromBytes(key.getBytes("UTF-8")))+ }.toMap++ val superVertices = commToId.map { case (ci, nid) =>+ FileVertex(nid, s"comm_$ci")+ }.toVector++ // Aggregate inter-community edge weights+ val edgeWeights = scala.collection.mutable.Map[(NodeId, NodeId), Double]()+ for ((src, neighbors) <- graph.adjMatrix) {+ val ci = finer.assignment.getOrElse(src, -1)+ if (ci >= 0) {+ val si = commToId(ci)+ for ((dst, w) <- neighbors) {+ val cj = finer.assignment.getOrElse(dst, -1)+ if (cj >= 0 && ci != cj) {+ val sj = commToId(cj)+ val key = if (si.value.compareTo(sj.value) < 0) (si, sj) else (sj, si)+ edgeWeights(key) = edgeWeights.getOrElse(key, 0.0) + w / 2.0+ }+ }+ }+ }++ val newAdj = scala.collection.mutable.Map[NodeId, scala.collection.mutable.Map[NodeId, Double]]()+ for (((si, sj), w) <- edgeWeights) {+ newAdj.getOrElseUpdate(si, scala.collection.mutable.Map())(sj) = w+ newAdj.getOrElseUpdate(sj, scala.collection.mutable.Map())(si) = w+ }+ val adjFinal = newAdj.map { case (k, m) => k -> m.toMap }.toMap+ val degrees = adjFinal.map { case (k, m) => k -> m.values.sum }+ val totalWeight = edgeWeights.values.sum++ val bridgeGraph = WeightedFileGraph(superVertices, adjFinal, degrees.toMap, totalWeight, Map.empty)++ // Try different resolutions to find intermediate community counts.+ // Higher resolution → more communities (finer); lower → fewer (coarser).+ val rng = new Random(seed + 999)+ val targetMin = coarserCount * 2+ val targetMax = finerCount / 2+ val inserted = scala.collection.mutable.ArrayBuffer[LevelPartition]()++ // Sweep resolutions: start very high (fine) and decrease toward coarse.+ // This explores intermediate granularities between the two existing levels.+ val resolutions = Vector(4.0, 2.0, 1.0, 0.5, 0.25, 0.1)+ for (res <- resolutions) {+ val assignment = absorbSingletons(louvainPass(bridgeGraph, res, rng), bridgeGraph)+ val projected = projectBridgePartition(assignment, commToId, finer)+ val count = projected.communities.count(_.size >= minCommunitySize)++ if (count > coarserCount && count < finerCount) {+ val isDuplicate = inserted.exists { existing =>+ val ec = existing.communities.count(_.size >= minCommunitySize)+ ec > 0 && (count.toDouble / ec > 0.7 && count.toDouble / ec < 1.4)+ }+ if (!isDuplicate) inserted += projected+ }+ }++ // Sort by community count descending (finest first) to maintain level ordering+ inserted.sortBy(p => -p.communities.count(_.size >= minCommunitySize)).toVector+ }++ /** Project a bridge-graph partition back to original file IDs. */+ private def projectBridgePartition(+ bridgeAssignment: Map[NodeId, Int],+ commToId: Map[Int, NodeId],+ finer: LevelPartition+ ): LevelPartition = {+ val idToComm = commToId.map(_.swap)+ // For each bridge community, collect all original file IDs from the fine-level communities it contains+ val bridgeComms = scala.collection.mutable.Map[Int, scala.collection.mutable.Set[NodeId]]()+ for ((nid, bridgeIdx) <- bridgeAssignment) {+ idToComm.get(nid).foreach { fineCommIdx =>+ val files = finer.communities(fineCommIdx)+ bridgeComms.getOrElseUpdate(bridgeIdx, scala.collection.mutable.Set()) ++= files+ }+ }+ val commVec = bridgeComms.toVector.sortBy(_._1).map(_._2.toSet)+ val reverseAssign = (for ((members, idx) <- commVec.zipWithIndex; f <- members) yield f -> idx).toMap+ LevelPartition(commVec, reverseAssign)+ }++ private[map] def adaptiveResolution(fileCount: Int): Double =+ if (fileCount < 50) 1.2+ else if (fileCount <= 500) 1.0+ else if (fileCount <= 2000) 0.8+ else if (fileCount <= 5000) 0.6+ else 0.4// ── Single Louvain pass ────────────────────────────────────────────@@ -242,6 +419,50 @@ object LouvainClustering {}}+ // ── Singleton absorption ───────────────────────────────────────────++ /**+ * After a Louvain pass, merge singleton communities into the neighbor+ * community they have the strongest total edge weight to. Nodes with+ * zero edges to any other community stay as singletons (truly isolated).+ */+ private def absorbSingletons(+ assignment: Map[NodeId, Int],+ graph: WeightedFileGraph+ ): Map[NodeId, Int] = {+ // Identify singleton communities+ val commMembers = assignment.groupBy(_._2).map { case (c, m) => c -> m.keySet }+ val singletonComms = commMembers.filter(_._2.size == 1).keySet++ if (singletonComms.isEmpty) return assignment++ val result = scala.collection.mutable.Map(assignment.toSeq: _*)++ for (comm <- singletonComms) {+ val node = commMembers(comm).head+ val neighbors = graph.adjMatrix.getOrElse(node, Map.empty)+ if (neighbors.nonEmpty) {+ // Sum edge weight to each neighboring community (excluding our own singleton)+ val weightByComm = scala.collection.mutable.Map.empty[Int, Double]+ for ((neighbor, w) <- neighbors) {+ val nc = result(neighbor)+ if (nc != comm) {+ weightByComm(nc) = weightByComm.getOrElse(nc, 0.0) + w+ }+ }+ // Merge into the community with the strongest total weight+ if (weightByComm.nonEmpty) {+ result(node) = weightByComm.maxBy(_._2)._1+ }+ }+ }++ // Re-index to keep dense community indices+ val usedComms = result.values.toVector.distinct.sorted+ val reindex = usedComms.zipWithIndex.toMap+ result.map { case (n, c) => n -> reindex(c) }.toMap+ }+// ── Projection ─────────────────────────────────────────────────────private def projectPartition(diff --git a/memory-layer/src/main/scala/ix/memory/map/MapService.scala b/memory-layer/src/main/scala/ix/memory/map/MapService.scalaindex b0c1d5fb..a0638a65 100644--- a/memory-layer/src/main/scala/ix/memory/map/MapService.scala+++ b/memory-layer/src/main/scala/ix/memory/map/MapService.scala@@ -33,8 +33,8 @@ class MapService(private val logger = Slf4jLogger.getLoggerFromName[IO]("ix.map")private val CrosscutThreshold = 0.10- // Cross-cut penalty multiplier (reduces edge weights involving crosscut files)- private val CrosscutLambda = 0.5+ // Cross-cut penalty base multiplier (reduces edge weights involving crosscut files)+ private val CrosscutLambdaBase = 0.5// High-degree threshold: files in the top N% by degree are crosscut candidatesprivate val CrosscutDegreeTop = 0.10// Path tokens that suggest cross-cutting utilities@@ -55,12 +55,13 @@ class MapService(private val preflight = new MapPreflight()private final class RetainedRegion(val level: Int, val communityIndex: Int, val region: Region)- // Label kinds by level+ // Label kinds by level — supports arbitrary depth.+ // Top level = "system", bottom level = "module", everything in between = "subsystem".private def labelKind(level: Int, maxLevel: Int): String =- if (maxLevel <= 1) "system"+ if (maxLevel <= 1) "system"else if (level == maxLevel) "system"- else if (level == 1) "module"- else "subsystem"+ else if (level == 1) "module"+ else "subsystem"def buildMap(forceRecompute: Boolean = false, forceFull: Boolean = false): IO[ArchitectureMap] =getOrBuildMap(forceRecompute, forceFull)@@ -184,15 +185,22 @@ class MapService(}}+ private def adaptiveCrosscutLambda(fileCount: Int): Double =+ if (fileCount <= 500) CrosscutLambdaBase // 0.5+ else if (fileCount <= 2000) CrosscutLambdaBase * 0.6 // 0.3+ else if (fileCount <= 5000) CrosscutLambdaBase * 0.4 // 0.2+ else CrosscutLambdaBase * 0.3 // 0.15+private def applyPenalty(graph: WeightedFileGraph,scores: Map[NodeId, Double]): WeightedFileGraph = {+ val lambda = adaptiveCrosscutLambda(graph.vertices.size)val newAdj = graph.adjMatrix.map { case (src, neighbors) =>val sCross = scores.getOrElse(src, 0.0)val adjusted = neighbors.map { case (dst, w) =>val dCross = scores.getOrElse(dst, 0.0)- val penalty = (sCross + dCross) / 2.0 * CrosscutLambda+ val penalty = (sCross + dCross) / 2.0 * lambdadst -> w * (1.0 - penalty)}src -> adjusted@@ -779,7 +787,7 @@ class MapService(crosscutScores <- IO.blocking(detectCrosscut(rawGraph))graph <- IO.blocking(applyPenalty(rawGraph, crosscutScores))t2 <- IO(System.currentTimeMillis())- levels <- IO.blocking(LouvainClustering.cluster(graph, maxLevels = 3, minCommunitySize = 3))+ levels <- IO.blocking(LouvainClustering.cluster(graph, maxLevels = 5, minCommunitySize = 3))t3 <- IO(System.currentTimeMillis())regions <- IO.blocking(buildRegions(graph, levels, crosscutScores, inputRev))edges <- IO.blocking(computeArchitectureEdges(regions, rawGraph))