Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: ix map — multi-scale architectural hierarchy from code structure

kagebinary143a20d10844
diff --git a/build.sbt b/build.sbt
index 1b129698..21817676 100644
--- a/build.sbt
+++ b/build.sbt
@@ -29,6 +29,11 @@ lazy val memoryLayer = (project in file("memory-layer"))
// ArangoDB
"com.arangodb" % "arangodb-java-driver" % "7.12.0",
+ // Tree-sitter (bonede/tree-sitter-ng — ARM64 macOS compatible)
+ "io.github.bonede" % "tree-sitter" % "0.26.6",
+ "io.github.bonede" % "tree-sitter-scala" % "0.24.0",
+ "io.github.bonede" % "tree-sitter-typescript" % "0.23.2",
+
// Logging
"org.typelevel" %% "log4cats-slf4j" % "2.7.0",
"ch.qos.logback" % "logback-classic" % "1.5.6",
diff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.ts
new file mode 100644
index 00000000..94893094
--- /dev/null
+++ b/ix-cli/src/cli/commands/map.ts
@@ -0,0 +1,140 @@
+import type { Command } from "commander";
+import chalk from "chalk";
+import { IxClient } from "../../client/api.js";
+import { getEndpoint } from "../config.js";
+
+interface MapRegion {
+ id: string;
+ label: string;
+ label_kind: string;
+ level: number;
+ file_count: number;
+ child_region_count: number;
+ parent_id: string | null;
+ cohesion: number;
+ external_coupling: number;
+ boundary_ratio: number;
+ confidence: number;
+ crosscut_score: number;
+ dominant_signals: string[];
+ interface_node_count: number;
+ children?: MapRegion[];
+}
+
+interface MapResult {
+ file_count: number;
+ region_count: number;
+ levels: number;
+ map_rev: number;
+ regions: MapRegion[];
+ hierarchy: MapRegion[];
+}
+
+export function registerMapCommand(program: Command): void {
+ program
+ .command("map")
+ .description("Infer multi-scale architectural hierarchy from code structure")
+ .option("--format <fmt>", "Output format (text|json)", "text")
+ .option("--level <n>", "Show only regions at this level (1=finest, higher=coarser)")
+ .option("--min-confidence <n>", "Only show regions above this confidence threshold (0-1)", "0")
+ .addHelpText(
+ "after",
+ `
+Runs Louvain community detection on the weighted file coupling graph to infer
+a multi-level architectural hierarchy. Persists results to the graph as Region
+nodes with IN_REGION edges.
+
+Levels:
+ 1 = module (fine-grained, ~5-20 files)
+ 2 = subsystem (mid-level, ~20-100 files)
+ 3 = system (top-level architectural regions)
+
+Examples:
+ ix map
+ ix map --format json
+ ix map --level 2
+ ix map --min-confidence 0.5`
+ )
+ .action(async (opts: { format: string; level?: string; minConfidence: string }) => {
+ const client = new IxClient(getEndpoint());
+
+ if (opts.format !== "json") {
+ process.stderr.write(chalk.dim("Computing architectural map...\n"));
+ }
+
+ let result: MapResult;
+ try {
+ result = await client.map() as MapResult;
+ } catch (err: any) {
+ console.error(chalk.red("Error:"), err.message);
+ process.exitCode = 1;
+ return;
+ }
+
+ const minConf = parseFloat(opts.minConfidence ?? "0");
+ const levelFilter = opts.level ? parseInt(opts.level, 10) : null;
+
+ let regions = result.regions;
+ if (levelFilter !== null) regions = regions.filter(r => r.level === levelFilter);
+ if (minConf > 0) regions = regions.filter(r => r.confidence >= minConf);
+
+ if (opts.format === "json") {
+ console.log(JSON.stringify({ ...result, regions }, null, 2));
+ return;
+ }
+
+ // Text output
+ console.log(
+ `\n${chalk.bold("Architectural Map")} ` +
+ chalk.dim(`rev ${result.map_rev} · ${result.file_count} files · `) +
+ chalk.dim(`${result.region_count} regions · ${result.levels} levels`)
+ );
+
+ if (regions.length === 0) {
+ console.log(chalk.dim("\n No regions found matching filters."));
+ return;
+ }
+
+ // Group by level, finest last for display (coarsest first = system overview at top)
+ const byLevel = new Map<number, MapRegion[]>();
+ for (const r of regions) {
+ if (!byLevel.has(r.level)) byLevel.set(r.level, []);
+ byLevel.get(r.level)!.push(r);
+ }
+
+ const sortedLevels = [...byLevel.keys()].sort((a, b) => b - a); // coarsest first
+
+ for (const lvl of sortedLevels) {
+ const lvlRegions = byLevel.get(lvl)!.sort((a, b) => b.confidence - a.confidence);
+ const kindLabel = lvlRegions[0]?.label_kind ?? "region";
+ console.log(`\n${chalk.bold(kindLabel.toUpperCase())} (level ${lvl})`);
+
+ for (const r of lvlRegions) {
+ const confBar = confidenceBar(r.confidence);
+ const signals = r.dominant_signals.slice(0, 2).join("+");
+ const name = chalk.bold(r.label.padEnd(20));
+ const files = chalk.dim(`${r.file_count}f`).padEnd(6);
+ const brVal = Math.min(r.boundary_ratio, 9999);
+ const br = brVal > 0
+ ? chalk.dim(`br=${brVal.toFixed(1)}`)
+ : chalk.dim("br=0.0");
+
+ console.log(
+ ` ${name} ${files} ${confBar} ${br} ${chalk.dim(signals)}`
+ );
+ }
+ }
+
+ console.log(
+ chalk.dim(`\nRegion nodes persisted to graph. Use 'ix entity <id>' to inspect a region.`)
+ );
+ });
+}
+
+/** Render a confidence score as a compact bar: ████░░ */
+function confidenceBar(conf: number): string {
+ const filled = Math.round(conf * 6);
+ const bar = "█".repeat(filled) + "░".repeat(6 - filled);
+ const color = conf >= 0.7 ? chalk.green : conf >= 0.4 ? chalk.yellow : chalk.red;
+ return color(bar);
+}
diff --git a/ix-cli/src/cli/register/oss.ts b/ix-cli/src/cli/register/oss.ts
index 6416e11b..fd6c5894 100644
--- a/ix-cli/src/cli/register/oss.ts
+++ b/ix-cli/src/cli/register/oss.ts
@@ -25,6 +25,7 @@ import { registerOverviewCommand } from "../commands/overview.js";
import { registerWatchCommand } from "../commands/watch.js";
import { registerDockerCommand } from "../commands/docker.js";
import { registerWorkflowsHelpCommand } from "../commands/workflows.js";
+import { registerMapCommand } from "../commands/map.js";
const PRO_COMMANDS: { name: string; desc: string }[] = [
{ name: "briefing", desc: "Session-resume briefing" },
@@ -76,6 +77,7 @@ export function registerOssCommands(program: Command): void {
registerWatchCommand(program);
registerDockerCommand(program);
registerWorkflowsHelpCommand(program);
+ registerMapCommand(program);
// Hide advanced commands from default help
const advancedSet = new Set(ADVANCED_COMMANDS);
diff --git a/ix-cli/src/client/api.ts b/ix-cli/src/client/api.ts
index d7a9f953..0e2e9138 100644
--- a/ix-cli/src/client/api.ts
+++ b/ix-cli/src/client/api.ts
@@ -174,6 +174,10 @@ export class IxClient {
return this.post("/v1/patch", patch);
}
+ async map(): Promise<any> {
+ return this.post("/v1/map", {});
+ }
+
async stats(): Promise<any> {
return this.get("/v1/stats");
}
diff --git a/memory-layer/src/main/scala/ix/memory/Main.scala b/memory-layer/src/main/scala/ix/memory/Main.scala
index 50133f18..1643959a 100644
--- a/memory-layer/src/main/scala/ix/memory/Main.scala
+++ b/memory-layer/src/main/scala/ix/memory/Main.scala
@@ -9,6 +9,7 @@ import ix.memory.api.Routes
import ix.memory.conflict.ConflictService
import ix.memory.context._
import ix.memory.db._
+import ix.memory.map.MapService
object Main extends IOApp.Simple {
@@ -37,9 +38,10 @@ object Main extends IOApp.Simple {
contextService = new ContextService(queryApi, seeder, expander,
claimCollector, confidenceScorer, conflictDetector)
conflictService = new ConflictService(client, queryApi, writeApi)
+ mapService = new MapService(client, queryApi, writeApi)
// 4. HTTP routes
- routes = Routes.all(contextService, queryApi, writeApi, conflictService, client)
+ routes = Routes.all(contextService, queryApi, writeApi, conflictService, client, mapService)
// 5. Server
server <- EmberServerBuilder
diff --git a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
new file mode 100644
index 00000000..d393c47d
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
@@ -0,0 +1,69 @@
+package ix.memory.api
+
+import java.nio.file.Paths
+
+import cats.effect.IO
+import io.circe.{Decoder, Encoder}
+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
+import org.http4s.HttpRoutes
+import org.http4s.circe.CirceEntityCodec._
+import org.http4s.dsl.io._
+
+import ix.memory.ingestion.{BulkIngestionService, IngestionProgress, IngestionService}
+import org.typelevel.log4cats.slf4j.Slf4jLogger
+
+case class IngestRequest(path: String, language: Option[String], recursive: Option[Boolean], force: Option[Boolean])
+
+object IngestRequest {
+ implicit val decoder: Decoder[IngestRequest] = deriveDecoder[IngestRequest]
+ implicit val encoder: Encoder[IngestRequest] = deriveEncoder[IngestRequest]
+}
+
+case class SkipReasonsResponse(unchanged: Int, emptyFile: Int, parseError: Int, tooLarge: Int)
+
+object SkipReasonsResponse {
+ implicit val encoder: Encoder[SkipReasonsResponse] = deriveEncoder[SkipReasonsResponse]
+ implicit val decoder: Decoder[SkipReasonsResponse] = deriveDecoder[SkipReasonsResponse]
+}
+
+case class IngestResponse(filesProcessed: Int, patchesApplied: Int, filesSkipped: Int,
+ entitiesCreated: Int, latestRev: Long, skipReasons: SkipReasonsResponse)
+
+object IngestResponse {
+ implicit val encoder: Encoder[IngestResponse] = deriveEncoder[IngestResponse]
+ implicit val decoder: Decoder[IngestResponse] = deriveDecoder[IngestResponse]
+}
+
+class IngestionRoutes(ingestionService: IngestionService, bulkIngestionService: BulkIngestionService) {
+
+ private val logger = Slf4jLogger.getLoggerFromName[IO]("ix.ingest.progress")
+
+ private val progressLog: IngestionProgress => IO[Unit] = { p =>
+ logger.info(s"discovered=${p.filesDiscovered} parsed=${p.filesParsed} chunks=${p.chunksWritten}/${p.totalChunks}")
+ }
+
+ val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {
+ case req @ POST -> Root / "v1" / "ingest" =>
+ (for {
+ body <- req.as[IngestRequest]
+ _ <- IO.raiseWhen(body.path.contains(".."))(
+ new IllegalArgumentException("Path traversal not allowed")
+ )
+ path = Paths.get(body.path)
+ result <- bulkIngestionService.ingestPath(path, body.language, body.recursive.getOrElse(false), progressLog, body.force.getOrElse(false))
+ resp <- Ok(IngestResponse(
+ filesProcessed = result.filesProcessed,
+ patchesApplied = result.patchesApplied,
+ filesSkipped = result.filesSkipped,
+ entitiesCreated = result.entitiesCreated,
+ latestRev = result.latestRev.value,
+ skipReasons = SkipReasonsResponse(
+ result.skipReasons.unchanged,
+ result.skipReasons.emptyFile,
+ result.skipReasons.parseError,
+ result.skipReasons.tooLarge
+ )
+ ))
+ } yield resp).handleErrorWith(ErrorHandler.handle(_))
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/api/MapRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/MapRoutes.scala
new file mode 100644
index 00000000..e4b53ed4
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/api/MapRoutes.scala
@@ -0,0 +1,80 @@
+package ix.memory.api
+
+import cats.effect.IO
+import io.circe.Json
+import io.circe.syntax._
+import org.http4s.HttpRoutes
+import org.http4s.circe.CirceEntityCodec._
+import org.http4s.dsl.io._
+
+import ix.memory.map.{ArchitectureMap, MapService, Region}
+import ix.memory.model.NodeId
+
+class MapRoutes(mapService: MapService) {
+
+ val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {
+
+ /** POST /v1/map — Run the full map pipeline and return the hierarchy. */
+ case POST -> Root / "v1" / "map" =>
+ (for {
+ archMap <- mapService.buildMap()
+ resp <- Ok(encodeMap(archMap))
+ } yield resp).handleErrorWith(ErrorHandler.handle(_))
+ }
+
+ private def encodeMap(m: ArchitectureMap): Json = {
+ val regionsByLevel = m.regions.groupBy(_.level).toVector.sortBy(_._1)
+
+ Json.obj(
+ "file_count" -> m.fileCount.asJson,
+ "region_count" -> m.regions.size.asJson,
+ "levels" -> regionsByLevel.size.asJson,
+ "map_rev" -> m.mapRev.asJson,
+ "regions" -> m.regions.sortBy(r => (r.level, r.label)).map(encodeRegion).asJson,
+ "hierarchy" -> buildHierarchyTree(m).asJson
+ )
+ }
+
+ private def encodeRegion(r: Region): Json =
+ Json.obj(
+ "id" -> r.id.value.toString.asJson,
+ "label" -> r.label.asJson,
+ "label_kind" -> r.labelKind.asJson,
+ "level" -> r.level.asJson,
+ "file_count" -> r.memberFiles.size.asJson,
+ "child_region_count" -> r.childRegionIds.size.asJson,
+ "parent_id" -> r.parentId.map(_.value.toString).asJson,
+ "cohesion" -> r.cohesion.asJson,
+ "external_coupling" -> r.externalCoupling.asJson,
+ "boundary_ratio" -> r.boundaryRatio.asJson,
+ "confidence" -> r.confidence.asJson,
+ "crosscut_score" -> r.crosscutScore.asJson,
+ "dominant_signals" -> r.dominantSignals.asJson,
+ "interface_node_count" -> r.interfaceNodeCount.asJson
+ )
+
+ /**
+ * Build a tree structure: top-level regions contain nested children.
+ * Returns a JSON array of root regions (no parent), each with a recursive
+ * `children` field.
+ */
+ private def buildHierarchyTree(m: ArchitectureMap): Json = {
+ val regionById: Map[NodeId, Region] = m.regions.map(r => r.id -> r).toMap
+
+ // Guard against cycles (can arise when Louvain levels have identical partitions)
+ def renderNode(r: Region, visited: Set[NodeId]): Json = {
+ val children =
+ if (visited.contains(r.id)) Vector.empty
+ else r.childRegionIds.toVector
+ .flatMap(cid => regionById.get(cid))
+ .filter(c => c.level < r.level) // strict level ordering: child must be finer
+ .sortBy(_.label)
+ .map(c => renderNode(c, visited + r.id))
+
+ encodeRegion(r).deepMerge(Json.obj("children" -> children.asJson))
+ }
+
+ val roots = m.regions.filter(_.parentId.isEmpty).sortBy(-_.level)
+ roots.map(r => renderNode(r, Set.empty)).asJson
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/api/Routes.scala b/memory-layer/src/main/scala/ix/memory/api/Routes.scala
index 8ac1c387..f2dd63db 100644
--- a/memory-layer/src/main/scala/ix/memory/api/Routes.scala
+++ b/memory-layer/src/main/scala/ix/memory/api/Routes.scala
@@ -11,6 +11,7 @@ import org.http4s.dsl.io._
import ix.memory.conflict.ConflictService
import ix.memory.context.ContextService
import ix.memory.db.{ArangoClient, GraphQueryApi, GraphWriteApi}
+import ix.memory.map.MapService
object Routes {
@@ -19,7 +20,8 @@ object Routes {
queryApi: GraphQueryApi,
writeApi: GraphWriteApi,
conflictService: ConflictService,
- client: ArangoClient
+ client: ArangoClient,
+ mapService: MapService
): HttpRoutes[IO] = {
val health = HttpRoutes.of[IO] {
@@ -41,10 +43,12 @@ object Routes {
val statsRoutes = new StatsRoutes(client).routes
val patchCommitRoutes = new PatchCommitRoutes(writeApi).routes
val listRoutes = new ListRoutes(queryApi).routes
+ val mapRoutes = new MapRoutes(mapService).routes
health <+> contextRoutes <+> entityRoutes <+>
diffRoutes <+> conflictRoutes <+> decideRoutes <+> searchRoutes <+>
truthRoutes <+> patchRoutes <+> decisionListRoutes <+> expandRoutes <+>
- statsRoutes <+> patchCommitRoutes <+> listRoutes <+> goalRoutes
+ statsRoutes <+> patchCommitRoutes <+> listRoutes <+> goalRoutes <+>
+ mapRoutes
}
}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
new file mode 100644
index 00000000..3d685cf8
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
@@ -0,0 +1,183 @@
+package ix.memory.ingestion
+
+import java.nio.file.{Files, Path}
+
+import cats.effect.IO
+import cats.syntax.all._
+import cats.effect.implicits._
+import io.circe.Json
+import io.circe.syntax._
+import org.typelevel.log4cats.slf4j.Slf4jLogger
+import org.typelevel.log4cats.Logger
+
+import ix.memory.db.{BulkWriteApi, CommitResult, CommitStatus, FileBatch, GraphQueryApi}
+import ix.memory.model._
+
+private[ingestion] sealed trait ParseOutcome
+private[ingestion] case class Parsed(batch: FileBatch) extends ParseOutcome
+private[ingestion] case class Skipped(reason: String) extends ParseOutcome
+
+/**
+ * High-performance ingestion service that:
+ * 1. Discovers files (with optional language filter)
+ * 2. Parses files in parallel (bounded by CPU cores)
+ * 3. Batches parsed results
+ * 4. Bulk-inserts via Document API
+ */
+class BulkIngestionService(
+ parserRouter: ParserRouter,
+ bulkWriteApi: BulkWriteApi,
+ queryApi: GraphQueryApi
+) {
+
+ private val parallelism = Runtime.getRuntime.availableProcessors.max(2)
+ private val MaxFileBytes: Long = 1024 * 1024 // 1 MB
+ private val mapper = new com.fasterxml.jackson.databind.ObjectMapper()
+ private val logger: Logger[IO] = Slf4jLogger.getLoggerFromName[IO]("ix.ingestion")
+
+ /**
+ * Ingest files under a path using parallel parsing and bulk writes.
+ */
+ def ingestPath(
+ path: Path,
+ language: Option[String],
+ recursive: Boolean,
+ onProgress: IngestionProgress => IO[Unit] = _ => IO.unit,
+ force: Boolean = false
+ ): IO[IngestionResult] = {
+ for {
+ files <- discoverFiles(path, language, recursive)
+ _ <- onProgress(IngestionProgress(files.size, 0, 0, 0))
+ hashMap <- if (force) IO.pure(Map.empty[String, String]) else loadExistingHashes(files)
+ outcomes <- files.parTraverseN(parallelism) { f =>
+ parseFile(f, hashMap).handleErrorWith { err =>
+ logger.warn(s"Parse error: ${f.toString} — ${err.getMessage}") *>
+ IO.pure(Skipped("parseError"): ParseOutcome)
+ }
+ }
+ validBatches = outcomes.collect { case Parsed(b) => b }
+ unchangedCount = outcomes.count { case Skipped("unchanged") => true; case _ => false }
+ emptyCount = outcomes.count { case Skipped("emptyFile") => true; case _ => false }
+ errorCount = outcomes.count { case Skipped("parseError") => true; case _ => false }
+ tooLargeCount = outcomes.count { case Skipped("tooLarge") => true; case _ => false }
+ skippedCount = unchangedCount + emptyCount + errorCount + tooLargeCount
+ totalChunks = if (validBatches.isEmpty) 0 else (validBatches.size + 99) / 100
+ _ <- onProgress(IngestionProgress(files.size, outcomes.size, 0, totalChunks))
+ latestRev <- queryApi.getLatestRev
+ result <- if (validBatches.isEmpty) IO.pure(CommitResult(latestRev, CommitStatus.Ok))
+ else commitChunkedWithProgress(validBatches.toVector, latestRev.value, onProgress, files.size, outcomes.size)
+ } yield IngestionResult(
+ filesProcessed = files.size,
+ patchesApplied = validBatches.size,
+ filesSkipped = skippedCount,
+ entitiesCreated = validBatches.flatMap(_.patch.ops.collect { case _: PatchOp.UpsertNode => 1 }).size,
+ latestRev = result.newRev,
+ skipReasons = SkipReasons(unchanged = unchangedCount, emptyFile = emptyCount, parseError = errorCount, tooLarge = tooLargeCount)
+ )
+ }
+
+ private def commitChunkedWithProgress(
+ batches: Vector[FileBatch],
+ baseRev: Long,
+ onProgress: IngestionProgress => IO[Unit],
+ discovered: Int,
+ parsed: Int
+ ): IO[CommitResult] = {
+ val chunkSize = 100
+ val chunks = batches.grouped(chunkSize).toVector
+ val totalChunks = chunks.size
+
+ chunks.zipWithIndex.foldLeft(IO.pure(baseRev)) { case (revIO, (chunk, idx)) =>
+ revIO.flatMap { currentRev =>
+ bulkWriteApi.commitBatch(chunk, currentRev).flatMap { result =>
+ onProgress(IngestionProgress(discovered, parsed, idx + 1, totalChunks)) *>
+ IO.pure(result.newRev.value)
+ }
+ }
+ }.map(finalRev => CommitResult(Rev(finalRev), CommitStatus.Ok))
+ }
+
+ /**
+ * Parse a single file into a FileBatch (or None if unchanged).
+ */
+ private def parseFile(filePath: Path, hashMap: Map[String, String]): IO[ParseOutcome] = {
+ for {
+ fileSize <- IO.blocking(Files.size(filePath))
+ result <- if (fileSize > MaxFileBytes) IO.pure(Skipped("tooLarge"): ParseOutcome)
+ else if (fileSize == 0L) IO.pure(Skipped("emptyFile"): ParseOutcome)
+ else {
+ for {
+ bytes <- IO.blocking(Files.readAllBytes(filePath))
+ hash = sha256Bytes(bytes)
+ outcome <- hashMap.get(filePath.toString) match {
+ case Some(existingHash) if existingHash == hash =>
+ IO.pure(Skipped("unchanged"): ParseOutcome)
+ case _ =>
+ for {
+ source <- IO.blocking {
+ try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)
+ catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }
+ }
+ parserOpt = parserRouter.parserFor(filePath.toString)
+ parseResult = parserOpt match {
+ case Some(p) => p.parse(filePath.getFileName.toString, source)
+ case None => genericTextParse(filePath.getFileName.toString, source)
+ }
+ patch = GraphPatchBuilder.build(filePath.toString, Some(hash), parseResult)
+ _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))
+ provenance = buildProvenanceMap(patch)
+ } yield Parsed(FileBatch(filePath.toString, Some(hash), patch, provenance)): ParseOutcome
+ }
+ } yield outcome
+ }
+ } yield result
+ }
+
+ private def loadExistingHashes(files: List[Path]): IO[Map[String, String]] = {
+ val paths = files.map(_.toString)
+ queryApi.getSourceHashes(paths)
+ }
+
+ private def buildProvenanceMap(patch: GraphPatch): java.util.Map[String, AnyRef] = {
+ val json = Json.obj(
+ "source_uri" -> Json.fromString(patch.source.uri),
+ "source_hash" -> patch.source.sourceHash.fold(Json.Null)(Json.fromString),
+ "extractor" -> Json.fromString(patch.source.extractor),
+ "source_type" -> Json.fromString(patch.source.sourceType.asJson.asString.getOrElse("code")),
+ "observed_at" -> Json.fromString(patch.timestamp.toString)
+ )
+ mapper.readValue(json.noSpaces, classOf[java.util.Map[String, AnyRef]])
+ }
+
+ private def discoverFiles(
+ path: Path,
+ language: Option[String],
+ recursive: Boolean
+ ): IO[List[Path]] = IO.blocking {
+ val extensions = FileDiscovery.extensionsFor(language)
+ if (Files.isRegularFile(path)) {
+ if (FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty
+ } else if (Files.isDirectory(path)) {
+ // Try git ls-files first (respects .gitignore), fall back to manual walk
+ FileDiscovery.tryGitLsFiles(path, recursive, extensions)
+ .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))
+ } else List.empty
+ }
+
+ private def sha256Bytes(bytes: Array[Byte]): String = {
+ val md = java.security.MessageDigest.getInstance("SHA-256")
+ md.digest(bytes).map("%02x".format(_)).mkString
+ }
+
+ private def genericTextParse(fileName: String, source: String): ParseResult = {
+ val lines = if (source.isEmpty) 1 else source.count(_ == '\n') + 1
+ ParseResult(
+ entities = Vector(ParsedEntity(
+ name = fileName, kind = NodeKind.File,
+ attrs = Map("content" -> Json.fromString(source)),
+ lineStart = 1, lineEnd = lines
+ )),
+ relationships = Vector.empty
+ )
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala b/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala
new file mode 100644
index 00000000..2a20186a
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala
@@ -0,0 +1,142 @@
+package ix.memory.ingestion
+
+import java.nio.file.{Files, Path}
+
+/**
+ * Discovers source files for ingestion.
+ *
+ * Primary strategy: `git ls-files` — respects .gitignore, fast, works for any repo.
+ * Fallback: manual file walk with hardcoded ignore list (for non-git directories).
+ */
+object FileDiscovery {
+
+ private val defaultExtensions = Set(
+ // code
+ ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",
+ ".java", ".scala", ".sc", ".go", ".rs", ".c", ".h", ".cc", ".cpp", ".hpp",
+ ".kt", ".kts", ".swift", ".rb", ".php", ".cs", ".ex", ".exs",
+ ".lua", ".zig", ".nim", ".jl", ".r", ".R", ".pl", ".pm",
+ ".sh", ".bash", ".zsh", ".fish",
+
+ // config / data
+ ".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".env", ".properties",
+ ".xml", ".plist", ".gradle",
+
+ // docs
+ ".md", ".mdx", ".rst", ".txt",
+
+ // build
+ ".sbt", ".gradle", ".cmake"
+ )
+
+ /** Files matched by name regardless of extension. */
+ private val specialFiles = Set(
+ "Dockerfile", "Makefile", "Rakefile", "Gemfile", "Procfile",
+ "README", "LICENSE", "NOTICE", "CHANGELOG",
+ "Cargo.toml", "go.mod", "go.sum",
+ "package.json", "tsconfig.json", "pyproject.toml", "setup.py", "setup.cfg",
+ "build.sbt", "build.gradle", "pom.xml", "CMakeLists.txt",
+ ".gitignore", ".dockerignore", ".editorconfig"
+ )
+
+ def extensionsFor(language: Option[String]): Set[String] = language match {
+ case Some("python") => Set(".py")
+ case Some("typescript") => Set(".ts", ".tsx")
+ case Some("javascript") => Set(".js", ".jsx", ".mjs", ".cjs")
+ case Some("scala") => Set(".scala", ".sc")
+ case Some("java") => Set(".java")
+ case Some("go") => Set(".go")
+ case Some("rust") => Set(".rs")
+ case Some("ruby") => Set(".rb")
+ case Some("config") => Set(".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".properties", ".xml")
+ case Some("markdown") => Set(".md", ".mdx")
+ case _ => defaultExtensions
+ }
+
+ def matchesFilter(file: Path, extensions: Set[String]): Boolean = {
+ val name = file.getFileName.toString
+ specialFiles.contains(name) || extensions.exists(name.endsWith)
+ }
+
+ /**
+ * Use `git ls-files` to discover tracked files. Returns None if not a git repo
+ * or git is not available.
+ */
+ def tryGitLsFiles(dir: Path, recursive: Boolean, extensions: Set[String]): Option[List[Path]] = {
+ try {
+ val pb = new ProcessBuilder("git", "ls-files", "--cached", "--others", "--exclude-standard")
+ pb.directory(dir.toFile)
+ pb.redirectErrorStream(true)
+ val proc = pb.start()
+ val reader = new java.io.BufferedReader(
+ new java.io.InputStreamReader(proc.getInputStream, java.nio.charset.StandardCharsets.UTF_8)
+ )
+ val result = scala.collection.mutable.ListBuffer.empty[Path]
+
+ try {
+ var line = reader.readLine()
+ while (line != null) {
+ if (line.nonEmpty) {
+ val p = dir.resolve(line).normalize()
+ val isUnder = if (recursive) true else p.getParent == dir
+ if (isUnder && Files.isRegularFile(p) && matchesFilter(p, extensions)) {
+ result += p
+ }
+ }
+ line = reader.readLine()
+ }
+ } finally {
+ reader.close()
+ }
+
+ val exitCode = proc.waitFor()
+ if (exitCode != 0) None else Some(result.toList)
+ } catch {
+ case _: Throwable => None
+ }
+ }
+
+ /**
+ * Fallback: walk the directory tree, skipping common non-source directories.
+ */
+ def walkFiles(dir: Path, recursive: Boolean, extensions: Set[String]): List[Path] = {
+ import scala.jdk.CollectionConverters._
+
+ if (!recursive) {
+ val stream = Files.list(dir)
+ try {
+ stream.iterator().asScala
+ .filter(Files.isRegularFile(_))
+ .filter(matchesFilter(_, extensions))
+ .toList
+ } finally stream.close()
+ } else {
+ // Directories to skip when not using git
+ val ignoredDirs = Set(
+ "node_modules", ".git", ".hg", ".svn",
+ "target", "dist", "build", "out", ".next",
+ "__pycache__", ".tox", ".venv", "venv", ".mypy_cache", ".pytest_cache",
+ ".gradle", ".idea", ".vscode", ".settings", ".vs",
+ ".cache", ".parcel-cache", "coverage", ".nyc_output",
+ "vendor", "Pods", ".dart_tool", ".pub-cache",
+ "bin", "obj", "pkg"
+ )
+ val result = scala.collection.mutable.ListBuffer.empty[Path]
+ Files.walkFileTree(dir, new java.nio.file.SimpleFileVisitor[Path] {
+ override def preVisitDirectory(d: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
+ val name = d.getFileName.toString
+ if (d != dir && (ignoredDirs.contains(name) || name.startsWith(".")))
+ java.nio.file.FileVisitResult.SKIP_SUBTREE
+ else
+ java.nio.file.FileVisitResult.CONTINUE
+ }
+ override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
+ if (matchesFilter(file, extensions))
+ result += file
+ java.nio.file.FileVisitResult.CONTINUE
+ }
+ })
+ result.toList
+ }
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala b/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala
new file mode 100644
index 00000000..8a7c4baf
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala
@@ -0,0 +1,20 @@
+package ix.memory.ingestion
+
+import java.security.MessageDigest
+
+object Fingerprint {
+
+ /** Compute a normalized content fingerprint from signature + body.
+ * Returns a 32-char hex string (first 16 bytes of SHA-256).
+ */
+ def compute(signature: String, body: String): String = {
+ val normalized = (signature.trim + "\n" + body.trim)
+ .replaceAll("\\s+", " ")
+ .trim
+ val digest = MessageDigest.getInstance("SHA-256")
+ digest.digest(normalized.getBytes("UTF-8"))
+ .take(16)
+ .map("%02x".format(_))
+ .mkString
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
new file mode 100644
index 00000000..c11587ef
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
@@ -0,0 +1,99 @@
+package ix.memory.ingestion
+
+import java.time.Instant
+import java.util.UUID
+
+import io.circe.Json
+
+import ix.memory.model._
+
+object GraphPatchBuilder {
+
+ /**
+ * Build a GraphPatch from a ParseResult and file metadata.
+ *
+ * Uses deterministic UUID generation (UUID v3 (MD5-based nameUUID)) so that
+ * re-parsing the same file produces the same node/edge IDs, enabling
+ * idempotent upserts.
+ */
+ def build(
+ filePath: String,
+ sourceHash: Option[String],
+ parseResult: ParseResult
+ ): GraphPatch = {
+
+ def nodeIdFor(name: String): NodeId =
+ NodeId(UUID.nameUUIDFromBytes(s"$filePath:$name".getBytes("UTF-8")))
+
+ def edgeIdFor(src: String, dst: String, predicate: String): EdgeId =
+ EdgeId(UUID.nameUUIDFromBytes(s"$filePath:$src:$dst:$predicate".getBytes("UTF-8")))
+
+ val extractor = if (filePath.endsWith(".py")) "tree-sitter-python/1.0"
+ else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) "typescript-parser/1.0"
+ else if (filePath.endsWith(".scala") || filePath.endsWith(".sc")) "scala-parser/1.0"
+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) "config-parser/1.0"
+ else if (filePath.endsWith(".md")) "markdown-parser/1.0"
+ else "unknown-parser/1.0"
+
+ val nodeOps = parseResult.entities.map { e =>
+ val baseAttrs = e.attrs +
+ ("line_start" -> Json.fromInt(e.lineStart)) +
+ ("line_end" -> Json.fromInt(e.lineEnd))
+ val attrsWithFp = e.contentFingerprint match {
+ case Some(fp) => baseAttrs + ("content_fingerprint" -> Json.fromString(fp))
+ case None => baseAttrs
+ }
+ PatchOp.UpsertNode(nodeIdFor(e.name), e.kind, e.name, attrsWithFp)
+ }
+
+ val edgeOps = parseResult.relationships.map { r =>
+ PatchOp.UpsertEdge(
+ edgeIdFor(r.srcName, r.dstName, r.predicate),
+ nodeIdFor(r.srcName),
+ nodeIdFor(r.dstName),
+ EdgePredicate(r.predicate),
+ Map.empty
+ )
+ }
+
+ // Generate claims so the confidence/conflict engine has data to operate on.
+ val relationshipClaims = parseResult.relationships.map { r =>
+ PatchOp.AssertClaim(
+ entityId = nodeIdFor(r.srcName),
+ field = s"${r.predicate.toLowerCase}:${r.dstName}",
+ value = Json.fromString(r.dstName),
+ confidence = None // will be computed at query time
+ )
+ }
+
+ val attributeClaims = parseResult.entities.flatMap { e =>
+ e.attrs.flatMap { case (key, value) =>
+ Vector(PatchOp.AssertClaim(
+ entityId = nodeIdFor(e.name),
+ field = key,
+ value = value,
+ confidence = None
+ ))
+ }
+ }
+
+ val claimOps = relationshipClaims ++ attributeClaims
+ val allOps = nodeOps ++ edgeOps ++ claimOps
+
+ val sourceType = if (filePath.endsWith(".py") || filePath.endsWith(".ts") || filePath.endsWith(".tsx") || filePath.endsWith(".scala") || filePath.endsWith(".sc")) SourceType.Code
+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) SourceType.Config
+ else if (filePath.endsWith(".md")) SourceType.Doc
+ else SourceType.Code
+
+ GraphPatch(
+ patchId = PatchId(UUID.randomUUID()),
+ actor = "ix/ingestion",
+ timestamp = Instant.now(),
+ source = PatchSource(filePath, sourceHash, extractor, sourceType),
+ baseRev = Rev(0L),
+ ops = allOps,
+ replaces = Vector.empty,
+ intent = Some(s"Parsed $filePath")
+ )
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
new file mode 100644
index 00000000..ea66a6bb
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
@@ -0,0 +1,277 @@
+package ix.memory.ingestion
+
+import java.nio.file.{Files, Path}
+import java.time.Instant
+import java.util.UUID
+
+import cats.effect.IO
+import cats.syntax.all._
+import ix.memory.db.{CommitResult, GraphQueryApi, GraphWriteApi}
+import ix.memory.model._
+import io.circe.Json
+
+/**
+ * Orchestrates file ingestion: reads source files, parses them,
+ * builds graph patches, and commits them through the write API.
+ */
+class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, queryApi: GraphQueryApi) {
+
+ /**
+ * Ingest a single file: read, parse, build patch, commit.
+ */
+ def ingestFile(filePath: Path): IO[CommitResult] = {
+ for {
+ bytes <- IO.blocking(Files.readAllBytes(filePath))
+ _ <- if (bytes.isEmpty) IO.raiseError(new RuntimeException(s"Skipping empty file: $filePath"))
+ else IO.unit
+ // Hash raw bytes so revisions are stable across encodings
+ hash = Some(sha256Bytes(bytes))
+ // Decode text with UTF-8, fallback to ISO-8859-1 to avoid hard failures
+ source <- IO.blocking {
+ try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)
+ catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }
+ }
+ // Use the language-specific parser when available; otherwise fall back to generic text
+ parserOpt = parserRouter.parserFor(filePath.toString)
+ result = parserOpt match {
+ case Some(p) => p.parse(filePath.getFileName.toString, source)
+ case None => genericTextParse(filePath.getFileName.toString, source)
+ }
+ val fp = filePath.toString
+ extractor = if (fp.endsWith(".py")) "tree-sitter-python/1.0"
+ else if (fp.endsWith(".ts") || fp.endsWith(".tsx")) "typescript-parser/1.0"
+ else if (fp.endsWith(".scala") || fp.endsWith(".sc")) "scala-parser/1.0"
+ else if (fp.endsWith(".json") || fp.endsWith(".yaml") || fp.endsWith(".yml") || fp.endsWith(".toml") || fp.endsWith(".ini") || fp.endsWith(".conf") || fp.endsWith(".properties") || fp.endsWith(".env")) "config-parser/1.0"
+ else if (fp.endsWith(".md") || fp.endsWith(".mdx") || fp.endsWith(".rst") || fp.endsWith(".txt")) "markdown-parser/1.0"
+ else "text-parser/1.0"
+ existing <- queryApi.getPatchesBySource(filePath.toString, extractor)
+ prevPatch = existing.headOption
+ prevHash = prevPatch.flatMap(_.hcursor.downField("data").downField("source").get[String]("sourceHash").toOption)
+ // If hash unchanged, skip (idempotent)
+ _ <- if (prevHash == hash) IO.raiseError(new RuntimeException("Idempotent — file unchanged"))
+ else IO.unit
+ prevPatchId = prevPatch.flatMap(_.hcursor.downField("data").get[String]("patchId").toOption)
+ .flatMap(s => scala.util.Try(java.util.UUID.fromString(s)).toOption)
+ .map(PatchId(_))
+ replaces = prevPatchId.toVector
+ patch = GraphPatchBuilder.build(filePath.toString, hash, result).copy(replaces = replaces)
+ _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))
+ commit <- writeApi.commitPatch(patch)
+ // Resolve cross-file import edges after main patch is committed
+ _ <- resolveImportEdges(filePath, result, hash).attempt.void
+ // Resolve named import symbols to their actual source entities
+ _ <- resolveNamedImports(filePath, result, hash).attempt.void
+ } yield commit
+ }
+
+ /**
+ * Ingest all matching files under a path.
+ *
+ * @param path root directory or single file
+ * @param language optional language filter (e.g. "python")
+ * @param recursive whether to recurse into subdirectories
+ * @return aggregated ingestion result
+ */
+ def ingestPath(
+ path: Path,
+ language: Option[String],
+ recursive: Boolean
+ ): IO[IngestionResult] = {
+ for {
+ files <- discoverFiles(path, language, recursive)
+ results <- files.traverse(f => ingestFile(f).attempt)
+ successes = results.collect { case Right(cr) => cr }
+ idempotentCount = results.count {
+ case Left(e: RuntimeException) if e.getMessage.startsWith("Idempotent") => true
+ case _ => false
+ }
+ latestRev = successes.lastOption.map(_.newRev).getOrElse(Rev(0L))
+ } yield IngestionResult(
+ filesProcessed = files.size,
+ patchesApplied = successes.size,
+ filesSkipped = idempotentCount,
+ entitiesCreated = 0, // Would need to count from parse results
+ latestRev = latestRev
+ )
+ }
+
+ private def discoverFiles(
+ path: Path,
+ language: Option[String],
+ recursive: Boolean
+ ): IO[List[Path]] = IO.blocking {
+ val extensions = FileDiscovery.extensionsFor(language)
+ if (Files.isRegularFile(path)) {
+ if (FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty
+ } else if (Files.isDirectory(path)) {
+ FileDiscovery.tryGitLsFiles(path, recursive, extensions)
+ .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))
+ } else {
+ List.empty
+ }
+ }
+
+ private def resolveImportEdges(
+ filePath: Path,
+ parseResult: ParseResult,
+ sourceHash: Option[String]
+ ): IO[Unit] = {
+ val importRels = parseResult.relationships.filter(_.predicate == "IMPORTS")
+ importRels.traverse_ { rel =>
+ val targetName = rel.dstName
+ // Try to resolve relative imports (./config, ../utils)
+ val resolvedPath = resolveRelativePath(filePath, targetName)
+ resolvedPath match {
+ case Some(resolved) =>
+ queryApi.searchNodes(resolved.getFileName.toString, limit = 1).flatMap {
+ case nodes if nodes.nonEmpty =>
+ val srcId = NodeId(UUID.nameUUIDFromBytes(s"${filePath}:${filePath.getFileName}".getBytes("UTF-8")))
+ val dstId = nodes.head.id
+ val edgeId = EdgeId(UUID.nameUUIDFromBytes(s"${srcId.value}:${dstId.value}:IMPORTS".getBytes("UTF-8")))
+ val patch = GraphPatch(
+ patchId = PatchId(UUID.randomUUID()),
+ actor = "ix/cross-file",
+ timestamp = Instant.now(),
+ source = PatchSource(filePath.toString, sourceHash, "cross-file-resolver/1.0", SourceType.Code),
+ baseRev = Rev(0L),
+ ops = Vector(PatchOp.UpsertEdge(edgeId, srcId, dstId, EdgePredicate("IMPORTS"), Map.empty)),
+ replaces = Vector.empty,
+ intent = Some(s"Cross-file import: ${filePath.getFileName} -> ${resolved.getFileName}")
+ )
+ writeApi.commitPatch(patch).void
+ case _ => IO.unit
+ }
+ case None => IO.unit
+ }
+ }
+ }
+
+ private def resolveNamedImports(
+ filePath: Path,
+ parseResult: ParseResult,
+ sourceHash: Option[String]
+ ): IO[Unit] = {
+ val namedImports = parseResult.relationships.filter { rel =>
+ rel.predicate == "IMPORTS" && !rel.dstName.contains("/") && !rel.dstName.contains(".")
+ }
+ namedImports.traverse_ { rel =>
+ queryApi.searchNodes(rel.dstName, limit = 5, nameOnly = true).flatMap { nodes =>
+ val external = nodes.filter { n =>
+ n.name == rel.dstName &&
+ n.provenance.sourceUri != filePath.toString
+ }
+ external.headOption match {
+ case Some(target) =>
+ val srcId = NodeId(UUID.nameUUIDFromBytes(
+ s"${filePath}:${filePath.getFileName}".getBytes("UTF-8")))
+ val edgeId = EdgeId(UUID.nameUUIDFromBytes(
+ s"${srcId.value}:${target.id.value}:IMPORTS".getBytes("UTF-8")))
+ val patch = GraphPatch(
+ patchId = PatchId(UUID.randomUUID()),
+ actor = "ix/cross-file",
+ timestamp = Instant.now(),
+ source = PatchSource(filePath.toString, sourceHash,
+ "cross-file-resolver/1.0", SourceType.Code),
+ baseRev = Rev(0L),
+ ops = Vector(PatchOp.UpsertEdge(
+ edgeId, srcId, target.id, EdgePredicate("IMPORTS"), Map.empty)),
+ replaces = Vector.empty,
+ intent = Some(s"Named import: ${filePath.getFileName} -> ${rel.dstName}")
+ )
+ writeApi.commitPatch(patch).void
+ case None => IO.unit
+ }
+ }
+ }
+ }
+
+ private def resolveRelativePath(from: Path, importTarget: String): Option[Path] = {
+ if (importTarget.startsWith("./") || importTarget.startsWith("../")) {
+ val base = from.getParent
+ if (base == null) return None
+
+ // 1. Try the import target as-is and with source extensions appended
+ val directExtensions = List("", ".ts", ".tsx", ".scala", ".py", ".js")
+ val directMatch = directExtensions.collectFirst {
+ case ext =>
+ val candidate = base.resolve(importTarget + ext).normalize()
+ if (Files.exists(candidate)) candidate else null
+ }.flatMap(Option(_))
+ if (directMatch.isDefined) return directMatch
+
+ // 2. Strip compiled extensions (.js/.mjs/.cjs) and try source extensions
+ val strippable = List(".js", ".mjs", ".cjs")
+ val stripped = strippable.collectFirst {
+ case ext if importTarget.endsWith(ext) =>
+ importTarget.stripSuffix(ext)
+ }
+ val strippedMatch = stripped.flatMap { baseName =>
+ val sourceExtensions = List(".ts", ".tsx")
+ sourceExtensions.collectFirst {
+ case ext =>
+ val candidate = base.resolve(baseName + ext).normalize()
+ if (Files.exists(candidate)) candidate else null
+ }.flatMap(Option(_))
+ }
+ if (strippedMatch.isDefined) return strippedMatch
+
+ // 3. Try directory index files (e.g., ./utils -> ./utils/index.ts)
+ val indexExtensions = List("index.ts", "index.tsx", "index.js")
+ indexExtensions.collectFirst {
+ case idx =>
+ val candidate = base.resolve(importTarget).resolve(idx).normalize()
+ if (Files.exists(candidate)) candidate else null
+ }.flatMap(Option(_))
+ } else None
+ }
+
+ private def sha256Bytes(bytes: Array[Byte]): String = {
+ val md = java.security.MessageDigest.getInstance("SHA-256")
+ md.digest(bytes).map("%02x".format(_)).mkString
+ }
+
+ /**
+ * Generic fallback parse: ensures every text file produces at least a File entity with content.
+ * This avoids needing a bespoke parser for every dev/text extension.
+ */
+ private def genericTextParse(fileName: String, source: String): ParseResult = {
+ val lines = if (source.isEmpty) 1 else source.count(_ == '\n') + 1
+ ParseResult(
+ entities = Vector(
+ ParsedEntity(
+ name = fileName,
+ kind = NodeKind.File,
+ attrs = Map(
+ "content" -> Json.fromString(source)
+ ),
+ lineStart = 1,
+ lineEnd = lines
+ )
+ ),
+ relationships = Vector.empty
+ )
+ }
+}
+
+case class IngestionProgress(
+ filesDiscovered: Int,
+ filesParsed: Int,
+ chunksWritten: Int,
+ totalChunks: Int
+)
+
+case class SkipReasons(
+ unchanged: Int = 0,
+ emptyFile: Int = 0,
+ parseError: Int = 0,
+ tooLarge: Int = 0
+)
+
+case class IngestionResult(
+ filesProcessed: Int,
+ patchesApplied: Int,
+ filesSkipped: Int,
+ entitiesCreated: Int,
+ latestRev: Rev,
+ skipReasons: SkipReasons = SkipReasons()
+)
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala
new file mode 100644
index 00000000..43ba1269
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala
@@ -0,0 +1,24 @@
+package ix.memory.ingestion
+
+import ix.memory.model.NodeKind
+import io.circe.Json
+
+case class ParsedEntity(
+ name: String,
+ kind: NodeKind,
+ attrs: Map[String, Json],
+ lineStart: Int,
+ lineEnd: Int,
+ contentFingerprint: Option[String] = None
+)
+
+case class ParsedRelationship(
+ srcName: String,
+ dstName: String,
+ predicate: String
+)
+
+case class ParseResult(
+ entities: Vector[ParsedEntity],
+ relationships: Vector[ParsedRelationship]
+)
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala
new file mode 100644
index 00000000..cec94982
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala
@@ -0,0 +1,9 @@
+package ix.memory.ingestion
+
+/**
+ * Common trait for language-specific source code parsers.
+ * Both TreeSitterPythonParser and TypeScriptParser implement this.
+ */
+trait Parser {
+ def parse(fileName: String, source: String): ParseResult
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
new file mode 100644
index 00000000..a5d24632
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
@@ -0,0 +1,25 @@
+package ix.memory.ingestion
+
+import ix.memory.ingestion.parsers.{ConfigParser, MarkdownParser, ScalaParser, TreeSitterPythonParser, TypeScriptParser}
+
+/**
+ * Routes file paths to the appropriate language parser.
+ * Supports Python (.py), TypeScript (.ts/.tsx), config files (.json/.yaml/.yml/.toml),
+ * and Markdown (.md).
+ */
+class ParserRouter {
+ private val pythonParser = new TreeSitterPythonParser()
+ private val tsParser = new TypeScriptParser()
+ private val configParser = new ConfigParser()
+ private val markdownParser = new MarkdownParser()
+ private val scalaParser = new ScalaParser()
+
+ def parserFor(filePath: String): Option[Parser] = {
+ if (filePath.endsWith(".py")) Some(pythonParser)
+ else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) Some(tsParser)
+ else if (filePath.endsWith(".scala") || filePath.endsWith(".sc")) Some(scalaParser)
+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) Some(configParser)
+ else if (filePath.endsWith(".md")) Some(markdownParser)
+ else None
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala b/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala
new file mode 100644
index 00000000..f2823309
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala
@@ -0,0 +1,16 @@
+package ix.memory.ingestion
+
+import ix.memory.model.GraphPatch
+
+/**
+ * Simple validation for graph patches before committing.
+ */
+object PatchValidator {
+
+ def validate(patch: GraphPatch): Either[String, GraphPatch] = {
+ if (patch.ops.isEmpty)
+ Left("Patch has no operations")
+ else
+ Right(patch)
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
new file mode 100644
index 00000000..bd396ed7
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
@@ -0,0 +1,190 @@
+package ix.memory.ingestion.parsers
+
+import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}
+import ix.memory.model.NodeKind
+import io.circe.Json
+
+/**
+ * Parser for configuration files (JSON, YAML, TOML).
+ *
+ * Flattens nested structures into dot-path entries so that each leaf value
+ * becomes a ConfigEntry node with a meaningful value (never "{" or "[").
+ */
+class ConfigParser extends Parser {
+
+ private val JsonExtensions = Set(".json")
+ private val YamlExtensions = Set(".yaml", ".yml")
+ private val TomlExtensions = Set(".toml")
+
+ def canParse(fileName: String): Boolean = {
+ val lower = fileName.toLowerCase
+ (JsonExtensions ++ YamlExtensions ++ TomlExtensions).exists(ext => lower.endsWith(ext))
+ }
+
+ def parse(fileName: String, source: String): ParseResult = {
+ val lower = fileName.toLowerCase
+
+ val entries: Vector[(String, String, Int)] =
+ if (JsonExtensions.exists(ext => lower.endsWith(ext))) parseJson(source)
+ else if (YamlExtensions.exists(ext => lower.endsWith(ext))) parseYaml(source)
+ else if (TomlExtensions.exists(ext => lower.endsWith(ext))) parseToml(source)
+ else Vector.empty
+
+ buildResult(fileName, source, entries)
+ }
+
+ // ---------------------------------------------------------------------------
+ // Result builder
+ // ---------------------------------------------------------------------------
+
+ private def buildResult(
+ fileName: String,
+ source: String,
+ entries: Vector[(String, String, Int)]
+ ): ParseResult = {
+ val lines = source.split("\n", -1)
+
+ val configEntity = ParsedEntity(
+ name = fileName,
+ kind = NodeKind.Config,
+ attrs = Map("format" -> Json.fromString(detectFormat(fileName))),
+ lineStart = 1,
+ lineEnd = lines.length
+ )
+
+ val entryEntities = entries.map { case (key, value, line) =>
+ ParsedEntity(
+ name = key,
+ kind = NodeKind.ConfigEntry,
+ attrs = Map("value" -> Json.fromString(value)),
+ lineStart = line,
+ lineEnd = line
+ )
+ }
+
+ val relationships = entries.map { case (key, _, _) =>
+ ParsedRelationship(fileName, key, "CONTAINS")
+ }
+
+ ParseResult(
+ entities = configEntity +: entryEntities,
+ relationships = relationships
+ )
+ }
+
+ private def detectFormat(fileName: String): String = {
+ val lower = fileName.toLowerCase
+ if (JsonExtensions.exists(ext => lower.endsWith(ext))) "json"
+ else if (YamlExtensions.exists(ext => lower.endsWith(ext))) "yaml"
+ else if (TomlExtensions.exists(ext => lower.endsWith(ext))) "toml"
+ else "unknown"
+ }
+
+ // ---------------------------------------------------------------------------
+ // JSON parsing — Circe-based with flattening
+ // ---------------------------------------------------------------------------
+
+ private def parseJson(source: String): Vector[(String, String, Int)] = {
+ io.circe.parser.parse(source) match {
+ case Right(json) => flattenJson("", json, 1)
+ case Left(_) => regexParseJson(source)
+ }
+ }
+
+ private def flattenJson(prefix: String, json: io.circe.Json, line: Int): Vector[(String, String, Int)] = {
+ json.fold(
+ jsonNull = Vector.empty,
+ jsonBoolean = b => Vector((prefix, b.toString, line)),
+ jsonNumber = n => Vector((prefix, n.toString, line)),
+ jsonString = s => Vector((prefix, s, line)),
+ jsonArray = arr => arr.zipWithIndex.flatMap { case (v, i) =>
+ val key = if (prefix.isEmpty) s"[$i]" else s"$prefix[$i]"
+ flattenJson(key, v, line)
+ }.toVector,
+ jsonObject = obj => obj.toVector.flatMap { case (k, v) =>
+ val key = if (prefix.isEmpty) k else s"$prefix.$k"
+ v.fold(
+ jsonNull = Vector((key, "null", line)),
+ jsonBoolean = b => Vector((key, b.toString, line)),
+ jsonNumber = n => Vector((key, n.toString, line)),
+ jsonString = s => Vector((key, s, line)),
+ jsonArray = _ => flattenJson(key, v, line),
+ jsonObject = _ => flattenJson(key, v, line)
+ )
+ }
+ )
+ }
+
+ /** Regex fallback for malformed JSON */
+ private def regexParseJson(source: String): Vector[(String, String, Int)] = {
+ val KvPattern = """"([^"]+)"\s*:\s*"([^"]*)"""".r
+ val lines = source.split("\n", -1)
+ lines.zipWithIndex.flatMap { case (line, idx) =>
+ KvPattern.findAllMatchIn(line).map { m =>
+ (m.group(1), m.group(2), idx + 1)
+ }
+ }.toVector
+ }
+
+ // ---------------------------------------------------------------------------
+ // YAML parsing — indentation-aware
+ // ---------------------------------------------------------------------------
+
+ private def parseYaml(source: String): Vector[(String, String, Int)] = {
+ val lines = source.split("\n", -1)
+ val result = Vector.newBuilder[(String, String, Int)]
+ val stack = scala.collection.mutable.Stack.empty[(Int, String)] // (indent, prefix)
+
+ val KvWithValue = """^([a-zA-Z_][\w.-]*)\s*:\s*(.+)$""".r
+ val KvNoValue = """^([a-zA-Z_][\w.-]*)\s*:\s*$""".r
+
+ lines.zipWithIndex.foreach { case (line, idx) =>
+ val trimmed = line.trim
+ if (!trimmed.startsWith("#") && trimmed.nonEmpty) {
+ val indent = line.takeWhile(_ == ' ').length
+ trimmed match {
+ case KvWithValue(key, value) =>
+ while (stack.nonEmpty && stack.top._1 >= indent) stack.pop()
+ val fullKey = if (stack.nonEmpty) s"${stack.top._2}.$key" else key
+ result += ((fullKey, value.trim, idx + 1))
+ case KvNoValue(key) =>
+ while (stack.nonEmpty && stack.top._1 >= indent) stack.pop()
+ val fullKey = if (stack.nonEmpty) s"${stack.top._2}.$key" else key
+ stack.push((indent, fullKey))
+ case _ =>
+ // skip list items, comments, etc.
+ }
+ }
+ }
+ result.result()
+ }
+
+ // ---------------------------------------------------------------------------
+ // TOML parsing — section-aware
+ // ---------------------------------------------------------------------------
+
+ private def parseToml(source: String): Vector[(String, String, Int)] = {
+ val lines = source.split("\n", -1)
+ val result = Vector.newBuilder[(String, String, Int)]
+ var currentSection = ""
+
+ val SectionPattern = """^\[([^\]]+)\]""".r
+ val KvPattern = """^([a-zA-Z_][\w.-]*)\s*=\s*(.+)$""".r
+
+ lines.zipWithIndex.foreach { case (line, idx) =>
+ val trimmed = line.trim
+ if (!trimmed.startsWith("#") && trimmed.nonEmpty) {
+ trimmed match {
+ case SectionPattern(section) =>
+ currentSection = section
+ case KvPattern(key, value) =>
+ val fullKey = if (currentSection.isEmpty) key else s"$currentSection.$key"
+ val cleanValue = value.trim.stripPrefix("\"").stripSuffix("\"")
+ result += ((fullKey, cleanValue, idx + 1))
+ case _ =>
+ }
+ }
+ }
+ result.result()
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
new file mode 100644
index 00000000..7d99504c
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
@@ -0,0 +1,78 @@
+package ix.memory.ingestion.parsers
+
+import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}
+import ix.memory.model.NodeKind
+import io.circe.Json
+
+import scala.util.matching.Regex
+
+/**
+ * Markdown parser that extracts structural entities from `.md` files.
+ *
+ * Each heading becomes a Doc entity with attrs for `level`, `title`, and `content`.
+ * Content captures all non-header lines in the section, up to 2000 characters.
+ * A top-level File entity is also emitted, with CONTAINS edges to each section.
+ */
+class MarkdownParser extends Parser {
+
+ private val HeadingPattern: Regex = """^(#{1,6})\s+(.+)$""".r
+
+ def parse(fileName: String, source: String): ParseResult = {
+ val lines = source.split("\n", -1)
+
+ // -- File entity --
+ val fileEntity = ParsedEntity(
+ name = fileName,
+ kind = NodeKind.File,
+ attrs = Map("language" -> Json.fromString("markdown")),
+ lineStart = 1,
+ lineEnd = lines.length
+ )
+
+ // Collect headings with their line indices
+ val headings: Vector[(Int, Int, String)] = lines.zipWithIndex.flatMap { case (line, idx) =>
+ HeadingPattern.findFirstMatchIn(line.trim).map { m =>
+ val level = m.group(1).length
+ val title = m.group(2).trim
+ (idx, level, title)
+ }
+ }.toVector
+
+ var entities = Vector(fileEntity)
+ var relationships = Vector.empty[ParsedRelationship]
+
+ for (i <- headings.indices) {
+ val (startIdx, level, title) = headings(i)
+ val endIdx = if (i + 1 < headings.length) headings(i + 1)._1 else lines.length
+
+ val content = extractContent(lines, startIdx, endIdx)
+
+ val entity = ParsedEntity(
+ name = title,
+ kind = NodeKind.Doc,
+ attrs = Map(
+ "level" -> Json.fromInt(level),
+ "title" -> Json.fromString(title),
+ "content" -> Json.fromString(content)
+ ),
+ lineStart = startIdx + 1,
+ lineEnd = endIdx
+ )
+
+ entities = entities :+ entity
+ relationships = relationships :+ ParsedRelationship(fileName, title, "CONTAINS")
+ }
+
+ ParseResult(entities, relationships)
+ }
+
+ /**
+ * Extract content from all non-header lines in the section, up to 2000 characters.
+ */
+ private def extractContent(lines: Array[String], startIdx: Int, endIdx: Int): String = {
+ val contentLines = lines.slice(startIdx + 1, endIdx).filterNot { line =>
+ HeadingPattern.findFirstMatchIn(line.trim).isDefined
+ }
+ contentLines.mkString("\n").trim.take(2000)
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala
new file mode 100644
index 00000000..deac202c
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala
@@ -0,0 +1,357 @@
+package ix.memory.ingestion.parsers
+
+import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
+import ix.memory.model.NodeKind
+import io.circe.Json
+
+import scala.util.matching.Regex
+
+/**
+ * Regex-based Scala source parser that extracts structural entities and relationships.
+ *
+ * Extracts: traits, classes, case classes, objects, methods (def), and imports.
+ * Uses brace counting for block boundary detection.
+ */
+class ScalaParser extends Parser {
+
+ // Patterns for Scala constructs
+ private val TraitPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*trait\s+(\w+)""".r
+ private val ClassPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*class\s+(\w+)""".r
+ private val CaseClassPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*case\s+class\s+(\w+)""".r
+ private val ObjectPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*(?:case\s+)?object\s+(\w+)""".r
+ private val DefPattern: Regex = """^\s*(?:override\s+)?(?:private(?:\[\w+\])?\s+|protected(?:\[\w+\])?\s+)?(?:final\s+)?def\s+(\w+)""".r
+ private val ImportPattern: Regex = """^\s*import\s+(.+)$""".r
+ private val CallPattern: Regex = """(?:(?:this|super)\.)?\b(\w+)\s*(?:\[.*?\])?\s*\(""".r
+ private val ExtendsPattern: Regex = """extends\s+(\w+)""".r
+ private val TypeRefPattern: Regex = """\b([A-Z]\w+)""".r
+ private val QualifiedRefPattern: Regex = """\b([A-Z]\w+)\.(\w+)""".r
+
+ private val ScalaBuiltinTypes: Set[String] = Set(
+ "String", "Int", "Long", "Double", "Float", "Boolean", "Unit", "Any", "AnyRef", "AnyVal",
+ "Nothing", "Null", "Option", "List", "Map", "Set", "Vector", "Seq", "Array", "Future",
+ "IO", "Either", "Try", "Some", "None", "Right", "Left", "Success", "Failure"
+ )
+
+ private val ScalaBuiltins: Set[String] = Set(
+ "println", "print", "require", "assert", "assume",
+ "Some", "None", "Option", "List", "Map", "Set", "Vector",
+ "Right", "Left", "Try", "Success", "Failure",
+ "Future", "IO", "pure", "flatMap", "map", "filter",
+ "foreach", "fold", "foldLeft", "foldRight",
+ "toString", "hashCode", "equals", "getClass",
+ "apply", "unapply", "copy", "andThen", "compose",
+ "getOrElse", "orElse", "contains", "exists", "forall",
+ "collect", "traverse", "sequence", "attempt", "handleErrorWith",
+ "raiseError", "blocking", "delay", "suspend", "async",
+ "mkString", "toList", "toVector", "toSet", "toMap",
+ "head", "tail", "size", "length", "isEmpty", "nonEmpty",
+ "take", "drop", "slice", "zip", "zipWithIndex",
+ "sortBy", "groupBy", "distinct", "flatten"
+ )
+
+ def parse(fileName: String, source: String): ParseResult = {
+ val lines = source.split("\n", -1)
+
+ val fileEntity = ParsedEntity(
+ name = fileName,
+ kind = NodeKind.File,
+ attrs = Map("language" -> Json.fromString("scala")),
+ lineStart = 1,
+ lineEnd = lines.length
+ )
+
+ var entities = Vector(fileEntity)
+ var relationships = Vector.empty[ParsedRelationship]
+
+ // Track type ranges for containment: (name, startLine, endLine)
+ var typeRanges = Vector.empty[(String, Int, Int)]
+
+ for ((line, idx) <- lines.zipWithIndex) {
+ val lineNum = idx + 1
+ val trimmed = line.trim
+
+ // Skip comment lines
+ if (trimmed.startsWith("//") || trimmed.startsWith("*") || trimmed.startsWith("/*")) {
+ // no-op
+ } else {
+
+ // Case class must be checked before class (since "case class" contains "class")
+ val handledAsType = CaseClassPattern.findFirstMatchIn(line) match {
+ case Some(m) =>
+ val name = m.group(1)
+ val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
+ val bodyText = lines.slice(idx, endLine).mkString("\n")
+ val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
+ entities = entities :+ ParsedEntity(
+ name = name,
+ kind = NodeKind.Class,
+ attrs = Map("scala_kind" -> Json.fromString("case_class"), "language" -> Json.fromString("scala"),
+ "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
+ lineStart = lineNum,
+ lineEnd = endLine,
+ contentFingerprint = Some(fp)
+ )
+ val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
+ val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
+ relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
+ typeRanges = typeRanges :+ (name, lineNum, endLine)
+ ExtendsPattern.findFirstMatchIn(line).foreach { em =>
+ relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
+ }
+ true
+
+ case None =>
+ // Check for trait
+ TraitPattern.findFirstMatchIn(line) match {
+ case Some(m) =>
+ val name = m.group(1)
+ val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
+ val bodyText = lines.slice(idx, endLine).mkString("\n")
+ val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
+ entities = entities :+ ParsedEntity(
+ name = name,
+ kind = NodeKind.Trait,
+ attrs = Map("scala_kind" -> Json.fromString("trait"), "language" -> Json.fromString("scala"),
+ "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
+ lineStart = lineNum,
+ lineEnd = endLine,
+ contentFingerprint = Some(fp)
+ )
+ val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
+ val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
+ relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
+ typeRanges = typeRanges :+ (name, lineNum, endLine)
+ ExtendsPattern.findFirstMatchIn(line).foreach { em =>
+ relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
+ }
+ true
+
+ case None =>
+ // Check for object (before class, since object keyword won't match class pattern)
+ ObjectPattern.findFirstMatchIn(line) match {
+ case Some(m) =>
+ val name = m.group(1)
+ val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
+ val bodyText = lines.slice(idx, endLine).mkString("\n")
+ val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
+ entities = entities :+ ParsedEntity(
+ name = name,
+ kind = NodeKind.Object,
+ attrs = Map("scala_kind" -> Json.fromString("object"), "language" -> Json.fromString("scala"),
+ "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
+ lineStart = lineNum,
+ lineEnd = endLine,
+ contentFingerprint = Some(fp)
+ )
+ val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
+ val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
+ relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
+ typeRanges = typeRanges :+ (name, lineNum, endLine)
+ ExtendsPattern.findFirstMatchIn(line).foreach { em =>
+ relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
+ }
+ true
+
+ case None =>
+ // Check for class
+ ClassPattern.findFirstMatchIn(line) match {
+ case Some(m) =>
+ val name = m.group(1)
+ val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
+ val bodyText = lines.slice(idx, endLine).mkString("\n")
+ val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
+ entities = entities :+ ParsedEntity(
+ name = name,
+ kind = NodeKind.Class,
+ attrs = Map("scala_kind" -> Json.fromString("class"), "language" -> Json.fromString("scala"),
+ "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
+ lineStart = lineNum,
+ lineEnd = endLine,
+ contentFingerprint = Some(fp)
+ )
+ val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
+ val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
+ relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
+ typeRanges = typeRanges :+ (name, lineNum, endLine)
+ ExtendsPattern.findFirstMatchIn(line).foreach { em =>
+ relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
+ }
+ true
+ case None => false
+ }
+ }
+ }
+ }
+
+ // Check for def (only if we didn't match a type on this line)
+ if (!handledAsType) {
+ DefPattern.findFirstMatchIn(line).foreach { m =>
+ val funcName = m.group(1)
+ val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
+
+ val signature = extractSignature(trimmed)
+ val visibility = extractVisibility(trimmed)
+
+ // Determine enclosing type
+ val enclosing = typeRanges.find { case (_, start, end) =>
+ lineNum > start && lineNum <= end
+ }
+
+ val (kind, edgePredicate, edgeSrc) = enclosing match {
+ case Some((typeName, _, _)) => (NodeKind.Method, "CONTAINS", typeName)
+ case None => (NodeKind.Function, "CONTAINS", fileName)
+ }
+
+ val bodyText = lines.slice(idx, endLine).mkString("\n")
+ val fp = Fingerprint.compute(signature, bodyText)
+
+ entities = entities :+ ParsedEntity(
+ name = funcName,
+ kind = kind,
+ attrs = Map(
+ "language" -> Json.fromString("scala"),
+ "summary" -> Json.fromString(signature),
+ "signature" -> Json.fromString(signature),
+ "visibility" -> Json.fromString(visibility)
+ ),
+ lineStart = lineNum,
+ lineEnd = endLine,
+ contentFingerprint = Some(fp)
+ )
+
+ relationships = relationships :+ ParsedRelationship(edgeSrc, funcName, edgePredicate)
+
+ // Extract CALLS and REFERENCES from method/function body
+ val callsInFunc = extractCalls(lines, idx, endLine, funcName)
+ relationships = relationships ++ callsInFunc
+
+ // Extract REFERENCES from method signature type annotations
+ val enclosingTypeName = enclosing.map(_._1).getOrElse("")
+ val sigRefs = TypeRefPattern.findAllMatchIn(signature).map(_.group(1)).toSet
+ for (ref <- sigRefs) {
+ if (!ScalaBuiltinTypes.contains(ref) && ref != funcName && ref != enclosingTypeName && ref != fileName) {
+ relationships = relationships :+ ParsedRelationship(funcName, ref, "REFERENCES")
+ }
+ }
+ }
+ }
+
+ // Check for imports
+ ImportPattern.findFirstMatchIn(line).foreach { m =>
+ val importPath = m.group(1).trim
+ // Parse import path: "ix.memory.model.{Claim, GraphNode}" -> module "ix.memory.model"
+ // "cats.effect.IO" -> module "cats.effect.IO"
+ val moduleName = if (importPath.contains("{")) {
+ // Grouped import: take the prefix before the braces
+ importPath.substring(0, importPath.indexOf('{')).stripSuffix(".").trim
+ } else {
+ importPath
+ }
+
+ // Avoid duplicate module entities
+ if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
+ entities = entities :+ ParsedEntity(
+ name = moduleName,
+ kind = NodeKind.Module,
+ attrs = Map.empty,
+ lineStart = lineNum,
+ lineEnd = lineNum
+ )
+ }
+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
+ }
+
+ } // end else (non-comment)
+ }
+
+ ParseResult(entities, relationships)
+ }
+
+ /**
+ * Find the end of a brace-delimited block starting at `startIdx`.
+ * Counts opening and closing braces. Returns None if no opening brace is found
+ * (e.g. abstract method in a trait with no body).
+ */
+ private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Option[Int] = {
+ var braceCount = 0
+ var foundOpen = false
+ var i = startIdx
+
+ while (i < lines.length) {
+ val line = lines(i)
+ for (ch <- line) {
+ ch match {
+ case '{' =>
+ braceCount += 1
+ foundOpen = true
+ case '}' =>
+ braceCount -= 1
+ if (foundOpen && braceCount == 0) {
+ return Some(i + 1) // 1-indexed
+ }
+ case _ => // skip
+ }
+ }
+ // If we found an opening brace and closed it, we'd have returned above
+ // If line ends and no brace found yet, keep going
+ i += 1
+ }
+
+ if (foundOpen) Some(lines.length) else None
+ }
+
+ /** Extract CALLS and qualified REFERENCES relationships from a method/function body. */
+ private def extractCalls(lines: Array[String], startIdx: Int, endIdx: Int, callerName: String): Vector[ParsedRelationship] = {
+ var calls = Vector.empty[ParsedRelationship]
+ val seen = scala.collection.mutable.Set.empty[String]
+ val refSeen = scala.collection.mutable.Set.empty[String]
+ for (i <- (startIdx + 1) until endIdx.min(lines.length)) {
+ val line = lines(i).trim
+ if (!line.startsWith("//") && !line.startsWith("*")) {
+ CallPattern.findAllMatchIn(line).foreach { m =>
+ val callee = m.group(1)
+ // Only include lowercase-starting identifiers (methods/functions, not types)
+ if (callee != callerName && !ScalaBuiltins.contains(callee) && !seen.contains(callee) && callee.head.isLower) {
+ seen += callee
+ calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
+ }
+ }
+ // Extract qualified REFERENCES (e.g., NodeKind.File, SourceType.Code)
+ QualifiedRefPattern.findAllMatchIn(line).foreach { m =>
+ val qualifier = m.group(1)
+ if (!ScalaBuiltinTypes.contains(qualifier) && !ScalaBuiltins.contains(qualifier) &&
+ !refSeen.contains(qualifier) && qualifier != callerName) {
+ refSeen += qualifier
+ calls = calls :+ ParsedRelationship(callerName, qualifier, "REFERENCES")
+ }
+ }
+ }
+ }
+ calls
+ }
+
+ /** Extract a type signature (trait/class/object), stripping body braces. */
+ private def extractTypeSignature(trimmedLine: String): String = {
+ val cleaned = trimmedLine
+ .replaceAll("""\s*\{\s*$""", "")
+ .trim
+ cleaned.take(120)
+ }
+
+ /** Extract a clean signature from a trimmed line, stripping trailing body markers. */
+ private def extractSignature(trimmedLine: String): String = {
+ val cleaned = trimmedLine
+ .replaceAll("""\s*=\s*\{?\s*$""", "")
+ .replaceAll("""\s*\{\s*$""", "")
+ .trim
+ cleaned.take(120)
+ }
+
+ /** Detect visibility from a trimmed line (after stripping `override`). */
+ private def extractVisibility(trimmedLine: String): String = {
+ val withoutOverride = trimmedLine.replaceFirst("""^\s*override\s+""", "").trim
+ if (withoutOverride.startsWith("private")) "private"
+ else if (withoutOverride.startsWith("protected")) "protected"
+ else "public"
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala
new file mode 100644
index 00000000..deb06bdd
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala
@@ -0,0 +1,250 @@
+package ix.memory.ingestion.parsers
+
+import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
+import ix.memory.model.NodeKind
+import io.circe.Json
+
+import scala.util.matching.Regex
+
+/**
+ * Python source parser that extracts structural entities and relationships.
+ *
+ * Attempts to use tree-sitter JNI first; falls back to regex-based extraction
+ * when the native library is unavailable (common on some platforms).
+ * The regex fallback is sufficient for Phase 1 since the goal is the pipeline
+ * structure, not production-grade parsing.
+ */
+class TreeSitterPythonParser extends Parser {
+
+ /**
+ * Parse a Python source file and extract entities and relationships.
+ *
+ * @param fileName the file name (e.g. "billing_service.py")
+ * @param source the full source code as a string
+ * @return ParseResult with entities and relationships
+ */
+ def parse(fileName: String, source: String): ParseResult = {
+ // Try tree-sitter first, fall back to regex
+ tryTreeSitter(fileName, source).getOrElse(regexParse(fileName, source))
+ }
+
+ // ---------------------------------------------------------------------------
+ // Tree-sitter JNI attempt
+ // ---------------------------------------------------------------------------
+
+ private def tryTreeSitter(fileName: String, source: String): Option[ParseResult] = {
+ try {
+ val langClass = Class.forName("ch.usi.si.seart.treesitter.Language")
+ val pythonLang = langClass.getMethod("valueOf", classOf[String]).invoke(null, "PYTHON")
+
+ val parserClass = Class.forName("ch.usi.si.seart.treesitter.Parser")
+ val parser = parserClass.getMethod("getFor", langClass).invoke(null, pythonLang)
+
+ val tree = parserClass.getMethod("parse", classOf[String]).invoke(parser, source)
+ val treeClass = Class.forName("ch.usi.si.seart.treesitter.Tree")
+ val root = treeClass.getMethod("getRootNode").invoke(tree)
+
+ // If we get here, tree-sitter loaded successfully
+ // We would walk the AST here, but for now this path is not reached
+ // on platforms where the JNI library is unavailable.
+ parserClass.getMethod("close").invoke(parser)
+ treeClass.getMethod("close").invoke(tree)
+ None // TODO: implement full tree-sitter AST walking when JNI works
+ } catch {
+ case _: Throwable => None // Catches both Exception and Error (e.g. UnsatisfiedLinkError)
+ }
+ }
+
+ // ---------------------------------------------------------------------------
+ // Regex-based fallback parser
+ // ---------------------------------------------------------------------------
+
+ private val ClassPattern: Regex = """^class\s+(\w+)""".r
+ private val FuncPattern: Regex = """^(\s*)def\s+(\w+)""".r
+ private val ImportPattern: Regex = """^import\s+(\w+)""".r
+ private val FromImportPattern: Regex = """^from\s+(\w+)\s+import""".r
+ private val CallPattern: Regex = """(?:self\.)?(\w+)\(""".r
+
+ private def regexParse(fileName: String, source: String): ParseResult = {
+ val lines = source.split("\n", -1)
+
+ // -- File entity --
+ val fileEntity = ParsedEntity(
+ name = fileName,
+ kind = NodeKind.File,
+ attrs = Map("language" -> Json.fromString("python")),
+ lineStart = 1,
+ lineEnd = lines.length
+ )
+
+ // -- Classes and functions --
+ var entities = Vector(fileEntity)
+ var relationships = Vector.empty[ParsedRelationship]
+
+ // Track class ranges: (className, lineStart, lineEnd) for containment checks
+ var classRanges = Vector.empty[(String, Int, Int)]
+
+ for ((line, idx) <- lines.zipWithIndex) {
+ val lineNum = idx + 1
+
+ // Skip comment lines — prevents matching patterns inside comments
+ if (line.trim.startsWith("#")) {
+ // no-op: skip this line entirely
+ } else {
+
+ // Check for class definition
+ ClassPattern.findFirstMatchIn(line).foreach { m =>
+ val className = m.group(1)
+ val classEnd = findBlockEnd(lines, idx)
+ val bodyText = lines.slice(idx, classEnd).mkString("\n")
+ val fp = Fingerprint.compute(line.trim.take(120), bodyText)
+ entities = entities :+ ParsedEntity(
+ name = className,
+ kind = NodeKind.Class,
+ attrs = Map("language" -> Json.fromString("python")),
+ lineStart = lineNum,
+ lineEnd = classEnd,
+ contentFingerprint = Some(fp)
+ )
+ // File CONTAINS class
+ relationships = relationships :+ ParsedRelationship(fileName, className, "CONTAINS")
+ classRanges = classRanges :+ (className, lineNum, classEnd)
+ }
+
+ // Check for function/method definition
+ FuncPattern.findFirstMatchIn(line).foreach { m =>
+ val funcName = m.group(2)
+ val funcEnd = findBlockEnd(lines, idx)
+ val bodyText = lines.slice(idx, funcEnd).mkString("\n")
+ val fp = Fingerprint.compute(line.trim.take(120), bodyText)
+
+ entities = entities :+ ParsedEntity(
+ name = funcName,
+ kind = NodeKind.Function,
+ attrs = Map("language" -> Json.fromString("python")),
+ lineStart = lineNum,
+ lineEnd = funcEnd,
+ contentFingerprint = Some(fp)
+ )
+
+ // Check if this function's line falls within any class range
+ val enclosingClass = classRanges.find { case (_, start, end) =>
+ lineNum > start && lineNum <= end
+ }
+ enclosingClass match {
+ case Some((className, _, _)) =>
+ // Method inside a class
+ relationships = relationships :+ ParsedRelationship(className, funcName, "CONTAINS")
+ case None =>
+ // Top-level function — file defines it
+ relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
+ }
+
+ // Extract function calls within the function body
+ val callsInFunc = extractCalls(lines, idx, funcName)
+ relationships = relationships ++ callsInFunc
+ }
+
+ // Check for imports
+ ImportPattern.findFirstMatchIn(line).foreach { m =>
+ val moduleName = m.group(1)
+ entities = entities :+ ParsedEntity(
+ name = moduleName,
+ kind = NodeKind.Module,
+ attrs = Map.empty,
+ lineStart = lineNum,
+ lineEnd = lineNum
+ )
+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
+ }
+
+ FromImportPattern.findFirstMatchIn(line).foreach { m =>
+ val moduleName = m.group(1)
+ // Avoid duplicate module entities
+ if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
+ entities = entities :+ ParsedEntity(
+ name = moduleName,
+ kind = NodeKind.Module,
+ attrs = Map.empty,
+ lineStart = lineNum,
+ lineEnd = lineNum
+ )
+ }
+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
+ }
+
+ } // end else (non-comment line)
+ }
+
+ ParseResult(entities, relationships)
+ }
+
+ /**
+ * Find the end line of a block starting at `startIdx` using indentation.
+ * Looks for the last line that has greater indentation than the definition line.
+ */
+ private def findBlockEnd(lines: Array[String], startIdx: Int): Int = {
+ val baseIndent = lines(startIdx).takeWhile(_ == ' ').length
+ var endIdx = startIdx
+ var i = startIdx + 1
+ while (i < lines.length) {
+ val line = lines(i)
+ if (line.trim.nonEmpty) {
+ val indent = line.takeWhile(_ == ' ').length
+ if (indent > baseIndent) {
+ endIdx = i
+ } else {
+ // Block ended
+ return endIdx + 1
+ }
+ }
+ i += 1
+ }
+ // If we reach end of file, the block extends to the end
+ endIdx + 1
+ }
+
+ /**
+ * Extract CALLS relationships from the body of a function starting at `funcIdx`.
+ * Uses a simple heuristic: look for `identifier(` patterns in subsequent indented lines.
+ */
+ private def extractCalls(lines: Array[String], funcIdx: Int, callerName: String): Vector[ParsedRelationship] = {
+ val baseIndent = lines(funcIdx).takeWhile(_ == ' ').length
+ var calls = Vector.empty[ParsedRelationship]
+ val seen = scala.collection.mutable.Set.empty[String]
+ var i = funcIdx + 1
+
+ while (i < lines.length) {
+ val line = lines(i)
+ if (line.trim.nonEmpty) {
+ val indent = line.takeWhile(_ == ' ').length
+ if (indent <= baseIndent) {
+ // Exited the function body
+ return calls
+ }
+ // Find calls in this line
+ CallPattern.findAllMatchIn(line).foreach { m =>
+ val callee = m.group(1)
+ // Filter out Python builtins and self-references, and skip 'self' prefix calls
+ // that match the method name itself
+ if (callee != callerName && !PythonBuiltins.contains(callee) && !seen.contains(callee)) {
+ seen += callee
+ calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
+ }
+ }
+ }
+ i += 1
+ }
+ calls
+ }
+
+ private val PythonBuiltins: Set[String] = Set(
+ "print", "len", "range", "int", "str", "float", "list", "dict",
+ "set", "tuple", "type", "isinstance", "issubclass", "hasattr",
+ "getattr", "setattr", "delattr", "super", "property", "staticmethod",
+ "classmethod", "enumerate", "zip", "map", "filter", "sorted",
+ "reversed", "any", "all", "min", "max", "sum", "abs", "round",
+ "input", "open", "format", "repr", "id", "hash", "callable",
+ "iter", "next", "not", "raise"
+ )
+}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala
new file mode 100644
index 00000000..fc2670a2
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala
@@ -0,0 +1,422 @@
+package ix.memory.ingestion.parsers
+
+import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
+import ix.memory.model.NodeKind
+import io.circe.Json
+
+import scala.util.matching.Regex
+
+/**
+ * TypeScript/JavaScript source parser that extracts structural entities
+ * and relationships using regex-based extraction.
+ */
+class TypeScriptParser extends Parser {
+
+ /**
+ * Parse a TypeScript source file and extract entities and relationships.
+ *
+ * @param fileName the file name (e.g. "api.ts")
+ * @param source the full source code as a string
+ * @return ParseResult with entities and relationships
+ */
+ def parse(fileName: String, source: String): ParseResult = {
+ val lines = source.split("\n", -1)
+
+ // -- File entity (no raw content stored) --
+ val fileEntity = ParsedEntity(
+ name = fileName,
+ kind = NodeKind.File,
+ attrs = Map("language" -> Json.fromString("typescript")),
+ lineStart = 1,
+ lineEnd = lines.length
+ )
+
+ var entities = Vector(fileEntity)
+ var relationships = Vector.empty[ParsedRelationship]
+
+ // Track class ranges for containment
+ var classRanges = Vector.empty[(String, Int, Int)]
+
+ for ((line, idx) <- lines.zipWithIndex) {
+ val lineNum = idx + 1
+ val trimmed = line.trim
+
+ // Skip comment lines
+ if (trimmed.startsWith("//") || trimmed.startsWith("*") || trimmed.startsWith("/*")) {
+ // no-op
+ } else {
+
+ // Check for class definition
+ ClassPattern.findFirstMatchIn(line).foreach { m =>
+ val className = m.group(1)
+ val classEnd = findBraceBlockEnd(lines, idx)
+ val bodyText = lines.slice(idx, classEnd).mkString("\n")
+ val fp = Fingerprint.compute(trimmed.replaceAll("""\s*\{\s*$""", "").take(120), bodyText)
+ val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
+ entities = entities :+ ParsedEntity(
+ name = className,
+ kind = NodeKind.Class,
+ attrs = Map("language" -> Json.fromString("typescript"), "exported" -> Json.fromBoolean(exported)),
+ lineStart = lineNum,
+ lineEnd = classEnd,
+ contentFingerprint = Some(fp)
+ )
+ relationships = relationships :+ ParsedRelationship(fileName, className, "CONTAINS")
+ classRanges = classRanges :+ (className, lineNum, classEnd)
+
+ // Extract methods from this class
+ val methods = extractMethods(lines, idx, classEnd, className)
+ entities = entities ++ methods._1
+ relationships = relationships ++ methods._2
+ }
+
+ // Check for top-level function (not inside a class)
+ if (!insideClass(lineNum, classRanges)) {
+ FuncPattern.findFirstMatchIn(line).foreach { m =>
+ val funcName = m.group(1)
+ val funcEnd = findBraceBlockEnd(lines, idx)
+ val summary = line.trim.take(120)
+ val bodyText = lines.slice(idx, funcEnd).mkString("\n")
+ val fp = Fingerprint.compute(summary, bodyText)
+ val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
+ entities = entities :+ ParsedEntity(
+ name = funcName,
+ kind = NodeKind.Function,
+ attrs = Map(
+ "language" -> Json.fromString("typescript"),
+ "summary" -> Json.fromString(summary),
+ "signature" -> Json.fromString(summary),
+ "visibility" -> Json.fromString("public"),
+ "exported" -> Json.fromBoolean(exported)
+ ),
+ lineStart = lineNum,
+ lineEnd = funcEnd,
+ contentFingerprint = Some(fp)
+ )
+ relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
+
+ // Extract calls within this function
+ val calls = extractCalls(lines, idx, funcEnd, funcName)
+ relationships = relationships ++ calls
+ }
+
+ // Check for arrow function / const function
+ val arrowMatch = ArrowFuncPattern.findFirstMatchIn(line)
+ val constFuncMatch = ConstFuncPattern.findFirstMatchIn(line)
+ val funcMatch = arrowMatch.orElse(constFuncMatch)
+ funcMatch.foreach { m =>
+ val funcName = m.group(1)
+ // Avoid duplicates if already captured by FuncPattern
+ if (!entities.exists(e => e.name == funcName && e.kind == NodeKind.Function)) {
+ val funcEnd = findBraceBlockEnd(lines, idx)
+ val summary = line.trim.take(120)
+ val bodyText = lines.slice(idx, funcEnd).mkString("\n")
+ val fp = Fingerprint.compute(summary, bodyText)
+ val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
+ entities = entities :+ ParsedEntity(
+ name = funcName,
+ kind = NodeKind.Function,
+ attrs = Map(
+ "language" -> Json.fromString("typescript"),
+ "summary" -> Json.fromString(summary),
+ "signature" -> Json.fromString(summary),
+ "visibility" -> Json.fromString("public"),
+ "exported" -> Json.fromBoolean(exported)
+ ),
+ lineStart = lineNum,
+ lineEnd = funcEnd,
+ contentFingerprint = Some(fp)
+ )
+ relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
+
+ val callRels = extractCalls(lines, idx, funcEnd, funcName)
+ relationships = relationships ++ callRels
+ }
+ }
+ }
+
+ // Check for imports
+ ImportPattern.findFirstMatchIn(line).foreach { m =>
+ val moduleName = m.group(1)
+ if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
+ entities = entities :+ ParsedEntity(
+ name = moduleName,
+ kind = NodeKind.Module,
+ attrs = Map.empty,
+ lineStart = lineNum,
+ lineEnd = lineNum
+ )
+ }
+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
+ }
+
+ // Extract named import bindings: import { foo, bar } from "mod"
+ NamedImportPattern.findFirstMatchIn(line).foreach { m =>
+ val names = m.group(1).split(",").map(_.trim).filter(_.nonEmpty)
+ for (rawName <- names) {
+ val symbolName = if (rawName.contains(" as ")) rawName.split("\\s+as\\s+").last.trim else rawName
+ if (symbolName.nonEmpty && !relationships.exists(r => r.srcName == fileName && r.dstName == symbolName && r.predicate == "IMPORTS")) {
+ relationships = relationships :+ ParsedRelationship(fileName, symbolName, "IMPORTS")
+ }
+ }
+ }
+
+ SideEffectImportPattern.findFirstMatchIn(line).foreach { m =>
+ val moduleName = m.group(1)
+ if (!relationships.exists(r => r.dstName == moduleName && r.predicate == "IMPORTS")) {
+ if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
+ entities = entities :+ ParsedEntity(
+ name = moduleName,
+ kind = NodeKind.Module,
+ attrs = Map.empty,
+ lineStart = lineNum,
+ lineEnd = lineNum
+ )
+ }
+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
+ }
+ }
+
+ // Check for interface definition
+ InterfacePattern.findFirstMatchIn(line).foreach { m =>
+ val ifaceName = m.group(1)
+ if (!entities.exists(e => e.name == ifaceName && e.kind == NodeKind.Interface)) {
+ val ifaceEnd = findBraceBlockEnd(lines, idx)
+ val bodyText = lines.slice(idx, ifaceEnd).mkString("\n")
+ val fp = Fingerprint.compute(trimmed.replaceAll("""\s*\{\s*$""", "").take(120), bodyText)
+ val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
+ entities = entities :+ ParsedEntity(
+ name = ifaceName,
+ kind = NodeKind.Interface,
+ attrs = Map(
+ "language" -> Json.fromString("typescript"),
+ "ts_kind" -> Json.fromString("interface"),
+ "exported" -> Json.fromBoolean(exported)
+ ),
+ lineStart = lineNum,
+ lineEnd = ifaceEnd,
+ contentFingerprint = Some(fp)
+ )
+ relationships = relationships :+ ParsedRelationship(fileName, ifaceName, "CONTAINS")
+ }
+ }
+
+ // Check for type alias
+ TypeAliasPattern.findFirstMatchIn(line).foreach { m =>
+ val typeName = m.group(1)
+ if (!trimmed.startsWith("import")) {
+ val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
+ val fp = Fingerprint.compute(trimmed.take(120), trimmed)
+ entities = entities :+ ParsedEntity(
+ name = typeName,
+ kind = NodeKind.Class,
+ attrs = Map(
+ "language" -> Json.fromString("typescript"),
+ "ts_kind" -> Json.fromString("type_alias"),
+ "exported" -> Json.fromBoolean(exported)
+ ),
+ lineStart = lineNum,
+ lineEnd = lineNum,
+ contentFingerprint = Some(fp)
+ )
+ relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
+ }
+ }
+
+ } // end else (non-comment)
+ }
+
+ // Extract file-level calls from code outside named functions/methods/classes
+ val coveredRanges: Vector[(Int, Int)] = entities
+ .filter(e => e.kind == NodeKind.Function || e.kind == NodeKind.Method || e.kind == NodeKind.Class)
+ .map(e => (e.lineStart, e.lineEnd))
+
+ val fileCallsSeen = scala.collection.mutable.Set.empty[String]
+ for ((line, idx) <- lines.zipWithIndex) {
+ val lineNum = idx + 1
+ val inNamedBody = coveredRanges.exists { case (s, e) => lineNum >= s && lineNum <= e }
+ if (!inNamedBody && !line.trim.startsWith("//") && !line.trim.startsWith("*")) {
+ CallPattern.findAllMatchIn(line).foreach { m =>
+ val callee = m.group(1)
+ if (!TypeScriptBuiltins.contains(callee) && !TypeScriptKeywords.contains(callee)
+ && !fileCallsSeen.contains(callee) && callee != fileName) {
+ fileCallsSeen += callee
+ relationships = relationships :+ ParsedRelationship(fileName, callee, "CALLS")
+ }
+ }
+ }
+ }
+
+ ParseResult(entities, relationships)
+ }
+
+ // ---------------------------------------------------------------------------
+ // Patterns
+ // ---------------------------------------------------------------------------
+
+ private val ClassPattern: Regex = """(?:export\s+)?(?:abstract\s+)?class\s+(\w+)""".r
+ private val FuncPattern: Regex = """(?:export\s+)?(?:async\s+)?function\s+(\w+)""".r
+ private val ArrowFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[a-zA-Z_]\w*)\s*(?::\s*[^=]+)?\s*=>""".r
+ private val ConstFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?function""".r
+ private val MethodPattern: Regex = """^\s+(?:async\s+)?(?:private\s+|public\s+|protected\s+|static\s+|readonly\s+)*(\w+)\s*(?:<[^>]+>)?\s*\(""".r
+ private val ImportPattern: Regex = """import\s+.*?from\s+['"]([^'"]+)['"]""".r
+ private val NamedImportPattern: Regex = """import\s+\{([^}]+)\}\s+from\s+['"]([^'"]+)['"]""".r
+ private val SideEffectImportPattern: Regex = """^import\s+['"]([^'"]+)['"]""".r
+ private val InterfacePattern: Regex = """(?:export\s+)?interface\s+(\w+)""".r
+ private val TypeAliasPattern: Regex = """(?:export\s+)?type\s+(\w+)\s*(?:<[^>]+>)?\s*=""".r
+ private val CallPattern: Regex = """(?:(?:this|self|await)\.)?\b(\w+)\s*\(""".r
+
+ // ---------------------------------------------------------------------------
+ // Helpers
+ // ---------------------------------------------------------------------------
+
+ private def insideClass(lineNum: Int, classRanges: Vector[(String, Int, Int)]): Boolean =
+ classRanges.exists { case (_, start, end) => lineNum > start && lineNum <= end }
+
+ private def extractVisibility(trimmedLine: String): String = {
+ if (trimmedLine.startsWith("private ") || trimmedLine.startsWith("private\t")) "private"
+ else if (trimmedLine.startsWith("protected ") || trimmedLine.startsWith("protected\t")) "protected"
+ else "public"
+ }
+
+ /**
+ * Extract methods from a class body.
+ */
+ private def extractMethods(
+ lines: Array[String],
+ classStartIdx: Int,
+ classEndLine: Int,
+ className: String
+ ): (Vector[ParsedEntity], Vector[ParsedRelationship]) = {
+ var entities = Vector.empty[ParsedEntity]
+ var relationships = Vector.empty[ParsedRelationship]
+
+ var i = classStartIdx + 1
+ while (i < lines.length && (i + 1) <= classEndLine) {
+ val line = lines(i)
+ val lineNum = i + 1
+
+ if (!line.trim.startsWith("//") && !line.trim.startsWith("*")) {
+ MethodPattern.findFirstMatchIn(line).foreach { m =>
+ val methodName = m.group(1)
+ // Skip keywords that look like methods
+ if (!TypeScriptKeywords.contains(methodName)) {
+ val methodEnd = findBraceBlockEnd(lines, i)
+ val summary = line.trim.take(120)
+ val visibility = extractVisibility(line.trim)
+ val bodyText = lines.slice(i, methodEnd).mkString("\n")
+ val fp = Fingerprint.compute(summary, bodyText)
+ entities = entities :+ ParsedEntity(
+ name = methodName,
+ kind = NodeKind.Method,
+ attrs = Map(
+ "language" -> Json.fromString("typescript"),
+ "summary" -> Json.fromString(summary),
+ "signature" -> Json.fromString(summary),
+ "visibility" -> Json.fromString(visibility)
+ ),
+ lineStart = lineNum,
+ lineEnd = methodEnd,
+ contentFingerprint = Some(fp)
+ )
+ relationships = relationships :+ ParsedRelationship(className, methodName, "CONTAINS")
+
+ // Extract calls within this method
+ val calls = extractCalls(lines, i, methodEnd, methodName)
+ relationships = relationships ++ calls
+ }
+ }
+ }
+ i += 1
+ }
+
+ (entities, relationships)
+ }
+
+ /**
+ * Find end of a brace-delimited block starting near `startIdx`.
+ *
+ * Uses end-of-line brace depth to detect block entry. This correctly
+ * skips inline TypeScript type annotations like `{ kind?: string }`
+ * that balance to zero on a single line, which would otherwise cause
+ * the parser to think the function body ended on the parameter line.
+ */
+ private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Int = {
+ var braceCount = 0
+ var entered = false
+ var i = startIdx
+ while (i < lines.length) {
+ for (ch <- lines(i)) {
+ if (ch == '{') braceCount += 1
+ if (ch == '}') braceCount -= 1
+ }
+ // Only mark as entered when braceCount > 0 at end of line.
+ // This skips inline type annotations like { a: string; b: number }
+ // that balance to 0 within the same line.
+ if (!entered && braceCount > 0) entered = true
+ if (entered && braceCount <= 0) return i + 1
+ i += 1
+ }
+ lines.length
+ }
+
+ /**
+ * Extract CALLS relationships from a function/method body.
+ */
+ private def extractCalls(
+ lines: Array[String],
+ funcStartIdx: Int,
+ funcEndLine: Int,
+ callerName: String
+ ): Vector[ParsedRelationship] = {
+ var calls = Vector.empty[ParsedRelationship]
+ val seen = scala.collection.mutable.Set.empty[String]
+ var i = funcStartIdx + 1
+
+ while (i < lines.length && (i + 1) <= funcEndLine) {
+ val line = lines(i)
+ CallPattern.findAllMatchIn(line).foreach { m =>
+ val callee = m.group(1)
+ if (callee != callerName && !TypeScriptBuiltins.contains(callee) &&
+ !TypeScriptKeywords.contains(callee) && !seen.contains(callee)) {
+ seen += callee
+ calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
+ }
+ }
+ i += 1
+ }
+ calls
+ }
+
+ private val TypeScriptKeywords: Set[String] = Set(
+ "if", "else", "for", "while", "do", "switch", "case", "break",
+ "continue", "return", "throw", "try", "catch", "finally",
+ "new", "delete", "typeof", "instanceof", "void", "in", "of",
+ "class", "extends", "implements", "interface", "enum", "type",
+ "const", "let", "var", "function", "async", "await",
+ "import", "export", "default", "from", "as",
+ "this", "super", "constructor", "static", "private", "public", "protected"
+ )
+
+ private val TypeScriptBuiltins: Set[String] = Set(
+ "console", "log", "error", "warn", "parseInt", "parseFloat",
+ "String", "Number", "Boolean", "Array", "Object", "Promise",
+ "JSON", "Math", "Date", "RegExp", "Error", "Map", "Set",
+ "WeakMap", "WeakSet", "Symbol",
+ "setTimeout", "setInterval", "clearTimeout", "clearInterval",
+ "require", "module", "exports",
+ "fetch", "Response", "Request", "URL", "Buffer", "process",
+ // Array/iterable methods
+ "map", "filter", "reduce", "forEach", "sort", "find", "some", "every",
+ "includes", "indexOf", "slice", "splice", "push", "pop", "shift", "unshift",
+ "concat", "join", "flat", "flatMap", "fill", "from",
+ // String methods
+ "split", "replace", "trim", "toUpperCase", "toLowerCase", "substring",
+ "charAt", "startsWith", "endsWith", "padStart", "padEnd", "match", "test",
+ // Object methods
+ "keys", "values", "entries", "assign", "create", "freeze",
+ "defineProperty", "getOwnPropertyNames", "hasOwnProperty",
+ // Promise methods
+ "then", "catch", "finally", "all", "race", "resolve", "reject"
+ )
+}
diff --git a/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala b/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala
new file mode 100644
index 00000000..b357b6ec
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala
@@ -0,0 +1,270 @@
+package ix.memory.map
+
+import java.util.UUID
+
+import ix.memory.model.NodeId
+
+import scala.util.Random
+
+/**
+ * Louvain community detection with recursive graph coarsening.
+ *
+ * Produces multi-level partitions: each level groups the original file nodes
+ * into progressively coarser communities.
+ *
+ * Algorithm:
+ * 1. Initialize each node in its own community.
+ * 2. For each node (random order), try moving it to a neighbor's community.
+ * Accept if the modularity gain ΔQ > 0.
+ * 3. Repeat until no improvement or max passes reached.
+ * 4. Collapse communities into supernodes; rebuild weighted graph.
+ * 5. Repeat on the coarsened graph.
+ * 6. Project all levels back to original file IDs.
+ *
+ * Modularity gain formula (standard Louvain):
+ * ΔQ(i → C) = k_i_in(C) / m − γ · Σ_tot(C) · k_i / (2m²)
+ *
+ * where:
+ * k_i_in(C) = sum of edge weights from node i to nodes in community C
+ * Σ_tot(C) = sum of all node degrees in C
+ * k_i = degree of node i
+ * m = total edge weight (each undirected edge counted once)
+ * γ = resolution parameter (1.0 by default)
+ */
+object LouvainClustering {
+
+ /** A partition of original file NodeIds into communities at one hierarchy level. */
+ final case class LevelPartition(
+ communities: Vector[Set[NodeId]], // each Set is a community of original file IDs
+ assignment: Map[NodeId, Int] // fileId → community index
+ )
+
+ /**
+ * Run multi-level Louvain clustering.
+ *
+ * Returns a vector of LevelPartitions, finest first (index 0 = most granular).
+ * Each partition covers ALL original file nodes.
+ */
+ def cluster(
+ graph: WeightedFileGraph,
+ maxLevels: Int = 4,
+ resolution: Double = 1.0,
+ minCommunitySize: Int = 2,
+ seed: Long = 42L
+ ): Vector[LevelPartition] = {
+ if (graph.vertices.isEmpty || graph.totalWeight == 0.0) return Vector.empty
+
+ val rng = new Random(seed)
+
+ // nodeExpansion: for the current working graph, maps each node → original file IDs it represents
+ var nodeExpansion: Map[NodeId, Set[NodeId]] =
+ graph.vertices.map(v => v.id -> Set(v.id)).toMap
+
+ var currentGraph = graph
+ val levels = scala.collection.mutable.ArrayBuffer.empty[LevelPartition]
+
+ var level = 0
+ while (level < maxLevels && currentGraph.vertices.size > 1) {
+ val rawAssignment = louvainPass(currentGraph, resolution, rng)
+
+ // Project raw assignment (current node ids) → original file id communities
+ val projected = projectPartition(rawAssignment, nodeExpansion)
+
+ val meaningful = projected.communities.count(_.size >= minCommunitySize)
+
+ if (meaningful <= 1 && levels.nonEmpty) {
+ // No further useful partitioning — stop
+ level = maxLevels
+ } else {
+ levels += projected
+
+ // Coarsen: collapse each community into a supernode
+ val (coarsenedGraph, newExpansion) =
+ coarsen(currentGraph, rawAssignment, nodeExpansion)
+
+ nodeExpansion = newExpansion
+ currentGraph = coarsenedGraph
+ level += 1
+
+ if (currentGraph.vertices.size < 3) level = maxLevels
+ }
+ }
+
+ levels.toVector
+ }
+
+ // ── Single Louvain pass ────────────────────────────────────────────
+
+ /**
+ * One full Louvain pass on the given graph.
+ * Returns assignment: nodeId → community index (indices are 0-based, dense).
+ */
+ private def louvainPass(
+ graph: WeightedFileGraph,
+ resolution: Double,
+ rng: Random
+ ): Map[NodeId, Int] = {
+ val nodes = graph.vertices.map(_.id)
+ val adj = graph.adjMatrix
+ val m = graph.totalWeight.max(1e-12)
+
+ // Initialize: node i → community i
+ val commOf = scala.collection.mutable.Map(nodes.zipWithIndex: _*)
+ val sumTot = scala.collection.mutable.Map(
+ nodes.zipWithIndex.map { case (n, i) => i -> graph.degrees.getOrElse(n, 0.0) }: _*
+ ).withDefaultValue(0.0)
+ val sumIn = scala.collection.mutable.Map[Int, Double]().withDefaultValue(0.0)
+ // communities: commId → members (mutable set)
+ val members = scala.collection.mutable.Map(
+ nodes.zipWithIndex.map { case (n, i) => i -> scala.collection.mutable.Set(n) }: _*
+ )
+
+ var improved = true
+ var passes = 0
+
+ while (improved && passes < 30) {
+ improved = false
+ passes += 1
+
+ for (node <- rng.shuffle(nodes)) {
+ val ci = commOf(node)
+ val ki = graph.degrees.getOrElse(node, 0.0)
+
+ // Compute weight from this node to each neighboring community
+ val wToComm = scala.collection.mutable.Map[Int, Double]().withDefaultValue(0.0)
+ for ((neighbor, w) <- adj.getOrElse(node, Map.empty)) {
+ wToComm(commOf(neighbor)) += w
+ }
+
+ // Remove node from current community
+ val kiInCi = wToComm.getOrElse(ci, 0.0)
+ sumIn(ci) -= 2.0 * kiInCi
+ sumTot(ci) -= ki
+ members(ci) -= node
+ // Temporarily isolate: treat as its own community (gains = 0 baseline)
+ commOf(node) = -1
+
+ var bestComm = ci
+ var bestGain = 0.0
+
+ for ((cj, kiInCj) <- wToComm if cj != -1) {
+ val dQ = kiInCj / m - resolution * sumTot.getOrElse(cj, 0.0) * ki / (2.0 * m * m)
+ if (dQ > bestGain) {
+ bestGain = dQ
+ bestComm = cj
+ }
+ }
+
+ // Place node in best community (or back in its old one if no improvement)
+ commOf(node) = bestComm
+ members.getOrElseUpdate(bestComm, scala.collection.mutable.Set()) += node
+ val kiInBest = wToComm.getOrElse(bestComm, 0.0)
+ sumIn(bestComm) += 2.0 * kiInBest
+ sumTot(bestComm) += ki
+
+ if (bestComm != ci) improved = true
+ }
+ }
+
+ // Re-index communities from 0 to N-1 (dense)
+ val nonEmpty = members.filter(_._2.nonEmpty).toMap
+ val reindex = nonEmpty.keys.toVector.sorted.zipWithIndex.toMap
+ val finalAssign = commOf.map { case (n, c) =>
+ n -> reindex.getOrElse(c, 0)
+ }.toMap
+
+ finalAssign
+ }
+
+ // ── Projection ─────────────────────────────────────────────────────
+
+ private def projectPartition(
+ assignment: Map[NodeId, Int],
+ nodeExpansion: Map[NodeId, Set[NodeId]]
+ ): LevelPartition = {
+ // Map: community index → set of original file IDs
+ val comms = scala.collection.mutable.Map[Int, scala.collection.mutable.Set[NodeId]]()
+ for ((node, commIdx) <- assignment) {
+ val origFiles = nodeExpansion.getOrElse(node, Set(node))
+ comms.getOrElseUpdate(commIdx, scala.collection.mutable.Set()) ++= origFiles
+ }
+
+ val commVec = comms.toVector.sortBy(_._1).map(_._2.toSet)
+ // Build reverse assignment: origFileId → commIdx
+ val reverseAssign = (for ((members, idx) <- commVec.zipWithIndex; f <- members) yield f -> idx).toMap
+
+ LevelPartition(commVec, reverseAssign)
+ }
+
+ // ── Graph coarsening ────────────────────────────────────────────────
+
+ /**
+ * Collapse each community in `assignment` into a single supernode.
+ * Returns the coarsened graph and an updated nodeExpansion map.
+ */
+ private def coarsen(
+ graph: WeightedFileGraph,
+ assignment: Map[NodeId, Int],
+ nodeExpansion: Map[NodeId, Set[NodeId]]
+ ): (WeightedFileGraph, Map[NodeId, Set[NodeId]]) = {
+ // Group current nodes by community
+ val commGroups: Map[Int, Set[NodeId]] =
+ assignment.groupBy(_._2).map { case (k, v) => k -> v.keySet }
+
+ // Assign a deterministic new NodeId to each community (based on sorted member UUIDs)
+ val commToSuper: Map[Int, NodeId] = commGroups.map { case (ci, members) =>
+ val key = "map:super:" + members.map(_.value.toString).toList.sorted.mkString(",")
+ ci -> NodeId(UUID.nameUUIDFromBytes(key.getBytes("UTF-8")))
+ }
+
+ // Build supernode expansion
+ val newExpansion: Map[NodeId, Set[NodeId]] = commGroups.map { case (ci, members) =>
+ val origFiles = members.flatMap(n => nodeExpansion.getOrElse(n, Set(n)))
+ commToSuper(ci) -> origFiles
+ }
+
+ // Build supernode vertices (use community index as label for debugging)
+ val superVertices = commToSuper.map { case (ci, superId) =>
+ FileVertex(superId, s"supernode_$ci")
+ }.toVector
+
+ // Aggregate inter-community edges
+ val edgeWeights =
+ scala.collection.mutable.Map[(NodeId, NodeId), Double]()
+
+ for ((src, neighbors) <- graph.adjMatrix) {
+ val ci = assignment.getOrElse(src, -1)
+ if (ci >= 0) {
+ val si = commToSuper(ci)
+ for ((dst, w) <- neighbors) {
+ val cj = assignment.getOrElse(dst, -1)
+ if (cj >= 0 && ci != cj) {
+ val sj = commToSuper(cj)
+ val key = if (si.value.compareTo(sj.value) < 0) (si, sj) else (sj, si)
+ edgeWeights(key) = edgeWeights.getOrElse(key, 0.0) + w / 2.0 // undirected: each pair counted twice in adj
+ }
+ }
+ }
+ }
+
+ // Build new undirected adjacency for the coarsened graph
+ val newAdj = scala.collection.mutable.Map[NodeId, scala.collection.mutable.Map[NodeId, Double]]()
+ for (((si, sj), w) <- edgeWeights) {
+ newAdj.getOrElseUpdate(si, scala.collection.mutable.Map())(sj) = w
+ newAdj.getOrElseUpdate(sj, scala.collection.mutable.Map())(si) = w
+ }
+
+ val newAdjFinal = newAdj.map { case (k, m) => k -> m.toMap }.toMap
+ val newDegrees = newAdjFinal.map { case (k, m) => k -> m.values.sum }
+ val newTotalWeight = edgeWeights.values.sum
+
+ val coarsened = WeightedFileGraph(
+ superVertices,
+ newAdjFinal,
+ newDegrees.toMap,
+ newTotalWeight
+ )
+
+ (coarsened, newExpansion)
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala b/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala
new file mode 100644
index 00000000..9a716c4a
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala
@@ -0,0 +1,196 @@
+package ix.memory.map
+
+import cats.effect.IO
+
+import ix.memory.db.ArangoClient
+import ix.memory.model.NodeId
+
+/**
+ * Builds a weighted undirected file-level coupling graph from ArangoDB.
+ *
+ * Key design choices:
+ * - Uses provenance.source_uri (absolute path) as the join key between file
+ * nodes and coupling edges, avoiding stale CONTAINS traversals.
+ * - Skips edges where either endpoint has no live provenance (external libs,
+ * tombstoned symbols with no replacement).
+ *
+ * Signal weights (from spec):
+ * α = 1.0 call coupling (highest importance)
+ * β = 0.7 import coupling
+ * γ = 0.9 extends/implements (type/interface cohesion proxy)
+ * ε = 0.3 path proximity (weak structural prior)
+ */
+class MapGraphBuilder(client: ArangoClient) {
+
+ private val Alpha = 1.0
+ private val Beta = 0.7
+ private val Gamma = 0.9
+ private val Epsilon = 0.3
+
+ def build(): IO[WeightedFileGraph] =
+ for {
+ files <- fetchFiles()
+ rawPairs <- fetchCouplingByUri()
+ graph = computeGraph(files, rawPairs)
+ } yield graph
+
+ // ── ArangoDB queries ───────────────────────────────────────────────
+
+ /**
+ * Returns file nodes using provenance.source_uri as the path key.
+ * Files without source_uri are skipped (unusual but defensive).
+ */
+ // Source-code extensions to include in the architectural map.
+ // Docs (.md), scripts (.sh), config (.yml/.json/.toml) etc. are excluded.
+ private val SourceExtensions =
+ Set("scala","java","kt","ts","tsx","js","jsx","py","go","rs","cs","cpp","c","h","hpp","rb")
+
+ private def fetchFiles(): IO[Vector[FileVertex]] =
+ client.query(
+ """FOR n IN nodes
+ | FILTER n.kind == "file"
+ | AND n.deleted_rev == null
+ | AND n.provenance.source_uri != null
+ | RETURN {id: n.logical_id, path: n.provenance.source_uri}""".stripMargin,
+ Map.empty
+ ).map { rows =>
+ rows.flatMap { json =>
+ val c = json.hcursor
+ for {
+ idStr <- c.get[String]("id").toOption
+ path <- c.get[String]("path").toOption
+ ext = path.split("\\.").lastOption.getOrElse("").toLowerCase
+ if SourceExtensions.contains(ext)
+ uuid <- try Some(java.util.UUID.fromString(idStr)) catch { case _: Exception => None }
+ } yield FileVertex(NodeId(uuid), path)
+ }.toVector
+ }
+
+ /**
+ * Returns cross-file coupling pairs keyed by absolute provenance.source_uri.
+ *
+ * Builds a unified logical_id → source_uri map over all live non-metadata
+ * nodes (files AND symbols), then resolves coupling edge endpoints via that
+ * map. Stale edges whose endpoints have been tombstoned are naturally
+ * excluded because deleted nodes have no entry in the map.
+ */
+ private def fetchCouplingByUri(): IO[Vector[RawFilePair]] =
+ client.query(
+ """LET uri_map = MERGE(
+ | FOR n IN nodes
+ | FILTER n.deleted_rev == null
+ | AND n.provenance.source_uri != null
+ | AND n.kind NOT IN ["module","config","config_entry",
+ | "doc","decision","intent","bug","plan",
+ | "task","goal","region"]
+ | RETURN {[n.logical_id]: n.provenance.source_uri}
+ |)
+ |FOR e IN edges
+ | FILTER e.predicate IN ["CALLS","IMPORTS","EXTENDS","IMPLEMENTS"]
+ | AND e.deleted_rev == null
+ | LET su = HAS(uri_map, e.src) ? uri_map[e.src] : null
+ | LET du = HAS(uri_map, e.dst) ? uri_map[e.dst] : null
+ | FILTER su != null AND du != null AND su != du
+ | COLLECT srcUri = su, dstUri = du, pred = e.predicate
+ | WITH COUNT INTO cnt
+ | RETURN {srcUri, dstUri, predicate: pred, count: cnt}""".stripMargin,
+ Map.empty
+ ).map { rows =>
+ rows.flatMap { json =>
+ val c = json.hcursor
+ for {
+ srcUri <- c.get[String]("srcUri").toOption
+ dstUri <- c.get[String]("dstUri").toOption
+ pred <- c.get[String]("predicate").toOption
+ cnt <- c.get[Int]("count").toOption
+ } yield RawFilePair(srcUri, dstUri, pred, cnt)
+ }.toVector
+ }
+
+ // ── Graph construction ─────────────────────────────────────────────
+
+ private def computeGraph(
+ files: Vector[FileVertex],
+ rawPairs: Vector[RawFilePair]
+ ): WeightedFileGraph = {
+ if (files.isEmpty) return WeightedFileGraph.empty
+
+ // Index files by their absolute source_uri path
+ val byUri: Map[String, FileVertex] = files.map(v => v.path -> v).toMap
+
+ // Accumulate coupling counts per (srcUri, dstUri) canonical pair
+ val pairAcc =
+ scala.collection.mutable.Map[(String, String), scala.collection.mutable.Map[String, Int]]()
+
+ for (pair <- rawPairs) {
+ val sv = byUri.get(pair.srcId) // srcId is source_uri here
+ val dv = byUri.get(pair.dstId)
+ (sv, dv) match {
+ case (Some(s), Some(d)) if s.id != d.id =>
+ // Canonical undirected key: lexicographically smaller path first
+ val key = if (s.path < d.path) (s.path, d.path) else (d.path, s.path)
+ val m = pairAcc.getOrElseUpdate(key, scala.collection.mutable.Map())
+ m(pair.predicate) = m.getOrElse(pair.predicate, 0) + pair.count
+ case _ =>
+ }
+ }
+
+ // Build adjacency with composite weights
+ val adjMut =
+ scala.collection.mutable.Map[NodeId, scala.collection.mutable.Map[NodeId, Double]]()
+
+ for (((sp, dp), predicateCounts) <- pairAcc) {
+ val sv = byUri(sp)
+ val dv = byUri(dp)
+
+ val callCount = predicateCounts.getOrElse("CALLS", 0)
+ val importCount = predicateCounts.getOrElse("IMPORTS", 0)
+ val typeCount = predicateCounts.getOrElse("EXTENDS", 0) +
+ predicateCounts.getOrElse("IMPLEMENTS", 0)
+
+ val sCall = Alpha * math.log1p(callCount)
+ val sImport = Beta * math.log1p(importCount)
+ val sType = Gamma * math.log1p(typeCount)
+ val sPath = Epsilon * pathProximity(sv.path, dv.path)
+
+ val w = sCall + sImport + sType + sPath
+ if (w > 0.0) {
+ adjMut.getOrElseUpdate(sv.id, scala.collection.mutable.Map())(dv.id) = w
+ adjMut.getOrElseUpdate(dv.id, scala.collection.mutable.Map())(sv.id) = w
+ }
+ }
+
+ // Path-proximity pass: add structural edges for same-directory file pairs
+ // that have no existing coupling edge. This ensures isolated files (no
+ // CALLS/IMPORTS/EXTENDS edges) can still be clustered by directory.
+ val filesByDir = files.groupBy(v => v.path.split("[/\\\\]").dropRight(1).mkString("/"))
+ for ((_, dirFiles) <- filesByDir if dirFiles.size >= 2) {
+ for (i <- dirFiles.indices; j <- (i + 1) until dirFiles.size) {
+ val sv = dirFiles(i)
+ val dv = dirFiles(j)
+ if (!adjMut.get(sv.id).exists(_.contains(dv.id))) {
+ val w = Epsilon * pathProximity(sv.path, dv.path)
+ if (w > 0.0) {
+ adjMut.getOrElseUpdate(sv.id, scala.collection.mutable.Map())(dv.id) = w
+ adjMut.getOrElseUpdate(dv.id, scala.collection.mutable.Map())(sv.id) = w
+ }
+ }
+ }
+ }
+
+ val adj = adjMut.map { case (k, m) => k -> m.toMap }.toMap
+ val degrees = adj.map { case (k, m) => k -> m.values.sum }
+ val totalWeight = degrees.values.sum / 2.0
+
+ WeightedFileGraph(files, adj, degrees.toMap, totalWeight)
+ }
+
+ /** Path distance heuristic: deeper common prefix → higher proximity. */
+ private def pathProximity(a: String, b: String): Double = {
+ val partsA = a.split("[/\\\\]").dropRight(1)
+ val partsB = b.split("[/\\\\]").dropRight(1)
+ val common = partsA.zip(partsB).takeWhile { case (x, y) => x == y }.length
+ val dist = (partsA.length - common) + (partsB.length - common)
+ 1.0 / (1.0 + dist)
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/map/MapService.scala b/memory-layer/src/main/scala/ix/memory/map/MapService.scala
new file mode 100644
index 00000000..a176a7b5
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/map/MapService.scala
@@ -0,0 +1,461 @@
+package ix.memory.map
+
+import java.time.Instant
+import java.util.UUID
+
+import cats.effect.IO
+import io.circe.Json
+import io.circe.syntax._
+
+import ix.memory.db.{ArangoClient, GraphQueryApi, GraphWriteApi}
+import ix.memory.model._
+
+/**
+ * Full pipeline for ix map:
+ * 1. Build weighted file graph
+ * 2. Detect cross-cutting nodes and apply penalty
+ * 3. Run multi-level Louvain clustering
+ * 4. Compute region metrics (cohesion, coupling, boundary ratio)
+ * 5. Infer labels
+ * 6. Score confidence
+ * 7. Persist Region nodes + IN_REGION edges to ArangoDB
+ * 8. Return ArchitectureMap
+ */
+class MapService(
+ client: ArangoClient,
+ queryApi: GraphQueryApi,
+ writeApi: GraphWriteApi
+) {
+
+ // Cross-cut penalty multiplier (reduces edge weights involving crosscut files)
+ private val CrosscutLambda = 0.5
+ // High-degree threshold: files in the top N% by degree are crosscut candidates
+ private val CrosscutDegreeTop = 0.10
+ // Path tokens that suggest cross-cutting utilities
+ private val CrosscutTokens = Set(
+ "util", "utils", "helper", "helpers", "common", "shared", "base",
+ "logging", "log", "logger", "config", "conf", "constants", "constant",
+ "extensions", "ext", "mixin", "mixins", "core", "framework"
+ )
+
+ private val builder = new MapGraphBuilder(client)
+
+ // Label kinds by level
+ private def labelKind(level: Int, maxLevel: Int): String =
+ if (maxLevel <= 1) "system"
+ else if (level == maxLevel) "system"
+ else if (level == 1) "module"
+ else "subsystem"
+
+ def buildMap(): IO[ArchitectureMap] =
+ for {
+ rev <- queryApi.getLatestRev
+ rawGraph <- builder.build()
+ _ <- IO.pure(()) // no-op; structured for easy step annotation
+ crosscutScores = detectCrosscut(rawGraph)
+ graph = applyPenalty(rawGraph, crosscutScores)
+ levels = LouvainClustering.cluster(graph, maxLevels = 3, minCommunitySize = 3)
+ regions = buildRegions(graph, levels, crosscutScores, rev)
+ _ <- persistMap(regions, rev)
+ } yield ArchitectureMap(regions, rawGraph.vertices.size, rev.value)
+
+ // ── Cross-cutting detection ─────────────────────────────────────────
+
+ private def detectCrosscut(graph: WeightedFileGraph): Map[NodeId, Double] = {
+ if (graph.degrees.isEmpty) return Map.empty
+ val maxDeg = graph.degrees.values.max.max(1e-10)
+ val threshold = maxDeg * (1.0 - CrosscutDegreeTop)
+
+ graph.vertices.map { v =>
+ val deg = graph.degrees.getOrElse(v.id, 0.0)
+ val nameLower = v.path.toLowerCase
+ val pathTokens = nameLower.split("[/\\\\._-]").toSet
+
+ val degreeSignal = if (deg >= threshold) (deg - threshold) / (maxDeg - threshold + 1e-10) else 0.0
+ val lexicalSignal = if (pathTokens.intersect(CrosscutTokens).nonEmpty) 0.5 else 0.0
+
+ val score = (0.7 * degreeSignal + 0.3 * lexicalSignal).min(1.0)
+ v.id -> score
+ }.toMap
+ }
+
+ private def applyPenalty(
+ graph: WeightedFileGraph,
+ scores: Map[NodeId, Double]
+ ): WeightedFileGraph = {
+ val newAdj = graph.adjMatrix.map { case (src, neighbors) =>
+ val sCross = scores.getOrElse(src, 0.0)
+ val adjusted = neighbors.map { case (dst, w) =>
+ val dCross = scores.getOrElse(dst, 0.0)
+ val penalty = (sCross + dCross) / 2.0 * CrosscutLambda
+ dst -> w * (1.0 - penalty)
+ }
+ src -> adjusted
+ }
+ val newDeg = newAdj.map { case (k, m) => k -> m.values.sum }
+ val newTotal = newDeg.values.sum / 2.0
+ graph.copy(adjMatrix = newAdj, degrees = newDeg.toMap, totalWeight = newTotal)
+ }
+
+ // ── Region construction ─────────────────────────────────────────────
+
+ private def buildRegions(
+ graph: WeightedFileGraph,
+ levels: Vector[LouvainClustering.LevelPartition],
+ crosscutScores: Map[NodeId, Double],
+ rev: Rev
+ ): Vector[Region] = {
+ if (levels.isEmpty) return Vector.empty
+
+ val fileVertexById = graph.vertices.map(v => v.id -> v).toMap
+ val maxLevel = levels.size
+
+ // Compute common absolute-path prefix to strip for label inference
+ val allPaths = graph.vertices.map(_.path)
+ val commonPfxLen = commonPathPrefixLength(allPaths)
+
+ // Build regions for each level, finest first.
+ // Apply minCommunitySize at every level — singleton/doubleton regions are noise.
+ val minSize = if (maxLevel >= 3) 3 else 2
+ val allRegions = levels.zipWithIndex.flatMap { case (partition, levelIdx) =>
+ val level = levelIdx + 1
+ partition.communities.filter(_.size >= minSize).map { memberFiles =>
+ buildOneRegion(
+ memberFiles, level, labelKind(level, maxLevel),
+ graph, fileVertexById, crosscutScores, rev, commonPfxLen
+ )
+ }
+ }
+
+ // Wire parent/child relationships
+ // For each region at level N, find which level-(N+1) region contains its files
+ val byLevel: Map[Int, Vector[Region]] =
+ allRegions.groupBy(_.level)
+
+ val withParents = allRegions.map { region =>
+ val parentOpt = byLevel.get(region.level + 1).flatMap { parents =>
+ // Parent must be strictly coarser: strictly larger member set (no equal sets)
+ parents
+ .filter(p => p.memberFiles.size > region.memberFiles.size &&
+ region.memberFiles.subsetOf(p.memberFiles))
+ .minByOption(_.memberFiles.size) // pick tightest enclosing region
+ }
+ region.copy(parentId = parentOpt.map(_.id))
+ }
+
+ // Fill childRegionIds
+ val childrenByParent: Map[NodeId, Set[NodeId]] =
+ withParents.flatMap(r => r.parentId.map(pid => pid -> r.id))
+ .groupBy(_._1)
+ .map { case (k, vs) => k -> vs.map(_._2).toSet }
+
+ val withChildren = withParents.map { r =>
+ r.copy(childRegionIds = childrenByParent.getOrElse(r.id, Set.empty))
+ }
+
+ // Disambiguate duplicate labels within each level.
+ // For each group of same-label regions, find a path segment unique to each.
+ val labelGroups = withChildren.groupBy(r => (r.level, r.label))
+ val needsDisambig = labelGroups.filter(_._2.size > 1)
+
+ if (needsDisambig.isEmpty) withChildren
+ else withChildren.map { r =>
+ needsDisambig.get((r.level, r.label)) match {
+ case Some(group) =>
+ val myPaths = r.memberFiles.flatMap(id => fileVertexById.get(id).map(_.path)).toSet
+ val otherPaths = group.filterNot(_.id == r.id)
+ .flatMap(_.memberFiles.flatMap(id => fileVertexById.get(id).map(_.path))).toSet
+
+ // Only use DIRECTORY segments (not filenames) for disambiguation
+ def dirSegs(paths: Set[String]): Set[String] =
+ paths.flatMap(p => p.split("[/\\\\]").dropRight(1).filterNot(_.isEmpty).map(_.toLowerCase))
+
+ val mySegs = dirSegs(myPaths)
+ val otherSegs = dirSegs(otherPaths)
+ val exclusive = (mySegs -- otherSegs).filterNot(LabelNoiseTokens.contains)
+
+ // Prefer shorter tokens (more likely to be a meaningful module name)
+ val suffix = exclusive.toVector.sortBy(s => (s.length, s)).headOption
+ .map(s => s" / ${s.capitalize}")
+ .getOrElse(s" (${r.memberFiles.size}f)")
+ r.copy(label = r.label + suffix)
+ case None => r
+ }
+ }
+ }
+
+ private val LabelNoiseTokens = Set(
+ "main", "scala", "java", "src", "test", "tests", "__tests__", "kotlin",
+ "python", "js", "ts", "tsx", "jsx", "com", "org", "io", "net",
+ "ix", "memory", "index", "mod", "lib", "pkg", "spec", "dist",
+ "app", "application", "impl", "implementation", "internal", "external",
+ "ix-memory", "memory-layer", "ix-cli", "core-ingestion", "ix-plugin",
+ "projects", "claude"
+ )
+
+ private def buildOneRegion(
+ memberFiles: Set[NodeId],
+ level: Int,
+ kind: String,
+ graph: WeightedFileGraph,
+ fileVertexById: Map[NodeId, FileVertex],
+ crosscutScores: Map[NodeId, Double],
+ rev: Rev,
+ commonPfxLen: Int = 0
+ ): Region = {
+ // Deterministic ID: stable across re-runs with same file set
+ val idKey = "map:region:" + memberFiles.map(_.value.toString).toList.sorted.mkString(",")
+ val nodeId = NodeId(UUID.nameUUIDFromBytes(idKey.getBytes("UTF-8")))
+
+ // Edge weight metrics
+ var internalWeight = 0.0
+ var externalWeight = 0.0
+ var boundaryNodes = Set.empty[NodeId]
+ var signalCounts = Map("calls" -> 0.0, "imports" -> 0.0, "type" -> 0.0)
+
+ for (fileId <- memberFiles) {
+ for ((neighbor, w) <- graph.adjMatrix.getOrElse(fileId, Map.empty)) {
+ if (memberFiles.contains(neighbor)) {
+ internalWeight += w
+ } else {
+ externalWeight += w
+ boundaryNodes += fileId
+ }
+ }
+ }
+ // internalWeight is double-counted (symmetric adj), halve it
+ internalWeight /= 2.0
+
+ val n = memberFiles.size.toDouble.max(1.0)
+ val cohesion = if (n > 1) internalWeight / (n * (n - 1) / 2.0) else 0.0
+ // Cap at 100 to avoid floating-point overflow for fully-isolated clusters
+ val boundaryRatio = if (externalWeight < 1e-6) math.min(internalWeight / 1e-6, 100.0)
+ else internalWeight / externalWeight
+
+ // Cross-cut score: average of member scores
+ val crosscutScore =
+ memberFiles.toList.map(f => crosscutScores.getOrElse(f, 0.0)).sum / n
+
+ val label = inferLabel(memberFiles, fileVertexById, commonPfxLen)
+ val confidence = computeConfidence(cohesion, boundaryRatio, memberFiles.size, crosscutScore)
+
+ Region(
+ id = nodeId,
+ label = label,
+ labelKind = kind,
+ level = level,
+ memberFiles = memberFiles,
+ childRegionIds = Set.empty,
+ parentId = None,
+ cohesion = cohesion,
+ externalCoupling = externalWeight,
+ boundaryRatio = boundaryRatio,
+ confidence = confidence,
+ crosscutScore = crosscutScore,
+ dominantSignals = List("calls", "imports", "type").sortBy(s => -signalCounts.getOrElse(s, 0.0)),
+ interfaceNodeCount = boundaryNodes.size,
+ mapRev = rev.value
+ )
+ }
+
+ // ── Label inference ─────────────────────────────────────────────────
+
+ /**
+ * Infer a human-readable label from file paths in the community.
+ *
+ * Strategy:
+ * 1. Extract all path segments and identifier tokens from filenames.
+ * 2. Score each token by frequency within cluster (TF) ×
+ * specificity (higher if not a common Java/Scala package name).
+ * 3. Return top token, Title-cased.
+ * Fallback: use the most common non-trivial directory name.
+ */
+ private def inferLabel(
+ memberFiles: Set[NodeId],
+ fileVertexById: Map[NodeId, FileVertex],
+ commonPfxLen: Int = 0
+ ): String = {
+ val rawPaths = memberFiles.flatMap(f => fileVertexById.get(f).map(_.path)).toVector
+ if (rawPaths.isEmpty) return "Unknown"
+ // Strip common absolute prefix (e.g. /home/user/project/) so labels are project-relative
+ val paths = rawPaths.map { p =>
+ val segs = p.split("[/\\\\]")
+ if (commonPfxLen > 0 && commonPfxLen < segs.length) segs.drop(commonPfxLen).mkString("/")
+ else p
+ }
+
+ val noiseTokens = LabelNoiseTokens
+
+ // Strategy 1: Longest common directory suffix across all paths
+ val dirSegments = paths.map(p => p.split("[/\\\\]").dropRight(1).toVector)
+ val commonDir = longestCommonSuffix(dirSegments)
+ .filterNot(s => noiseTokens.contains(s.toLowerCase))
+ .headOption
+
+ if (commonDir.isDefined) return commonDir.get.capitalize
+
+ // Strategy 2: Most discriminative non-noise directory segment.
+ // Prefer deeper (more specific) segments by scoring: frequency × depth-weight.
+ // Split each path into (depth, segment) pairs so we can weight deeper dirs higher.
+ val allDirsWithDepth: Vector[(Int, String)] = paths.flatMap { p =>
+ val segs = p.split("[/\\\\]").dropRight(1)
+ segs.zipWithIndex.map { case (s, i) => (i, s) }
+ .filter { case (_, s) => !noiseTokens.contains(s.toLowerCase) && s.nonEmpty }
+ }
+ // Score: count * (1 + depth / maxDepth) — deeper segments score higher
+ val maxDepth = allDirsWithDepth.map(_._1).maxOption.getOrElse(1).toDouble.max(1.0)
+ val scored = allDirsWithDepth.groupBy(_._2).map { case (seg, entries) =>
+ seg -> entries.length * (1.0 + entries.map(_._1 / maxDepth).sum / entries.length)
+ }.toVector.sortBy(-_._2)
+
+ scored.headOption.map(_._1.capitalize).getOrElse {
+ // Strategy 3: Most frequent meaningful filename token
+ val fileTokens = paths.flatMap { p =>
+ val name = p.split("[/\\\\]").lastOption.getOrElse("")
+ val noExt = name.replaceAll("\\.[^.]+$", "")
+ splitCamel(noExt).filter(t => t.length > 3).map(_.toLowerCase)
+ }
+ fileTokens.filterNot(noiseTokens.contains)
+ .groupBy(identity).maxByOption(_._2.size)
+ .map(_._1.capitalize).getOrElse("Region")
+ }
+ }
+
+ /** Returns the number of leading path segments shared by all paths (the common abs prefix). */
+ private def commonPathPrefixLength(paths: Vector[String]): Int = {
+ if (paths.size <= 1) return 0
+ val split = paths.map(_.split("[/\\\\]").toVector)
+ val minLen = split.map(_.length).min
+ (0 until minLen).takeWhile(i => split.map(_(i)).distinct.size == 1).length
+ }
+
+ /** Returns the longest common suffix of all path segment vectors, innermost first. */
+ private def longestCommonSuffix(segLists: Vector[Vector[String]]): Vector[String] = {
+ if (segLists.isEmpty || segLists.exists(_.isEmpty)) return Vector.empty
+ val reversed = segLists.map(_.reverse)
+ val maxLen = reversed.map(_.length).min
+ (0 until maxLen).toVector
+ .takeWhile(i => reversed.map(_(i)).distinct.size == 1)
+ .map(i => reversed.head(i))
+ }
+
+ private def splitCamel(s: String): Array[String] =
+ s.split("(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|[_\\-.]")
+
+ // ── Confidence scoring ──────────────────────────────────────────────
+
+ private def computeConfidence(
+ cohesion: Double,
+ boundaryRatio: Double,
+ size: Int,
+ crosscutScore: Double
+ ): Double = {
+ // Normalize boundary ratio: high ratio → high confidence
+ val brScore = math.tanh(boundaryRatio / 3.0) // saturates around ratio = 6
+ val cohScore = cohesion.min(1.0)
+ val sizeBonus = math.tanh(size / 10.0) // more files → more signal
+ val crossPenalty = crosscutScore * 0.3
+
+ ((0.4 * brScore + 0.3 * cohScore + 0.3 * sizeBonus) - crossPenalty).max(0.0).min(1.0)
+ }
+
+ // ── Persistence ─────────────────────────────────────────────────────
+
+ private def persistMap(regions: Vector[Region], rev: Rev): IO[Unit] = {
+ if (regions.isEmpty) return IO.unit
+
+ for {
+ oldRegions <- queryApi.findNodesByKind(NodeKind.Region, limit = 5000)
+ _ <- if (oldRegions.nonEmpty) submitDeletePatch(oldRegions, rev) else IO.unit
+ _ <- submitUpsertPatch(regions, rev)
+ } yield ()
+ }
+
+ private def submitDeletePatch(oldNodes: Vector[GraphNode], rev: Rev): IO[Unit] = {
+ val ops = oldNodes.map(n => PatchOp.DeleteNode(n.id))
+ val patch = GraphPatch(
+ patchId = PatchId(UUID.randomUUID()),
+ actor = "map-service",
+ timestamp = Instant.now(),
+ source = PatchSource("ix:map", None, "map-service", SourceType.Inferred),
+ baseRev = rev,
+ ops = ops.toVector,
+ replaces = Vector.empty,
+ intent = Some("Clear previous map regions before re-computing")
+ )
+ writeApi.commitPatch(patch).void
+ }
+
+ private def submitUpsertPatch(regions: Vector[Region], rev: Rev): IO[Unit] = {
+ val nodeOps: Vector[PatchOp] = regions.map { r =>
+ PatchOp.UpsertNode(
+ id = r.id,
+ kind = NodeKind.Region,
+ name = r.label,
+ attrs = Map(
+ "map_generated" -> Json.True,
+ "level" -> r.level.asJson,
+ "label_kind" -> r.labelKind.asJson,
+ "file_count" -> r.memberFiles.size.asJson,
+ "member_files" -> r.memberFiles.map(_.value.toString).asJson,
+ "cohesion" -> r.cohesion.asJson,
+ "external_coupling" -> r.externalCoupling.asJson,
+ "boundary_ratio" -> r.boundaryRatio.asJson,
+ "confidence" -> r.confidence.asJson,
+ "crosscut_score" -> r.crosscutScore.asJson,
+ "dominant_signals" -> r.dominantSignals.asJson,
+ "interface_node_count" -> r.interfaceNodeCount.asJson,
+ "map_rev" -> r.mapRev.asJson
+ )
+ )
+ }
+
+ // IN_REGION edges: file → owning region (level-1 regions only)
+ val fileEdgeOps: Vector[PatchOp] = regions
+ .filter(_.level == 1)
+ .flatMap { r =>
+ r.memberFiles.toVector.map { fileId =>
+ val edgeId = EdgeId(
+ UUID.nameUUIDFromBytes(s"in_region:${fileId.value}:${r.id.value}".getBytes("UTF-8"))
+ )
+ PatchOp.UpsertEdge(
+ id = edgeId,
+ src = fileId,
+ dst = r.id,
+ predicate = EdgePredicate("IN_REGION"),
+ attrs = Map.empty
+ )
+ }
+ }
+
+ // IN_REGION edges: child region → parent region
+ val regionEdgeOps: Vector[PatchOp] = regions.flatMap { r =>
+ r.parentId.map { pid =>
+ val edgeId = EdgeId(
+ UUID.nameUUIDFromBytes(s"in_region:${r.id.value}:${pid.value}".getBytes("UTF-8"))
+ )
+ PatchOp.UpsertEdge(
+ id = edgeId,
+ src = r.id,
+ dst = pid,
+ predicate = EdgePredicate("IN_REGION"),
+ attrs = Map.empty
+ )
+ }
+ }
+
+ val allOps = nodeOps ++ fileEdgeOps ++ regionEdgeOps
+ val patch = GraphPatch(
+ patchId = PatchId(UUID.nameUUIDFromBytes(s"map:${rev.value}".getBytes("UTF-8"))),
+ actor = "map-service",
+ timestamp = Instant.now(),
+ source = PatchSource("ix:map", None, "map-service", SourceType.Inferred),
+ baseRev = rev,
+ ops = allOps,
+ replaces = Vector.empty,
+ intent = Some(s"Architectural map computed at rev ${rev.value}")
+ )
+
+ writeApi.commitPatch(patch).void
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/map/MapTypes.scala b/memory-layer/src/main/scala/ix/memory/map/MapTypes.scala
new file mode 100644
index 00000000..2b7dd9b3
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/map/MapTypes.scala
@@ -0,0 +1,61 @@
+package ix.memory.map
+
+import ix.memory.model.NodeId
+
+/** A file node with its UUID and path (used as label source). */
+final case class FileVertex(id: NodeId, path: String)
+
+/** Raw edge from ArangoDB before weight computation. */
+final case class RawFilePair(srcId: String, dstId: String, predicate: String, count: Int)
+
+/** Weighted edge between two file nodes (symmetric — both directions stored). */
+final case class FileCouplingEdge(src: NodeId, dst: NodeId, weight: Double, dominant: String)
+
+/**
+ * Weighted, undirected file-level coupling graph.
+ *
+ * adjMatrix is symmetric: adjMatrix(a)(b) == adjMatrix(b)(a)
+ * totalWeight is the sum of all weights counting each edge once.
+ */
+final case class WeightedFileGraph(
+ vertices: Vector[FileVertex],
+ adjMatrix: Map[NodeId, Map[NodeId, Double]],
+ degrees: Map[NodeId, Double],
+ totalWeight: Double
+)
+
+object WeightedFileGraph {
+ val empty: WeightedFileGraph =
+ WeightedFileGraph(Vector.empty, Map.empty, Map.empty, 0.0)
+}
+
+/**
+ * A single inferred architectural region at one level of the hierarchy.
+ *
+ * `memberFiles` always refers to the original file NodeIds.
+ * `parentId` is set for regions at level >= 2 (their containing region at level+1).
+ */
+final case class Region(
+ id: NodeId,
+ label: String,
+ labelKind: String, // "module" | "subsystem" | "system"
+ level: Int, // 1 = finest, 2 = mid, 3 = coarse
+ memberFiles: Set[NodeId],
+ childRegionIds: Set[NodeId], // direct children at level-1 (empty for level 1)
+ parentId: Option[NodeId],
+ cohesion: Double,
+ externalCoupling: Double,
+ boundaryRatio: Double,
+ confidence: Double,
+ crosscutScore: Double,
+ dominantSignals: List[String],
+ interfaceNodeCount: Int,
+ mapRev: Long
+)
+
+/** The full multi-level architecture map returned by MapService. */
+final case class ArchitectureMap(
+ regions: Vector[Region],
+ fileCount: Int,
+ mapRev: Long
+)
diff --git a/memory-layer/src/main/scala/ix/memory/model/Node.scala b/memory-layer/src/main/scala/ix/memory/model/Node.scala
index 1bff21f3..164417b6 100644
--- a/memory-layer/src/main/scala/ix/memory/model/Node.scala
+++ b/memory-layer/src/main/scala/ix/memory/model/Node.scala
@@ -27,6 +27,7 @@ object NodeKind {
case object Task extends NodeKind
case object Bug extends NodeKind
case object Goal extends NodeKind
+ case object Region extends NodeKind
private val nameMap: Map[String, NodeKind] = Map(
"module" -> Module,
@@ -48,7 +49,8 @@ object NodeKind {
"plan" -> Plan,
"task" -> Task,
"bug" -> Bug,
- "goal" -> Goal
+ "goal" -> Goal,
+ "region" -> Region
)
implicit val encoder: Encoder[NodeKind] = Encoder[String].contramap {
@@ -72,6 +74,7 @@ object NodeKind {
case Task => "task"
case Bug => "bug"
case Goal => "goal"
+ case Region => "region"
}
implicit val decoder: Decoder[NodeKind] = Decoder[String].emap { s =>