Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat: ix map — multi-scale architectural hierarchy from code structure
kagebinary143a20d10844
diff --git a/build.sbt b/build.sbtindex 1b129698..21817676 100644--- a/build.sbt+++ b/build.sbt@@ -29,6 +29,11 @@ lazy val memoryLayer = (project in file("memory-layer"))// ArangoDB"com.arangodb" % "arangodb-java-driver" % "7.12.0",+ // Tree-sitter (bonede/tree-sitter-ng — ARM64 macOS compatible)+ "io.github.bonede" % "tree-sitter" % "0.26.6",+ "io.github.bonede" % "tree-sitter-scala" % "0.24.0",+ "io.github.bonede" % "tree-sitter-typescript" % "0.23.2",+// Logging"org.typelevel" %% "log4cats-slf4j" % "2.7.0","ch.qos.logback" % "logback-classic" % "1.5.6",diff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.tsnew file mode 100644index 00000000..94893094--- /dev/null+++ b/ix-cli/src/cli/commands/map.ts@@ -0,0 +1,140 @@+import type { Command } from "commander";+import chalk from "chalk";+import { IxClient } from "../../client/api.js";+import { getEndpoint } from "../config.js";++interface MapRegion {+ id: string;+ label: string;+ label_kind: string;+ level: number;+ file_count: number;+ child_region_count: number;+ parent_id: string | null;+ cohesion: number;+ external_coupling: number;+ boundary_ratio: number;+ confidence: number;+ crosscut_score: number;+ dominant_signals: string[];+ interface_node_count: number;+ children?: MapRegion[];+}++interface MapResult {+ file_count: number;+ region_count: number;+ levels: number;+ map_rev: number;+ regions: MapRegion[];+ hierarchy: MapRegion[];+}++export function registerMapCommand(program: Command): void {+ program+ .command("map")+ .description("Infer multi-scale architectural hierarchy from code structure")+ .option("--format <fmt>", "Output format (text|json)", "text")+ .option("--level <n>", "Show only regions at this level (1=finest, higher=coarser)")+ .option("--min-confidence <n>", "Only show regions above this confidence threshold (0-1)", "0")+ .addHelpText(+ "after",+ `+Runs Louvain community detection on the weighted file coupling graph to infer+a multi-level architectural hierarchy. Persists results to the graph as Region+nodes with IN_REGION edges.++Levels:+ 1 = module (fine-grained, ~5-20 files)+ 2 = subsystem (mid-level, ~20-100 files)+ 3 = system (top-level architectural regions)++Examples:+ ix map+ ix map --format json+ ix map --level 2+ ix map --min-confidence 0.5`+ )+ .action(async (opts: { format: string; level?: string; minConfidence: string }) => {+ const client = new IxClient(getEndpoint());++ if (opts.format !== "json") {+ process.stderr.write(chalk.dim("Computing architectural map...\n"));+ }++ let result: MapResult;+ try {+ result = await client.map() as MapResult;+ } catch (err: any) {+ console.error(chalk.red("Error:"), err.message);+ process.exitCode = 1;+ return;+ }++ const minConf = parseFloat(opts.minConfidence ?? "0");+ const levelFilter = opts.level ? parseInt(opts.level, 10) : null;++ let regions = result.regions;+ if (levelFilter !== null) regions = regions.filter(r => r.level === levelFilter);+ if (minConf > 0) regions = regions.filter(r => r.confidence >= minConf);++ if (opts.format === "json") {+ console.log(JSON.stringify({ ...result, regions }, null, 2));+ return;+ }++ // Text output+ console.log(+ `\n${chalk.bold("Architectural Map")} ` ++ chalk.dim(`rev ${result.map_rev} · ${result.file_count} files · `) ++ chalk.dim(`${result.region_count} regions · ${result.levels} levels`)+ );++ if (regions.length === 0) {+ console.log(chalk.dim("\n No regions found matching filters."));+ return;+ }++ // Group by level, finest last for display (coarsest first = system overview at top)+ const byLevel = new Map<number, MapRegion[]>();+ for (const r of regions) {+ if (!byLevel.has(r.level)) byLevel.set(r.level, []);+ byLevel.get(r.level)!.push(r);+ }++ const sortedLevels = [...byLevel.keys()].sort((a, b) => b - a); // coarsest first++ for (const lvl of sortedLevels) {+ const lvlRegions = byLevel.get(lvl)!.sort((a, b) => b.confidence - a.confidence);+ const kindLabel = lvlRegions[0]?.label_kind ?? "region";+ console.log(`\n${chalk.bold(kindLabel.toUpperCase())} (level ${lvl})`);++ for (const r of lvlRegions) {+ const confBar = confidenceBar(r.confidence);+ const signals = r.dominant_signals.slice(0, 2).join("+");+ const name = chalk.bold(r.label.padEnd(20));+ const files = chalk.dim(`${r.file_count}f`).padEnd(6);+ const brVal = Math.min(r.boundary_ratio, 9999);+ const br = brVal > 0+ ? chalk.dim(`br=${brVal.toFixed(1)}`)+ : chalk.dim("br=0.0");++ console.log(+ ` ${name} ${files} ${confBar} ${br} ${chalk.dim(signals)}`+ );+ }+ }++ console.log(+ chalk.dim(`\nRegion nodes persisted to graph. Use 'ix entity <id>' to inspect a region.`)+ );+ });+}++/** Render a confidence score as a compact bar: ████░░ */+function confidenceBar(conf: number): string {+ const filled = Math.round(conf * 6);+ const bar = "█".repeat(filled) + "░".repeat(6 - filled);+ const color = conf >= 0.7 ? chalk.green : conf >= 0.4 ? chalk.yellow : chalk.red;+ return color(bar);+}diff --git a/ix-cli/src/cli/register/oss.ts b/ix-cli/src/cli/register/oss.tsindex 6416e11b..fd6c5894 100644--- a/ix-cli/src/cli/register/oss.ts+++ b/ix-cli/src/cli/register/oss.ts@@ -25,6 +25,7 @@ import { registerOverviewCommand } from "../commands/overview.js";import { registerWatchCommand } from "../commands/watch.js";import { registerDockerCommand } from "../commands/docker.js";import { registerWorkflowsHelpCommand } from "../commands/workflows.js";+import { registerMapCommand } from "../commands/map.js";const PRO_COMMANDS: { name: string; desc: string }[] = [{ name: "briefing", desc: "Session-resume briefing" },@@ -76,6 +77,7 @@ export function registerOssCommands(program: Command): void {registerWatchCommand(program);registerDockerCommand(program);registerWorkflowsHelpCommand(program);+ registerMapCommand(program);// Hide advanced commands from default helpconst advancedSet = new Set(ADVANCED_COMMANDS);diff --git a/ix-cli/src/client/api.ts b/ix-cli/src/client/api.tsindex d7a9f953..0e2e9138 100644--- a/ix-cli/src/client/api.ts+++ b/ix-cli/src/client/api.ts@@ -174,6 +174,10 @@ export class IxClient {return this.post("/v1/patch", patch);}+ async map(): Promise<any> {+ return this.post("/v1/map", {});+ }+async stats(): Promise<any> {return this.get("/v1/stats");}diff --git a/memory-layer/src/main/scala/ix/memory/Main.scala b/memory-layer/src/main/scala/ix/memory/Main.scalaindex 50133f18..1643959a 100644--- a/memory-layer/src/main/scala/ix/memory/Main.scala+++ b/memory-layer/src/main/scala/ix/memory/Main.scala@@ -9,6 +9,7 @@ import ix.memory.api.Routesimport ix.memory.conflict.ConflictServiceimport ix.memory.context._import ix.memory.db._+import ix.memory.map.MapServiceobject Main extends IOApp.Simple {@@ -37,9 +38,10 @@ object Main extends IOApp.Simple {contextService = new ContextService(queryApi, seeder, expander,claimCollector, confidenceScorer, conflictDetector)conflictService = new ConflictService(client, queryApi, writeApi)+ mapService = new MapService(client, queryApi, writeApi)// 4. HTTP routes- routes = Routes.all(contextService, queryApi, writeApi, conflictService, client)+ routes = Routes.all(contextService, queryApi, writeApi, conflictService, client, mapService)// 5. Serverserver <- EmberServerBuilderdiff --git a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scalanew file mode 100644index 00000000..d393c47d--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala@@ -0,0 +1,69 @@+package ix.memory.api++import java.nio.file.Paths++import cats.effect.IO+import io.circe.{Decoder, Encoder}+import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}+import org.http4s.HttpRoutes+import org.http4s.circe.CirceEntityCodec._+import org.http4s.dsl.io._++import ix.memory.ingestion.{BulkIngestionService, IngestionProgress, IngestionService}+import org.typelevel.log4cats.slf4j.Slf4jLogger++case class IngestRequest(path: String, language: Option[String], recursive: Option[Boolean], force: Option[Boolean])++object IngestRequest {+ implicit val decoder: Decoder[IngestRequest] = deriveDecoder[IngestRequest]+ implicit val encoder: Encoder[IngestRequest] = deriveEncoder[IngestRequest]+}++case class SkipReasonsResponse(unchanged: Int, emptyFile: Int, parseError: Int, tooLarge: Int)++object SkipReasonsResponse {+ implicit val encoder: Encoder[SkipReasonsResponse] = deriveEncoder[SkipReasonsResponse]+ implicit val decoder: Decoder[SkipReasonsResponse] = deriveDecoder[SkipReasonsResponse]+}++case class IngestResponse(filesProcessed: Int, patchesApplied: Int, filesSkipped: Int,+ entitiesCreated: Int, latestRev: Long, skipReasons: SkipReasonsResponse)++object IngestResponse {+ implicit val encoder: Encoder[IngestResponse] = deriveEncoder[IngestResponse]+ implicit val decoder: Decoder[IngestResponse] = deriveDecoder[IngestResponse]+}++class IngestionRoutes(ingestionService: IngestionService, bulkIngestionService: BulkIngestionService) {++ private val logger = Slf4jLogger.getLoggerFromName[IO]("ix.ingest.progress")++ private val progressLog: IngestionProgress => IO[Unit] = { p =>+ logger.info(s"discovered=${p.filesDiscovered} parsed=${p.filesParsed} chunks=${p.chunksWritten}/${p.totalChunks}")+ }++ val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {+ case req @ POST -> Root / "v1" / "ingest" =>+ (for {+ body <- req.as[IngestRequest]+ _ <- IO.raiseWhen(body.path.contains(".."))(+ new IllegalArgumentException("Path traversal not allowed")+ )+ path = Paths.get(body.path)+ result <- bulkIngestionService.ingestPath(path, body.language, body.recursive.getOrElse(false), progressLog, body.force.getOrElse(false))+ resp <- Ok(IngestResponse(+ filesProcessed = result.filesProcessed,+ patchesApplied = result.patchesApplied,+ filesSkipped = result.filesSkipped,+ entitiesCreated = result.entitiesCreated,+ latestRev = result.latestRev.value,+ skipReasons = SkipReasonsResponse(+ result.skipReasons.unchanged,+ result.skipReasons.emptyFile,+ result.skipReasons.parseError,+ result.skipReasons.tooLarge+ )+ ))+ } yield resp).handleErrorWith(ErrorHandler.handle(_))+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/api/MapRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/MapRoutes.scalanew file mode 100644index 00000000..e4b53ed4--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/api/MapRoutes.scala@@ -0,0 +1,80 @@+package ix.memory.api++import cats.effect.IO+import io.circe.Json+import io.circe.syntax._+import org.http4s.HttpRoutes+import org.http4s.circe.CirceEntityCodec._+import org.http4s.dsl.io._++import ix.memory.map.{ArchitectureMap, MapService, Region}+import ix.memory.model.NodeId++class MapRoutes(mapService: MapService) {++ val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {++ /** POST /v1/map — Run the full map pipeline and return the hierarchy. */+ case POST -> Root / "v1" / "map" =>+ (for {+ archMap <- mapService.buildMap()+ resp <- Ok(encodeMap(archMap))+ } yield resp).handleErrorWith(ErrorHandler.handle(_))+ }++ private def encodeMap(m: ArchitectureMap): Json = {+ val regionsByLevel = m.regions.groupBy(_.level).toVector.sortBy(_._1)++ Json.obj(+ "file_count" -> m.fileCount.asJson,+ "region_count" -> m.regions.size.asJson,+ "levels" -> regionsByLevel.size.asJson,+ "map_rev" -> m.mapRev.asJson,+ "regions" -> m.regions.sortBy(r => (r.level, r.label)).map(encodeRegion).asJson,+ "hierarchy" -> buildHierarchyTree(m).asJson+ )+ }++ private def encodeRegion(r: Region): Json =+ Json.obj(+ "id" -> r.id.value.toString.asJson,+ "label" -> r.label.asJson,+ "label_kind" -> r.labelKind.asJson,+ "level" -> r.level.asJson,+ "file_count" -> r.memberFiles.size.asJson,+ "child_region_count" -> r.childRegionIds.size.asJson,+ "parent_id" -> r.parentId.map(_.value.toString).asJson,+ "cohesion" -> r.cohesion.asJson,+ "external_coupling" -> r.externalCoupling.asJson,+ "boundary_ratio" -> r.boundaryRatio.asJson,+ "confidence" -> r.confidence.asJson,+ "crosscut_score" -> r.crosscutScore.asJson,+ "dominant_signals" -> r.dominantSignals.asJson,+ "interface_node_count" -> r.interfaceNodeCount.asJson+ )++ /**+ * Build a tree structure: top-level regions contain nested children.+ * Returns a JSON array of root regions (no parent), each with a recursive+ * `children` field.+ */+ private def buildHierarchyTree(m: ArchitectureMap): Json = {+ val regionById: Map[NodeId, Region] = m.regions.map(r => r.id -> r).toMap++ // Guard against cycles (can arise when Louvain levels have identical partitions)+ def renderNode(r: Region, visited: Set[NodeId]): Json = {+ val children =+ if (visited.contains(r.id)) Vector.empty+ else r.childRegionIds.toVector+ .flatMap(cid => regionById.get(cid))+ .filter(c => c.level < r.level) // strict level ordering: child must be finer+ .sortBy(_.label)+ .map(c => renderNode(c, visited + r.id))++ encodeRegion(r).deepMerge(Json.obj("children" -> children.asJson))+ }++ val roots = m.regions.filter(_.parentId.isEmpty).sortBy(-_.level)+ roots.map(r => renderNode(r, Set.empty)).asJson+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/api/Routes.scala b/memory-layer/src/main/scala/ix/memory/api/Routes.scalaindex 8ac1c387..f2dd63db 100644--- a/memory-layer/src/main/scala/ix/memory/api/Routes.scala+++ b/memory-layer/src/main/scala/ix/memory/api/Routes.scala@@ -11,6 +11,7 @@ import org.http4s.dsl.io._import ix.memory.conflict.ConflictServiceimport ix.memory.context.ContextServiceimport ix.memory.db.{ArangoClient, GraphQueryApi, GraphWriteApi}+import ix.memory.map.MapServiceobject Routes {@@ -19,7 +20,8 @@ object Routes {queryApi: GraphQueryApi,writeApi: GraphWriteApi,conflictService: ConflictService,- client: ArangoClient+ client: ArangoClient,+ mapService: MapService): HttpRoutes[IO] = {val health = HttpRoutes.of[IO] {@@ -41,10 +43,12 @@ object Routes {val statsRoutes = new StatsRoutes(client).routesval patchCommitRoutes = new PatchCommitRoutes(writeApi).routesval listRoutes = new ListRoutes(queryApi).routes+ val mapRoutes = new MapRoutes(mapService).routeshealth <+> contextRoutes <+> entityRoutes <+>diffRoutes <+> conflictRoutes <+> decideRoutes <+> searchRoutes <+>truthRoutes <+> patchRoutes <+> decisionListRoutes <+> expandRoutes <+>- statsRoutes <+> patchCommitRoutes <+> listRoutes <+> goalRoutes+ statsRoutes <+> patchCommitRoutes <+> listRoutes <+> goalRoutes <+>+ mapRoutes}}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scalanew file mode 100644index 00000000..3d685cf8--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala@@ -0,0 +1,183 @@+package ix.memory.ingestion++import java.nio.file.{Files, Path}++import cats.effect.IO+import cats.syntax.all._+import cats.effect.implicits._+import io.circe.Json+import io.circe.syntax._+import org.typelevel.log4cats.slf4j.Slf4jLogger+import org.typelevel.log4cats.Logger++import ix.memory.db.{BulkWriteApi, CommitResult, CommitStatus, FileBatch, GraphQueryApi}+import ix.memory.model._++private[ingestion] sealed trait ParseOutcome+private[ingestion] case class Parsed(batch: FileBatch) extends ParseOutcome+private[ingestion] case class Skipped(reason: String) extends ParseOutcome++/**+ * High-performance ingestion service that:+ * 1. Discovers files (with optional language filter)+ * 2. Parses files in parallel (bounded by CPU cores)+ * 3. Batches parsed results+ * 4. Bulk-inserts via Document API+ */+class BulkIngestionService(+ parserRouter: ParserRouter,+ bulkWriteApi: BulkWriteApi,+ queryApi: GraphQueryApi+) {++ private val parallelism = Runtime.getRuntime.availableProcessors.max(2)+ private val MaxFileBytes: Long = 1024 * 1024 // 1 MB+ private val mapper = new com.fasterxml.jackson.databind.ObjectMapper()+ private val logger: Logger[IO] = Slf4jLogger.getLoggerFromName[IO]("ix.ingestion")++ /**+ * Ingest files under a path using parallel parsing and bulk writes.+ */+ def ingestPath(+ path: Path,+ language: Option[String],+ recursive: Boolean,+ onProgress: IngestionProgress => IO[Unit] = _ => IO.unit,+ force: Boolean = false+ ): IO[IngestionResult] = {+ for {+ files <- discoverFiles(path, language, recursive)+ _ <- onProgress(IngestionProgress(files.size, 0, 0, 0))+ hashMap <- if (force) IO.pure(Map.empty[String, String]) else loadExistingHashes(files)+ outcomes <- files.parTraverseN(parallelism) { f =>+ parseFile(f, hashMap).handleErrorWith { err =>+ logger.warn(s"Parse error: ${f.toString} — ${err.getMessage}") *>+ IO.pure(Skipped("parseError"): ParseOutcome)+ }+ }+ validBatches = outcomes.collect { case Parsed(b) => b }+ unchangedCount = outcomes.count { case Skipped("unchanged") => true; case _ => false }+ emptyCount = outcomes.count { case Skipped("emptyFile") => true; case _ => false }+ errorCount = outcomes.count { case Skipped("parseError") => true; case _ => false }+ tooLargeCount = outcomes.count { case Skipped("tooLarge") => true; case _ => false }+ skippedCount = unchangedCount + emptyCount + errorCount + tooLargeCount+ totalChunks = if (validBatches.isEmpty) 0 else (validBatches.size + 99) / 100+ _ <- onProgress(IngestionProgress(files.size, outcomes.size, 0, totalChunks))+ latestRev <- queryApi.getLatestRev+ result <- if (validBatches.isEmpty) IO.pure(CommitResult(latestRev, CommitStatus.Ok))+ else commitChunkedWithProgress(validBatches.toVector, latestRev.value, onProgress, files.size, outcomes.size)+ } yield IngestionResult(+ filesProcessed = files.size,+ patchesApplied = validBatches.size,+ filesSkipped = skippedCount,+ entitiesCreated = validBatches.flatMap(_.patch.ops.collect { case _: PatchOp.UpsertNode => 1 }).size,+ latestRev = result.newRev,+ skipReasons = SkipReasons(unchanged = unchangedCount, emptyFile = emptyCount, parseError = errorCount, tooLarge = tooLargeCount)+ )+ }++ private def commitChunkedWithProgress(+ batches: Vector[FileBatch],+ baseRev: Long,+ onProgress: IngestionProgress => IO[Unit],+ discovered: Int,+ parsed: Int+ ): IO[CommitResult] = {+ val chunkSize = 100+ val chunks = batches.grouped(chunkSize).toVector+ val totalChunks = chunks.size++ chunks.zipWithIndex.foldLeft(IO.pure(baseRev)) { case (revIO, (chunk, idx)) =>+ revIO.flatMap { currentRev =>+ bulkWriteApi.commitBatch(chunk, currentRev).flatMap { result =>+ onProgress(IngestionProgress(discovered, parsed, idx + 1, totalChunks)) *>+ IO.pure(result.newRev.value)+ }+ }+ }.map(finalRev => CommitResult(Rev(finalRev), CommitStatus.Ok))+ }++ /**+ * Parse a single file into a FileBatch (or None if unchanged).+ */+ private def parseFile(filePath: Path, hashMap: Map[String, String]): IO[ParseOutcome] = {+ for {+ fileSize <- IO.blocking(Files.size(filePath))+ result <- if (fileSize > MaxFileBytes) IO.pure(Skipped("tooLarge"): ParseOutcome)+ else if (fileSize == 0L) IO.pure(Skipped("emptyFile"): ParseOutcome)+ else {+ for {+ bytes <- IO.blocking(Files.readAllBytes(filePath))+ hash = sha256Bytes(bytes)+ outcome <- hashMap.get(filePath.toString) match {+ case Some(existingHash) if existingHash == hash =>+ IO.pure(Skipped("unchanged"): ParseOutcome)+ case _ =>+ for {+ source <- IO.blocking {+ try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)+ catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }+ }+ parserOpt = parserRouter.parserFor(filePath.toString)+ parseResult = parserOpt match {+ case Some(p) => p.parse(filePath.getFileName.toString, source)+ case None => genericTextParse(filePath.getFileName.toString, source)+ }+ patch = GraphPatchBuilder.build(filePath.toString, Some(hash), parseResult)+ _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))+ provenance = buildProvenanceMap(patch)+ } yield Parsed(FileBatch(filePath.toString, Some(hash), patch, provenance)): ParseOutcome+ }+ } yield outcome+ }+ } yield result+ }++ private def loadExistingHashes(files: List[Path]): IO[Map[String, String]] = {+ val paths = files.map(_.toString)+ queryApi.getSourceHashes(paths)+ }++ private def buildProvenanceMap(patch: GraphPatch): java.util.Map[String, AnyRef] = {+ val json = Json.obj(+ "source_uri" -> Json.fromString(patch.source.uri),+ "source_hash" -> patch.source.sourceHash.fold(Json.Null)(Json.fromString),+ "extractor" -> Json.fromString(patch.source.extractor),+ "source_type" -> Json.fromString(patch.source.sourceType.asJson.asString.getOrElse("code")),+ "observed_at" -> Json.fromString(patch.timestamp.toString)+ )+ mapper.readValue(json.noSpaces, classOf[java.util.Map[String, AnyRef]])+ }++ private def discoverFiles(+ path: Path,+ language: Option[String],+ recursive: Boolean+ ): IO[List[Path]] = IO.blocking {+ val extensions = FileDiscovery.extensionsFor(language)+ if (Files.isRegularFile(path)) {+ if (FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty+ } else if (Files.isDirectory(path)) {+ // Try git ls-files first (respects .gitignore), fall back to manual walk+ FileDiscovery.tryGitLsFiles(path, recursive, extensions)+ .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))+ } else List.empty+ }++ private def sha256Bytes(bytes: Array[Byte]): String = {+ val md = java.security.MessageDigest.getInstance("SHA-256")+ md.digest(bytes).map("%02x".format(_)).mkString+ }++ private def genericTextParse(fileName: String, source: String): ParseResult = {+ val lines = if (source.isEmpty) 1 else source.count(_ == '\n') + 1+ ParseResult(+ entities = Vector(ParsedEntity(+ name = fileName, kind = NodeKind.File,+ attrs = Map("content" -> Json.fromString(source)),+ lineStart = 1, lineEnd = lines+ )),+ relationships = Vector.empty+ )+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala b/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scalanew file mode 100644index 00000000..2a20186a--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala@@ -0,0 +1,142 @@+package ix.memory.ingestion++import java.nio.file.{Files, Path}++/**+ * Discovers source files for ingestion.+ *+ * Primary strategy: `git ls-files` — respects .gitignore, fast, works for any repo.+ * Fallback: manual file walk with hardcoded ignore list (for non-git directories).+ */+object FileDiscovery {++ private val defaultExtensions = Set(+ // code+ ".py", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",+ ".java", ".scala", ".sc", ".go", ".rs", ".c", ".h", ".cc", ".cpp", ".hpp",+ ".kt", ".kts", ".swift", ".rb", ".php", ".cs", ".ex", ".exs",+ ".lua", ".zig", ".nim", ".jl", ".r", ".R", ".pl", ".pm",+ ".sh", ".bash", ".zsh", ".fish",++ // config / data+ ".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".env", ".properties",+ ".xml", ".plist", ".gradle",++ // docs+ ".md", ".mdx", ".rst", ".txt",++ // build+ ".sbt", ".gradle", ".cmake"+ )++ /** Files matched by name regardless of extension. */+ private val specialFiles = Set(+ "Dockerfile", "Makefile", "Rakefile", "Gemfile", "Procfile",+ "README", "LICENSE", "NOTICE", "CHANGELOG",+ "Cargo.toml", "go.mod", "go.sum",+ "package.json", "tsconfig.json", "pyproject.toml", "setup.py", "setup.cfg",+ "build.sbt", "build.gradle", "pom.xml", "CMakeLists.txt",+ ".gitignore", ".dockerignore", ".editorconfig"+ )++ def extensionsFor(language: Option[String]): Set[String] = language match {+ case Some("python") => Set(".py")+ case Some("typescript") => Set(".ts", ".tsx")+ case Some("javascript") => Set(".js", ".jsx", ".mjs", ".cjs")+ case Some("scala") => Set(".scala", ".sc")+ case Some("java") => Set(".java")+ case Some("go") => Set(".go")+ case Some("rust") => Set(".rs")+ case Some("ruby") => Set(".rb")+ case Some("config") => Set(".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".properties", ".xml")+ case Some("markdown") => Set(".md", ".mdx")+ case _ => defaultExtensions+ }++ def matchesFilter(file: Path, extensions: Set[String]): Boolean = {+ val name = file.getFileName.toString+ specialFiles.contains(name) || extensions.exists(name.endsWith)+ }++ /**+ * Use `git ls-files` to discover tracked files. Returns None if not a git repo+ * or git is not available.+ */+ def tryGitLsFiles(dir: Path, recursive: Boolean, extensions: Set[String]): Option[List[Path]] = {+ try {+ val pb = new ProcessBuilder("git", "ls-files", "--cached", "--others", "--exclude-standard")+ pb.directory(dir.toFile)+ pb.redirectErrorStream(true)+ val proc = pb.start()+ val reader = new java.io.BufferedReader(+ new java.io.InputStreamReader(proc.getInputStream, java.nio.charset.StandardCharsets.UTF_8)+ )+ val result = scala.collection.mutable.ListBuffer.empty[Path]++ try {+ var line = reader.readLine()+ while (line != null) {+ if (line.nonEmpty) {+ val p = dir.resolve(line).normalize()+ val isUnder = if (recursive) true else p.getParent == dir+ if (isUnder && Files.isRegularFile(p) && matchesFilter(p, extensions)) {+ result += p+ }+ }+ line = reader.readLine()+ }+ } finally {+ reader.close()+ }++ val exitCode = proc.waitFor()+ if (exitCode != 0) None else Some(result.toList)+ } catch {+ case _: Throwable => None+ }+ }++ /**+ * Fallback: walk the directory tree, skipping common non-source directories.+ */+ def walkFiles(dir: Path, recursive: Boolean, extensions: Set[String]): List[Path] = {+ import scala.jdk.CollectionConverters._++ if (!recursive) {+ val stream = Files.list(dir)+ try {+ stream.iterator().asScala+ .filter(Files.isRegularFile(_))+ .filter(matchesFilter(_, extensions))+ .toList+ } finally stream.close()+ } else {+ // Directories to skip when not using git+ val ignoredDirs = Set(+ "node_modules", ".git", ".hg", ".svn",+ "target", "dist", "build", "out", ".next",+ "__pycache__", ".tox", ".venv", "venv", ".mypy_cache", ".pytest_cache",+ ".gradle", ".idea", ".vscode", ".settings", ".vs",+ ".cache", ".parcel-cache", "coverage", ".nyc_output",+ "vendor", "Pods", ".dart_tool", ".pub-cache",+ "bin", "obj", "pkg"+ )+ val result = scala.collection.mutable.ListBuffer.empty[Path]+ Files.walkFileTree(dir, new java.nio.file.SimpleFileVisitor[Path] {+ override def preVisitDirectory(d: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {+ val name = d.getFileName.toString+ if (d != dir && (ignoredDirs.contains(name) || name.startsWith(".")))+ java.nio.file.FileVisitResult.SKIP_SUBTREE+ else+ java.nio.file.FileVisitResult.CONTINUE+ }+ override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {+ if (matchesFilter(file, extensions))+ result += file+ java.nio.file.FileVisitResult.CONTINUE+ }+ })+ result.toList+ }+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala b/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scalanew file mode 100644index 00000000..8a7c4baf--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala@@ -0,0 +1,20 @@+package ix.memory.ingestion++import java.security.MessageDigest++object Fingerprint {++ /** Compute a normalized content fingerprint from signature + body.+ * Returns a 32-char hex string (first 16 bytes of SHA-256).+ */+ def compute(signature: String, body: String): String = {+ val normalized = (signature.trim + "\n" + body.trim)+ .replaceAll("\\s+", " ")+ .trim+ val digest = MessageDigest.getInstance("SHA-256")+ digest.digest(normalized.getBytes("UTF-8"))+ .take(16)+ .map("%02x".format(_))+ .mkString+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scalanew file mode 100644index 00000000..c11587ef--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala@@ -0,0 +1,99 @@+package ix.memory.ingestion++import java.time.Instant+import java.util.UUID++import io.circe.Json++import ix.memory.model._++object GraphPatchBuilder {++ /**+ * Build a GraphPatch from a ParseResult and file metadata.+ *+ * Uses deterministic UUID generation (UUID v3 (MD5-based nameUUID)) so that+ * re-parsing the same file produces the same node/edge IDs, enabling+ * idempotent upserts.+ */+ def build(+ filePath: String,+ sourceHash: Option[String],+ parseResult: ParseResult+ ): GraphPatch = {++ def nodeIdFor(name: String): NodeId =+ NodeId(UUID.nameUUIDFromBytes(s"$filePath:$name".getBytes("UTF-8")))++ def edgeIdFor(src: String, dst: String, predicate: String): EdgeId =+ EdgeId(UUID.nameUUIDFromBytes(s"$filePath:$src:$dst:$predicate".getBytes("UTF-8")))++ val extractor = if (filePath.endsWith(".py")) "tree-sitter-python/1.0"+ else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) "typescript-parser/1.0"+ else if (filePath.endsWith(".scala") || filePath.endsWith(".sc")) "scala-parser/1.0"+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) "config-parser/1.0"+ else if (filePath.endsWith(".md")) "markdown-parser/1.0"+ else "unknown-parser/1.0"++ val nodeOps = parseResult.entities.map { e =>+ val baseAttrs = e.attrs ++ ("line_start" -> Json.fromInt(e.lineStart)) ++ ("line_end" -> Json.fromInt(e.lineEnd))+ val attrsWithFp = e.contentFingerprint match {+ case Some(fp) => baseAttrs + ("content_fingerprint" -> Json.fromString(fp))+ case None => baseAttrs+ }+ PatchOp.UpsertNode(nodeIdFor(e.name), e.kind, e.name, attrsWithFp)+ }++ val edgeOps = parseResult.relationships.map { r =>+ PatchOp.UpsertEdge(+ edgeIdFor(r.srcName, r.dstName, r.predicate),+ nodeIdFor(r.srcName),+ nodeIdFor(r.dstName),+ EdgePredicate(r.predicate),+ Map.empty+ )+ }++ // Generate claims so the confidence/conflict engine has data to operate on.+ val relationshipClaims = parseResult.relationships.map { r =>+ PatchOp.AssertClaim(+ entityId = nodeIdFor(r.srcName),+ field = s"${r.predicate.toLowerCase}:${r.dstName}",+ value = Json.fromString(r.dstName),+ confidence = None // will be computed at query time+ )+ }++ val attributeClaims = parseResult.entities.flatMap { e =>+ e.attrs.flatMap { case (key, value) =>+ Vector(PatchOp.AssertClaim(+ entityId = nodeIdFor(e.name),+ field = key,+ value = value,+ confidence = None+ ))+ }+ }++ val claimOps = relationshipClaims ++ attributeClaims+ val allOps = nodeOps ++ edgeOps ++ claimOps++ val sourceType = if (filePath.endsWith(".py") || filePath.endsWith(".ts") || filePath.endsWith(".tsx") || filePath.endsWith(".scala") || filePath.endsWith(".sc")) SourceType.Code+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) SourceType.Config+ else if (filePath.endsWith(".md")) SourceType.Doc+ else SourceType.Code++ GraphPatch(+ patchId = PatchId(UUID.randomUUID()),+ actor = "ix/ingestion",+ timestamp = Instant.now(),+ source = PatchSource(filePath, sourceHash, extractor, sourceType),+ baseRev = Rev(0L),+ ops = allOps,+ replaces = Vector.empty,+ intent = Some(s"Parsed $filePath")+ )+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scalanew file mode 100644index 00000000..ea66a6bb--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala@@ -0,0 +1,277 @@+package ix.memory.ingestion++import java.nio.file.{Files, Path}+import java.time.Instant+import java.util.UUID++import cats.effect.IO+import cats.syntax.all._+import ix.memory.db.{CommitResult, GraphQueryApi, GraphWriteApi}+import ix.memory.model._+import io.circe.Json++/**+ * Orchestrates file ingestion: reads source files, parses them,+ * builds graph patches, and commits them through the write API.+ */+class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, queryApi: GraphQueryApi) {++ /**+ * Ingest a single file: read, parse, build patch, commit.+ */+ def ingestFile(filePath: Path): IO[CommitResult] = {+ for {+ bytes <- IO.blocking(Files.readAllBytes(filePath))+ _ <- if (bytes.isEmpty) IO.raiseError(new RuntimeException(s"Skipping empty file: $filePath"))+ else IO.unit+ // Hash raw bytes so revisions are stable across encodings+ hash = Some(sha256Bytes(bytes))+ // Decode text with UTF-8, fallback to ISO-8859-1 to avoid hard failures+ source <- IO.blocking {+ try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)+ catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }+ }+ // Use the language-specific parser when available; otherwise fall back to generic text+ parserOpt = parserRouter.parserFor(filePath.toString)+ result = parserOpt match {+ case Some(p) => p.parse(filePath.getFileName.toString, source)+ case None => genericTextParse(filePath.getFileName.toString, source)+ }+ val fp = filePath.toString+ extractor = if (fp.endsWith(".py")) "tree-sitter-python/1.0"+ else if (fp.endsWith(".ts") || fp.endsWith(".tsx")) "typescript-parser/1.0"+ else if (fp.endsWith(".scala") || fp.endsWith(".sc")) "scala-parser/1.0"+ else if (fp.endsWith(".json") || fp.endsWith(".yaml") || fp.endsWith(".yml") || fp.endsWith(".toml") || fp.endsWith(".ini") || fp.endsWith(".conf") || fp.endsWith(".properties") || fp.endsWith(".env")) "config-parser/1.0"+ else if (fp.endsWith(".md") || fp.endsWith(".mdx") || fp.endsWith(".rst") || fp.endsWith(".txt")) "markdown-parser/1.0"+ else "text-parser/1.0"+ existing <- queryApi.getPatchesBySource(filePath.toString, extractor)+ prevPatch = existing.headOption+ prevHash = prevPatch.flatMap(_.hcursor.downField("data").downField("source").get[String]("sourceHash").toOption)+ // If hash unchanged, skip (idempotent)+ _ <- if (prevHash == hash) IO.raiseError(new RuntimeException("Idempotent — file unchanged"))+ else IO.unit+ prevPatchId = prevPatch.flatMap(_.hcursor.downField("data").get[String]("patchId").toOption)+ .flatMap(s => scala.util.Try(java.util.UUID.fromString(s)).toOption)+ .map(PatchId(_))+ replaces = prevPatchId.toVector+ patch = GraphPatchBuilder.build(filePath.toString, hash, result).copy(replaces = replaces)+ _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))+ commit <- writeApi.commitPatch(patch)+ // Resolve cross-file import edges after main patch is committed+ _ <- resolveImportEdges(filePath, result, hash).attempt.void+ // Resolve named import symbols to their actual source entities+ _ <- resolveNamedImports(filePath, result, hash).attempt.void+ } yield commit+ }++ /**+ * Ingest all matching files under a path.+ *+ * @param path root directory or single file+ * @param language optional language filter (e.g. "python")+ * @param recursive whether to recurse into subdirectories+ * @return aggregated ingestion result+ */+ def ingestPath(+ path: Path,+ language: Option[String],+ recursive: Boolean+ ): IO[IngestionResult] = {+ for {+ files <- discoverFiles(path, language, recursive)+ results <- files.traverse(f => ingestFile(f).attempt)+ successes = results.collect { case Right(cr) => cr }+ idempotentCount = results.count {+ case Left(e: RuntimeException) if e.getMessage.startsWith("Idempotent") => true+ case _ => false+ }+ latestRev = successes.lastOption.map(_.newRev).getOrElse(Rev(0L))+ } yield IngestionResult(+ filesProcessed = files.size,+ patchesApplied = successes.size,+ filesSkipped = idempotentCount,+ entitiesCreated = 0, // Would need to count from parse results+ latestRev = latestRev+ )+ }++ private def discoverFiles(+ path: Path,+ language: Option[String],+ recursive: Boolean+ ): IO[List[Path]] = IO.blocking {+ val extensions = FileDiscovery.extensionsFor(language)+ if (Files.isRegularFile(path)) {+ if (FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty+ } else if (Files.isDirectory(path)) {+ FileDiscovery.tryGitLsFiles(path, recursive, extensions)+ .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))+ } else {+ List.empty+ }+ }++ private def resolveImportEdges(+ filePath: Path,+ parseResult: ParseResult,+ sourceHash: Option[String]+ ): IO[Unit] = {+ val importRels = parseResult.relationships.filter(_.predicate == "IMPORTS")+ importRels.traverse_ { rel =>+ val targetName = rel.dstName+ // Try to resolve relative imports (./config, ../utils)+ val resolvedPath = resolveRelativePath(filePath, targetName)+ resolvedPath match {+ case Some(resolved) =>+ queryApi.searchNodes(resolved.getFileName.toString, limit = 1).flatMap {+ case nodes if nodes.nonEmpty =>+ val srcId = NodeId(UUID.nameUUIDFromBytes(s"${filePath}:${filePath.getFileName}".getBytes("UTF-8")))+ val dstId = nodes.head.id+ val edgeId = EdgeId(UUID.nameUUIDFromBytes(s"${srcId.value}:${dstId.value}:IMPORTS".getBytes("UTF-8")))+ val patch = GraphPatch(+ patchId = PatchId(UUID.randomUUID()),+ actor = "ix/cross-file",+ timestamp = Instant.now(),+ source = PatchSource(filePath.toString, sourceHash, "cross-file-resolver/1.0", SourceType.Code),+ baseRev = Rev(0L),+ ops = Vector(PatchOp.UpsertEdge(edgeId, srcId, dstId, EdgePredicate("IMPORTS"), Map.empty)),+ replaces = Vector.empty,+ intent = Some(s"Cross-file import: ${filePath.getFileName} -> ${resolved.getFileName}")+ )+ writeApi.commitPatch(patch).void+ case _ => IO.unit+ }+ case None => IO.unit+ }+ }+ }++ private def resolveNamedImports(+ filePath: Path,+ parseResult: ParseResult,+ sourceHash: Option[String]+ ): IO[Unit] = {+ val namedImports = parseResult.relationships.filter { rel =>+ rel.predicate == "IMPORTS" && !rel.dstName.contains("/") && !rel.dstName.contains(".")+ }+ namedImports.traverse_ { rel =>+ queryApi.searchNodes(rel.dstName, limit = 5, nameOnly = true).flatMap { nodes =>+ val external = nodes.filter { n =>+ n.name == rel.dstName &&+ n.provenance.sourceUri != filePath.toString+ }+ external.headOption match {+ case Some(target) =>+ val srcId = NodeId(UUID.nameUUIDFromBytes(+ s"${filePath}:${filePath.getFileName}".getBytes("UTF-8")))+ val edgeId = EdgeId(UUID.nameUUIDFromBytes(+ s"${srcId.value}:${target.id.value}:IMPORTS".getBytes("UTF-8")))+ val patch = GraphPatch(+ patchId = PatchId(UUID.randomUUID()),+ actor = "ix/cross-file",+ timestamp = Instant.now(),+ source = PatchSource(filePath.toString, sourceHash,+ "cross-file-resolver/1.0", SourceType.Code),+ baseRev = Rev(0L),+ ops = Vector(PatchOp.UpsertEdge(+ edgeId, srcId, target.id, EdgePredicate("IMPORTS"), Map.empty)),+ replaces = Vector.empty,+ intent = Some(s"Named import: ${filePath.getFileName} -> ${rel.dstName}")+ )+ writeApi.commitPatch(patch).void+ case None => IO.unit+ }+ }+ }+ }++ private def resolveRelativePath(from: Path, importTarget: String): Option[Path] = {+ if (importTarget.startsWith("./") || importTarget.startsWith("../")) {+ val base = from.getParent+ if (base == null) return None++ // 1. Try the import target as-is and with source extensions appended+ val directExtensions = List("", ".ts", ".tsx", ".scala", ".py", ".js")+ val directMatch = directExtensions.collectFirst {+ case ext =>+ val candidate = base.resolve(importTarget + ext).normalize()+ if (Files.exists(candidate)) candidate else null+ }.flatMap(Option(_))+ if (directMatch.isDefined) return directMatch++ // 2. Strip compiled extensions (.js/.mjs/.cjs) and try source extensions+ val strippable = List(".js", ".mjs", ".cjs")+ val stripped = strippable.collectFirst {+ case ext if importTarget.endsWith(ext) =>+ importTarget.stripSuffix(ext)+ }+ val strippedMatch = stripped.flatMap { baseName =>+ val sourceExtensions = List(".ts", ".tsx")+ sourceExtensions.collectFirst {+ case ext =>+ val candidate = base.resolve(baseName + ext).normalize()+ if (Files.exists(candidate)) candidate else null+ }.flatMap(Option(_))+ }+ if (strippedMatch.isDefined) return strippedMatch++ // 3. Try directory index files (e.g., ./utils -> ./utils/index.ts)+ val indexExtensions = List("index.ts", "index.tsx", "index.js")+ indexExtensions.collectFirst {+ case idx =>+ val candidate = base.resolve(importTarget).resolve(idx).normalize()+ if (Files.exists(candidate)) candidate else null+ }.flatMap(Option(_))+ } else None+ }++ private def sha256Bytes(bytes: Array[Byte]): String = {+ val md = java.security.MessageDigest.getInstance("SHA-256")+ md.digest(bytes).map("%02x".format(_)).mkString+ }++ /**+ * Generic fallback parse: ensures every text file produces at least a File entity with content.+ * This avoids needing a bespoke parser for every dev/text extension.+ */+ private def genericTextParse(fileName: String, source: String): ParseResult = {+ val lines = if (source.isEmpty) 1 else source.count(_ == '\n') + 1+ ParseResult(+ entities = Vector(+ ParsedEntity(+ name = fileName,+ kind = NodeKind.File,+ attrs = Map(+ "content" -> Json.fromString(source)+ ),+ lineStart = 1,+ lineEnd = lines+ )+ ),+ relationships = Vector.empty+ )+ }+}++case class IngestionProgress(+ filesDiscovered: Int,+ filesParsed: Int,+ chunksWritten: Int,+ totalChunks: Int+)++case class SkipReasons(+ unchanged: Int = 0,+ emptyFile: Int = 0,+ parseError: Int = 0,+ tooLarge: Int = 0+)++case class IngestionResult(+ filesProcessed: Int,+ patchesApplied: Int,+ filesSkipped: Int,+ entitiesCreated: Int,+ latestRev: Rev,+ skipReasons: SkipReasons = SkipReasons()+)diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scalanew file mode 100644index 00000000..43ba1269--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala@@ -0,0 +1,24 @@+package ix.memory.ingestion++import ix.memory.model.NodeKind+import io.circe.Json++case class ParsedEntity(+ name: String,+ kind: NodeKind,+ attrs: Map[String, Json],+ lineStart: Int,+ lineEnd: Int,+ contentFingerprint: Option[String] = None+)++case class ParsedRelationship(+ srcName: String,+ dstName: String,+ predicate: String+)++case class ParseResult(+ entities: Vector[ParsedEntity],+ relationships: Vector[ParsedRelationship]+)diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scalanew file mode 100644index 00000000..cec94982--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala@@ -0,0 +1,9 @@+package ix.memory.ingestion++/**+ * Common trait for language-specific source code parsers.+ * Both TreeSitterPythonParser and TypeScriptParser implement this.+ */+trait Parser {+ def parse(fileName: String, source: String): ParseResult+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scalanew file mode 100644index 00000000..a5d24632--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala@@ -0,0 +1,25 @@+package ix.memory.ingestion++import ix.memory.ingestion.parsers.{ConfigParser, MarkdownParser, ScalaParser, TreeSitterPythonParser, TypeScriptParser}++/**+ * Routes file paths to the appropriate language parser.+ * Supports Python (.py), TypeScript (.ts/.tsx), config files (.json/.yaml/.yml/.toml),+ * and Markdown (.md).+ */+class ParserRouter {+ private val pythonParser = new TreeSitterPythonParser()+ private val tsParser = new TypeScriptParser()+ private val configParser = new ConfigParser()+ private val markdownParser = new MarkdownParser()+ private val scalaParser = new ScalaParser()++ def parserFor(filePath: String): Option[Parser] = {+ if (filePath.endsWith(".py")) Some(pythonParser)+ else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) Some(tsParser)+ else if (filePath.endsWith(".scala") || filePath.endsWith(".sc")) Some(scalaParser)+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) Some(configParser)+ else if (filePath.endsWith(".md")) Some(markdownParser)+ else None+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala b/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scalanew file mode 100644index 00000000..f2823309--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala@@ -0,0 +1,16 @@+package ix.memory.ingestion++import ix.memory.model.GraphPatch++/**+ * Simple validation for graph patches before committing.+ */+object PatchValidator {++ def validate(patch: GraphPatch): Either[String, GraphPatch] = {+ if (patch.ops.isEmpty)+ Left("Patch has no operations")+ else+ Right(patch)+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scalanew file mode 100644index 00000000..bd396ed7--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala@@ -0,0 +1,190 @@+package ix.memory.ingestion.parsers++import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}+import ix.memory.model.NodeKind+import io.circe.Json++/**+ * Parser for configuration files (JSON, YAML, TOML).+ *+ * Flattens nested structures into dot-path entries so that each leaf value+ * becomes a ConfigEntry node with a meaningful value (never "{" or "[").+ */+class ConfigParser extends Parser {++ private val JsonExtensions = Set(".json")+ private val YamlExtensions = Set(".yaml", ".yml")+ private val TomlExtensions = Set(".toml")++ def canParse(fileName: String): Boolean = {+ val lower = fileName.toLowerCase+ (JsonExtensions ++ YamlExtensions ++ TomlExtensions).exists(ext => lower.endsWith(ext))+ }++ def parse(fileName: String, source: String): ParseResult = {+ val lower = fileName.toLowerCase++ val entries: Vector[(String, String, Int)] =+ if (JsonExtensions.exists(ext => lower.endsWith(ext))) parseJson(source)+ else if (YamlExtensions.exists(ext => lower.endsWith(ext))) parseYaml(source)+ else if (TomlExtensions.exists(ext => lower.endsWith(ext))) parseToml(source)+ else Vector.empty++ buildResult(fileName, source, entries)+ }++ // ---------------------------------------------------------------------------+ // Result builder+ // ---------------------------------------------------------------------------++ private def buildResult(+ fileName: String,+ source: String,+ entries: Vector[(String, String, Int)]+ ): ParseResult = {+ val lines = source.split("\n", -1)++ val configEntity = ParsedEntity(+ name = fileName,+ kind = NodeKind.Config,+ attrs = Map("format" -> Json.fromString(detectFormat(fileName))),+ lineStart = 1,+ lineEnd = lines.length+ )++ val entryEntities = entries.map { case (key, value, line) =>+ ParsedEntity(+ name = key,+ kind = NodeKind.ConfigEntry,+ attrs = Map("value" -> Json.fromString(value)),+ lineStart = line,+ lineEnd = line+ )+ }++ val relationships = entries.map { case (key, _, _) =>+ ParsedRelationship(fileName, key, "CONTAINS")+ }++ ParseResult(+ entities = configEntity +: entryEntities,+ relationships = relationships+ )+ }++ private def detectFormat(fileName: String): String = {+ val lower = fileName.toLowerCase+ if (JsonExtensions.exists(ext => lower.endsWith(ext))) "json"+ else if (YamlExtensions.exists(ext => lower.endsWith(ext))) "yaml"+ else if (TomlExtensions.exists(ext => lower.endsWith(ext))) "toml"+ else "unknown"+ }++ // ---------------------------------------------------------------------------+ // JSON parsing — Circe-based with flattening+ // ---------------------------------------------------------------------------++ private def parseJson(source: String): Vector[(String, String, Int)] = {+ io.circe.parser.parse(source) match {+ case Right(json) => flattenJson("", json, 1)+ case Left(_) => regexParseJson(source)+ }+ }++ private def flattenJson(prefix: String, json: io.circe.Json, line: Int): Vector[(String, String, Int)] = {+ json.fold(+ jsonNull = Vector.empty,+ jsonBoolean = b => Vector((prefix, b.toString, line)),+ jsonNumber = n => Vector((prefix, n.toString, line)),+ jsonString = s => Vector((prefix, s, line)),+ jsonArray = arr => arr.zipWithIndex.flatMap { case (v, i) =>+ val key = if (prefix.isEmpty) s"[$i]" else s"$prefix[$i]"+ flattenJson(key, v, line)+ }.toVector,+ jsonObject = obj => obj.toVector.flatMap { case (k, v) =>+ val key = if (prefix.isEmpty) k else s"$prefix.$k"+ v.fold(+ jsonNull = Vector((key, "null", line)),+ jsonBoolean = b => Vector((key, b.toString, line)),+ jsonNumber = n => Vector((key, n.toString, line)),+ jsonString = s => Vector((key, s, line)),+ jsonArray = _ => flattenJson(key, v, line),+ jsonObject = _ => flattenJson(key, v, line)+ )+ }+ )+ }++ /** Regex fallback for malformed JSON */+ private def regexParseJson(source: String): Vector[(String, String, Int)] = {+ val KvPattern = """"([^"]+)"\s*:\s*"([^"]*)"""".r+ val lines = source.split("\n", -1)+ lines.zipWithIndex.flatMap { case (line, idx) =>+ KvPattern.findAllMatchIn(line).map { m =>+ (m.group(1), m.group(2), idx + 1)+ }+ }.toVector+ }++ // ---------------------------------------------------------------------------+ // YAML parsing — indentation-aware+ // ---------------------------------------------------------------------------++ private def parseYaml(source: String): Vector[(String, String, Int)] = {+ val lines = source.split("\n", -1)+ val result = Vector.newBuilder[(String, String, Int)]+ val stack = scala.collection.mutable.Stack.empty[(Int, String)] // (indent, prefix)++ val KvWithValue = """^([a-zA-Z_][\w.-]*)\s*:\s*(.+)$""".r+ val KvNoValue = """^([a-zA-Z_][\w.-]*)\s*:\s*$""".r++ lines.zipWithIndex.foreach { case (line, idx) =>+ val trimmed = line.trim+ if (!trimmed.startsWith("#") && trimmed.nonEmpty) {+ val indent = line.takeWhile(_ == ' ').length+ trimmed match {+ case KvWithValue(key, value) =>+ while (stack.nonEmpty && stack.top._1 >= indent) stack.pop()+ val fullKey = if (stack.nonEmpty) s"${stack.top._2}.$key" else key+ result += ((fullKey, value.trim, idx + 1))+ case KvNoValue(key) =>+ while (stack.nonEmpty && stack.top._1 >= indent) stack.pop()+ val fullKey = if (stack.nonEmpty) s"${stack.top._2}.$key" else key+ stack.push((indent, fullKey))+ case _ =>+ // skip list items, comments, etc.+ }+ }+ }+ result.result()+ }++ // ---------------------------------------------------------------------------+ // TOML parsing — section-aware+ // ---------------------------------------------------------------------------++ private def parseToml(source: String): Vector[(String, String, Int)] = {+ val lines = source.split("\n", -1)+ val result = Vector.newBuilder[(String, String, Int)]+ var currentSection = ""++ val SectionPattern = """^\[([^\]]+)\]""".r+ val KvPattern = """^([a-zA-Z_][\w.-]*)\s*=\s*(.+)$""".r++ lines.zipWithIndex.foreach { case (line, idx) =>+ val trimmed = line.trim+ if (!trimmed.startsWith("#") && trimmed.nonEmpty) {+ trimmed match {+ case SectionPattern(section) =>+ currentSection = section+ case KvPattern(key, value) =>+ val fullKey = if (currentSection.isEmpty) key else s"$currentSection.$key"+ val cleanValue = value.trim.stripPrefix("\"").stripSuffix("\"")+ result += ((fullKey, cleanValue, idx + 1))+ case _ =>+ }+ }+ }+ result.result()+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scalanew file mode 100644index 00000000..7d99504c--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala@@ -0,0 +1,78 @@+package ix.memory.ingestion.parsers++import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}+import ix.memory.model.NodeKind+import io.circe.Json++import scala.util.matching.Regex++/**+ * Markdown parser that extracts structural entities from `.md` files.+ *+ * Each heading becomes a Doc entity with attrs for `level`, `title`, and `content`.+ * Content captures all non-header lines in the section, up to 2000 characters.+ * A top-level File entity is also emitted, with CONTAINS edges to each section.+ */+class MarkdownParser extends Parser {++ private val HeadingPattern: Regex = """^(#{1,6})\s+(.+)$""".r++ def parse(fileName: String, source: String): ParseResult = {+ val lines = source.split("\n", -1)++ // -- File entity --+ val fileEntity = ParsedEntity(+ name = fileName,+ kind = NodeKind.File,+ attrs = Map("language" -> Json.fromString("markdown")),+ lineStart = 1,+ lineEnd = lines.length+ )++ // Collect headings with their line indices+ val headings: Vector[(Int, Int, String)] = lines.zipWithIndex.flatMap { case (line, idx) =>+ HeadingPattern.findFirstMatchIn(line.trim).map { m =>+ val level = m.group(1).length+ val title = m.group(2).trim+ (idx, level, title)+ }+ }.toVector++ var entities = Vector(fileEntity)+ var relationships = Vector.empty[ParsedRelationship]++ for (i <- headings.indices) {+ val (startIdx, level, title) = headings(i)+ val endIdx = if (i + 1 < headings.length) headings(i + 1)._1 else lines.length++ val content = extractContent(lines, startIdx, endIdx)++ val entity = ParsedEntity(+ name = title,+ kind = NodeKind.Doc,+ attrs = Map(+ "level" -> Json.fromInt(level),+ "title" -> Json.fromString(title),+ "content" -> Json.fromString(content)+ ),+ lineStart = startIdx + 1,+ lineEnd = endIdx+ )++ entities = entities :+ entity+ relationships = relationships :+ ParsedRelationship(fileName, title, "CONTAINS")+ }++ ParseResult(entities, relationships)+ }++ /**+ * Extract content from all non-header lines in the section, up to 2000 characters.+ */+ private def extractContent(lines: Array[String], startIdx: Int, endIdx: Int): String = {+ val contentLines = lines.slice(startIdx + 1, endIdx).filterNot { line =>+ HeadingPattern.findFirstMatchIn(line.trim).isDefined+ }+ contentLines.mkString("\n").trim.take(2000)+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scalanew file mode 100644index 00000000..deac202c--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala@@ -0,0 +1,357 @@+package ix.memory.ingestion.parsers++import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}+import ix.memory.model.NodeKind+import io.circe.Json++import scala.util.matching.Regex++/**+ * Regex-based Scala source parser that extracts structural entities and relationships.+ *+ * Extracts: traits, classes, case classes, objects, methods (def), and imports.+ * Uses brace counting for block boundary detection.+ */+class ScalaParser extends Parser {++ // Patterns for Scala constructs+ private val TraitPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*trait\s+(\w+)""".r+ private val ClassPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*class\s+(\w+)""".r+ private val CaseClassPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*case\s+class\s+(\w+)""".r+ private val ObjectPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*(?:case\s+)?object\s+(\w+)""".r+ private val DefPattern: Regex = """^\s*(?:override\s+)?(?:private(?:\[\w+\])?\s+|protected(?:\[\w+\])?\s+)?(?:final\s+)?def\s+(\w+)""".r+ private val ImportPattern: Regex = """^\s*import\s+(.+)$""".r+ private val CallPattern: Regex = """(?:(?:this|super)\.)?\b(\w+)\s*(?:\[.*?\])?\s*\(""".r+ private val ExtendsPattern: Regex = """extends\s+(\w+)""".r+ private val TypeRefPattern: Regex = """\b([A-Z]\w+)""".r+ private val QualifiedRefPattern: Regex = """\b([A-Z]\w+)\.(\w+)""".r++ private val ScalaBuiltinTypes: Set[String] = Set(+ "String", "Int", "Long", "Double", "Float", "Boolean", "Unit", "Any", "AnyRef", "AnyVal",+ "Nothing", "Null", "Option", "List", "Map", "Set", "Vector", "Seq", "Array", "Future",+ "IO", "Either", "Try", "Some", "None", "Right", "Left", "Success", "Failure"+ )++ private val ScalaBuiltins: Set[String] = Set(+ "println", "print", "require", "assert", "assume",+ "Some", "None", "Option", "List", "Map", "Set", "Vector",+ "Right", "Left", "Try", "Success", "Failure",+ "Future", "IO", "pure", "flatMap", "map", "filter",+ "foreach", "fold", "foldLeft", "foldRight",+ "toString", "hashCode", "equals", "getClass",+ "apply", "unapply", "copy", "andThen", "compose",+ "getOrElse", "orElse", "contains", "exists", "forall",+ "collect", "traverse", "sequence", "attempt", "handleErrorWith",+ "raiseError", "blocking", "delay", "suspend", "async",+ "mkString", "toList", "toVector", "toSet", "toMap",+ "head", "tail", "size", "length", "isEmpty", "nonEmpty",+ "take", "drop", "slice", "zip", "zipWithIndex",+ "sortBy", "groupBy", "distinct", "flatten"+ )++ def parse(fileName: String, source: String): ParseResult = {+ val lines = source.split("\n", -1)++ val fileEntity = ParsedEntity(+ name = fileName,+ kind = NodeKind.File,+ attrs = Map("language" -> Json.fromString("scala")),+ lineStart = 1,+ lineEnd = lines.length+ )++ var entities = Vector(fileEntity)+ var relationships = Vector.empty[ParsedRelationship]++ // Track type ranges for containment: (name, startLine, endLine)+ var typeRanges = Vector.empty[(String, Int, Int)]++ for ((line, idx) <- lines.zipWithIndex) {+ val lineNum = idx + 1+ val trimmed = line.trim++ // Skip comment lines+ if (trimmed.startsWith("//") || trimmed.startsWith("*") || trimmed.startsWith("/*")) {+ // no-op+ } else {++ // Case class must be checked before class (since "case class" contains "class")+ val handledAsType = CaseClassPattern.findFirstMatchIn(line) match {+ case Some(m) =>+ val name = m.group(1)+ val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)+ val bodyText = lines.slice(idx, endLine).mkString("\n")+ val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)+ entities = entities :+ ParsedEntity(+ name = name,+ kind = NodeKind.Class,+ attrs = Map("scala_kind" -> Json.fromString("case_class"), "language" -> Json.fromString("scala"),+ "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),+ lineStart = lineNum,+ lineEnd = endLine,+ contentFingerprint = Some(fp)+ )+ val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }+ val containsSrc = enclosingType.map(_._1).getOrElse(fileName)+ relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")+ typeRanges = typeRanges :+ (name, lineNum, endLine)+ ExtendsPattern.findFirstMatchIn(line).foreach { em =>+ relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")+ }+ true++ case None =>+ // Check for trait+ TraitPattern.findFirstMatchIn(line) match {+ case Some(m) =>+ val name = m.group(1)+ val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)+ val bodyText = lines.slice(idx, endLine).mkString("\n")+ val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)+ entities = entities :+ ParsedEntity(+ name = name,+ kind = NodeKind.Trait,+ attrs = Map("scala_kind" -> Json.fromString("trait"), "language" -> Json.fromString("scala"),+ "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),+ lineStart = lineNum,+ lineEnd = endLine,+ contentFingerprint = Some(fp)+ )+ val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }+ val containsSrc = enclosingType.map(_._1).getOrElse(fileName)+ relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")+ typeRanges = typeRanges :+ (name, lineNum, endLine)+ ExtendsPattern.findFirstMatchIn(line).foreach { em =>+ relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")+ }+ true++ case None =>+ // Check for object (before class, since object keyword won't match class pattern)+ ObjectPattern.findFirstMatchIn(line) match {+ case Some(m) =>+ val name = m.group(1)+ val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)+ val bodyText = lines.slice(idx, endLine).mkString("\n")+ val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)+ entities = entities :+ ParsedEntity(+ name = name,+ kind = NodeKind.Object,+ attrs = Map("scala_kind" -> Json.fromString("object"), "language" -> Json.fromString("scala"),+ "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),+ lineStart = lineNum,+ lineEnd = endLine,+ contentFingerprint = Some(fp)+ )+ val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }+ val containsSrc = enclosingType.map(_._1).getOrElse(fileName)+ relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")+ typeRanges = typeRanges :+ (name, lineNum, endLine)+ ExtendsPattern.findFirstMatchIn(line).foreach { em =>+ relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")+ }+ true++ case None =>+ // Check for class+ ClassPattern.findFirstMatchIn(line) match {+ case Some(m) =>+ val name = m.group(1)+ val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)+ val bodyText = lines.slice(idx, endLine).mkString("\n")+ val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)+ entities = entities :+ ParsedEntity(+ name = name,+ kind = NodeKind.Class,+ attrs = Map("scala_kind" -> Json.fromString("class"), "language" -> Json.fromString("scala"),+ "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),+ lineStart = lineNum,+ lineEnd = endLine,+ contentFingerprint = Some(fp)+ )+ val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }+ val containsSrc = enclosingType.map(_._1).getOrElse(fileName)+ relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")+ typeRanges = typeRanges :+ (name, lineNum, endLine)+ ExtendsPattern.findFirstMatchIn(line).foreach { em =>+ relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")+ }+ true+ case None => false+ }+ }+ }+ }++ // Check for def (only if we didn't match a type on this line)+ if (!handledAsType) {+ DefPattern.findFirstMatchIn(line).foreach { m =>+ val funcName = m.group(1)+ val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)++ val signature = extractSignature(trimmed)+ val visibility = extractVisibility(trimmed)++ // Determine enclosing type+ val enclosing = typeRanges.find { case (_, start, end) =>+ lineNum > start && lineNum <= end+ }++ val (kind, edgePredicate, edgeSrc) = enclosing match {+ case Some((typeName, _, _)) => (NodeKind.Method, "CONTAINS", typeName)+ case None => (NodeKind.Function, "CONTAINS", fileName)+ }++ val bodyText = lines.slice(idx, endLine).mkString("\n")+ val fp = Fingerprint.compute(signature, bodyText)++ entities = entities :+ ParsedEntity(+ name = funcName,+ kind = kind,+ attrs = Map(+ "language" -> Json.fromString("scala"),+ "summary" -> Json.fromString(signature),+ "signature" -> Json.fromString(signature),+ "visibility" -> Json.fromString(visibility)+ ),+ lineStart = lineNum,+ lineEnd = endLine,+ contentFingerprint = Some(fp)+ )++ relationships = relationships :+ ParsedRelationship(edgeSrc, funcName, edgePredicate)++ // Extract CALLS and REFERENCES from method/function body+ val callsInFunc = extractCalls(lines, idx, endLine, funcName)+ relationships = relationships ++ callsInFunc++ // Extract REFERENCES from method signature type annotations+ val enclosingTypeName = enclosing.map(_._1).getOrElse("")+ val sigRefs = TypeRefPattern.findAllMatchIn(signature).map(_.group(1)).toSet+ for (ref <- sigRefs) {+ if (!ScalaBuiltinTypes.contains(ref) && ref != funcName && ref != enclosingTypeName && ref != fileName) {+ relationships = relationships :+ ParsedRelationship(funcName, ref, "REFERENCES")+ }+ }+ }+ }++ // Check for imports+ ImportPattern.findFirstMatchIn(line).foreach { m =>+ val importPath = m.group(1).trim+ // Parse import path: "ix.memory.model.{Claim, GraphNode}" -> module "ix.memory.model"+ // "cats.effect.IO" -> module "cats.effect.IO"+ val moduleName = if (importPath.contains("{")) {+ // Grouped import: take the prefix before the braces+ importPath.substring(0, importPath.indexOf('{')).stripSuffix(".").trim+ } else {+ importPath+ }++ // Avoid duplicate module entities+ if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {+ entities = entities :+ ParsedEntity(+ name = moduleName,+ kind = NodeKind.Module,+ attrs = Map.empty,+ lineStart = lineNum,+ lineEnd = lineNum+ )+ }+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")+ }++ } // end else (non-comment)+ }++ ParseResult(entities, relationships)+ }++ /**+ * Find the end of a brace-delimited block starting at `startIdx`.+ * Counts opening and closing braces. Returns None if no opening brace is found+ * (e.g. abstract method in a trait with no body).+ */+ private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Option[Int] = {+ var braceCount = 0+ var foundOpen = false+ var i = startIdx++ while (i < lines.length) {+ val line = lines(i)+ for (ch <- line) {+ ch match {+ case '{' =>+ braceCount += 1+ foundOpen = true+ case '}' =>+ braceCount -= 1+ if (foundOpen && braceCount == 0) {+ return Some(i + 1) // 1-indexed+ }+ case _ => // skip+ }+ }+ // If we found an opening brace and closed it, we'd have returned above+ // If line ends and no brace found yet, keep going+ i += 1+ }++ if (foundOpen) Some(lines.length) else None+ }++ /** Extract CALLS and qualified REFERENCES relationships from a method/function body. */+ private def extractCalls(lines: Array[String], startIdx: Int, endIdx: Int, callerName: String): Vector[ParsedRelationship] = {+ var calls = Vector.empty[ParsedRelationship]+ val seen = scala.collection.mutable.Set.empty[String]+ val refSeen = scala.collection.mutable.Set.empty[String]+ for (i <- (startIdx + 1) until endIdx.min(lines.length)) {+ val line = lines(i).trim+ if (!line.startsWith("//") && !line.startsWith("*")) {+ CallPattern.findAllMatchIn(line).foreach { m =>+ val callee = m.group(1)+ // Only include lowercase-starting identifiers (methods/functions, not types)+ if (callee != callerName && !ScalaBuiltins.contains(callee) && !seen.contains(callee) && callee.head.isLower) {+ seen += callee+ calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")+ }+ }+ // Extract qualified REFERENCES (e.g., NodeKind.File, SourceType.Code)+ QualifiedRefPattern.findAllMatchIn(line).foreach { m =>+ val qualifier = m.group(1)+ if (!ScalaBuiltinTypes.contains(qualifier) && !ScalaBuiltins.contains(qualifier) &&+ !refSeen.contains(qualifier) && qualifier != callerName) {+ refSeen += qualifier+ calls = calls :+ ParsedRelationship(callerName, qualifier, "REFERENCES")+ }+ }+ }+ }+ calls+ }++ /** Extract a type signature (trait/class/object), stripping body braces. */+ private def extractTypeSignature(trimmedLine: String): String = {+ val cleaned = trimmedLine+ .replaceAll("""\s*\{\s*$""", "")+ .trim+ cleaned.take(120)+ }++ /** Extract a clean signature from a trimmed line, stripping trailing body markers. */+ private def extractSignature(trimmedLine: String): String = {+ val cleaned = trimmedLine+ .replaceAll("""\s*=\s*\{?\s*$""", "")+ .replaceAll("""\s*\{\s*$""", "")+ .trim+ cleaned.take(120)+ }++ /** Detect visibility from a trimmed line (after stripping `override`). */+ private def extractVisibility(trimmedLine: String): String = {+ val withoutOverride = trimmedLine.replaceFirst("""^\s*override\s+""", "").trim+ if (withoutOverride.startsWith("private")) "private"+ else if (withoutOverride.startsWith("protected")) "protected"+ else "public"+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scalanew file mode 100644index 00000000..deb06bdd--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala@@ -0,0 +1,250 @@+package ix.memory.ingestion.parsers++import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}+import ix.memory.model.NodeKind+import io.circe.Json++import scala.util.matching.Regex++/**+ * Python source parser that extracts structural entities and relationships.+ *+ * Attempts to use tree-sitter JNI first; falls back to regex-based extraction+ * when the native library is unavailable (common on some platforms).+ * The regex fallback is sufficient for Phase 1 since the goal is the pipeline+ * structure, not production-grade parsing.+ */+class TreeSitterPythonParser extends Parser {++ /**+ * Parse a Python source file and extract entities and relationships.+ *+ * @param fileName the file name (e.g. "billing_service.py")+ * @param source the full source code as a string+ * @return ParseResult with entities and relationships+ */+ def parse(fileName: String, source: String): ParseResult = {+ // Try tree-sitter first, fall back to regex+ tryTreeSitter(fileName, source).getOrElse(regexParse(fileName, source))+ }++ // ---------------------------------------------------------------------------+ // Tree-sitter JNI attempt+ // ---------------------------------------------------------------------------++ private def tryTreeSitter(fileName: String, source: String): Option[ParseResult] = {+ try {+ val langClass = Class.forName("ch.usi.si.seart.treesitter.Language")+ val pythonLang = langClass.getMethod("valueOf", classOf[String]).invoke(null, "PYTHON")++ val parserClass = Class.forName("ch.usi.si.seart.treesitter.Parser")+ val parser = parserClass.getMethod("getFor", langClass).invoke(null, pythonLang)++ val tree = parserClass.getMethod("parse", classOf[String]).invoke(parser, source)+ val treeClass = Class.forName("ch.usi.si.seart.treesitter.Tree")+ val root = treeClass.getMethod("getRootNode").invoke(tree)++ // If we get here, tree-sitter loaded successfully+ // We would walk the AST here, but for now this path is not reached+ // on platforms where the JNI library is unavailable.+ parserClass.getMethod("close").invoke(parser)+ treeClass.getMethod("close").invoke(tree)+ None // TODO: implement full tree-sitter AST walking when JNI works+ } catch {+ case _: Throwable => None // Catches both Exception and Error (e.g. UnsatisfiedLinkError)+ }+ }++ // ---------------------------------------------------------------------------+ // Regex-based fallback parser+ // ---------------------------------------------------------------------------++ private val ClassPattern: Regex = """^class\s+(\w+)""".r+ private val FuncPattern: Regex = """^(\s*)def\s+(\w+)""".r+ private val ImportPattern: Regex = """^import\s+(\w+)""".r+ private val FromImportPattern: Regex = """^from\s+(\w+)\s+import""".r+ private val CallPattern: Regex = """(?:self\.)?(\w+)\(""".r++ private def regexParse(fileName: String, source: String): ParseResult = {+ val lines = source.split("\n", -1)++ // -- File entity --+ val fileEntity = ParsedEntity(+ name = fileName,+ kind = NodeKind.File,+ attrs = Map("language" -> Json.fromString("python")),+ lineStart = 1,+ lineEnd = lines.length+ )++ // -- Classes and functions --+ var entities = Vector(fileEntity)+ var relationships = Vector.empty[ParsedRelationship]++ // Track class ranges: (className, lineStart, lineEnd) for containment checks+ var classRanges = Vector.empty[(String, Int, Int)]++ for ((line, idx) <- lines.zipWithIndex) {+ val lineNum = idx + 1++ // Skip comment lines — prevents matching patterns inside comments+ if (line.trim.startsWith("#")) {+ // no-op: skip this line entirely+ } else {++ // Check for class definition+ ClassPattern.findFirstMatchIn(line).foreach { m =>+ val className = m.group(1)+ val classEnd = findBlockEnd(lines, idx)+ val bodyText = lines.slice(idx, classEnd).mkString("\n")+ val fp = Fingerprint.compute(line.trim.take(120), bodyText)+ entities = entities :+ ParsedEntity(+ name = className,+ kind = NodeKind.Class,+ attrs = Map("language" -> Json.fromString("python")),+ lineStart = lineNum,+ lineEnd = classEnd,+ contentFingerprint = Some(fp)+ )+ // File CONTAINS class+ relationships = relationships :+ ParsedRelationship(fileName, className, "CONTAINS")+ classRanges = classRanges :+ (className, lineNum, classEnd)+ }++ // Check for function/method definition+ FuncPattern.findFirstMatchIn(line).foreach { m =>+ val funcName = m.group(2)+ val funcEnd = findBlockEnd(lines, idx)+ val bodyText = lines.slice(idx, funcEnd).mkString("\n")+ val fp = Fingerprint.compute(line.trim.take(120), bodyText)++ entities = entities :+ ParsedEntity(+ name = funcName,+ kind = NodeKind.Function,+ attrs = Map("language" -> Json.fromString("python")),+ lineStart = lineNum,+ lineEnd = funcEnd,+ contentFingerprint = Some(fp)+ )++ // Check if this function's line falls within any class range+ val enclosingClass = classRanges.find { case (_, start, end) =>+ lineNum > start && lineNum <= end+ }+ enclosingClass match {+ case Some((className, _, _)) =>+ // Method inside a class+ relationships = relationships :+ ParsedRelationship(className, funcName, "CONTAINS")+ case None =>+ // Top-level function — file defines it+ relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")+ }++ // Extract function calls within the function body+ val callsInFunc = extractCalls(lines, idx, funcName)+ relationships = relationships ++ callsInFunc+ }++ // Check for imports+ ImportPattern.findFirstMatchIn(line).foreach { m =>+ val moduleName = m.group(1)+ entities = entities :+ ParsedEntity(+ name = moduleName,+ kind = NodeKind.Module,+ attrs = Map.empty,+ lineStart = lineNum,+ lineEnd = lineNum+ )+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")+ }++ FromImportPattern.findFirstMatchIn(line).foreach { m =>+ val moduleName = m.group(1)+ // Avoid duplicate module entities+ if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {+ entities = entities :+ ParsedEntity(+ name = moduleName,+ kind = NodeKind.Module,+ attrs = Map.empty,+ lineStart = lineNum,+ lineEnd = lineNum+ )+ }+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")+ }++ } // end else (non-comment line)+ }++ ParseResult(entities, relationships)+ }++ /**+ * Find the end line of a block starting at `startIdx` using indentation.+ * Looks for the last line that has greater indentation than the definition line.+ */+ private def findBlockEnd(lines: Array[String], startIdx: Int): Int = {+ val baseIndent = lines(startIdx).takeWhile(_ == ' ').length+ var endIdx = startIdx+ var i = startIdx + 1+ while (i < lines.length) {+ val line = lines(i)+ if (line.trim.nonEmpty) {+ val indent = line.takeWhile(_ == ' ').length+ if (indent > baseIndent) {+ endIdx = i+ } else {+ // Block ended+ return endIdx + 1+ }+ }+ i += 1+ }+ // If we reach end of file, the block extends to the end+ endIdx + 1+ }++ /**+ * Extract CALLS relationships from the body of a function starting at `funcIdx`.+ * Uses a simple heuristic: look for `identifier(` patterns in subsequent indented lines.+ */+ private def extractCalls(lines: Array[String], funcIdx: Int, callerName: String): Vector[ParsedRelationship] = {+ val baseIndent = lines(funcIdx).takeWhile(_ == ' ').length+ var calls = Vector.empty[ParsedRelationship]+ val seen = scala.collection.mutable.Set.empty[String]+ var i = funcIdx + 1++ while (i < lines.length) {+ val line = lines(i)+ if (line.trim.nonEmpty) {+ val indent = line.takeWhile(_ == ' ').length+ if (indent <= baseIndent) {+ // Exited the function body+ return calls+ }+ // Find calls in this line+ CallPattern.findAllMatchIn(line).foreach { m =>+ val callee = m.group(1)+ // Filter out Python builtins and self-references, and skip 'self' prefix calls+ // that match the method name itself+ if (callee != callerName && !PythonBuiltins.contains(callee) && !seen.contains(callee)) {+ seen += callee+ calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")+ }+ }+ }+ i += 1+ }+ calls+ }++ private val PythonBuiltins: Set[String] = Set(+ "print", "len", "range", "int", "str", "float", "list", "dict",+ "set", "tuple", "type", "isinstance", "issubclass", "hasattr",+ "getattr", "setattr", "delattr", "super", "property", "staticmethod",+ "classmethod", "enumerate", "zip", "map", "filter", "sorted",+ "reversed", "any", "all", "min", "max", "sum", "abs", "round",+ "input", "open", "format", "repr", "id", "hash", "callable",+ "iter", "next", "not", "raise"+ )+}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scalanew file mode 100644index 00000000..fc2670a2--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala@@ -0,0 +1,422 @@+package ix.memory.ingestion.parsers++import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}+import ix.memory.model.NodeKind+import io.circe.Json++import scala.util.matching.Regex++/**+ * TypeScript/JavaScript source parser that extracts structural entities+ * and relationships using regex-based extraction.+ */+class TypeScriptParser extends Parser {++ /**+ * Parse a TypeScript source file and extract entities and relationships.+ *+ * @param fileName the file name (e.g. "api.ts")+ * @param source the full source code as a string+ * @return ParseResult with entities and relationships+ */+ def parse(fileName: String, source: String): ParseResult = {+ val lines = source.split("\n", -1)++ // -- File entity (no raw content stored) --+ val fileEntity = ParsedEntity(+ name = fileName,+ kind = NodeKind.File,+ attrs = Map("language" -> Json.fromString("typescript")),+ lineStart = 1,+ lineEnd = lines.length+ )++ var entities = Vector(fileEntity)+ var relationships = Vector.empty[ParsedRelationship]++ // Track class ranges for containment+ var classRanges = Vector.empty[(String, Int, Int)]++ for ((line, idx) <- lines.zipWithIndex) {+ val lineNum = idx + 1+ val trimmed = line.trim++ // Skip comment lines+ if (trimmed.startsWith("//") || trimmed.startsWith("*") || trimmed.startsWith("/*")) {+ // no-op+ } else {++ // Check for class definition+ ClassPattern.findFirstMatchIn(line).foreach { m =>+ val className = m.group(1)+ val classEnd = findBraceBlockEnd(lines, idx)+ val bodyText = lines.slice(idx, classEnd).mkString("\n")+ val fp = Fingerprint.compute(trimmed.replaceAll("""\s*\{\s*$""", "").take(120), bodyText)+ val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")+ entities = entities :+ ParsedEntity(+ name = className,+ kind = NodeKind.Class,+ attrs = Map("language" -> Json.fromString("typescript"), "exported" -> Json.fromBoolean(exported)),+ lineStart = lineNum,+ lineEnd = classEnd,+ contentFingerprint = Some(fp)+ )+ relationships = relationships :+ ParsedRelationship(fileName, className, "CONTAINS")+ classRanges = classRanges :+ (className, lineNum, classEnd)++ // Extract methods from this class+ val methods = extractMethods(lines, idx, classEnd, className)+ entities = entities ++ methods._1+ relationships = relationships ++ methods._2+ }++ // Check for top-level function (not inside a class)+ if (!insideClass(lineNum, classRanges)) {+ FuncPattern.findFirstMatchIn(line).foreach { m =>+ val funcName = m.group(1)+ val funcEnd = findBraceBlockEnd(lines, idx)+ val summary = line.trim.take(120)+ val bodyText = lines.slice(idx, funcEnd).mkString("\n")+ val fp = Fingerprint.compute(summary, bodyText)+ val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")+ entities = entities :+ ParsedEntity(+ name = funcName,+ kind = NodeKind.Function,+ attrs = Map(+ "language" -> Json.fromString("typescript"),+ "summary" -> Json.fromString(summary),+ "signature" -> Json.fromString(summary),+ "visibility" -> Json.fromString("public"),+ "exported" -> Json.fromBoolean(exported)+ ),+ lineStart = lineNum,+ lineEnd = funcEnd,+ contentFingerprint = Some(fp)+ )+ relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")++ // Extract calls within this function+ val calls = extractCalls(lines, idx, funcEnd, funcName)+ relationships = relationships ++ calls+ }++ // Check for arrow function / const function+ val arrowMatch = ArrowFuncPattern.findFirstMatchIn(line)+ val constFuncMatch = ConstFuncPattern.findFirstMatchIn(line)+ val funcMatch = arrowMatch.orElse(constFuncMatch)+ funcMatch.foreach { m =>+ val funcName = m.group(1)+ // Avoid duplicates if already captured by FuncPattern+ if (!entities.exists(e => e.name == funcName && e.kind == NodeKind.Function)) {+ val funcEnd = findBraceBlockEnd(lines, idx)+ val summary = line.trim.take(120)+ val bodyText = lines.slice(idx, funcEnd).mkString("\n")+ val fp = Fingerprint.compute(summary, bodyText)+ val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")+ entities = entities :+ ParsedEntity(+ name = funcName,+ kind = NodeKind.Function,+ attrs = Map(+ "language" -> Json.fromString("typescript"),+ "summary" -> Json.fromString(summary),+ "signature" -> Json.fromString(summary),+ "visibility" -> Json.fromString("public"),+ "exported" -> Json.fromBoolean(exported)+ ),+ lineStart = lineNum,+ lineEnd = funcEnd,+ contentFingerprint = Some(fp)+ )+ relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")++ val callRels = extractCalls(lines, idx, funcEnd, funcName)+ relationships = relationships ++ callRels+ }+ }+ }++ // Check for imports+ ImportPattern.findFirstMatchIn(line).foreach { m =>+ val moduleName = m.group(1)+ if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {+ entities = entities :+ ParsedEntity(+ name = moduleName,+ kind = NodeKind.Module,+ attrs = Map.empty,+ lineStart = lineNum,+ lineEnd = lineNum+ )+ }+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")+ }++ // Extract named import bindings: import { foo, bar } from "mod"+ NamedImportPattern.findFirstMatchIn(line).foreach { m =>+ val names = m.group(1).split(",").map(_.trim).filter(_.nonEmpty)+ for (rawName <- names) {+ val symbolName = if (rawName.contains(" as ")) rawName.split("\\s+as\\s+").last.trim else rawName+ if (symbolName.nonEmpty && !relationships.exists(r => r.srcName == fileName && r.dstName == symbolName && r.predicate == "IMPORTS")) {+ relationships = relationships :+ ParsedRelationship(fileName, symbolName, "IMPORTS")+ }+ }+ }++ SideEffectImportPattern.findFirstMatchIn(line).foreach { m =>+ val moduleName = m.group(1)+ if (!relationships.exists(r => r.dstName == moduleName && r.predicate == "IMPORTS")) {+ if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {+ entities = entities :+ ParsedEntity(+ name = moduleName,+ kind = NodeKind.Module,+ attrs = Map.empty,+ lineStart = lineNum,+ lineEnd = lineNum+ )+ }+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")+ }+ }++ // Check for interface definition+ InterfacePattern.findFirstMatchIn(line).foreach { m =>+ val ifaceName = m.group(1)+ if (!entities.exists(e => e.name == ifaceName && e.kind == NodeKind.Interface)) {+ val ifaceEnd = findBraceBlockEnd(lines, idx)+ val bodyText = lines.slice(idx, ifaceEnd).mkString("\n")+ val fp = Fingerprint.compute(trimmed.replaceAll("""\s*\{\s*$""", "").take(120), bodyText)+ val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")+ entities = entities :+ ParsedEntity(+ name = ifaceName,+ kind = NodeKind.Interface,+ attrs = Map(+ "language" -> Json.fromString("typescript"),+ "ts_kind" -> Json.fromString("interface"),+ "exported" -> Json.fromBoolean(exported)+ ),+ lineStart = lineNum,+ lineEnd = ifaceEnd,+ contentFingerprint = Some(fp)+ )+ relationships = relationships :+ ParsedRelationship(fileName, ifaceName, "CONTAINS")+ }+ }++ // Check for type alias+ TypeAliasPattern.findFirstMatchIn(line).foreach { m =>+ val typeName = m.group(1)+ if (!trimmed.startsWith("import")) {+ val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")+ val fp = Fingerprint.compute(trimmed.take(120), trimmed)+ entities = entities :+ ParsedEntity(+ name = typeName,+ kind = NodeKind.Class,+ attrs = Map(+ "language" -> Json.fromString("typescript"),+ "ts_kind" -> Json.fromString("type_alias"),+ "exported" -> Json.fromBoolean(exported)+ ),+ lineStart = lineNum,+ lineEnd = lineNum,+ contentFingerprint = Some(fp)+ )+ relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")+ }+ }++ } // end else (non-comment)+ }++ // Extract file-level calls from code outside named functions/methods/classes+ val coveredRanges: Vector[(Int, Int)] = entities+ .filter(e => e.kind == NodeKind.Function || e.kind == NodeKind.Method || e.kind == NodeKind.Class)+ .map(e => (e.lineStart, e.lineEnd))++ val fileCallsSeen = scala.collection.mutable.Set.empty[String]+ for ((line, idx) <- lines.zipWithIndex) {+ val lineNum = idx + 1+ val inNamedBody = coveredRanges.exists { case (s, e) => lineNum >= s && lineNum <= e }+ if (!inNamedBody && !line.trim.startsWith("//") && !line.trim.startsWith("*")) {+ CallPattern.findAllMatchIn(line).foreach { m =>+ val callee = m.group(1)+ if (!TypeScriptBuiltins.contains(callee) && !TypeScriptKeywords.contains(callee)+ && !fileCallsSeen.contains(callee) && callee != fileName) {+ fileCallsSeen += callee+ relationships = relationships :+ ParsedRelationship(fileName, callee, "CALLS")+ }+ }+ }+ }++ ParseResult(entities, relationships)+ }++ // ---------------------------------------------------------------------------+ // Patterns+ // ---------------------------------------------------------------------------++ private val ClassPattern: Regex = """(?:export\s+)?(?:abstract\s+)?class\s+(\w+)""".r+ private val FuncPattern: Regex = """(?:export\s+)?(?:async\s+)?function\s+(\w+)""".r+ private val ArrowFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[a-zA-Z_]\w*)\s*(?::\s*[^=]+)?\s*=>""".r+ private val ConstFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?function""".r+ private val MethodPattern: Regex = """^\s+(?:async\s+)?(?:private\s+|public\s+|protected\s+|static\s+|readonly\s+)*(\w+)\s*(?:<[^>]+>)?\s*\(""".r+ private val ImportPattern: Regex = """import\s+.*?from\s+['"]([^'"]+)['"]""".r+ private val NamedImportPattern: Regex = """import\s+\{([^}]+)\}\s+from\s+['"]([^'"]+)['"]""".r+ private val SideEffectImportPattern: Regex = """^import\s+['"]([^'"]+)['"]""".r+ private val InterfacePattern: Regex = """(?:export\s+)?interface\s+(\w+)""".r+ private val TypeAliasPattern: Regex = """(?:export\s+)?type\s+(\w+)\s*(?:<[^>]+>)?\s*=""".r+ private val CallPattern: Regex = """(?:(?:this|self|await)\.)?\b(\w+)\s*\(""".r++ // ---------------------------------------------------------------------------+ // Helpers+ // ---------------------------------------------------------------------------++ private def insideClass(lineNum: Int, classRanges: Vector[(String, Int, Int)]): Boolean =+ classRanges.exists { case (_, start, end) => lineNum > start && lineNum <= end }++ private def extractVisibility(trimmedLine: String): String = {+ if (trimmedLine.startsWith("private ") || trimmedLine.startsWith("private\t")) "private"+ else if (trimmedLine.startsWith("protected ") || trimmedLine.startsWith("protected\t")) "protected"+ else "public"+ }++ /**+ * Extract methods from a class body.+ */+ private def extractMethods(+ lines: Array[String],+ classStartIdx: Int,+ classEndLine: Int,+ className: String+ ): (Vector[ParsedEntity], Vector[ParsedRelationship]) = {+ var entities = Vector.empty[ParsedEntity]+ var relationships = Vector.empty[ParsedRelationship]++ var i = classStartIdx + 1+ while (i < lines.length && (i + 1) <= classEndLine) {+ val line = lines(i)+ val lineNum = i + 1++ if (!line.trim.startsWith("//") && !line.trim.startsWith("*")) {+ MethodPattern.findFirstMatchIn(line).foreach { m =>+ val methodName = m.group(1)+ // Skip keywords that look like methods+ if (!TypeScriptKeywords.contains(methodName)) {+ val methodEnd = findBraceBlockEnd(lines, i)+ val summary = line.trim.take(120)+ val visibility = extractVisibility(line.trim)+ val bodyText = lines.slice(i, methodEnd).mkString("\n")+ val fp = Fingerprint.compute(summary, bodyText)+ entities = entities :+ ParsedEntity(+ name = methodName,+ kind = NodeKind.Method,+ attrs = Map(+ "language" -> Json.fromString("typescript"),+ "summary" -> Json.fromString(summary),+ "signature" -> Json.fromString(summary),+ "visibility" -> Json.fromString(visibility)+ ),+ lineStart = lineNum,+ lineEnd = methodEnd,+ contentFingerprint = Some(fp)+ )+ relationships = relationships :+ ParsedRelationship(className, methodName, "CONTAINS")++ // Extract calls within this method+ val calls = extractCalls(lines, i, methodEnd, methodName)+ relationships = relationships ++ calls+ }+ }+ }+ i += 1+ }++ (entities, relationships)+ }++ /**+ * Find end of a brace-delimited block starting near `startIdx`.+ *+ * Uses end-of-line brace depth to detect block entry. This correctly+ * skips inline TypeScript type annotations like `{ kind?: string }`+ * that balance to zero on a single line, which would otherwise cause+ * the parser to think the function body ended on the parameter line.+ */+ private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Int = {+ var braceCount = 0+ var entered = false+ var i = startIdx+ while (i < lines.length) {+ for (ch <- lines(i)) {+ if (ch == '{') braceCount += 1+ if (ch == '}') braceCount -= 1+ }+ // Only mark as entered when braceCount > 0 at end of line.+ // This skips inline type annotations like { a: string; b: number }+ // that balance to 0 within the same line.+ if (!entered && braceCount > 0) entered = true+ if (entered && braceCount <= 0) return i + 1+ i += 1+ }+ lines.length+ }++ /**+ * Extract CALLS relationships from a function/method body.+ */+ private def extractCalls(+ lines: Array[String],+ funcStartIdx: Int,+ funcEndLine: Int,+ callerName: String+ ): Vector[ParsedRelationship] = {+ var calls = Vector.empty[ParsedRelationship]+ val seen = scala.collection.mutable.Set.empty[String]+ var i = funcStartIdx + 1++ while (i < lines.length && (i + 1) <= funcEndLine) {+ val line = lines(i)+ CallPattern.findAllMatchIn(line).foreach { m =>+ val callee = m.group(1)+ if (callee != callerName && !TypeScriptBuiltins.contains(callee) &&+ !TypeScriptKeywords.contains(callee) && !seen.contains(callee)) {+ seen += callee+ calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")+ }+ }+ i += 1+ }+ calls+ }++ private val TypeScriptKeywords: Set[String] = Set(+ "if", "else", "for", "while", "do", "switch", "case", "break",+ "continue", "return", "throw", "try", "catch", "finally",+ "new", "delete", "typeof", "instanceof", "void", "in", "of",+ "class", "extends", "implements", "interface", "enum", "type",+ "const", "let", "var", "function", "async", "await",+ "import", "export", "default", "from", "as",+ "this", "super", "constructor", "static", "private", "public", "protected"+ )++ private val TypeScriptBuiltins: Set[String] = Set(+ "console", "log", "error", "warn", "parseInt", "parseFloat",+ "String", "Number", "Boolean", "Array", "Object", "Promise",+ "JSON", "Math", "Date", "RegExp", "Error", "Map", "Set",+ "WeakMap", "WeakSet", "Symbol",+ "setTimeout", "setInterval", "clearTimeout", "clearInterval",+ "require", "module", "exports",+ "fetch", "Response", "Request", "URL", "Buffer", "process",+ // Array/iterable methods+ "map", "filter", "reduce", "forEach", "sort", "find", "some", "every",+ "includes", "indexOf", "slice", "splice", "push", "pop", "shift", "unshift",+ "concat", "join", "flat", "flatMap", "fill", "from",+ // String methods+ "split", "replace", "trim", "toUpperCase", "toLowerCase", "substring",+ "charAt", "startsWith", "endsWith", "padStart", "padEnd", "match", "test",+ // Object methods+ "keys", "values", "entries", "assign", "create", "freeze",+ "defineProperty", "getOwnPropertyNames", "hasOwnProperty",+ // Promise methods+ "then", "catch", "finally", "all", "race", "resolve", "reject"+ )+}diff --git a/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala b/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scalanew file mode 100644index 00000000..b357b6ec--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/map/LouvainClustering.scala@@ -0,0 +1,270 @@+package ix.memory.map++import java.util.UUID++import ix.memory.model.NodeId++import scala.util.Random++/**+ * Louvain community detection with recursive graph coarsening.+ *+ * Produces multi-level partitions: each level groups the original file nodes+ * into progressively coarser communities.+ *+ * Algorithm:+ * 1. Initialize each node in its own community.+ * 2. For each node (random order), try moving it to a neighbor's community.+ * Accept if the modularity gain ΔQ > 0.+ * 3. Repeat until no improvement or max passes reached.+ * 4. Collapse communities into supernodes; rebuild weighted graph.+ * 5. Repeat on the coarsened graph.+ * 6. Project all levels back to original file IDs.+ *+ * Modularity gain formula (standard Louvain):+ * ΔQ(i → C) = k_i_in(C) / m − γ · Σ_tot(C) · k_i / (2m²)+ *+ * where:+ * k_i_in(C) = sum of edge weights from node i to nodes in community C+ * Σ_tot(C) = sum of all node degrees in C+ * k_i = degree of node i+ * m = total edge weight (each undirected edge counted once)+ * γ = resolution parameter (1.0 by default)+ */+object LouvainClustering {++ /** A partition of original file NodeIds into communities at one hierarchy level. */+ final case class LevelPartition(+ communities: Vector[Set[NodeId]], // each Set is a community of original file IDs+ assignment: Map[NodeId, Int] // fileId → community index+ )++ /**+ * Run multi-level Louvain clustering.+ *+ * Returns a vector of LevelPartitions, finest first (index 0 = most granular).+ * Each partition covers ALL original file nodes.+ */+ def cluster(+ graph: WeightedFileGraph,+ maxLevels: Int = 4,+ resolution: Double = 1.0,+ minCommunitySize: Int = 2,+ seed: Long = 42L+ ): Vector[LevelPartition] = {+ if (graph.vertices.isEmpty || graph.totalWeight == 0.0) return Vector.empty++ val rng = new Random(seed)++ // nodeExpansion: for the current working graph, maps each node → original file IDs it represents+ var nodeExpansion: Map[NodeId, Set[NodeId]] =+ graph.vertices.map(v => v.id -> Set(v.id)).toMap++ var currentGraph = graph+ val levels = scala.collection.mutable.ArrayBuffer.empty[LevelPartition]++ var level = 0+ while (level < maxLevels && currentGraph.vertices.size > 1) {+ val rawAssignment = louvainPass(currentGraph, resolution, rng)++ // Project raw assignment (current node ids) → original file id communities+ val projected = projectPartition(rawAssignment, nodeExpansion)++ val meaningful = projected.communities.count(_.size >= minCommunitySize)++ if (meaningful <= 1 && levels.nonEmpty) {+ // No further useful partitioning — stop+ level = maxLevels+ } else {+ levels += projected++ // Coarsen: collapse each community into a supernode+ val (coarsenedGraph, newExpansion) =+ coarsen(currentGraph, rawAssignment, nodeExpansion)++ nodeExpansion = newExpansion+ currentGraph = coarsenedGraph+ level += 1++ if (currentGraph.vertices.size < 3) level = maxLevels+ }+ }++ levels.toVector+ }++ // ── Single Louvain pass ────────────────────────────────────────────++ /**+ * One full Louvain pass on the given graph.+ * Returns assignment: nodeId → community index (indices are 0-based, dense).+ */+ private def louvainPass(+ graph: WeightedFileGraph,+ resolution: Double,+ rng: Random+ ): Map[NodeId, Int] = {+ val nodes = graph.vertices.map(_.id)+ val adj = graph.adjMatrix+ val m = graph.totalWeight.max(1e-12)++ // Initialize: node i → community i+ val commOf = scala.collection.mutable.Map(nodes.zipWithIndex: _*)+ val sumTot = scala.collection.mutable.Map(+ nodes.zipWithIndex.map { case (n, i) => i -> graph.degrees.getOrElse(n, 0.0) }: _*+ ).withDefaultValue(0.0)+ val sumIn = scala.collection.mutable.Map[Int, Double]().withDefaultValue(0.0)+ // communities: commId → members (mutable set)+ val members = scala.collection.mutable.Map(+ nodes.zipWithIndex.map { case (n, i) => i -> scala.collection.mutable.Set(n) }: _*+ )++ var improved = true+ var passes = 0++ while (improved && passes < 30) {+ improved = false+ passes += 1++ for (node <- rng.shuffle(nodes)) {+ val ci = commOf(node)+ val ki = graph.degrees.getOrElse(node, 0.0)++ // Compute weight from this node to each neighboring community+ val wToComm = scala.collection.mutable.Map[Int, Double]().withDefaultValue(0.0)+ for ((neighbor, w) <- adj.getOrElse(node, Map.empty)) {+ wToComm(commOf(neighbor)) += w+ }++ // Remove node from current community+ val kiInCi = wToComm.getOrElse(ci, 0.0)+ sumIn(ci) -= 2.0 * kiInCi+ sumTot(ci) -= ki+ members(ci) -= node+ // Temporarily isolate: treat as its own community (gains = 0 baseline)+ commOf(node) = -1++ var bestComm = ci+ var bestGain = 0.0++ for ((cj, kiInCj) <- wToComm if cj != -1) {+ val dQ = kiInCj / m - resolution * sumTot.getOrElse(cj, 0.0) * ki / (2.0 * m * m)+ if (dQ > bestGain) {+ bestGain = dQ+ bestComm = cj+ }+ }++ // Place node in best community (or back in its old one if no improvement)+ commOf(node) = bestComm+ members.getOrElseUpdate(bestComm, scala.collection.mutable.Set()) += node+ val kiInBest = wToComm.getOrElse(bestComm, 0.0)+ sumIn(bestComm) += 2.0 * kiInBest+ sumTot(bestComm) += ki++ if (bestComm != ci) improved = true+ }+ }++ // Re-index communities from 0 to N-1 (dense)+ val nonEmpty = members.filter(_._2.nonEmpty).toMap+ val reindex = nonEmpty.keys.toVector.sorted.zipWithIndex.toMap+ val finalAssign = commOf.map { case (n, c) =>+ n -> reindex.getOrElse(c, 0)+ }.toMap++ finalAssign+ }++ // ── Projection ─────────────────────────────────────────────────────++ private def projectPartition(+ assignment: Map[NodeId, Int],+ nodeExpansion: Map[NodeId, Set[NodeId]]+ ): LevelPartition = {+ // Map: community index → set of original file IDs+ val comms = scala.collection.mutable.Map[Int, scala.collection.mutable.Set[NodeId]]()+ for ((node, commIdx) <- assignment) {+ val origFiles = nodeExpansion.getOrElse(node, Set(node))+ comms.getOrElseUpdate(commIdx, scala.collection.mutable.Set()) ++= origFiles+ }++ val commVec = comms.toVector.sortBy(_._1).map(_._2.toSet)+ // Build reverse assignment: origFileId → commIdx+ val reverseAssign = (for ((members, idx) <- commVec.zipWithIndex; f <- members) yield f -> idx).toMap++ LevelPartition(commVec, reverseAssign)+ }++ // ── Graph coarsening ────────────────────────────────────────────────++ /**+ * Collapse each community in `assignment` into a single supernode.+ * Returns the coarsened graph and an updated nodeExpansion map.+ */+ private def coarsen(+ graph: WeightedFileGraph,+ assignment: Map[NodeId, Int],+ nodeExpansion: Map[NodeId, Set[NodeId]]+ ): (WeightedFileGraph, Map[NodeId, Set[NodeId]]) = {+ // Group current nodes by community+ val commGroups: Map[Int, Set[NodeId]] =+ assignment.groupBy(_._2).map { case (k, v) => k -> v.keySet }++ // Assign a deterministic new NodeId to each community (based on sorted member UUIDs)+ val commToSuper: Map[Int, NodeId] = commGroups.map { case (ci, members) =>+ val key = "map:super:" + members.map(_.value.toString).toList.sorted.mkString(",")+ ci -> NodeId(UUID.nameUUIDFromBytes(key.getBytes("UTF-8")))+ }++ // Build supernode expansion+ val newExpansion: Map[NodeId, Set[NodeId]] = commGroups.map { case (ci, members) =>+ val origFiles = members.flatMap(n => nodeExpansion.getOrElse(n, Set(n)))+ commToSuper(ci) -> origFiles+ }++ // Build supernode vertices (use community index as label for debugging)+ val superVertices = commToSuper.map { case (ci, superId) =>+ FileVertex(superId, s"supernode_$ci")+ }.toVector++ // Aggregate inter-community edges+ val edgeWeights =+ scala.collection.mutable.Map[(NodeId, NodeId), Double]()++ for ((src, neighbors) <- graph.adjMatrix) {+ val ci = assignment.getOrElse(src, -1)+ if (ci >= 0) {+ val si = commToSuper(ci)+ for ((dst, w) <- neighbors) {+ val cj = assignment.getOrElse(dst, -1)+ if (cj >= 0 && ci != cj) {+ val sj = commToSuper(cj)+ val key = if (si.value.compareTo(sj.value) < 0) (si, sj) else (sj, si)+ edgeWeights(key) = edgeWeights.getOrElse(key, 0.0) + w / 2.0 // undirected: each pair counted twice in adj+ }+ }+ }+ }++ // Build new undirected adjacency for the coarsened graph+ val newAdj = scala.collection.mutable.Map[NodeId, scala.collection.mutable.Map[NodeId, Double]]()+ for (((si, sj), w) <- edgeWeights) {+ newAdj.getOrElseUpdate(si, scala.collection.mutable.Map())(sj) = w+ newAdj.getOrElseUpdate(sj, scala.collection.mutable.Map())(si) = w+ }++ val newAdjFinal = newAdj.map { case (k, m) => k -> m.toMap }.toMap+ val newDegrees = newAdjFinal.map { case (k, m) => k -> m.values.sum }+ val newTotalWeight = edgeWeights.values.sum++ val coarsened = WeightedFileGraph(+ superVertices,+ newAdjFinal,+ newDegrees.toMap,+ newTotalWeight+ )++ (coarsened, newExpansion)+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala b/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scalanew file mode 100644index 00000000..9a716c4a--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/map/MapGraphBuilder.scala@@ -0,0 +1,196 @@+package ix.memory.map++import cats.effect.IO++import ix.memory.db.ArangoClient+import ix.memory.model.NodeId++/**+ * Builds a weighted undirected file-level coupling graph from ArangoDB.+ *+ * Key design choices:+ * - Uses provenance.source_uri (absolute path) as the join key between file+ * nodes and coupling edges, avoiding stale CONTAINS traversals.+ * - Skips edges where either endpoint has no live provenance (external libs,+ * tombstoned symbols with no replacement).+ *+ * Signal weights (from spec):+ * α = 1.0 call coupling (highest importance)+ * β = 0.7 import coupling+ * γ = 0.9 extends/implements (type/interface cohesion proxy)+ * ε = 0.3 path proximity (weak structural prior)+ */+class MapGraphBuilder(client: ArangoClient) {++ private val Alpha = 1.0+ private val Beta = 0.7+ private val Gamma = 0.9+ private val Epsilon = 0.3++ def build(): IO[WeightedFileGraph] =+ for {+ files <- fetchFiles()+ rawPairs <- fetchCouplingByUri()+ graph = computeGraph(files, rawPairs)+ } yield graph++ // ── ArangoDB queries ───────────────────────────────────────────────++ /**+ * Returns file nodes using provenance.source_uri as the path key.+ * Files without source_uri are skipped (unusual but defensive).+ */+ // Source-code extensions to include in the architectural map.+ // Docs (.md), scripts (.sh), config (.yml/.json/.toml) etc. are excluded.+ private val SourceExtensions =+ Set("scala","java","kt","ts","tsx","js","jsx","py","go","rs","cs","cpp","c","h","hpp","rb")++ private def fetchFiles(): IO[Vector[FileVertex]] =+ client.query(+ """FOR n IN nodes+ | FILTER n.kind == "file"+ | AND n.deleted_rev == null+ | AND n.provenance.source_uri != null+ | RETURN {id: n.logical_id, path: n.provenance.source_uri}""".stripMargin,+ Map.empty+ ).map { rows =>+ rows.flatMap { json =>+ val c = json.hcursor+ for {+ idStr <- c.get[String]("id").toOption+ path <- c.get[String]("path").toOption+ ext = path.split("\\.").lastOption.getOrElse("").toLowerCase+ if SourceExtensions.contains(ext)+ uuid <- try Some(java.util.UUID.fromString(idStr)) catch { case _: Exception => None }+ } yield FileVertex(NodeId(uuid), path)+ }.toVector+ }++ /**+ * Returns cross-file coupling pairs keyed by absolute provenance.source_uri.+ *+ * Builds a unified logical_id → source_uri map over all live non-metadata+ * nodes (files AND symbols), then resolves coupling edge endpoints via that+ * map. Stale edges whose endpoints have been tombstoned are naturally+ * excluded because deleted nodes have no entry in the map.+ */+ private def fetchCouplingByUri(): IO[Vector[RawFilePair]] =+ client.query(+ """LET uri_map = MERGE(+ | FOR n IN nodes+ | FILTER n.deleted_rev == null+ | AND n.provenance.source_uri != null+ | AND n.kind NOT IN ["module","config","config_entry",+ | "doc","decision","intent","bug","plan",+ | "task","goal","region"]+ | RETURN {[n.logical_id]: n.provenance.source_uri}+ |)+ |FOR e IN edges+ | FILTER e.predicate IN ["CALLS","IMPORTS","EXTENDS","IMPLEMENTS"]+ | AND e.deleted_rev == null+ | LET su = HAS(uri_map, e.src) ? uri_map[e.src] : null+ | LET du = HAS(uri_map, e.dst) ? uri_map[e.dst] : null+ | FILTER su != null AND du != null AND su != du+ | COLLECT srcUri = su, dstUri = du, pred = e.predicate+ | WITH COUNT INTO cnt+ | RETURN {srcUri, dstUri, predicate: pred, count: cnt}""".stripMargin,+ Map.empty+ ).map { rows =>+ rows.flatMap { json =>+ val c = json.hcursor+ for {+ srcUri <- c.get[String]("srcUri").toOption+ dstUri <- c.get[String]("dstUri").toOption+ pred <- c.get[String]("predicate").toOption+ cnt <- c.get[Int]("count").toOption+ } yield RawFilePair(srcUri, dstUri, pred, cnt)+ }.toVector+ }++ // ── Graph construction ─────────────────────────────────────────────++ private def computeGraph(+ files: Vector[FileVertex],+ rawPairs: Vector[RawFilePair]+ ): WeightedFileGraph = {+ if (files.isEmpty) return WeightedFileGraph.empty++ // Index files by their absolute source_uri path+ val byUri: Map[String, FileVertex] = files.map(v => v.path -> v).toMap++ // Accumulate coupling counts per (srcUri, dstUri) canonical pair+ val pairAcc =+ scala.collection.mutable.Map[(String, String), scala.collection.mutable.Map[String, Int]]()++ for (pair <- rawPairs) {+ val sv = byUri.get(pair.srcId) // srcId is source_uri here+ val dv = byUri.get(pair.dstId)+ (sv, dv) match {+ case (Some(s), Some(d)) if s.id != d.id =>+ // Canonical undirected key: lexicographically smaller path first+ val key = if (s.path < d.path) (s.path, d.path) else (d.path, s.path)+ val m = pairAcc.getOrElseUpdate(key, scala.collection.mutable.Map())+ m(pair.predicate) = m.getOrElse(pair.predicate, 0) + pair.count+ case _ =>+ }+ }++ // Build adjacency with composite weights+ val adjMut =+ scala.collection.mutable.Map[NodeId, scala.collection.mutable.Map[NodeId, Double]]()++ for (((sp, dp), predicateCounts) <- pairAcc) {+ val sv = byUri(sp)+ val dv = byUri(dp)++ val callCount = predicateCounts.getOrElse("CALLS", 0)+ val importCount = predicateCounts.getOrElse("IMPORTS", 0)+ val typeCount = predicateCounts.getOrElse("EXTENDS", 0) ++ predicateCounts.getOrElse("IMPLEMENTS", 0)++ val sCall = Alpha * math.log1p(callCount)+ val sImport = Beta * math.log1p(importCount)+ val sType = Gamma * math.log1p(typeCount)+ val sPath = Epsilon * pathProximity(sv.path, dv.path)++ val w = sCall + sImport + sType + sPath+ if (w > 0.0) {+ adjMut.getOrElseUpdate(sv.id, scala.collection.mutable.Map())(dv.id) = w+ adjMut.getOrElseUpdate(dv.id, scala.collection.mutable.Map())(sv.id) = w+ }+ }++ // Path-proximity pass: add structural edges for same-directory file pairs+ // that have no existing coupling edge. This ensures isolated files (no+ // CALLS/IMPORTS/EXTENDS edges) can still be clustered by directory.+ val filesByDir = files.groupBy(v => v.path.split("[/\\\\]").dropRight(1).mkString("/"))+ for ((_, dirFiles) <- filesByDir if dirFiles.size >= 2) {+ for (i <- dirFiles.indices; j <- (i + 1) until dirFiles.size) {+ val sv = dirFiles(i)+ val dv = dirFiles(j)+ if (!adjMut.get(sv.id).exists(_.contains(dv.id))) {+ val w = Epsilon * pathProximity(sv.path, dv.path)+ if (w > 0.0) {+ adjMut.getOrElseUpdate(sv.id, scala.collection.mutable.Map())(dv.id) = w+ adjMut.getOrElseUpdate(dv.id, scala.collection.mutable.Map())(sv.id) = w+ }+ }+ }+ }++ val adj = adjMut.map { case (k, m) => k -> m.toMap }.toMap+ val degrees = adj.map { case (k, m) => k -> m.values.sum }+ val totalWeight = degrees.values.sum / 2.0++ WeightedFileGraph(files, adj, degrees.toMap, totalWeight)+ }++ /** Path distance heuristic: deeper common prefix → higher proximity. */+ private def pathProximity(a: String, b: String): Double = {+ val partsA = a.split("[/\\\\]").dropRight(1)+ val partsB = b.split("[/\\\\]").dropRight(1)+ val common = partsA.zip(partsB).takeWhile { case (x, y) => x == y }.length+ val dist = (partsA.length - common) + (partsB.length - common)+ 1.0 / (1.0 + dist)+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/map/MapService.scala b/memory-layer/src/main/scala/ix/memory/map/MapService.scalanew file mode 100644index 00000000..a176a7b5--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/map/MapService.scala@@ -0,0 +1,461 @@+package ix.memory.map++import java.time.Instant+import java.util.UUID++import cats.effect.IO+import io.circe.Json+import io.circe.syntax._++import ix.memory.db.{ArangoClient, GraphQueryApi, GraphWriteApi}+import ix.memory.model._++/**+ * Full pipeline for ix map:+ * 1. Build weighted file graph+ * 2. Detect cross-cutting nodes and apply penalty+ * 3. Run multi-level Louvain clustering+ * 4. Compute region metrics (cohesion, coupling, boundary ratio)+ * 5. Infer labels+ * 6. Score confidence+ * 7. Persist Region nodes + IN_REGION edges to ArangoDB+ * 8. Return ArchitectureMap+ */+class MapService(+ client: ArangoClient,+ queryApi: GraphQueryApi,+ writeApi: GraphWriteApi+) {++ // Cross-cut penalty multiplier (reduces edge weights involving crosscut files)+ private val CrosscutLambda = 0.5+ // High-degree threshold: files in the top N% by degree are crosscut candidates+ private val CrosscutDegreeTop = 0.10+ // Path tokens that suggest cross-cutting utilities+ private val CrosscutTokens = Set(+ "util", "utils", "helper", "helpers", "common", "shared", "base",+ "logging", "log", "logger", "config", "conf", "constants", "constant",+ "extensions", "ext", "mixin", "mixins", "core", "framework"+ )++ private val builder = new MapGraphBuilder(client)++ // Label kinds by level+ private def labelKind(level: Int, maxLevel: Int): String =+ if (maxLevel <= 1) "system"+ else if (level == maxLevel) "system"+ else if (level == 1) "module"+ else "subsystem"++ def buildMap(): IO[ArchitectureMap] =+ for {+ rev <- queryApi.getLatestRev+ rawGraph <- builder.build()+ _ <- IO.pure(()) // no-op; structured for easy step annotation+ crosscutScores = detectCrosscut(rawGraph)+ graph = applyPenalty(rawGraph, crosscutScores)+ levels = LouvainClustering.cluster(graph, maxLevels = 3, minCommunitySize = 3)+ regions = buildRegions(graph, levels, crosscutScores, rev)+ _ <- persistMap(regions, rev)+ } yield ArchitectureMap(regions, rawGraph.vertices.size, rev.value)++ // ── Cross-cutting detection ─────────────────────────────────────────++ private def detectCrosscut(graph: WeightedFileGraph): Map[NodeId, Double] = {+ if (graph.degrees.isEmpty) return Map.empty+ val maxDeg = graph.degrees.values.max.max(1e-10)+ val threshold = maxDeg * (1.0 - CrosscutDegreeTop)++ graph.vertices.map { v =>+ val deg = graph.degrees.getOrElse(v.id, 0.0)+ val nameLower = v.path.toLowerCase+ val pathTokens = nameLower.split("[/\\\\._-]").toSet++ val degreeSignal = if (deg >= threshold) (deg - threshold) / (maxDeg - threshold + 1e-10) else 0.0+ val lexicalSignal = if (pathTokens.intersect(CrosscutTokens).nonEmpty) 0.5 else 0.0++ val score = (0.7 * degreeSignal + 0.3 * lexicalSignal).min(1.0)+ v.id -> score+ }.toMap+ }++ private def applyPenalty(+ graph: WeightedFileGraph,+ scores: Map[NodeId, Double]+ ): WeightedFileGraph = {+ val newAdj = graph.adjMatrix.map { case (src, neighbors) =>+ val sCross = scores.getOrElse(src, 0.0)+ val adjusted = neighbors.map { case (dst, w) =>+ val dCross = scores.getOrElse(dst, 0.0)+ val penalty = (sCross + dCross) / 2.0 * CrosscutLambda+ dst -> w * (1.0 - penalty)+ }+ src -> adjusted+ }+ val newDeg = newAdj.map { case (k, m) => k -> m.values.sum }+ val newTotal = newDeg.values.sum / 2.0+ graph.copy(adjMatrix = newAdj, degrees = newDeg.toMap, totalWeight = newTotal)+ }++ // ── Region construction ─────────────────────────────────────────────++ private def buildRegions(+ graph: WeightedFileGraph,+ levels: Vector[LouvainClustering.LevelPartition],+ crosscutScores: Map[NodeId, Double],+ rev: Rev+ ): Vector[Region] = {+ if (levels.isEmpty) return Vector.empty++ val fileVertexById = graph.vertices.map(v => v.id -> v).toMap+ val maxLevel = levels.size++ // Compute common absolute-path prefix to strip for label inference+ val allPaths = graph.vertices.map(_.path)+ val commonPfxLen = commonPathPrefixLength(allPaths)++ // Build regions for each level, finest first.+ // Apply minCommunitySize at every level — singleton/doubleton regions are noise.+ val minSize = if (maxLevel >= 3) 3 else 2+ val allRegions = levels.zipWithIndex.flatMap { case (partition, levelIdx) =>+ val level = levelIdx + 1+ partition.communities.filter(_.size >= minSize).map { memberFiles =>+ buildOneRegion(+ memberFiles, level, labelKind(level, maxLevel),+ graph, fileVertexById, crosscutScores, rev, commonPfxLen+ )+ }+ }++ // Wire parent/child relationships+ // For each region at level N, find which level-(N+1) region contains its files+ val byLevel: Map[Int, Vector[Region]] =+ allRegions.groupBy(_.level)++ val withParents = allRegions.map { region =>+ val parentOpt = byLevel.get(region.level + 1).flatMap { parents =>+ // Parent must be strictly coarser: strictly larger member set (no equal sets)+ parents+ .filter(p => p.memberFiles.size > region.memberFiles.size &&+ region.memberFiles.subsetOf(p.memberFiles))+ .minByOption(_.memberFiles.size) // pick tightest enclosing region+ }+ region.copy(parentId = parentOpt.map(_.id))+ }++ // Fill childRegionIds+ val childrenByParent: Map[NodeId, Set[NodeId]] =+ withParents.flatMap(r => r.parentId.map(pid => pid -> r.id))+ .groupBy(_._1)+ .map { case (k, vs) => k -> vs.map(_._2).toSet }++ val withChildren = withParents.map { r =>+ r.copy(childRegionIds = childrenByParent.getOrElse(r.id, Set.empty))+ }++ // Disambiguate duplicate labels within each level.+ // For each group of same-label regions, find a path segment unique to each.+ val labelGroups = withChildren.groupBy(r => (r.level, r.label))+ val needsDisambig = labelGroups.filter(_._2.size > 1)++ if (needsDisambig.isEmpty) withChildren+ else withChildren.map { r =>+ needsDisambig.get((r.level, r.label)) match {+ case Some(group) =>+ val myPaths = r.memberFiles.flatMap(id => fileVertexById.get(id).map(_.path)).toSet+ val otherPaths = group.filterNot(_.id == r.id)+ .flatMap(_.memberFiles.flatMap(id => fileVertexById.get(id).map(_.path))).toSet++ // Only use DIRECTORY segments (not filenames) for disambiguation+ def dirSegs(paths: Set[String]): Set[String] =+ paths.flatMap(p => p.split("[/\\\\]").dropRight(1).filterNot(_.isEmpty).map(_.toLowerCase))++ val mySegs = dirSegs(myPaths)+ val otherSegs = dirSegs(otherPaths)+ val exclusive = (mySegs -- otherSegs).filterNot(LabelNoiseTokens.contains)++ // Prefer shorter tokens (more likely to be a meaningful module name)+ val suffix = exclusive.toVector.sortBy(s => (s.length, s)).headOption+ .map(s => s" / ${s.capitalize}")+ .getOrElse(s" (${r.memberFiles.size}f)")+ r.copy(label = r.label + suffix)+ case None => r+ }+ }+ }++ private val LabelNoiseTokens = Set(+ "main", "scala", "java", "src", "test", "tests", "__tests__", "kotlin",+ "python", "js", "ts", "tsx", "jsx", "com", "org", "io", "net",+ "ix", "memory", "index", "mod", "lib", "pkg", "spec", "dist",+ "app", "application", "impl", "implementation", "internal", "external",+ "ix-memory", "memory-layer", "ix-cli", "core-ingestion", "ix-plugin",+ "projects", "claude"+ )++ private def buildOneRegion(+ memberFiles: Set[NodeId],+ level: Int,+ kind: String,+ graph: WeightedFileGraph,+ fileVertexById: Map[NodeId, FileVertex],+ crosscutScores: Map[NodeId, Double],+ rev: Rev,+ commonPfxLen: Int = 0+ ): Region = {+ // Deterministic ID: stable across re-runs with same file set+ val idKey = "map:region:" + memberFiles.map(_.value.toString).toList.sorted.mkString(",")+ val nodeId = NodeId(UUID.nameUUIDFromBytes(idKey.getBytes("UTF-8")))++ // Edge weight metrics+ var internalWeight = 0.0+ var externalWeight = 0.0+ var boundaryNodes = Set.empty[NodeId]+ var signalCounts = Map("calls" -> 0.0, "imports" -> 0.0, "type" -> 0.0)++ for (fileId <- memberFiles) {+ for ((neighbor, w) <- graph.adjMatrix.getOrElse(fileId, Map.empty)) {+ if (memberFiles.contains(neighbor)) {+ internalWeight += w+ } else {+ externalWeight += w+ boundaryNodes += fileId+ }+ }+ }+ // internalWeight is double-counted (symmetric adj), halve it+ internalWeight /= 2.0++ val n = memberFiles.size.toDouble.max(1.0)+ val cohesion = if (n > 1) internalWeight / (n * (n - 1) / 2.0) else 0.0+ // Cap at 100 to avoid floating-point overflow for fully-isolated clusters+ val boundaryRatio = if (externalWeight < 1e-6) math.min(internalWeight / 1e-6, 100.0)+ else internalWeight / externalWeight++ // Cross-cut score: average of member scores+ val crosscutScore =+ memberFiles.toList.map(f => crosscutScores.getOrElse(f, 0.0)).sum / n++ val label = inferLabel(memberFiles, fileVertexById, commonPfxLen)+ val confidence = computeConfidence(cohesion, boundaryRatio, memberFiles.size, crosscutScore)++ Region(+ id = nodeId,+ label = label,+ labelKind = kind,+ level = level,+ memberFiles = memberFiles,+ childRegionIds = Set.empty,+ parentId = None,+ cohesion = cohesion,+ externalCoupling = externalWeight,+ boundaryRatio = boundaryRatio,+ confidence = confidence,+ crosscutScore = crosscutScore,+ dominantSignals = List("calls", "imports", "type").sortBy(s => -signalCounts.getOrElse(s, 0.0)),+ interfaceNodeCount = boundaryNodes.size,+ mapRev = rev.value+ )+ }++ // ── Label inference ─────────────────────────────────────────────────++ /**+ * Infer a human-readable label from file paths in the community.+ *+ * Strategy:+ * 1. Extract all path segments and identifier tokens from filenames.+ * 2. Score each token by frequency within cluster (TF) ×+ * specificity (higher if not a common Java/Scala package name).+ * 3. Return top token, Title-cased.+ * Fallback: use the most common non-trivial directory name.+ */+ private def inferLabel(+ memberFiles: Set[NodeId],+ fileVertexById: Map[NodeId, FileVertex],+ commonPfxLen: Int = 0+ ): String = {+ val rawPaths = memberFiles.flatMap(f => fileVertexById.get(f).map(_.path)).toVector+ if (rawPaths.isEmpty) return "Unknown"+ // Strip common absolute prefix (e.g. /home/user/project/) so labels are project-relative+ val paths = rawPaths.map { p =>+ val segs = p.split("[/\\\\]")+ if (commonPfxLen > 0 && commonPfxLen < segs.length) segs.drop(commonPfxLen).mkString("/")+ else p+ }++ val noiseTokens = LabelNoiseTokens++ // Strategy 1: Longest common directory suffix across all paths+ val dirSegments = paths.map(p => p.split("[/\\\\]").dropRight(1).toVector)+ val commonDir = longestCommonSuffix(dirSegments)+ .filterNot(s => noiseTokens.contains(s.toLowerCase))+ .headOption++ if (commonDir.isDefined) return commonDir.get.capitalize++ // Strategy 2: Most discriminative non-noise directory segment.+ // Prefer deeper (more specific) segments by scoring: frequency × depth-weight.+ // Split each path into (depth, segment) pairs so we can weight deeper dirs higher.+ val allDirsWithDepth: Vector[(Int, String)] = paths.flatMap { p =>+ val segs = p.split("[/\\\\]").dropRight(1)+ segs.zipWithIndex.map { case (s, i) => (i, s) }+ .filter { case (_, s) => !noiseTokens.contains(s.toLowerCase) && s.nonEmpty }+ }+ // Score: count * (1 + depth / maxDepth) — deeper segments score higher+ val maxDepth = allDirsWithDepth.map(_._1).maxOption.getOrElse(1).toDouble.max(1.0)+ val scored = allDirsWithDepth.groupBy(_._2).map { case (seg, entries) =>+ seg -> entries.length * (1.0 + entries.map(_._1 / maxDepth).sum / entries.length)+ }.toVector.sortBy(-_._2)++ scored.headOption.map(_._1.capitalize).getOrElse {+ // Strategy 3: Most frequent meaningful filename token+ val fileTokens = paths.flatMap { p =>+ val name = p.split("[/\\\\]").lastOption.getOrElse("")+ val noExt = name.replaceAll("\\.[^.]+$", "")+ splitCamel(noExt).filter(t => t.length > 3).map(_.toLowerCase)+ }+ fileTokens.filterNot(noiseTokens.contains)+ .groupBy(identity).maxByOption(_._2.size)+ .map(_._1.capitalize).getOrElse("Region")+ }+ }++ /** Returns the number of leading path segments shared by all paths (the common abs prefix). */+ private def commonPathPrefixLength(paths: Vector[String]): Int = {+ if (paths.size <= 1) return 0+ val split = paths.map(_.split("[/\\\\]").toVector)+ val minLen = split.map(_.length).min+ (0 until minLen).takeWhile(i => split.map(_(i)).distinct.size == 1).length+ }++ /** Returns the longest common suffix of all path segment vectors, innermost first. */+ private def longestCommonSuffix(segLists: Vector[Vector[String]]): Vector[String] = {+ if (segLists.isEmpty || segLists.exists(_.isEmpty)) return Vector.empty+ val reversed = segLists.map(_.reverse)+ val maxLen = reversed.map(_.length).min+ (0 until maxLen).toVector+ .takeWhile(i => reversed.map(_(i)).distinct.size == 1)+ .map(i => reversed.head(i))+ }++ private def splitCamel(s: String): Array[String] =+ s.split("(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|[_\\-.]")++ // ── Confidence scoring ──────────────────────────────────────────────++ private def computeConfidence(+ cohesion: Double,+ boundaryRatio: Double,+ size: Int,+ crosscutScore: Double+ ): Double = {+ // Normalize boundary ratio: high ratio → high confidence+ val brScore = math.tanh(boundaryRatio / 3.0) // saturates around ratio = 6+ val cohScore = cohesion.min(1.0)+ val sizeBonus = math.tanh(size / 10.0) // more files → more signal+ val crossPenalty = crosscutScore * 0.3++ ((0.4 * brScore + 0.3 * cohScore + 0.3 * sizeBonus) - crossPenalty).max(0.0).min(1.0)+ }++ // ── Persistence ─────────────────────────────────────────────────────++ private def persistMap(regions: Vector[Region], rev: Rev): IO[Unit] = {+ if (regions.isEmpty) return IO.unit++ for {+ oldRegions <- queryApi.findNodesByKind(NodeKind.Region, limit = 5000)+ _ <- if (oldRegions.nonEmpty) submitDeletePatch(oldRegions, rev) else IO.unit+ _ <- submitUpsertPatch(regions, rev)+ } yield ()+ }++ private def submitDeletePatch(oldNodes: Vector[GraphNode], rev: Rev): IO[Unit] = {+ val ops = oldNodes.map(n => PatchOp.DeleteNode(n.id))+ val patch = GraphPatch(+ patchId = PatchId(UUID.randomUUID()),+ actor = "map-service",+ timestamp = Instant.now(),+ source = PatchSource("ix:map", None, "map-service", SourceType.Inferred),+ baseRev = rev,+ ops = ops.toVector,+ replaces = Vector.empty,+ intent = Some("Clear previous map regions before re-computing")+ )+ writeApi.commitPatch(patch).void+ }++ private def submitUpsertPatch(regions: Vector[Region], rev: Rev): IO[Unit] = {+ val nodeOps: Vector[PatchOp] = regions.map { r =>+ PatchOp.UpsertNode(+ id = r.id,+ kind = NodeKind.Region,+ name = r.label,+ attrs = Map(+ "map_generated" -> Json.True,+ "level" -> r.level.asJson,+ "label_kind" -> r.labelKind.asJson,+ "file_count" -> r.memberFiles.size.asJson,+ "member_files" -> r.memberFiles.map(_.value.toString).asJson,+ "cohesion" -> r.cohesion.asJson,+ "external_coupling" -> r.externalCoupling.asJson,+ "boundary_ratio" -> r.boundaryRatio.asJson,+ "confidence" -> r.confidence.asJson,+ "crosscut_score" -> r.crosscutScore.asJson,+ "dominant_signals" -> r.dominantSignals.asJson,+ "interface_node_count" -> r.interfaceNodeCount.asJson,+ "map_rev" -> r.mapRev.asJson+ )+ )+ }++ // IN_REGION edges: file → owning region (level-1 regions only)+ val fileEdgeOps: Vector[PatchOp] = regions+ .filter(_.level == 1)+ .flatMap { r =>+ r.memberFiles.toVector.map { fileId =>+ val edgeId = EdgeId(+ UUID.nameUUIDFromBytes(s"in_region:${fileId.value}:${r.id.value}".getBytes("UTF-8"))+ )+ PatchOp.UpsertEdge(+ id = edgeId,+ src = fileId,+ dst = r.id,+ predicate = EdgePredicate("IN_REGION"),+ attrs = Map.empty+ )+ }+ }++ // IN_REGION edges: child region → parent region+ val regionEdgeOps: Vector[PatchOp] = regions.flatMap { r =>+ r.parentId.map { pid =>+ val edgeId = EdgeId(+ UUID.nameUUIDFromBytes(s"in_region:${r.id.value}:${pid.value}".getBytes("UTF-8"))+ )+ PatchOp.UpsertEdge(+ id = edgeId,+ src = r.id,+ dst = pid,+ predicate = EdgePredicate("IN_REGION"),+ attrs = Map.empty+ )+ }+ }++ val allOps = nodeOps ++ fileEdgeOps ++ regionEdgeOps+ val patch = GraphPatch(+ patchId = PatchId(UUID.nameUUIDFromBytes(s"map:${rev.value}".getBytes("UTF-8"))),+ actor = "map-service",+ timestamp = Instant.now(),+ source = PatchSource("ix:map", None, "map-service", SourceType.Inferred),+ baseRev = rev,+ ops = allOps,+ replaces = Vector.empty,+ intent = Some(s"Architectural map computed at rev ${rev.value}")+ )++ writeApi.commitPatch(patch).void+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/map/MapTypes.scala b/memory-layer/src/main/scala/ix/memory/map/MapTypes.scalanew file mode 100644index 00000000..2b7dd9b3--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/map/MapTypes.scala@@ -0,0 +1,61 @@+package ix.memory.map++import ix.memory.model.NodeId++/** A file node with its UUID and path (used as label source). */+final case class FileVertex(id: NodeId, path: String)++/** Raw edge from ArangoDB before weight computation. */+final case class RawFilePair(srcId: String, dstId: String, predicate: String, count: Int)++/** Weighted edge between two file nodes (symmetric — both directions stored). */+final case class FileCouplingEdge(src: NodeId, dst: NodeId, weight: Double, dominant: String)++/**+ * Weighted, undirected file-level coupling graph.+ *+ * adjMatrix is symmetric: adjMatrix(a)(b) == adjMatrix(b)(a)+ * totalWeight is the sum of all weights counting each edge once.+ */+final case class WeightedFileGraph(+ vertices: Vector[FileVertex],+ adjMatrix: Map[NodeId, Map[NodeId, Double]],+ degrees: Map[NodeId, Double],+ totalWeight: Double+)++object WeightedFileGraph {+ val empty: WeightedFileGraph =+ WeightedFileGraph(Vector.empty, Map.empty, Map.empty, 0.0)+}++/**+ * A single inferred architectural region at one level of the hierarchy.+ *+ * `memberFiles` always refers to the original file NodeIds.+ * `parentId` is set for regions at level >= 2 (their containing region at level+1).+ */+final case class Region(+ id: NodeId,+ label: String,+ labelKind: String, // "module" | "subsystem" | "system"+ level: Int, // 1 = finest, 2 = mid, 3 = coarse+ memberFiles: Set[NodeId],+ childRegionIds: Set[NodeId], // direct children at level-1 (empty for level 1)+ parentId: Option[NodeId],+ cohesion: Double,+ externalCoupling: Double,+ boundaryRatio: Double,+ confidence: Double,+ crosscutScore: Double,+ dominantSignals: List[String],+ interfaceNodeCount: Int,+ mapRev: Long+)++/** The full multi-level architecture map returned by MapService. */+final case class ArchitectureMap(+ regions: Vector[Region],+ fileCount: Int,+ mapRev: Long+)diff --git a/memory-layer/src/main/scala/ix/memory/model/Node.scala b/memory-layer/src/main/scala/ix/memory/model/Node.scalaindex 1bff21f3..164417b6 100644--- a/memory-layer/src/main/scala/ix/memory/model/Node.scala+++ b/memory-layer/src/main/scala/ix/memory/model/Node.scala@@ -27,6 +27,7 @@ object NodeKind {case object Task extends NodeKindcase object Bug extends NodeKindcase object Goal extends NodeKind+ case object Region extends NodeKindprivate val nameMap: Map[String, NodeKind] = Map("module" -> Module,@@ -48,7 +49,8 @@ object NodeKind {"plan" -> Plan,"task" -> Task,"bug" -> Bug,- "goal" -> Goal+ "goal" -> Goal,+ "region" -> Region)implicit val encoder: Encoder[NodeKind] = Encoder[String].contramap {@@ -72,6 +74,7 @@ object NodeKind {case Task => "task"case Bug => "bug"case Goal => "goal"+ case Region => "region"}implicit val decoder: Decoder[NodeKind] = Decoder[String].emap { s =>