Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

Remove Scala backend ingestion services

Joseph Mikhail246d46f49d89
diff --git a/build.sbt b/build.sbt
index 4ccc51e6..1b129698 100644
--- a/build.sbt
+++ b/build.sbt
@@ -29,12 +29,6 @@ lazy val memoryLayer = (project in file("memory-layer"))
// ArangoDB
"com.arangodb" % "arangodb-java-driver" % "7.12.0",
- // Tree-sitter (bonede/tree-sitter-ng — ARM64 macOS compatible)
- "io.github.bonede" % "tree-sitter" % "0.26.6",
- "io.github.bonede" % "tree-sitter-scala" % "0.24.0",
- "io.github.bonede" % "tree-sitter-typescript" % "0.23.2",
- "io.github.bonede" % "tree-sitter-go" % "0.23.3",
-
// Logging
"org.typelevel" %% "log4cats-slf4j" % "2.7.0",
"ch.qos.logback" % "logback-classic" % "1.5.6",
diff --git a/memory-layer/src/main/scala/ix/memory/Main.scala b/memory-layer/src/main/scala/ix/memory/Main.scala
index 9ce6d3db..6020f93a 100644
--- a/memory-layer/src/main/scala/ix/memory/Main.scala
+++ b/memory-layer/src/main/scala/ix/memory/Main.scala
@@ -9,7 +9,6 @@ import ix.memory.api.Routes
import ix.memory.conflict.ConflictService
import ix.memory.context._
import ix.memory.db._
-import ix.memory.ingestion._
import ix.memory.map.MapService
import ix.memory.smell.SmellService
import ix.memory.subsystem.SubsystemScoringService
@@ -33,10 +32,7 @@ object Main extends IOApp.Simple {
queryApi = new ArangoGraphQueryApi(client)
// 3. Services
- parserRouter = new ParserRouter()
- ingestionService = new IngestionService(parserRouter, writeApi, queryApi)
bulkWriteApi = new BulkWriteApi(client)
- bulkIngestionService = new BulkIngestionService(parserRouter, bulkWriteApi, queryApi)
seeder = new GraphSeeder(queryApi)
expander = new GraphExpander(queryApi)
claimCollector = new ClaimCollector(queryApi)
@@ -50,7 +46,7 @@ object Main extends IOApp.Simple {
subsystemService = new SubsystemScoringService(client, writeApi, mapService)
// 4. HTTP routes
- routes = Routes.all(contextService, ingestionService, bulkIngestionService, queryApi, writeApi, conflictService, client, mapService, bulkWriteApi, smellService, subsystemService)
+ routes = Routes.all(contextService, queryApi, writeApi, conflictService, client, mapService, bulkWriteApi, smellService, subsystemService)
// 5. Server
server <- EmberServerBuilder
diff --git a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala b/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
deleted file mode 100644
index d393c47d..00000000
--- a/memory-layer/src/main/scala/ix/memory/api/IngestionRoutes.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-package ix.memory.api
-
-import java.nio.file.Paths
-
-import cats.effect.IO
-import io.circe.{Decoder, Encoder}
-import io.circe.generic.semiauto.{deriveDecoder, deriveEncoder}
-import org.http4s.HttpRoutes
-import org.http4s.circe.CirceEntityCodec._
-import org.http4s.dsl.io._
-
-import ix.memory.ingestion.{BulkIngestionService, IngestionProgress, IngestionService}
-import org.typelevel.log4cats.slf4j.Slf4jLogger
-
-case class IngestRequest(path: String, language: Option[String], recursive: Option[Boolean], force: Option[Boolean])
-
-object IngestRequest {
- implicit val decoder: Decoder[IngestRequest] = deriveDecoder[IngestRequest]
- implicit val encoder: Encoder[IngestRequest] = deriveEncoder[IngestRequest]
-}
-
-case class SkipReasonsResponse(unchanged: Int, emptyFile: Int, parseError: Int, tooLarge: Int)
-
-object SkipReasonsResponse {
- implicit val encoder: Encoder[SkipReasonsResponse] = deriveEncoder[SkipReasonsResponse]
- implicit val decoder: Decoder[SkipReasonsResponse] = deriveDecoder[SkipReasonsResponse]
-}
-
-case class IngestResponse(filesProcessed: Int, patchesApplied: Int, filesSkipped: Int,
- entitiesCreated: Int, latestRev: Long, skipReasons: SkipReasonsResponse)
-
-object IngestResponse {
- implicit val encoder: Encoder[IngestResponse] = deriveEncoder[IngestResponse]
- implicit val decoder: Decoder[IngestResponse] = deriveDecoder[IngestResponse]
-}
-
-class IngestionRoutes(ingestionService: IngestionService, bulkIngestionService: BulkIngestionService) {
-
- private val logger = Slf4jLogger.getLoggerFromName[IO]("ix.ingest.progress")
-
- private val progressLog: IngestionProgress => IO[Unit] = { p =>
- logger.info(s"discovered=${p.filesDiscovered} parsed=${p.filesParsed} chunks=${p.chunksWritten}/${p.totalChunks}")
- }
-
- val routes: HttpRoutes[IO] = HttpRoutes.of[IO] {
- case req @ POST -> Root / "v1" / "ingest" =>
- (for {
- body <- req.as[IngestRequest]
- _ <- IO.raiseWhen(body.path.contains(".."))(
- new IllegalArgumentException("Path traversal not allowed")
- )
- path = Paths.get(body.path)
- result <- bulkIngestionService.ingestPath(path, body.language, body.recursive.getOrElse(false), progressLog, body.force.getOrElse(false))
- resp <- Ok(IngestResponse(
- filesProcessed = result.filesProcessed,
- patchesApplied = result.patchesApplied,
- filesSkipped = result.filesSkipped,
- entitiesCreated = result.entitiesCreated,
- latestRev = result.latestRev.value,
- skipReasons = SkipReasonsResponse(
- result.skipReasons.unchanged,
- result.skipReasons.emptyFile,
- result.skipReasons.parseError,
- result.skipReasons.tooLarge
- )
- ))
- } yield resp).handleErrorWith(ErrorHandler.handle(_))
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/api/Routes.scala b/memory-layer/src/main/scala/ix/memory/api/Routes.scala
index 376a16c9..d8dd22dd 100644
--- a/memory-layer/src/main/scala/ix/memory/api/Routes.scala
+++ b/memory-layer/src/main/scala/ix/memory/api/Routes.scala
@@ -11,7 +11,6 @@ import org.http4s.dsl.io._
import ix.memory.conflict.ConflictService
import ix.memory.context.ContextService
import ix.memory.db.{ArangoClient, BulkWriteApi, GraphQueryApi, GraphWriteApi}
-import ix.memory.ingestion.{BulkIngestionService, IngestionService}
import ix.memory.map.MapService
import ix.memory.smell.SmellService
import ix.memory.subsystem.SubsystemScoringService
@@ -20,8 +19,6 @@ object Routes {
def all(
contextService: ContextService,
- ingestionService: IngestionService,
- bulkIngestionService: BulkIngestionService,
queryApi: GraphQueryApi,
writeApi: GraphWriteApi,
conflictService: ConflictService,
@@ -38,7 +35,6 @@ object Routes {
}
val contextRoutes = new ContextRoutes(contextService).routes
- val ingestionRoutes = new IngestionRoutes(ingestionService, bulkIngestionService).routes
val entityRoutes = new EntityRoutes(queryApi).routes
val diffRoutes = new DiffRoutes(queryApi).routes
val conflictRoutes = new ConflictRoutes(conflictService).routes
@@ -59,7 +55,7 @@ object Routes {
val smellRoutes = new SmellRoutes(smellService).routes
val subsystemRoutes = new SubsystemRoutes(subsystemService, mapService).routes
- health <+> contextRoutes <+> ingestionRoutes <+> entityRoutes <+>
+ health <+> contextRoutes <+> entityRoutes <+>
diffRoutes <+> conflictRoutes <+> decideRoutes <+> searchRoutes <+>
truthRoutes <+> patchRoutes <+> decisionListRoutes <+> expandRoutes <+>
statsRoutes <+> patchCommitRoutes <+> listRoutes <+> goalRoutes <+>
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
deleted file mode 100644
index d94e39fe..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/BulkIngestionService.scala
+++ /dev/null
@@ -1,245 +0,0 @@
-package ix.memory.ingestion
-
-import java.nio.file.{Files, Path}
-
-import cats.effect.IO
-import cats.syntax.all._
-import cats.effect.implicits._
-import io.circe.Json
-import io.circe.syntax._
-import org.typelevel.log4cats.slf4j.Slf4jLogger
-import org.typelevel.log4cats.Logger
-
-import ix.memory.db.{BulkWriteApi, FileBatch, GraphQueryApi}
-import ix.memory.model._
-
-private[ingestion] sealed trait ParseOutcome
-private[ingestion] case class Parsed(batch: FileBatch) extends ParseOutcome
-private[ingestion] case class Skipped(reason: String) extends ParseOutcome
-
-/**
- * High-performance ingestion service that:
- * 1. Discovers files (with optional language filter)
- * 2. Parses files in parallel (bounded by CPU cores)
- * 3. Batches parsed results
- * 4. Bulk-inserts via Document API
- */
-class BulkIngestionService(
- parserRouter: ParserRouter,
- bulkWriteApi: BulkWriteApi,
- queryApi: GraphQueryApi
-) {
-
- private val parallelism = Runtime.getRuntime.availableProcessors.max(2)
- private val MaxFileBytes: Long = 1024 * 1024 // 1 MB
- private val ParseBatchSize = 100
- private val MaxCommitFiles = 100
- private val MaxCommitPayloadBytes = 1024 * 1024
- private val mapper = new com.fasterxml.jackson.databind.ObjectMapper()
- private val logger: Logger[IO] = Slf4jLogger.getLoggerFromName[IO]("ix.ingestion")
-
- /**
- * Ingest files under a path using parallel parsing and bulk writes.
- */
- def ingestPath(
- path: Path,
- language: Option[String],
- recursive: Boolean,
- onProgress: IngestionProgress => IO[Unit] = _ => IO.unit,
- force: Boolean = false
- ): IO[IngestionResult] = {
- for {
- files <- discoverFiles(path, language, recursive)
- _ <- onProgress(IngestionProgress(files.size, 0, 0, 0))
- latestRev <- queryApi.getLatestRev
- state <- files.grouped(ParseBatchSize).toVector.foldLeft(IO.pure(BulkIngestionState.empty(latestRev.value))) {
- case (stateIO, batch) =>
- stateIO.flatMap(processBatch(batch, force, files.size, onProgress, _))
- }
- } yield IngestionResult(
- filesProcessed = files.size,
- patchesApplied = state.patchesApplied,
- filesSkipped = state.skipped.total,
- entitiesCreated = state.entitiesCreated,
- latestRev = Rev(state.currentRev),
- skipReasons = state.skipped
- )
- }
-
- private def processBatch(
- files: List[Path],
- force: Boolean,
- discovered: Int,
- onProgress: IngestionProgress => IO[Unit],
- state: BulkIngestionState
- ): IO[BulkIngestionState] = {
- for {
- hashMap <- if (force) IO.pure(Map.empty[String, String]) else loadExistingHashes(files)
- outcomes <- files.parTraverseN(parallelism) { f =>
- parseFile(f, hashMap).handleErrorWith { err =>
- logger.warn(s"Parse error: ${f.toString} - ${err.getMessage}") *>
- IO.pure(Skipped("parseError"): ParseOutcome)
- }
- }
- validBatches = outcomes.collect { case Parsed(batch) => batch }.toVector
- batchSkipped = SkipReasons(
- unchanged = outcomes.count { case Skipped("unchanged") => true; case _ => false },
- emptyFile = outcomes.count { case Skipped("emptyFile") => true; case _ => false },
- parseError = outcomes.count { case Skipped("parseError") => true; case _ => false },
- tooLarge = outcomes.count { case Skipped("tooLarge") => true; case _ => false }
- )
- parsedFiles = state.filesParsed + outcomes.size
- commitChunks = chunkForCommit(validBatches)
- totalChunks = state.totalChunksPlanned + commitChunks.size
- _ <- onProgress(IngestionProgress(discovered, parsedFiles, state.chunksWritten, totalChunks))
- committed <- commitChunks.foldLeft(IO.pure((state.currentRev, state.chunksWritten))) {
- case (revIO, chunk) =>
- revIO.flatMap { case (currentRev, chunksWritten) =>
- bulkWriteApi.commitBatch(chunk, currentRev).flatMap { result =>
- val updatedChunks = chunksWritten + 1
- onProgress(IngestionProgress(discovered, parsedFiles, updatedChunks, totalChunks)) *>
- IO.pure((result.newRev.value, updatedChunks))
- }
- }
- }
- entitiesCreated = validBatches.flatMap(_.patch.ops.collect { case _: PatchOp.UpsertNode => 1 }).size
- } yield state.copy(
- currentRev = committed._1,
- filesParsed = parsedFiles,
- patchesApplied = state.patchesApplied + validBatches.size,
- chunksWritten = committed._2,
- totalChunksPlanned = totalChunks,
- entitiesCreated = state.entitiesCreated + entitiesCreated,
- skipped = combineSkipReasons(state.skipped, batchSkipped)
- )
- }
-
- /**
- * Parse a single file into a FileBatch (or None if unchanged).
- */
- private def parseFile(filePath: Path, hashMap: Map[String, String]): IO[ParseOutcome] = {
- for {
- fileSize <- IO.blocking(Files.size(filePath))
- result <- if (fileSize > MaxFileBytes) IO.pure(Skipped("tooLarge"): ParseOutcome)
- else if (fileSize == 0L) IO.pure(Skipped("emptyFile"): ParseOutcome)
- else {
- for {
- bytes <- IO.blocking(Files.readAllBytes(filePath))
- hash = sha256Bytes(bytes)
- outcome <- hashMap.get(filePath.toString) match {
- case Some(existingHash) if existingHash == hash =>
- IO.pure(Skipped("unchanged"): ParseOutcome)
- case _ =>
- for {
- source <- IO.blocking {
- try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)
- catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }
- }
- parserOpt = parserRouter.parserFor(filePath.toString)
- parseResult = parserOpt match {
- case Some(p) => p.parse(filePath.getFileName.toString, source)
- case None => GenericTextFallback.parse(filePath.getFileName.toString, source)
- }
- patch = GraphPatchBuilder.build(filePath.toString, Some(hash), parseResult)
- _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))
- provenance = buildProvenanceMap(patch)
- } yield Parsed(FileBatch(filePath.toString, Some(hash), patch, provenance)): ParseOutcome
- }
- } yield outcome
- }
- } yield result
- }
-
- private def loadExistingHashes(files: Seq[Path]): IO[Map[String, String]] = {
- val paths = files.map(_.toString)
- queryApi.getSourceHashes(paths)
- }
-
- private def buildProvenanceMap(patch: GraphPatch): java.util.Map[String, AnyRef] = {
- val json = Json.obj(
- "source_uri" -> Json.fromString(patch.source.uri),
- "source_hash" -> patch.source.sourceHash.fold(Json.Null)(Json.fromString),
- "extractor" -> Json.fromString(patch.source.extractor),
- "source_type" -> Json.fromString(patch.source.sourceType.asJson.asString.getOrElse("code")),
- "observed_at" -> Json.fromString(patch.timestamp.toString)
- )
- mapper.readValue(json.noSpaces, classOf[java.util.Map[String, AnyRef]])
- }
-
- private def discoverFiles(
- path: Path,
- language: Option[String],
- recursive: Boolean
- ): IO[List[Path]] = IO.blocking {
- val extensions = FileDiscovery.extensionsFor(language)
- if (Files.isRegularFile(path)) {
- if (language.isEmpty || FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty
- } else if (Files.isDirectory(path)) {
- // Try git ls-files first (respects .gitignore), fall back to manual walk
- FileDiscovery.tryGitLsFiles(path, recursive, extensions)
- .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))
- } else List.empty
- }
-
- private def chunkForCommit(batches: Vector[FileBatch]): Vector[Vector[FileBatch]] = {
- val chunks = Vector.newBuilder[Vector[FileBatch]]
- var current = Vector.empty[FileBatch]
- var currentBytes = 0
-
- batches.foreach { batch =>
- val batchBytes = batch.estimatedPayloadBytes
- val exceedsPayload = current.nonEmpty && currentBytes + batchBytes > MaxCommitPayloadBytes
- val exceedsCount = current.size >= MaxCommitFiles
- if (exceedsPayload || exceedsCount) {
- chunks += current
- current = Vector.empty
- currentBytes = 0
- }
- current = current :+ batch
- currentBytes += batchBytes
- }
-
- if (current.nonEmpty) {
- chunks += current
- }
-
- chunks.result()
- }
-
- private def combineSkipReasons(left: SkipReasons, right: SkipReasons): SkipReasons =
- SkipReasons(
- unchanged = left.unchanged + right.unchanged,
- emptyFile = left.emptyFile + right.emptyFile,
- parseError = left.parseError + right.parseError,
- tooLarge = left.tooLarge + right.tooLarge
- )
-
- private def sha256Bytes(bytes: Array[Byte]): String = {
- val md = java.security.MessageDigest.getInstance("SHA-256")
- md.digest(bytes).map("%02x".format(_)).mkString
- }
-
-}
-
-private[ingestion] final case class BulkIngestionState(
- currentRev: Long,
- filesParsed: Int,
- patchesApplied: Int,
- chunksWritten: Int,
- totalChunksPlanned: Int,
- entitiesCreated: Int,
- skipped: SkipReasons
-)
-
-private[ingestion] object BulkIngestionState {
- def empty(baseRev: Long): BulkIngestionState =
- BulkIngestionState(
- currentRev = baseRev,
- filesParsed = 0,
- patchesApplied = 0,
- chunksWritten = 0,
- totalChunksPlanned = 0,
- entitiesCreated = 0,
- skipped = SkipReasons()
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala b/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala
deleted file mode 100644
index 26e1d212..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/FileDiscovery.scala
+++ /dev/null
@@ -1,120 +0,0 @@
-package ix.memory.ingestion
-
-import java.nio.file.{Files, Path}
-
-/**
- * Discovers source files for ingestion.
- *
- * Primary strategy: `git ls-files` — respects .gitignore, fast, works for any repo.
- * Fallback: manual file walk with hardcoded ignore list (for non-git directories).
- */
-object FileDiscovery {
-
- private val defaultExtensions = Set(
- ".py", ".ts", ".tsx",
- ".scala", ".sc",
- ".json", ".yaml", ".yml", ".toml",
- ".md"
- )
-
- def extensionsFor(language: Option[String]): Set[String] = language match {
- case Some("python") => Set(".py")
- case Some("typescript") => Set(".ts", ".tsx")
- case Some("javascript") => Set(".js", ".jsx", ".mjs", ".cjs")
- case Some("scala") => Set(".scala", ".sc")
- case Some("java") => Set(".java")
- case Some("go") => Set(".go")
- case Some("rust") => Set(".rs")
- case Some("ruby") => Set(".rb")
- case Some("config") => Set(".json", ".yaml", ".yml", ".toml", ".ini", ".conf", ".properties", ".xml")
- case Some("markdown") => Set(".md", ".mdx")
- case _ => defaultExtensions
- }
-
- def matchesFilter(file: Path, extensions: Set[String]): Boolean = {
- val name = file.getFileName.toString
- extensions.exists(name.endsWith)
- }
-
- /**
- * Use `git ls-files` to discover tracked files. Returns None if not a git repo
- * or git is not available.
- */
- def tryGitLsFiles(dir: Path, recursive: Boolean, extensions: Set[String]): Option[List[Path]] = {
- try {
- val pb = new ProcessBuilder("git", "ls-files", "--cached", "--others", "--exclude-standard")
- pb.directory(dir.toFile)
- pb.redirectErrorStream(true)
- val proc = pb.start()
- val reader = new java.io.BufferedReader(
- new java.io.InputStreamReader(proc.getInputStream, java.nio.charset.StandardCharsets.UTF_8)
- )
- val result = scala.collection.mutable.ListBuffer.empty[Path]
-
- try {
- var line = reader.readLine()
- while (line != null) {
- if (line.nonEmpty) {
- val p = dir.resolve(line).normalize()
- val isUnder = if (recursive) true else p.getParent == dir
- if (isUnder && Files.isRegularFile(p) && matchesFilter(p, extensions)) {
- result += p
- }
- }
- line = reader.readLine()
- }
- } finally {
- reader.close()
- }
-
- val exitCode = proc.waitFor()
- if (exitCode != 0) None else Some(result.toList)
- } catch {
- case _: Throwable => None
- }
- }
-
- /**
- * Fallback: walk the directory tree, skipping common non-source directories.
- */
- def walkFiles(dir: Path, recursive: Boolean, extensions: Set[String]): List[Path] = {
- import scala.jdk.CollectionConverters._
-
- if (!recursive) {
- val stream = Files.list(dir)
- try {
- stream.iterator().asScala
- .filter(Files.isRegularFile(_))
- .filter(matchesFilter(_, extensions))
- .toList
- } finally stream.close()
- } else {
- // Directories to skip when not using git
- val ignoredDirs = Set(
- "node_modules", ".git", ".hg", ".svn",
- "target", "dist", "build", "out", ".next",
- "__pycache__", ".tox", ".venv", "venv", ".mypy_cache", ".pytest_cache",
- ".gradle", ".idea", ".vscode", ".settings", ".vs",
- ".cache", ".parcel-cache", "coverage", ".nyc_output",
- "vendor", "Pods", ".dart_tool", ".pub-cache",
- "bin", "obj", "pkg"
- )
- val result = scala.collection.mutable.ListBuffer.empty[Path]
- Files.walkFileTree(dir, new java.nio.file.SimpleFileVisitor[Path] {
- override def preVisitDirectory(d: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
- val name = d.getFileName.toString
- if (d != dir && (ignoredDirs.contains(name) || name.startsWith(".")))
- java.nio.file.FileVisitResult.SKIP_SUBTREE
- else
- java.nio.file.FileVisitResult.CONTINUE
- }
- override def visitFile(file: Path, attrs: java.nio.file.attribute.BasicFileAttributes): java.nio.file.FileVisitResult = {
- if (matchesFilter(file, extensions))
- result += file
- java.nio.file.FileVisitResult.CONTINUE
- }
- })
- result.toList
- }
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala b/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala
deleted file mode 100644
index 8a7c4baf..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/Fingerprint.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-package ix.memory.ingestion
-
-import java.security.MessageDigest
-
-object Fingerprint {
-
- /** Compute a normalized content fingerprint from signature + body.
- * Returns a 32-char hex string (first 16 bytes of SHA-256).
- */
- def compute(signature: String, body: String): String = {
- val normalized = (signature.trim + "\n" + body.trim)
- .replaceAll("\\s+", " ")
- .trim
- val digest = MessageDigest.getInstance("SHA-256")
- digest.digest(normalized.getBytes("UTF-8"))
- .take(16)
- .map("%02x".format(_))
- .mkString
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/GenericTextFallback.scala b/memory-layer/src/main/scala/ix/memory/ingestion/GenericTextFallback.scala
deleted file mode 100644
index 29d670ea..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/GenericTextFallback.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-package ix.memory.ingestion
-
-import io.circe.Json
-import ix.memory.model.NodeKind
-
-object GenericTextFallback {
- private val PreviewLimit = 240
-
- def parse(fileName: String, source: String): ParseResult = {
- val lines = if (source.isEmpty) 1 else source.count(_ == '\n') + 1
- val preview = source.linesIterator.find(_.trim.nonEmpty).map(_.trim.take(PreviewLimit))
-
- val attrs = Map(
- "file_name" -> Json.fromString(fileName),
- "line_count" -> Json.fromInt(lines)
- ) ++ preview.filter(_.nonEmpty).map(value => "preview" -> Json.fromString(value))
-
- ParseResult(
- entities = Vector(
- ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = attrs,
- lineStart = 1,
- lineEnd = lines,
- contentFingerprint = Some(Fingerprint.compute(fileName, source))
- )
- ),
- relationships = Vector.empty
- )
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
deleted file mode 100644
index 3a903acd..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
+++ /dev/null
@@ -1,100 +0,0 @@
-package ix.memory.ingestion
-
-import java.time.Instant
-import java.util.UUID
-
-import io.circe.Json
-
-import ix.memory.model._
-
-object GraphPatchBuilder {
-
- /**
- * Build a GraphPatch from a ParseResult and file metadata.
- *
- * Uses deterministic UUID generation (UUID v3 (MD5-based nameUUID)) so that
- * re-parsing the same file produces the same node/edge IDs, enabling
- * idempotent upserts.
- */
- def build(
- filePath: String,
- sourceHash: Option[String],
- parseResult: ParseResult
- ): GraphPatch = {
-
- def nodeIdFor(name: String): NodeId =
- NodeId(UUID.nameUUIDFromBytes(s"$filePath:$name".getBytes("UTF-8")))
-
- def edgeIdFor(src: String, dst: String, predicate: String): EdgeId =
- EdgeId(UUID.nameUUIDFromBytes(s"$filePath:$src:$dst:$predicate".getBytes("UTF-8")))
-
- val extractor = if (filePath.endsWith(".py")) "tree-sitter-python/1.0"
- else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) "typescript-parser/1.0"
- else if (filePath.endsWith(".scala") || filePath.endsWith(".sc")) "scala-parser/1.0"
- else if (filePath.endsWith(".go")) "tree-sitter-go/1.0"
- else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) "config-parser/1.0"
- else if (filePath.endsWith(".md")) "markdown-parser/1.0"
- else "unknown-parser/1.0"
-
- val nodeOps = parseResult.entities.map { e =>
- val baseAttrs = e.attrs +
- ("line_start" -> Json.fromInt(e.lineStart)) +
- ("line_end" -> Json.fromInt(e.lineEnd))
- val attrsWithFp = e.contentFingerprint match {
- case Some(fp) => baseAttrs + ("content_fingerprint" -> Json.fromString(fp))
- case None => baseAttrs
- }
- PatchOp.UpsertNode(nodeIdFor(e.name), e.kind, e.name, attrsWithFp)
- }
-
- val edgeOps = parseResult.relationships.map { r =>
- PatchOp.UpsertEdge(
- edgeIdFor(r.srcName, r.dstName, r.predicate),
- nodeIdFor(r.srcName),
- nodeIdFor(r.dstName),
- EdgePredicate(r.predicate),
- Map.empty
- )
- }
-
- // Generate claims so the confidence/conflict engine has data to operate on.
- val relationshipClaims = parseResult.relationships.map { r =>
- PatchOp.AssertClaim(
- entityId = nodeIdFor(r.srcName),
- field = s"${r.predicate.toLowerCase}:${r.dstName}",
- value = Json.fromString(r.dstName),
- confidence = None // will be computed at query time
- )
- }
-
- val attributeClaims = parseResult.entities.flatMap { e =>
- e.attrs.flatMap { case (key, value) =>
- Vector(PatchOp.AssertClaim(
- entityId = nodeIdFor(e.name),
- field = key,
- value = value,
- confidence = None
- ))
- }
- }
-
- val claimOps = relationshipClaims ++ attributeClaims
- val allOps = nodeOps ++ edgeOps ++ claimOps
-
- val sourceType = if (filePath.endsWith(".py") || filePath.endsWith(".ts") || filePath.endsWith(".tsx") || filePath.endsWith(".scala") || filePath.endsWith(".sc") || filePath.endsWith(".go")) SourceType.Code
- else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) SourceType.Config
- else if (filePath.endsWith(".md")) SourceType.Doc
- else SourceType.Code
-
- GraphPatch(
- patchId = PatchId(UUID.randomUUID()),
- actor = "ix/ingestion",
- timestamp = Instant.now(),
- source = PatchSource(filePath, sourceHash, extractor, sourceType),
- baseRev = Rev(0L),
- ops = allOps,
- replaces = Vector.empty,
- intent = Some(s"Parsed $filePath")
- )
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
deleted file mode 100644
index 363fbc0d..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
+++ /dev/null
@@ -1,256 +0,0 @@
-package ix.memory.ingestion
-
-import java.nio.file.{Files, Path}
-import java.time.Instant
-import java.util.UUID
-
-import cats.effect.IO
-import cats.syntax.all._
-import ix.memory.db.{CommitResult, GraphQueryApi, GraphWriteApi}
-import ix.memory.model._
-/**
- * Orchestrates file ingestion: reads source files, parses them,
- * builds graph patches, and commits them through the write API.
- */
-class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, queryApi: GraphQueryApi) {
-
- /**
- * Ingest a single file: read, parse, build patch, commit.
- */
- def ingestFile(filePath: Path): IO[CommitResult] = {
- for {
- bytes <- IO.blocking(Files.readAllBytes(filePath))
- _ <- if (bytes.isEmpty) IO.raiseError(new RuntimeException(s"Skipping empty file: $filePath"))
- else IO.unit
- // Hash raw bytes so revisions are stable across encodings
- hash = Some(sha256Bytes(bytes))
- // Decode text with UTF-8, fallback to ISO-8859-1 to avoid hard failures
- source <- IO.blocking {
- try new String(bytes, java.nio.charset.StandardCharsets.UTF_8)
- catch { case _: Throwable => new String(bytes, java.nio.charset.StandardCharsets.ISO_8859_1) }
- }
- // Use the language-specific parser when available; otherwise fall back to generic text
- parserOpt = parserRouter.parserFor(filePath.toString)
- result = parserOpt match {
- case Some(p) => p.parse(filePath.getFileName.toString, source)
- case None => GenericTextFallback.parse(filePath.getFileName.toString, source)
- }
- val fp = filePath.toString
- extractor = if (fp.endsWith(".py")) "tree-sitter-python/1.0"
- else if (fp.endsWith(".ts") || fp.endsWith(".tsx")) "typescript-parser/1.0"
- else if (fp.endsWith(".scala") || fp.endsWith(".sc")) "scala-parser/1.0"
- else if (fp.endsWith(".go")) "tree-sitter-go/1.0"
- else if (fp.endsWith(".json") || fp.endsWith(".yaml") || fp.endsWith(".yml") || fp.endsWith(".toml") || fp.endsWith(".ini") || fp.endsWith(".conf") || fp.endsWith(".properties") || fp.endsWith(".env")) "config-parser/1.0"
- else if (fp.endsWith(".md") || fp.endsWith(".mdx") || fp.endsWith(".rst") || fp.endsWith(".txt")) "markdown-parser/1.0"
- else "text-parser/1.0"
- existing <- queryApi.getPatchesBySource(filePath.toString, extractor)
- prevPatch = existing.headOption
- prevHash = prevPatch.flatMap(_.hcursor.downField("data").downField("source").get[String]("sourceHash").toOption)
- // If hash unchanged, skip (idempotent)
- _ <- if (prevHash == hash) IO.raiseError(new RuntimeException("Idempotent — file unchanged"))
- else IO.unit
- prevPatchId = prevPatch.flatMap(_.hcursor.downField("data").get[String]("patchId").toOption)
- .flatMap(s => scala.util.Try(java.util.UUID.fromString(s)).toOption)
- .map(PatchId(_))
- replaces = prevPatchId.toVector
- patch = GraphPatchBuilder.build(filePath.toString, hash, result).copy(replaces = replaces)
- _ <- IO.fromEither(PatchValidator.validate(patch).left.map(msg => new IllegalStateException(msg)))
- commit <- writeApi.commitPatch(patch)
- // Resolve cross-file import edges after main patch is committed
- _ <- resolveImportEdges(filePath, result, hash).attempt.void
- // Resolve named import symbols to their actual source entities
- _ <- resolveNamedImports(filePath, result, hash).attempt.void
- } yield commit
- }
-
- /**
- * Ingest all matching files under a path.
- *
- * @param path root directory or single file
- * @param language optional language filter (e.g. "python")
- * @param recursive whether to recurse into subdirectories
- * @return aggregated ingestion result
- */
- def ingestPath(
- path: Path,
- language: Option[String],
- recursive: Boolean
- ): IO[IngestionResult] = {
- for {
- files <- discoverFiles(path, language, recursive)
- results <- files.traverse(f => ingestFile(f).attempt)
- successes = results.collect { case Right(cr) => cr }
- idempotentCount = results.count {
- case Left(e: RuntimeException) if e.getMessage.startsWith("Idempotent") => true
- case _ => false
- }
- latestRev = successes.lastOption.map(_.newRev).getOrElse(Rev(0L))
- } yield IngestionResult(
- filesProcessed = files.size,
- patchesApplied = successes.size,
- filesSkipped = idempotentCount,
- entitiesCreated = 0, // Would need to count from parse results
- latestRev = latestRev
- )
- }
-
- private def discoverFiles(
- path: Path,
- language: Option[String],
- recursive: Boolean
- ): IO[List[Path]] = IO.blocking {
- val extensions = FileDiscovery.extensionsFor(language)
- if (Files.isRegularFile(path)) {
- if (language.isEmpty || FileDiscovery.matchesFilter(path, extensions)) List(path) else List.empty
- } else if (Files.isDirectory(path)) {
- FileDiscovery.tryGitLsFiles(path, recursive, extensions)
- .getOrElse(FileDiscovery.walkFiles(path, recursive, extensions))
- } else {
- List.empty
- }
- }
-
- private def resolveImportEdges(
- filePath: Path,
- parseResult: ParseResult,
- sourceHash: Option[String]
- ): IO[Unit] = {
- val importRels = parseResult.relationships.filter(_.predicate == "IMPORTS")
- importRels.traverse_ { rel =>
- val targetName = rel.dstName
- // Try to resolve relative imports (./config, ../utils)
- val resolvedPath = resolveRelativePath(filePath, targetName)
- resolvedPath match {
- case Some(resolved) =>
- queryApi.searchNodes(resolved.getFileName.toString, limit = 1).flatMap {
- case nodes if nodes.nonEmpty =>
- val srcId = NodeId(UUID.nameUUIDFromBytes(s"${filePath}:${filePath.getFileName}".getBytes("UTF-8")))
- val dstId = nodes.head.id
- val edgeId = EdgeId(UUID.nameUUIDFromBytes(s"${srcId.value}:${dstId.value}:IMPORTS".getBytes("UTF-8")))
- val patch = GraphPatch(
- patchId = PatchId(UUID.randomUUID()),
- actor = "ix/cross-file",
- timestamp = Instant.now(),
- source = PatchSource(filePath.toString, sourceHash, "cross-file-resolver/1.0", SourceType.Code),
- baseRev = Rev(0L),
- ops = Vector(PatchOp.UpsertEdge(edgeId, srcId, dstId, EdgePredicate("IMPORTS"), Map.empty)),
- replaces = Vector.empty,
- intent = Some(s"Cross-file import: ${filePath.getFileName} -> ${resolved.getFileName}")
- )
- writeApi.commitPatch(patch).void
- case _ => IO.unit
- }
- case None => IO.unit
- }
- }
- }
-
- private def resolveNamedImports(
- filePath: Path,
- parseResult: ParseResult,
- sourceHash: Option[String]
- ): IO[Unit] = {
- val namedImports = parseResult.relationships.filter { rel =>
- rel.predicate == "IMPORTS" && !rel.dstName.contains("/") && !rel.dstName.contains(".")
- }
- namedImports.traverse_ { rel =>
- queryApi.searchNodes(rel.dstName, limit = 5, nameOnly = true).flatMap { nodes =>
- val external = nodes.filter { n =>
- n.name == rel.dstName &&
- n.provenance.sourceUri != filePath.toString
- }
- external.headOption match {
- case Some(target) =>
- val srcId = NodeId(UUID.nameUUIDFromBytes(
- s"${filePath}:${filePath.getFileName}".getBytes("UTF-8")))
- val edgeId = EdgeId(UUID.nameUUIDFromBytes(
- s"${srcId.value}:${target.id.value}:IMPORTS".getBytes("UTF-8")))
- val patch = GraphPatch(
- patchId = PatchId(UUID.randomUUID()),
- actor = "ix/cross-file",
- timestamp = Instant.now(),
- source = PatchSource(filePath.toString, sourceHash,
- "cross-file-resolver/1.0", SourceType.Code),
- baseRev = Rev(0L),
- ops = Vector(PatchOp.UpsertEdge(
- edgeId, srcId, target.id, EdgePredicate("IMPORTS"), Map.empty)),
- replaces = Vector.empty,
- intent = Some(s"Named import: ${filePath.getFileName} -> ${rel.dstName}")
- )
- writeApi.commitPatch(patch).void
- case None => IO.unit
- }
- }
- }
- }
-
- private def resolveRelativePath(from: Path, importTarget: String): Option[Path] = {
- if (importTarget.startsWith("./") || importTarget.startsWith("../")) {
- val base = from.getParent
- if (base == null) return None
-
- // 1. Try the import target as-is and with source extensions appended
- val directExtensions = List("", ".ts", ".tsx", ".scala", ".py", ".js")
- val directMatch = directExtensions.collectFirst {
- case ext =>
- val candidate = base.resolve(importTarget + ext).normalize()
- if (Files.exists(candidate)) candidate else null
- }.flatMap(Option(_))
- if (directMatch.isDefined) return directMatch
-
- // 2. Strip compiled extensions (.js/.mjs/.cjs) and try source extensions
- val strippable = List(".js", ".mjs", ".cjs")
- val stripped = strippable.collectFirst {
- case ext if importTarget.endsWith(ext) =>
- importTarget.stripSuffix(ext)
- }
- val strippedMatch = stripped.flatMap { baseName =>
- val sourceExtensions = List(".ts", ".tsx")
- sourceExtensions.collectFirst {
- case ext =>
- val candidate = base.resolve(baseName + ext).normalize()
- if (Files.exists(candidate)) candidate else null
- }.flatMap(Option(_))
- }
- if (strippedMatch.isDefined) return strippedMatch
-
- // 3. Try directory index files (e.g., ./utils -> ./utils/index.ts)
- val indexExtensions = List("index.ts", "index.tsx", "index.js")
- indexExtensions.collectFirst {
- case idx =>
- val candidate = base.resolve(importTarget).resolve(idx).normalize()
- if (Files.exists(candidate)) candidate else null
- }.flatMap(Option(_))
- } else None
- }
-
- private def sha256Bytes(bytes: Array[Byte]): String = {
- val md = java.security.MessageDigest.getInstance("SHA-256")
- md.digest(bytes).map("%02x".format(_)).mkString
- }
-}
-
-case class IngestionProgress(
- filesDiscovered: Int,
- filesParsed: Int,
- chunksWritten: Int,
- totalChunks: Int
-)
-
-case class SkipReasons(
- unchanged: Int = 0,
- emptyFile: Int = 0,
- parseError: Int = 0,
- tooLarge: Int = 0
-) {
- def total: Int = unchanged + emptyFile + parseError + tooLarge
-}
-
-case class IngestionResult(
- filesProcessed: Int,
- patchesApplied: Int,
- filesSkipped: Int,
- entitiesCreated: Int,
- latestRev: Rev,
- skipReasons: SkipReasons = SkipReasons()
-)
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala
deleted file mode 100644
index 43ba1269..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/ParseResult.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-package ix.memory.ingestion
-
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-case class ParsedEntity(
- name: String,
- kind: NodeKind,
- attrs: Map[String, Json],
- lineStart: Int,
- lineEnd: Int,
- contentFingerprint: Option[String] = None
-)
-
-case class ParsedRelationship(
- srcName: String,
- dstName: String,
- predicate: String
-)
-
-case class ParseResult(
- entities: Vector[ParsedEntity],
- relationships: Vector[ParsedRelationship]
-)
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala
deleted file mode 100644
index cec94982..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/Parser.scala
+++ /dev/null
@@ -1,9 +0,0 @@
-package ix.memory.ingestion
-
-/**
- * Common trait for language-specific source code parsers.
- * Both TreeSitterPythonParser and TypeScriptParser implement this.
- */
-trait Parser {
- def parse(fileName: String, source: String): ParseResult
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
deleted file mode 100644
index 74639975..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-package ix.memory.ingestion
-
-import ix.memory.ingestion.parsers.{ConfigParser, MarkdownParser, ScalaParser, TreeSitterGoParser, TreeSitterPythonParser, TypeScriptParser}
-
-/**
- * Routes file paths to the appropriate language parser.
- * Supports Python (.py), TypeScript (.ts/.tsx), config files (.json/.yaml/.yml/.toml),
- * and Markdown (.md).
- */
-class ParserRouter {
- private val pythonParser = new TreeSitterPythonParser()
- private val tsParser = new TypeScriptParser()
- private val configParser = new ConfigParser()
- private val markdownParser = new MarkdownParser()
- private val scalaParser = new ScalaParser()
- private val goParser = new TreeSitterGoParser()
-
- def parserFor(filePath: String): Option[Parser] = {
- if (filePath.endsWith(".py")) Some(pythonParser)
- else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) Some(tsParser)
- else if (filePath.endsWith(".scala") || filePath.endsWith(".sc")) Some(scalaParser)
- else if (filePath.endsWith(".go")) Some(goParser)
- else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) Some(configParser)
- else if (filePath.endsWith(".md")) Some(markdownParser)
- else None
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala b/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala
deleted file mode 100644
index f2823309..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/PatchValidator.scala
+++ /dev/null
@@ -1,16 +0,0 @@
-package ix.memory.ingestion
-
-import ix.memory.model.GraphPatch
-
-/**
- * Simple validation for graph patches before committing.
- */
-object PatchValidator {
-
- def validate(patch: GraphPatch): Either[String, GraphPatch] = {
- if (patch.ops.isEmpty)
- Left("Patch has no operations")
- else
- Right(patch)
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
deleted file mode 100644
index bd396ed7..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
+++ /dev/null
@@ -1,190 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-/**
- * Parser for configuration files (JSON, YAML, TOML).
- *
- * Flattens nested structures into dot-path entries so that each leaf value
- * becomes a ConfigEntry node with a meaningful value (never "{" or "[").
- */
-class ConfigParser extends Parser {
-
- private val JsonExtensions = Set(".json")
- private val YamlExtensions = Set(".yaml", ".yml")
- private val TomlExtensions = Set(".toml")
-
- def canParse(fileName: String): Boolean = {
- val lower = fileName.toLowerCase
- (JsonExtensions ++ YamlExtensions ++ TomlExtensions).exists(ext => lower.endsWith(ext))
- }
-
- def parse(fileName: String, source: String): ParseResult = {
- val lower = fileName.toLowerCase
-
- val entries: Vector[(String, String, Int)] =
- if (JsonExtensions.exists(ext => lower.endsWith(ext))) parseJson(source)
- else if (YamlExtensions.exists(ext => lower.endsWith(ext))) parseYaml(source)
- else if (TomlExtensions.exists(ext => lower.endsWith(ext))) parseToml(source)
- else Vector.empty
-
- buildResult(fileName, source, entries)
- }
-
- // ---------------------------------------------------------------------------
- // Result builder
- // ---------------------------------------------------------------------------
-
- private def buildResult(
- fileName: String,
- source: String,
- entries: Vector[(String, String, Int)]
- ): ParseResult = {
- val lines = source.split("\n", -1)
-
- val configEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.Config,
- attrs = Map("format" -> Json.fromString(detectFormat(fileName))),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- val entryEntities = entries.map { case (key, value, line) =>
- ParsedEntity(
- name = key,
- kind = NodeKind.ConfigEntry,
- attrs = Map("value" -> Json.fromString(value)),
- lineStart = line,
- lineEnd = line
- )
- }
-
- val relationships = entries.map { case (key, _, _) =>
- ParsedRelationship(fileName, key, "CONTAINS")
- }
-
- ParseResult(
- entities = configEntity +: entryEntities,
- relationships = relationships
- )
- }
-
- private def detectFormat(fileName: String): String = {
- val lower = fileName.toLowerCase
- if (JsonExtensions.exists(ext => lower.endsWith(ext))) "json"
- else if (YamlExtensions.exists(ext => lower.endsWith(ext))) "yaml"
- else if (TomlExtensions.exists(ext => lower.endsWith(ext))) "toml"
- else "unknown"
- }
-
- // ---------------------------------------------------------------------------
- // JSON parsing — Circe-based with flattening
- // ---------------------------------------------------------------------------
-
- private def parseJson(source: String): Vector[(String, String, Int)] = {
- io.circe.parser.parse(source) match {
- case Right(json) => flattenJson("", json, 1)
- case Left(_) => regexParseJson(source)
- }
- }
-
- private def flattenJson(prefix: String, json: io.circe.Json, line: Int): Vector[(String, String, Int)] = {
- json.fold(
- jsonNull = Vector.empty,
- jsonBoolean = b => Vector((prefix, b.toString, line)),
- jsonNumber = n => Vector((prefix, n.toString, line)),
- jsonString = s => Vector((prefix, s, line)),
- jsonArray = arr => arr.zipWithIndex.flatMap { case (v, i) =>
- val key = if (prefix.isEmpty) s"[$i]" else s"$prefix[$i]"
- flattenJson(key, v, line)
- }.toVector,
- jsonObject = obj => obj.toVector.flatMap { case (k, v) =>
- val key = if (prefix.isEmpty) k else s"$prefix.$k"
- v.fold(
- jsonNull = Vector((key, "null", line)),
- jsonBoolean = b => Vector((key, b.toString, line)),
- jsonNumber = n => Vector((key, n.toString, line)),
- jsonString = s => Vector((key, s, line)),
- jsonArray = _ => flattenJson(key, v, line),
- jsonObject = _ => flattenJson(key, v, line)
- )
- }
- )
- }
-
- /** Regex fallback for malformed JSON */
- private def regexParseJson(source: String): Vector[(String, String, Int)] = {
- val KvPattern = """"([^"]+)"\s*:\s*"([^"]*)"""".r
- val lines = source.split("\n", -1)
- lines.zipWithIndex.flatMap { case (line, idx) =>
- KvPattern.findAllMatchIn(line).map { m =>
- (m.group(1), m.group(2), idx + 1)
- }
- }.toVector
- }
-
- // ---------------------------------------------------------------------------
- // YAML parsing — indentation-aware
- // ---------------------------------------------------------------------------
-
- private def parseYaml(source: String): Vector[(String, String, Int)] = {
- val lines = source.split("\n", -1)
- val result = Vector.newBuilder[(String, String, Int)]
- val stack = scala.collection.mutable.Stack.empty[(Int, String)] // (indent, prefix)
-
- val KvWithValue = """^([a-zA-Z_][\w.-]*)\s*:\s*(.+)$""".r
- val KvNoValue = """^([a-zA-Z_][\w.-]*)\s*:\s*$""".r
-
- lines.zipWithIndex.foreach { case (line, idx) =>
- val trimmed = line.trim
- if (!trimmed.startsWith("#") && trimmed.nonEmpty) {
- val indent = line.takeWhile(_ == ' ').length
- trimmed match {
- case KvWithValue(key, value) =>
- while (stack.nonEmpty && stack.top._1 >= indent) stack.pop()
- val fullKey = if (stack.nonEmpty) s"${stack.top._2}.$key" else key
- result += ((fullKey, value.trim, idx + 1))
- case KvNoValue(key) =>
- while (stack.nonEmpty && stack.top._1 >= indent) stack.pop()
- val fullKey = if (stack.nonEmpty) s"${stack.top._2}.$key" else key
- stack.push((indent, fullKey))
- case _ =>
- // skip list items, comments, etc.
- }
- }
- }
- result.result()
- }
-
- // ---------------------------------------------------------------------------
- // TOML parsing — section-aware
- // ---------------------------------------------------------------------------
-
- private def parseToml(source: String): Vector[(String, String, Int)] = {
- val lines = source.split("\n", -1)
- val result = Vector.newBuilder[(String, String, Int)]
- var currentSection = ""
-
- val SectionPattern = """^\[([^\]]+)\]""".r
- val KvPattern = """^([a-zA-Z_][\w.-]*)\s*=\s*(.+)$""".r
-
- lines.zipWithIndex.foreach { case (line, idx) =>
- val trimmed = line.trim
- if (!trimmed.startsWith("#") && trimmed.nonEmpty) {
- trimmed match {
- case SectionPattern(section) =>
- currentSection = section
- case KvPattern(key, value) =>
- val fullKey = if (currentSection.isEmpty) key else s"$currentSection.$key"
- val cleanValue = value.trim.stripPrefix("\"").stripSuffix("\"")
- result += ((fullKey, cleanValue, idx + 1))
- case _ =>
- }
- }
- }
- result.result()
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/GoParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/GoParser.scala
deleted file mode 100644
index 2abd19ae..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/GoParser.scala
+++ /dev/null
@@ -1,485 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-import scala.util.matching.Regex
-
-/**
- * Regex-based Go source parser (fallback for when tree-sitter is unavailable).
- * Extracts: functions, methods, structs, interfaces, type declarations,
- * imports, const/var declarations, and basic call edges.
- */
-class GoParser extends Parser {
-
- private val PackagePattern: Regex = """^\s*package\s+(\w+)""".r
- private val FuncPattern: Regex = """^\s*func\s+(\w+)\s*(?:\[.*?\])?\s*\(""".r
- private val MethodPattern: Regex = """^\s*func\s+\(\s*\w+\s+\*?(\w+)\s*\)\s+(\w+)\s*(?:\[.*?\])?\s*\(""".r
- private val StructPattern: Regex = """^\s*type\s+(\w+)\s+struct\s*\{""".r
- private val InterfacePattern: Regex = """^\s*type\s+(\w+)\s+interface\s*\{""".r
- private val TypeAliasPattern: Regex = """^\s*type\s+(\w+)\s*=\s*(.+)""".r
- private val NamedTypePattern: Regex = """^\s*type\s+(\w+)\s+(\w+)""".r
- private val ImportSinglePattern: Regex = """^\s*import\s+"([^"]+)"""".r
- private val ImportGroupStart: Regex = """^\s*import\s*\(""".r
- private val ImportSpecPattern: Regex = """^\s*(?:(\w+|\.)\s+)?"([^"]+)"""".r
- private val ConstPattern: Regex = """^\s*const\s+(\w+)\s""".r
- private val VarPattern: Regex = """^\s*var\s+(\w+)\s""".r
- private val ConstGroupStart: Regex = """^\s*const\s*\(""".r
- private val VarGroupStart: Regex = """^\s*var\s*\(""".r
- private val ConstSpecPattern: Regex = """^\s*(\w+)""".r
- private val CallPattern: Regex = """\b(\w+)\s*\(""".r
-
- private val GoBuiltins: Set[String] = Set(
- "make", "len", "cap", "append", "copy", "close", "delete",
- "complex", "real", "imag", "new", "panic", "recover",
- "print", "println", "error", "string", "int", "float64",
- "bool", "byte", "rune", "nil", "true", "false", "iota",
- "int8", "int16", "int32", "int64",
- "uint", "uint8", "uint16", "uint32", "uint64", "uintptr",
- "float32", "complex64", "complex128", "any", "comparable"
- )
-
- private val GoKeywords: Set[String] = Set(
- "break", "case", "chan", "const", "continue", "default", "defer",
- "else", "fallthrough", "for", "func", "go", "goto", "if",
- "import", "interface", "map", "package", "range", "return",
- "select", "struct", "switch", "type", "var"
- )
-
- def parse(fileName: String, source: String): ParseResult = {
- val lines = source.split("\n", -1)
-
- var fileAttrs: Map[String, Json] = Map("language" -> Json.fromString("go"))
- var entities = Vector.empty[ParsedEntity]
- var relationships = Vector.empty[ParsedRelationship]
-
- // Track type ranges for containment
- var typeRanges = Vector.empty[(String, Int, Int)]
-
- var i = 0
- while (i < lines.length) {
- val line = lines(i)
- val lineNum = i + 1
- val trimmed = line.trim
-
- // Skip comments
- if (trimmed.startsWith("//") || trimmed.startsWith("/*") || trimmed.startsWith("*")) {
- i += 1
- } else {
- var handled = false
-
- // Package
- if (!handled) {
- PackagePattern.findFirstMatchIn(line).foreach { m =>
- fileAttrs = fileAttrs + ("package" -> Json.fromString(m.group(1)))
- handled = true
- }
- }
-
- // Method (must check before function)
- if (!handled) {
- MethodPattern.findFirstMatchIn(line).foreach { m =>
- val receiverType = m.group(1)
- val methodName = m.group(2)
- val endLine = findBraceBlockEnd(lines, i).getOrElse(lineNum)
- val sig = trimmed.replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = lines.slice(i, endLine).mkString("\n")
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = methodName.nonEmpty && methodName.head.isUpper
-
- entities = entities :+ ParsedEntity(
- name = methodName,
- kind = NodeKind.Method,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "summary" -> Json.fromString(sig),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported),
- "receiver" -> Json.fromString(receiverType)
- ),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(receiverType, methodName, "CONTAINS")
-
- // Extract calls
- relationships = relationships ++ extractCalls(lines, i, endLine, methodName)
-
- i = endLine
- handled = true
- }
- }
-
- // Function
- if (!handled) {
- FuncPattern.findFirstMatchIn(line).foreach { m =>
- val funcName = m.group(1)
- val endLine = findBraceBlockEnd(lines, i).getOrElse(lineNum)
- val sig = trimmed.replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = lines.slice(i, endLine).mkString("\n")
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = funcName.nonEmpty && funcName.head.isUpper
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "summary" -> Json.fromString(sig),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
-
- // Extract calls
- relationships = relationships ++ extractCalls(lines, i, endLine, funcName)
-
- i = endLine
- handled = true
- }
- }
-
- // Struct
- if (!handled) {
- StructPattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, i).getOrElse(lineNum)
- val sig = trimmed.replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = lines.slice(i, endLine).mkString("\n")
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = name.nonEmpty && name.head.isUpper
-
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("struct"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
-
- i = endLine
- handled = true
- }
- }
-
- // Interface
- if (!handled) {
- InterfacePattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, i).getOrElse(lineNum)
- val sig = trimmed.replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = lines.slice(i, endLine).mkString("\n")
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = name.nonEmpty && name.head.isUpper
-
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Interface,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("interface"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
-
- i = endLine
- handled = true
- }
- }
-
- // Type alias
- if (!handled) {
- TypeAliasPattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- val exported = name.nonEmpty && name.head.isUpper
- val sig = trimmed.take(120)
- val fp = Fingerprint.compute(sig, sig)
-
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("type_alias"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = lineNum,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- handled = true
- }
- }
-
- // Named type (must be after struct/interface/alias checks)
- if (!handled) {
- NamedTypePattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- val baseType = m.group(2)
- if (baseType != "struct" && baseType != "interface") {
- val exported = name.nonEmpty && name.head.isUpper
- val sig = trimmed.take(120)
- val fp = Fingerprint.compute(sig, sig)
-
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("named_type"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = lineNum,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- handled = true
- }
- }
- }
-
- // Import group
- if (!handled) {
- ImportGroupStart.findFirstMatchIn(line).foreach { _ =>
- i += 1
- while (i < lines.length && !lines(i).trim.startsWith(")")) {
- ImportSpecPattern.findFirstMatchIn(lines(i)).foreach { m =>
- val importPath = m.group(2)
- if (!entities.exists(e => e.name == importPath && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = importPath,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = i + 1,
- lineEnd = i + 1
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, importPath, "IMPORTS")
- }
- i += 1
- }
- handled = true
- }
- }
-
- // Single import
- if (!handled) {
- ImportSinglePattern.findFirstMatchIn(line).foreach { m =>
- val importPath = m.group(1)
- if (!entities.exists(e => e.name == importPath && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = importPath,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, importPath, "IMPORTS")
- handled = true
- }
- }
-
- // Const (single)
- if (!handled) {
- ConstGroupStart.findFirstMatchIn(line).foreach { _ =>
- i += 1
- while (i < lines.length && !lines(i).trim.startsWith(")")) {
- ConstSpecPattern.findFirstMatchIn(lines(i).trim).foreach { m =>
- val name = m.group(1)
- if (name != "//" && name != "/*" && name != "_") {
- val exported = name.nonEmpty && name.head.isUpper
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Variable,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("const"),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = i + 1,
- lineEnd = i + 1
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- }
- }
- i += 1
- }
- handled = true
- }
- }
-
- if (!handled) {
- ConstPattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- if (name != "(" && name != "_") {
- val exported = name.nonEmpty && name.head.isUpper
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Variable,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("const"),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = lineNum
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- handled = true
- }
- }
- }
-
- // Var group
- if (!handled) {
- VarGroupStart.findFirstMatchIn(line).foreach { _ =>
- i += 1
- while (i < lines.length && !lines(i).trim.startsWith(")")) {
- val varLine = lines(i).trim
- if (varLine.nonEmpty && !varLine.startsWith("//") && !varLine.startsWith("/*")) {
- varLine.split("\\s+").headOption.foreach { name =>
- if (name != "_" && name.matches("\\w+")) {
- val exported = name.nonEmpty && name.head.isUpper
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Variable,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("var"),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = i + 1,
- lineEnd = i + 1
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- }
- }
- }
- i += 1
- }
- handled = true
- }
- }
-
- if (!handled) {
- VarPattern.findFirstMatchIn(line).foreach { m =>
- val name = m.group(1)
- if (name != "(" && name != "_") {
- val exported = name.nonEmpty && name.head.isUpper
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Variable,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("var"),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = lineNum
- )
- relationships = relationships :+ ParsedRelationship(fileName, name, "CONTAINS")
- handled = true
- }
- }
- }
-
- if (!handled) i += 1
- else if (i == lineNum - 1) i += 1 // advance if we didn't already
- }
- }
-
- val fileEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = fileAttrs,
- lineStart = 1,
- lineEnd = lines.length
- )
-
- ParseResult(fileEntity +: entities, relationships)
- }
-
- private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Option[Int] = {
- var braceCount = 0
- var foundOpen = false
- var i = startIdx
-
- while (i < lines.length) {
- val line = lines(i)
- for (ch <- line) {
- ch match {
- case '{' =>
- braceCount += 1
- foundOpen = true
- case '}' =>
- braceCount -= 1
- if (foundOpen && braceCount == 0) {
- return Some(i + 1)
- }
- case _ =>
- }
- }
- i += 1
- }
-
- if (foundOpen) Some(lines.length) else None
- }
-
- private def extractCalls(lines: Array[String], startIdx: Int, endIdx: Int, callerName: String): Vector[ParsedRelationship] = {
- var calls = Vector.empty[ParsedRelationship]
- val seen = scala.collection.mutable.Set.empty[String]
- for (i <- (startIdx + 1) until endIdx.min(lines.length)) {
- val line = lines(i).trim
- if (!line.startsWith("//") && !line.startsWith("/*")) {
- CallPattern.findAllMatchIn(line).foreach { m =>
- val callee = m.group(1)
- if (callee != callerName && !GoBuiltins.contains(callee) &&
- !GoKeywords.contains(callee) && !seen.contains(callee)) {
- seen += callee
- calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
- }
- }
- }
- }
- calls
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
deleted file mode 100644
index 7d99504c..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-import scala.util.matching.Regex
-
-/**
- * Markdown parser that extracts structural entities from `.md` files.
- *
- * Each heading becomes a Doc entity with attrs for `level`, `title`, and `content`.
- * Content captures all non-header lines in the section, up to 2000 characters.
- * A top-level File entity is also emitted, with CONTAINS edges to each section.
- */
-class MarkdownParser extends Parser {
-
- private val HeadingPattern: Regex = """^(#{1,6})\s+(.+)$""".r
-
- def parse(fileName: String, source: String): ParseResult = {
- val lines = source.split("\n", -1)
-
- // -- File entity --
- val fileEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("markdown")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- // Collect headings with their line indices
- val headings: Vector[(Int, Int, String)] = lines.zipWithIndex.flatMap { case (line, idx) =>
- HeadingPattern.findFirstMatchIn(line.trim).map { m =>
- val level = m.group(1).length
- val title = m.group(2).trim
- (idx, level, title)
- }
- }.toVector
-
- var entities = Vector(fileEntity)
- var relationships = Vector.empty[ParsedRelationship]
-
- for (i <- headings.indices) {
- val (startIdx, level, title) = headings(i)
- val endIdx = if (i + 1 < headings.length) headings(i + 1)._1 else lines.length
-
- val content = extractContent(lines, startIdx, endIdx)
-
- val entity = ParsedEntity(
- name = title,
- kind = NodeKind.Doc,
- attrs = Map(
- "level" -> Json.fromInt(level),
- "title" -> Json.fromString(title),
- "content" -> Json.fromString(content)
- ),
- lineStart = startIdx + 1,
- lineEnd = endIdx
- )
-
- entities = entities :+ entity
- relationships = relationships :+ ParsedRelationship(fileName, title, "CONTAINS")
- }
-
- ParseResult(entities, relationships)
- }
-
- /**
- * Extract content from all non-header lines in the section, up to 2000 characters.
- */
- private def extractContent(lines: Array[String], startIdx: Int, endIdx: Int): String = {
- val contentLines = lines.slice(startIdx + 1, endIdx).filterNot { line =>
- HeadingPattern.findFirstMatchIn(line.trim).isDefined
- }
- contentLines.mkString("\n").trim.take(2000)
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala
deleted file mode 100644
index deac202c..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ScalaParser.scala
+++ /dev/null
@@ -1,357 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-import scala.util.matching.Regex
-
-/**
- * Regex-based Scala source parser that extracts structural entities and relationships.
- *
- * Extracts: traits, classes, case classes, objects, methods (def), and imports.
- * Uses brace counting for block boundary detection.
- */
-class ScalaParser extends Parser {
-
- // Patterns for Scala constructs
- private val TraitPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*trait\s+(\w+)""".r
- private val ClassPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*class\s+(\w+)""".r
- private val CaseClassPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*case\s+class\s+(\w+)""".r
- private val ObjectPattern: Regex = """^\s*(?:sealed\s+|abstract\s+|private\s+|protected\s+)*(?:case\s+)?object\s+(\w+)""".r
- private val DefPattern: Regex = """^\s*(?:override\s+)?(?:private(?:\[\w+\])?\s+|protected(?:\[\w+\])?\s+)?(?:final\s+)?def\s+(\w+)""".r
- private val ImportPattern: Regex = """^\s*import\s+(.+)$""".r
- private val CallPattern: Regex = """(?:(?:this|super)\.)?\b(\w+)\s*(?:\[.*?\])?\s*\(""".r
- private val ExtendsPattern: Regex = """extends\s+(\w+)""".r
- private val TypeRefPattern: Regex = """\b([A-Z]\w+)""".r
- private val QualifiedRefPattern: Regex = """\b([A-Z]\w+)\.(\w+)""".r
-
- private val ScalaBuiltinTypes: Set[String] = Set(
- "String", "Int", "Long", "Double", "Float", "Boolean", "Unit", "Any", "AnyRef", "AnyVal",
- "Nothing", "Null", "Option", "List", "Map", "Set", "Vector", "Seq", "Array", "Future",
- "IO", "Either", "Try", "Some", "None", "Right", "Left", "Success", "Failure"
- )
-
- private val ScalaBuiltins: Set[String] = Set(
- "println", "print", "require", "assert", "assume",
- "Some", "None", "Option", "List", "Map", "Set", "Vector",
- "Right", "Left", "Try", "Success", "Failure",
- "Future", "IO", "pure", "flatMap", "map", "filter",
- "foreach", "fold", "foldLeft", "foldRight",
- "toString", "hashCode", "equals", "getClass",
- "apply", "unapply", "copy", "andThen", "compose",
- "getOrElse", "orElse", "contains", "exists", "forall",
- "collect", "traverse", "sequence", "attempt", "handleErrorWith",
- "raiseError", "blocking", "delay", "suspend", "async",
- "mkString", "toList", "toVector", "toSet", "toMap",
- "head", "tail", "size", "length", "isEmpty", "nonEmpty",
- "take", "drop", "slice", "zip", "zipWithIndex",
- "sortBy", "groupBy", "distinct", "flatten"
- )
-
- def parse(fileName: String, source: String): ParseResult = {
- val lines = source.split("\n", -1)
-
- val fileEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("scala")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- var entities = Vector(fileEntity)
- var relationships = Vector.empty[ParsedRelationship]
-
- // Track type ranges for containment: (name, startLine, endLine)
- var typeRanges = Vector.empty[(String, Int, Int)]
-
- for ((line, idx) <- lines.zipWithIndex) {
- val lineNum = idx + 1
- val trimmed = line.trim
-
- // Skip comment lines
- if (trimmed.startsWith("//") || trimmed.startsWith("*") || trimmed.startsWith("/*")) {
- // no-op
- } else {
-
- // Case class must be checked before class (since "case class" contains "class")
- val handledAsType = CaseClassPattern.findFirstMatchIn(line) match {
- case Some(m) =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
- val bodyText = lines.slice(idx, endLine).mkString("\n")
- val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Class,
- attrs = Map("scala_kind" -> Json.fromString("case_class"), "language" -> Json.fromString("scala"),
- "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
- val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
- ExtendsPattern.findFirstMatchIn(line).foreach { em =>
- relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
- }
- true
-
- case None =>
- // Check for trait
- TraitPattern.findFirstMatchIn(line) match {
- case Some(m) =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
- val bodyText = lines.slice(idx, endLine).mkString("\n")
- val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Trait,
- attrs = Map("scala_kind" -> Json.fromString("trait"), "language" -> Json.fromString("scala"),
- "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
- val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
- ExtendsPattern.findFirstMatchIn(line).foreach { em =>
- relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
- }
- true
-
- case None =>
- // Check for object (before class, since object keyword won't match class pattern)
- ObjectPattern.findFirstMatchIn(line) match {
- case Some(m) =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
- val bodyText = lines.slice(idx, endLine).mkString("\n")
- val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Object,
- attrs = Map("scala_kind" -> Json.fromString("object"), "language" -> Json.fromString("scala"),
- "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
- val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
- ExtendsPattern.findFirstMatchIn(line).foreach { em =>
- relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
- }
- true
-
- case None =>
- // Check for class
- ClassPattern.findFirstMatchIn(line) match {
- case Some(m) =>
- val name = m.group(1)
- val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
- val bodyText = lines.slice(idx, endLine).mkString("\n")
- val fp = Fingerprint.compute(extractTypeSignature(trimmed), bodyText)
- entities = entities :+ ParsedEntity(
- name = name,
- kind = NodeKind.Class,
- attrs = Map("scala_kind" -> Json.fromString("class"), "language" -> Json.fromString("scala"),
- "signature" -> Json.fromString(extractTypeSignature(trimmed)), "visibility" -> Json.fromString(extractVisibility(trimmed))),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
- val enclosingType = typeRanges.find { case (_, start, end) => lineNum > start && lineNum <= end }
- val containsSrc = enclosingType.map(_._1).getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
- typeRanges = typeRanges :+ (name, lineNum, endLine)
- ExtendsPattern.findFirstMatchIn(line).foreach { em =>
- relationships = relationships :+ ParsedRelationship(name, em.group(1), "EXTENDS")
- }
- true
- case None => false
- }
- }
- }
- }
-
- // Check for def (only if we didn't match a type on this line)
- if (!handledAsType) {
- DefPattern.findFirstMatchIn(line).foreach { m =>
- val funcName = m.group(1)
- val endLine = findBraceBlockEnd(lines, idx).getOrElse(lineNum)
-
- val signature = extractSignature(trimmed)
- val visibility = extractVisibility(trimmed)
-
- // Determine enclosing type
- val enclosing = typeRanges.find { case (_, start, end) =>
- lineNum > start && lineNum <= end
- }
-
- val (kind, edgePredicate, edgeSrc) = enclosing match {
- case Some((typeName, _, _)) => (NodeKind.Method, "CONTAINS", typeName)
- case None => (NodeKind.Function, "CONTAINS", fileName)
- }
-
- val bodyText = lines.slice(idx, endLine).mkString("\n")
- val fp = Fingerprint.compute(signature, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = kind,
- attrs = Map(
- "language" -> Json.fromString("scala"),
- "summary" -> Json.fromString(signature),
- "signature" -> Json.fromString(signature),
- "visibility" -> Json.fromString(visibility)
- ),
- lineStart = lineNum,
- lineEnd = endLine,
- contentFingerprint = Some(fp)
- )
-
- relationships = relationships :+ ParsedRelationship(edgeSrc, funcName, edgePredicate)
-
- // Extract CALLS and REFERENCES from method/function body
- val callsInFunc = extractCalls(lines, idx, endLine, funcName)
- relationships = relationships ++ callsInFunc
-
- // Extract REFERENCES from method signature type annotations
- val enclosingTypeName = enclosing.map(_._1).getOrElse("")
- val sigRefs = TypeRefPattern.findAllMatchIn(signature).map(_.group(1)).toSet
- for (ref <- sigRefs) {
- if (!ScalaBuiltinTypes.contains(ref) && ref != funcName && ref != enclosingTypeName && ref != fileName) {
- relationships = relationships :+ ParsedRelationship(funcName, ref, "REFERENCES")
- }
- }
- }
- }
-
- // Check for imports
- ImportPattern.findFirstMatchIn(line).foreach { m =>
- val importPath = m.group(1).trim
- // Parse import path: "ix.memory.model.{Claim, GraphNode}" -> module "ix.memory.model"
- // "cats.effect.IO" -> module "cats.effect.IO"
- val moduleName = if (importPath.contains("{")) {
- // Grouped import: take the prefix before the braces
- importPath.substring(0, importPath.indexOf('{')).stripSuffix(".").trim
- } else {
- importPath
- }
-
- // Avoid duplicate module entities
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
-
- } // end else (non-comment)
- }
-
- ParseResult(entities, relationships)
- }
-
- /**
- * Find the end of a brace-delimited block starting at `startIdx`.
- * Counts opening and closing braces. Returns None if no opening brace is found
- * (e.g. abstract method in a trait with no body).
- */
- private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Option[Int] = {
- var braceCount = 0
- var foundOpen = false
- var i = startIdx
-
- while (i < lines.length) {
- val line = lines(i)
- for (ch <- line) {
- ch match {
- case '{' =>
- braceCount += 1
- foundOpen = true
- case '}' =>
- braceCount -= 1
- if (foundOpen && braceCount == 0) {
- return Some(i + 1) // 1-indexed
- }
- case _ => // skip
- }
- }
- // If we found an opening brace and closed it, we'd have returned above
- // If line ends and no brace found yet, keep going
- i += 1
- }
-
- if (foundOpen) Some(lines.length) else None
- }
-
- /** Extract CALLS and qualified REFERENCES relationships from a method/function body. */
- private def extractCalls(lines: Array[String], startIdx: Int, endIdx: Int, callerName: String): Vector[ParsedRelationship] = {
- var calls = Vector.empty[ParsedRelationship]
- val seen = scala.collection.mutable.Set.empty[String]
- val refSeen = scala.collection.mutable.Set.empty[String]
- for (i <- (startIdx + 1) until endIdx.min(lines.length)) {
- val line = lines(i).trim
- if (!line.startsWith("//") && !line.startsWith("*")) {
- CallPattern.findAllMatchIn(line).foreach { m =>
- val callee = m.group(1)
- // Only include lowercase-starting identifiers (methods/functions, not types)
- if (callee != callerName && !ScalaBuiltins.contains(callee) && !seen.contains(callee) && callee.head.isLower) {
- seen += callee
- calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
- }
- }
- // Extract qualified REFERENCES (e.g., NodeKind.File, SourceType.Code)
- QualifiedRefPattern.findAllMatchIn(line).foreach { m =>
- val qualifier = m.group(1)
- if (!ScalaBuiltinTypes.contains(qualifier) && !ScalaBuiltins.contains(qualifier) &&
- !refSeen.contains(qualifier) && qualifier != callerName) {
- refSeen += qualifier
- calls = calls :+ ParsedRelationship(callerName, qualifier, "REFERENCES")
- }
- }
- }
- }
- calls
- }
-
- /** Extract a type signature (trait/class/object), stripping body braces. */
- private def extractTypeSignature(trimmedLine: String): String = {
- val cleaned = trimmedLine
- .replaceAll("""\s*\{\s*$""", "")
- .trim
- cleaned.take(120)
- }
-
- /** Extract a clean signature from a trimmed line, stripping trailing body markers. */
- private def extractSignature(trimmedLine: String): String = {
- val cleaned = trimmedLine
- .replaceAll("""\s*=\s*\{?\s*$""", "")
- .replaceAll("""\s*\{\s*$""", "")
- .trim
- cleaned.take(120)
- }
-
- /** Detect visibility from a trimmed line (after stripping `override`). */
- private def extractVisibility(trimmedLine: String): String = {
- val withoutOverride = trimmedLine.replaceFirst("""^\s*override\s+""", "").trim
- if (withoutOverride.startsWith("private")) "private"
- else if (withoutOverride.startsWith("protected")) "protected"
- else "public"
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterGoParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterGoParser.scala
deleted file mode 100644
index 39b94b57..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterGoParser.scala
+++ /dev/null
@@ -1,523 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-import org.treesitter.{TSNode, TreeSitterGo}
-
-/**
- * Tree-sitter AST-based Go parser with regex fallback.
- * Extracts packages, functions, methods, structs, interfaces, type aliases,
- * const/var declarations, imports, call edges, and struct embedding (EXTENDS).
- */
-class TreeSitterGoParser extends Parser {
-
- private val regexFallback = new GoParser()
-
- def parse(fileName: String, source: String): ParseResult = {
- try treeSitterParse(fileName, source)
- catch { case _: Throwable => regexFallback.parse(fileName, source) }
- }
-
- private def treeSitterParse(fileName: String, source: String): ParseResult = {
- TreeSitterUtils.withParse(new TreeSitterGo(), source) { root =>
- val ctx = new GoExtractionContext(fileName, source, root)
- ctx.result()
- }
- }
-
- // -------------------------------------------------------------------------
- // Extraction context
- // -------------------------------------------------------------------------
-
- private class GoExtractionContext(
- fileName: String,
- source: String,
- root: TSNode
- ) {
- import TreeSitterUtils._
-
- var entities = Vector.empty[ParsedEntity]
- var relationships = Vector.empty[ParsedRelationship]
-
- private val lines = source.split("\n", -1)
-
- // Initialize with file entity
- entities = entities :+ ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("go")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- // Walk the program
- walkProgram(root)
-
- def result(): ParseResult = ParseResult(entities, relationships)
-
- private def walkProgram(node: TSNode): Unit = {
- for (child <- namedChildren(node)) {
- child.getType() match {
- case "package_clause" =>
- handlePackageClause(child)
-
- case "import_declaration" =>
- handleImportDeclaration(child)
-
- case "function_declaration" =>
- handleFunctionDeclaration(child)
-
- case "method_declaration" =>
- handleMethodDeclaration(child)
-
- case "type_declaration" =>
- handleTypeDeclaration(child)
-
- case "const_declaration" =>
- handleConstDeclaration(child)
-
- case "var_declaration" =>
- handleVarDeclaration(child)
-
- case _ =>
- // Ignore other top-level nodes (comments, etc.)
- }
- }
- }
-
- // --- Package handling ---
-
- private def handlePackageClause(node: TSNode): Unit = {
- fieldChild(node, "name").orElse(findChild(node, "package_identifier")).foreach { nameNode =>
- val pkgName = nodeText(nameNode, source)
- // Set package as attribute on the File entity
- entities = entities.map { e =>
- if (e.name == fileName && e.kind == NodeKind.File) {
- e.copy(attrs = e.attrs + ("package" -> Json.fromString(pkgName)))
- } else e
- }
- }
- }
-
- // --- Import handling ---
-
- private def handleImportDeclaration(node: TSNode): Unit = {
- // Single import: import "fmt"
- findChild(node, "import_spec").foreach(handleImportSpec)
-
- // Grouped import: import ( "fmt"; "os" )
- findChild(node, "import_spec_list").foreach { specList =>
- for (spec <- childrenOfType(specList, "import_spec")) {
- handleImportSpec(spec)
- }
- }
- }
-
- private def handleImportSpec(spec: TSNode): Unit = {
- // import_spec has optional name (alias) and path (interpreted_string_literal)
- val pathOpt = fieldChild(spec, "path")
- .orElse(findChild(spec, "interpreted_string_literal"))
- .map(n => nodeText(n, source).stripPrefix("\"").stripSuffix("\""))
-
- pathOpt.foreach { importPath =>
- val aliasOpt = fieldChild(spec, "name")
- .map(n => nodeText(n, source))
-
- val entityName = aliasOpt match {
- case Some(".") => importPath // dot import
- case Some("_") => importPath // side-effect import
- case Some(alias) => alias // named import
- case None => importPath // bare import
- }
-
- if (!entities.exists(e => e.name == importPath && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = importPath,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineStart(spec),
- lineEnd = lineEnd(spec)
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, importPath, "IMPORTS")
- }
- }
-
- // --- Function handling ---
-
- private def handleFunctionDeclaration(node: TSNode): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { funcName =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val sig = extractFirstLine(node).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = funcName.nonEmpty && funcName.head.isUpper
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "summary" -> Json.fromString(sig),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
-
- // Extract calls from function body
- fieldChild(node, "body").foreach { body =>
- extractCallsFromBody(body, funcName)
- }
- }
- }
-
- // --- Method handling ---
-
- private def handleMethodDeclaration(node: TSNode): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { methodName =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val sig = extractFirstLine(node).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
- val exported = methodName.nonEmpty && methodName.head.isUpper
-
- // Extract receiver type name
- val receiverType = fieldChild(node, "receiver")
- .orElse(findChild(node, "parameter_list"))
- .flatMap(extractReceiverType)
-
- val attrs = Map(
- "language" -> Json.fromString("go"),
- "summary" -> Json.fromString(sig),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ) ++ receiverType.map(r => "receiver" -> Json.fromString(r))
-
- entities = entities :+ ParsedEntity(
- name = methodName,
- kind = NodeKind.Method,
- attrs = attrs,
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
-
- // Method belongs to receiver type, not file
- val containsSrc = receiverType.getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, methodName, "CONTAINS")
-
- // Extract calls from method body
- fieldChild(node, "body").foreach { body =>
- extractCallsFromBody(body, methodName)
- }
- }
- }
-
- private def extractReceiverType(paramList: TSNode): Option[String] = {
- // The receiver parameter list contains a single parameter_declaration
- // e.g., (r *Receiver) or (r Receiver)
- val params = namedChildren(paramList)
- params.headOption.flatMap { param =>
- // Look for type_identifier or pointer_type → type_identifier
- val typeNode = fieldChild(param, "type").getOrElse(param)
- extractTypeName(typeNode)
- }
- }
-
- private def extractTypeName(node: TSNode): Option[String] = {
- node.getType() match {
- case "type_identifier" =>
- Some(nodeText(node, source))
- case "pointer_type" =>
- // *Type — recurse to get the type name
- namedChildren(node).headOption.flatMap(extractTypeName)
- case "parameter_declaration" =>
- // Get the type field
- fieldChild(node, "type").flatMap(extractTypeName)
- case "generic_type" =>
- // Type[T] — extract the base type
- fieldChild(node, "type").flatMap(extractTypeName)
- case _ =>
- // Fallback: try to find a type_identifier descendant
- findChild(node, "type_identifier").map(n => nodeText(n, source))
- }
- }
-
- // --- Type declaration handling ---
-
- private def handleTypeDeclaration(node: TSNode): Unit = {
- // Single type: type Foo struct { ... }
- findChild(node, "type_spec").foreach(handleTypeSpec)
-
- // Grouped type: type ( Foo struct {}; Bar interface {} )
- for (child <- namedChildren(node)) {
- if (child.getType() == "type_spec") {
- handleTypeSpec(child)
- }
- }
- }
-
- private def handleTypeSpec(spec: TSNode): Unit = {
- val nameOpt = extractName(spec, source)
- nameOpt.foreach { typeName =>
- val start = lineStart(spec)
- val end = lineEnd(spec)
- val exported = typeName.nonEmpty && typeName.head.isUpper
- val bodyText = nodeText(spec, source)
-
- // Determine the type kind from the type expression
- val typeNode = fieldChild(spec, "type")
- val typeKind = typeNode.map(_.getType()).getOrElse("")
-
- typeKind match {
- case "struct_type" =>
- val sig = extractFirstLine(spec).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val fp = Fingerprint.compute(sig, bodyText)
- entities = entities :+ ParsedEntity(
- name = typeName,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("struct"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
-
- // Extract struct embedding (EXTENDS)
- typeNode.foreach(extractStructEmbedding(_, typeName))
-
- case "interface_type" =>
- val sig = extractFirstLine(spec).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val fp = Fingerprint.compute(sig, bodyText)
- entities = entities :+ ParsedEntity(
- name = typeName,
- kind = NodeKind.Interface,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString("interface"),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
-
- // Extract interface embedding
- typeNode.foreach(extractInterfaceEmbedding(_, typeName))
-
- case _ =>
- // Type alias (type Foo = Bar) or named type (type Foo int)
- val isAlias = allChildren(spec).exists(c => nodeText(c, source) == "=")
- val goKind = if (isAlias) "type_alias" else "named_type"
- val sig = nodeText(spec, source).trim.take(120)
- val fp = Fingerprint.compute(sig, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = typeName,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString(goKind),
- "signature" -> Json.fromString(sig),
- "summary" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
- }
- }
- }
-
- private def extractStructEmbedding(structType: TSNode, typeName: String): Unit = {
- // Walk field_declaration_list for embedded fields (no field name)
- findChild(structType, "field_declaration_list").foreach { fieldList =>
- for (field <- childrenOfType(fieldList, "field_declaration")) {
- // An embedded field has a type but no name field
- val hasName = fieldChild(field, "name").isDefined
- if (!hasName) {
- // The type is the embedded type
- fieldChild(field, "type").foreach { typeNode =>
- extractTypeName(typeNode).foreach { embeddedName =>
- if (embeddedName != typeName && !GoBuiltins.contains(embeddedName)) {
- relationships = relationships :+ ParsedRelationship(typeName, embeddedName, "EXTENDS")
- }
- }
- }
- }
- }
- }
- }
-
- private def extractInterfaceEmbedding(ifaceType: TSNode, typeName: String): Unit = {
- // Walk interface body for bare type identifiers (embedded interfaces)
- for (child <- namedChildren(ifaceType)) {
- child.getType() match {
- case "type_identifier" =>
- val embeddedName = nodeText(child, source)
- if (embeddedName != typeName && !GoBuiltins.contains(embeddedName)) {
- relationships = relationships :+ ParsedRelationship(typeName, embeddedName, "EXTENDS")
- }
- case "qualified_type" =>
- // pkg.Type embedded
- fieldChild(child, "name")
- .orElse(namedChildren(child).lastOption)
- .foreach { nameNode =>
- val embeddedName = nodeText(nameNode, source)
- if (embeddedName != typeName) {
- relationships = relationships :+ ParsedRelationship(typeName, embeddedName, "EXTENDS")
- }
- }
- case _ => // method specs, etc. — skip
- }
- }
- }
-
- // --- Const/Var handling ---
-
- private def handleConstDeclaration(node: TSNode): Unit = {
- // Single const or grouped const ( ... )
- val specs = childrenOfType(node, "const_spec")
- if (specs.isEmpty) {
- // May have a single const_spec as direct structure
- handleConstOrVarSpecs(namedChildren(node).filter(_.getType() == "const_spec"), "const")
- } else {
- handleConstOrVarSpecs(specs, "const")
- }
- }
-
- private def handleVarDeclaration(node: TSNode): Unit = {
- val specs = childrenOfType(node, "var_spec")
- if (specs.isEmpty) {
- handleConstOrVarSpecs(namedChildren(node).filter(_.getType() == "var_spec"), "var")
- } else {
- handleConstOrVarSpecs(specs, "var")
- }
- }
-
- private def handleConstOrVarSpecs(specs: Vector[TSNode], declKind: String): Unit = {
- for (spec <- specs) {
- // Each spec may have multiple names: const a, b = 1, 2
- val names = fieldChild(spec, "name")
- .map(Vector(_))
- .getOrElse(childrenOfType(spec, "identifier"))
-
- for (nameNode <- names) {
- val varName = nodeText(nameNode, source)
- if (varName.nonEmpty && varName != "_") {
- val start = lineStart(spec)
- val end = lineEnd(spec)
- val exported = varName.head.isUpper
- val sig = nodeText(spec, source).trim.take(120)
- val fp = Fingerprint.compute(sig, sig)
-
- entities = entities :+ ParsedEntity(
- name = varName,
- kind = NodeKind.Variable,
- attrs = Map(
- "language" -> Json.fromString("go"),
- "go_kind" -> Json.fromString(declKind),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(if (exported) "exported" else "unexported"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, varName, "CONTAINS")
- }
- }
- }
- }
-
- // --- CALLS extraction ---
-
- private def extractCallsFromBody(bodyNode: TSNode, callerName: String): Unit = {
- val seen = scala.collection.mutable.Set.empty[String]
- val callExprs = collectDescendants(bodyNode, _.getType() == "call_expression")
-
- for (callExpr <- callExprs) {
- fieldChild(callExpr, "function").foreach { funcNode =>
- val callee = resolveCalleeName(funcNode)
- callee.foreach { name =>
- if (name != callerName && !GoBuiltins.contains(name) &&
- !GoKeywords.contains(name) && !seen.contains(name) && name != fileName) {
- seen += name
- relationships = relationships :+ ParsedRelationship(callerName, name, "CALLS")
- }
- }
- }
- }
- }
-
- private def resolveCalleeName(funcNode: TSNode): Option[String] = {
- funcNode.getType() match {
- case "identifier" =>
- Some(nodeText(funcNode, source))
- case "selector_expression" =>
- // e.g., pkg.Func() or obj.Method() — extract the field (rightmost)
- fieldChild(funcNode, "field").map(n => nodeText(n, source))
- case "call_expression" =>
- // Chained calls — recurse
- fieldChild(funcNode, "function").flatMap(resolveCalleeName)
- case "parenthesized_expression" =>
- // (func)() — recurse into the inner expression
- namedChildren(funcNode).headOption.flatMap(resolveCalleeName)
- case _ =>
- None
- }
- }
-
- // --- Helpers ---
-
- private def extractFirstLine(node: TSNode): String = {
- val text = nodeText(node, source)
- text.split("\n").head
- }
- }
-
- // --- Shared constants ---
-
- private val GoKeywords: Set[String] = Set(
- "break", "case", "chan", "const", "continue", "default", "defer",
- "else", "fallthrough", "for", "func", "go", "goto", "if",
- "import", "interface", "map", "package", "range", "return",
- "select", "struct", "switch", "type", "var"
- )
-
- private val GoBuiltins: Set[String] = Set(
- "make", "len", "cap", "append", "copy", "close", "delete",
- "complex", "real", "imag", "new", "panic", "recover",
- "print", "println", "error", "string", "int", "float64",
- "bool", "byte", "rune", "nil", "true", "false", "iota",
- "int8", "int16", "int32", "int64",
- "uint", "uint8", "uint16", "uint32", "uint64", "uintptr",
- "float32", "complex64", "complex128", "any", "comparable"
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala
deleted file mode 100644
index deb06bdd..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterPythonParser.scala
+++ /dev/null
@@ -1,250 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-import scala.util.matching.Regex
-
-/**
- * Python source parser that extracts structural entities and relationships.
- *
- * Attempts to use tree-sitter JNI first; falls back to regex-based extraction
- * when the native library is unavailable (common on some platforms).
- * The regex fallback is sufficient for Phase 1 since the goal is the pipeline
- * structure, not production-grade parsing.
- */
-class TreeSitterPythonParser extends Parser {
-
- /**
- * Parse a Python source file and extract entities and relationships.
- *
- * @param fileName the file name (e.g. "billing_service.py")
- * @param source the full source code as a string
- * @return ParseResult with entities and relationships
- */
- def parse(fileName: String, source: String): ParseResult = {
- // Try tree-sitter first, fall back to regex
- tryTreeSitter(fileName, source).getOrElse(regexParse(fileName, source))
- }
-
- // ---------------------------------------------------------------------------
- // Tree-sitter JNI attempt
- // ---------------------------------------------------------------------------
-
- private def tryTreeSitter(fileName: String, source: String): Option[ParseResult] = {
- try {
- val langClass = Class.forName("ch.usi.si.seart.treesitter.Language")
- val pythonLang = langClass.getMethod("valueOf", classOf[String]).invoke(null, "PYTHON")
-
- val parserClass = Class.forName("ch.usi.si.seart.treesitter.Parser")
- val parser = parserClass.getMethod("getFor", langClass).invoke(null, pythonLang)
-
- val tree = parserClass.getMethod("parse", classOf[String]).invoke(parser, source)
- val treeClass = Class.forName("ch.usi.si.seart.treesitter.Tree")
- val root = treeClass.getMethod("getRootNode").invoke(tree)
-
- // If we get here, tree-sitter loaded successfully
- // We would walk the AST here, but for now this path is not reached
- // on platforms where the JNI library is unavailable.
- parserClass.getMethod("close").invoke(parser)
- treeClass.getMethod("close").invoke(tree)
- None // TODO: implement full tree-sitter AST walking when JNI works
- } catch {
- case _: Throwable => None // Catches both Exception and Error (e.g. UnsatisfiedLinkError)
- }
- }
-
- // ---------------------------------------------------------------------------
- // Regex-based fallback parser
- // ---------------------------------------------------------------------------
-
- private val ClassPattern: Regex = """^class\s+(\w+)""".r
- private val FuncPattern: Regex = """^(\s*)def\s+(\w+)""".r
- private val ImportPattern: Regex = """^import\s+(\w+)""".r
- private val FromImportPattern: Regex = """^from\s+(\w+)\s+import""".r
- private val CallPattern: Regex = """(?:self\.)?(\w+)\(""".r
-
- private def regexParse(fileName: String, source: String): ParseResult = {
- val lines = source.split("\n", -1)
-
- // -- File entity --
- val fileEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("python")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- // -- Classes and functions --
- var entities = Vector(fileEntity)
- var relationships = Vector.empty[ParsedRelationship]
-
- // Track class ranges: (className, lineStart, lineEnd) for containment checks
- var classRanges = Vector.empty[(String, Int, Int)]
-
- for ((line, idx) <- lines.zipWithIndex) {
- val lineNum = idx + 1
-
- // Skip comment lines — prevents matching patterns inside comments
- if (line.trim.startsWith("#")) {
- // no-op: skip this line entirely
- } else {
-
- // Check for class definition
- ClassPattern.findFirstMatchIn(line).foreach { m =>
- val className = m.group(1)
- val classEnd = findBlockEnd(lines, idx)
- val bodyText = lines.slice(idx, classEnd).mkString("\n")
- val fp = Fingerprint.compute(line.trim.take(120), bodyText)
- entities = entities :+ ParsedEntity(
- name = className,
- kind = NodeKind.Class,
- attrs = Map("language" -> Json.fromString("python")),
- lineStart = lineNum,
- lineEnd = classEnd,
- contentFingerprint = Some(fp)
- )
- // File CONTAINS class
- relationships = relationships :+ ParsedRelationship(fileName, className, "CONTAINS")
- classRanges = classRanges :+ (className, lineNum, classEnd)
- }
-
- // Check for function/method definition
- FuncPattern.findFirstMatchIn(line).foreach { m =>
- val funcName = m.group(2)
- val funcEnd = findBlockEnd(lines, idx)
- val bodyText = lines.slice(idx, funcEnd).mkString("\n")
- val fp = Fingerprint.compute(line.trim.take(120), bodyText)
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map("language" -> Json.fromString("python")),
- lineStart = lineNum,
- lineEnd = funcEnd,
- contentFingerprint = Some(fp)
- )
-
- // Check if this function's line falls within any class range
- val enclosingClass = classRanges.find { case (_, start, end) =>
- lineNum > start && lineNum <= end
- }
- enclosingClass match {
- case Some((className, _, _)) =>
- // Method inside a class
- relationships = relationships :+ ParsedRelationship(className, funcName, "CONTAINS")
- case None =>
- // Top-level function — file defines it
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
- }
-
- // Extract function calls within the function body
- val callsInFunc = extractCalls(lines, idx, funcName)
- relationships = relationships ++ callsInFunc
- }
-
- // Check for imports
- ImportPattern.findFirstMatchIn(line).foreach { m =>
- val moduleName = m.group(1)
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
-
- FromImportPattern.findFirstMatchIn(line).foreach { m =>
- val moduleName = m.group(1)
- // Avoid duplicate module entities
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
-
- } // end else (non-comment line)
- }
-
- ParseResult(entities, relationships)
- }
-
- /**
- * Find the end line of a block starting at `startIdx` using indentation.
- * Looks for the last line that has greater indentation than the definition line.
- */
- private def findBlockEnd(lines: Array[String], startIdx: Int): Int = {
- val baseIndent = lines(startIdx).takeWhile(_ == ' ').length
- var endIdx = startIdx
- var i = startIdx + 1
- while (i < lines.length) {
- val line = lines(i)
- if (line.trim.nonEmpty) {
- val indent = line.takeWhile(_ == ' ').length
- if (indent > baseIndent) {
- endIdx = i
- } else {
- // Block ended
- return endIdx + 1
- }
- }
- i += 1
- }
- // If we reach end of file, the block extends to the end
- endIdx + 1
- }
-
- /**
- * Extract CALLS relationships from the body of a function starting at `funcIdx`.
- * Uses a simple heuristic: look for `identifier(` patterns in subsequent indented lines.
- */
- private def extractCalls(lines: Array[String], funcIdx: Int, callerName: String): Vector[ParsedRelationship] = {
- val baseIndent = lines(funcIdx).takeWhile(_ == ' ').length
- var calls = Vector.empty[ParsedRelationship]
- val seen = scala.collection.mutable.Set.empty[String]
- var i = funcIdx + 1
-
- while (i < lines.length) {
- val line = lines(i)
- if (line.trim.nonEmpty) {
- val indent = line.takeWhile(_ == ' ').length
- if (indent <= baseIndent) {
- // Exited the function body
- return calls
- }
- // Find calls in this line
- CallPattern.findAllMatchIn(line).foreach { m =>
- val callee = m.group(1)
- // Filter out Python builtins and self-references, and skip 'self' prefix calls
- // that match the method name itself
- if (callee != callerName && !PythonBuiltins.contains(callee) && !seen.contains(callee)) {
- seen += callee
- calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
- }
- }
- }
- i += 1
- }
- calls
- }
-
- private val PythonBuiltins: Set[String] = Set(
- "print", "len", "range", "int", "str", "float", "list", "dict",
- "set", "tuple", "type", "isinstance", "issubclass", "hasattr",
- "getattr", "setattr", "delattr", "super", "property", "staticmethod",
- "classmethod", "enumerate", "zip", "map", "filter", "sorted",
- "reversed", "any", "all", "min", "max", "sum", "abs", "round",
- "input", "open", "format", "repr", "id", "hash", "callable",
- "iter", "next", "not", "raise"
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterScalaParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterScalaParser.scala
deleted file mode 100644
index 09693c7a..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterScalaParser.scala
+++ /dev/null
@@ -1,350 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-import org.treesitter.{TSNode, TreeSitterScala}
-
-import scala.util.matching.Regex
-
-/**
- * Tree-sitter AST-based Scala parser with regex fallback.
- * Produces the same ParseResult shape as the original regex ScalaParser.
- */
-class TreeSitterScalaParser extends Parser {
-
- private val regexFallback = new ScalaParser()
-
- def parse(fileName: String, source: String): ParseResult = {
- try treeSitterParse(fileName, source)
- catch { case _: Throwable => regexFallback.parse(fileName, source) }
- }
-
- private def treeSitterParse(fileName: String, source: String): ParseResult = {
- TreeSitterUtils.withParse(new TreeSitterScala(), source) { root =>
- val ctx = new ScalaExtractionContext(fileName, source, root)
- ctx.result()
- }
- }
-
- // -------------------------------------------------------------------------
- // Extraction context — accumulates entities and relationships
- // -------------------------------------------------------------------------
-
- private class ScalaExtractionContext(
- fileName: String,
- source: String,
- root: TSNode
- ) {
- import TreeSitterUtils._
-
- var entities = Vector.empty[ParsedEntity]
- var relationships = Vector.empty[ParsedRelationship]
-
- // Initialize with file entity
- entities = entities :+ ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("scala")),
- lineStart = 1,
- lineEnd = source.split("\n", -1).length
- )
-
- // Walk the root
- walkChildren(root, enclosing = None)
-
- def result(): ParseResult = ParseResult(entities, relationships)
-
- private def walkChildren(node: TSNode, enclosing: Option[String]): Unit = {
- for (child <- namedChildren(node)) {
- walkNode(child, enclosing)
- }
- }
-
- private def walkNode(node: TSNode, enclosing: Option[String]): Unit = {
- node.getType() match {
- case "object_definition" =>
- handleTypeDefinition(node, enclosing, NodeKind.Object, "object")
- case "class_definition" =>
- val isCaseClass = hasCaseModifier(node)
- val scalaKind = if (isCaseClass) "case_class" else "class"
- handleTypeDefinition(node, enclosing, NodeKind.Class, scalaKind)
- case "trait_definition" =>
- handleTypeDefinition(node, enclosing, NodeKind.Trait, "trait")
- case "function_definition" =>
- handleFunctionDefinition(node, enclosing)
- case "import_declaration" =>
- handleImport(node)
- case "template_body" =>
- walkChildren(node, enclosing)
- case "package_clause" =>
- // recurse into package body
- walkChildren(node, enclosing)
- case _ =>
- // Recurse into other nodes to catch nested definitions
- walkChildren(node, enclosing)
- }
- }
-
- private def handleTypeDefinition(
- node: TSNode,
- enclosing: Option[String],
- kind: NodeKind,
- scalaKind: String
- ): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { name =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val sig = extractTypeSignature(node)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
- val vis = extractVisibility(node)
-
- entities = entities :+ ParsedEntity(
- name = name,
- kind = kind,
- attrs = Map(
- "scala_kind" -> Json.fromString(scalaKind),
- "language" -> Json.fromString("scala"),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(vis)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
-
- // CONTAINS edge
- val containsSrc = enclosing.getOrElse(fileName)
- relationships = relationships :+ ParsedRelationship(containsSrc, name, "CONTAINS")
-
- // EXTENDS edge
- findChild(node, "extends_clause").foreach { extendsNode =>
- // First type in extends clause is the parent
- val types = collectDescendants(extendsNode, n =>
- n.getType() == "type_identifier" || n.getType() == "identifier"
- )
- types.headOption.foreach { typeNode =>
- val parentName = nodeText(typeNode, source)
- if (parentName.nonEmpty && parentName.head.isUpper) {
- relationships = relationships :+ ParsedRelationship(name, parentName, "EXTENDS")
- }
- }
- }
-
- // Recurse into template_body
- findChild(node, "template_body").orElse(findChild(node, "body")).foreach { body =>
- walkChildren(body, Some(name))
- }
- }
- }
-
- private def handleFunctionDefinition(node: TSNode, enclosing: Option[String]): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { funcName =>
- val start = lineStart(node)
- val end = lineEnd(node)
-
- val (kind, containsSrc) = enclosing match {
- case Some(typeName) => (NodeKind.Method, typeName)
- case None => (NodeKind.Function, fileName)
- }
-
- val sig = extractFunctionSignature(node)
- val vis = extractVisibility(node)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = kind,
- attrs = Map(
- "language" -> Json.fromString("scala"),
- "summary" -> Json.fromString(sig),
- "signature" -> Json.fromString(sig),
- "visibility" -> Json.fromString(vis)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
-
- relationships = relationships :+ ParsedRelationship(containsSrc, funcName, "CONTAINS")
-
- // Extract CALLS from body
- fieldChild(node, "body").foreach { body =>
- extractCalls(body, funcName, enclosing.getOrElse(""))
- }
-
- // Extract REFERENCES from signature types
- extractSignatureRefs(node, funcName, enclosing.getOrElse(""))
- }
- }
-
- private def extractCalls(bodyNode: TSNode, callerName: String, enclosingTypeName: String): Unit = {
- val seen = scala.collection.mutable.Set.empty[String]
- val refSeen = scala.collection.mutable.Set.empty[String]
-
- val callExprs = collectDescendants(bodyNode, _.getType() == "call_expression")
- for (callExpr <- callExprs) {
- // Scala grammar: call_expression first named child is the function
- val funcNodeOpt = fieldChild(callExpr, "function")
- .orElse(if (callExpr.getNamedChildCount() > 0) Some(callExpr.getNamedChild(0)) else None)
-
- funcNodeOpt.foreach { funcNode =>
- val callee = resolveScalaCallee(funcNode)
- callee.foreach { name =>
- if (name != callerName && !ScalaBuiltins.contains(name) &&
- !seen.contains(name) && name.nonEmpty && name.head.isLower) {
- seen += name
- relationships = relationships :+ ParsedRelationship(callerName, name, "CALLS")
- }
- }
- }
- }
-
- // Extract qualified REFERENCES (e.g., NodeKind.File, SourceType.Code)
- // Scala tree-sitter grammar: field_expression has plain identifier children
- // (first child = object, third child = field, with "." in between)
- val fieldExprs = collectDescendants(bodyNode, _.getType() == "field_expression")
- for (fe <- fieldExprs) {
- // Try named field first, then fall back to first named child as qualifier
- val objNode = fieldChild(fe, "object")
- .orElse(fieldChild(fe, "qualifier"))
- .orElse {
- val named = namedChildren(fe)
- named.headOption.filter(_.getType() == "identifier")
- }
- objNode.foreach { on =>
- if (on.getType() == "identifier") {
- val qualifier = nodeText(on, source)
- if (qualifier.nonEmpty && qualifier.head.isUpper &&
- !ScalaBuiltinTypes.contains(qualifier) && !ScalaBuiltins.contains(qualifier) &&
- !refSeen.contains(qualifier) && qualifier != callerName &&
- qualifier != enclosingTypeName) {
- refSeen += qualifier
- relationships = relationships :+ ParsedRelationship(callerName, qualifier, "REFERENCES")
- }
- }
- }
- }
- }
-
- private def resolveScalaCallee(funcNode: TSNode): Option[String] = {
- funcNode.getType() match {
- case "identifier" =>
- Some(nodeText(funcNode, source))
- case "field_expression" =>
- // Scala field_expression: children are [object, ".", field]
- // Last named child is the field/method name
- val named = namedChildren(funcNode)
- named.lastOption.map(n => nodeText(n, source))
- case "call_expression" =>
- // Chained calls — recurse
- val inner = if (funcNode.getNamedChildCount() > 0) Some(funcNode.getNamedChild(0)) else None
- inner.flatMap(resolveScalaCallee)
- case _ => None
- }
- }
-
- private def extractSignatureRefs(node: TSNode, funcName: String, enclosingTypeName: String): Unit = {
- // Use regex on the signature text for type references — pragmatic hybrid approach
- val sig = extractFunctionSignature(node)
- val TypeRefPattern: Regex = """\b([A-Z]\w+)""".r
- val refs = TypeRefPattern.findAllMatchIn(sig).map(_.group(1)).toSet
- for (ref <- refs) {
- if (!ScalaBuiltinTypes.contains(ref) && ref != funcName &&
- ref != enclosingTypeName && ref != fileName) {
- relationships = relationships :+ ParsedRelationship(funcName, ref, "REFERENCES")
- }
- }
- }
-
- private def handleImport(node: TSNode): Unit = {
- val importText = nodeText(node, source)
- .stripPrefix("import ")
- .trim
-
- val moduleName = if (importText.contains("{")) {
- importText.substring(0, importText.indexOf('{')).stripSuffix(".").trim
- } else {
- importText
- }
-
- if (moduleName.nonEmpty && !entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineStart(node),
- lineEnd = lineEnd(node)
- )
- }
- if (moduleName.nonEmpty) {
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
- }
-
- // --- Helpers ---
-
- private def hasCaseModifier(node: TSNode): Boolean = {
- // Check for "case" modifier in the node text before the keyword
- val text = nodeText(node, source)
- text.trim.startsWith("case ") || text.contains(" case class") ||
- // Check modifiers children
- allChildren(node).exists { child =>
- child.getType() == "case" ||
- (child.getType() == "modifiers" && nodeText(child, source).contains("case"))
- }
- }
-
- private def extractTypeSignature(node: TSNode): String = {
- val text = nodeText(node, source)
- val firstLine = text.split("\n").head
- firstLine.replaceAll("""\s*\{\s*$""", "").trim.take(120)
- }
-
- private def extractFunctionSignature(node: TSNode): String = {
- val text = nodeText(node, source)
- val sig = text.split("\n").head
- .replaceAll("""\s*=\s*\{?\s*$""", "")
- .replaceAll("""\s*\{\s*$""", "")
- .trim
- sig.take(120)
- }
-
- private def extractVisibility(node: TSNode): String = {
- val text = nodeText(node, source).trim
- val withoutOverride = text.replaceFirst("""^\s*override\s+""", "").trim
- if (withoutOverride.startsWith("private")) "private"
- else if (withoutOverride.startsWith("protected")) "protected"
- else "public"
- }
- }
-
- // --- Shared constants (reused from ScalaParser) ---
-
- private val ScalaBuiltinTypes: Set[String] = Set(
- "String", "Int", "Long", "Double", "Float", "Boolean", "Unit", "Any", "AnyRef", "AnyVal",
- "Nothing", "Null", "Option", "List", "Map", "Set", "Vector", "Seq", "Array", "Future",
- "IO", "Either", "Try", "Some", "None", "Right", "Left", "Success", "Failure"
- )
-
- private val ScalaBuiltins: Set[String] = Set(
- "println", "print", "require", "assert", "assume",
- "Some", "None", "Option", "List", "Map", "Set", "Vector",
- "Right", "Left", "Try", "Success", "Failure",
- "Future", "IO", "pure", "flatMap", "map", "filter",
- "foreach", "fold", "foldLeft", "foldRight",
- "toString", "hashCode", "equals", "getClass",
- "apply", "unapply", "copy", "andThen", "compose",
- "getOrElse", "orElse", "contains", "exists", "forall",
- "collect", "traverse", "sequence", "attempt", "handleErrorWith",
- "raiseError", "blocking", "delay", "suspend", "async",
- "mkString", "toList", "toVector", "toSet", "toMap",
- "head", "tail", "size", "length", "isEmpty", "nonEmpty",
- "take", "drop", "slice", "zip", "zipWithIndex",
- "sortBy", "groupBy", "distinct", "flatten"
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterTypeScriptParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterTypeScriptParser.scala
deleted file mode 100644
index 69cdac18..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterTypeScriptParser.scala
+++ /dev/null
@@ -1,531 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-import org.treesitter.{TSNode, TreeSitterTypescript}
-
-/**
- * Tree-sitter AST-based TypeScript parser with regex fallback.
- * Produces the same ParseResult shape as the original regex TypeScriptParser.
- */
-class TreeSitterTypeScriptParser extends Parser {
-
- private val regexFallback = new TypeScriptParser()
-
- def parse(fileName: String, source: String): ParseResult = {
- try treeSitterParse(fileName, source)
- catch { case _: Throwable => regexFallback.parse(fileName, source) }
- }
-
- private def treeSitterParse(fileName: String, source: String): ParseResult = {
- TreeSitterUtils.withParse(new TreeSitterTypescript(), source) { root =>
- val ctx = new TSExtractionContext(fileName, source, root)
- ctx.result()
- }
- }
-
- // -------------------------------------------------------------------------
- // Extraction context
- // -------------------------------------------------------------------------
-
- private class TSExtractionContext(
- fileName: String,
- source: String,
- root: TSNode
- ) {
- import TreeSitterUtils._
-
- var entities = Vector.empty[ParsedEntity]
- var relationships = Vector.empty[ParsedRelationship]
-
- private val lines = source.split("\n", -1)
-
- // Initialize with file entity
- entities = entities :+ ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("typescript")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- // Walk the program
- walkProgram(root)
-
- def result(): ParseResult = ParseResult(entities, relationships)
-
- private def walkProgram(node: TSNode): Unit = {
- for (child <- namedChildren(node)) {
- walkTopLevel(child, exported = false)
- }
- }
-
- private def walkTopLevel(node: TSNode, exported: Boolean): Unit = {
- node.getType() match {
- case "export_statement" =>
- // The actual declaration is a child
- for (child <- namedChildren(node)) {
- walkTopLevel(child, exported = true)
- }
-
- case "class_declaration" | "abstract_class_declaration" =>
- handleClassDeclaration(node, exported)
-
- case "function_declaration" =>
- handleFunctionDeclaration(node, exported)
-
- case "lexical_declaration" | "variable_declaration" =>
- handleVariableDeclaration(node, exported)
-
- case "interface_declaration" =>
- handleInterfaceDeclaration(node, exported)
-
- case "type_alias_declaration" =>
- handleTypeAliasDeclaration(node, exported)
-
- case "import_statement" =>
- handleImport(node)
-
- case "expression_statement" =>
- // File-level code — extract CALLS
- extractFileLevelCalls(node)
-
- case "enum_declaration" =>
- handleEnumDeclaration(node, exported)
-
- case _ =>
- // Ignore other top-level nodes
- }
- }
-
- // --- Class handling ---
-
- private def handleClassDeclaration(node: TSNode, exported: Boolean): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { className =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val sig = extractFirstLine(node).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = className,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, className, "CONTAINS")
-
- // Extract methods from class body
- findChild(node, "class_body").foreach { body =>
- extractMethods(body, className)
- }
- }
- }
-
- private def extractMethods(classBody: TSNode, className: String): Unit = {
- for (child <- namedChildren(classBody)) {
- child.getType() match {
- case "method_definition" =>
- handleMethodDefinition(child, className)
- case "public_field_definition" | "property_definition" =>
- // Skip property definitions
- case _ =>
- // Skip other class body members
- }
- }
- }
-
- private def handleMethodDefinition(node: TSNode, className: String): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { methodName =>
- // Skip constructor
- if (methodName != "constructor" && !TypeScriptKeywords.contains(methodName)) {
- val start = lineStart(node)
- val end = lineEnd(node)
- val summary = extractFirstLine(node).trim.take(120)
- val vis = extractVisibility(node)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(summary, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = methodName,
- kind = NodeKind.Method,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString(vis)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(className, methodName, "CONTAINS")
-
- // Extract calls from method body
- fieldChild(node, "body").foreach { body =>
- extractCallsFromBody(body, methodName)
- }
- }
- }
- }
-
- // --- Function handling ---
-
- private def handleFunctionDeclaration(node: TSNode, exported: Boolean): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { funcName =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val summary = extractFirstLine(node).trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(summary, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString("public"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
-
- // Extract calls from function body
- fieldChild(node, "body").foreach { body =>
- extractCallsFromBody(body, funcName)
- }
- }
- }
-
- // --- Variable/arrow function handling ---
-
- private def handleVariableDeclaration(node: TSNode, exported: Boolean): Unit = {
- for (declarator <- childrenOfType(node, "variable_declarator")) {
- val nameOpt = extractName(declarator, source)
- nameOpt.foreach { varName =>
- // Check if the value is an arrow function or function expression
- fieldChild(declarator, "value").foreach { value =>
- val isFunc = value.getType() == "arrow_function" ||
- value.getType() == "function" ||
- value.getType() == "function_expression"
- if (isFunc && !entities.exists(e => e.name == varName && e.kind == NodeKind.Function)) {
- val start = lineStart(node)
- val end = lineEnd(node)
- val summary = extractFirstLine(node).trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(summary, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = varName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString("public"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, varName, "CONTAINS")
-
- // Extract calls from function body
- fieldChild(value, "body").foreach { body =>
- extractCallsFromBody(body, varName)
- }
- }
- }
- }
- }
- }
-
- // --- Interface handling ---
-
- private def handleInterfaceDeclaration(node: TSNode, exported: Boolean): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { ifaceName =>
- if (!entities.exists(e => e.name == ifaceName && e.kind == NodeKind.Interface)) {
- val start = lineStart(node)
- val end = lineEnd(node)
- val sig = extractFirstLine(node).replaceAll("""\s*\{\s*$""", "").trim.take(120)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(sig, bodyText)
-
- entities = entities :+ ParsedEntity(
- name = ifaceName,
- kind = NodeKind.Interface,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("interface"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, ifaceName, "CONTAINS")
- }
- }
- }
-
- // --- Type alias handling ---
-
- private def handleTypeAliasDeclaration(node: TSNode, exported: Boolean): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { typeName =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val text = nodeText(node, source).trim
- val fp = Fingerprint.compute(text.take(120), text)
-
- entities = entities :+ ParsedEntity(
- name = typeName,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("type_alias"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
- }
- }
-
- // --- Enum handling ---
-
- private def handleEnumDeclaration(node: TSNode, exported: Boolean): Unit = {
- val nameOpt = extractName(node, source)
- nameOpt.foreach { enumName =>
- val start = lineStart(node)
- val end = lineEnd(node)
- val bodyText = nodeText(node, source)
- val fp = Fingerprint.compute(extractFirstLine(node).trim.take(120), bodyText)
-
- entities = entities :+ ParsedEntity(
- name = enumName,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("enum"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = start,
- lineEnd = end,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, enumName, "CONTAINS")
- }
- }
-
- // --- Import handling ---
-
- private def handleImport(node: TSNode): Unit = {
- // Extract module path from the source/string child
- val moduleOpt = findChild(node, "string").map(n => nodeText(n, source))
- .map(_.stripPrefix("\"").stripSuffix("\"").stripPrefix("'").stripSuffix("'"))
-
- moduleOpt.foreach { moduleName =>
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineStart(node),
- lineEnd = lineEnd(node)
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
-
- // Extract named imports
- findChild(node, "import_clause").foreach { clause =>
- // Default import: import Foo from "mod"
- findChild(clause, "identifier").foreach { id =>
- val name = nodeText(id, source)
- if (name.nonEmpty && !relationships.exists(r =>
- r.srcName == fileName && r.dstName == name && r.predicate == "IMPORTS")) {
- relationships = relationships :+ ParsedRelationship(fileName, name, "IMPORTS")
- }
- }
-
- // Named imports: import { a, b as c } from "mod"
- findChild(clause, "named_imports").foreach { namedImports =>
- for (spec <- childrenOfType(namedImports, "import_specifier")) {
- // Use alias if present, otherwise use name
- val alias = fieldChild(spec, "alias").map(n => nodeText(n, source))
- val importName = alias.orElse(extractName(spec, source))
- importName.foreach { name =>
- if (!relationships.exists(r =>
- r.srcName == fileName && r.dstName == name && r.predicate == "IMPORTS")) {
- relationships = relationships :+ ParsedRelationship(fileName, name, "IMPORTS")
- }
- }
- }
- }
-
- // Namespace import: import * as foo from "mod"
- findChild(clause, "namespace_import").foreach { nsImport =>
- findChild(nsImport, "identifier").foreach { id =>
- val name = nodeText(id, source)
- if (name.nonEmpty && !relationships.exists(r =>
- r.srcName == fileName && r.dstName == name && r.predicate == "IMPORTS")) {
- relationships = relationships :+ ParsedRelationship(fileName, name, "IMPORTS")
- }
- }
- }
- }
- }
-
- // Handle side-effect imports (no import clause, just `import "mod"`)
- if (findChild(node, "import_clause").isEmpty) {
- findChild(node, "string").foreach { strNode =>
- val moduleName = nodeText(strNode, source)
- .stripPrefix("\"").stripSuffix("\"")
- .stripPrefix("'").stripSuffix("'")
- if (moduleName.nonEmpty) {
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineStart(node),
- lineEnd = lineEnd(node)
- )
- }
- if (!relationships.exists(r => r.dstName == moduleName && r.predicate == "IMPORTS")) {
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
- }
- }
- }
- }
-
- // --- CALLS extraction ---
-
- private def extractCallsFromBody(bodyNode: TSNode, callerName: String): Unit = {
- val seen = scala.collection.mutable.Set.empty[String]
- val callExprs = collectDescendants(bodyNode, _.getType() == "call_expression")
-
- for (callExpr <- callExprs) {
- fieldChild(callExpr, "function").foreach { funcNode =>
- val callee = resolveCalleeName(funcNode)
- callee.foreach { name =>
- if (name != callerName && !TypeScriptBuiltins.contains(name) &&
- !TypeScriptKeywords.contains(name) && !seen.contains(name) && name != fileName) {
- seen += name
- relationships = relationships :+ ParsedRelationship(callerName, name, "CALLS")
- }
- }
- }
- }
- }
-
- private def extractFileLevelCalls(exprStatement: TSNode): Unit = {
- val callExprs = collectDescendants(exprStatement, _.getType() == "call_expression")
- for (callExpr <- callExprs) {
- fieldChild(callExpr, "function").foreach { funcNode =>
- val callee = resolveCalleeName(funcNode)
- callee.foreach { name =>
- if (!TypeScriptBuiltins.contains(name) && !TypeScriptKeywords.contains(name) &&
- name != fileName && !relationships.exists(r =>
- r.srcName == fileName && r.dstName == name && r.predicate == "CALLS")) {
- relationships = relationships :+ ParsedRelationship(fileName, name, "CALLS")
- }
- }
- }
- }
- }
-
- private def resolveCalleeName(funcNode: TSNode): Option[String] = {
- funcNode.getType() match {
- case "identifier" =>
- Some(nodeText(funcNode, source))
- case "member_expression" =>
- // e.g., this.method(), obj.method() — extract the property name
- fieldChild(funcNode, "property").map(n => nodeText(n, source))
- case "call_expression" =>
- // Chained: foo.bar().baz() — take the outermost function
- fieldChild(funcNode, "function").flatMap(resolveCalleeName)
- case _ =>
- None
- }
- }
-
- // --- Helpers ---
-
- private def extractFirstLine(node: TSNode): String = {
- val text = nodeText(node, source)
- text.split("\n").head
- }
-
- private def extractVisibility(node: TSNode): String = {
- // Check for accessibility_modifier child
- val modifier = allChildren(node).find(c =>
- c.getType() == "accessibility_modifier" ||
- c.getType() == "private" || c.getType() == "public" || c.getType() == "protected"
- )
- modifier match {
- case Some(m) =>
- val text = nodeText(m, source).trim
- if (text == "private") "private"
- else if (text == "protected") "protected"
- else "public"
- case None =>
- // Also check the first line text
- val firstLine = extractFirstLine(node).trim
- if (firstLine.startsWith("private ") || firstLine.startsWith("private\t")) "private"
- else if (firstLine.startsWith("protected ") || firstLine.startsWith("protected\t")) "protected"
- else "public"
- }
- }
- }
-
- // --- Shared constants (reused from TypeScriptParser) ---
-
- private val TypeScriptKeywords: Set[String] = Set(
- "if", "else", "for", "while", "do", "switch", "case", "break",
- "continue", "return", "throw", "try", "catch", "finally",
- "new", "delete", "typeof", "instanceof", "void", "in", "of",
- "class", "extends", "implements", "interface", "enum", "type",
- "const", "let", "var", "function", "async", "await",
- "import", "export", "default", "from", "as",
- "this", "super", "constructor", "static", "private", "public", "protected"
- )
-
- private val TypeScriptBuiltins: Set[String] = Set(
- "console", "log", "error", "warn", "parseInt", "parseFloat",
- "String", "Number", "Boolean", "Array", "Object", "Promise",
- "JSON", "Math", "Date", "RegExp", "Error", "Map", "Set",
- "WeakMap", "WeakSet", "Symbol",
- "setTimeout", "setInterval", "clearTimeout", "clearInterval",
- "require", "module", "exports",
- "fetch", "Response", "Request", "URL", "Buffer", "process",
- "map", "filter", "reduce", "forEach", "sort", "find", "some", "every",
- "includes", "indexOf", "slice", "splice", "push", "pop", "shift", "unshift",
- "concat", "join", "flat", "flatMap", "fill", "from",
- "split", "replace", "trim", "toUpperCase", "toLowerCase", "substring",
- "charAt", "startsWith", "endsWith", "padStart", "padEnd", "match", "test",
- "keys", "values", "entries", "assign", "create", "freeze",
- "defineProperty", "getOwnPropertyNames", "hasOwnProperty",
- "then", "catch", "finally", "all", "race", "resolve", "reject"
- )
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterUtils.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterUtils.scala
deleted file mode 100644
index 7c69a61c..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TreeSitterUtils.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import org.treesitter.{TSNode, TSParser, TSTree, TSLanguage}
-
-object TreeSitterUtils {
-
- /** 1-based line start */
- def lineStart(node: TSNode): Int = node.getStartPoint().getRow() + 1
-
- /** 1-based line end */
- def lineEnd(node: TSNode): Int = node.getEndPoint().getRow() + 1
-
- /** Extract source text for a node */
- def nodeText(node: TSNode, source: String): String =
- source.substring(node.getStartByte(), node.getEndByte())
-
- /** Find named child by field name, returning None if null */
- def fieldChild(node: TSNode, field: String): Option[TSNode] = {
- val child = node.getChildByFieldName(field)
- if (child == null || child.isNull()) None else Some(child)
- }
-
- /** Find first named child with given type */
- def findChild(node: TSNode, nodeType: String): Option[TSNode] =
- namedChildren(node).find(_.getType() == nodeType)
-
- /** All named children */
- def namedChildren(node: TSNode): Vector[TSNode] =
- (0 until node.getNamedChildCount()).map(node.getNamedChild).toVector
-
- /** All named children of a given type */
- def childrenOfType(node: TSNode, nodeType: String): Vector[TSNode] =
- namedChildren(node).filter(_.getType() == nodeType)
-
- /** All children (named and anonymous) */
- def allChildren(node: TSNode): Vector[TSNode] =
- (0 until node.getChildCount()).map(node.getChild).toVector
-
- /** Recursively collect all descendants matching a predicate */
- def collectDescendants(node: TSNode, pred: TSNode => Boolean): Vector[TSNode] = {
- val buf = Vector.newBuilder[TSNode]
- def walk(n: TSNode): Unit = {
- if (pred(n)) buf += n
- for (i <- 0 until n.getChildCount()) walk(n.getChild(i))
- }
- walk(node)
- buf.result()
- }
-
- /** Extract identifier name from a node's "name" field */
- def extractName(node: TSNode, source: String): Option[String] =
- fieldChild(node, "name").map(n => nodeText(n, source))
-
- /** Loan pattern: parse source, run body, close parser+tree */
- def withParse[A](lang: TSLanguage, source: String)(body: TSNode => A): A = {
- val parser = new TSParser()
- parser.setLanguage(lang)
- val tree = parser.parseString(null, source)
- try body(tree.getRootNode())
- finally { tree.close(); parser.close() }
- }
-}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala
deleted file mode 100644
index fc2670a2..00000000
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala
+++ /dev/null
@@ -1,422 +0,0 @@
-package ix.memory.ingestion.parsers
-
-import ix.memory.ingestion.{Fingerprint, Parser, ParseResult, ParsedEntity, ParsedRelationship}
-import ix.memory.model.NodeKind
-import io.circe.Json
-
-import scala.util.matching.Regex
-
-/**
- * TypeScript/JavaScript source parser that extracts structural entities
- * and relationships using regex-based extraction.
- */
-class TypeScriptParser extends Parser {
-
- /**
- * Parse a TypeScript source file and extract entities and relationships.
- *
- * @param fileName the file name (e.g. "api.ts")
- * @param source the full source code as a string
- * @return ParseResult with entities and relationships
- */
- def parse(fileName: String, source: String): ParseResult = {
- val lines = source.split("\n", -1)
-
- // -- File entity (no raw content stored) --
- val fileEntity = ParsedEntity(
- name = fileName,
- kind = NodeKind.File,
- attrs = Map("language" -> Json.fromString("typescript")),
- lineStart = 1,
- lineEnd = lines.length
- )
-
- var entities = Vector(fileEntity)
- var relationships = Vector.empty[ParsedRelationship]
-
- // Track class ranges for containment
- var classRanges = Vector.empty[(String, Int, Int)]
-
- for ((line, idx) <- lines.zipWithIndex) {
- val lineNum = idx + 1
- val trimmed = line.trim
-
- // Skip comment lines
- if (trimmed.startsWith("//") || trimmed.startsWith("*") || trimmed.startsWith("/*")) {
- // no-op
- } else {
-
- // Check for class definition
- ClassPattern.findFirstMatchIn(line).foreach { m =>
- val className = m.group(1)
- val classEnd = findBraceBlockEnd(lines, idx)
- val bodyText = lines.slice(idx, classEnd).mkString("\n")
- val fp = Fingerprint.compute(trimmed.replaceAll("""\s*\{\s*$""", "").take(120), bodyText)
- val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
- entities = entities :+ ParsedEntity(
- name = className,
- kind = NodeKind.Class,
- attrs = Map("language" -> Json.fromString("typescript"), "exported" -> Json.fromBoolean(exported)),
- lineStart = lineNum,
- lineEnd = classEnd,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, className, "CONTAINS")
- classRanges = classRanges :+ (className, lineNum, classEnd)
-
- // Extract methods from this class
- val methods = extractMethods(lines, idx, classEnd, className)
- entities = entities ++ methods._1
- relationships = relationships ++ methods._2
- }
-
- // Check for top-level function (not inside a class)
- if (!insideClass(lineNum, classRanges)) {
- FuncPattern.findFirstMatchIn(line).foreach { m =>
- val funcName = m.group(1)
- val funcEnd = findBraceBlockEnd(lines, idx)
- val summary = line.trim.take(120)
- val bodyText = lines.slice(idx, funcEnd).mkString("\n")
- val fp = Fingerprint.compute(summary, bodyText)
- val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString("public"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = funcEnd,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
-
- // Extract calls within this function
- val calls = extractCalls(lines, idx, funcEnd, funcName)
- relationships = relationships ++ calls
- }
-
- // Check for arrow function / const function
- val arrowMatch = ArrowFuncPattern.findFirstMatchIn(line)
- val constFuncMatch = ConstFuncPattern.findFirstMatchIn(line)
- val funcMatch = arrowMatch.orElse(constFuncMatch)
- funcMatch.foreach { m =>
- val funcName = m.group(1)
- // Avoid duplicates if already captured by FuncPattern
- if (!entities.exists(e => e.name == funcName && e.kind == NodeKind.Function)) {
- val funcEnd = findBraceBlockEnd(lines, idx)
- val summary = line.trim.take(120)
- val bodyText = lines.slice(idx, funcEnd).mkString("\n")
- val fp = Fingerprint.compute(summary, bodyText)
- val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
- entities = entities :+ ParsedEntity(
- name = funcName,
- kind = NodeKind.Function,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString("public"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = funcEnd,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, funcName, "CONTAINS")
-
- val callRels = extractCalls(lines, idx, funcEnd, funcName)
- relationships = relationships ++ callRels
- }
- }
- }
-
- // Check for imports
- ImportPattern.findFirstMatchIn(line).foreach { m =>
- val moduleName = m.group(1)
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
-
- // Extract named import bindings: import { foo, bar } from "mod"
- NamedImportPattern.findFirstMatchIn(line).foreach { m =>
- val names = m.group(1).split(",").map(_.trim).filter(_.nonEmpty)
- for (rawName <- names) {
- val symbolName = if (rawName.contains(" as ")) rawName.split("\\s+as\\s+").last.trim else rawName
- if (symbolName.nonEmpty && !relationships.exists(r => r.srcName == fileName && r.dstName == symbolName && r.predicate == "IMPORTS")) {
- relationships = relationships :+ ParsedRelationship(fileName, symbolName, "IMPORTS")
- }
- }
- }
-
- SideEffectImportPattern.findFirstMatchIn(line).foreach { m =>
- val moduleName = m.group(1)
- if (!relationships.exists(r => r.dstName == moduleName && r.predicate == "IMPORTS")) {
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
- }
-
- // Check for interface definition
- InterfacePattern.findFirstMatchIn(line).foreach { m =>
- val ifaceName = m.group(1)
- if (!entities.exists(e => e.name == ifaceName && e.kind == NodeKind.Interface)) {
- val ifaceEnd = findBraceBlockEnd(lines, idx)
- val bodyText = lines.slice(idx, ifaceEnd).mkString("\n")
- val fp = Fingerprint.compute(trimmed.replaceAll("""\s*\{\s*$""", "").take(120), bodyText)
- val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
- entities = entities :+ ParsedEntity(
- name = ifaceName,
- kind = NodeKind.Interface,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("interface"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = ifaceEnd,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, ifaceName, "CONTAINS")
- }
- }
-
- // Check for type alias
- TypeAliasPattern.findFirstMatchIn(line).foreach { m =>
- val typeName = m.group(1)
- if (!trimmed.startsWith("import")) {
- val exported = trimmed.startsWith("export ") || trimmed.startsWith("export{")
- val fp = Fingerprint.compute(trimmed.take(120), trimmed)
- entities = entities :+ ParsedEntity(
- name = typeName,
- kind = NodeKind.Class,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("type_alias"),
- "exported" -> Json.fromBoolean(exported)
- ),
- lineStart = lineNum,
- lineEnd = lineNum,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(fileName, typeName, "CONTAINS")
- }
- }
-
- } // end else (non-comment)
- }
-
- // Extract file-level calls from code outside named functions/methods/classes
- val coveredRanges: Vector[(Int, Int)] = entities
- .filter(e => e.kind == NodeKind.Function || e.kind == NodeKind.Method || e.kind == NodeKind.Class)
- .map(e => (e.lineStart, e.lineEnd))
-
- val fileCallsSeen = scala.collection.mutable.Set.empty[String]
- for ((line, idx) <- lines.zipWithIndex) {
- val lineNum = idx + 1
- val inNamedBody = coveredRanges.exists { case (s, e) => lineNum >= s && lineNum <= e }
- if (!inNamedBody && !line.trim.startsWith("//") && !line.trim.startsWith("*")) {
- CallPattern.findAllMatchIn(line).foreach { m =>
- val callee = m.group(1)
- if (!TypeScriptBuiltins.contains(callee) && !TypeScriptKeywords.contains(callee)
- && !fileCallsSeen.contains(callee) && callee != fileName) {
- fileCallsSeen += callee
- relationships = relationships :+ ParsedRelationship(fileName, callee, "CALLS")
- }
- }
- }
- }
-
- ParseResult(entities, relationships)
- }
-
- // ---------------------------------------------------------------------------
- // Patterns
- // ---------------------------------------------------------------------------
-
- private val ClassPattern: Regex = """(?:export\s+)?(?:abstract\s+)?class\s+(\w+)""".r
- private val FuncPattern: Regex = """(?:export\s+)?(?:async\s+)?function\s+(\w+)""".r
- private val ArrowFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[a-zA-Z_]\w*)\s*(?::\s*[^=]+)?\s*=>""".r
- private val ConstFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?function""".r
- private val MethodPattern: Regex = """^\s+(?:async\s+)?(?:private\s+|public\s+|protected\s+|static\s+|readonly\s+)*(\w+)\s*(?:<[^>]+>)?\s*\(""".r
- private val ImportPattern: Regex = """import\s+.*?from\s+['"]([^'"]+)['"]""".r
- private val NamedImportPattern: Regex = """import\s+\{([^}]+)\}\s+from\s+['"]([^'"]+)['"]""".r
- private val SideEffectImportPattern: Regex = """^import\s+['"]([^'"]+)['"]""".r
- private val InterfacePattern: Regex = """(?:export\s+)?interface\s+(\w+)""".r
- private val TypeAliasPattern: Regex = """(?:export\s+)?type\s+(\w+)\s*(?:<[^>]+>)?\s*=""".r
- private val CallPattern: Regex = """(?:(?:this|self|await)\.)?\b(\w+)\s*\(""".r
-
- // ---------------------------------------------------------------------------
- // Helpers
- // ---------------------------------------------------------------------------
-
- private def insideClass(lineNum: Int, classRanges: Vector[(String, Int, Int)]): Boolean =
- classRanges.exists { case (_, start, end) => lineNum > start && lineNum <= end }
-
- private def extractVisibility(trimmedLine: String): String = {
- if (trimmedLine.startsWith("private ") || trimmedLine.startsWith("private\t")) "private"
- else if (trimmedLine.startsWith("protected ") || trimmedLine.startsWith("protected\t")) "protected"
- else "public"
- }
-
- /**
- * Extract methods from a class body.
- */
- private def extractMethods(
- lines: Array[String],
- classStartIdx: Int,
- classEndLine: Int,
- className: String
- ): (Vector[ParsedEntity], Vector[ParsedRelationship]) = {
- var entities = Vector.empty[ParsedEntity]
- var relationships = Vector.empty[ParsedRelationship]
-
- var i = classStartIdx + 1
- while (i < lines.length && (i + 1) <= classEndLine) {
- val line = lines(i)
- val lineNum = i + 1
-
- if (!line.trim.startsWith("//") && !line.trim.startsWith("*")) {
- MethodPattern.findFirstMatchIn(line).foreach { m =>
- val methodName = m.group(1)
- // Skip keywords that look like methods
- if (!TypeScriptKeywords.contains(methodName)) {
- val methodEnd = findBraceBlockEnd(lines, i)
- val summary = line.trim.take(120)
- val visibility = extractVisibility(line.trim)
- val bodyText = lines.slice(i, methodEnd).mkString("\n")
- val fp = Fingerprint.compute(summary, bodyText)
- entities = entities :+ ParsedEntity(
- name = methodName,
- kind = NodeKind.Method,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "summary" -> Json.fromString(summary),
- "signature" -> Json.fromString(summary),
- "visibility" -> Json.fromString(visibility)
- ),
- lineStart = lineNum,
- lineEnd = methodEnd,
- contentFingerprint = Some(fp)
- )
- relationships = relationships :+ ParsedRelationship(className, methodName, "CONTAINS")
-
- // Extract calls within this method
- val calls = extractCalls(lines, i, methodEnd, methodName)
- relationships = relationships ++ calls
- }
- }
- }
- i += 1
- }
-
- (entities, relationships)
- }
-
- /**
- * Find end of a brace-delimited block starting near `startIdx`.
- *
- * Uses end-of-line brace depth to detect block entry. This correctly
- * skips inline TypeScript type annotations like `{ kind?: string }`
- * that balance to zero on a single line, which would otherwise cause
- * the parser to think the function body ended on the parameter line.
- */
- private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Int = {
- var braceCount = 0
- var entered = false
- var i = startIdx
- while (i < lines.length) {
- for (ch <- lines(i)) {
- if (ch == '{') braceCount += 1
- if (ch == '}') braceCount -= 1
- }
- // Only mark as entered when braceCount > 0 at end of line.
- // This skips inline type annotations like { a: string; b: number }
- // that balance to 0 within the same line.
- if (!entered && braceCount > 0) entered = true
- if (entered && braceCount <= 0) return i + 1
- i += 1
- }
- lines.length
- }
-
- /**
- * Extract CALLS relationships from a function/method body.
- */
- private def extractCalls(
- lines: Array[String],
- funcStartIdx: Int,
- funcEndLine: Int,
- callerName: String
- ): Vector[ParsedRelationship] = {
- var calls = Vector.empty[ParsedRelationship]
- val seen = scala.collection.mutable.Set.empty[String]
- var i = funcStartIdx + 1
-
- while (i < lines.length && (i + 1) <= funcEndLine) {
- val line = lines(i)
- CallPattern.findAllMatchIn(line).foreach { m =>
- val callee = m.group(1)
- if (callee != callerName && !TypeScriptBuiltins.contains(callee) &&
- !TypeScriptKeywords.contains(callee) && !seen.contains(callee)) {
- seen += callee
- calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
- }
- }
- i += 1
- }
- calls
- }
-
- private val TypeScriptKeywords: Set[String] = Set(
- "if", "else", "for", "while", "do", "switch", "case", "break",
- "continue", "return", "throw", "try", "catch", "finally",
- "new", "delete", "typeof", "instanceof", "void", "in", "of",
- "class", "extends", "implements", "interface", "enum", "type",
- "const", "let", "var", "function", "async", "await",
- "import", "export", "default", "from", "as",
- "this", "super", "constructor", "static", "private", "public", "protected"
- )
-
- private val TypeScriptBuiltins: Set[String] = Set(
- "console", "log", "error", "warn", "parseInt", "parseFloat",
- "String", "Number", "Boolean", "Array", "Object", "Promise",
- "JSON", "Math", "Date", "RegExp", "Error", "Map", "Set",
- "WeakMap", "WeakSet", "Symbol",
- "setTimeout", "setInterval", "clearTimeout", "clearInterval",
- "require", "module", "exports",
- "fetch", "Response", "Request", "URL", "Buffer", "process",
- // Array/iterable methods
- "map", "filter", "reduce", "forEach", "sort", "find", "some", "every",
- "includes", "indexOf", "slice", "splice", "push", "pop", "shift", "unshift",
- "concat", "join", "flat", "flatMap", "fill", "from",
- // String methods
- "split", "replace", "trim", "toUpperCase", "toLowerCase", "substring",
- "charAt", "startsWith", "endsWith", "padStart", "padEnd", "match", "test",
- // Object methods
- "keys", "values", "entries", "assign", "create", "freeze",
- "defineProperty", "getOwnPropertyNames", "hasOwnProperty",
- // Promise methods
- "then", "catch", "finally", "all", "race", "resolve", "reject"
- )
-}
diff --git a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
index 35ec4613..0deac771 100644
--- a/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/api/RoutesSpec.scala
@@ -17,7 +17,6 @@ import ix.memory.TestDbHelper
import ix.memory.conflict.ConflictService
import ix.memory.context._
import ix.memory.db._
-import ix.memory.ingestion._
import ix.memory.map.MapService
import ix.memory.model._
import ix.memory.smell.SmellService
@@ -65,18 +64,13 @@ class RoutesSpec extends AsyncFlatSpec with AsyncIOSpec with Matchers with TestD
new ConfidenceScorerImpl(),
new ConflictDetectorImpl()
)
- val parserRouter = new ParserRouter()
- val ingestionService = new IngestionService(parserRouter, writeApi, queryApi)
val bulkWriteApi = new BulkWriteApi(client)
- val bulkIngestionService = new BulkIngestionService(parserRouter, bulkWriteApi, queryApi)
val mapService = new MapService(client, queryApi, writeApi)
val smellService = new SmellService(client, writeApi)
val subsystemService = new SubsystemScoringService(client, writeApi, mapService)
Routes.all(
contextService,
- ingestionService,
- bulkIngestionService,
queryApi,
writeApi,
conflictService,
diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/FileDiscoverySpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/FileDiscoverySpec.scala
deleted file mode 100644
index 806e275c..00000000
--- a/memory-layer/src/test/scala/ix/memory/ingestion/FileDiscoverySpec.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-package ix.memory.ingestion
-
-import java.nio.file.Paths
-
-import org.scalatest.flatspec.AnyFlatSpec
-import org.scalatest.matchers.should.Matchers
-
-class FileDiscoverySpec extends AnyFlatSpec with Matchers {
-
- "extensionsFor(None)" should "default to parser-supported file types" in {
- val extensions = FileDiscovery.extensionsFor(None)
-
- extensions should contain allOf (".py", ".ts", ".tsx", ".scala", ".sc", ".json", ".yaml", ".yml", ".toml", ".md")
- extensions should not contain ".txt"
- extensions should not contain ".ini"
- extensions should not contain ".sbt"
- }
-
- "matchesFilter" should "exclude unsupported generic-text files by default" in {
- val extensions = FileDiscovery.extensionsFor(None)
-
- FileDiscovery.matchesFilter(Paths.get("README.md"), extensions) shouldBe true
- FileDiscovery.matchesFilter(Paths.get("Dockerfile"), extensions) shouldBe false
- FileDiscovery.matchesFilter(Paths.get("notes.txt"), extensions) shouldBe false
- }
-}
diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/GenericTextFallbackSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/GenericTextFallbackSpec.scala
deleted file mode 100644
index 14fca5c8..00000000
--- a/memory-layer/src/test/scala/ix/memory/ingestion/GenericTextFallbackSpec.scala
+++ /dev/null
@@ -1,17 +0,0 @@
-package ix.memory.ingestion
-
-import org.scalatest.flatspec.AnyFlatSpec
-import org.scalatest.matchers.should.Matchers
-
-class GenericTextFallbackSpec extends AnyFlatSpec with Matchers {
-
- "GenericTextFallback" should "store compact metadata instead of raw content" in {
- val parsed = GenericTextFallback.parse("notes.txt", "first line\nsecond line\n")
- val entity = parsed.entities.head
-
- entity.attrs.keySet should contain allOf ("file_name", "line_count", "preview")
- entity.attrs.keySet should not contain "content"
- entity.attrs("preview").asString shouldBe Some("first line")
- entity.contentFingerprint should not be empty
- }
-}