Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat: config parser for YAML, JSON, and TOML ingestion
riley0227246957d35bac
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scalaindex 13035545..d9adfcf0 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala@@ -30,6 +30,7 @@ object GraphPatchBuilder {val extractor = if (filePath.endsWith(".py")) "tree-sitter-python/1.0"else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) "typescript-parser/1.0"+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) "config-parser/1.0"else "unknown-parser/1.0"val nodeOps = parseResult.entities.map { e =>@@ -70,11 +71,14 @@ object GraphPatchBuilder {val claimOps = relationshipClaims ++ attributeClaimsval allOps = nodeOps ++ edgeOps ++ claimOps+ val sourceType = if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) SourceType.Config+ else SourceType.Code+GraphPatch(patchId = PatchId(UUID.randomUUID()),actor = "ix/ingestion",timestamp = Instant.now(),- source = PatchSource(filePath, sourceHash, extractor, SourceType.Code),+ source = PatchSource(filePath, sourceHash, extractor, sourceType),baseRev = Rev(0L),ops = allOps,replaces = Vector.empty,diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scalaindex 366ef8e2..52f4462c 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala@@ -35,6 +35,7 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quer// Check for existing patch from same source+extractorextractor = if (filePath.toString.endsWith(".py")) "tree-sitter-python/1.0"else if (filePath.toString.endsWith(".ts") || filePath.toString.endsWith(".tsx")) "typescript-parser/1.0"+ else if (filePath.toString.endsWith(".json") || filePath.toString.endsWith(".yaml") || filePath.toString.endsWith(".yml") || filePath.toString.endsWith(".toml")) "config-parser/1.0"else "unknown-parser/1.0"existing <- queryApi.getPatchesBySource(filePath.toString, extractor)prevPatch = existing.headOption@@ -93,7 +94,8 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, querval extensions = language match {case Some("python") => Set(".py")case Some("typescript") => Set(".ts", ".tsx")- case _ => Set(".py", ".ts", ".tsx") // all supported+ case Some("config") => Set(".json", ".yaml", ".yml", ".toml")+ case _ => Set(".py", ".ts", ".tsx", ".json", ".yaml", ".yml", ".toml") // all supported}if (Files.isRegularFile(path)) {diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scalaindex 385b8e10..c1df0f32 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala@@ -1,18 +1,20 @@package ix.memory.ingestion-import ix.memory.ingestion.parsers.{TreeSitterPythonParser, TypeScriptParser}+import ix.memory.ingestion.parsers.{ConfigParser, TreeSitterPythonParser, TypeScriptParser}/*** Routes file paths to the appropriate language parser.- * Supports Python (.py), TypeScript (.ts), and TSX (.tsx) files.+ * Supports Python (.py), TypeScript (.ts/.tsx), and config files (.json/.yaml/.yml/.toml).*/class ParserRouter {private val pythonParser = new TreeSitterPythonParser()private val tsParser = new TypeScriptParser()+ private val configParser = new ConfigParser()def parserFor(filePath: String): Option[Parser] = {if (filePath.endsWith(".py")) Some(pythonParser)else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) Some(tsParser)+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) Some(configParser)else None}}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scalanew file mode 100644index 00000000..552c83c2--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala@@ -0,0 +1,133 @@+package ix.memory.ingestion.parsers++import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}+import ix.memory.model.NodeKind+import io.circe.Json++/**+ * Config file parser for YAML, JSON, and TOML files.+ * Extracts a Config node for the file and ConfigEntry nodes for top-level keys.+ * Uses simple regex heuristics — not a full YAML/TOML parser.+ */+class ConfigParser extends Parser {++ def parse(fileName: String, source: String): ParseResult = {+ val format = detectFormat(fileName)+ val entries = format match {+ case "json" => parseJson(source)+ case "yaml" => parseYaml(source)+ case "toml" => parseToml(source)+ case _ => Vector.empty+ }++ // File-level Config node+ val configEntity = ParsedEntity(+ name = fileName,+ kind = NodeKind.Config,+ attrs = Map("format" -> Json.fromString(format)),+ lineStart = 1,+ lineEnd = source.split("\n", -1).length+ )++ // ConfigEntry nodes for each top-level key+ val entryEntities = entries.map { case (key, value, line) =>+ ParsedEntity(+ name = key,+ kind = NodeKind.ConfigEntry,+ attrs = Map(+ "value" -> Json.fromString(value),+ "format" -> Json.fromString(format)+ ),+ lineStart = line,+ lineEnd = line+ )+ }++ // Relationships: Config DEFINES each ConfigEntry+ val relationships = entries.map { case (key, _, _) =>+ ParsedRelationship(fileName, key, "DEFINES")+ }++ ParseResult(+ entities = configEntity +: entryEntities,+ relationships = relationships+ )+ }++ private def detectFormat(fileName: String): String = {+ if (fileName.endsWith(".json")) "json"+ else if (fileName.endsWith(".yaml") || fileName.endsWith(".yml")) "yaml"+ else if (fileName.endsWith(".toml")) "toml"+ else "unknown"+ }++ /** Parse JSON: extract top-level keys from objects. Simple regex approach. */+ private def parseJson(source: String): Vector[(String, String, Int)] = {+ val lines = source.split("\n", -1)+ var results = Vector.empty[(String, String, Int)]+ // Match top-level JSON object keys: "key": value+ // Only match lines at indent level 2 (1 level into the root object)+ val KeyPattern = """^\s{0,4}"(\w[\w.-]*)"\s*:\s*(.+?),?\s*$""".r+ for ((line, idx) <- lines.zipWithIndex) {+ KeyPattern.findFirstMatchIn(line).foreach { m =>+ val key = m.group(1)+ val value = m.group(2).trim.stripPrefix("\"").stripSuffix("\"").stripSuffix(",")+ results = results :+ (key, value, idx + 1)+ }+ }+ results+ }++ /** Parse YAML: extract top-level keys (no indentation). */+ private def parseYaml(source: String): Vector[(String, String, Int)] = {+ val lines = source.split("\n", -1)+ var results = Vector.empty[(String, String, Int)]+ // Top-level YAML keys start at column 0 (no indentation)+ val KeyPattern = """^([a-zA-Z_][\w.-]*)\s*:\s*(.*)$""".r+ for ((line, idx) <- lines.zipWithIndex) {+ // Skip comments and empty lines+ if (!line.trim.startsWith("#") && line.trim.nonEmpty) {+ KeyPattern.findFirstMatchIn(line).foreach { m =>+ val key = m.group(1)+ val value = m.group(2).trim+ // Skip keys that start a nested block (value is empty or just a comment)+ val cleanValue = if (value.isEmpty || value.startsWith("#")) "(block)" else value+ results = results :+ (key, cleanValue, idx + 1)+ }+ }+ }+ results+ }++ /** Parse TOML: extract top-level key-value pairs and section keys. */+ private def parseToml(source: String): Vector[(String, String, Int)] = {+ val lines = source.split("\n", -1)+ var results = Vector.empty[(String, String, Int)]+ var inSection = false // Track if we're inside a [section]+ val SectionPattern = """^\[.*\]""".r+ val KeyPattern = """^([a-zA-Z_][\w.-]*)\s*=\s*(.+)$""".r+ for ((line, idx) <- lines.zipWithIndex) {+ if (!line.trim.startsWith("#") && line.trim.nonEmpty) {+ SectionPattern.findFirstMatchIn(line).foreach { _ =>+ inSection = true+ }+ if (!inSection) {+ KeyPattern.findFirstMatchIn(line).foreach { m =>+ val key = m.group(1)+ val value = m.group(2).trim.stripPrefix("\"").stripSuffix("\"")+ results = results :+ (key, value, idx + 1)+ }+ }+ // Also capture section-level keys as section.key+ if (inSection) {+ KeyPattern.findFirstMatchIn(line).foreach { m =>+ val key = m.group(1)+ val value = m.group(2).trim.stripPrefix("\"").stripSuffix("\"")+ results = results :+ (key, value, idx + 1)+ }+ }+ }+ }+ results+ }+}diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/parsers/ConfigParserSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/ConfigParserSpec.scalanew file mode 100644index 00000000..5a8c5980--- /dev/null+++ b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/ConfigParserSpec.scala@@ -0,0 +1,118 @@+package ix.memory.ingestion.parsers++import org.scalatest.flatspec.AnyFlatSpec+import org.scalatest.matchers.should.Matchers+import ix.memory.model.NodeKind++class ConfigParserSpec extends AnyFlatSpec with Matchers {++ val parser = new ConfigParser()++ // Test 1: JSON — creates Config entity for file+ "ConfigParser" should "create a Config entity for a JSON file" in {+ val source = """{ "name": "my-app", "version": "1.0.0" }"""+ val result = parser.parse("package.json", source)+ result.entities.exists(e => e.name == "package.json" && e.kind == NodeKind.Config) shouldBe true+ val config = result.entities.find(_.kind == NodeKind.Config).get+ config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"json\"")+ }++ // Test 2: JSON — extracts top-level keys as ConfigEntry+ it should "extract top-level JSON keys as ConfigEntry nodes" in {+ val source = """{+ | "name": "my-app",+ | "version": "1.0.0",+ | "description": "A test app"+ |}""".stripMargin+ val result = parser.parse("package.json", source)+ result.entities.exists(e => e.name == "name" && e.kind == NodeKind.ConfigEntry) shouldBe true+ result.entities.exists(e => e.name == "version" && e.kind == NodeKind.ConfigEntry) shouldBe true+ result.entities.exists(e => e.name == "description" && e.kind == NodeKind.ConfigEntry) shouldBe true+ }++ // Test 3: JSON — DEFINES relationships+ it should "generate DEFINES relationships for JSON entries" in {+ val source = """{+ | "name": "test"+ |}""".stripMargin+ val result = parser.parse("config.json", source)+ result.relationships.exists(r => r.srcName == "config.json" && r.dstName == "name" && r.predicate == "DEFINES") shouldBe true+ }++ // Test 4: YAML — extracts top-level keys+ it should "extract top-level YAML keys" in {+ val source = """endpoint: http://localhost:8090+ |format: text+ |debug: true""".stripMargin+ val result = parser.parse("config.yaml", source)+ result.entities.exists(e => e.name == "endpoint" && e.kind == NodeKind.ConfigEntry) shouldBe true+ result.entities.exists(e => e.name == "format" && e.kind == NodeKind.ConfigEntry) shouldBe true+ result.entities.exists(e => e.name == "debug" && e.kind == NodeKind.ConfigEntry) shouldBe true+ }++ // Test 5: YAML — skips comments+ it should "skip YAML comment lines" in {+ val source = """# This is a comment+ |name: test+ |# Another comment+ |port: 8080""".stripMargin+ val result = parser.parse("config.yml", source)+ val entryNames = result.entities.filter(_.kind == NodeKind.ConfigEntry).map(_.name)+ entryNames should contain("name")+ entryNames should contain("port")+ entryNames.size shouldBe 2+ }++ // Test 6: YAML — format attr is "yaml"+ it should "set format to yaml for .yaml files" in {+ val result = parser.parse("app.yaml", "key: value")+ val config = result.entities.find(_.kind == NodeKind.Config).get+ config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"yaml\"")+ }++ // Test 7: YAML — format attr is "yaml" for .yml too+ it should "set format to yaml for .yml files" in {+ val result = parser.parse("app.yml", "key: value")+ val config = result.entities.find(_.kind == NodeKind.Config).get+ config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"yaml\"")+ }++ // Test 8: TOML — extracts top-level keys+ it should "extract top-level TOML keys" in {+ val source = """name = "my-app"+ |version = "1.0.0"+ |+ |[database]+ |host = "localhost"+ |port = 5432""".stripMargin+ val result = parser.parse("config.toml", source)+ result.entities.exists(e => e.name == "name" && e.kind == NodeKind.ConfigEntry) shouldBe true+ result.entities.exists(e => e.name == "version" && e.kind == NodeKind.ConfigEntry) shouldBe true+ // Section keys should also be extracted+ result.entities.exists(e => e.name == "host" && e.kind == NodeKind.ConfigEntry) shouldBe true+ result.entities.exists(e => e.name == "port" && e.kind == NodeKind.ConfigEntry) shouldBe true+ }++ // Test 9: TOML — format attr is "toml"+ it should "set format to toml for .toml files" in {+ val result = parser.parse("config.toml", "key = \"value\"")+ val config = result.entities.find(_.kind == NodeKind.Config).get+ config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"toml\"")+ }++ // Test 10: Empty config+ it should "handle empty config files" in {+ val result = parser.parse("empty.json", "")+ result.entities.exists(_.kind == NodeKind.Config) shouldBe true+ result.entities.count(_.kind == NodeKind.ConfigEntry) shouldBe 0+ }++ // Test 11: ConfigEntry has value attr+ it should "store the value in ConfigEntry attrs" in {+ val source = "port: 8080"+ val result = parser.parse("config.yaml", source)+ val entry = result.entities.find(e => e.name == "port" && e.kind == NodeKind.ConfigEntry)+ entry shouldBe defined+ entry.get.attrs.get("value").map(_.noSpaces) shouldBe Some("\"8080\"")+ }+}