Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: config parser for YAML, JSON, and TOML ingestion

riley0227246957d35bac
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
index 13035545..d9adfcf0 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
@@ -30,6 +30,7 @@ object GraphPatchBuilder {
val extractor = if (filePath.endsWith(".py")) "tree-sitter-python/1.0"
else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) "typescript-parser/1.0"
+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) "config-parser/1.0"
else "unknown-parser/1.0"
val nodeOps = parseResult.entities.map { e =>
@@ -70,11 +71,14 @@ object GraphPatchBuilder {
val claimOps = relationshipClaims ++ attributeClaims
val allOps = nodeOps ++ edgeOps ++ claimOps
+ val sourceType = if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) SourceType.Config
+ else SourceType.Code
+
GraphPatch(
patchId = PatchId(UUID.randomUUID()),
actor = "ix/ingestion",
timestamp = Instant.now(),
- source = PatchSource(filePath, sourceHash, extractor, SourceType.Code),
+ source = PatchSource(filePath, sourceHash, extractor, sourceType),
baseRev = Rev(0L),
ops = allOps,
replaces = Vector.empty,
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
index 366ef8e2..52f4462c 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
@@ -35,6 +35,7 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quer
// Check for existing patch from same source+extractor
extractor = if (filePath.toString.endsWith(".py")) "tree-sitter-python/1.0"
else if (filePath.toString.endsWith(".ts") || filePath.toString.endsWith(".tsx")) "typescript-parser/1.0"
+ else if (filePath.toString.endsWith(".json") || filePath.toString.endsWith(".yaml") || filePath.toString.endsWith(".yml") || filePath.toString.endsWith(".toml")) "config-parser/1.0"
else "unknown-parser/1.0"
existing <- queryApi.getPatchesBySource(filePath.toString, extractor)
prevPatch = existing.headOption
@@ -93,7 +94,8 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quer
val extensions = language match {
case Some("python") => Set(".py")
case Some("typescript") => Set(".ts", ".tsx")
- case _ => Set(".py", ".ts", ".tsx") // all supported
+ case Some("config") => Set(".json", ".yaml", ".yml", ".toml")
+ case _ => Set(".py", ".ts", ".tsx", ".json", ".yaml", ".yml", ".toml") // all supported
}
if (Files.isRegularFile(path)) {
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
index 385b8e10..c1df0f32 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
@@ -1,18 +1,20 @@
package ix.memory.ingestion
-import ix.memory.ingestion.parsers.{TreeSitterPythonParser, TypeScriptParser}
+import ix.memory.ingestion.parsers.{ConfigParser, TreeSitterPythonParser, TypeScriptParser}
/**
* Routes file paths to the appropriate language parser.
- * Supports Python (.py), TypeScript (.ts), and TSX (.tsx) files.
+ * Supports Python (.py), TypeScript (.ts/.tsx), and config files (.json/.yaml/.yml/.toml).
*/
class ParserRouter {
private val pythonParser = new TreeSitterPythonParser()
private val tsParser = new TypeScriptParser()
+ private val configParser = new ConfigParser()
def parserFor(filePath: String): Option[Parser] = {
if (filePath.endsWith(".py")) Some(pythonParser)
else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) Some(tsParser)
+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) Some(configParser)
else None
}
}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
new file mode 100644
index 00000000..552c83c2
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
@@ -0,0 +1,133 @@
+package ix.memory.ingestion.parsers
+
+import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}
+import ix.memory.model.NodeKind
+import io.circe.Json
+
+/**
+ * Config file parser for YAML, JSON, and TOML files.
+ * Extracts a Config node for the file and ConfigEntry nodes for top-level keys.
+ * Uses simple regex heuristics — not a full YAML/TOML parser.
+ */
+class ConfigParser extends Parser {
+
+ def parse(fileName: String, source: String): ParseResult = {
+ val format = detectFormat(fileName)
+ val entries = format match {
+ case "json" => parseJson(source)
+ case "yaml" => parseYaml(source)
+ case "toml" => parseToml(source)
+ case _ => Vector.empty
+ }
+
+ // File-level Config node
+ val configEntity = ParsedEntity(
+ name = fileName,
+ kind = NodeKind.Config,
+ attrs = Map("format" -> Json.fromString(format)),
+ lineStart = 1,
+ lineEnd = source.split("\n", -1).length
+ )
+
+ // ConfigEntry nodes for each top-level key
+ val entryEntities = entries.map { case (key, value, line) =>
+ ParsedEntity(
+ name = key,
+ kind = NodeKind.ConfigEntry,
+ attrs = Map(
+ "value" -> Json.fromString(value),
+ "format" -> Json.fromString(format)
+ ),
+ lineStart = line,
+ lineEnd = line
+ )
+ }
+
+ // Relationships: Config DEFINES each ConfigEntry
+ val relationships = entries.map { case (key, _, _) =>
+ ParsedRelationship(fileName, key, "DEFINES")
+ }
+
+ ParseResult(
+ entities = configEntity +: entryEntities,
+ relationships = relationships
+ )
+ }
+
+ private def detectFormat(fileName: String): String = {
+ if (fileName.endsWith(".json")) "json"
+ else if (fileName.endsWith(".yaml") || fileName.endsWith(".yml")) "yaml"
+ else if (fileName.endsWith(".toml")) "toml"
+ else "unknown"
+ }
+
+ /** Parse JSON: extract top-level keys from objects. Simple regex approach. */
+ private def parseJson(source: String): Vector[(String, String, Int)] = {
+ val lines = source.split("\n", -1)
+ var results = Vector.empty[(String, String, Int)]
+ // Match top-level JSON object keys: "key": value
+ // Only match lines at indent level 2 (1 level into the root object)
+ val KeyPattern = """^\s{0,4}"(\w[\w.-]*)"\s*:\s*(.+?),?\s*$""".r
+ for ((line, idx) <- lines.zipWithIndex) {
+ KeyPattern.findFirstMatchIn(line).foreach { m =>
+ val key = m.group(1)
+ val value = m.group(2).trim.stripPrefix("\"").stripSuffix("\"").stripSuffix(",")
+ results = results :+ (key, value, idx + 1)
+ }
+ }
+ results
+ }
+
+ /** Parse YAML: extract top-level keys (no indentation). */
+ private def parseYaml(source: String): Vector[(String, String, Int)] = {
+ val lines = source.split("\n", -1)
+ var results = Vector.empty[(String, String, Int)]
+ // Top-level YAML keys start at column 0 (no indentation)
+ val KeyPattern = """^([a-zA-Z_][\w.-]*)\s*:\s*(.*)$""".r
+ for ((line, idx) <- lines.zipWithIndex) {
+ // Skip comments and empty lines
+ if (!line.trim.startsWith("#") && line.trim.nonEmpty) {
+ KeyPattern.findFirstMatchIn(line).foreach { m =>
+ val key = m.group(1)
+ val value = m.group(2).trim
+ // Skip keys that start a nested block (value is empty or just a comment)
+ val cleanValue = if (value.isEmpty || value.startsWith("#")) "(block)" else value
+ results = results :+ (key, cleanValue, idx + 1)
+ }
+ }
+ }
+ results
+ }
+
+ /** Parse TOML: extract top-level key-value pairs and section keys. */
+ private def parseToml(source: String): Vector[(String, String, Int)] = {
+ val lines = source.split("\n", -1)
+ var results = Vector.empty[(String, String, Int)]
+ var inSection = false // Track if we're inside a [section]
+ val SectionPattern = """^\[.*\]""".r
+ val KeyPattern = """^([a-zA-Z_][\w.-]*)\s*=\s*(.+)$""".r
+ for ((line, idx) <- lines.zipWithIndex) {
+ if (!line.trim.startsWith("#") && line.trim.nonEmpty) {
+ SectionPattern.findFirstMatchIn(line).foreach { _ =>
+ inSection = true
+ }
+ if (!inSection) {
+ KeyPattern.findFirstMatchIn(line).foreach { m =>
+ val key = m.group(1)
+ val value = m.group(2).trim.stripPrefix("\"").stripSuffix("\"")
+ results = results :+ (key, value, idx + 1)
+ }
+ }
+ // Also capture section-level keys as section.key
+ if (inSection) {
+ KeyPattern.findFirstMatchIn(line).foreach { m =>
+ val key = m.group(1)
+ val value = m.group(2).trim.stripPrefix("\"").stripSuffix("\"")
+ results = results :+ (key, value, idx + 1)
+ }
+ }
+ }
+ }
+ results
+ }
+}
diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/parsers/ConfigParserSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/ConfigParserSpec.scala
new file mode 100644
index 00000000..5a8c5980
--- /dev/null
+++ b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/ConfigParserSpec.scala
@@ -0,0 +1,118 @@
+package ix.memory.ingestion.parsers
+
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+import ix.memory.model.NodeKind
+
+class ConfigParserSpec extends AnyFlatSpec with Matchers {
+
+ val parser = new ConfigParser()
+
+ // Test 1: JSON — creates Config entity for file
+ "ConfigParser" should "create a Config entity for a JSON file" in {
+ val source = """{ "name": "my-app", "version": "1.0.0" }"""
+ val result = parser.parse("package.json", source)
+ result.entities.exists(e => e.name == "package.json" && e.kind == NodeKind.Config) shouldBe true
+ val config = result.entities.find(_.kind == NodeKind.Config).get
+ config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"json\"")
+ }
+
+ // Test 2: JSON — extracts top-level keys as ConfigEntry
+ it should "extract top-level JSON keys as ConfigEntry nodes" in {
+ val source = """{
+ | "name": "my-app",
+ | "version": "1.0.0",
+ | "description": "A test app"
+ |}""".stripMargin
+ val result = parser.parse("package.json", source)
+ result.entities.exists(e => e.name == "name" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ result.entities.exists(e => e.name == "version" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ result.entities.exists(e => e.name == "description" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ }
+
+ // Test 3: JSON — DEFINES relationships
+ it should "generate DEFINES relationships for JSON entries" in {
+ val source = """{
+ | "name": "test"
+ |}""".stripMargin
+ val result = parser.parse("config.json", source)
+ result.relationships.exists(r => r.srcName == "config.json" && r.dstName == "name" && r.predicate == "DEFINES") shouldBe true
+ }
+
+ // Test 4: YAML — extracts top-level keys
+ it should "extract top-level YAML keys" in {
+ val source = """endpoint: http://localhost:8090
+ |format: text
+ |debug: true""".stripMargin
+ val result = parser.parse("config.yaml", source)
+ result.entities.exists(e => e.name == "endpoint" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ result.entities.exists(e => e.name == "format" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ result.entities.exists(e => e.name == "debug" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ }
+
+ // Test 5: YAML — skips comments
+ it should "skip YAML comment lines" in {
+ val source = """# This is a comment
+ |name: test
+ |# Another comment
+ |port: 8080""".stripMargin
+ val result = parser.parse("config.yml", source)
+ val entryNames = result.entities.filter(_.kind == NodeKind.ConfigEntry).map(_.name)
+ entryNames should contain("name")
+ entryNames should contain("port")
+ entryNames.size shouldBe 2
+ }
+
+ // Test 6: YAML — format attr is "yaml"
+ it should "set format to yaml for .yaml files" in {
+ val result = parser.parse("app.yaml", "key: value")
+ val config = result.entities.find(_.kind == NodeKind.Config).get
+ config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"yaml\"")
+ }
+
+ // Test 7: YAML — format attr is "yaml" for .yml too
+ it should "set format to yaml for .yml files" in {
+ val result = parser.parse("app.yml", "key: value")
+ val config = result.entities.find(_.kind == NodeKind.Config).get
+ config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"yaml\"")
+ }
+
+ // Test 8: TOML — extracts top-level keys
+ it should "extract top-level TOML keys" in {
+ val source = """name = "my-app"
+ |version = "1.0.0"
+ |
+ |[database]
+ |host = "localhost"
+ |port = 5432""".stripMargin
+ val result = parser.parse("config.toml", source)
+ result.entities.exists(e => e.name == "name" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ result.entities.exists(e => e.name == "version" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ // Section keys should also be extracted
+ result.entities.exists(e => e.name == "host" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ result.entities.exists(e => e.name == "port" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ }
+
+ // Test 9: TOML — format attr is "toml"
+ it should "set format to toml for .toml files" in {
+ val result = parser.parse("config.toml", "key = \"value\"")
+ val config = result.entities.find(_.kind == NodeKind.Config).get
+ config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"toml\"")
+ }
+
+ // Test 10: Empty config
+ it should "handle empty config files" in {
+ val result = parser.parse("empty.json", "")
+ result.entities.exists(_.kind == NodeKind.Config) shouldBe true
+ result.entities.count(_.kind == NodeKind.ConfigEntry) shouldBe 0
+ }
+
+ // Test 11: ConfigEntry has value attr
+ it should "store the value in ConfigEntry attrs" in {
+ val source = "port: 8080"
+ val result = parser.parse("config.yaml", source)
+ val entry = result.entities.find(e => e.name == "port" && e.kind == NodeKind.ConfigEntry)
+ entry shouldBe defined
+ entry.get.attrs.get("value").map(_.noSpaces) shouldBe Some("\"8080\"")
+ }
+}