Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: markdown parser for ingestion pipeline (.md)

riley0227ecde2798b985
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
index d9adfcf0..a267fb0d 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala
@@ -31,6 +31,7 @@ object GraphPatchBuilder {
val extractor = if (filePath.endsWith(".py")) "tree-sitter-python/1.0"
else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) "typescript-parser/1.0"
else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) "config-parser/1.0"
+ else if (filePath.endsWith(".md")) "markdown-parser/1.0"
else "unknown-parser/1.0"
val nodeOps = parseResult.entities.map { e =>
@@ -71,7 +72,9 @@ object GraphPatchBuilder {
val claimOps = relationshipClaims ++ attributeClaims
val allOps = nodeOps ++ edgeOps ++ claimOps
- val sourceType = if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) SourceType.Config
+ val sourceType = if (filePath.endsWith(".py") || filePath.endsWith(".ts") || filePath.endsWith(".tsx")) SourceType.Code
+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) SourceType.Config
+ else if (filePath.endsWith(".md")) SourceType.Doc
else SourceType.Code
GraphPatch(
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
index 52f4462c..49200a64 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala
@@ -36,6 +36,7 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quer
extractor = if (filePath.toString.endsWith(".py")) "tree-sitter-python/1.0"
else if (filePath.toString.endsWith(".ts") || filePath.toString.endsWith(".tsx")) "typescript-parser/1.0"
else if (filePath.toString.endsWith(".json") || filePath.toString.endsWith(".yaml") || filePath.toString.endsWith(".yml") || filePath.toString.endsWith(".toml")) "config-parser/1.0"
+ else if (filePath.toString.endsWith(".md")) "markdown-parser/1.0"
else "unknown-parser/1.0"
existing <- queryApi.getPatchesBySource(filePath.toString, extractor)
prevPatch = existing.headOption
@@ -95,7 +96,8 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quer
case Some("python") => Set(".py")
case Some("typescript") => Set(".ts", ".tsx")
case Some("config") => Set(".json", ".yaml", ".yml", ".toml")
- case _ => Set(".py", ".ts", ".tsx", ".json", ".yaml", ".yml", ".toml") // all supported
+ case Some("markdown") => Set(".md")
+ case _ => Set(".py", ".ts", ".tsx", ".json", ".yaml", ".yml", ".toml", ".md") // all supported
}
if (Files.isRegularFile(path)) {
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
index c1df0f32..5df0539c 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala
@@ -1,20 +1,23 @@
package ix.memory.ingestion
-import ix.memory.ingestion.parsers.{ConfigParser, TreeSitterPythonParser, TypeScriptParser}
+import ix.memory.ingestion.parsers.{ConfigParser, MarkdownParser, TreeSitterPythonParser, TypeScriptParser}
/**
* Routes file paths to the appropriate language parser.
- * Supports Python (.py), TypeScript (.ts/.tsx), and config files (.json/.yaml/.yml/.toml).
+ * Supports Python (.py), TypeScript (.ts/.tsx), config files (.json/.yaml/.yml/.toml),
+ * and Markdown (.md).
*/
class ParserRouter {
- private val pythonParser = new TreeSitterPythonParser()
- private val tsParser = new TypeScriptParser()
- private val configParser = new ConfigParser()
+ private val pythonParser = new TreeSitterPythonParser()
+ private val tsParser = new TypeScriptParser()
+ private val configParser = new ConfigParser()
+ private val markdownParser = new MarkdownParser()
def parserFor(filePath: String): Option[Parser] = {
if (filePath.endsWith(".py")) Some(pythonParser)
else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) Some(tsParser)
else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) Some(configParser)
+ else if (filePath.endsWith(".md")) Some(markdownParser)
else None
}
}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
new file mode 100644
index 00000000..270ef04d
--- /dev/null
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
@@ -0,0 +1,97 @@
+package ix.memory.ingestion.parsers
+
+import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}
+import ix.memory.model.NodeKind
+import io.circe.Json
+
+import scala.util.matching.Regex
+
+/**
+ * Markdown parser that extracts document structure from .md files.
+ * Each header becomes a Doc node. Sections are connected by DEFINES relationships
+ * based on heading hierarchy (## is defined by #, ### is defined by ##).
+ */
+class MarkdownParser extends Parser {
+
+ private val HeaderPattern: Regex = """^(#{1,6})\s+(.+)$""".r
+
+ def parse(fileName: String, source: String): ParseResult = {
+ val lines = source.split("\n", -1)
+
+ // File-level Doc node
+ val fileEntity = ParsedEntity(
+ name = fileName,
+ kind = NodeKind.File,
+ attrs = Map("language" -> Json.fromString("markdown")),
+ lineStart = 1,
+ lineEnd = lines.length
+ )
+
+ var entities = Vector(fileEntity)
+ var relationships = Vector.empty[ParsedRelationship]
+
+ // Track heading hierarchy for parent-child relationships
+ // Stack of (level, name) — most recent heading at each level
+ var headingStack = Vector.empty[(Int, String)]
+
+ for ((line, idx) <- lines.zipWithIndex) {
+ val lineNum = idx + 1
+
+ HeaderPattern.findFirstMatchIn(line).foreach { m =>
+ val level = m.group(1).length // 1 for #, 2 for ##, etc.
+ val title = m.group(2).trim
+
+ // Find the end of this section (next heading of same or higher level)
+ val sectionEnd = findSectionEnd(lines, idx, level)
+
+ // Extract a content summary (first non-empty, non-header line after the heading)
+ val contentSummary = extractSummary(lines, idx + 1, sectionEnd)
+
+ val attrs = Map(
+ "level" -> Json.fromInt(level),
+ "content" -> Json.fromString(contentSummary)
+ )
+
+ entities = entities :+ ParsedEntity(
+ name = title,
+ kind = NodeKind.Doc,
+ attrs = attrs,
+ lineStart = lineNum,
+ lineEnd = sectionEnd
+ )
+
+ // Determine parent: find nearest heading with lower level number
+ headingStack = headingStack.filter(_._1 < level)
+ val parent = headingStack.lastOption.map(_._2).getOrElse(fileName)
+
+ relationships = relationships :+ ParsedRelationship(parent, title, "DEFINES")
+
+ headingStack = headingStack :+ (level, title)
+ }
+ }
+
+ ParseResult(entities, relationships)
+ }
+
+ /** Find the end line of a section: the next heading line (any level). Returns 1-based line number. */
+ private def findSectionEnd(lines: Array[String], startIdx: Int, level: Int): Int = {
+ var i = startIdx + 1
+ while (i < lines.length) {
+ if (HeaderPattern.findFirstMatchIn(lines(i)).isDefined) {
+ return i + 1 // Convert 0-indexed to 1-based line number
+ }
+ i += 1
+ }
+ lines.length // Section extends to end of file
+ }
+
+ /** Extract a summary from the first paragraph after a heading (up to 200 chars). */
+ private def extractSummary(lines: Array[String], startIdx: Int, endIdx: Int): String = {
+ val contentLines = lines.slice(startIdx, endIdx)
+ .map(_.trim)
+ .filter(l => l.nonEmpty && !l.startsWith("#"))
+ .take(3) // Take first 3 content lines for summary
+ val summary = contentLines.mkString(" ").take(200)
+ summary
+ }
+}
diff --git a/memory-layer/src/main/scala/ix/memory/model/Node.scala b/memory-layer/src/main/scala/ix/memory/model/Node.scala
index d3ad52e7..360cf677 100644
--- a/memory-layer/src/main/scala/ix/memory/model/Node.scala
+++ b/memory-layer/src/main/scala/ix/memory/model/Node.scala
@@ -18,6 +18,7 @@ object NodeKind {
case object Endpoint extends NodeKind
case object Intent extends NodeKind
case object Decision extends NodeKind
+ case object Doc extends NodeKind
private val nameMap: Map[String, NodeKind] = Map(
"module" -> Module,
@@ -30,7 +31,8 @@ object NodeKind {
"service" -> Service,
"endpoint" -> Endpoint,
"intent" -> Intent,
- "decision" -> Decision
+ "decision" -> Decision,
+ "doc" -> Doc
)
implicit val encoder: Encoder[NodeKind] = Encoder[String].contramap {
@@ -45,6 +47,7 @@ object NodeKind {
case Endpoint => "endpoint"
case Intent => "intent"
case Decision => "decision"
+ case Doc => "doc"
}
implicit val decoder: Decoder[NodeKind] = Decoder[String].emap { s =>
diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/parsers/MarkdownParserSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/MarkdownParserSpec.scala
new file mode 100644
index 00000000..fee9c09f
--- /dev/null
+++ b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/MarkdownParserSpec.scala
@@ -0,0 +1,114 @@
+package ix.memory.ingestion.parsers
+
+import org.scalatest.flatspec.AnyFlatSpec
+import org.scalatest.matchers.should.Matchers
+import ix.memory.model.NodeKind
+
+class MarkdownParserSpec extends AnyFlatSpec with Matchers {
+
+ val parser = new MarkdownParser()
+
+ // Test 1: Creates File entity
+ "MarkdownParser" should "create a File entity for the markdown file" in {
+ val result = parser.parse("README.md", "# Hello")
+ result.entities.exists(e => e.name == "README.md" && e.kind == NodeKind.File) shouldBe true
+ val file = result.entities.find(_.kind == NodeKind.File).get
+ file.attrs.get("language").map(_.noSpaces) shouldBe Some("\"markdown\"")
+ }
+
+ // Test 2: Extracts headers as Doc nodes
+ it should "extract headers as Doc nodes" in {
+ val source = """# Overview
+ |Some intro text.
+ |
+ |## Architecture
+ |Details about architecture.
+ |
+ |## API Design
+ |Details about API.""".stripMargin
+ val result = parser.parse("design.md", source)
+ result.entities.exists(e => e.name == "Overview" && e.kind == NodeKind.Doc) shouldBe true
+ result.entities.exists(e => e.name == "Architecture" && e.kind == NodeKind.Doc) shouldBe true
+ result.entities.exists(e => e.name == "API Design" && e.kind == NodeKind.Doc) shouldBe true
+ }
+
+ // Test 3: Hierarchy — ## under # creates DEFINES relationship
+ it should "create DEFINES relationships based on heading hierarchy" in {
+ val source = """# Top
+ |## Sub
+ |### SubSub""".stripMargin
+ val result = parser.parse("doc.md", source)
+ // File DEFINES Top (# is top-level)
+ result.relationships.exists(r => r.srcName == "doc.md" && r.dstName == "Top" && r.predicate == "DEFINES") shouldBe true
+ // Top DEFINES Sub
+ result.relationships.exists(r => r.srcName == "Top" && r.dstName == "Sub" && r.predicate == "DEFINES") shouldBe true
+ // Sub DEFINES SubSub
+ result.relationships.exists(r => r.srcName == "Sub" && r.dstName == "SubSub" && r.predicate == "DEFINES") shouldBe true
+ }
+
+ // Test 4: Level attr is correct
+ it should "store the heading level in attrs" in {
+ val source = """# Level 1
+ |## Level 2
+ |### Level 3""".stripMargin
+ val result = parser.parse("doc.md", source)
+ val h1 = result.entities.find(_.name == "Level 1").get
+ val h2 = result.entities.find(_.name == "Level 2").get
+ val h3 = result.entities.find(_.name == "Level 3").get
+ h1.attrs.get("level").flatMap(_.asNumber).flatMap(_.toInt) shouldBe Some(1)
+ h2.attrs.get("level").flatMap(_.asNumber).flatMap(_.toInt) shouldBe Some(2)
+ h3.attrs.get("level").flatMap(_.asNumber).flatMap(_.toInt) shouldBe Some(3)
+ }
+
+ // Test 5: Content summary extracted
+ it should "extract content summary from section body" in {
+ val source = """# Introduction
+ |This is the introduction to the project.
+ |It explains the main goals.""".stripMargin
+ val result = parser.parse("readme.md", source)
+ val intro = result.entities.find(_.name == "Introduction").get
+ val content = intro.attrs.get("content").flatMap(_.asString).getOrElse("")
+ content should include("introduction")
+ }
+
+ // Test 6: Empty markdown
+ it should "handle empty markdown files" in {
+ val result = parser.parse("empty.md", "")
+ result.entities.exists(_.kind == NodeKind.File) shouldBe true
+ result.entities.count(_.kind == NodeKind.Doc) shouldBe 0
+ }
+
+ // Test 7: Markdown with no headers
+ it should "handle markdown with no headers" in {
+ val source = "Just some text with no headers."
+ val result = parser.parse("notes.md", source)
+ result.entities.count(_.kind == NodeKind.Doc) shouldBe 0
+ result.relationships shouldBe empty
+ }
+
+ // Test 8: Multiple sections at same level
+ it should "handle sibling sections at the same level" in {
+ val source = """# Main
+ |## Section A
+ |Content A
+ |## Section B
+ |Content B""".stripMargin
+ val result = parser.parse("doc.md", source)
+ // Both should be DEFINES'd by Main
+ result.relationships.exists(r => r.srcName == "Main" && r.dstName == "Section A") shouldBe true
+ result.relationships.exists(r => r.srcName == "Main" && r.dstName == "Section B") shouldBe true
+ }
+
+ // Test 9: Section line ranges
+ it should "compute correct line ranges for sections" in {
+ val source = """# Intro
+ |Line 2
+ |Line 3
+ |## Details
+ |Line 5""".stripMargin
+ val result = parser.parse("doc.md", source)
+ val intro = result.entities.find(_.name == "Intro").get
+ intro.lineStart shouldBe 1
+ intro.lineEnd shouldBe 4 // Ends at line before ## Details
+ }
+}