Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat: markdown parser for ingestion pipeline (.md)
riley0227ecde2798b985
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scalaindex d9adfcf0..a267fb0d 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/GraphPatchBuilder.scala@@ -31,6 +31,7 @@ object GraphPatchBuilder {val extractor = if (filePath.endsWith(".py")) "tree-sitter-python/1.0"else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) "typescript-parser/1.0"else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) "config-parser/1.0"+ else if (filePath.endsWith(".md")) "markdown-parser/1.0"else "unknown-parser/1.0"val nodeOps = parseResult.entities.map { e =>@@ -71,7 +72,9 @@ object GraphPatchBuilder {val claimOps = relationshipClaims ++ attributeClaimsval allOps = nodeOps ++ edgeOps ++ claimOps- val sourceType = if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) SourceType.Config+ val sourceType = if (filePath.endsWith(".py") || filePath.endsWith(".ts") || filePath.endsWith(".tsx")) SourceType.Code+ else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) SourceType.Config+ else if (filePath.endsWith(".md")) SourceType.Docelse SourceType.CodeGraphPatch(diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scalaindex 52f4462c..49200a64 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/IngestionService.scala@@ -36,6 +36,7 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, querextractor = if (filePath.toString.endsWith(".py")) "tree-sitter-python/1.0"else if (filePath.toString.endsWith(".ts") || filePath.toString.endsWith(".tsx")) "typescript-parser/1.0"else if (filePath.toString.endsWith(".json") || filePath.toString.endsWith(".yaml") || filePath.toString.endsWith(".yml") || filePath.toString.endsWith(".toml")) "config-parser/1.0"+ else if (filePath.toString.endsWith(".md")) "markdown-parser/1.0"else "unknown-parser/1.0"existing <- queryApi.getPatchesBySource(filePath.toString, extractor)prevPatch = existing.headOption@@ -95,7 +96,8 @@ class IngestionService(parserRouter: ParserRouter, writeApi: GraphWriteApi, quercase Some("python") => Set(".py")case Some("typescript") => Set(".ts", ".tsx")case Some("config") => Set(".json", ".yaml", ".yml", ".toml")- case _ => Set(".py", ".ts", ".tsx", ".json", ".yaml", ".yml", ".toml") // all supported+ case Some("markdown") => Set(".md")+ case _ => Set(".py", ".ts", ".tsx", ".json", ".yaml", ".yml", ".toml", ".md") // all supported}if (Files.isRegularFile(path)) {diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scalaindex c1df0f32..5df0539c 100644--- a/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala+++ b/memory-layer/src/main/scala/ix/memory/ingestion/ParserRouter.scala@@ -1,20 +1,23 @@package ix.memory.ingestion-import ix.memory.ingestion.parsers.{ConfigParser, TreeSitterPythonParser, TypeScriptParser}+import ix.memory.ingestion.parsers.{ConfigParser, MarkdownParser, TreeSitterPythonParser, TypeScriptParser}/*** Routes file paths to the appropriate language parser.- * Supports Python (.py), TypeScript (.ts/.tsx), and config files (.json/.yaml/.yml/.toml).+ * Supports Python (.py), TypeScript (.ts/.tsx), config files (.json/.yaml/.yml/.toml),+ * and Markdown (.md).*/class ParserRouter {- private val pythonParser = new TreeSitterPythonParser()- private val tsParser = new TypeScriptParser()- private val configParser = new ConfigParser()+ private val pythonParser = new TreeSitterPythonParser()+ private val tsParser = new TypeScriptParser()+ private val configParser = new ConfigParser()+ private val markdownParser = new MarkdownParser()def parserFor(filePath: String): Option[Parser] = {if (filePath.endsWith(".py")) Some(pythonParser)else if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) Some(tsParser)else if (filePath.endsWith(".json") || filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".toml")) Some(configParser)+ else if (filePath.endsWith(".md")) Some(markdownParser)else None}}diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scalanew file mode 100644index 00000000..270ef04d--- /dev/null+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala@@ -0,0 +1,97 @@+package ix.memory.ingestion.parsers++import ix.memory.ingestion.{Parser, ParseResult, ParsedEntity, ParsedRelationship}+import ix.memory.model.NodeKind+import io.circe.Json++import scala.util.matching.Regex++/**+ * Markdown parser that extracts document structure from .md files.+ * Each header becomes a Doc node. Sections are connected by DEFINES relationships+ * based on heading hierarchy (## is defined by #, ### is defined by ##).+ */+class MarkdownParser extends Parser {++ private val HeaderPattern: Regex = """^(#{1,6})\s+(.+)$""".r++ def parse(fileName: String, source: String): ParseResult = {+ val lines = source.split("\n", -1)++ // File-level Doc node+ val fileEntity = ParsedEntity(+ name = fileName,+ kind = NodeKind.File,+ attrs = Map("language" -> Json.fromString("markdown")),+ lineStart = 1,+ lineEnd = lines.length+ )++ var entities = Vector(fileEntity)+ var relationships = Vector.empty[ParsedRelationship]++ // Track heading hierarchy for parent-child relationships+ // Stack of (level, name) — most recent heading at each level+ var headingStack = Vector.empty[(Int, String)]++ for ((line, idx) <- lines.zipWithIndex) {+ val lineNum = idx + 1++ HeaderPattern.findFirstMatchIn(line).foreach { m =>+ val level = m.group(1).length // 1 for #, 2 for ##, etc.+ val title = m.group(2).trim++ // Find the end of this section (next heading of same or higher level)+ val sectionEnd = findSectionEnd(lines, idx, level)++ // Extract a content summary (first non-empty, non-header line after the heading)+ val contentSummary = extractSummary(lines, idx + 1, sectionEnd)++ val attrs = Map(+ "level" -> Json.fromInt(level),+ "content" -> Json.fromString(contentSummary)+ )++ entities = entities :+ ParsedEntity(+ name = title,+ kind = NodeKind.Doc,+ attrs = attrs,+ lineStart = lineNum,+ lineEnd = sectionEnd+ )++ // Determine parent: find nearest heading with lower level number+ headingStack = headingStack.filter(_._1 < level)+ val parent = headingStack.lastOption.map(_._2).getOrElse(fileName)++ relationships = relationships :+ ParsedRelationship(parent, title, "DEFINES")++ headingStack = headingStack :+ (level, title)+ }+ }++ ParseResult(entities, relationships)+ }++ /** Find the end line of a section: the next heading line (any level). Returns 1-based line number. */+ private def findSectionEnd(lines: Array[String], startIdx: Int, level: Int): Int = {+ var i = startIdx + 1+ while (i < lines.length) {+ if (HeaderPattern.findFirstMatchIn(lines(i)).isDefined) {+ return i + 1 // Convert 0-indexed to 1-based line number+ }+ i += 1+ }+ lines.length // Section extends to end of file+ }++ /** Extract a summary from the first paragraph after a heading (up to 200 chars). */+ private def extractSummary(lines: Array[String], startIdx: Int, endIdx: Int): String = {+ val contentLines = lines.slice(startIdx, endIdx)+ .map(_.trim)+ .filter(l => l.nonEmpty && !l.startsWith("#"))+ .take(3) // Take first 3 content lines for summary+ val summary = contentLines.mkString(" ").take(200)+ summary+ }+}diff --git a/memory-layer/src/main/scala/ix/memory/model/Node.scala b/memory-layer/src/main/scala/ix/memory/model/Node.scalaindex d3ad52e7..360cf677 100644--- a/memory-layer/src/main/scala/ix/memory/model/Node.scala+++ b/memory-layer/src/main/scala/ix/memory/model/Node.scala@@ -18,6 +18,7 @@ object NodeKind {case object Endpoint extends NodeKindcase object Intent extends NodeKindcase object Decision extends NodeKind+ case object Doc extends NodeKindprivate val nameMap: Map[String, NodeKind] = Map("module" -> Module,@@ -30,7 +31,8 @@ object NodeKind {"service" -> Service,"endpoint" -> Endpoint,"intent" -> Intent,- "decision" -> Decision+ "decision" -> Decision,+ "doc" -> Doc)implicit val encoder: Encoder[NodeKind] = Encoder[String].contramap {@@ -45,6 +47,7 @@ object NodeKind {case Endpoint => "endpoint"case Intent => "intent"case Decision => "decision"+ case Doc => "doc"}implicit val decoder: Decoder[NodeKind] = Decoder[String].emap { s =>diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/parsers/MarkdownParserSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/MarkdownParserSpec.scalanew file mode 100644index 00000000..fee9c09f--- /dev/null+++ b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/MarkdownParserSpec.scala@@ -0,0 +1,114 @@+package ix.memory.ingestion.parsers++import org.scalatest.flatspec.AnyFlatSpec+import org.scalatest.matchers.should.Matchers+import ix.memory.model.NodeKind++class MarkdownParserSpec extends AnyFlatSpec with Matchers {++ val parser = new MarkdownParser()++ // Test 1: Creates File entity+ "MarkdownParser" should "create a File entity for the markdown file" in {+ val result = parser.parse("README.md", "# Hello")+ result.entities.exists(e => e.name == "README.md" && e.kind == NodeKind.File) shouldBe true+ val file = result.entities.find(_.kind == NodeKind.File).get+ file.attrs.get("language").map(_.noSpaces) shouldBe Some("\"markdown\"")+ }++ // Test 2: Extracts headers as Doc nodes+ it should "extract headers as Doc nodes" in {+ val source = """# Overview+ |Some intro text.+ |+ |## Architecture+ |Details about architecture.+ |+ |## API Design+ |Details about API.""".stripMargin+ val result = parser.parse("design.md", source)+ result.entities.exists(e => e.name == "Overview" && e.kind == NodeKind.Doc) shouldBe true+ result.entities.exists(e => e.name == "Architecture" && e.kind == NodeKind.Doc) shouldBe true+ result.entities.exists(e => e.name == "API Design" && e.kind == NodeKind.Doc) shouldBe true+ }++ // Test 3: Hierarchy — ## under # creates DEFINES relationship+ it should "create DEFINES relationships based on heading hierarchy" in {+ val source = """# Top+ |## Sub+ |### SubSub""".stripMargin+ val result = parser.parse("doc.md", source)+ // File DEFINES Top (# is top-level)+ result.relationships.exists(r => r.srcName == "doc.md" && r.dstName == "Top" && r.predicate == "DEFINES") shouldBe true+ // Top DEFINES Sub+ result.relationships.exists(r => r.srcName == "Top" && r.dstName == "Sub" && r.predicate == "DEFINES") shouldBe true+ // Sub DEFINES SubSub+ result.relationships.exists(r => r.srcName == "Sub" && r.dstName == "SubSub" && r.predicate == "DEFINES") shouldBe true+ }++ // Test 4: Level attr is correct+ it should "store the heading level in attrs" in {+ val source = """# Level 1+ |## Level 2+ |### Level 3""".stripMargin+ val result = parser.parse("doc.md", source)+ val h1 = result.entities.find(_.name == "Level 1").get+ val h2 = result.entities.find(_.name == "Level 2").get+ val h3 = result.entities.find(_.name == "Level 3").get+ h1.attrs.get("level").flatMap(_.asNumber).flatMap(_.toInt) shouldBe Some(1)+ h2.attrs.get("level").flatMap(_.asNumber).flatMap(_.toInt) shouldBe Some(2)+ h3.attrs.get("level").flatMap(_.asNumber).flatMap(_.toInt) shouldBe Some(3)+ }++ // Test 5: Content summary extracted+ it should "extract content summary from section body" in {+ val source = """# Introduction+ |This is the introduction to the project.+ |It explains the main goals.""".stripMargin+ val result = parser.parse("readme.md", source)+ val intro = result.entities.find(_.name == "Introduction").get+ val content = intro.attrs.get("content").flatMap(_.asString).getOrElse("")+ content should include("introduction")+ }++ // Test 6: Empty markdown+ it should "handle empty markdown files" in {+ val result = parser.parse("empty.md", "")+ result.entities.exists(_.kind == NodeKind.File) shouldBe true+ result.entities.count(_.kind == NodeKind.Doc) shouldBe 0+ }++ // Test 7: Markdown with no headers+ it should "handle markdown with no headers" in {+ val source = "Just some text with no headers."+ val result = parser.parse("notes.md", source)+ result.entities.count(_.kind == NodeKind.Doc) shouldBe 0+ result.relationships shouldBe empty+ }++ // Test 8: Multiple sections at same level+ it should "handle sibling sections at the same level" in {+ val source = """# Main+ |## Section A+ |Content A+ |## Section B+ |Content B""".stripMargin+ val result = parser.parse("doc.md", source)+ // Both should be DEFINES'd by Main+ result.relationships.exists(r => r.srcName == "Main" && r.dstName == "Section A") shouldBe true+ result.relationships.exists(r => r.srcName == "Main" && r.dstName == "Section B") shouldBe true+ }++ // Test 9: Section line ranges+ it should "compute correct line ranges for sections" in {+ val source = """# Intro+ |Line 2+ |Line 3+ |## Details+ |Line 5""".stripMargin+ val result = parser.parse("doc.md", source)+ val intro = result.entities.find(_.name == "Intro").get+ intro.lineStart shouldBe 1+ intro.lineEnd shouldBe 4 // Ends at line before ## Details+ }+}