Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat: improve TS/Markdown/Config parsers — summaries, section content, dot-path flattening

riley02278bb61e7a6cdb
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
index 552c83c2..5978f604 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/ConfigParser.scala
@@ -5,45 +5,63 @@ import ix.memory.model.NodeKind
import io.circe.Json
/**
- * Config file parser for YAML, JSON, and TOML files.
- * Extracts a Config node for the file and ConfigEntry nodes for top-level keys.
- * Uses simple regex heuristics — not a full YAML/TOML parser.
+ * Parser for configuration files (JSON, YAML, TOML).
+ *
+ * Flattens nested structures into dot-path entries so that each leaf value
+ * becomes a ConfigEntry node with a meaningful value (never "{" or "[").
*/
class ConfigParser extends Parser {
+ private val JsonExtensions = Set(".json")
+ private val YamlExtensions = Set(".yaml", ".yml")
+ private val TomlExtensions = Set(".toml")
+
+ def canParse(fileName: String): Boolean = {
+ val lower = fileName.toLowerCase
+ (JsonExtensions ++ YamlExtensions ++ TomlExtensions).exists(ext => lower.endsWith(ext))
+ }
+
def parse(fileName: String, source: String): ParseResult = {
- val format = detectFormat(fileName)
- val entries = format match {
- case "json" => parseJson(source)
- case "yaml" => parseYaml(source)
- case "toml" => parseToml(source)
- case _ => Vector.empty
- }
+ val lower = fileName.toLowerCase
+
+ val entries: Vector[(String, String, Int)] =
+ if (JsonExtensions.exists(ext => lower.endsWith(ext))) parseJson(source)
+ else if (YamlExtensions.exists(ext => lower.endsWith(ext))) parseYaml(source)
+ else if (TomlExtensions.exists(ext => lower.endsWith(ext))) parseToml(source)
+ else Vector.empty
+
+ buildResult(fileName, source, entries)
+ }
+
+ // ---------------------------------------------------------------------------
+ // Result builder
+ // ---------------------------------------------------------------------------
+
+ private def buildResult(
+ fileName: String,
+ source: String,
+ entries: Vector[(String, String, Int)]
+ ): ParseResult = {
+ val lines = source.split("\n", -1)
- // File-level Config node
val configEntity = ParsedEntity(
name = fileName,
kind = NodeKind.Config,
- attrs = Map("format" -> Json.fromString(format)),
+ attrs = Map("format" -> Json.fromString(detectFormat(fileName))),
lineStart = 1,
- lineEnd = source.split("\n", -1).length
+ lineEnd = lines.length
)
- // ConfigEntry nodes for each top-level key
val entryEntities = entries.map { case (key, value, line) =>
ParsedEntity(
name = key,
kind = NodeKind.ConfigEntry,
- attrs = Map(
- "value" -> Json.fromString(value),
- "format" -> Json.fromString(format)
- ),
+ attrs = Map("value" -> Json.fromString(value)),
lineStart = line,
lineEnd = line
)
}
- // Relationships: Config DEFINES each ConfigEntry
val relationships = entries.map { case (key, _, _) =>
ParsedRelationship(fileName, key, "DEFINES")
}
@@ -55,79 +73,118 @@ class ConfigParser extends Parser {
}
private def detectFormat(fileName: String): String = {
- if (fileName.endsWith(".json")) "json"
- else if (fileName.endsWith(".yaml") || fileName.endsWith(".yml")) "yaml"
- else if (fileName.endsWith(".toml")) "toml"
+ val lower = fileName.toLowerCase
+ if (JsonExtensions.exists(ext => lower.endsWith(ext))) "json"
+ else if (YamlExtensions.exists(ext => lower.endsWith(ext))) "yaml"
+ else if (TomlExtensions.exists(ext => lower.endsWith(ext))) "toml"
else "unknown"
}
- /** Parse JSON: extract top-level keys from objects. Simple regex approach. */
+ // ---------------------------------------------------------------------------
+ // JSON parsing — Circe-based with flattening
+ // ---------------------------------------------------------------------------
+
private def parseJson(source: String): Vector[(String, String, Int)] = {
+ io.circe.parser.parse(source) match {
+ case Right(json) => flattenJson("", json, 1)
+ case Left(_) => regexParseJson(source)
+ }
+ }
+
+ private def flattenJson(prefix: String, json: io.circe.Json, line: Int): Vector[(String, String, Int)] = {
+ json.fold(
+ jsonNull = Vector.empty,
+ jsonBoolean = b => Vector((prefix, b.toString, line)),
+ jsonNumber = n => Vector((prefix, n.toString, line)),
+ jsonString = s => Vector((prefix, s, line)),
+ jsonArray = arr => arr.zipWithIndex.flatMap { case (v, i) =>
+ val key = if (prefix.isEmpty) s"[$i]" else s"$prefix[$i]"
+ flattenJson(key, v, line)
+ }.toVector,
+ jsonObject = obj => obj.toVector.flatMap { case (k, v) =>
+ val key = if (prefix.isEmpty) k else s"$prefix.$k"
+ v.fold(
+ jsonNull = Vector((key, "null", line)),
+ jsonBoolean = b => Vector((key, b.toString, line)),
+ jsonNumber = n => Vector((key, n.toString, line)),
+ jsonString = s => Vector((key, s, line)),
+ jsonArray = _ => flattenJson(key, v, line),
+ jsonObject = _ => flattenJson(key, v, line)
+ )
+ }
+ )
+ }
+
+ /** Regex fallback for malformed JSON */
+ private def regexParseJson(source: String): Vector[(String, String, Int)] = {
+ val KvPattern = """"([^"]+)"\s*:\s*"([^"]*)"""".r
val lines = source.split("\n", -1)
- var results = Vector.empty[(String, String, Int)]
- // Match top-level JSON object keys: "key": value
- // Only match lines at indent level 2 (1 level into the root object)
- val KeyPattern = """^\s{0,4}"(\w[\w.-]*)"\s*:\s*(.+?),?\s*$""".r
- for ((line, idx) <- lines.zipWithIndex) {
- KeyPattern.findFirstMatchIn(line).foreach { m =>
- val key = m.group(1)
- val value = m.group(2).trim.stripPrefix("\"").stripSuffix("\"").stripSuffix(",")
- results = results :+ (key, value, idx + 1)
+ lines.zipWithIndex.flatMap { case (line, idx) =>
+ KvPattern.findAllMatchIn(line).map { m =>
+ (m.group(1), m.group(2), idx + 1)
}
- }
- results
+ }.toVector
}
- /** Parse YAML: extract top-level keys (no indentation). */
+ // ---------------------------------------------------------------------------
+ // YAML parsing — indentation-aware
+ // ---------------------------------------------------------------------------
+
private def parseYaml(source: String): Vector[(String, String, Int)] = {
val lines = source.split("\n", -1)
- var results = Vector.empty[(String, String, Int)]
- // Top-level YAML keys start at column 0 (no indentation)
- val KeyPattern = """^([a-zA-Z_][\w.-]*)\s*:\s*(.*)$""".r
- for ((line, idx) <- lines.zipWithIndex) {
- // Skip comments and empty lines
- if (!line.trim.startsWith("#") && line.trim.nonEmpty) {
- KeyPattern.findFirstMatchIn(line).foreach { m =>
- val key = m.group(1)
- val value = m.group(2).trim
- // Skip keys that start a nested block (value is empty or just a comment)
- val cleanValue = if (value.isEmpty || value.startsWith("#")) "(block)" else value
- results = results :+ (key, cleanValue, idx + 1)
+ val result = Vector.newBuilder[(String, String, Int)]
+ val stack = scala.collection.mutable.Stack.empty[(Int, String)] // (indent, prefix)
+
+ val KvWithValue = """^([a-zA-Z_][\w.-]*)\s*:\s*(.+)$""".r
+ val KvNoValue = """^([a-zA-Z_][\w.-]*)\s*:\s*$""".r
+
+ lines.zipWithIndex.foreach { case (line, idx) =>
+ val trimmed = line.trim
+ if (!trimmed.startsWith("#") && trimmed.nonEmpty) {
+ val indent = line.takeWhile(_ == ' ').length
+ trimmed match {
+ case KvWithValue(key, value) =>
+ while (stack.nonEmpty && stack.top._1 >= indent) stack.pop()
+ val fullKey = if (stack.nonEmpty) s"${stack.top._2}.$key" else key
+ result += ((fullKey, value.trim, idx + 1))
+ case KvNoValue(key) =>
+ while (stack.nonEmpty && stack.top._1 >= indent) stack.pop()
+ val fullKey = if (stack.nonEmpty) s"${stack.top._2}.$key" else key
+ stack.push((indent, fullKey))
+ case _ =>
+ // skip list items, comments, etc.
}
}
}
- results
+ result.result()
}
- /** Parse TOML: extract top-level key-value pairs and section keys. */
+ // ---------------------------------------------------------------------------
+ // TOML parsing — section-aware
+ // ---------------------------------------------------------------------------
+
private def parseToml(source: String): Vector[(String, String, Int)] = {
val lines = source.split("\n", -1)
- var results = Vector.empty[(String, String, Int)]
- var inSection = false // Track if we're inside a [section]
- val SectionPattern = """^\[.*\]""".r
- val KeyPattern = """^([a-zA-Z_][\w.-]*)\s*=\s*(.+)$""".r
- for ((line, idx) <- lines.zipWithIndex) {
- if (!line.trim.startsWith("#") && line.trim.nonEmpty) {
- SectionPattern.findFirstMatchIn(line).foreach { _ =>
- inSection = true
- }
- if (!inSection) {
- KeyPattern.findFirstMatchIn(line).foreach { m =>
- val key = m.group(1)
- val value = m.group(2).trim.stripPrefix("\"").stripSuffix("\"")
- results = results :+ (key, value, idx + 1)
- }
- }
- // Also capture section-level keys as section.key
- if (inSection) {
- KeyPattern.findFirstMatchIn(line).foreach { m =>
- val key = m.group(1)
- val value = m.group(2).trim.stripPrefix("\"").stripSuffix("\"")
- results = results :+ (key, value, idx + 1)
- }
+ val result = Vector.newBuilder[(String, String, Int)]
+ var currentSection = ""
+
+ val SectionPattern = """^\[([^\]]+)\]""".r
+ val KvPattern = """^([a-zA-Z_][\w.-]*)\s*=\s*(.+)$""".r
+
+ lines.zipWithIndex.foreach { case (line, idx) =>
+ val trimmed = line.trim
+ if (!trimmed.startsWith("#") && trimmed.nonEmpty) {
+ trimmed match {
+ case SectionPattern(section) =>
+ currentSection = section
+ case KvPattern(key, value) =>
+ val fullKey = if (currentSection.isEmpty) key else s"$currentSection.$key"
+ val cleanValue = value.trim.stripPrefix("\"").stripSuffix("\"")
+ result += ((fullKey, cleanValue, idx + 1))
+ case _ =>
}
}
}
- results
+ result.result()
}
}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
index 270ef04d..7d99504c 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/MarkdownParser.scala
@@ -7,18 +7,20 @@ import io.circe.Json
import scala.util.matching.Regex
/**
- * Markdown parser that extracts document structure from .md files.
- * Each header becomes a Doc node. Sections are connected by DEFINES relationships
- * based on heading hierarchy (## is defined by #, ### is defined by ##).
+ * Markdown parser that extracts structural entities from `.md` files.
+ *
+ * Each heading becomes a Doc entity with attrs for `level`, `title`, and `content`.
+ * Content captures all non-header lines in the section, up to 2000 characters.
+ * A top-level File entity is also emitted, with CONTAINS edges to each section.
*/
class MarkdownParser extends Parser {
- private val HeaderPattern: Regex = """^(#{1,6})\s+(.+)$""".r
+ private val HeadingPattern: Regex = """^(#{1,6})\s+(.+)$""".r
def parse(fileName: String, source: String): ParseResult = {
val lines = source.split("\n", -1)
- // File-level Doc node
+ // -- File entity --
val fileEntity = ParsedEntity(
name = fileName,
kind = NodeKind.File,
@@ -27,71 +29,50 @@ class MarkdownParser extends Parser {
lineEnd = lines.length
)
+ // Collect headings with their line indices
+ val headings: Vector[(Int, Int, String)] = lines.zipWithIndex.flatMap { case (line, idx) =>
+ HeadingPattern.findFirstMatchIn(line.trim).map { m =>
+ val level = m.group(1).length
+ val title = m.group(2).trim
+ (idx, level, title)
+ }
+ }.toVector
+
var entities = Vector(fileEntity)
var relationships = Vector.empty[ParsedRelationship]
- // Track heading hierarchy for parent-child relationships
- // Stack of (level, name) — most recent heading at each level
- var headingStack = Vector.empty[(Int, String)]
-
- for ((line, idx) <- lines.zipWithIndex) {
- val lineNum = idx + 1
+ for (i <- headings.indices) {
+ val (startIdx, level, title) = headings(i)
+ val endIdx = if (i + 1 < headings.length) headings(i + 1)._1 else lines.length
- HeaderPattern.findFirstMatchIn(line).foreach { m =>
- val level = m.group(1).length // 1 for #, 2 for ##, etc.
- val title = m.group(2).trim
-
- // Find the end of this section (next heading of same or higher level)
- val sectionEnd = findSectionEnd(lines, idx, level)
+ val content = extractContent(lines, startIdx, endIdx)
- // Extract a content summary (first non-empty, non-header line after the heading)
- val contentSummary = extractSummary(lines, idx + 1, sectionEnd)
-
- val attrs = Map(
+ val entity = ParsedEntity(
+ name = title,
+ kind = NodeKind.Doc,
+ attrs = Map(
"level" -> Json.fromInt(level),
- "content" -> Json.fromString(contentSummary)
- )
-
- entities = entities :+ ParsedEntity(
- name = title,
- kind = NodeKind.Doc,
- attrs = attrs,
- lineStart = lineNum,
- lineEnd = sectionEnd
- )
-
- // Determine parent: find nearest heading with lower level number
- headingStack = headingStack.filter(_._1 < level)
- val parent = headingStack.lastOption.map(_._2).getOrElse(fileName)
-
- relationships = relationships :+ ParsedRelationship(parent, title, "DEFINES")
-
- headingStack = headingStack :+ (level, title)
- }
+ "title" -> Json.fromString(title),
+ "content" -> Json.fromString(content)
+ ),
+ lineStart = startIdx + 1,
+ lineEnd = endIdx
+ )
+
+ entities = entities :+ entity
+ relationships = relationships :+ ParsedRelationship(fileName, title, "CONTAINS")
}
ParseResult(entities, relationships)
}
- /** Find the end line of a section: the next heading line (any level). Returns 1-based line number. */
- private def findSectionEnd(lines: Array[String], startIdx: Int, level: Int): Int = {
- var i = startIdx + 1
- while (i < lines.length) {
- if (HeaderPattern.findFirstMatchIn(lines(i)).isDefined) {
- return i + 1 // Convert 0-indexed to 1-based line number
- }
- i += 1
+ /**
+ * Extract content from all non-header lines in the section, up to 2000 characters.
+ */
+ private def extractContent(lines: Array[String], startIdx: Int, endIdx: Int): String = {
+ val contentLines = lines.slice(startIdx + 1, endIdx).filterNot { line =>
+ HeadingPattern.findFirstMatchIn(line.trim).isDefined
}
- lines.length // Section extends to end of file
- }
-
- /** Extract a summary from the first paragraph after a heading (up to 200 chars). */
- private def extractSummary(lines: Array[String], startIdx: Int, endIdx: Int): String = {
- val contentLines = lines.slice(startIdx, endIdx)
- .map(_.trim)
- .filter(l => l.nonEmpty && !l.startsWith("#"))
- .take(3) // Take first 3 content lines for summary
- val summary = contentLines.mkString(" ").take(200)
- summary
+ contentLines.mkString("\n").trim.take(2000)
}
}
diff --git a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala
index 7f0276a1..4b648b40 100644
--- a/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala
+++ b/memory-layer/src/main/scala/ix/memory/ingestion/parsers/TypeScriptParser.scala
@@ -7,60 +7,26 @@ import io.circe.Json
import scala.util.matching.Regex
/**
- * TypeScript/TSX source parser that extracts structural entities and relationships.
- *
- * Uses regex-based extraction for .ts/.tsx files, following the same pattern as
- * TreeSitterPythonParser. Uses brace counting for block boundaries instead of
- * indentation.
+ * TypeScript/JavaScript source parser that extracts structural entities
+ * and relationships using regex-based extraction.
*/
class TypeScriptParser extends Parser {
+ /**
+ * Parse a TypeScript source file and extract entities and relationships.
+ *
+ * @param fileName the file name (e.g. "api.ts")
+ * @param source the full source code as a string
+ * @return ParseResult with entities and relationships
+ */
def parse(fileName: String, source: String): ParseResult = {
- regexParse(fileName, source)
- }
-
- // ---------------------------------------------------------------------------
- // Regex patterns
- // ---------------------------------------------------------------------------
-
- private val ClassPattern: Regex = """(?:export\s+)?(?:abstract\s+)?class\s+(\w+)""".r
- private val FuncPattern: Regex = """(?:export\s+)?(?:async\s+)?function\s+(\w+)""".r
- private val ArrowFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[a-zA-Z_]\w*)\s*(?::\s*[^=]+)?\s*=>""".r
- private val ConstFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?function""".r
- private val InterfacePattern: Regex = """(?:export\s+)?interface\s+(\w+)""".r
- private val TypeAliasPattern: Regex = """(?:export\s+)?type\s+(\w+)\s*(?:<[^>]+>)?\s*=""".r
- private val ImportPattern: Regex = """import\s+.*?from\s+['"]([^'"]+)['"]""".r
- private val SideEffectImportPattern: Regex = """^import\s+['"]([^'"]+)['"]""".r
- private val MethodPattern: Regex = """^\s+(?:async\s+)?(?:private\s+|public\s+|protected\s+|static\s+|readonly\s+)*(\w+)\s*(?:<[^>]+>)?\s*\(""".r
- private val CallPattern: Regex = """(?:(?:this|self|await)\.)?\b(\w+)\s*\(""".r
-
- // ---------------------------------------------------------------------------
- // Builtins to filter from CALLS
- // ---------------------------------------------------------------------------
-
- private val TsBuiltins: Set[String] = Set(
- "console", "parseInt", "parseFloat", "String", "Number", "Boolean",
- "Array", "Object", "Promise", "JSON", "Math", "Date", "Error",
- "RegExp", "Map", "Set", "WeakMap", "WeakSet", "Symbol",
- "require", "setTimeout", "setInterval", "clearTimeout", "clearInterval",
- "fetch", "Response", "Request", "URL", "Buffer", "process"
- )
-
- // ---------------------------------------------------------------------------
- // Regex-based parser
- // ---------------------------------------------------------------------------
-
- private def regexParse(fileName: String, source: String): ParseResult = {
val lines = source.split("\n", -1)
- // -- File entity --
+ // -- File entity (no raw content stored) --
val fileEntity = ParsedEntity(
name = fileName,
kind = NodeKind.File,
- attrs = Map(
- "language" -> Json.fromString("typescript"),
- "content" -> Json.fromString(source)
- ),
+ attrs = Map("language" -> Json.fromString("typescript")),
lineStart = 1,
lineEnd = lines.length
)
@@ -68,7 +34,7 @@ class TypeScriptParser extends Parser {
var entities = Vector(fileEntity)
var relationships = Vector.empty[ParsedRelationship]
- // Track class ranges for containment checks
+ // Track class ranges for containment
var classRanges = Vector.empty[(String, Int, Int)]
for ((line, idx) <- lines.zipWithIndex) {
@@ -94,29 +60,33 @@ class TypeScriptParser extends Parser {
relationships = relationships :+ ParsedRelationship(fileName, className, "DEFINES")
classRanges = classRanges :+ (className, lineNum, classEnd)
- // Extract methods inside this class
+ // Extract methods from this class
val methods = extractMethods(lines, idx, classEnd, className)
entities = entities ++ methods._1
relationships = relationships ++ methods._2
}
- // Check for function definition (but not if it's a method inside a class)
- if (!isInsideClass(lineNum, classRanges)) {
+ // Check for top-level function (not inside a class)
+ if (!insideClass(lineNum, classRanges)) {
FuncPattern.findFirstMatchIn(line).foreach { m =>
val funcName = m.group(1)
val funcEnd = findBraceBlockEnd(lines, idx)
+ val summary = line.trim.take(120)
entities = entities :+ ParsedEntity(
name = funcName,
kind = NodeKind.Function,
- attrs = Map("language" -> Json.fromString("typescript")),
+ attrs = Map(
+ "language" -> Json.fromString("typescript"),
+ "summary" -> Json.fromString(summary)
+ ),
lineStart = lineNum,
lineEnd = funcEnd
)
relationships = relationships :+ ParsedRelationship(fileName, funcName, "DEFINES")
// Extract calls within this function
- val callRels = extractCalls(lines, idx, funcEnd, funcName)
- relationships = relationships ++ callRels
+ val calls = extractCalls(lines, idx, funcEnd, funcName)
+ relationships = relationships ++ calls
}
// Check for arrow function / const function
@@ -128,10 +98,14 @@ class TypeScriptParser extends Parser {
// Avoid duplicates if already captured by FuncPattern
if (!entities.exists(e => e.name == funcName && e.kind == NodeKind.Function)) {
val funcEnd = findBraceBlockEnd(lines, idx)
+ val summary = line.trim.take(120)
entities = entities :+ ParsedEntity(
name = funcName,
kind = NodeKind.Function,
- attrs = Map("language" -> Json.fromString("typescript")),
+ attrs = Map(
+ "language" -> Json.fromString("typescript"),
+ "summary" -> Json.fromString(summary)
+ ),
lineStart = lineNum,
lineEnd = funcEnd
)
@@ -143,18 +117,48 @@ class TypeScriptParser extends Parser {
}
}
+ // Check for imports
+ ImportPattern.findFirstMatchIn(line).foreach { m =>
+ val moduleName = m.group(1)
+ if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
+ entities = entities :+ ParsedEntity(
+ name = moduleName,
+ kind = NodeKind.Module,
+ attrs = Map.empty,
+ lineStart = lineNum,
+ lineEnd = lineNum
+ )
+ }
+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
+ }
+
+ SideEffectImportPattern.findFirstMatchIn(line).foreach { m =>
+ val moduleName = m.group(1)
+ if (!relationships.exists(r => r.dstName == moduleName && r.predicate == "IMPORTS")) {
+ if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
+ entities = entities :+ ParsedEntity(
+ name = moduleName,
+ kind = NodeKind.Module,
+ attrs = Map.empty,
+ lineStart = lineNum,
+ lineEnd = lineNum
+ )
+ }
+ relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
+ }
+ }
+
// Check for interface definition
InterfacePattern.findFirstMatchIn(line).foreach { m =>
val ifaceName = m.group(1)
- // Don't match if this is already matched as a class pattern
- if (!entities.exists(e => e.name == ifaceName && e.kind == NodeKind.Class && !e.attrs.contains("ts_kind"))) {
+ if (!entities.exists(e => e.name == ifaceName && e.kind == NodeKind.Class)) {
val ifaceEnd = findBraceBlockEnd(lines, idx)
entities = entities :+ ParsedEntity(
name = ifaceName,
- kind = NodeKind.Class,
+ kind = NodeKind.Class, // model interfaces as Class kind
attrs = Map(
- "language" -> Json.fromString("typescript"),
- "ts_kind" -> Json.fromString("interface")
+ "language" -> Json.fromString("typescript"),
+ "ts_kind" -> Json.fromString("interface")
),
lineStart = lineNum,
lineEnd = ifaceEnd
@@ -166,7 +170,6 @@ class TypeScriptParser extends Parser {
// Check for type alias
TypeAliasPattern.findFirstMatchIn(line).foreach { m =>
val typeName = m.group(1)
- // Avoid matching 'import type' lines
if (!trimmed.startsWith("import")) {
entities = entities :+ ParsedEntity(
name = typeName,
@@ -182,120 +185,103 @@ class TypeScriptParser extends Parser {
}
}
- // Check for imports
- ImportPattern.findFirstMatchIn(line).foreach { m =>
- val moduleName = m.group(1)
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
-
- SideEffectImportPattern.findFirstMatchIn(line).foreach { m =>
- val moduleName = m.group(1)
- // Only add if not already captured by the regular ImportPattern
- if (!relationships.exists(r => r.dstName == moduleName && r.predicate == "IMPORTS")) {
- if (!entities.exists(e => e.name == moduleName && e.kind == NodeKind.Module)) {
- entities = entities :+ ParsedEntity(
- name = moduleName,
- kind = NodeKind.Module,
- attrs = Map.empty,
- lineStart = lineNum,
- lineEnd = lineNum
- )
- }
- relationships = relationships :+ ParsedRelationship(fileName, moduleName, "IMPORTS")
- }
- }
-
- } // end else (non-comment line)
+ } // end else (non-comment)
}
ParseResult(entities, relationships)
}
// ---------------------------------------------------------------------------
- // Block boundary detection using brace counting
+ // Patterns
// ---------------------------------------------------------------------------
- private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Int = {
- var braceCount = 0
- var foundOpen = false
- var i = startIdx
- while (i < lines.length) {
- for (ch <- lines(i)) {
- if (ch == '{') { braceCount += 1; foundOpen = true }
- else if (ch == '}') { braceCount -= 1 }
- }
- if (foundOpen && braceCount <= 0) return i + 1
- i += 1
- }
- lines.length
- }
+ private val ClassPattern: Regex = """(?:export\s+)?(?:abstract\s+)?class\s+(\w+)""".r
+ private val FuncPattern: Regex = """(?:export\s+)?(?:async\s+)?function\s+(\w+)""".r
+ private val ArrowFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[a-zA-Z_]\w*)\s*(?::\s*[^=]+)?\s*=>""".r
+ private val ConstFuncPattern: Regex = """(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?function""".r
+ private val MethodPattern: Regex = """^\s+(?:async\s+)?(?:private\s+|public\s+|protected\s+|static\s+|readonly\s+)*(\w+)\s*(?:<[^>]+>)?\s*\(""".r
+ private val ImportPattern: Regex = """import\s+.*?from\s+['"]([^'"]+)['"]""".r
+ private val SideEffectImportPattern: Regex = """^import\s+['"]([^'"]+)['"]""".r
+ private val InterfacePattern: Regex = """(?:export\s+)?interface\s+(\w+)""".r
+ private val TypeAliasPattern: Regex = """(?:export\s+)?type\s+(\w+)\s*(?:<[^>]+>)?\s*=""".r
+ private val CallPattern: Regex = """(?:(?:this|self|await)\.)?\b(\w+)\s*\(""".r
// ---------------------------------------------------------------------------
- // Helper: check if a line is inside a class range
+ // Helpers
// ---------------------------------------------------------------------------
- private def isInsideClass(lineNum: Int, classRanges: Vector[(String, Int, Int)]): Boolean = {
- classRanges.exists { case (_, start, end) =>
- lineNum > start && lineNum <= end
- }
- }
-
- // ---------------------------------------------------------------------------
- // Extract methods inside a class block
- // ---------------------------------------------------------------------------
+ private def insideClass(lineNum: Int, classRanges: Vector[(String, Int, Int)]): Boolean =
+ classRanges.exists { case (_, start, end) => lineNum > start && lineNum <= end }
+ /**
+ * Extract methods from a class body.
+ */
private def extractMethods(
lines: Array[String],
classStartIdx: Int,
classEndLine: Int,
className: String
): (Vector[ParsedEntity], Vector[ParsedRelationship]) = {
- var methodEntities = Vector.empty[ParsedEntity]
- var methodRels = Vector.empty[ParsedRelationship]
+ var entities = Vector.empty[ParsedEntity]
+ var relationships = Vector.empty[ParsedRelationship]
var i = classStartIdx + 1
while (i < lines.length && (i + 1) <= classEndLine) {
val line = lines(i)
- MethodPattern.findFirstMatchIn(line).foreach { m =>
- val methodName = m.group(1)
- // Skip constructor and common keywords that look like methods
- if (methodName != "constructor" && methodName != "if" && methodName != "for" &&
- methodName != "while" && methodName != "switch" && methodName != "return" &&
- methodName != "new" && methodName != "throw" && methodName != "catch") {
- val methodEnd = findBraceBlockEnd(lines, i)
- methodEntities = methodEntities :+ ParsedEntity(
- name = methodName,
- kind = NodeKind.Function,
- attrs = Map("language" -> Json.fromString("typescript")),
- lineStart = i + 1,
- lineEnd = methodEnd
- )
- methodRels = methodRels :+ ParsedRelationship(className, methodName, "DEFINES")
+ val lineNum = i + 1
+
+ if (!line.trim.startsWith("//") && !line.trim.startsWith("*")) {
+ MethodPattern.findFirstMatchIn(line).foreach { m =>
+ val methodName = m.group(1)
+ // Skip keywords that look like methods
+ if (!TypeScriptKeywords.contains(methodName)) {
+ val methodEnd = findBraceBlockEnd(lines, i)
+ val summary = line.trim.take(120)
+ entities = entities :+ ParsedEntity(
+ name = methodName,
+ kind = NodeKind.Function,
+ attrs = Map(
+ "language" -> Json.fromString("typescript"),
+ "summary" -> Json.fromString(summary)
+ ),
+ lineStart = lineNum,
+ lineEnd = methodEnd
+ )
+ relationships = relationships :+ ParsedRelationship(className, methodName, "DEFINES")
- // Extract calls within this method
- val callRels = extractCalls(lines, i, methodEnd, methodName)
- methodRels = methodRels ++ callRels
+ // Extract calls within this method
+ val calls = extractCalls(lines, i, methodEnd, methodName)
+ relationships = relationships ++ calls
+ }
}
}
i += 1
}
- (methodEntities, methodRels)
+ (entities, relationships)
}
- // ---------------------------------------------------------------------------
- // Extract CALLS relationships from a function/method body
- // ---------------------------------------------------------------------------
+ /**
+ * Find end of a brace-delimited block starting near `startIdx`.
+ */
+ private def findBraceBlockEnd(lines: Array[String], startIdx: Int): Int = {
+ var braceCount = 0
+ var foundOpen = false
+ var i = startIdx
+ while (i < lines.length) {
+ for (ch <- lines(i)) {
+ if (ch == '{') { braceCount += 1; foundOpen = true }
+ if (ch == '}') braceCount -= 1
+ }
+ if (foundOpen && braceCount <= 0) return i + 1
+ i += 1
+ }
+ lines.length
+ }
+ /**
+ * Extract CALLS relationships from a function/method body.
+ */
private def extractCalls(
lines: Array[String],
funcStartIdx: Int,
@@ -310,10 +296,8 @@ class TypeScriptParser extends Parser {
val line = lines(i)
CallPattern.findAllMatchIn(line).foreach { m =>
val callee = m.group(1)
- if (callee != callerName && !TsBuiltins.contains(callee) && !seen.contains(callee) &&
- callee != "if" && callee != "for" && callee != "while" && callee != "switch" &&
- callee != "return" && callee != "new" && callee != "throw" && callee != "catch" &&
- callee != "constructor") {
+ if (callee != callerName && !TypeScriptBuiltins.contains(callee) &&
+ !TypeScriptKeywords.contains(callee) && !seen.contains(callee)) {
seen += callee
calls = calls :+ ParsedRelationship(callerName, callee, "CALLS")
}
@@ -322,4 +306,24 @@ class TypeScriptParser extends Parser {
}
calls
}
+
+ private val TypeScriptKeywords: Set[String] = Set(
+ "if", "else", "for", "while", "do", "switch", "case", "break",
+ "continue", "return", "throw", "try", "catch", "finally",
+ "new", "delete", "typeof", "instanceof", "void", "in", "of",
+ "class", "extends", "implements", "interface", "enum", "type",
+ "const", "let", "var", "function", "async", "await",
+ "import", "export", "default", "from", "as",
+ "this", "super", "constructor", "static", "private", "public", "protected"
+ )
+
+ private val TypeScriptBuiltins: Set[String] = Set(
+ "console", "log", "error", "warn", "parseInt", "parseFloat",
+ "String", "Number", "Boolean", "Array", "Object", "Promise",
+ "JSON", "Math", "Date", "RegExp", "Error", "Map", "Set",
+ "WeakMap", "WeakSet", "Symbol",
+ "setTimeout", "setInterval", "clearTimeout", "clearInterval",
+ "require", "module", "exports",
+ "fetch", "Response", "Request", "URL", "Buffer", "process"
+ )
}
diff --git a/memory-layer/src/test/resources/fixtures/api.ts b/memory-layer/src/test/resources/fixtures/api.ts
new file mode 100644
index 00000000..73304cdf
--- /dev/null
+++ b/memory-layer/src/test/resources/fixtures/api.ts
@@ -0,0 +1,39 @@
+import axios from "axios";
+import { Config } from "./config";
+
+interface UserResponse {
+ id: string;
+ name: string;
+ email: string;
+}
+
+export class ApiClient {
+ private baseUrl: string;
+
+ constructor(private config: Config) {
+ this.baseUrl = config.baseUrl;
+ }
+
+ async getUser(userId: string): Promise<UserResponse> {
+ const response = await axios.get(`${this.baseUrl}/users/${userId}`);
+ return this.parseResponse(response);
+ }
+
+ async updateUser(userId: string, data: Partial<UserResponse>): Promise<UserResponse> {
+ const response = await axios.put(`${this.baseUrl}/users/${userId}`, data);
+ return this.parseResponse(response);
+ }
+
+ private parseResponse(response: any): UserResponse {
+ return response.data as UserResponse;
+ }
+}
+
+export function createClient(config: Config): ApiClient {
+ return new ApiClient(config);
+}
+
+export async function fetchAllUsers(client: ApiClient): Promise<UserResponse[]> {
+ const response = await client.getUser("all");
+ return [response];
+}
diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/parsers/ConfigParserSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/ConfigParserSpec.scala
index 5a8c5980..5df07086 100644
--- a/memory-layer/src/test/scala/ix/memory/ingestion/parsers/ConfigParserSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/ConfigParserSpec.scala
@@ -2,14 +2,92 @@ package ix.memory.ingestion.parsers
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
+
import ix.memory.model.NodeKind
class ConfigParserSpec extends AnyFlatSpec with Matchers {
-
val parser = new ConfigParser()
- // Test 1: JSON — creates Config entity for file
- "ConfigParser" should "create a Config entity for a JSON file" in {
+ // --- Worktree tests: dot-path flattening ---
+
+ "ConfigParser" should "parse simple JSON key-value pairs" in {
+ val json = """{"name": "ix-memory", "version": "1.0"}"""
+ val result = parser.parse("package.json", json)
+ val entries = result.entities.filter(_.kind == NodeKind.ConfigEntry)
+ val names = entries.map(_.name)
+ names should contain("name")
+ names should contain("version")
+ }
+
+ it should "flatten nested JSON objects into dot-path entries" in {
+ val json = """{"mcpServers": {"ix-memory": {"command": "node", "args": ["server.js"]}}, "debug": true}"""
+ val result = parser.parse("config.json", json)
+ val entries = result.entities.filter(_.kind == NodeKind.ConfigEntry)
+ val names = entries.map(_.name)
+ names should contain("mcpServers.ix-memory.command")
+ names should contain("mcpServers.ix-memory.args[0]")
+ names should contain("debug")
+ entries.foreach { e =>
+ val v = e.attrs.get("value").flatMap(_.asString).getOrElse("")
+ v should not be "{"
+ }
+ }
+
+ it should "flatten nested YAML into dot-path entries" in {
+ val yaml = "server:\n host: localhost\n port: 8080\ndebug: true"
+ val result = parser.parse("config.yaml", yaml)
+ val entries = result.entities.filter(_.kind == NodeKind.ConfigEntry)
+ val names = entries.map(_.name)
+ names should contain("server.host")
+ names should contain("server.port")
+ names should contain("debug")
+ }
+
+ it should "parse simple YAML key-value pairs" in {
+ val yaml = "name: my-app\nversion: 2.0\nenabled: true"
+ val result = parser.parse("config.yml", yaml)
+ val entries = result.entities.filter(_.kind == NodeKind.ConfigEntry)
+ entries.map(_.name) should contain allOf ("name", "version", "enabled")
+ }
+
+ it should "parse TOML with sections as dot-path entries" in {
+ val toml = "[database]\nhost = \"localhost\"\nport = 5432\n\n[server]\nworkers = 4"
+ val result = parser.parse("config.toml", toml)
+ val entries = result.entities.filter(_.kind == NodeKind.ConfigEntry)
+ val names = entries.map(_.name)
+ names should contain("database.host")
+ names should contain("database.port")
+ names should contain("server.workers")
+ }
+
+ it should "create a Config file-level entity" in {
+ val json = """{"key": "value"}"""
+ val result = parser.parse("settings.json", json)
+ val configs = result.entities.filter(_.kind == NodeKind.Config)
+ configs should have size 1
+ configs.head.name shouldBe "settings.json"
+ }
+
+ it should "create DEFINES relationships from Config to ConfigEntry" in {
+ val json = """{"a": "1", "b": "2"}"""
+ val result = parser.parse("app.json", json)
+ val defines = result.relationships.filter(_.predicate == "DEFINES")
+ defines should have size 2
+ defines.foreach(_.srcName shouldBe "app.json")
+ }
+
+ it should "report canParse correctly for config file extensions" in {
+ parser.canParse("config.json") shouldBe true
+ parser.canParse("config.yaml") shouldBe true
+ parser.canParse("config.yml") shouldBe true
+ parser.canParse("config.toml") shouldBe true
+ parser.canParse("main.py") shouldBe false
+ parser.canParse("README.md") shouldBe false
+ }
+
+ // --- Existing tests ---
+
+ it should "create a Config entity with format attr for JSON" in {
val source = """{ "name": "my-app", "version": "1.0.0" }"""
val result = parser.parse("package.json", source)
result.entities.exists(e => e.name == "package.json" && e.kind == NodeKind.Config) shouldBe true
@@ -17,29 +95,6 @@ class ConfigParserSpec extends AnyFlatSpec with Matchers {
config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"json\"")
}
- // Test 2: JSON — extracts top-level keys as ConfigEntry
- it should "extract top-level JSON keys as ConfigEntry nodes" in {
- val source = """{
- | "name": "my-app",
- | "version": "1.0.0",
- | "description": "A test app"
- |}""".stripMargin
- val result = parser.parse("package.json", source)
- result.entities.exists(e => e.name == "name" && e.kind == NodeKind.ConfigEntry) shouldBe true
- result.entities.exists(e => e.name == "version" && e.kind == NodeKind.ConfigEntry) shouldBe true
- result.entities.exists(e => e.name == "description" && e.kind == NodeKind.ConfigEntry) shouldBe true
- }
-
- // Test 3: JSON — DEFINES relationships
- it should "generate DEFINES relationships for JSON entries" in {
- val source = """{
- | "name": "test"
- |}""".stripMargin
- val result = parser.parse("config.json", source)
- result.relationships.exists(r => r.srcName == "config.json" && r.dstName == "name" && r.predicate == "DEFINES") shouldBe true
- }
-
- // Test 4: YAML — extracts top-level keys
it should "extract top-level YAML keys" in {
val source = """endpoint: http://localhost:8090
|format: text
@@ -50,7 +105,6 @@ class ConfigParserSpec extends AnyFlatSpec with Matchers {
result.entities.exists(e => e.name == "debug" && e.kind == NodeKind.ConfigEntry) shouldBe true
}
- // Test 5: YAML — skips comments
it should "skip YAML comment lines" in {
val source = """# This is a comment
|name: test
@@ -63,22 +117,19 @@ class ConfigParserSpec extends AnyFlatSpec with Matchers {
entryNames.size shouldBe 2
}
- // Test 6: YAML — format attr is "yaml"
it should "set format to yaml for .yaml files" in {
val result = parser.parse("app.yaml", "key: value")
val config = result.entities.find(_.kind == NodeKind.Config).get
config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"yaml\"")
}
- // Test 7: YAML — format attr is "yaml" for .yml too
it should "set format to yaml for .yml files" in {
val result = parser.parse("app.yml", "key: value")
val config = result.entities.find(_.kind == NodeKind.Config).get
config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"yaml\"")
}
- // Test 8: TOML — extracts top-level keys
- it should "extract top-level TOML keys" in {
+ it should "extract TOML keys with section prefix" in {
val source = """name = "my-app"
|version = "1.0.0"
|
@@ -88,26 +139,22 @@ class ConfigParserSpec extends AnyFlatSpec with Matchers {
val result = parser.parse("config.toml", source)
result.entities.exists(e => e.name == "name" && e.kind == NodeKind.ConfigEntry) shouldBe true
result.entities.exists(e => e.name == "version" && e.kind == NodeKind.ConfigEntry) shouldBe true
- // Section keys should also be extracted
- result.entities.exists(e => e.name == "host" && e.kind == NodeKind.ConfigEntry) shouldBe true
- result.entities.exists(e => e.name == "port" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ result.entities.exists(e => e.name == "database.host" && e.kind == NodeKind.ConfigEntry) shouldBe true
+ result.entities.exists(e => e.name == "database.port" && e.kind == NodeKind.ConfigEntry) shouldBe true
}
- // Test 9: TOML — format attr is "toml"
it should "set format to toml for .toml files" in {
val result = parser.parse("config.toml", "key = \"value\"")
val config = result.entities.find(_.kind == NodeKind.Config).get
config.attrs.get("format").map(_.noSpaces) shouldBe Some("\"toml\"")
}
- // Test 10: Empty config
it should "handle empty config files" in {
val result = parser.parse("empty.json", "")
result.entities.exists(_.kind == NodeKind.Config) shouldBe true
result.entities.count(_.kind == NodeKind.ConfigEntry) shouldBe 0
}
- // Test 11: ConfigEntry has value attr
it should "store the value in ConfigEntry attrs" in {
val source = "port: 8080"
val result = parser.parse("config.yaml", source)
diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/parsers/MarkdownParserSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/MarkdownParserSpec.scala
index fee9c09f..457e0663 100644
--- a/memory-layer/src/test/scala/ix/memory/ingestion/parsers/MarkdownParserSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/MarkdownParserSpec.scala
@@ -2,21 +2,85 @@ package ix.memory.ingestion.parsers
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
+import io.circe.Json
+
import ix.memory.model.NodeKind
class MarkdownParserSpec extends AnyFlatSpec with Matchers {
-
val parser = new MarkdownParser()
- // Test 1: Creates File entity
- "MarkdownParser" should "create a File entity for the markdown file" in {
+ private def strAttr(attrs: Map[String, Json], key: String): Option[String] =
+ attrs.get(key).flatMap(_.asString)
+
+ private def intAttr(attrs: Map[String, Json], key: String): Option[Int] =
+ attrs.get(key).flatMap(_.asNumber).flatMap(_.toInt)
+
+ // --- Worktree tests: section content, title attr, 2000-char limit ---
+
+ "MarkdownParser" should "extract headings as Doc entities" in {
+ val md = "# Introduction\n\nSome text\n\n## Details\n\nMore text\n"
+ val result = parser.parse("readme.md", md)
+ val names = result.entities.map(_.name)
+ names should contain("Introduction")
+ names should contain("Details")
+ }
+
+ it should "create a File entity for the document" in {
+ val md = "# Title\n\nContent\n"
+ val result = parser.parse("test.md", md)
+ val file = result.entities.find(_.name == "test.md")
+ file shouldBe defined
+ }
+
+ it should "create CONTAINS relationships from file to sections" in {
+ val md = "# Title\n\nContent\n\n## Sub\n\nMore\n"
+ val result = parser.parse("test.md", md)
+ val contains = result.relationships.filter(_.predicate == "CONTAINS")
+ contains.map(_.srcName) should contain only "test.md"
+ contains.map(_.dstName) should contain allOf ("Title", "Sub")
+ }
+
+ it should "store full section content up to 2000 chars" in {
+ val longContent = "word " * 200 // 1000 chars
+ val md = s"# Title\n\n$longContent\n\n## Next"
+ val result = parser.parse("long.md", md)
+ val titleDoc = result.entities.find(_.name == "Title").get
+ val content = strAttr(titleDoc.attrs, "content").get
+ content.length should be > 200
+ }
+
+ it should "store heading title as separate attr" in {
+ val md = "# My Title\n\nSome content\n"
+ val result = parser.parse("test.md", md)
+ val doc = result.entities.find(_.name == "My Title").get
+ strAttr(doc.attrs, "title") shouldBe Some("My Title")
+ }
+
+ it should "truncate content at 2000 chars" in {
+ val longContent = "a" * 3000
+ val md = s"# Big Section\n\n$longContent\n"
+ val result = parser.parse("big.md", md)
+ val doc = result.entities.find(_.name == "Big Section").get
+ val content = strAttr(doc.attrs, "content").get
+ content.length shouldBe 2000
+ }
+
+ it should "include level attr for headings" in {
+ val md = "## Level Two\n\nText\n"
+ val result = parser.parse("test.md", md)
+ val doc = result.entities.find(_.name == "Level Two").get
+ intAttr(doc.attrs, "level") shouldBe Some(2)
+ }
+
+ // --- Existing tests: basic structure, empty files ---
+
+ it should "create a File entity with language=markdown" in {
val result = parser.parse("README.md", "# Hello")
result.entities.exists(e => e.name == "README.md" && e.kind == NodeKind.File) shouldBe true
val file = result.entities.find(_.kind == NodeKind.File).get
file.attrs.get("language").map(_.noSpaces) shouldBe Some("\"markdown\"")
}
- // Test 2: Extracts headers as Doc nodes
it should "extract headers as Doc nodes" in {
val source = """# Overview
|Some intro text.
@@ -32,21 +96,6 @@ class MarkdownParserSpec extends AnyFlatSpec with Matchers {
result.entities.exists(e => e.name == "API Design" && e.kind == NodeKind.Doc) shouldBe true
}
- // Test 3: Hierarchy — ## under # creates DEFINES relationship
- it should "create DEFINES relationships based on heading hierarchy" in {
- val source = """# Top
- |## Sub
- |### SubSub""".stripMargin
- val result = parser.parse("doc.md", source)
- // File DEFINES Top (# is top-level)
- result.relationships.exists(r => r.srcName == "doc.md" && r.dstName == "Top" && r.predicate == "DEFINES") shouldBe true
- // Top DEFINES Sub
- result.relationships.exists(r => r.srcName == "Top" && r.dstName == "Sub" && r.predicate == "DEFINES") shouldBe true
- // Sub DEFINES SubSub
- result.relationships.exists(r => r.srcName == "Sub" && r.dstName == "SubSub" && r.predicate == "DEFINES") shouldBe true
- }
-
- // Test 4: Level attr is correct
it should "store the heading level in attrs" in {
val source = """# Level 1
|## Level 2
@@ -60,8 +109,7 @@ class MarkdownParserSpec extends AnyFlatSpec with Matchers {
h3.attrs.get("level").flatMap(_.asNumber).flatMap(_.toInt) shouldBe Some(3)
}
- // Test 5: Content summary extracted
- it should "extract content summary from section body" in {
+ it should "extract content from section body" in {
val source = """# Introduction
|This is the introduction to the project.
|It explains the main goals.""".stripMargin
@@ -71,14 +119,12 @@ class MarkdownParserSpec extends AnyFlatSpec with Matchers {
content should include("introduction")
}
- // Test 6: Empty markdown
it should "handle empty markdown files" in {
val result = parser.parse("empty.md", "")
result.entities.exists(_.kind == NodeKind.File) shouldBe true
result.entities.count(_.kind == NodeKind.Doc) shouldBe 0
}
- // Test 7: Markdown with no headers
it should "handle markdown with no headers" in {
val source = "Just some text with no headers."
val result = parser.parse("notes.md", source)
@@ -86,7 +132,6 @@ class MarkdownParserSpec extends AnyFlatSpec with Matchers {
result.relationships shouldBe empty
}
- // Test 8: Multiple sections at same level
it should "handle sibling sections at the same level" in {
val source = """# Main
|## Section A
@@ -94,21 +139,8 @@ class MarkdownParserSpec extends AnyFlatSpec with Matchers {
|## Section B
|Content B""".stripMargin
val result = parser.parse("doc.md", source)
- // Both should be DEFINES'd by Main
- result.relationships.exists(r => r.srcName == "Main" && r.dstName == "Section A") shouldBe true
- result.relationships.exists(r => r.srcName == "Main" && r.dstName == "Section B") shouldBe true
- }
-
- // Test 9: Section line ranges
- it should "compute correct line ranges for sections" in {
- val source = """# Intro
- |Line 2
- |Line 3
- |## Details
- |Line 5""".stripMargin
- val result = parser.parse("doc.md", source)
- val intro = result.entities.find(_.name == "Intro").get
- intro.lineStart shouldBe 1
- intro.lineEnd shouldBe 4 // Ends at line before ## Details
+ // Both should be CONTAINS'd by file
+ result.relationships.exists(r => r.srcName == "doc.md" && r.dstName == "Section A") shouldBe true
+ result.relationships.exists(r => r.srcName == "doc.md" && r.dstName == "Section B") shouldBe true
}
}
diff --git a/memory-layer/src/test/scala/ix/memory/ingestion/parsers/TypeScriptParserSpec.scala b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/TypeScriptParserSpec.scala
index 9065511a..ff95e02a 100644
--- a/memory-layer/src/test/scala/ix/memory/ingestion/parsers/TypeScriptParserSpec.scala
+++ b/memory-layer/src/test/scala/ix/memory/ingestion/parsers/TypeScriptParserSpec.scala
@@ -2,19 +2,95 @@ package ix.memory.ingestion.parsers
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
+
import ix.memory.model.NodeKind
class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
-
val parser = new TypeScriptParser()
- // Test 1: Extract file entity
- "TypeScriptParser" should "create a File entity for the source file" in {
+ val sampleCode: String = scala.io.Source.fromResource("fixtures/api.ts").mkString
+
+ // --- Tests using fixture file ---
+
+ "TypeScriptParser" should "extract classes and functions from TypeScript source" in {
+ val result = parser.parse("api.ts", sampleCode)
+ val names = result.entities.map(_.name)
+ names should contain("ApiClient")
+ names should contain("createClient")
+ names should contain("fetchAllUsers")
+ }
+
+ it should "extract methods from classes" in {
+ val result = parser.parse("api.ts", sampleCode)
+ val names = result.entities.map(_.name)
+ names should contain("getUser")
+ names should contain("updateUser")
+ names should contain("parseResponse")
+ }
+
+ it should "set language attr to typescript" in {
+ val result = parser.parse("api.ts", sampleCode)
+ val file = result.entities.find(_.kind == NodeKind.File).get
+ file.attrs("language").asString shouldBe Some("typescript")
+ }
+
+ it should "extract import relationships" in {
+ val result = parser.parse("api.ts", sampleCode)
+ val imports = result.relationships.filter(_.predicate == "IMPORTS")
+ imports.size should be >= 2
+ }
+
+ it should "extract class-to-method DEFINES edges" in {
+ val result = parser.parse("api.ts", sampleCode)
+ val defines = result.relationships.filter(_.predicate == "DEFINES")
+ val classDefines = defines.filter(_.srcName == "ApiClient")
+ classDefines.size should be >= 3
+ }
+
+ it should "extract function call relationships" in {
+ val result = parser.parse("api.ts", sampleCode)
+ val calls = result.relationships.filter(_.predicate == "CALLS")
+ calls.size should be >= 1
+ }
+
+ it should "extract interface definitions" in {
+ val result = parser.parse("api.ts", sampleCode)
+ val names = result.entities.map(_.name)
+ names should contain("UserResponse")
+ }
+
+ it should "store method/function signature as summary attr" in {
+ val result = parser.parse("api.ts", sampleCode)
+ val funcs = result.entities.filter(_.kind == NodeKind.Function)
+ funcs should not be empty
+ funcs.foreach { f =>
+ f.attrs.get("summary") shouldBe defined
+ f.attrs("summary").asString.get should not be empty
+ }
+ }
+
+ it should "not store raw source content on File entity" in {
+ val result = parser.parse("api.ts", sampleCode)
+ val file = result.entities.find(_.kind == NodeKind.File).get
+ file.attrs.get("content") shouldBe None
+ }
+
+ it should "truncate summary to 120 chars" in {
+ val longSignature = "export async function veryLongFunctionNameThatExceedsTheLimit(paramA: string, paramB: number, paramC: boolean, paramD: SomeType, paramE: AnotherType): Promise<Result> {"
+ val longSource = longSignature + "\n return null;\n}\n"
+ val result = parser.parse("long.ts", longSource)
+ val func = result.entities.find(_.kind == NodeKind.Function).get
+ val summary = func.attrs("summary").asString.get
+ summary.length should be <= 120
+ }
+
+ // --- Tests using inline source ---
+
+ it should "create a File entity for the source file" in {
val result = parser.parse("app.ts", "const x = 1;")
result.entities.exists(e => e.name == "app.ts" && e.kind == NodeKind.File) shouldBe true
}
- // Test 2: Extract class
it should "extract class definitions" in {
val source = """
|export class UserService {
@@ -27,11 +103,9 @@ class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
""".stripMargin
val result = parser.parse("service.ts", source)
result.entities.exists(e => e.name == "UserService" && e.kind == NodeKind.Class) shouldBe true
- // File DEFINES UserService
result.relationships.exists(r => r.srcName == "service.ts" && r.dstName == "UserService" && r.predicate == "DEFINES") shouldBe true
}
- // Test 3: Extract functions
it should "extract function definitions" in {
val source = """
|export function calculateTotal(items: Item[]): number {
@@ -47,7 +121,6 @@ class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
result.entities.exists(e => e.name == "fetchData" && e.kind == NodeKind.Function) shouldBe true
}
- // Test 4: Extract arrow functions
it should "extract arrow function declarations" in {
val source = """
|export const handler = async (req: Request): Promise<Response> => {
@@ -58,8 +131,7 @@ class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
result.entities.exists(e => e.name == "handler" && e.kind == NodeKind.Function) shouldBe true
}
- // Test 5: Extract interfaces
- it should "extract interface definitions" in {
+ it should "extract interface definitions with ts_kind attr" in {
val source = """
|export interface UserProps {
| name: string;
@@ -69,11 +141,10 @@ class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
val result = parser.parse("types.ts", source)
val iface = result.entities.find(e => e.name == "UserProps")
iface shouldBe defined
- iface.get.kind shouldBe NodeKind.Class // using Class for interfaces
+ iface.get.kind shouldBe NodeKind.Class
iface.get.attrs.get("ts_kind").map(_.noSpaces) shouldBe Some("\"interface\"")
}
- // Test 6: Extract type aliases
it should "extract type alias definitions" in {
val source = """
|export type UserId = string;
@@ -84,8 +155,7 @@ class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
result.entities.exists(e => e.name == "Config") shouldBe true
}
- // Test 7: Extract imports
- it should "extract import relationships" in {
+ it should "extract import relationships from various import styles" in {
val source = """
|import { Router } from 'express';
|import * as path from 'node:path';
@@ -96,7 +166,6 @@ class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
result.relationships.exists(r => r.dstName == "node:path" && r.predicate == "IMPORTS") shouldBe true
}
- // Test 8: Extract method-in-class
it should "detect methods inside classes" in {
val source = """
|class Api {
@@ -110,13 +179,11 @@ class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
|}
""".stripMargin
val result = parser.parse("api.ts", source)
- // Api DEFINES fetch
result.relationships.exists(r => r.srcName == "Api" && r.dstName == "fetch" && r.predicate == "DEFINES") shouldBe true
result.relationships.exists(r => r.srcName == "Api" && r.dstName == "parse" && r.predicate == "DEFINES") shouldBe true
}
- // Test 9: Extract function calls
- it should "extract function call relationships" in {
+ it should "extract function call relationships from function bodies" in {
val source = """
|function main() {
| const data = loadConfig();
@@ -128,7 +195,6 @@ class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
result.relationships.exists(r => r.srcName == "main" && r.dstName == "processData" && r.predicate == "CALLS") shouldBe true
}
- // Test 10: Filter builtins from calls
it should "filter TypeScript builtins from CALLS" in {
val source = """
|function test() {
@@ -143,7 +209,6 @@ class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
result.relationships.exists(r => r.predicate == "CALLS" && r.dstName == "myFunction") shouldBe true
}
- // Test 11: Handles .tsx files
it should "parse .tsx files with JSX" in {
val source = """
|import React from 'react';
@@ -161,12 +226,4 @@ class TypeScriptParserSpec extends AnyFlatSpec with Matchers {
result.entities.exists(e => e.name == "Props") shouldBe true
result.relationships.exists(r => r.dstName == "react" && r.predicate == "IMPORTS") shouldBe true
}
-
- // Test 12: Language attrs set to "typescript"
- it should "set language attr to typescript" in {
- val result = parser.parse("app.ts", "const x = 1;")
- val fileEntity = result.entities.find(_.name == "app.ts")
- fileEntity shouldBe defined
- fileEntity.get.attrs.get("language").map(_.noSpaces) shouldBe Some("\"typescript\"")
- }
}