Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: improve file type detection and C/C++ AST extraction (#497)

yangxinxin-7b7740770e82f
diff --git a/openviking/parse/parsers/code/ast/languages/cpp.py b/openviking/parse/parsers/code/ast/languages/cpp.py
index b3237675f..b48c4993a 100644
--- a/openviking/parse/parsers/code/ast/languages/cpp.py
+++ b/openviking/parse/parsers/code/ast/languages/cpp.py
@@ -2,7 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
"""C/C++ AST extractor using tree-sitter-cpp."""
-from typing import List
+from typing import List, Optional
from openviking.parse.parsers.code.ast.languages.base import LanguageExtractor
from openviking.parse.parsers.code.ast.skeleton import ClassSkeleton, CodeSkeleton, FunctionSig
@@ -134,6 +134,60 @@ def _extract_class(node, content_bytes: bytes, docstring: str = "") -> ClassSkel
return ClassSkeleton(name=name, bases=bases, docstring=docstring, methods=methods)
+def _extract_typedef_struct(node, content_bytes: bytes, docstring: str = "") -> Optional[ClassSkeleton]:
+ """Handle typedef struct { ... } Name; and typedef struct Tag { ... } Name;
+
+ tree-sitter-cpp emits this as a 'type_definition' node with children:
+ 'typedef', struct_specifier, type_identifier (the alias), ';'
+ """
+ struct_node = None
+ typedef_name = ""
+
+ for child in node.children:
+ if child.type in ("struct_specifier", "class_specifier"):
+ struct_node = child
+ elif child.type == "type_identifier" and struct_node is not None:
+ typedef_name = _node_text(child, content_bytes)
+
+ if struct_node is None:
+ return None
+
+ skeleton = _extract_class(struct_node, content_bytes, docstring=docstring)
+ # Prefer the typedef alias as the canonical name
+ if typedef_name:
+ skeleton.name = typedef_name
+ return skeleton if skeleton.name else None
+
+
+def _extract_function_proto(node, content_bytes: bytes, docstring: str = "") -> Optional[FunctionSig]:
+ """Extract a function prototype from a top-level declaration node."""
+ fn = _extract_function(node, content_bytes, docstring=docstring)
+ return fn if fn.name else None
+
+
+def _process_siblings(
+ siblings: list,
+ content_bytes: bytes,
+ classes: List[ClassSkeleton],
+ functions: List[FunctionSig],
+) -> None:
+ """Extract classes and functions from a list of sibling nodes (shared by top-level and namespace)."""
+ for idx, child in enumerate(siblings):
+ doc = _preceding_doc(siblings, idx, content_bytes)
+ if child.type in ("class_specifier", "struct_specifier"):
+ classes.append(_extract_class(child, content_bytes, docstring=doc))
+ elif child.type == "function_definition":
+ functions.append(_extract_function(child, content_bytes, docstring=doc))
+ elif child.type == "type_definition":
+ cls = _extract_typedef_struct(child, content_bytes, docstring=doc)
+ if cls:
+ classes.append(cls)
+ elif child.type == "declaration":
+ fn = _extract_function_proto(child, content_bytes, docstring=doc)
+ if fn:
+ functions.append(fn)
+
+
class CppExtractor(LanguageExtractor):
def __init__(self):
import tree_sitter_cpp as tscpp
@@ -152,31 +206,20 @@ class CppExtractor(LanguageExtractor):
functions: List[FunctionSig] = []
siblings = list(root.children)
+ top_level = []
for idx, child in enumerate(siblings):
if child.type == "preproc_include":
for sub in child.children:
if sub.type in ("string_literal", "system_lib_string"):
raw = _node_text(sub, content_bytes).strip().strip('"<>')
imports.append(raw)
- elif child.type in ("class_specifier", "struct_specifier"):
- doc = _preceding_doc(siblings, idx, content_bytes)
- classes.append(_extract_class(child, content_bytes, docstring=doc))
- elif child.type == "function_definition":
- doc = _preceding_doc(siblings, idx, content_bytes)
- functions.append(_extract_function(child, content_bytes, docstring=doc))
elif child.type == "namespace_definition":
for sub in child.children:
if sub.type == "declaration_list":
- inner = list(sub.children)
- for i2, s2 in enumerate(inner):
- if s2.type in ("class_specifier", "struct_specifier"):
- doc = _preceding_doc(inner, i2, content_bytes)
- classes.append(_extract_class(s2, content_bytes, docstring=doc))
- elif s2.type == "function_definition":
- doc = _preceding_doc(inner, i2, content_bytes)
- functions.append(
- _extract_function(s2, content_bytes, docstring=doc)
- )
+ _process_siblings(list(sub.children), content_bytes, classes, functions)
+ else:
+ top_level.append(child)
+ _process_siblings(top_level, content_bytes, classes, functions)
return CodeSkeleton(
file_name=file_name,
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index ca2ba36a9..1e918eb43 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -30,8 +30,11 @@ def _owner_space_for_uri(uri: str, ctx: RequestContext) -> str:
return ""
-def get_resource_content_type(file_name: str) -> ResourceContentType:
- """Determine resource content type based on file extension."""
+def get_resource_content_type(file_name: str) -> Optional[ResourceContentType]:
+ """Determine resource content type based on file extension.
+
+ Returns None if the file type is not recognized.
+ """
file_name = file_name.lower()
text_extensions = {
@@ -49,6 +52,48 @@ def get_resource_content_type(file_name: str) -> ResourceContentType:
".h",
".go",
".rs",
+ ".lua",
+ ".rb",
+ ".php",
+ ".sh",
+ ".bash",
+ ".zsh",
+ ".fish",
+ ".sql",
+ ".kt",
+ ".swift",
+ ".scala",
+ ".r",
+ ".m",
+ ".pl",
+ ".toml",
+ ".yaml",
+ ".yml",
+ ".ini",
+ ".cfg",
+ ".conf",
+ ".tsx",
+ ".jsx",
+ ".cs",
+ ".env",
+ ".properties",
+ ".rst",
+ ".tf",
+ ".proto",
+ ".gradle",
+ ".cc",
+ ".cxx",
+ ".hpp",
+ ".hh",
+ ".dart",
+ ".vue",
+ ".groovy",
+ ".ps1",
+ ".ex",
+ ".exs",
+ ".erl",
+ ".jl",
+ ".mm",
}
image_extensions = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".svg", ".webp"}
video_extensions = {".mp4", ".avi", ".mov", ".wmv", ".flv"}
@@ -63,7 +108,7 @@ def get_resource_content_type(file_name: str) -> ResourceContentType:
elif any(file_name.endswith(ext) for ext in audio_extensions):
return ResourceContentType.AUDIO
- return ResourceContentType.UNKNOWN
+ return None
async def vectorize_directory_meta(
@@ -163,7 +208,19 @@ async def vectorize_file(
)
content_type = get_resource_content_type(file_name)
- if content_type == ResourceContentType.TEXT:
+ if content_type is None:
+ # Unsupported file type: fall back to summary if available
+ if summary:
+ logger.warning(
+ f"Unsupported file type for {file_path}, falling back to summary for vectorization"
+ )
+ context.set_vectorize(Vectorize(text=summary))
+ else:
+ logger.warning(
+ f"Unsupported file type for {file_path} and no summary available, skipping vectorization"
+ )
+ return
+ elif content_type == ResourceContentType.TEXT:
# For text files, try to read content
try:
content = await viking_fs.read_file(file_path, ctx=ctx)
diff --git a/tests/parse/test_ast_extractor.py b/tests/parse/test_ast_extractor.py
index d887f7cfe..99c17d3a7 100644
--- a/tests/parse/test_ast_extractor.py
+++ b/tests/parse/test_ast_extractor.py
@@ -705,6 +705,120 @@ int add(int a, int b) {
assert "@param a First operand" in text
assert "@return Sum of a and b" in text
+ def test_typedef_struct_anonymous(self):
+ code = """
+/* A 2D point. */
+typedef struct {
+ float x;
+ float y;
+} Point;
+
+/* An RGB color value. */
+typedef struct {
+ int r;
+ int g;
+ int b;
+} Color;
+"""
+ sk = self.e.extract("types.h", code)
+ names = {c.name for c in sk.classes}
+ assert "Point" in names
+ assert "Color" in names
+
+ def test_typedef_struct_named_tag(self):
+ # typedef struct Node { ... } Node; — tag and alias are the same
+ code = """
+typedef struct Node {
+ int value;
+ struct Node *next;
+} Node;
+"""
+ sk = self.e.extract("list.h", code)
+ names = {c.name for c in sk.classes}
+ assert "Node" in names
+
+ def test_typedef_struct_docstring(self):
+ code = """
+/** Represents a rectangle with width and height. */
+typedef struct {
+ int width;
+ int height;
+} Rect;
+"""
+ sk = self.e.extract("rect.h", code)
+ rect = next((c for c in sk.classes if c.name == "Rect"), None)
+ assert rect is not None
+ assert "Represents a rectangle" in rect.docstring
+
+ def test_function_prototype_top_level(self):
+ # .h header with only function declarations (no bodies)
+ code = """
+#include <stddef.h>
+
+/* Allocate n bytes of memory. */
+void *my_malloc(size_t n);
+
+/* Free previously allocated memory. */
+void my_free(void *ptr);
+"""
+ sk = self.e.extract("mem.h", code)
+ names = {f.name for f in sk.functions}
+ assert "my_malloc" in names
+ assert "my_free" in names
+
+ def test_function_prototype_return_type(self):
+ code = """
+int compute(int a, int b);
+void reset(void);
+"""
+ sk = self.e.extract("utils.h", code)
+ print(sk.to_text())
+ fns = {f.name: f for f in sk.functions}
+ assert fns["compute"].return_type == "int"
+ assert fns["reset"].return_type == "void"
+
+ def test_function_prototype_docstring(self):
+ code = """
+/** Add two integers and return the result. */
+int add(int a, int b);
+"""
+ sk = self.e.extract("math.h", code)
+ fns = {f.name: f for f in sk.functions}
+ assert "Add two integers" in fns["add"].docstring
+
+ def test_namespace_typedef_and_proto(self):
+ code = """
+namespace utils {
+
+typedef struct {
+ int id;
+} Handle;
+
+int create(int flags);
+
+}
+"""
+ sk = self.e.extract("utils.cpp", code)
+ names = {c.name for c in sk.classes}
+ assert "Handle" in names
+ fns = {f.name for f in sk.functions}
+ assert "create" in fns
+
+ def test_declaration_and_definition_both_extracted(self):
+ # Forward declaration + definition in the same file — both appear in skeleton
+ code = """
+/* Forward declaration */
+int add(int a, int b);
+
+/* Definition */
+int add(int a, int b) {
+ return a + b;
+}
+"""
+ sk = self.e.extract("math.cpp", code)
+ names = [f.name for f in sk.functions]
+ assert names.count("add") == 2
+
# ---------------------------------------------------------------------------
# Rust