Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: improve file type detection and C/C++ AST extraction (#497)
yangxinxin-7b7740770e82f
diff --git a/openviking/parse/parsers/code/ast/languages/cpp.py b/openviking/parse/parsers/code/ast/languages/cpp.pyindex b3237675f..b48c4993a 100644--- a/openviking/parse/parsers/code/ast/languages/cpp.py+++ b/openviking/parse/parsers/code/ast/languages/cpp.py@@ -2,7 +2,7 @@# SPDX-License-Identifier: Apache-2.0"""C/C++ AST extractor using tree-sitter-cpp."""-from typing import List+from typing import List, Optionalfrom openviking.parse.parsers.code.ast.languages.base import LanguageExtractorfrom openviking.parse.parsers.code.ast.skeleton import ClassSkeleton, CodeSkeleton, FunctionSig@@ -134,6 +134,60 @@ def _extract_class(node, content_bytes: bytes, docstring: str = "") -> ClassSkelreturn ClassSkeleton(name=name, bases=bases, docstring=docstring, methods=methods)+def _extract_typedef_struct(node, content_bytes: bytes, docstring: str = "") -> Optional[ClassSkeleton]:+ """Handle typedef struct { ... } Name; and typedef struct Tag { ... } Name;++ tree-sitter-cpp emits this as a 'type_definition' node with children:+ 'typedef', struct_specifier, type_identifier (the alias), ';'+ """+ struct_node = None+ typedef_name = ""++ for child in node.children:+ if child.type in ("struct_specifier", "class_specifier"):+ struct_node = child+ elif child.type == "type_identifier" and struct_node is not None:+ typedef_name = _node_text(child, content_bytes)++ if struct_node is None:+ return None++ skeleton = _extract_class(struct_node, content_bytes, docstring=docstring)+ # Prefer the typedef alias as the canonical name+ if typedef_name:+ skeleton.name = typedef_name+ return skeleton if skeleton.name else None+++def _extract_function_proto(node, content_bytes: bytes, docstring: str = "") -> Optional[FunctionSig]:+ """Extract a function prototype from a top-level declaration node."""+ fn = _extract_function(node, content_bytes, docstring=docstring)+ return fn if fn.name else None+++def _process_siblings(+ siblings: list,+ content_bytes: bytes,+ classes: List[ClassSkeleton],+ functions: List[FunctionSig],+) -> None:+ """Extract classes and functions from a list of sibling nodes (shared by top-level and namespace)."""+ for idx, child in enumerate(siblings):+ doc = _preceding_doc(siblings, idx, content_bytes)+ if child.type in ("class_specifier", "struct_specifier"):+ classes.append(_extract_class(child, content_bytes, docstring=doc))+ elif child.type == "function_definition":+ functions.append(_extract_function(child, content_bytes, docstring=doc))+ elif child.type == "type_definition":+ cls = _extract_typedef_struct(child, content_bytes, docstring=doc)+ if cls:+ classes.append(cls)+ elif child.type == "declaration":+ fn = _extract_function_proto(child, content_bytes, docstring=doc)+ if fn:+ functions.append(fn)++class CppExtractor(LanguageExtractor):def __init__(self):import tree_sitter_cpp as tscpp@@ -152,31 +206,20 @@ class CppExtractor(LanguageExtractor):functions: List[FunctionSig] = []siblings = list(root.children)+ top_level = []for idx, child in enumerate(siblings):if child.type == "preproc_include":for sub in child.children:if sub.type in ("string_literal", "system_lib_string"):raw = _node_text(sub, content_bytes).strip().strip('"<>')imports.append(raw)- elif child.type in ("class_specifier", "struct_specifier"):- doc = _preceding_doc(siblings, idx, content_bytes)- classes.append(_extract_class(child, content_bytes, docstring=doc))- elif child.type == "function_definition":- doc = _preceding_doc(siblings, idx, content_bytes)- functions.append(_extract_function(child, content_bytes, docstring=doc))elif child.type == "namespace_definition":for sub in child.children:if sub.type == "declaration_list":- inner = list(sub.children)- for i2, s2 in enumerate(inner):- if s2.type in ("class_specifier", "struct_specifier"):- doc = _preceding_doc(inner, i2, content_bytes)- classes.append(_extract_class(s2, content_bytes, docstring=doc))- elif s2.type == "function_definition":- doc = _preceding_doc(inner, i2, content_bytes)- functions.append(- _extract_function(s2, content_bytes, docstring=doc)- )+ _process_siblings(list(sub.children), content_bytes, classes, functions)+ else:+ top_level.append(child)+ _process_siblings(top_level, content_bytes, classes, functions)return CodeSkeleton(file_name=file_name,diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex ca2ba36a9..1e918eb43 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -30,8 +30,11 @@ def _owner_space_for_uri(uri: str, ctx: RequestContext) -> str:return ""-def get_resource_content_type(file_name: str) -> ResourceContentType:- """Determine resource content type based on file extension."""+def get_resource_content_type(file_name: str) -> Optional[ResourceContentType]:+ """Determine resource content type based on file extension.++ Returns None if the file type is not recognized.+ """file_name = file_name.lower()text_extensions = {@@ -49,6 +52,48 @@ def get_resource_content_type(file_name: str) -> ResourceContentType:".h",".go",".rs",+ ".lua",+ ".rb",+ ".php",+ ".sh",+ ".bash",+ ".zsh",+ ".fish",+ ".sql",+ ".kt",+ ".swift",+ ".scala",+ ".r",+ ".m",+ ".pl",+ ".toml",+ ".yaml",+ ".yml",+ ".ini",+ ".cfg",+ ".conf",+ ".tsx",+ ".jsx",+ ".cs",+ ".env",+ ".properties",+ ".rst",+ ".tf",+ ".proto",+ ".gradle",+ ".cc",+ ".cxx",+ ".hpp",+ ".hh",+ ".dart",+ ".vue",+ ".groovy",+ ".ps1",+ ".ex",+ ".exs",+ ".erl",+ ".jl",+ ".mm",}image_extensions = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".svg", ".webp"}video_extensions = {".mp4", ".avi", ".mov", ".wmv", ".flv"}@@ -63,7 +108,7 @@ def get_resource_content_type(file_name: str) -> ResourceContentType:elif any(file_name.endswith(ext) for ext in audio_extensions):return ResourceContentType.AUDIO- return ResourceContentType.UNKNOWN+ return Noneasync def vectorize_directory_meta(@@ -163,7 +208,19 @@ async def vectorize_file()content_type = get_resource_content_type(file_name)- if content_type == ResourceContentType.TEXT:+ if content_type is None:+ # Unsupported file type: fall back to summary if available+ if summary:+ logger.warning(+ f"Unsupported file type for {file_path}, falling back to summary for vectorization"+ )+ context.set_vectorize(Vectorize(text=summary))+ else:+ logger.warning(+ f"Unsupported file type for {file_path} and no summary available, skipping vectorization"+ )+ return+ elif content_type == ResourceContentType.TEXT:# For text files, try to read contenttry:content = await viking_fs.read_file(file_path, ctx=ctx)diff --git a/tests/parse/test_ast_extractor.py b/tests/parse/test_ast_extractor.pyindex d887f7cfe..99c17d3a7 100644--- a/tests/parse/test_ast_extractor.py+++ b/tests/parse/test_ast_extractor.py@@ -705,6 +705,120 @@ int add(int a, int b) {assert "@param a First operand" in textassert "@return Sum of a and b" in text+ def test_typedef_struct_anonymous(self):+ code = """+/* A 2D point. */+typedef struct {+ float x;+ float y;+} Point;++/* An RGB color value. */+typedef struct {+ int r;+ int g;+ int b;+} Color;+"""+ sk = self.e.extract("types.h", code)+ names = {c.name for c in sk.classes}+ assert "Point" in names+ assert "Color" in names++ def test_typedef_struct_named_tag(self):+ # typedef struct Node { ... } Node; — tag and alias are the same+ code = """+typedef struct Node {+ int value;+ struct Node *next;+} Node;+"""+ sk = self.e.extract("list.h", code)+ names = {c.name for c in sk.classes}+ assert "Node" in names++ def test_typedef_struct_docstring(self):+ code = """+/** Represents a rectangle with width and height. */+typedef struct {+ int width;+ int height;+} Rect;+"""+ sk = self.e.extract("rect.h", code)+ rect = next((c for c in sk.classes if c.name == "Rect"), None)+ assert rect is not None+ assert "Represents a rectangle" in rect.docstring++ def test_function_prototype_top_level(self):+ # .h header with only function declarations (no bodies)+ code = """+#include <stddef.h>++/* Allocate n bytes of memory. */+void *my_malloc(size_t n);++/* Free previously allocated memory. */+void my_free(void *ptr);+"""+ sk = self.e.extract("mem.h", code)+ names = {f.name for f in sk.functions}+ assert "my_malloc" in names+ assert "my_free" in names++ def test_function_prototype_return_type(self):+ code = """+int compute(int a, int b);+void reset(void);+"""+ sk = self.e.extract("utils.h", code)+ print(sk.to_text())+ fns = {f.name: f for f in sk.functions}+ assert fns["compute"].return_type == "int"+ assert fns["reset"].return_type == "void"++ def test_function_prototype_docstring(self):+ code = """+/** Add two integers and return the result. */+int add(int a, int b);+"""+ sk = self.e.extract("math.h", code)+ fns = {f.name: f for f in sk.functions}+ assert "Add two integers" in fns["add"].docstring++ def test_namespace_typedef_and_proto(self):+ code = """+namespace utils {++typedef struct {+ int id;+} Handle;++int create(int flags);++}+"""+ sk = self.e.extract("utils.cpp", code)+ names = {c.name for c in sk.classes}+ assert "Handle" in names+ fns = {f.name for f in sk.functions}+ assert "create" in fns++ def test_declaration_and_definition_both_extracted(self):+ # Forward declaration + definition in the same file — both appear in skeleton+ code = """+/* Forward declaration */+int add(int a, int b);++/* Definition */+int add(int a, int b) {+ return a + b;+}+"""+ sk = self.e.extract("math.cpp", code)+ names = [f.name for f in sk.functions]+ assert names.count("add") == 2+# ---------------------------------------------------------------------------# Rust