Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(embedder): use summary for file embedding in semantic pipeline (#765)
yangxinxin-759352f825f7c
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex 79213f718..55cacc691 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -57,6 +57,7 @@ class VectorizeTask:file_path: Optional[str] = Nonesummary_dict: Optional[Dict[str, str]] = Noneparent_uri: Optional[str] = None+ use_summary: bool = False# For directory tasksabstract: Optional[str] = Noneoverview: Optional[str] = None@@ -76,6 +77,7 @@ class SemanticDagExecutor:semantic_msg_id: Optional[str] = None,recursive: bool = True,lifecycle_lock_handle_id: str = "",+ is_code_repo: bool = False,):self._processor = processorself._context_type = context_type@@ -86,6 +88,7 @@ class SemanticDagExecutor:self._semantic_msg_id = semantic_msg_idself._recursive = recursiveself._lifecycle_lock_handle_id = lifecycle_lock_handle_id+ self._is_code_repo = is_code_repoself._llm_sem = asyncio.Semaphore(max_concurrent_llm)self._viking_fs = get_viking_fs()self._nodes: Dict[str, DirNode] = {}@@ -192,6 +195,7 @@ class SemanticDagExecutor:summary_dict=task.summary_dict,ctx=task.ctx,semantic_msg_id=task.semantic_msg_id,+ use_summary=task.use_summary,))else:@@ -432,6 +436,7 @@ class SemanticDagExecutor:try:if need_vectorize:+ use_summary = self._is_code_repo and bool(summary_dict.get("summary"))task = VectorizeTask(task_type="file",uri=file_path,@@ -441,6 +446,7 @@ class SemanticDagExecutor:file_path=file_path,summary_dict=summary_dict,parent_uri=parent_uri,+ use_summary=use_summary,)await self._add_vectorize_task(task)except Exception as e:diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.pyindex 720948e81..fbd56647e 100644--- a/openviking/storage/queuefs/semantic_msg.py+++ b/openviking/storage/queuefs/semantic_msg.py@@ -40,6 +40,7 @@ class SemanticMsg:telemetry_id: str = ""target_uri: str = ""lifecycle_lock_handle_id: str = ""+ is_code_repo: bool = Falsechanges: Optional[Dict[str, List[str]]] = (None # {"added": [...], "modified": [...], "deleted": [...]})@@ -57,6 +58,7 @@ class SemanticMsg:telemetry_id: str = "",target_uri: str = "",lifecycle_lock_handle_id: str = "",+ is_code_repo: bool = False,changes: Optional[Dict[str, List[str]]] = None,):self.id = str(uuid4())@@ -71,6 +73,7 @@ class SemanticMsg:self.telemetry_id = telemetry_idself.target_uri = target_uriself.lifecycle_lock_handle_id = lifecycle_lock_handle_id+ self.is_code_repo = is_code_repoself.changes = changesdef to_dict(self) -> Dict[str, Any]:@@ -110,6 +113,7 @@ class SemanticMsg:telemetry_id=data.get("telemetry_id", ""),target_uri=data.get("target_uri", ""),lifecycle_lock_handle_id=data.get("lifecycle_lock_handle_id", ""),+ is_code_repo=data.get("is_code_repo", False),changes=data.get("changes"),)if "id" in data and data["id"]:diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 2b92b73b9..cf164177a 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -264,6 +264,7 @@ class SemanticProcessor(DequeueHandlerBase):semantic_msg_id=msg.id,recursive=msg.recursive,lifecycle_lock_handle_id=msg.lifecycle_lock_handle_id,+ is_code_repo=msg.is_code_repo,)self._dag_executor = executorawait executor.run(msg.uri)@@ -651,6 +652,9 @@ class SemanticProcessor(DequeueHandlerBase):verbose = code_mode == "ast_llm"skeleton_text = extract_skeleton(file_name, content, verbose=verbose)if skeleton_text:+ max_skeleton_chars = get_openviking_config().semantic.max_skeleton_chars+ if len(skeleton_text) > max_skeleton_chars:+ skeleton_text = skeleton_text[:max_skeleton_chars]if code_mode == "ast":return {"name": file_name, "summary": skeleton_text}else: # ast_llm@@ -1069,6 +1073,7 @@ class SemanticProcessor(DequeueHandlerBase):summary_dict: Dict[str, str],ctx: Optional[RequestContext] = None,semantic_msg_id: Optional[str] = None,+ use_summary: bool = False,) -> None:"""Vectorize a single file using its content or summary."""from openviking.utils.embedding_utils import vectorize_file@@ -1081,4 +1086,5 @@ class SemanticProcessor(DequeueHandlerBase):context_type=context_type,ctx=active_ctx,semantic_msg_id=semantic_msg_id,+ use_summary=use_summary,)diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex 9a2f9d2d5..35aeadccb 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -213,12 +213,14 @@ async def vectorize_file(context_type: str = "resource",ctx: Optional[RequestContext] = None,semantic_msg_id: Optional[str] = None,+ use_summary: bool = False,) -> None:"""Vectorize a single file.Creates Context object for the file and enqueues it.- Reads content for TEXT files, otherwise uses summary.+ If use_summary=True and summary is available, uses summary for TEXT files (e.g. code scenario).+ Otherwise reads raw file content for TEXT files, falls back to summary on failure."""enqueued = False@@ -260,21 +262,27 @@ async def vectorize_file()returnelif content_type == ResourceContentType.TEXT:- # For text files, try to read content- try:- content = await viking_fs.read_file(file_path, ctx=ctx)- if isinstance(content, bytes):- content = content.decode("utf-8", errors="replace")- context.set_vectorize(Vectorize(text=content))- except Exception as e:- logger.warning(- f"Failed to read file content for {file_path}, falling back to summary: {e}"- )- if summary:- context.set_vectorize(Vectorize(text=summary))- else:- logger.warning(f"No summary available for {file_path}, skipping vectorization")- return+ if use_summary and summary:+ # Code scenario: use pre-generated summary (e.g. AST skeleton) for embedding+ context.set_vectorize(Vectorize(text=summary))+ else:+ # Default: read raw file content+ try:+ content = await viking_fs.read_file(file_path, ctx=ctx)+ if isinstance(content, bytes):+ content = content.decode("utf-8", errors="replace")+ context.set_vectorize(Vectorize(text=content))+ except Exception as e:+ logger.warning(+ f"Failed to read file content for {file_path}, falling back to summary: {e}"+ )+ if summary:+ context.set_vectorize(Vectorize(text=summary))+ else:+ logger.warning(+ f"No summary available for {file_path}, skipping vectorization"+ )+ returnelif summary:# For non-text files, use summarycontext.set_vectorize(Vectorize(text=summary))diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex ef6185dc3..9dd61f971 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -265,6 +265,7 @@ class ResourceProcessor:should_summarize = summarize or build_indexif should_summarize:skip_vec = not build_index+ is_code_repo = parse_result.source_format == "repository"try:with telemetry.measure("resource.summarize"):await self._get_summarizer().summarize(@@ -273,6 +274,7 @@ class ResourceProcessor:skip_vectorization=skip_vec,lifecycle_lock_handle_id=lifecycle_lock_handle_id,temp_uris=[temp_uri_for_summarize],+ is_code_repo=is_code_repo,**kwargs,)except Exception as e:diff --git a/openviking/utils/summarizer.py b/openviking/utils/summarizer.pyindex e9a1cb20c..dff099e62 100644--- a/openviking/utils/summarizer.py+++ b/openviking/utils/summarizer.py@@ -74,6 +74,7 @@ class Summarizer:telemetry_id=telemetry.telemetry_id if telemetry.enabled else "",target_uri=uri if uri != temp_uri else None,lifecycle_lock_handle_id=lifecycle_lock_handle_id,+ is_code_repo=kwargs.get("is_code_repo", False),)await semantic_queue.enqueue(msg)enqueued_count += 1diff --git a/openviking_cli/utils/config/parser_config.py b/openviking_cli/utils/config/parser_config.pyindex 5cf3860f2..717375646 100644--- a/openviking_cli/utils/config/parser_config.py+++ b/openviking_cli/utils/config/parser_config.py@@ -492,6 +492,9 @@ class SemanticConfig:max_file_content_chars: int = 30000"""Maximum characters of file content sent to LLM for summary generation."""+ max_skeleton_chars: int = 12000+ """Maximum characters of AST skeleton used for embedding (~3000 tokens)."""+max_overview_prompt_chars: int = 60000"""Maximum characters allowed in the overview generation prompt.If exceeded, file summaries are batched and merged."""