Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: dag trigger embedding (#123)
kkkwjx6a2ab4ffefc8
diff --git a/openviking/service/core.py b/openviking/service/core.pyindex e229faf6c..7077528a1 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -150,6 +150,11 @@ class OpenVikingService:"""Get SearchService instance."""return self._search_service+ @property+ def user(self) -> UserIdentifier:+ """Get current user identifier."""+ return self._user+@propertydef resources(self) -> ResourceService:"""Get ResourceService instance."""diff --git a/openviking/storage/observers/queue_observer.py b/openviking/storage/observers/queue_observer.pyindex 6665e6869..7b6713bb3 100644--- a/openviking/storage/observers/queue_observer.py+++ b/openviking/storage/observers/queue_observer.py@@ -82,9 +82,7 @@ class QueueObserver(BaseObserver):{"Queue": "Semantic-Nodes","Pending": getattr(dag_stats, "pending_nodes", 0) if dag_stats else 0,- "In Progress": getattr(dag_stats, "overview_scheduled_nodes", 0)- if dag_stats- else 0,+ "In Progress": getattr(dag_stats, "in_progress_nodes", 0) if dag_stats else 0,"Processed": getattr(dag_stats, "done_nodes", 0) if dag_stats else 0,"Errors": 0,"Total": getattr(dag_stats, "total_nodes", 0) if dag_stats else 0,diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex 98c432173..5954841d5 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -34,7 +34,7 @@ class DirNode:class DagStats:total_nodes: int = 0pending_nodes: int = 0- overview_scheduled_nodes: int = 0+ in_progress_nodes: int = 0done_nodes: int = 0@@ -94,7 +94,10 @@ class SemanticDagExecutor:for file_path in file_paths:self._stats.total_nodes += 1+ # File nodes are scheduled immediately: pending -> in_progress.self._stats.pending_nodes += 1+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)+ self._stats.in_progress_nodes += 1asyncio.create_task(self._file_summary_task(dir_uri, file_path))for child_uri in children_dirs:@@ -142,10 +145,23 @@ class SemanticDagExecutor:summary_dict = {"name": file_name, "summary": ""}finally:self._stats.done_nodes += 1- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)+ self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)await self._on_file_done(parent_uri, file_path, summary_dict)+ # Vectorize file as soon as summary is ready to avoid waiting for overview.+ try:+ asyncio.create_task(+ self._processor._vectorize_single_file(+ parent_uri=parent_uri,+ context_type=self._context_type,+ file_path=file_path,+ summary_dict=summary_dict,+ )+ )+ except Exception as e:+ logger.error(f"Failed to schedule vectorization for {file_path}: {e}", exc_info=True)+async def _on_file_done(self, parent_uri: str, file_path: str, summary_dict: Dict[str, str]) -> None:@@ -160,7 +176,8 @@ class SemanticDagExecutor:node.pending -= 1if node.pending == 0 and not node.overview_scheduled:node.overview_scheduled = True- self._stats.overview_scheduled_nodes += 1+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)+ self._stats.in_progress_nodes += 1asyncio.create_task(self._overview_task(parent_uri))async def _on_child_done(self, parent_uri: str, child_uri: str, abstract: str) -> None:@@ -176,7 +193,8 @@ class SemanticDagExecutor:node.pending -= 1if node.pending == 0 and not node.overview_scheduled:node.overview_scheduled = True- self._stats.overview_scheduled_nodes += 1+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)+ self._stats.in_progress_nodes += 1asyncio.create_task(self._overview_task(parent_uri))def _schedule_overview(self, dir_uri: str) -> None:@@ -186,7 +204,8 @@ class SemanticDagExecutor:if node.overview_scheduled:returnnode.overview_scheduled = True- self._stats.overview_scheduled_nodes += 1+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)+ self._stats.in_progress_nodes += 1asyncio.create_task(self._overview_task(dir_uri))def _finalize_file_summaries(self, node: DirNode) -> List[Dict[str, str]]:@@ -238,19 +257,12 @@ class SemanticDagExecutor:except Exception as e:logger.error(f"Failed to vectorize directory {dir_uri}: {e}", exc_info=True)- try:- await self._processor._vectorize_files(- dir_uri, self._context_type, node.file_paths, file_summaries- )- except Exception as e:- logger.error(f"Failed to vectorize files in {dir_uri}: {e}", exc_info=True)-except Exception as e:logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)abstract = ""finally:self._stats.done_nodes += 1- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)+ self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)parent_uri = self._parent.get(dir_uri)if parent_uri is None:@@ -264,7 +276,7 @@ class SemanticDagExecutor:return DagStats(total_nodes=self._stats.total_nodes,pending_nodes=self._stats.pending_nodes,- overview_scheduled_nodes=self._stats.overview_scheduled_nodes,+ in_progress_nodes=self._stats.in_progress_nodes,done_nodes=self._stats.done_nodes,)diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 9922b6f2a..2e5d1aac8 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -206,7 +206,9 @@ class SemanticProcessor(DequeueHandlerBase):children_abstracts = await self._collect_children_abstracts(children_uris)# 2. Concurrently generate summaries for files in directory- file_summaries = await self._generate_file_summaries(file_paths)+ file_summaries = await self._generate_file_summaries(+ file_paths, context_type=context_type, parent_uri=uri, enqueue_files=True+ )# 3. Generate .overview.md (contains brief description)overview = await self._generate_overview(uri, file_summaries, children_abstracts)@@ -226,13 +228,6 @@ class SemanticProcessor(DequeueHandlerBase):except Exception as e:logger.error(f"Failed to vectorize directory {uri}: {e}", exc_info=True)- # 7. Vectorize files- # Requires retrieval refactoring before enabling- try:- await self._vectorize_files(uri, context_type, file_paths, file_summaries)- except Exception as e:- logger.error(f"Failed to vectorize files in {uri}: {e}", exc_info=True)-async def _collect_children_abstracts(self, children_uris: List[str]) -> List[Dict[str, str]]:"""Collect .abstract.md from subdirectories."""viking_fs = get_viking_fs()@@ -244,7 +239,13 @@ class SemanticProcessor(DequeueHandlerBase):results.append({"name": dir_name, "abstract": abstract})return results- async def _generate_file_summaries(self, file_paths: List[str]) -> List[Dict[str, str]]:+ async def _generate_file_summaries(+ self,+ file_paths: List[str],+ context_type: Optional[str] = None,+ parent_uri: Optional[str] = None,+ enqueue_files: bool = False,+ ) -> List[Dict[str, str]]:"""Concurrently generate file summaries."""if not file_paths:return []@@ -253,7 +254,21 @@ class SemanticProcessor(DequeueHandlerBase):async def generate_one_summary(file_path: str) -> Dict[str, str]:async with sem:- return await self._generate_single_file_summary(file_path)+ summary = await self._generate_single_file_summary(file_path)+ if enqueue_files and context_type and parent_uri:+ try:+ await self._vectorize_single_file(+ parent_uri=parent_uri,+ context_type=context_type,+ file_path=file_path,+ summary_dict=summary,+ )+ except Exception as e:+ logger.error(+ f"Failed to vectorize file {file_path}: {e}",+ exc_info=True,+ )+ return summarytasks = [generate_one_summary(fp) for fp in file_paths]return await asyncio.gather(*tasks)@@ -435,37 +450,66 @@ class SemanticProcessor(DequeueHandlerBase):file_summaries: List[Dict[str, str]],) -> None:"""Vectorize files in directory."""- from datetime import datetime-- from openviking.core.context import Contextfrom openviking.storage.queuefs import get_queue_manager- from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverterqueue_manager = get_queue_manager()embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)for file_path, file_summary_dict in zip(file_paths, file_summaries):- file_name = file_summary_dict["name"]- summary = file_summary_dict["summary"]+ await self._vectorize_single_file(+ parent_uri=uri,+ context_type=context_type,+ file_path=file_path,+ summary_dict=file_summary_dict,+ embedding_queue=embedding_queue,+ )++ async def _vectorize_single_file(+ self,+ parent_uri: str,+ context_type: str,+ file_path: str,+ summary_dict: Dict[str, str],+ embedding_queue: Optional[Any] = None,+ ) -> None:+ """Vectorize a single file using its content or summary."""+ from datetime import datetime++ from openviking.storage.queuefs import get_queue_manager+ from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter++ try:+ file_name = summary_dict.get("name") or file_path.split("/")[-1]+ summary = summary_dict.get("summary", "")++ if embedding_queue is None:+ queue_manager = get_queue_manager()+ embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)context = Context(uri=file_path,- parent_uri=uri,+ parent_uri=parent_uri,is_leaf=True,abstract=summary,context_type=context_type,created_at=datetime.now(),)+if self.get_resource_content_type(file_name) == ResourceContentType.TEXT:content = await get_viking_fs().read_file(file_path)context.set_vectorize(Vectorize(text=content))elif summary:context.set_vectorize(Vectorize(text=summary))else:- continue+ return+embedding_msg = EmbeddingMsgConverter.from_context(context)+ if not embedding_msg:+ returnawait embedding_queue.enqueue(embedding_msg) # type: ignorelogger.debug(f"Enqueued file for vectorization: {file_path}")+ except Exception as e:+ logger.error(f"Failed to vectorize file {file_path}: {e}", exc_info=True)def get_resource_content_type(self, file_name: str) -> ResourceContentType:def _is_image_file(file_name: str) -> bool: