Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: dag trigger embedding (#123)

kkkwjx6a2ab4ffefc8
diff --git a/openviking/service/core.py b/openviking/service/core.py
index e229faf6c..7077528a1 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -150,6 +150,11 @@ class OpenVikingService:
"""Get SearchService instance."""
return self._search_service
+ @property
+ def user(self) -> UserIdentifier:
+ """Get current user identifier."""
+ return self._user
+
@property
def resources(self) -> ResourceService:
"""Get ResourceService instance."""
diff --git a/openviking/storage/observers/queue_observer.py b/openviking/storage/observers/queue_observer.py
index 6665e6869..7b6713bb3 100644
--- a/openviking/storage/observers/queue_observer.py
+++ b/openviking/storage/observers/queue_observer.py
@@ -82,9 +82,7 @@ class QueueObserver(BaseObserver):
{
"Queue": "Semantic-Nodes",
"Pending": getattr(dag_stats, "pending_nodes", 0) if dag_stats else 0,
- "In Progress": getattr(dag_stats, "overview_scheduled_nodes", 0)
- if dag_stats
- else 0,
+ "In Progress": getattr(dag_stats, "in_progress_nodes", 0) if dag_stats else 0,
"Processed": getattr(dag_stats, "done_nodes", 0) if dag_stats else 0,
"Errors": 0,
"Total": getattr(dag_stats, "total_nodes", 0) if dag_stats else 0,
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
index 98c432173..5954841d5 100644
--- a/openviking/storage/queuefs/semantic_dag.py
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -34,7 +34,7 @@ class DirNode:
class DagStats:
total_nodes: int = 0
pending_nodes: int = 0
- overview_scheduled_nodes: int = 0
+ in_progress_nodes: int = 0
done_nodes: int = 0
@@ -94,7 +94,10 @@ class SemanticDagExecutor:
for file_path in file_paths:
self._stats.total_nodes += 1
+ # File nodes are scheduled immediately: pending -> in_progress.
self._stats.pending_nodes += 1
+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
+ self._stats.in_progress_nodes += 1
asyncio.create_task(self._file_summary_task(dir_uri, file_path))
for child_uri in children_dirs:
@@ -142,10 +145,23 @@ class SemanticDagExecutor:
summary_dict = {"name": file_name, "summary": ""}
finally:
self._stats.done_nodes += 1
- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
+ self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)
await self._on_file_done(parent_uri, file_path, summary_dict)
+ # Vectorize file as soon as summary is ready to avoid waiting for overview.
+ try:
+ asyncio.create_task(
+ self._processor._vectorize_single_file(
+ parent_uri=parent_uri,
+ context_type=self._context_type,
+ file_path=file_path,
+ summary_dict=summary_dict,
+ )
+ )
+ except Exception as e:
+ logger.error(f"Failed to schedule vectorization for {file_path}: {e}", exc_info=True)
+
async def _on_file_done(
self, parent_uri: str, file_path: str, summary_dict: Dict[str, str]
) -> None:
@@ -160,7 +176,8 @@ class SemanticDagExecutor:
node.pending -= 1
if node.pending == 0 and not node.overview_scheduled:
node.overview_scheduled = True
- self._stats.overview_scheduled_nodes += 1
+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
+ self._stats.in_progress_nodes += 1
asyncio.create_task(self._overview_task(parent_uri))
async def _on_child_done(self, parent_uri: str, child_uri: str, abstract: str) -> None:
@@ -176,7 +193,8 @@ class SemanticDagExecutor:
node.pending -= 1
if node.pending == 0 and not node.overview_scheduled:
node.overview_scheduled = True
- self._stats.overview_scheduled_nodes += 1
+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
+ self._stats.in_progress_nodes += 1
asyncio.create_task(self._overview_task(parent_uri))
def _schedule_overview(self, dir_uri: str) -> None:
@@ -186,7 +204,8 @@ class SemanticDagExecutor:
if node.overview_scheduled:
return
node.overview_scheduled = True
- self._stats.overview_scheduled_nodes += 1
+ self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
+ self._stats.in_progress_nodes += 1
asyncio.create_task(self._overview_task(dir_uri))
def _finalize_file_summaries(self, node: DirNode) -> List[Dict[str, str]]:
@@ -238,19 +257,12 @@ class SemanticDagExecutor:
except Exception as e:
logger.error(f"Failed to vectorize directory {dir_uri}: {e}", exc_info=True)
- try:
- await self._processor._vectorize_files(
- dir_uri, self._context_type, node.file_paths, file_summaries
- )
- except Exception as e:
- logger.error(f"Failed to vectorize files in {dir_uri}: {e}", exc_info=True)
-
except Exception as e:
logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)
abstract = ""
finally:
self._stats.done_nodes += 1
- self._stats.pending_nodes = max(0, self._stats.pending_nodes - 1)
+ self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)
parent_uri = self._parent.get(dir_uri)
if parent_uri is None:
@@ -264,7 +276,7 @@ class SemanticDagExecutor:
return DagStats(
total_nodes=self._stats.total_nodes,
pending_nodes=self._stats.pending_nodes,
- overview_scheduled_nodes=self._stats.overview_scheduled_nodes,
+ in_progress_nodes=self._stats.in_progress_nodes,
done_nodes=self._stats.done_nodes,
)
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 9922b6f2a..2e5d1aac8 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -206,7 +206,9 @@ class SemanticProcessor(DequeueHandlerBase):
children_abstracts = await self._collect_children_abstracts(children_uris)
# 2. Concurrently generate summaries for files in directory
- file_summaries = await self._generate_file_summaries(file_paths)
+ file_summaries = await self._generate_file_summaries(
+ file_paths, context_type=context_type, parent_uri=uri, enqueue_files=True
+ )
# 3. Generate .overview.md (contains brief description)
overview = await self._generate_overview(uri, file_summaries, children_abstracts)
@@ -226,13 +228,6 @@ class SemanticProcessor(DequeueHandlerBase):
except Exception as e:
logger.error(f"Failed to vectorize directory {uri}: {e}", exc_info=True)
- # 7. Vectorize files
- # Requires retrieval refactoring before enabling
- try:
- await self._vectorize_files(uri, context_type, file_paths, file_summaries)
- except Exception as e:
- logger.error(f"Failed to vectorize files in {uri}: {e}", exc_info=True)
-
async def _collect_children_abstracts(self, children_uris: List[str]) -> List[Dict[str, str]]:
"""Collect .abstract.md from subdirectories."""
viking_fs = get_viking_fs()
@@ -244,7 +239,13 @@ class SemanticProcessor(DequeueHandlerBase):
results.append({"name": dir_name, "abstract": abstract})
return results
- async def _generate_file_summaries(self, file_paths: List[str]) -> List[Dict[str, str]]:
+ async def _generate_file_summaries(
+ self,
+ file_paths: List[str],
+ context_type: Optional[str] = None,
+ parent_uri: Optional[str] = None,
+ enqueue_files: bool = False,
+ ) -> List[Dict[str, str]]:
"""Concurrently generate file summaries."""
if not file_paths:
return []
@@ -253,7 +254,21 @@ class SemanticProcessor(DequeueHandlerBase):
async def generate_one_summary(file_path: str) -> Dict[str, str]:
async with sem:
- return await self._generate_single_file_summary(file_path)
+ summary = await self._generate_single_file_summary(file_path)
+ if enqueue_files and context_type and parent_uri:
+ try:
+ await self._vectorize_single_file(
+ parent_uri=parent_uri,
+ context_type=context_type,
+ file_path=file_path,
+ summary_dict=summary,
+ )
+ except Exception as e:
+ logger.error(
+ f"Failed to vectorize file {file_path}: {e}",
+ exc_info=True,
+ )
+ return summary
tasks = [generate_one_summary(fp) for fp in file_paths]
return await asyncio.gather(*tasks)
@@ -435,37 +450,66 @@ class SemanticProcessor(DequeueHandlerBase):
file_summaries: List[Dict[str, str]],
) -> None:
"""Vectorize files in directory."""
- from datetime import datetime
-
- from openviking.core.context import Context
from openviking.storage.queuefs import get_queue_manager
- from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
queue_manager = get_queue_manager()
embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)
for file_path, file_summary_dict in zip(file_paths, file_summaries):
- file_name = file_summary_dict["name"]
- summary = file_summary_dict["summary"]
+ await self._vectorize_single_file(
+ parent_uri=uri,
+ context_type=context_type,
+ file_path=file_path,
+ summary_dict=file_summary_dict,
+ embedding_queue=embedding_queue,
+ )
+
+ async def _vectorize_single_file(
+ self,
+ parent_uri: str,
+ context_type: str,
+ file_path: str,
+ summary_dict: Dict[str, str],
+ embedding_queue: Optional[Any] = None,
+ ) -> None:
+ """Vectorize a single file using its content or summary."""
+ from datetime import datetime
+
+ from openviking.storage.queuefs import get_queue_manager
+ from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
+
+ try:
+ file_name = summary_dict.get("name") or file_path.split("/")[-1]
+ summary = summary_dict.get("summary", "")
+
+ if embedding_queue is None:
+ queue_manager = get_queue_manager()
+ embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)
context = Context(
uri=file_path,
- parent_uri=uri,
+ parent_uri=parent_uri,
is_leaf=True,
abstract=summary,
context_type=context_type,
created_at=datetime.now(),
)
+
if self.get_resource_content_type(file_name) == ResourceContentType.TEXT:
content = await get_viking_fs().read_file(file_path)
context.set_vectorize(Vectorize(text=content))
elif summary:
context.set_vectorize(Vectorize(text=summary))
else:
- continue
+ return
+
embedding_msg = EmbeddingMsgConverter.from_context(context)
+ if not embedding_msg:
+ return
await embedding_queue.enqueue(embedding_msg) # type: ignore
logger.debug(f"Enqueued file for vectorization: {file_path}")
+ except Exception as e:
+ logger.error(f"Failed to vectorize file {file_path}: {e}", exc_info=True)
def get_resource_content_type(self, file_name: str) -> ResourceContentType:
def _is_image_file(file_name: str) -> bool: