Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Feature/add resource increment (#659)

yepper2e1619c352fd
diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs
index 81126ae96..e20dd339a 100644
--- a/crates/ov_cli/src/main.rs
+++ b/crates/ov_cli/src/main.rs
@@ -697,7 +697,17 @@ async fn handle_add_resource(
let strict = !no_strict;
let directly_upload_media = !no_directly_upload_media;
- let client = ctx.get_client();
+ let effective_timeout = if wait {
+ timeout.unwrap_or(60.0).max(ctx.config.timeout)
+ } else {
+ ctx.config.timeout
+ };
+ let client = client::HttpClient::new(
+ &ctx.config.url,
+ ctx.config.api_key.clone(),
+ ctx.config.agent_id.clone(),
+ effective_timeout,
+ );
commands::resources::add_resource(
&client,
&path,
diff --git a/openviking/core/context.py b/openviking/core/context.py
index 94d47b1fd..763085703 100644
--- a/openviking/core/context.py
+++ b/openviking/core/context.py
@@ -56,6 +56,7 @@ class Context:
self,
uri: str,
parent_uri: Optional[str] = None,
+ temp_uri: Optional[str] = None,
is_leaf: bool = False,
abstract: str = "",
context_type: Optional[str] = None,
@@ -78,6 +79,7 @@ class Context:
self.id = id or str(uuid4())
self.uri = uri
self.parent_uri = parent_uri
+ self.temp_uri = temp_uri
self.is_leaf = is_leaf
self.abstract = abstract
self.context_type = context_type or self._derive_context_type()
@@ -159,6 +161,7 @@ class Context:
"id": self.id,
"uri": self.uri,
"parent_uri": self.parent_uri,
+ "temp_uri": self.temp_uri,
"is_leaf": self.is_leaf,
"abstract": self.abstract,
"context_type": self.context_type,
@@ -194,6 +197,7 @@ class Context:
obj = cls(
uri=data["uri"],
parent_uri=data.get("parent_uri"),
+ temp_uri=data.get("temp_uri"),
is_leaf=data.get("is_leaf", False),
abstract=data.get("abstract", ""),
context_type=data.get("context_type"),
diff --git a/openviking/parse/parsers/constants.py b/openviking/parse/parsers/constants.py
index 311e545a2..174df57b9 100644
--- a/openviking/parse/parsers/constants.py
+++ b/openviking/parse/parsers/constants.py
@@ -191,6 +191,7 @@ DOCUMENTATION_EXTENSIONS = {
".texi",
".texinfo",
".wiki",
+ ".conf",
}
# File type constants for consistent return values
diff --git a/openviking/parse/parsers/upload_utils.py b/openviking/parse/parsers/upload_utils.py
index 50c80ed02..d18701735 100644
--- a/openviking/parse/parsers/upload_utils.py
+++ b/openviking/parse/parsers/upload_utils.py
@@ -40,6 +40,7 @@ _EXTENSIONLESS_TEXT_NAMES: Set[str] = {
"NEWS",
"NOTICE",
"TODO",
+ "BUILD",
}
diff --git a/openviking/parse/tree_builder.py b/openviking/parse/tree_builder.py
index 9a2543ce4..4260065e4 100644
--- a/openviking/parse/tree_builder.py
+++ b/openviking/parse/tree_builder.py
@@ -20,7 +20,6 @@ IMPORTANT (v5.0 Architecture):
- Content splitting is handled by Parser, not TreeBuilder
"""
-import asyncio
import logging
from typing import TYPE_CHECKING, Optional
@@ -28,9 +27,7 @@ from openviking.core.building_tree import BuildingTree
from openviking.core.context import Context
from openviking.parse.parsers.media.utils import get_media_base_uri, get_media_type
from openviking.server.identity import RequestContext
-from openviking.storage.queuefs import SemanticMsg, get_queue_manager
from openviking.storage.viking_fs import get_viking_fs
-from openviking.telemetry import get_current_telemetry
from openviking.utils import parse_code_hosting_url
from openviking_cli.utils.uri import VikingURI
@@ -94,7 +91,6 @@ class TreeBuilder:
parent_uri: Optional[str] = None,
source_path: Optional[str] = None,
source_format: Optional[str] = None,
- trigger_semantic: bool = False,
) -> "BuildingTree":
"""
Finalize processing by moving from temp to AGFS.
@@ -102,8 +98,6 @@ class TreeBuilder:
Args:
to_uri: Exact target URI (must not exist)
parent_uri: Target parent URI (must exist)
- trigger_semantic: Whether to automatically trigger semantic generation.
- Default is False (handled by ResourceProcessor/Summarizer).
"""
viking_fs = get_viking_fs()
@@ -139,16 +133,6 @@ class TreeBuilder:
base_uri = parent_uri or auto_base_uri
# 3. Determine candidate_uri
if to_uri:
- # Exact target URI: must not exist yet
- try:
- await viking_fs.stat(to_uri, ctx=ctx)
- # If we get here, it already exists
- raise FileExistsError(f"Target URI already exists: {to_uri}")
- except FileExistsError:
- raise
- except Exception:
- # It doesn't exist, good to use
- pass
candidate_uri = to_uri
else:
if parent_uri:
@@ -161,35 +145,8 @@ class TreeBuilder:
raise ValueError(f"Parent URI is not a directory: {parent_uri}")
candidate_uri = VikingURI(base_uri).join(final_doc_name).uri
- final_uri = await self._resolve_unique_uri(candidate_uri, ctx=ctx)
+ final_uri = candidate_uri
- if final_uri != candidate_uri:
- logger.info(f"[TreeBuilder] Resolved name conflict: {candidate_uri} -> {final_uri}")
- else:
- logger.info(f"[TreeBuilder] Finalizing from temp: {final_uri}")
-
- # 4. Move directory tree from temp to final location in AGFS
- await self._move_temp_to_dest(viking_fs, temp_doc_uri, final_uri, ctx=ctx)
- logger.info(f"[TreeBuilder] Moved temp tree: {temp_doc_uri} -> {final_uri}")
-
- # 5. Cleanup temporary root directory
- try:
- await viking_fs.delete_temp(temp_uri, ctx=ctx)
- logger.info(f"[TreeBuilder] Cleaned up temp root: {temp_uri}")
- except Exception as e:
- logger.warning(f"[TreeBuilder] Failed to cleanup temp root: {e}")
-
- # 6. Enqueue to SemanticQueue for async semantic generation
- if trigger_semantic:
- try:
- await self._enqueue_semantic_generation(final_uri, "resource", ctx=ctx)
- logger.info(f"[TreeBuilder] Enqueued semantic generation for: {final_uri}")
- except Exception as e:
- logger.error(
- f"[TreeBuilder] Failed to enqueue semantic generation: {e}", exc_info=True
- )
-
- # 7. Return simple BuildingTree (no scanning needed)
tree = BuildingTree(
source_path=source_path,
source_format=source_format,
@@ -197,121 +154,7 @@ class TreeBuilder:
tree._root_uri = final_uri
# Create a minimal Context object for the root so that tree.root is not None
- root_context = Context(uri=final_uri)
+ root_context = Context(uri=final_uri, temp_uri=temp_doc_uri)
tree.add_context(root_context)
return tree
-
- async def _resolve_unique_uri(
- self, uri: str, max_attempts: int = 100, ctx: Optional[RequestContext] = None
- ) -> str:
- """Return a URI that does not collide with an existing resource.
-
- If *uri* is free, return it unchanged. Otherwise append ``_1``,
- ``_2``, … until a free name is found (like macOS Finder / Windows
- Explorer).
- """
- viking_fs = get_viking_fs()
-
- async def _exists(u: str) -> bool:
- try:
- await viking_fs.stat(u, ctx=ctx)
- return True
- except Exception:
- return False
-
- if not await _exists(uri):
- return uri
-
- for i in range(1, max_attempts + 1):
- candidate = f"{uri}_{i}"
- if not await _exists(candidate):
- return candidate
-
- raise FileExistsError(f"Cannot resolve unique name for {uri} after {max_attempts} attempts")
-
- async def _move_temp_to_dest(
- self, viking_fs, src_uri: str, dst_uri: str, ctx: RequestContext
- ) -> None:
- """Move temp directory to final destination using a single native AGFS mv call.
-
- Temp files have no vector records yet, so no vector index update is needed.
- """
- src_path = viking_fs._uri_to_path(src_uri, ctx=ctx)
- dst_path = viking_fs._uri_to_path(dst_uri, ctx=ctx)
- await self._ensure_parent_dirs(dst_uri, ctx=ctx)
- await asyncio.to_thread(viking_fs.agfs.mv, src_path, dst_path)
-
- async def _ensure_parent_dirs(self, uri: str, ctx: RequestContext) -> None:
- """Recursively create parent directories."""
- viking_fs = get_viking_fs()
- parent = VikingURI(uri).parent
- if not parent:
- return
- parent_uri = parent.uri
- # Recursively ensure parent's parent exists
- await self._ensure_parent_dirs(parent_uri, ctx=ctx)
-
- # Create parent directory (ignore if already exists)
- try:
- await viking_fs.mkdir(parent_uri, exist_ok=True, ctx=ctx)
- logger.debug(f"Created parent directory: {parent_uri}")
- except Exception as e:
- # Directory may already exist, ignore error
- if "exist" not in str(e).lower():
- logger.debug(f"Parent dir {parent_uri} may already exist: {e}")
-
- async def _enqueue_semantic_generation(
- self, uri: str, context_type: str, ctx: RequestContext
- ) -> None:
- """
- Enqueue a directory for semantic generation.
-
- Args:
- uri: Directory URI to enqueue
- context_type: resource/memory/skill
- """
-
- queue_manager = get_queue_manager()
-
- # Get semantic queue
- semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
-
- # Sort by depth (descending) for bottom-up processing
- msg = SemanticMsg(
- uri=uri,
- context_type=context_type,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- agent_id=ctx.user.agent_id,
- role=ctx.role.value,
- telemetry_id=get_current_telemetry().telemetry_id,
- )
- await semantic_queue.enqueue(msg)
-
- async def _load_content(self, uri: str, content_type: str) -> str:
- """Helper to load content with proper type handling"""
- import json
-
- if content_type == "abstract":
- result = await get_viking_fs().abstract(uri)
- elif content_type == "overview":
- result = await get_viking_fs().overview(uri)
- elif content_type == "detail":
- result = await get_viking_fs().read_file(uri)
- else:
- return ""
-
- # Handle different return types
- if isinstance(result, str):
- return result
- elif isinstance(result, bytes):
- return result.decode("utf-8")
- elif hasattr(result, "to_dict") and not isinstance(result, list):
- # Handle FindResult by converting to dict (skip lists)
- return str(result.to_dict())
- elif isinstance(result, list):
- # Handle list results
- return json.dumps(result)
- else:
- return str(result)
diff --git a/openviking/server/models.py b/openviking/server/models.py
index a97b086bb..445da590b 100644
--- a/openviking/server/models.py
+++ b/openviking/server/models.py
@@ -31,6 +31,7 @@ ERROR_CODE_TO_HTTP_STATUS = {
"INVALID_URI": 400,
"NOT_FOUND": 404,
"ALREADY_EXISTS": 409,
+ "CONFLICT": 409,
"PERMISSION_DENIED": 403,
"UNAUTHENTICATED": 401,
"RESOURCE_EXHAUSTED": 429,
diff --git a/openviking/session/compressor.py b/openviking/session/compressor.py
index 692612cdb..243eba8b0 100644
--- a/openviking/session/compressor.py
+++ b/openviking/session/compressor.py
@@ -66,16 +66,100 @@ class SessionCompressor:
self.vikingdb = vikingdb
self.extractor = MemoryExtractor()
self.deduplicator = MemoryDeduplicator(vikingdb=vikingdb)
+ self._pending_semantic_changes: Dict[str, Dict[str, set]] = {}
- async def _index_memory(self, memory: Context, ctx: RequestContext) -> bool:
- """Add memory to vectorization queue and trigger parent directory semantic generation."""
+ def _record_semantic_change(
+ self, file_uri: str, change_type: str, parent_uri: Optional[str] = None
+ ) -> None:
+ """Record a file change for batch semantic processing.
+
+ Args:
+ file_uri: The URI of the file that changed
+ change_type: One of "added", "modified", "deleted"
+ parent_uri: Optional parent directory URI. If not provided, will be derived from file_uri
+ """
+ if change_type not in ("added", "modified", "deleted"):
+ logger.warning(f"Invalid change_type: {change_type}, skipping")
+ return
+
+ if not parent_uri:
+ parent_uri = "/".join(file_uri.rsplit("/", 1)[:-1])
+
+ if not parent_uri:
+ logger.warning(f"Could not determine parent URI for {file_uri}, skipping")
+ return
+
+ if parent_uri not in self._pending_semantic_changes:
+ self._pending_semantic_changes[parent_uri] = {
+ "added": set(),
+ "modified": set(),
+ "deleted": set(),
+ }
+
+ self._pending_semantic_changes[parent_uri][change_type].add(file_uri)
+ logger.debug(f"Recorded semantic change: {change_type} {file_uri} in {parent_uri}")
+
+ async def _flush_semantic_operations(self, ctx: RequestContext) -> None:
+ """Flush all pending semantic operations.
+
+ This method should be called after all memory changes are complete.
+ It will deduplicate parent URIs and enqueue semantic operations with change info.
+ """
+ if not self._pending_semantic_changes:
+ return
+
+ try:
+ from openviking.storage.queuefs import get_queue_manager
+ from openviking.storage.queuefs.semantic_msg import SemanticMsg
+
+ queue_manager = get_queue_manager()
+ semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
+
+ for parent_uri, changes in self._pending_semantic_changes.items():
+ changes_dict = {
+ "added": list(changes["added"]),
+ "modified": list(changes["modified"]),
+ "deleted": list(changes["deleted"]),
+ }
+
+ msg = SemanticMsg(
+ uri=parent_uri,
+ context_type="memory",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ agent_id=ctx.user.agent_id,
+ role=ctx.role.value,
+ changes=changes_dict,
+ )
+ await semantic_queue.enqueue(msg)
+ logger.info(
+ f"Enqueued semantic generation for {parent_uri} with changes: "
+ f"added={len(changes['added'])}, modified={len(changes['modified'])}, "
+ f"deleted={len(changes['deleted'])}"
+ )
+
+ except Exception as e:
+ logger.error(f"Failed to flush semantic operations: {e}", exc_info=True)
+ finally:
+ self._pending_semantic_changes.clear()
+
+ async def _index_memory(
+ self, memory: Context, ctx: RequestContext, change_type: str = "added"
+ ) -> bool:
+ """Add memory to vectorization queue and record semantic change.
+
+ Args:
+ memory: The memory context to index
+ ctx: Request context
+ change_type: One of "added" or "modified"
+ """
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
embedding_msg = EmbeddingMsgConverter.from_context(memory)
await self.vikingdb.enqueue_embedding_msg(embedding_msg)
logger.info(f"Enqueued memory for vectorization: {memory.uri}")
- await self.extractor._enqueue_semantic_for_parent(memory.uri, ctx)
+ self._record_semantic_change(memory.uri, change_type, parent_uri=memory.parent_uri)
return True
async def _merge_into_existing(
@@ -108,7 +192,7 @@ class SessionCompressor:
"Merged memory %s with abstract %s", target_memory.uri, target_memory.abstract
)
target_memory.set_vectorize(Vectorize(text=payload.content))
- await self._index_memory(target_memory, ctx)
+ await self._index_memory(target_memory, ctx, change_type="modified")
return True
except Exception as e:
logger.error(f"Failed to merge memory {target_memory.uri}: {e}")
@@ -129,6 +213,8 @@ class SessionCompressor:
await self.vikingdb.delete_uris(ctx, [memory.uri])
except Exception as e:
logger.warning(f"Failed to remove vector record for {memory.uri}: {e}")
+
+ self._record_semantic_change(memory.uri, "deleted", parent_uri=memory.parent_uri)
return True
async def extract_long_term_memories(
@@ -147,162 +233,177 @@ class SessionCompressor:
if not ctx:
return []
- if strict_extract_errors:
- # Intentionally let extraction errors bubble up so caller (task tracker)
- # can mark background commit tasks as failed with an explicit error.
- candidates = await self.extractor.extract_strict(context, user, session_id)
- else:
- candidates = await self.extractor.extract(context, user, session_id)
-
- if not candidates:
- return []
-
- memories: List[Context] = []
- stats = ExtractionStats()
- viking_fs = get_viking_fs()
-
- tool_parts = self._extract_tool_parts(messages)
- from .tool_skill_utils import collect_skill_stats, collect_tool_stats
-
- tool_stats_map = collect_tool_stats(tool_parts)
- skill_stats_map = collect_skill_stats(tool_parts)
-
- for candidate in candidates:
- # Profile: skip dedup, always merge
- if candidate.category in ALWAYS_MERGE_CATEGORIES:
- memory = await self.extractor.create_memory(candidate, user, session_id, ctx=ctx)
- if memory:
- memories.append(memory)
- stats.created += 1
- await self._index_memory(memory, ctx)
- else:
- stats.skipped += 1
- continue
+ self._pending_semantic_changes.clear()
- # Tool/Skill Memory: 特殊合并逻辑
- if candidate.category in TOOL_SKILL_CATEGORIES:
- if isinstance(candidate, ToolSkillCandidateMemory):
- tool_name, skill_name, tool_status = self._get_tool_skill_info(
- candidate, tool_parts
+ try:
+ if strict_extract_errors:
+ # Intentionally let extraction errors bubble up so caller (task tracker)
+ # can mark background commit tasks as failed with an explicit error.
+ candidates = await self.extractor.extract_strict(context, user, session_id)
+ else:
+ candidates = await self.extractor.extract(context, user, session_id)
+
+ if not candidates:
+ return []
+
+ memories: List[Context] = []
+ stats = ExtractionStats()
+ viking_fs = get_viking_fs()
+
+ tool_parts = self._extract_tool_parts(messages)
+ from .tool_skill_utils import collect_skill_stats, collect_tool_stats
+
+ tool_stats_map = collect_tool_stats(tool_parts)
+ skill_stats_map = collect_skill_stats(tool_parts)
+
+ for candidate in candidates:
+ # Profile: skip dedup, always merge
+ if candidate.category in ALWAYS_MERGE_CATEGORIES:
+ memory = await self.extractor.create_memory(
+ candidate, user, session_id, ctx=ctx
)
- candidate.tool_status = tool_status
- if tool_name:
- candidate.tool_name = tool_name
- if skill_name:
- candidate.skill_name = skill_name
-
- if tool_name and candidate.call_time == 0:
- tool_stats = tool_stats_map.get(tool_name, {})
- candidate.call_time = tool_stats.get("call_count", candidate.call_time)
- candidate.success_time = tool_stats.get(
- "success_time", candidate.success_time
- )
- candidate.duration_ms = tool_stats.get("duration_ms", candidate.duration_ms)
- candidate.prompt_tokens = tool_stats.get(
- "prompt_tokens", candidate.prompt_tokens
- )
- candidate.completion_tokens = tool_stats.get(
- "completion_tokens", candidate.completion_tokens
- )
-
- if skill_name and candidate.call_time == 0:
- skill_stats = skill_stats_map.get(skill_name, {})
- candidate.call_time = skill_stats.get("call_count", candidate.call_time)
- candidate.success_time = skill_stats.get(
- "success_time", candidate.success_time
- )
- if skill_name:
- memory = await self.extractor._merge_skill_memory(
- skill_name, candidate, ctx=ctx
- )
- elif tool_name:
- memory = await self.extractor._merge_tool_memory(
- tool_name, candidate, ctx=ctx
- )
- else:
- logger.warning("No tool_name or skill_name found, skipping")
- stats.skipped += 1
- continue
if memory:
memories.append(memory)
- stats.merged += 1
+ stats.created += 1
await self._index_memory(memory, ctx)
- continue
-
- # Dedup check for other categories
- result = await self.deduplicator.deduplicate(candidate, ctx)
- actions = result.actions or []
- decision = result.decision
-
- # Safety net: create+merge should be treated as none.
- if decision == DedupDecision.CREATE and any(
- a.decision == MemoryActionDecision.MERGE for a in actions
- ):
- logger.warning(
- f"Dedup returned create with merge action, normalizing to none: "
- f"{candidate.abstract}"
- )
- decision = DedupDecision.NONE
+ else:
+ stats.skipped += 1
+ continue
+
+ # Tool/Skill Memory: 特殊合并逻辑
+ if candidate.category in TOOL_SKILL_CATEGORIES:
+ if isinstance(candidate, ToolSkillCandidateMemory):
+ tool_name, skill_name, tool_status = self._get_tool_skill_info(
+ candidate, tool_parts
+ )
+ candidate.tool_status = tool_status
+ if tool_name:
+ candidate.tool_name = tool_name
+ if skill_name:
+ candidate.skill_name = skill_name
+
+ if tool_name and candidate.call_time == 0:
+ tool_stats = tool_stats_map.get(tool_name, {})
+ candidate.call_time = tool_stats.get("call_count", candidate.call_time)
+ candidate.success_time = tool_stats.get(
+ "success_time", candidate.success_time
+ )
+ candidate.duration_ms = tool_stats.get(
+ "duration_ms", candidate.duration_ms
+ )
+ candidate.prompt_tokens = tool_stats.get(
+ "prompt_tokens", candidate.prompt_tokens
+ )
+ candidate.completion_tokens = tool_stats.get(
+ "completion_tokens", candidate.completion_tokens
+ )
+
+ if skill_name and candidate.call_time == 0:
+ skill_stats = skill_stats_map.get(skill_name, {})
+ candidate.call_time = skill_stats.get("call_count", candidate.call_time)
+ candidate.success_time = skill_stats.get(
+ "success_time", candidate.success_time
+ )
+ if skill_name:
+ memory = await self.extractor._merge_skill_memory(
+ skill_name, candidate, ctx=ctx
+ )
+ elif tool_name:
+ memory = await self.extractor._merge_tool_memory(
+ tool_name, candidate, ctx=ctx
+ )
+ else:
+ logger.warning("No tool_name or skill_name found, skipping")
+ stats.skipped += 1
+ continue
+ if memory:
+ memories.append(memory)
+ stats.merged += 1
+ await self._index_memory(memory, ctx, change_type="modified")
+ continue
- if decision == DedupDecision.SKIP:
- stats.skipped += 1
- continue
+ # Dedup check for other categories
+ result = await self.deduplicator.deduplicate(candidate, ctx)
+ actions = result.actions or []
+ decision = result.decision
+
+ # Safety net: create+merge should be treated as none.
+ if decision == DedupDecision.CREATE and any(
+ a.decision == MemoryActionDecision.MERGE for a in actions
+ ):
+ logger.warning(
+ f"Dedup returned create with merge action, normalizing to none: "
+ f"{candidate.abstract}"
+ )
+ decision = DedupDecision.NONE
- if decision == DedupDecision.NONE:
- if not actions:
+ if decision == DedupDecision.SKIP:
stats.skipped += 1
continue
- for action in actions:
- if action.decision == MemoryActionDecision.DELETE:
- if viking_fs and await self._delete_existing_memory(
- action.memory, viking_fs, ctx=ctx
- ):
- stats.deleted += 1
- else:
- stats.skipped += 1
- elif action.decision == MemoryActionDecision.MERGE:
- if candidate.category in MERGE_SUPPORTED_CATEGORIES and viking_fs:
- if await self._merge_into_existing(
- candidate, action.memory, viking_fs, ctx=ctx
+ if decision == DedupDecision.NONE:
+ if not actions:
+ stats.skipped += 1
+ continue
+
+ for action in actions:
+ if action.decision == MemoryActionDecision.DELETE:
+ if viking_fs and await self._delete_existing_memory(
+ action.memory, viking_fs, ctx=ctx
):
- stats.merged += 1
+ stats.deleted += 1
else:
stats.skipped += 1
- else:
- # events/cases don't support MERGE, treat as SKIP
- stats.skipped += 1
- continue
-
- if decision == DedupDecision.CREATE:
- # create can optionally include delete actions (delete first, then create)
- for action in actions:
- if action.decision == MemoryActionDecision.DELETE:
- if viking_fs and await self._delete_existing_memory(
- action.memory, viking_fs, ctx=ctx
- ):
- stats.deleted += 1
- else:
- stats.skipped += 1
+ elif action.decision == MemoryActionDecision.MERGE:
+ if candidate.category in MERGE_SUPPORTED_CATEGORIES and viking_fs:
+ if await self._merge_into_existing(
+ candidate, action.memory, viking_fs, ctx=ctx
+ ):
+ stats.merged += 1
+ else:
+ stats.skipped += 1
+ else:
+ # events/cases don't support MERGE, treat as SKIP
+ stats.skipped += 1
+ continue
- memory = await self.extractor.create_memory(candidate, user, session_id, ctx=ctx)
- if memory:
- memories.append(memory)
- stats.created += 1
- await self._index_memory(memory, ctx)
- else:
- stats.skipped += 1
+ if decision == DedupDecision.CREATE:
+ # create can optionally include delete actions (delete first, then create)
+ for action in actions:
+ if action.decision == MemoryActionDecision.DELETE:
+ if viking_fs and await self._delete_existing_memory(
+ action.memory, viking_fs, ctx=ctx
+ ):
+ stats.deleted += 1
+ else:
+ stats.skipped += 1
- # Extract URIs used in messages, create relations
- used_uris = self._extract_used_uris(messages)
- if used_uris and memories:
- await self._create_relations(memories, used_uris, ctx=ctx)
+ memory = await self.extractor.create_memory(
+ candidate, user, session_id, ctx=ctx
+ )
+ if memory:
+ memories.append(memory)
+ stats.created += 1
+ await self._index_memory(memory, ctx)
+ else:
+ stats.skipped += 1
+
+ # Extract URIs used in messages, create relations
+ used_uris = self._extract_used_uris(messages)
+ if used_uris and memories:
+ await self._create_relations(memories, used_uris, ctx=ctx)
+
+ await self._flush_semantic_operations(ctx)
- logger.info(
- f"Memory extraction: created={stats.created}, "
- f"merged={stats.merged}, deleted={stats.deleted}, skipped={stats.skipped}"
- )
- return memories
+ logger.info(
+ f"Memory extraction: created={stats.created}, "
+ f"merged={stats.merged}, deleted={stats.deleted}, skipped={stats.skipped}"
+ )
+ return memories
+
+ except Exception:
+ self._pending_semantic_changes.clear()
+ raise
def _extract_tool_parts(self, messages: List[Message]) -> List:
"""Extract all ToolPart from messages."""
diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.py
index dda6196bb..99d4a1215 100644
--- a/openviking/session/memory_extractor.py
+++ b/openviking/session/memory_extractor.py
@@ -727,28 +727,6 @@ class MemoryExtractor:
await viking_fs.write_file(uri=uri, content=merged_content, ctx=ctx)
return self._create_tool_context(uri, candidate, ctx, abstract_override=abstract_override)
- async def _enqueue_semantic_for_parent(self, file_uri: str, ctx: "RequestContext") -> None:
- """Enqueue semantic generation for parent directory."""
- try:
- from openviking.storage.queuefs import get_queue_manager
- from openviking.storage.queuefs.semantic_msg import SemanticMsg
-
- parent_uri = "/".join(file_uri.rsplit("/", 1)[:-1])
- queue_manager = get_queue_manager()
- semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
- msg = SemanticMsg(
- uri=parent_uri,
- context_type="memory",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- agent_id=ctx.user.agent_id,
- role=ctx.role.value,
- )
- await semantic_queue.enqueue(msg)
- logger.debug(f"Enqueued semantic generation for: {parent_uri}")
- except Exception as e:
- logger.warning(f"Failed to enqueue semantic generation for {file_uri}: {e}")
-
def _compute_statistics_derived(self, stats: dict) -> dict:
"""计算派生统计数据(平均值、成功率)"""
if stats["total_calls"] > 0:
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index ea33b5d9b..c46f2d769 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -353,3 +353,12 @@ class TextEmbeddingHandler(DequeueHandlerBase):
self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
self.report_error(str(e), data)
return None
+ finally:
+ if embedding_msg and embedding_msg.semantic_msg_id:
+ from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker
+
+ tracker = EmbeddingTaskTracker.get_instance()
+ try:
+ await tracker.decrement(embedding_msg.semantic_msg_id)
+ except Exception as tracker_err:
+ logger.warning(f"Failed to decrement embedding tracker: {tracker_err}")
diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.py
index b73a01d7f..87514a830 100644
--- a/openviking/storage/queuefs/__init__.py
+++ b/openviking/storage/queuefs/__init__.py
@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
from .embedding_msg import EmbeddingMsg
from .embedding_queue import EmbeddingQueue
+from .embedding_tracker import EmbeddingTaskTracker
from .named_queue import NamedQueue, QueueError, QueueStatus
from .queue_manager import QueueManager, get_queue_manager, init_queue_manager
from .semantic_dag import SemanticDagExecutor
@@ -18,6 +19,7 @@ __all__ = [
"QueueError",
"EmbeddingQueue",
"EmbeddingMsg",
+ "EmbeddingTaskTracker",
"SemanticQueue",
"SemanticDagExecutor",
"SemanticMsg",
diff --git a/openviking/storage/queuefs/embedding_msg.py b/openviking/storage/queuefs/embedding_msg.py
index c100d569f..af0e95eea 100644
--- a/openviking/storage/queuefs/embedding_msg.py
+++ b/openviking/storage/queuefs/embedding_msg.py
@@ -2,7 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
import json
from dataclasses import asdict, dataclass
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, Optional, Union
from uuid import uuid4
@@ -11,17 +11,20 @@ class EmbeddingMsg:
message: Union[str, List[Dict[str, Any]]]
context_data: Dict[str, Any]
telemetry_id: str = ""
+ semantic_msg_id: Optional[str] = None
def __init__(
self,
message: Union[str, List[Dict[str, Any]]],
context_data: Dict[str, Any],
telemetry_id: str = "",
+ semantic_msg_id: Optional[str] = None,
):
self.id = str(uuid4())
self.message = message
self.context_data = context_data
self.telemetry_id = telemetry_id
+ self.semantic_msg_id = semantic_msg_id
def to_dict(self) -> Dict[str, Any]:
"""Convert embedding message to dictionary format."""
@@ -38,6 +41,7 @@ class EmbeddingMsg:
message=data["message"],
context_data=data["context_data"],
telemetry_id=data.get("telemetry_id", ""),
+ semantic_msg_id=data.get("semantic_msg_id"),
)
obj.id = data.get("id", obj.id)
return obj
diff --git a/openviking/storage/queuefs/embedding_queue.py b/openviking/storage/queuefs/embedding_queue.py
index d44595e8f..5fb1f4af4 100644
--- a/openviking/storage/queuefs/embedding_queue.py
+++ b/openviking/storage/queuefs/embedding_queue.py
@@ -21,6 +21,7 @@ class EmbeddingQueue(NamedQueue):
if msg is None:
logger.warning("Embedding message is None, skipping enqueuing")
return ""
+ logger.debug(f"Enqueued embedding message: {msg}")
return await super().enqueue(msg.to_dict())
async def dequeue(self) -> Optional[EmbeddingMsg]:
diff --git a/openviking/storage/queuefs/embedding_tracker.py b/openviking/storage/queuefs/embedding_tracker.py
new file mode 100644
index 000000000..bfee27b79
--- /dev/null
+++ b/openviking/storage/queuefs/embedding_tracker.py
@@ -0,0 +1,179 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Embedding Task Tracker for tracking embedding task completion status."""
+
+import asyncio
+from typing import Any, Callable, Dict, Optional
+
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class EmbeddingTaskTracker:
+ """Track embedding task completion status for each SemanticMsg.
+
+ This tracker maintains a global registry of embedding tasks associated
+ with each SemanticMsg. When all embedding tasks for a SemanticMsg are
+ completed, it triggers the registered callback and removes the entry.
+ """
+
+ _instance: Optional["EmbeddingTaskTracker"] = None
+ _initialized: bool = False
+
+ def __new__(cls) -> "EmbeddingTaskTracker":
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+
+ def __init__(self):
+ if self._initialized:
+ return
+ self._lock: asyncio.Lock = asyncio.Lock()
+ self._tasks: Dict[str, Dict[str, Any]] = {}
+ self._initialized = True
+
+ @classmethod
+ def get_instance(cls) -> "EmbeddingTaskTracker":
+ """Get the singleton instance of EmbeddingTaskTracker."""
+ if cls._instance is None:
+ cls._instance = cls()
+ return cls._instance
+
+ async def register(
+ self,
+ semantic_msg_id: str,
+ total_count: int,
+ on_complete: Optional[Callable[[], Any]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ ) -> None:
+ """Register a SemanticMsg with its total embedding task count.
+
+ Args:
+ semantic_msg_id: The ID of the SemanticMsg
+ total_count: Total number of embedding tasks for this SemanticMsg
+ on_complete: Optional callback when all tasks complete
+ metadata: Optional metadata to store with the task
+ """
+ async with self._lock:
+ self._tasks[semantic_msg_id] = {
+ "remaining": total_count,
+ "total": total_count,
+ "on_complete": on_complete,
+ "metadata": metadata or {},
+ }
+ logger.info(
+ f"Registered embedding tracker for SemanticMsg {semantic_msg_id}: "
+ f"{total_count} tasks"
+ )
+
+ if total_count <= 0 and on_complete:
+ del self._tasks[semantic_msg_id]
+ logger.info(
+ f"No embedding tasks for SemanticMsg {semantic_msg_id}, "
+ f"triggering on_complete immediately"
+ )
+
+ if total_count <= 0 and on_complete:
+ try:
+ result = on_complete()
+ if asyncio.iscoroutine(result):
+ await result
+ except Exception as e:
+ logger.error(
+ f"Error in completion callback for {semantic_msg_id}: {e}",
+ exc_info=True,
+ )
+
+ async def decrement(self, semantic_msg_id: str) -> Optional[int]:
+ """Decrement the remaining task count for a SemanticMsg.
+
+ This method should be called when an embedding task is completed.
+ When the count reaches zero, the registered callback is executed
+ and the entry is removed from the tracker.
+
+ Args:
+ semantic_msg_id: The ID of the SemanticMsg
+
+ Returns:
+ The remaining count after decrement, or None if not found
+ """
+ on_complete = None
+
+ async with self._lock:
+ if semantic_msg_id not in self._tasks:
+ return None
+
+ task_info = self._tasks[semantic_msg_id]
+ task_info["remaining"] -= 1
+ remaining = task_info["remaining"]
+
+ if remaining <= 0:
+ on_complete = task_info.get("on_complete")
+
+ del self._tasks[semantic_msg_id]
+ logger.info(
+ f"All embedding tasks({task_info['total']}) completed for SemanticMsg {semantic_msg_id}"
+ )
+
+ if on_complete:
+ try:
+ result = on_complete()
+ if asyncio.iscoroutine(result):
+ await result
+ except Exception as e:
+ logger.error(
+ f"Error in completion callback for {semantic_msg_id}: {e}",
+ exc_info=True,
+ )
+ return remaining
+
+ async def get_status(self, semantic_msg_id: str) -> Optional[Dict[str, Any]]:
+ """Get the current status of a SemanticMsg's embedding tasks.
+
+ Args:
+ semantic_msg_id: The ID of the SemanticMsg
+
+ Returns:
+ Dict with 'remaining', 'total', 'metadata' or None if not found
+ """
+ async with self._lock:
+ if semantic_msg_id not in self._tasks:
+ return None
+ task_info = self._tasks[semantic_msg_id]
+ return {
+ "remaining": task_info["remaining"],
+ "total": task_info["total"],
+ "metadata": task_info.get("metadata", {}),
+ }
+
+ async def remove(self, semantic_msg_id: str) -> bool:
+ """Remove a SemanticMsg from the tracker.
+
+ Args:
+ semantic_msg_id: The ID of the SemanticMsg
+
+ Returns:
+ True if removed, False if not found
+ """
+ async with self._lock:
+ if semantic_msg_id in self._tasks:
+ del self._tasks[semantic_msg_id]
+ return True
+ return False
+
+ async def get_all_tracked(self) -> Dict[str, Dict[str, Any]]:
+ """Get all currently tracked SemanticMsgs.
+
+ Returns:
+ Dict of semantic_msg_id -> task info
+ """
+ async with self._lock:
+ return {
+ msg_id: {
+ "remaining": info["remaining"],
+ "total": info["total"],
+ "metadata": info.get("metadata", {}),
+ }
+ for msg_id, info in self._tasks.items()
+ }
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
index 212cffd99..9ed037b6d 100644
--- a/openviking/storage/queuefs/semantic_dag.py
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -4,7 +4,7 @@
import asyncio
from dataclasses import dataclass, field
-from typing import Dict, List, Optional
+from typing import Awaitable, Callable, Dict, List, Optional
from openviking.server.identity import RequestContext
from openviking.storage.viking_fs import get_viking_fs
@@ -44,6 +44,24 @@ class DagStats:
done_nodes: int = 0
+@dataclass
+class VectorizeTask:
+ """Vectorize task information."""
+
+ task_type: str # "file" or "directory"
+ uri: str
+ context_type: str
+ ctx: "RequestContext"
+ semantic_msg_id: Optional[str] = None
+ # For file tasks
+ file_path: Optional[str] = None
+ summary_dict: Optional[Dict[str, str]] = None
+ parent_uri: Optional[str] = None
+ # For directory tasks
+ abstract: Optional[str] = None
+ overview: Optional[str] = None
+
+
class SemanticDagExecutor:
"""Execute semantic generation with DAG-style, event-driven lazy dispatch."""
@@ -53,11 +71,19 @@ class SemanticDagExecutor:
context_type: str,
max_concurrent_llm: int,
ctx: RequestContext,
+ incremental_update: bool = False,
+ target_uri: Optional[str] = None,
+ semantic_msg_id: Optional[str] = None,
+ recursive: bool = True,
):
self._processor = processor
self._context_type = context_type
self._max_concurrent_llm = max_concurrent_llm
self._ctx = ctx
+ self._incremental_update = incremental_update
+ self._target_uri = target_uri
+ self._semantic_msg_id = semantic_msg_id
+ self._recursive = recursive
self._llm_sem = asyncio.Semaphore(max_concurrent_llm)
self._viking_fs = get_viking_fs()
self._nodes: Dict[str, DirNode] = {}
@@ -65,6 +91,70 @@ class SemanticDagExecutor:
self._root_uri: Optional[str] = None
self._root_done: Optional[asyncio.Event] = None
self._stats = DagStats()
+ self._vectorize_task_count: int = 0
+ self._pending_vectorize_tasks: List[VectorizeTask] = []
+ self._vectorize_lock = asyncio.Lock()
+ self._file_change_status: Dict[str, bool] = {}
+ self._dir_change_status: Dict[str, bool] = {}
+ self._overview_cache: Dict[str, Dict[str, str]] = {}
+ self._overview_cache_lock = asyncio.Lock()
+
+ def _create_on_complete_callback(self) -> Optional[Callable[[], Awaitable[None]]]:
+ """Create on_complete callback for incremental update or full update."""
+ if not self._target_uri or not self._root_uri:
+ return None
+
+ if self._incremental_update:
+
+ async def sync_diff_callback() -> None:
+ try:
+ diff = await self._processor._sync_topdown_recursive(
+ self._root_uri,
+ self._target_uri,
+ ctx=self._ctx,
+ file_change_status=self._file_change_status,
+ )
+ logger.info(
+ f"[SyncDiff] Diff computed: "
+ f"added_files={len(diff.added_files)}, "
+ f"deleted_files={len(diff.deleted_files)}, "
+ f"updated_files={len(diff.updated_files)}, "
+ f"added_dirs={len(diff.added_dirs)}, "
+ f"deleted_dirs={len(diff.deleted_dirs)}"
+ )
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Error in sync_diff_callback: "
+ f"root_uri={self._root_uri}, target_uri={self._target_uri} "
+ f"error={e}",
+ exc_info=True,
+ )
+
+ return sync_diff_callback
+ else:
+
+ async def move_temp_to_target_callback() -> None:
+ try:
+ target_exists = await self._viking_fs.exists(self._target_uri, ctx=self._ctx)
+ if target_exists:
+ await self._viking_fs.rm(self._target_uri, recursive=True, ctx=self._ctx)
+ parent_uri = "/".join(self._target_uri.rsplit("/", 1)[:-1])
+ if parent_uri:
+ await self._viking_fs.mkdir(parent_uri, exist_ok=True, ctx=self._ctx)
+ await self._viking_fs.mv(self._root_uri, self._target_uri, ctx=self._ctx)
+ logger.info(
+ f"[MoveTemp] Moved temp uri to target uri: "
+ f"root_uri={self._root_uri}, target_uri={self._target_uri}"
+ )
+ except Exception as e:
+ logger.error(
+ f"[MoveTemp] Error moving temp uri to target uri: "
+ f"root_uri={self._root_uri}, target_uri={self._target_uri} "
+ f"error={e}",
+ exc_info=True,
+ )
+
+ return move_temp_to_target_callback
async def run(self, root_uri: str) -> None:
"""Run DAG execution starting from root_uri."""
@@ -73,6 +163,52 @@ class SemanticDagExecutor:
await self._dispatch_dir(root_uri, parent_uri=None)
await self._root_done.wait()
+ on_complete = self._create_on_complete_callback()
+
+ async with self._vectorize_lock:
+ task_count = self._vectorize_task_count
+ tasks = list(self._pending_vectorize_tasks)
+
+ if task_count > 0:
+ from .embedding_tracker import EmbeddingTaskTracker
+
+ tracker = EmbeddingTaskTracker.get_instance()
+ await tracker.register(
+ semantic_msg_id=self._semantic_msg_id,
+ total_count=task_count,
+ on_complete=on_complete,
+ metadata={"uri": root_uri},
+ )
+
+ for task in tasks:
+ if task.task_type == "file":
+ asyncio.create_task(
+ self._processor._vectorize_single_file(
+ parent_uri=task.parent_uri,
+ context_type=task.context_type,
+ file_path=task.file_path,
+ summary_dict=task.summary_dict,
+ ctx=task.ctx,
+ semantic_msg_id=task.semantic_msg_id,
+ )
+ )
+ else:
+ asyncio.create_task(
+ self._processor._vectorize_directory(
+ task.uri,
+ task.context_type,
+ task.abstract,
+ task.overview,
+ ctx=task.ctx,
+ semantic_msg_id=task.semantic_msg_id,
+ )
+ )
+ elif on_complete:
+ try:
+ await on_complete()
+ except Exception as e:
+ logger.error(f"Error in on_complete callback: {e}", exc_info=True)
+
async def _dispatch_dir(self, dir_uri: str, parent_uri: Optional[str]) -> None:
"""Lazy-dispatch tasks for a directory when it is triggered."""
if dir_uri in self._nodes:
@@ -84,7 +220,10 @@ class SemanticDagExecutor:
children_dirs, file_paths = await self._list_dir(dir_uri)
file_index = {path: idx for idx, path in enumerate(file_paths)}
child_index = {path: idx for idx, path in enumerate(children_dirs)}
- pending = len(children_dirs) + len(file_paths)
+ if self._recursive:
+ pending = len(children_dirs) + len(file_paths)
+ else:
+ pending = len(file_paths)
node = DirNode(
uri=dir_uri,
@@ -113,8 +252,10 @@ class SemanticDagExecutor:
self._stats.in_progress_nodes += 1
asyncio.create_task(self._file_summary_task(dir_uri, file_path))
- for child_uri in children_dirs:
- asyncio.create_task(self._dispatch_dir(child_uri, dir_uri))
+ if children_dirs:
+ if self._recursive:
+ for child_uri in children_dirs:
+ asyncio.create_task(self._dispatch_dir(child_uri, dir_uri))
except Exception as e:
logger.error(f"Failed to dispatch directory {dir_uri}: {e}", exc_info=True)
if parent_uri:
@@ -146,13 +287,133 @@ class SemanticDagExecutor:
return children_dirs, file_paths
+ def _get_target_file_path(self, current_uri: str) -> Optional[str]:
+ if not self._incremental_update or not self._target_uri or not self._root_uri:
+ logger.warning(
+ f"Invalid target_uri or root_uri for incremental update: target_uri={self._target_uri}, root_uri={self._root_uri}"
+ )
+ return None
+ try:
+ relative_path = current_uri[len(self._root_uri) :]
+ if relative_path.startswith("/"):
+ relative_path = relative_path[1:]
+ return f"{self._target_uri}/{relative_path}" if relative_path else self._target_uri
+ except Exception:
+ return None
+
+ async def _check_file_content_changed(self, file_path: str) -> bool:
+ target_path = self._get_target_file_path(file_path)
+ if not target_path:
+ return True
+ try:
+ current_content = await self._viking_fs.read_file(file_path, ctx=self._ctx)
+ target_content = await self._viking_fs.read_file(target_path, ctx=self._ctx)
+ return current_content != target_content
+ except Exception:
+ return True
+
+ async def _read_existing_summary(self, file_path: str) -> Optional[Dict[str, str]]:
+ """Read existing summary from parent directory's .overview.md.
+
+ Args:
+ file_path: Current file path
+
+ Returns:
+ Summary dict with 'name' and 'summary' keys, or None if not found
+ """
+ target_path = self._get_target_file_path(file_path)
+ if not target_path:
+ return None
+
+ try:
+ parent_uri = "/".join(target_path.rsplit("/", 1)[:-1])
+ if not parent_uri:
+ return None
+
+ if parent_uri not in self._overview_cache:
+ async with self._overview_cache_lock:
+ if parent_uri not in self._overview_cache:
+ overview_path = f"{parent_uri}/.overview.md"
+ overview_content = await self._viking_fs.read_file(
+ overview_path, ctx=self._ctx
+ )
+ if overview_content:
+ self._overview_cache[parent_uri] = self._processor._parse_overview_md(
+ overview_content
+ )
+ else:
+ self._overview_cache[parent_uri] = {}
+
+ existing_summaries = self._overview_cache.get(parent_uri, {})
+ file_name = file_path.split("/")[-1]
+
+ if file_name in existing_summaries:
+ return {"name": file_name, "summary": existing_summaries[file_name]}
+
+ except Exception as e:
+ logger.debug(f"Failed to read existing summary from overview.md for {file_path}: {e}")
+
+ return None
+
+ async def _check_dir_children_changed(
+ self, dir_uri: str, current_files: List[str], current_dirs: List[str]
+ ) -> bool:
+ target_path = self._get_target_file_path(dir_uri)
+ if not target_path:
+ return True
+ try:
+ target_dirs, target_files = await self._list_dir(target_path)
+ current_file_names = {f.split("/")[-1] for f in current_files}
+ target_file_names = {f.split("/")[-1] for f in target_files}
+ if current_file_names != target_file_names:
+ return True
+ current_dir_names = {d.split("/")[-1] for d in current_dirs}
+ target_dir_names = {d.split("/")[-1] for d in target_dirs}
+ if current_dir_names != target_dir_names:
+ return True
+ for current_file in current_files:
+ if self._file_change_status.get(current_file, True):
+ return True
+ for current_dir in current_dirs:
+ if self._dir_change_status.get(current_dir, True):
+ return True
+ return False
+ except Exception:
+ return True
+
+ async def _read_existing_overview_abstract(
+ self, dir_uri: str
+ ) -> tuple[Optional[str], Optional[str]]:
+ target_path = self._get_target_file_path(dir_uri)
+ if not target_path:
+ return None, None
+ try:
+ overview = await self._viking_fs.read_file(f"{target_path}/.overview.md", ctx=self._ctx)
+ abstract = await self._viking_fs.read_file(f"{target_path}/.abstract.md", ctx=self._ctx)
+ return overview, abstract
+ except Exception:
+ return None, None
+
async def _file_summary_task(self, parent_uri: str, file_path: str) -> None:
"""Generate file summary and notify parent completion."""
+
file_name = file_path.split("/")[-1]
+ need_vectorize = True
try:
- summary_dict = await self._processor._generate_single_file_summary(
- file_path, llm_sem=self._llm_sem, ctx=self._ctx
- )
+ summary_dict = None
+ if self._incremental_update:
+ content_changed = await self._check_file_content_changed(file_path)
+ self._file_change_status[file_path] = content_changed
+
+ if not content_changed:
+ summary_dict = await self._read_existing_summary(file_path)
+ need_vectorize = False
+ else:
+ self._file_change_status[file_path] = True
+ if summary_dict is None:
+ summary_dict = await self._processor._generate_single_file_summary(
+ file_path, llm_sem=self._llm_sem, ctx=self._ctx
+ )
except Exception as e:
logger.warning(f"Failed to generate summary for {file_path}: {e}")
summary_dict = {"name": file_name, "summary": ""}
@@ -160,21 +421,22 @@ class SemanticDagExecutor:
self._stats.done_nodes += 1
self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)
- await self._on_file_done(parent_uri, file_path, summary_dict)
-
- # Vectorize file as soon as summary is ready to avoid waiting for overview.
try:
- asyncio.create_task(
- self._processor._vectorize_single_file(
- parent_uri=parent_uri,
+ if need_vectorize:
+ task = VectorizeTask(
+ task_type="file",
+ uri=file_path,
context_type=self._context_type,
+ ctx=self._ctx,
+ semantic_msg_id=self._semantic_msg_id,
file_path=file_path,
summary_dict=summary_dict,
- ctx=self._ctx,
+ parent_uri=parent_uri,
)
- )
+ await self._add_vectorize_task(task)
except Exception as e:
logger.error(f"Failed to schedule vectorization for {file_path}: {e}", exc_info=True)
+ await self._on_file_done(parent_uri, file_path, summary_dict)
async def _on_file_done(
self, parent_uri: str, file_path: str, summary_dict: Dict[str, str]
@@ -246,17 +508,28 @@ class SemanticDagExecutor:
node = self._nodes.get(dir_uri)
if not node:
return
-
- async with node.lock:
- file_summaries = self._finalize_file_summaries(node)
- children_abstracts = self._finalize_children_abstracts(node)
-
+ need_vectorize = True
+ children_changed = True
try:
- async with self._llm_sem:
- overview = await self._processor._generate_overview(
- dir_uri, file_summaries, children_abstracts
+ overview = None
+ abstract = None
+ if self._incremental_update:
+ children_changed = await self._check_dir_children_changed(
+ dir_uri, node.file_paths, node.children_dirs
)
- abstract = self._processor._extract_abstract_from_overview(overview)
+
+ if not children_changed:
+ need_vectorize = False
+ overview, abstract = await self._read_existing_overview_abstract(dir_uri)
+ if overview is None or abstract is None:
+ async with node.lock:
+ file_summaries = self._finalize_file_summaries(node)
+ children_abstracts = self._finalize_children_abstracts(node)
+ async with self._llm_sem:
+ overview = await self._processor._generate_overview(
+ dir_uri, file_summaries, children_abstracts
+ )
+ abstract = self._processor._extract_abstract_from_overview(overview)
try:
await self._viking_fs.write_file(f"{dir_uri}/.overview.md", overview, ctx=self._ctx)
@@ -265,11 +538,19 @@ class SemanticDagExecutor:
logger.warning(f"Failed to write overview/abstract for {dir_uri}: {e}")
try:
- await self._processor._vectorize_directory_simple(
- dir_uri, self._context_type, abstract, overview, ctx=self._ctx
- )
+ if need_vectorize:
+ task = VectorizeTask(
+ task_type="directory",
+ uri=dir_uri,
+ context_type=self._context_type,
+ ctx=self._ctx,
+ semantic_msg_id=self._semantic_msg_id,
+ abstract=abstract,
+ overview=overview,
+ )
+ await self._add_vectorize_task(task)
except Exception as e:
- logger.error(f"Failed to vectorize directory {dir_uri}: {e}", exc_info=True)
+ logger.error(f"Failed to schedule vectorization for {dir_uri}: {e}", exc_info=True)
except Exception as e:
logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)
@@ -278,6 +559,8 @@ class SemanticDagExecutor:
self._stats.done_nodes += 1
self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)
+ self._dir_change_status[dir_uri] = children_changed
+
parent_uri = self._parent.get(dir_uri)
if parent_uri is None:
if self._root_done:
@@ -286,6 +569,15 @@ class SemanticDagExecutor:
await self._on_child_done(parent_uri, dir_uri, abstract)
+ async def _add_vectorize_task(self, task: VectorizeTask) -> None:
+ """Add a vectorize task to the pending list."""
+ async with self._vectorize_lock:
+ self._pending_vectorize_tasks.append(task)
+ if task.task_type == "file":
+ self._vectorize_task_count += 1
+ else: # directory
+ self._vectorize_task_count += 2
+
def get_stats(self) -> DagStats:
return DagStats(
total_nodes=self._stats.total_nodes,
diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.py
index 59c4ba94e..f6acdaf4e 100644
--- a/openviking/storage/queuefs/semantic_msg.py
+++ b/openviking/storage/queuefs/semantic_msg.py
@@ -5,7 +5,7 @@
import json
from dataclasses import asdict, dataclass
from datetime import datetime
-from typing import Any, Dict
+from typing import Any, Dict, List, Optional
from uuid import uuid4
@@ -16,7 +16,7 @@ class SemanticMsg:
Attributes:
id: Unique identifier (UUID)
uri: Directory URI to process
- context_type: Type of context (resource, memory, skill)
+ context_type: Type of context (resource, memory, skill, session)
status: Processing status (pending/processing/completed)
timestamp: Creation timestamp
recursive: Whether to recursively process subdirectories.
@@ -27,7 +27,7 @@ class SemanticMsg:
id: str # UUID
uri: str # Directory URI
- context_type: str # resource, memory, skill
+ context_type: str # resource, memory, skill, session
status: str = "pending" # pending/processing/completed
timestamp: int = int(datetime.now().timestamp())
recursive: bool = True # Whether to recursively process subdirectories
@@ -38,6 +38,10 @@ class SemanticMsg:
# Additional flags
skip_vectorization: bool = False
telemetry_id: str = ""
+ target_uri: str = ""
+ changes: Optional[Dict[str, List[str]]] = (
+ None # {"added": [...], "modified": [...], "deleted": [...]}
+ )
def __init__(
self,
@@ -50,6 +54,8 @@ class SemanticMsg:
role: str = "root",
skip_vectorization: bool = False,
telemetry_id: str = "",
+ target_uri: str = "",
+ changes: Optional[Dict[str, List[str]]] = None,
):
self.id = str(uuid4())
self.uri = uri
@@ -61,6 +67,8 @@ class SemanticMsg:
self.role = role
self.skip_vectorization = skip_vectorization
self.telemetry_id = telemetry_id
+ self.target_uri = target_uri
+ self.changes = changes
def to_dict(self) -> Dict[str, Any]:
"""Convert object to dictionary."""
@@ -97,6 +105,8 @@ class SemanticMsg:
role=data.get("role", "root"),
skip_vectorization=data.get("skip_vectorization", False),
telemetry_id=data.get("telemetry_id", ""),
+ target_uri=data.get("target_uri", ""),
+ changes=data.get("changes"),
)
if "id" in data and data["id"]:
obj.id = data["id"]
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 2e861bcc3..60dc06e64 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -5,8 +5,8 @@
import asyncio
import threading
from contextlib import nullcontext
-from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Tuple
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Set, Tuple
from openviking.parse.parsers.constants import (
CODE_EXTENSIONS,
@@ -37,6 +37,16 @@ logger = get_logger(__name__)
@dataclass
+class DiffResult:
+ """Directory diff result for sync operations."""
+
+ added_files: List[str] = field(default_factory=list)
+ deleted_files: List[str] = field(default_factory=list)
+ updated_files: List[str] = field(default_factory=list)
+ added_dirs: List[str] = field(default_factory=list)
+ deleted_dirs: List[str] = field(default_factory=list)
+
+
class RequestQueueStats:
processed: int = 0
error_count: int = 0
@@ -176,52 +186,17 @@ class SemanticProcessor(DequeueHandlerBase):
# Default to other
return FILE_TYPE_OTHER
- async def _enqueue_semantic_msg(self, msg: SemanticMsg) -> None:
- """Enqueue a SemanticMsg to the semantic queue for processing."""
- from openviking.storage.queuefs import get_queue_manager
-
- queue_manager = get_queue_manager()
- semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC)
- # The queue manager returns SemanticQueue but method signature says NamedQueue
- # We need to ignore the type error for the enqueue call
- await semantic_queue.enqueue(msg) # type: ignore
- logger.debug(f"Enqueued semantic message for processing: {msg.uri}")
-
- async def _collect_directory_info(
- self,
- uri: str,
- result: List[Tuple[str, List[str], List[str]]],
- ) -> None:
- """Recursively collect directory info, post-order traversal ensures bottom-up order."""
+ async def _check_file_content_changed(
+ self, file_path: str, target_file: str, ctx: Optional[RequestContext] = None
+ ) -> bool:
+ """Check if file content has changed compared to target file."""
viking_fs = get_viking_fs()
-
try:
- entries = await viking_fs.ls(uri, ctx=self._current_ctx)
- except Exception as e:
- logger.warning(f"Failed to list directory {uri}: {e}")
- return
-
- children_uris = []
- file_paths = []
-
- for entry in entries:
- name = entry.get("name", "")
- if not name or name.startswith(".") or name in [".", ".."]:
- continue
-
- item_uri = VikingURI(uri).join(name).uri
-
- if entry.get("isDir", False):
- # Child directory
- children_uris.append(item_uri)
- # Recursively collect children
- await self._collect_directory_info(item_uri, result)
- else:
- # File (not starting with .)
- file_paths.append(item_uri)
-
- # Add current directory info
- result.append((uri, children_uris, file_paths))
+ current_content = await viking_fs.read_file(file_path, ctx=ctx)
+ target_content = await viking_fs.read_file(target_file, ctx=ctx)
+ return current_content != target_content
+ except Exception:
+ return True
async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
"""Process dequeued SemanticMsg, recursively process all subdirectories."""
@@ -236,12 +211,10 @@ class SemanticProcessor(DequeueHandlerBase):
if "data" in data and isinstance(data["data"], str):
data = json.loads(data["data"])
- # data is guaranteed to be not None at this point
assert data is not None
msg = SemanticMsg.from_dict(data)
collector = resolve_telemetry(msg.telemetry_id)
telemetry_ctx = bind_telemetry(collector) if collector is not None else nullcontext()
-
with telemetry_ctx:
self._current_msg = msg
self._current_ctx = self._ctx_from_semantic_msg(msg)
@@ -249,12 +222,32 @@ class SemanticProcessor(DequeueHandlerBase):
f"Processing semantic generation for: {msg.uri} (recursive={msg.recursive})"
)
- if msg.recursive:
+ logger.info(f"Processing semantic generation for: {msg})")
+
+ if msg.context_type == "memory":
+ await self._process_memory_directory(msg)
+ else:
+ is_incremental = False
+ viking_fs = get_viking_fs()
+ if msg.target_uri:
+ target_exists = await viking_fs.exists(
+ msg.target_uri, ctx=self._current_ctx
+ )
+ if target_exists:
+ is_incremental = True
+ logger.info(
+ f"Target URI exists, using incremental update: {msg.target_uri}"
+ )
+
executor = SemanticDagExecutor(
processor=self,
context_type=msg.context_type,
max_concurrent_llm=self.max_concurrent_llm,
ctx=self._current_ctx,
+ incremental_update=is_incremental,
+ target_uri=msg.target_uri,
+ semantic_msg_id=msg.id,
+ recursive=msg.recursive,
)
self._dag_executor = executor
await executor.run(msg.uri)
@@ -263,44 +256,6 @@ class SemanticProcessor(DequeueHandlerBase):
msg.uri,
executor.get_stats(),
)
- else:
- children_uris = []
- file_paths = []
-
- viking_fs = get_viking_fs()
- try:
- entries = await viking_fs.ls(msg.uri, ctx=self._current_ctx)
- for entry in entries:
- name = entry.get("name", "")
- if not name or name.startswith(".") or name in [".", ".."]:
- continue
-
- item_uri = VikingURI(msg.uri).join(name).uri
-
- if entry.get("isDir", False):
- children_uris.append(item_uri)
- else:
- file_paths.append(item_uri)
- except Exception as e:
- logger.warning(f"Failed to list directory {msg.uri}: {e}")
-
- await self._process_single_directory(
- uri=msg.uri,
- context_type=msg.context_type,
- children_uris=children_uris,
- file_paths=file_paths,
- )
- self._cache_dag_stats(
- msg.telemetry_id,
- msg.uri,
- DagStats(
- total_nodes=1,
- pending_nodes=0,
- in_progress_nodes=0,
- done_nodes=1,
- ),
- )
-
self._merge_request_stats(msg.telemetry_id, processed=1)
logger.info(f"Completed semantic generation for: {msg.uri}")
self.report_success()
@@ -314,90 +269,293 @@ class SemanticProcessor(DequeueHandlerBase):
return None
finally:
self._current_msg = None
+ self._current_ctx = None
def get_dag_stats(self) -> Optional["DagStats"]:
if not self._dag_executor:
return None
return self._dag_executor.get_stats()
- async def _process_single_directory(
- self,
- uri: str,
- context_type: str,
- children_uris: List[str],
- file_paths: List[str],
- ) -> None:
- """Process single directory, generate .abstract.md and .overview.md."""
+ async def _process_memory_directory(self, msg: SemanticMsg) -> None:
+ """Process a memory directory with special handling.
+
+ For memory directories:
+ - Memory files are already vectorized via embedding queue
+ - Only generate abstract.md and overview.md
+ - Vectorize the generated abstract.md and overview.md
+
+ Args:
+ msg: The semantic message containing directory info and changes
+ """
viking_fs = get_viking_fs()
+ dir_uri = msg.uri
+ ctx = self._current_ctx
+ llm_sem = asyncio.Semaphore(self.max_concurrent_llm)
- # 1. Collect .abstract.md from subdirectories (already processed earlier)
- children_abstracts = await self._collect_children_abstracts(children_uris)
+ try:
+ entries = await viking_fs.ls(dir_uri, ctx=ctx)
+ except Exception as e:
+ logger.warning(f"Failed to list memory directory {dir_uri}: {e}")
+ return
- # 2. Concurrently generate summaries for files in directory
- file_summaries = await self._generate_file_summaries(
- file_paths, context_type=context_type, parent_uri=uri, enqueue_files=True
- )
+ file_paths: List[str] = []
+ for entry in entries:
+ name = entry.get("name", "")
+ if not name or name.startswith(".") or name in [".", ".."]:
+ continue
+ if not entry.get("isDir", False):
+ item_uri = VikingURI(dir_uri).join(name).uri
+ file_paths.append(item_uri)
- # 3. Generate .overview.md (contains brief description)
- overview = await self._generate_overview(uri, file_summaries, children_abstracts)
+ if not file_paths:
+ logger.info(f"No memory files found in {dir_uri}")
+ return
- # 4. Extract abstract from overview
+ file_summaries: List[Dict[str, str]] = []
+ existing_summaries: Dict[str, str] = {}
+
+ if msg.changes:
+ try:
+ old_overview = await viking_fs.read_file(f"{dir_uri}/.overview.md", ctx=ctx)
+ if old_overview:
+ existing_summaries = self._parse_overview_md(old_overview)
+ logger.info(
+ f"Parsed {len(existing_summaries)} existing summaries from overview.md"
+ )
+ except Exception as e:
+ logger.debug(f"No existing overview.md found for {dir_uri}: {e}")
+
+ changed_files: Set[str] = set()
+ if msg.changes:
+ changed_files = set(msg.changes.get("added", []) + msg.changes.get("modified", []))
+ deleted_files = set(msg.changes.get("deleted", []))
+ logger.info(
+ f"Processing memory directory {dir_uri} with changes: "
+ f"added={len(msg.changes.get('added', []))}, "
+ f"modified={len(msg.changes.get('modified', []))}, "
+ f"deleted={len(deleted_files)}"
+ )
+
+ for file_path in file_paths:
+ file_name = file_path.split("/")[-1]
+
+ if file_path not in changed_files and file_name in existing_summaries:
+ file_summaries.append({"name": file_name, "summary": existing_summaries[file_name]})
+ logger.debug(f"Reused existing summary for {file_name}")
+ else:
+ try:
+ summary_dict = await self._generate_single_file_summary(
+ file_path, llm_sem=llm_sem, ctx=ctx
+ )
+ file_summaries.append(summary_dict)
+ logger.debug(f"Generated summary for {file_name}")
+ except Exception as e:
+ logger.warning(f"Failed to generate summary for {file_path}: {e}")
+ file_summaries.append({"name": file_name, "summary": ""})
+
+ overview = await self._generate_overview(dir_uri, file_summaries, [])
abstract = self._extract_abstract_from_overview(overview)
- # 5. Write files
- await viking_fs.write_file(f"{uri}/.overview.md", overview, ctx=self._current_ctx)
- await viking_fs.write_file(f"{uri}/.abstract.md", abstract, ctx=self._current_ctx)
+ try:
+ await viking_fs.write_file(f"{dir_uri}/.overview.md", overview, ctx=ctx)
+ await viking_fs.write_file(f"{dir_uri}/.abstract.md", abstract, ctx=ctx)
+ logger.info(f"Generated abstract.md and overview.md for {dir_uri}")
+ except Exception as e:
+ logger.error(f"Failed to write abstract/overview for {dir_uri}: {e}")
+ return
- logger.debug(f"Generated overview and abstract for {uri}")
+ await self._vectorize_directory(
+ uri=dir_uri,
+ context_type="memory",
+ abstract=abstract,
+ overview=overview,
+ ctx=ctx,
+ semantic_msg_id=msg.id,
+ )
+ logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")
- # 6. Vectorize directory
+ async def _sync_topdown_recursive(
+ self,
+ root_uri: str,
+ target_uri: str,
+ ctx: Optional[RequestContext] = None,
+ file_change_status: Optional[Dict[str, bool]] = None,
+ ) -> DiffResult:
+ viking_fs = get_viking_fs()
+ diff = DiffResult()
+
+ async def list_children(dir_uri: str) -> Tuple[Dict[str, str], Dict[str, str]]:
+ files: Dict[str, str] = {}
+ dirs: Dict[str, str] = {}
+ try:
+ entries = await viking_fs.ls(dir_uri, show_all_hidden=True, ctx=ctx)
+ except Exception as e:
+ logger.error(f"[SyncDiff] Failed to list {dir_uri}: {e}")
+ return files, dirs
+
+ for entry in entries:
+ name = entry.get("name", "")
+ if not name or name in [".", ".."]:
+ continue
+ if name.startswith(".") and name not in [".abstract.md", ".overview.md"]:
+ continue
+ item_uri = VikingURI(dir_uri).join(name).uri
+ if entry.get("isDir", False):
+ dirs[name] = item_uri
+ else:
+ files[name] = item_uri
+ return files, dirs
+
+ async def sync_dir(root_dir: str, target_dir: str) -> None:
+ root_files, root_dirs = await list_children(root_dir)
+ target_files, target_dirs = await list_children(target_dir)
+
+ file_names = set(root_files.keys()) | set(target_files.keys())
+ for name in sorted(file_names):
+ root_file = root_files.get(name)
+ target_file = target_files.get(name)
+
+ if root_file and name in target_dirs:
+ target_conflict_dir = target_dirs[name]
+ try:
+ await viking_fs.rm(target_conflict_dir, recursive=True, ctx=ctx)
+ diff.deleted_dirs.append(target_conflict_dir)
+ target_dirs.pop(name, None)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to delete directory for file conflict: {target_conflict_dir}, error={e}"
+ )
+ target_file = None
+
+ if target_file and name in root_dirs and not root_file:
+ try:
+ await viking_fs.rm(target_file, ctx=ctx)
+ diff.deleted_files.append(target_file)
+ target_files.pop(name, None)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to delete file for dir conflict: {target_file}, error={e}"
+ )
+ continue
+
+ if target_file and not root_file:
+ try:
+ await viking_fs.rm(target_file, ctx=ctx)
+ diff.deleted_files.append(target_file)
+ except Exception as e:
+ logger.error(f"[SyncDiff] Failed to delete file: {target_file}, error={e}")
+ continue
+
+ if root_file and target_file:
+ changed = False
+ if file_change_status and root_file in file_change_status:
+ changed = file_change_status[root_file]
+ else:
+ try:
+ changed = await self._check_file_content_changed(
+ root_file, target_file, ctx=ctx
+ )
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to compare file content for {root_file}: {e}, treating as unchanged"
+ )
+ changed = False
+ if changed:
+ diff.updated_files.append(root_file)
+ try:
+ await viking_fs.rm(target_file, ctx=ctx)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to remove old file before update: {target_file}, error={e}"
+ )
+ try:
+ await viking_fs.mv(root_file, target_file, ctx=ctx)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to move updated file: {root_file} -> {target_file}, error={e}"
+ )
+ continue
+
+ if root_file and not target_file:
+ diff.added_files.append(root_file)
+ target_file_uri = VikingURI(target_dir).join(name).uri
+ try:
+ await viking_fs.mv(root_file, target_file_uri, ctx=ctx)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to move added file: {root_file} -> {target_file_uri}, error={e}"
+ )
+
+ dir_names = set(root_dirs.keys()) | set(target_dirs.keys())
+ for name in sorted(dir_names):
+ root_subdir = root_dirs.get(name)
+ target_subdir = target_dirs.get(name)
+
+ if root_subdir and name in target_files:
+ target_conflict_file = target_files[name]
+ try:
+ await viking_fs.rm(target_conflict_file, ctx=ctx)
+ diff.deleted_files.append(target_conflict_file)
+ target_files.pop(name, None)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to delete file for dir conflict: {target_conflict_file}, error={e}"
+ )
+ target_subdir = None
+
+ if target_subdir and not root_subdir:
+ try:
+ await viking_fs.rm(target_subdir, recursive=True, ctx=ctx)
+ diff.deleted_dirs.append(target_subdir)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to delete directory: {target_subdir}, error={e}"
+ )
+ continue
+
+ if root_subdir and not target_subdir:
+ diff.added_dirs.append(root_subdir)
+ target_subdir_uri = VikingURI(target_dir).join(name).uri
+ try:
+ await viking_fs.mv(root_subdir, target_subdir_uri, ctx=ctx)
+ except Exception as e:
+ logger.error(
+ f"[SyncDiff] Failed to move added directory: {root_subdir} -> {target_subdir_uri}, error={e}"
+ )
+ continue
+
+ if root_subdir and target_subdir:
+ await sync_dir(root_subdir, target_subdir)
+
+ target_exists = await viking_fs.exists(target_uri, ctx=ctx)
+ if not target_exists:
+ parent_uri = VikingURI(target_uri).parent
+ if parent_uri:
+ await viking_fs.mkdir(parent_uri.uri, exist_ok=True, ctx=ctx)
+ diff.added_dirs.append(root_uri)
+ await viking_fs.mv(root_uri, target_uri, ctx=ctx)
+ return diff
+
+ await sync_dir(root_uri, target_uri)
try:
- await self._vectorize_directory_simple(uri, context_type, abstract, overview)
+ await viking_fs.rm(root_uri, recursive=True, ctx=ctx)
except Exception as e:
- logger.error(f"Failed to vectorize directory {uri}: {e}", exc_info=True)
+ logger.error(f"[SyncDiff] Failed to delete root directory {root_uri}: {e}")
+ return diff
- async def _collect_children_abstracts(self, children_uris: List[str]) -> List[Dict[str, str]]:
+ async def _collect_children_abstracts(
+ self, children_uris: List[str], ctx: Optional[RequestContext] = None
+ ) -> List[Dict[str, str]]:
"""Collect .abstract.md from subdirectories."""
viking_fs = get_viking_fs()
results = []
for child_uri in children_uris:
- abstract = await viking_fs.abstract(child_uri, ctx=self._current_ctx)
+ abstract = await viking_fs.abstract(child_uri, ctx=ctx)
dir_name = child_uri.split("/")[-1]
results.append({"name": dir_name, "abstract": abstract})
return results
- async def _generate_file_summaries(
- self,
- file_paths: List[str],
- context_type: Optional[str] = None,
- parent_uri: Optional[str] = None,
- enqueue_files: bool = False,
- ) -> List[Dict[str, str]]:
- """Concurrently generate file summaries."""
- if not file_paths:
- return []
-
- async def generate_one_summary(file_path: str) -> Dict[str, str]:
- summary = await self._generate_single_file_summary(file_path, ctx=self._current_ctx)
- if enqueue_files and context_type and parent_uri:
- try:
- await self._vectorize_single_file(
- parent_uri=parent_uri,
- context_type=context_type,
- file_path=file_path,
- summary_dict=summary,
- )
- except Exception as e:
- logger.error(
- f"Failed to vectorize file {file_path}: {e}",
- exc_info=True,
- )
- return summary
-
- tasks = [generate_one_summary(fp) for fp in file_paths]
- return await asyncio.gather(*tasks)
-
async def _generate_text_summary(
self,
file_path: str,
@@ -410,11 +568,6 @@ class SemanticProcessor(DequeueHandlerBase):
vlm = get_openviking_config().vlm
active_ctx = ctx or self._current_ctx
- # Read file content (limit length)
- content = await viking_fs.read_file(file_path, ctx=active_ctx)
-
- # Limit content length (about 10000 tokens)
- max_chars = 30000
content = await viking_fs.read_file(file_path, ctx=active_ctx)
if isinstance(content, bytes):
# Try to decode with error handling for text files
@@ -533,6 +686,59 @@ class SemanticProcessor(DequeueHandlerBase):
return "\n".join(content_lines).strip()
+ def _parse_overview_md(self, overview_content: str) -> Dict[str, str]:
+ """Parse overview.md and extract file summaries.
+
+ Args:
+ overview_content: Content of the overview.md file
+
+ Returns:
+ Dictionary mapping file names to their summaries
+ """
+ import re
+
+ summaries: Dict[str, str] = {}
+
+ if not overview_content or not overview_content.strip():
+ return summaries
+
+ lines = overview_content.split("\n")
+ current_file = None
+ current_summary_lines: List[str] = []
+
+ for line in lines:
+ header_match = re.match(r"^###\s+(.+?)\s*$", line)
+ if header_match:
+ if current_file and current_summary_lines:
+ summaries[current_file] = " ".join(current_summary_lines).strip()
+
+ file_name = header_match.group(1).strip()
+ parts = file_name.split()
+ if len(parts) >= 2 and parts[0] == parts[1]:
+ file_name = parts[0]
+
+ current_file = file_name
+ current_summary_lines = []
+ continue
+
+ numbered_match = re.match(r"^\[(\d+)\]\s+(.+?):\s*(.+)$", line)
+ if numbered_match:
+ if current_file and current_summary_lines:
+ summaries[current_file] = " ".join(current_summary_lines).strip()
+ current_file = numbered_match.group(2).strip()
+ current_summary_lines = [numbered_match.group(3).strip()]
+ continue
+
+ if current_file:
+ stripped = line.strip()
+ if stripped and not stripped.startswith("#"):
+ current_summary_lines.append(stripped)
+
+ if current_file and current_summary_lines:
+ summaries[current_file] = " ".join(current_summary_lines).strip()
+
+ return summaries
+
async def _generate_overview(
self,
dir_uri: str,
@@ -598,13 +804,14 @@ class SemanticProcessor(DequeueHandlerBase):
logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)
return f"# {dir_uri.split('/')[-1]}\n\nDirectory overview"
- async def _vectorize_directory_simple(
+ async def _vectorize_directory(
self,
uri: str,
context_type: str,
abstract: str,
overview: str,
ctx: Optional[RequestContext] = None,
+ semantic_msg_id: Optional[str] = None,
) -> None:
"""Create directory Context and enqueue to EmbeddingQueue."""
@@ -621,40 +828,17 @@ class SemanticProcessor(DequeueHandlerBase):
overview=overview,
context_type=context_type,
ctx=active_ctx,
+ semantic_msg_id=semantic_msg_id,
)
- async def _vectorize_files(
- self,
- uri: str,
- context_type: str,
- file_paths: List[str],
- file_summaries: List[Dict[str, str]],
- ctx: Optional[RequestContext] = None,
- ) -> None:
- """Vectorize files in directory."""
- from openviking.storage.queuefs import get_queue_manager
-
- queue_manager = get_queue_manager()
- embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)
-
- for file_path, file_summary_dict in zip(file_paths, file_summaries):
- await self._vectorize_single_file(
- parent_uri=uri,
- context_type=context_type,
- file_path=file_path,
- summary_dict=file_summary_dict,
- embedding_queue=embedding_queue,
- ctx=ctx,
- )
-
async def _vectorize_single_file(
self,
parent_uri: str,
context_type: str,
file_path: str,
summary_dict: Dict[str, str],
- embedding_queue: Optional[Any] = None,
ctx: Optional[RequestContext] = None,
+ semantic_msg_id: Optional[str] = None,
) -> None:
"""Vectorize a single file using its content or summary."""
from openviking.utils.embedding_utils import vectorize_file
@@ -666,4 +850,5 @@ class SemanticProcessor(DequeueHandlerBase):
parent_uri=parent_uri,
context_type=context_type,
ctx=active_ctx,
+ semantic_msg_id=semantic_msg_id,
)
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 62cf15fa7..7c45f5441 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -361,6 +361,22 @@ class VikingFS:
path = self._uri_to_path(uri, ctx=ctx)
return self.agfs.stat(path)
+ async def exists(self, uri: str, ctx: Optional[RequestContext] = None) -> bool:
+ """Check if a URI exists.
+
+ Args:
+ uri: Viking URI
+ ctx: Request context
+
+ Returns:
+ bool: True if the URI exists, False otherwise
+ """
+ try:
+ await self.stat(uri, ctx=ctx)
+ return True
+ except Exception:
+ return False
+
async def glob(
self,
pattern: str,
@@ -1086,7 +1102,7 @@ class VikingFS:
async def _collect_uris(
self, path: str, recursive: bool, ctx: Optional[RequestContext] = None
) -> List[str]:
- """Recursively collect all URIs (for rm/mv)."""
+ """Recursively collect all URIs (for rm/mv), including directories."""
uris = []
async def _collect(p: str):
@@ -1097,6 +1113,7 @@ class VikingFS:
continue
full_path = f"{p}/{name}".replace("//", "/")
if entry.get("isDir"):
+ uris.append(self._path_to_uri(full_path, ctx=ctx))
if recursive:
await _collect(full_path)
else:
@@ -1146,23 +1163,8 @@ class VikingFS:
for uri in uris:
try:
- records = await vector_store.get_context_by_uri(
- uri=uri,
- limit=1,
- ctx=self._ctx_or_default(ctx),
- )
-
- if not records or "id" not in records[0]:
- continue
-
- record = records[0]
-
new_uri = uri.replace(old_base_uri, new_base_uri, 1)
-
- old_parent_uri = record.get("parent_uri", "")
- new_parent_uri = (
- old_parent_uri.replace(old_base_uri, new_base_uri, 1) if old_parent_uri else ""
- )
+ new_parent_uri = VikingURI(new_uri).parent.uri
await vector_store.update_uri_mapping(
ctx=self._ctx_or_default(ctx),
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index 5effb6cbc..880af30b7 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -830,15 +830,54 @@ class VikingVectorIndexBackend:
new_uri: str,
new_parent_uri: str,
) -> bool:
+ import hashlib
+
records = await self.filter(
filter=And([Eq("uri", uri), Eq("account_id", ctx.account_id)]),
- limit=1,
+ limit=100,
ctx=ctx,
)
- if not records or "id" not in records[0]:
+ if not records:
return False
- updated = {**records[0], "uri": new_uri, "parent_uri": new_parent_uri}
- return bool(await self.upsert(updated, ctx=ctx))
+
+ def _seed_uri_for_id(uri: str, level: int) -> str:
+ if level == 0:
+ return uri if uri.endswith("/.abstract.md") else f"{uri}/.abstract.md"
+ if level == 1:
+ return uri if uri.endswith("/.overview.md") else f"{uri}/.overview.md"
+ return uri
+
+ success = False
+ ids_to_delete: List[str] = []
+ for record in records:
+ if "id" not in record:
+ continue
+ raw_level = record.get("level", 2)
+ try:
+ level = int(raw_level)
+ except (TypeError, ValueError):
+ level = 2
+
+ seed_uri = _seed_uri_for_id(new_uri, level)
+ id_seed = f"{ctx.account_id}:{seed_uri}"
+ new_id = hashlib.md5(id_seed.encode("utf-8")).hexdigest()
+
+ updated = {
+ **record,
+ "id": new_id,
+ "uri": new_uri,
+ "parent_uri": new_parent_uri,
+ }
+ if await self.upsert(updated, ctx=ctx):
+ success = True
+ old_id = record.get("id")
+ if old_id and old_id != new_id:
+ ids_to_delete.append(old_id)
+
+ if ids_to_delete:
+ await self.delete(list(set(ids_to_delete)), ctx=ctx)
+
+ return success
async def increment_active_count(self, ctx: RequestContext, uris: List[str]) -> int:
updated = 0
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index 1dffc1c8a..9a2f9d2d5 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -20,6 +20,22 @@ from openviking_cli.utils import VikingURI, get_logger
logger = get_logger(__name__)
+async def _decrement_embedding_tracker(semantic_msg_id: Optional[str], count: int) -> None:
+ if not semantic_msg_id or count <= 0:
+ return
+ try:
+ from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker
+
+ tracker = EmbeddingTaskTracker.get_instance()
+ for _ in range(count):
+ await tracker.decrement(semantic_msg_id)
+ except Exception as e:
+ logger.error(
+ f"Failed to decrement embedding tracker for semantic_msg_id={semantic_msg_id}: {e}",
+ exc_info=True,
+ )
+
+
def _owner_space_for_uri(uri: str, ctx: RequestContext) -> str:
"""Derive owner_space from a URI."""
if uri.startswith("viking://agent/"):
@@ -116,57 +132,78 @@ async def vectorize_directory_meta(
overview: str,
context_type: str = "resource",
ctx: Optional[RequestContext] = None,
+ semantic_msg_id: Optional[str] = None,
) -> None:
"""
Vectorize directory metadata (.abstract.md and .overview.md).
Creates Context objects for abstract and overview and enqueues them.
"""
- if not ctx:
- logger.warning("No context provided for vectorization")
- return
+ enqueued = 0
+ try:
+ if not ctx:
+ logger.warning("No context provided for vectorization")
+ return
- queue_manager = get_queue_manager()
- embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)
-
- parent_uri = VikingURI(uri).parent.uri
- owner_space = _owner_space_for_uri(uri, ctx)
-
- # Vectorize L0: .abstract.md (abstract)
- context_abstract = Context(
- uri=uri,
- parent_uri=parent_uri,
- is_leaf=False,
- abstract=abstract,
- context_type=context_type,
- level=ContextLevel.ABSTRACT,
- user=ctx.user,
- account_id=ctx.account_id,
- owner_space=owner_space,
- )
- context_abstract.set_vectorize(Vectorize(text=abstract))
- msg_abstract = EmbeddingMsgConverter.from_context(context_abstract)
- if msg_abstract:
- await embedding_queue.enqueue(msg_abstract)
- logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {uri}")
-
- # Vectorize L1: .overview.md (overview)
- context_overview = Context(
- uri=uri,
- parent_uri=parent_uri,
- is_leaf=False,
- abstract=abstract,
- context_type=context_type,
- level=ContextLevel.OVERVIEW,
- user=ctx.user,
- account_id=ctx.account_id,
- owner_space=owner_space,
- )
- context_overview.set_vectorize(Vectorize(text=overview))
- msg_overview = EmbeddingMsgConverter.from_context(context_overview)
- if msg_overview:
- await embedding_queue.enqueue(msg_overview)
- logger.debug(f"Enqueued directory L1 (overview) for vectorization: {uri}")
+ queue_manager = get_queue_manager()
+ embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)
+
+ parent_uri = VikingURI(uri).parent.uri
+ owner_space = _owner_space_for_uri(uri, ctx)
+
+ # Vectorize L0: .abstract.md (abstract)
+ context_abstract = Context(
+ uri=uri,
+ parent_uri=parent_uri,
+ is_leaf=False,
+ abstract=abstract,
+ context_type=context_type,
+ level=ContextLevel.ABSTRACT,
+ user=ctx.user,
+ account_id=ctx.account_id,
+ owner_space=owner_space,
+ )
+ context_abstract.set_vectorize(Vectorize(text=abstract))
+ msg_abstract = EmbeddingMsgConverter.from_context(context_abstract)
+ if msg_abstract:
+ msg_abstract.semantic_msg_id = semantic_msg_id
+ try:
+ await embedding_queue.enqueue(msg_abstract)
+ enqueued += 1
+ logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {uri}")
+ except Exception as e:
+ logger.error(
+ f"Failed to enqueue directory L0 (abstract) for vectorization: {uri}: {e}",
+ exc_info=True,
+ )
+
+ # Vectorize L1: .overview.md (overview)
+ context_overview = Context(
+ uri=uri,
+ parent_uri=parent_uri,
+ is_leaf=False,
+ abstract=abstract,
+ context_type=context_type,
+ level=ContextLevel.OVERVIEW,
+ user=ctx.user,
+ account_id=ctx.account_id,
+ owner_space=owner_space,
+ )
+ context_overview.set_vectorize(Vectorize(text=overview))
+ msg_overview = EmbeddingMsgConverter.from_context(context_overview)
+ if msg_overview:
+ msg_overview.semantic_msg_id = semantic_msg_id
+ try:
+ await embedding_queue.enqueue(msg_overview)
+ enqueued += 1
+ logger.debug(f"Enqueued directory L1 (overview) for vectorization: {uri}")
+ except Exception as e:
+ logger.error(
+ f"Failed to enqueue directory L1 (overview) for vectorization: {uri}: {e}",
+ exc_info=True,
+ )
+ finally:
+ await _decrement_embedding_tracker(semantic_msg_id, 2 - enqueued)
async def vectorize_file(
@@ -175,6 +212,7 @@ async def vectorize_file(
parent_uri: str,
context_type: str = "resource",
ctx: Optional[RequestContext] = None,
+ semantic_msg_id: Optional[str] = None,
) -> None:
"""
Vectorize a single file.
@@ -182,15 +220,17 @@ async def vectorize_file(
Creates Context object for the file and enqueues it.
Reads content for TEXT files, otherwise uses summary.
"""
- if not ctx:
- logger.warning("No context provided for vectorization")
- return
-
- queue_manager = get_queue_manager()
- embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)
- viking_fs = get_viking_fs()
+ enqueued = False
try:
+ if not ctx:
+ logger.warning("No context provided for vectorization")
+ return
+
+ queue_manager = get_queue_manager()
+ embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)
+ viking_fs = get_viking_fs()
+
file_name = summary_dict.get("name") or os.path.basename(file_path)
summary = summary_dict.get("summary", "")
@@ -246,11 +286,16 @@ async def vectorize_file(
if not embedding_msg:
return
+ embedding_msg.semantic_msg_id = semantic_msg_id
await embedding_queue.enqueue(embedding_msg)
+ enqueued = True
logger.debug(f"Enqueued file for vectorization: {file_path}")
except Exception as e:
logger.error(f"Failed to vectorize file {file_path}: {e}", exc_info=True)
+ finally:
+ if not enqueued:
+ await _decrement_embedding_tracker(semantic_msg_id, 1)
async def index_resource(
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index 668ecb0ff..bd56a2eae 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -123,7 +123,7 @@ class ResourceProcessor:
}
telemetry = get_current_telemetry()
- # ============ Phase 1: Parse source (Parser generates L0/L1 and writes to temp) ============
+ # ============ Phase 1: Parse source and writes to temp viking fs ============
try:
parse_start = time.perf_counter()
media_processor = self._get_media_processor()
@@ -187,10 +187,10 @@ class ResourceProcessor:
parent_uri=parent,
source_path=parse_result.source_path,
source_format=parse_result.source_format,
- trigger_semantic=True,
)
if context_tree and context_tree.root:
result["root_uri"] = context_tree.root.uri
+ result["temp_uri"] = context_tree.root.temp_uri
telemetry.set(
"resource.finalize.duration_ms",
round((time.perf_counter() - finalize_start) * 1000, 3),
@@ -211,6 +211,7 @@ class ResourceProcessor:
# ============ Phase 4: Optional Steps ============
build_index = kwargs.get("build_index", True)
+ temp_uri_for_summarize = result.get("temp_uri") or parse_result.temp_dir_path
if summarize:
# Explicit summarization request.
# If build_index is ALSO True, we want vectorization.
@@ -221,6 +222,7 @@ class ResourceProcessor:
resource_uris=[result["root_uri"]],
ctx=ctx,
skip_vectorization=skip_vec,
+ temp_uris=[temp_uri_for_summarize],
**kwargs,
)
except Exception as e:
@@ -232,7 +234,11 @@ class ResourceProcessor:
# We assume this means "Ingest and Index", which requires summarization.
try:
await self._get_summarizer().summarize(
- resource_uris=[result["root_uri"]], ctx=ctx, skip_vectorization=False, **kwargs
+ resource_uris=[result["root_uri"]],
+ ctx=ctx,
+ skip_vectorization=False,
+ temp_uris=[temp_uri_for_summarize],
+ **kwargs,
)
except Exception as e:
logger.error(f"Auto-index failed: {e}")
diff --git a/openviking/utils/summarizer.py b/openviking/utils/summarizer.py
index 3c6a39c68..22076f925 100644
--- a/openviking/utils/summarizer.py
+++ b/openviking/utils/summarizer.py
@@ -40,9 +40,21 @@ class Summarizer:
queue_manager = get_queue_manager()
semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
+ temp_uris = kwargs.get("temp_uris", [])
+ if not temp_uris:
+ temp_uris = resource_uris
+ if len(temp_uris) != len(resource_uris):
+ logger.error(
+ f"temp_uris length ({len(temp_uris)}) must match resource_uris length ({len(resource_uris)})"
+ )
+ return {
+ "status": "error",
+ "message": "temp_uris length must match resource_uris length",
+ }
enqueued_count = 0
+
telemetry = get_current_telemetry()
- for uri in resource_uris:
+ for uri, temp_uri in zip(resource_uris, temp_uris):
# Determine context_type based on URI
context_type = "resource"
if uri.startswith("viking://memory/"):
@@ -51,7 +63,7 @@ class Summarizer:
context_type = "skill"
msg = SemanticMsg(
- uri=uri,
+ uri=temp_uri,
context_type=context_type,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
@@ -59,6 +71,7 @@ class Summarizer:
role=ctx.role.value,
skip_vectorization=skip_vectorization,
telemetry_id=telemetry.telemetry_id if telemetry.enabled else "",
+ target_uri=uri,
)
await semantic_queue.enqueue(msg)
enqueued_count += 1
diff --git a/openviking_cli/exceptions.py b/openviking_cli/exceptions.py
index 807d317e5..cd432552c 100644
--- a/openviking_cli/exceptions.py
+++ b/openviking_cli/exceptions.py
@@ -70,6 +70,14 @@ class AlreadyExistsError(OpenVikingError):
)
+class ConflictError(OpenVikingError):
+ """Resource conflict (e.g., locked by another operation)."""
+
+ def __init__(self, message: str, resource: Optional[str] = None):
+ details = {"resource": resource} if resource else {}
+ super().__init__(message, code="CONFLICT", details=details)
+
+
# ============= Authentication Errors =============
diff --git a/tests/storage/test_semantic_dag_skip_files.py b/tests/storage/test_semantic_dag_skip_files.py
index 75b233146..6b52fa4e9 100644
--- a/tests/storage/test_semantic_dag_skip_files.py
+++ b/tests/storage/test_semantic_dag_skip_files.py
@@ -35,19 +35,29 @@ class _FakeProcessor:
def _extract_abstract_from_overview(self, overview):
return "abstract"
- async def _vectorize_directory_simple(self, uri, context_type, abstract, overview, ctx=None):
+ async def _vectorize_directory(
+ self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None
+ ):
pass
+ async def _vectorize_directory_simple(self, uri, context_type, abstract, overview, ctx=None):
+ await self._vectorize_directory(uri, context_type, abstract, overview, ctx=ctx)
+
async def _vectorize_single_file(
- self, parent_uri, context_type, file_path, summary_dict, ctx=None
+ self, parent_uri, context_type, file_path, summary_dict, ctx=None, semantic_msg_id=None
):
self.vectorized_files.append(file_path)
+class _DummyTracker:
+ async def register(self, **_kwargs):
+ return None
+
+
@pytest.mark.asyncio
async def test_messages_jsonl_excluded_from_summary(monkeypatch):
"""messages.jsonl should be skipped by _list_dir and never summarized."""
- root_uri = "viking://sessions/test-session"
+ root_uri = "viking://session/test-session"
tree = {
root_uri: [
{"name": "messages.jsonl", "isDir": False},
@@ -57,6 +67,10 @@ async def test_messages_jsonl_excluded_from_summary(monkeypatch):
}
fake_fs = _FakeVikingFS(tree)
monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
+ lambda: _DummyTracker(),
+ )
processor = _FakeProcessor()
ctx = RequestContext(user=UserIdentifier("acc1", "user1", "agent1"), role=Role.USER)
@@ -77,7 +91,7 @@ async def test_messages_jsonl_excluded_from_summary(monkeypatch):
@pytest.mark.asyncio
async def test_messages_jsonl_excluded_in_subdirectory(monkeypatch):
"""messages.jsonl in a subdirectory should also be skipped."""
- root_uri = "viking://sessions/test-session"
+ root_uri = "viking://session/test-session"
tree = {
root_uri: [
{"name": "subdir", "isDir": True},
@@ -89,6 +103,10 @@ async def test_messages_jsonl_excluded_in_subdirectory(monkeypatch):
}
fake_fs = _FakeVikingFS(tree)
monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
+ lambda: _DummyTracker(),
+ )
processor = _FakeProcessor()
ctx = RequestContext(user=UserIdentifier("acc1", "user1", "agent1"), role=Role.USER)
diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.py
index 10f06c226..8cb8c1f8f 100644
--- a/tests/storage/test_semantic_dag_stats.py
+++ b/tests/storage/test_semantic_dag_stats.py
@@ -1,6 +1,8 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
+import asyncio
+
import pytest
from openviking.server.identity import RequestContext, Role
@@ -34,14 +36,24 @@ class _FakeProcessor:
def _extract_abstract_from_overview(self, overview):
return "abstract"
- async def _vectorize_directory_simple(self, uri, context_type, abstract, overview, ctx=None):
+ async def _vectorize_directory(
+ self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None
+ ):
self.vectorized_dirs.append(uri)
async def _vectorize_single_file(
- self, parent_uri, context_type, file_path, summary_dict, ctx=None
+ self, parent_uri, context_type, file_path, summary_dict, ctx=None, semantic_msg_id=None
):
self.vectorized_files.append(file_path)
+ async def _vectorize_directory_simple(self, uri, context_type, abstract, overview, ctx=None):
+ await self._vectorize_directory(uri, context_type, abstract, overview, ctx=ctx)
+
+
+class _DummyTracker:
+ async def register(self, **_kwargs):
+ return None
+
@pytest.mark.asyncio
async def test_semantic_dag_stats_collects_nodes(monkeypatch):
@@ -58,6 +70,10 @@ async def test_semantic_dag_stats_collects_nodes(monkeypatch):
}
fake_fs = _FakeVikingFS(tree)
monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",
+ lambda: _DummyTracker(),
+ )
processor = _FakeProcessor()
ctx = RequestContext(user=UserIdentifier("acc1", "user1", "agent1"), role=Role.USER)
@@ -68,6 +84,7 @@ async def test_semantic_dag_stats_collects_nodes(monkeypatch):
ctx=ctx,
)
await executor.run(root_uri)
+ await asyncio.sleep(0)
stats = executor.get_stats()
assert isinstance(stats, DagStats)