Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Feature/add resource increment (#659)
yepper2e1619c352fd
diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rsindex 81126ae96..e20dd339a 100644--- a/crates/ov_cli/src/main.rs+++ b/crates/ov_cli/src/main.rs@@ -697,7 +697,17 @@ async fn handle_add_resource(let strict = !no_strict;let directly_upload_media = !no_directly_upload_media;- let client = ctx.get_client();+ let effective_timeout = if wait {+ timeout.unwrap_or(60.0).max(ctx.config.timeout)+ } else {+ ctx.config.timeout+ };+ let client = client::HttpClient::new(+ &ctx.config.url,+ ctx.config.api_key.clone(),+ ctx.config.agent_id.clone(),+ effective_timeout,+ );commands::resources::add_resource(&client,&path,diff --git a/openviking/core/context.py b/openviking/core/context.pyindex 94d47b1fd..763085703 100644--- a/openviking/core/context.py+++ b/openviking/core/context.py@@ -56,6 +56,7 @@ class Context:self,uri: str,parent_uri: Optional[str] = None,+ temp_uri: Optional[str] = None,is_leaf: bool = False,abstract: str = "",context_type: Optional[str] = None,@@ -78,6 +79,7 @@ class Context:self.id = id or str(uuid4())self.uri = uriself.parent_uri = parent_uri+ self.temp_uri = temp_uriself.is_leaf = is_leafself.abstract = abstractself.context_type = context_type or self._derive_context_type()@@ -159,6 +161,7 @@ class Context:"id": self.id,"uri": self.uri,"parent_uri": self.parent_uri,+ "temp_uri": self.temp_uri,"is_leaf": self.is_leaf,"abstract": self.abstract,"context_type": self.context_type,@@ -194,6 +197,7 @@ class Context:obj = cls(uri=data["uri"],parent_uri=data.get("parent_uri"),+ temp_uri=data.get("temp_uri"),is_leaf=data.get("is_leaf", False),abstract=data.get("abstract", ""),context_type=data.get("context_type"),diff --git a/openviking/parse/parsers/constants.py b/openviking/parse/parsers/constants.pyindex 311e545a2..174df57b9 100644--- a/openviking/parse/parsers/constants.py+++ b/openviking/parse/parsers/constants.py@@ -191,6 +191,7 @@ DOCUMENTATION_EXTENSIONS = {".texi",".texinfo",".wiki",+ ".conf",}# File type constants for consistent return valuesdiff --git a/openviking/parse/parsers/upload_utils.py b/openviking/parse/parsers/upload_utils.pyindex 50c80ed02..d18701735 100644--- a/openviking/parse/parsers/upload_utils.py+++ b/openviking/parse/parsers/upload_utils.py@@ -40,6 +40,7 @@ _EXTENSIONLESS_TEXT_NAMES: Set[str] = {"NEWS","NOTICE","TODO",+ "BUILD",}diff --git a/openviking/parse/tree_builder.py b/openviking/parse/tree_builder.pyindex 9a2543ce4..4260065e4 100644--- a/openviking/parse/tree_builder.py+++ b/openviking/parse/tree_builder.py@@ -20,7 +20,6 @@ IMPORTANT (v5.0 Architecture):- Content splitting is handled by Parser, not TreeBuilder"""-import asyncioimport loggingfrom typing import TYPE_CHECKING, Optional@@ -28,9 +27,7 @@ from openviking.core.building_tree import BuildingTreefrom openviking.core.context import Contextfrom openviking.parse.parsers.media.utils import get_media_base_uri, get_media_typefrom openviking.server.identity import RequestContext-from openviking.storage.queuefs import SemanticMsg, get_queue_managerfrom openviking.storage.viking_fs import get_viking_fs-from openviking.telemetry import get_current_telemetryfrom openviking.utils import parse_code_hosting_urlfrom openviking_cli.utils.uri import VikingURI@@ -94,7 +91,6 @@ class TreeBuilder:parent_uri: Optional[str] = None,source_path: Optional[str] = None,source_format: Optional[str] = None,- trigger_semantic: bool = False,) -> "BuildingTree":"""Finalize processing by moving from temp to AGFS.@@ -102,8 +98,6 @@ class TreeBuilder:Args:to_uri: Exact target URI (must not exist)parent_uri: Target parent URI (must exist)- trigger_semantic: Whether to automatically trigger semantic generation.- Default is False (handled by ResourceProcessor/Summarizer)."""viking_fs = get_viking_fs()@@ -139,16 +133,6 @@ class TreeBuilder:base_uri = parent_uri or auto_base_uri# 3. Determine candidate_uriif to_uri:- # Exact target URI: must not exist yet- try:- await viking_fs.stat(to_uri, ctx=ctx)- # If we get here, it already exists- raise FileExistsError(f"Target URI already exists: {to_uri}")- except FileExistsError:- raise- except Exception:- # It doesn't exist, good to use- passcandidate_uri = to_urielse:if parent_uri:@@ -161,35 +145,8 @@ class TreeBuilder:raise ValueError(f"Parent URI is not a directory: {parent_uri}")candidate_uri = VikingURI(base_uri).join(final_doc_name).uri- final_uri = await self._resolve_unique_uri(candidate_uri, ctx=ctx)+ final_uri = candidate_uri- if final_uri != candidate_uri:- logger.info(f"[TreeBuilder] Resolved name conflict: {candidate_uri} -> {final_uri}")- else:- logger.info(f"[TreeBuilder] Finalizing from temp: {final_uri}")-- # 4. Move directory tree from temp to final location in AGFS- await self._move_temp_to_dest(viking_fs, temp_doc_uri, final_uri, ctx=ctx)- logger.info(f"[TreeBuilder] Moved temp tree: {temp_doc_uri} -> {final_uri}")-- # 5. Cleanup temporary root directory- try:- await viking_fs.delete_temp(temp_uri, ctx=ctx)- logger.info(f"[TreeBuilder] Cleaned up temp root: {temp_uri}")- except Exception as e:- logger.warning(f"[TreeBuilder] Failed to cleanup temp root: {e}")-- # 6. Enqueue to SemanticQueue for async semantic generation- if trigger_semantic:- try:- await self._enqueue_semantic_generation(final_uri, "resource", ctx=ctx)- logger.info(f"[TreeBuilder] Enqueued semantic generation for: {final_uri}")- except Exception as e:- logger.error(- f"[TreeBuilder] Failed to enqueue semantic generation: {e}", exc_info=True- )-- # 7. Return simple BuildingTree (no scanning needed)tree = BuildingTree(source_path=source_path,source_format=source_format,@@ -197,121 +154,7 @@ class TreeBuilder:tree._root_uri = final_uri# Create a minimal Context object for the root so that tree.root is not None- root_context = Context(uri=final_uri)+ root_context = Context(uri=final_uri, temp_uri=temp_doc_uri)tree.add_context(root_context)return tree-- async def _resolve_unique_uri(- self, uri: str, max_attempts: int = 100, ctx: Optional[RequestContext] = None- ) -> str:- """Return a URI that does not collide with an existing resource.-- If *uri* is free, return it unchanged. Otherwise append ``_1``,- ``_2``, … until a free name is found (like macOS Finder / Windows- Explorer).- """- viking_fs = get_viking_fs()-- async def _exists(u: str) -> bool:- try:- await viking_fs.stat(u, ctx=ctx)- return True- except Exception:- return False-- if not await _exists(uri):- return uri-- for i in range(1, max_attempts + 1):- candidate = f"{uri}_{i}"- if not await _exists(candidate):- return candidate-- raise FileExistsError(f"Cannot resolve unique name for {uri} after {max_attempts} attempts")-- async def _move_temp_to_dest(- self, viking_fs, src_uri: str, dst_uri: str, ctx: RequestContext- ) -> None:- """Move temp directory to final destination using a single native AGFS mv call.-- Temp files have no vector records yet, so no vector index update is needed.- """- src_path = viking_fs._uri_to_path(src_uri, ctx=ctx)- dst_path = viking_fs._uri_to_path(dst_uri, ctx=ctx)- await self._ensure_parent_dirs(dst_uri, ctx=ctx)- await asyncio.to_thread(viking_fs.agfs.mv, src_path, dst_path)-- async def _ensure_parent_dirs(self, uri: str, ctx: RequestContext) -> None:- """Recursively create parent directories."""- viking_fs = get_viking_fs()- parent = VikingURI(uri).parent- if not parent:- return- parent_uri = parent.uri- # Recursively ensure parent's parent exists- await self._ensure_parent_dirs(parent_uri, ctx=ctx)-- # Create parent directory (ignore if already exists)- try:- await viking_fs.mkdir(parent_uri, exist_ok=True, ctx=ctx)- logger.debug(f"Created parent directory: {parent_uri}")- except Exception as e:- # Directory may already exist, ignore error- if "exist" not in str(e).lower():- logger.debug(f"Parent dir {parent_uri} may already exist: {e}")-- async def _enqueue_semantic_generation(- self, uri: str, context_type: str, ctx: RequestContext- ) -> None:- """- Enqueue a directory for semantic generation.-- Args:- uri: Directory URI to enqueue- context_type: resource/memory/skill- """-- queue_manager = get_queue_manager()-- # Get semantic queue- semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)-- # Sort by depth (descending) for bottom-up processing- msg = SemanticMsg(- uri=uri,- context_type=context_type,- account_id=ctx.account_id,- user_id=ctx.user.user_id,- agent_id=ctx.user.agent_id,- role=ctx.role.value,- telemetry_id=get_current_telemetry().telemetry_id,- )- await semantic_queue.enqueue(msg)-- async def _load_content(self, uri: str, content_type: str) -> str:- """Helper to load content with proper type handling"""- import json-- if content_type == "abstract":- result = await get_viking_fs().abstract(uri)- elif content_type == "overview":- result = await get_viking_fs().overview(uri)- elif content_type == "detail":- result = await get_viking_fs().read_file(uri)- else:- return ""-- # Handle different return types- if isinstance(result, str):- return result- elif isinstance(result, bytes):- return result.decode("utf-8")- elif hasattr(result, "to_dict") and not isinstance(result, list):- # Handle FindResult by converting to dict (skip lists)- return str(result.to_dict())- elif isinstance(result, list):- # Handle list results- return json.dumps(result)- else:- return str(result)diff --git a/openviking/server/models.py b/openviking/server/models.pyindex a97b086bb..445da590b 100644--- a/openviking/server/models.py+++ b/openviking/server/models.py@@ -31,6 +31,7 @@ ERROR_CODE_TO_HTTP_STATUS = {"INVALID_URI": 400,"NOT_FOUND": 404,"ALREADY_EXISTS": 409,+ "CONFLICT": 409,"PERMISSION_DENIED": 403,"UNAUTHENTICATED": 401,"RESOURCE_EXHAUSTED": 429,diff --git a/openviking/session/compressor.py b/openviking/session/compressor.pyindex 692612cdb..243eba8b0 100644--- a/openviking/session/compressor.py+++ b/openviking/session/compressor.py@@ -66,16 +66,100 @@ class SessionCompressor:self.vikingdb = vikingdbself.extractor = MemoryExtractor()self.deduplicator = MemoryDeduplicator(vikingdb=vikingdb)+ self._pending_semantic_changes: Dict[str, Dict[str, set]] = {}- async def _index_memory(self, memory: Context, ctx: RequestContext) -> bool:- """Add memory to vectorization queue and trigger parent directory semantic generation."""+ def _record_semantic_change(+ self, file_uri: str, change_type: str, parent_uri: Optional[str] = None+ ) -> None:+ """Record a file change for batch semantic processing.++ Args:+ file_uri: The URI of the file that changed+ change_type: One of "added", "modified", "deleted"+ parent_uri: Optional parent directory URI. If not provided, will be derived from file_uri+ """+ if change_type not in ("added", "modified", "deleted"):+ logger.warning(f"Invalid change_type: {change_type}, skipping")+ return++ if not parent_uri:+ parent_uri = "/".join(file_uri.rsplit("/", 1)[:-1])++ if not parent_uri:+ logger.warning(f"Could not determine parent URI for {file_uri}, skipping")+ return++ if parent_uri not in self._pending_semantic_changes:+ self._pending_semantic_changes[parent_uri] = {+ "added": set(),+ "modified": set(),+ "deleted": set(),+ }++ self._pending_semantic_changes[parent_uri][change_type].add(file_uri)+ logger.debug(f"Recorded semantic change: {change_type} {file_uri} in {parent_uri}")++ async def _flush_semantic_operations(self, ctx: RequestContext) -> None:+ """Flush all pending semantic operations.++ This method should be called after all memory changes are complete.+ It will deduplicate parent URIs and enqueue semantic operations with change info.+ """+ if not self._pending_semantic_changes:+ return++ try:+ from openviking.storage.queuefs import get_queue_manager+ from openviking.storage.queuefs.semantic_msg import SemanticMsg++ queue_manager = get_queue_manager()+ semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)++ for parent_uri, changes in self._pending_semantic_changes.items():+ changes_dict = {+ "added": list(changes["added"]),+ "modified": list(changes["modified"]),+ "deleted": list(changes["deleted"]),+ }++ msg = SemanticMsg(+ uri=parent_uri,+ context_type="memory",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ agent_id=ctx.user.agent_id,+ role=ctx.role.value,+ changes=changes_dict,+ )+ await semantic_queue.enqueue(msg)+ logger.info(+ f"Enqueued semantic generation for {parent_uri} with changes: "+ f"added={len(changes['added'])}, modified={len(changes['modified'])}, "+ f"deleted={len(changes['deleted'])}"+ )++ except Exception as e:+ logger.error(f"Failed to flush semantic operations: {e}", exc_info=True)+ finally:+ self._pending_semantic_changes.clear()++ async def _index_memory(+ self, memory: Context, ctx: RequestContext, change_type: str = "added"+ ) -> bool:+ """Add memory to vectorization queue and record semantic change.++ Args:+ memory: The memory context to index+ ctx: Request context+ change_type: One of "added" or "modified"+ """from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverterembedding_msg = EmbeddingMsgConverter.from_context(memory)await self.vikingdb.enqueue_embedding_msg(embedding_msg)logger.info(f"Enqueued memory for vectorization: {memory.uri}")- await self.extractor._enqueue_semantic_for_parent(memory.uri, ctx)+ self._record_semantic_change(memory.uri, change_type, parent_uri=memory.parent_uri)return Trueasync def _merge_into_existing(@@ -108,7 +192,7 @@ class SessionCompressor:"Merged memory %s with abstract %s", target_memory.uri, target_memory.abstract)target_memory.set_vectorize(Vectorize(text=payload.content))- await self._index_memory(target_memory, ctx)+ await self._index_memory(target_memory, ctx, change_type="modified")return Trueexcept Exception as e:logger.error(f"Failed to merge memory {target_memory.uri}: {e}")@@ -129,6 +213,8 @@ class SessionCompressor:await self.vikingdb.delete_uris(ctx, [memory.uri])except Exception as e:logger.warning(f"Failed to remove vector record for {memory.uri}: {e}")++ self._record_semantic_change(memory.uri, "deleted", parent_uri=memory.parent_uri)return Trueasync def extract_long_term_memories(@@ -147,162 +233,177 @@ class SessionCompressor:if not ctx:return []- if strict_extract_errors:- # Intentionally let extraction errors bubble up so caller (task tracker)- # can mark background commit tasks as failed with an explicit error.- candidates = await self.extractor.extract_strict(context, user, session_id)- else:- candidates = await self.extractor.extract(context, user, session_id)-- if not candidates:- return []-- memories: List[Context] = []- stats = ExtractionStats()- viking_fs = get_viking_fs()-- tool_parts = self._extract_tool_parts(messages)- from .tool_skill_utils import collect_skill_stats, collect_tool_stats-- tool_stats_map = collect_tool_stats(tool_parts)- skill_stats_map = collect_skill_stats(tool_parts)-- for candidate in candidates:- # Profile: skip dedup, always merge- if candidate.category in ALWAYS_MERGE_CATEGORIES:- memory = await self.extractor.create_memory(candidate, user, session_id, ctx=ctx)- if memory:- memories.append(memory)- stats.created += 1- await self._index_memory(memory, ctx)- else:- stats.skipped += 1- continue+ self._pending_semantic_changes.clear()- # Tool/Skill Memory: 特殊合并逻辑- if candidate.category in TOOL_SKILL_CATEGORIES:- if isinstance(candidate, ToolSkillCandidateMemory):- tool_name, skill_name, tool_status = self._get_tool_skill_info(- candidate, tool_parts+ try:+ if strict_extract_errors:+ # Intentionally let extraction errors bubble up so caller (task tracker)+ # can mark background commit tasks as failed with an explicit error.+ candidates = await self.extractor.extract_strict(context, user, session_id)+ else:+ candidates = await self.extractor.extract(context, user, session_id)++ if not candidates:+ return []++ memories: List[Context] = []+ stats = ExtractionStats()+ viking_fs = get_viking_fs()++ tool_parts = self._extract_tool_parts(messages)+ from .tool_skill_utils import collect_skill_stats, collect_tool_stats++ tool_stats_map = collect_tool_stats(tool_parts)+ skill_stats_map = collect_skill_stats(tool_parts)++ for candidate in candidates:+ # Profile: skip dedup, always merge+ if candidate.category in ALWAYS_MERGE_CATEGORIES:+ memory = await self.extractor.create_memory(+ candidate, user, session_id, ctx=ctx)- candidate.tool_status = tool_status- if tool_name:- candidate.tool_name = tool_name- if skill_name:- candidate.skill_name = skill_name-- if tool_name and candidate.call_time == 0:- tool_stats = tool_stats_map.get(tool_name, {})- candidate.call_time = tool_stats.get("call_count", candidate.call_time)- candidate.success_time = tool_stats.get(- "success_time", candidate.success_time- )- candidate.duration_ms = tool_stats.get("duration_ms", candidate.duration_ms)- candidate.prompt_tokens = tool_stats.get(- "prompt_tokens", candidate.prompt_tokens- )- candidate.completion_tokens = tool_stats.get(- "completion_tokens", candidate.completion_tokens- )-- if skill_name and candidate.call_time == 0:- skill_stats = skill_stats_map.get(skill_name, {})- candidate.call_time = skill_stats.get("call_count", candidate.call_time)- candidate.success_time = skill_stats.get(- "success_time", candidate.success_time- )- if skill_name:- memory = await self.extractor._merge_skill_memory(- skill_name, candidate, ctx=ctx- )- elif tool_name:- memory = await self.extractor._merge_tool_memory(- tool_name, candidate, ctx=ctx- )- else:- logger.warning("No tool_name or skill_name found, skipping")- stats.skipped += 1- continueif memory:memories.append(memory)- stats.merged += 1+ stats.created += 1await self._index_memory(memory, ctx)- continue-- # Dedup check for other categories- result = await self.deduplicator.deduplicate(candidate, ctx)- actions = result.actions or []- decision = result.decision-- # Safety net: create+merge should be treated as none.- if decision == DedupDecision.CREATE and any(- a.decision == MemoryActionDecision.MERGE for a in actions- ):- logger.warning(- f"Dedup returned create with merge action, normalizing to none: "- f"{candidate.abstract}"- )- decision = DedupDecision.NONE+ else:+ stats.skipped += 1+ continue++ # Tool/Skill Memory: 特殊合并逻辑+ if candidate.category in TOOL_SKILL_CATEGORIES:+ if isinstance(candidate, ToolSkillCandidateMemory):+ tool_name, skill_name, tool_status = self._get_tool_skill_info(+ candidate, tool_parts+ )+ candidate.tool_status = tool_status+ if tool_name:+ candidate.tool_name = tool_name+ if skill_name:+ candidate.skill_name = skill_name++ if tool_name and candidate.call_time == 0:+ tool_stats = tool_stats_map.get(tool_name, {})+ candidate.call_time = tool_stats.get("call_count", candidate.call_time)+ candidate.success_time = tool_stats.get(+ "success_time", candidate.success_time+ )+ candidate.duration_ms = tool_stats.get(+ "duration_ms", candidate.duration_ms+ )+ candidate.prompt_tokens = tool_stats.get(+ "prompt_tokens", candidate.prompt_tokens+ )+ candidate.completion_tokens = tool_stats.get(+ "completion_tokens", candidate.completion_tokens+ )++ if skill_name and candidate.call_time == 0:+ skill_stats = skill_stats_map.get(skill_name, {})+ candidate.call_time = skill_stats.get("call_count", candidate.call_time)+ candidate.success_time = skill_stats.get(+ "success_time", candidate.success_time+ )+ if skill_name:+ memory = await self.extractor._merge_skill_memory(+ skill_name, candidate, ctx=ctx+ )+ elif tool_name:+ memory = await self.extractor._merge_tool_memory(+ tool_name, candidate, ctx=ctx+ )+ else:+ logger.warning("No tool_name or skill_name found, skipping")+ stats.skipped += 1+ continue+ if memory:+ memories.append(memory)+ stats.merged += 1+ await self._index_memory(memory, ctx, change_type="modified")+ continue- if decision == DedupDecision.SKIP:- stats.skipped += 1- continue+ # Dedup check for other categories+ result = await self.deduplicator.deduplicate(candidate, ctx)+ actions = result.actions or []+ decision = result.decision++ # Safety net: create+merge should be treated as none.+ if decision == DedupDecision.CREATE and any(+ a.decision == MemoryActionDecision.MERGE for a in actions+ ):+ logger.warning(+ f"Dedup returned create with merge action, normalizing to none: "+ f"{candidate.abstract}"+ )+ decision = DedupDecision.NONE- if decision == DedupDecision.NONE:- if not actions:+ if decision == DedupDecision.SKIP:stats.skipped += 1continue- for action in actions:- if action.decision == MemoryActionDecision.DELETE:- if viking_fs and await self._delete_existing_memory(- action.memory, viking_fs, ctx=ctx- ):- stats.deleted += 1- else:- stats.skipped += 1- elif action.decision == MemoryActionDecision.MERGE:- if candidate.category in MERGE_SUPPORTED_CATEGORIES and viking_fs:- if await self._merge_into_existing(- candidate, action.memory, viking_fs, ctx=ctx+ if decision == DedupDecision.NONE:+ if not actions:+ stats.skipped += 1+ continue++ for action in actions:+ if action.decision == MemoryActionDecision.DELETE:+ if viking_fs and await self._delete_existing_memory(+ action.memory, viking_fs, ctx=ctx):- stats.merged += 1+ stats.deleted += 1else:stats.skipped += 1- else:- # events/cases don't support MERGE, treat as SKIP- stats.skipped += 1- continue-- if decision == DedupDecision.CREATE:- # create can optionally include delete actions (delete first, then create)- for action in actions:- if action.decision == MemoryActionDecision.DELETE:- if viking_fs and await self._delete_existing_memory(- action.memory, viking_fs, ctx=ctx- ):- stats.deleted += 1- else:- stats.skipped += 1+ elif action.decision == MemoryActionDecision.MERGE:+ if candidate.category in MERGE_SUPPORTED_CATEGORIES and viking_fs:+ if await self._merge_into_existing(+ candidate, action.memory, viking_fs, ctx=ctx+ ):+ stats.merged += 1+ else:+ stats.skipped += 1+ else:+ # events/cases don't support MERGE, treat as SKIP+ stats.skipped += 1+ continue- memory = await self.extractor.create_memory(candidate, user, session_id, ctx=ctx)- if memory:- memories.append(memory)- stats.created += 1- await self._index_memory(memory, ctx)- else:- stats.skipped += 1+ if decision == DedupDecision.CREATE:+ # create can optionally include delete actions (delete first, then create)+ for action in actions:+ if action.decision == MemoryActionDecision.DELETE:+ if viking_fs and await self._delete_existing_memory(+ action.memory, viking_fs, ctx=ctx+ ):+ stats.deleted += 1+ else:+ stats.skipped += 1- # Extract URIs used in messages, create relations- used_uris = self._extract_used_uris(messages)- if used_uris and memories:- await self._create_relations(memories, used_uris, ctx=ctx)+ memory = await self.extractor.create_memory(+ candidate, user, session_id, ctx=ctx+ )+ if memory:+ memories.append(memory)+ stats.created += 1+ await self._index_memory(memory, ctx)+ else:+ stats.skipped += 1++ # Extract URIs used in messages, create relations+ used_uris = self._extract_used_uris(messages)+ if used_uris and memories:+ await self._create_relations(memories, used_uris, ctx=ctx)++ await self._flush_semantic_operations(ctx)- logger.info(- f"Memory extraction: created={stats.created}, "- f"merged={stats.merged}, deleted={stats.deleted}, skipped={stats.skipped}"- )- return memories+ logger.info(+ f"Memory extraction: created={stats.created}, "+ f"merged={stats.merged}, deleted={stats.deleted}, skipped={stats.skipped}"+ )+ return memories++ except Exception:+ self._pending_semantic_changes.clear()+ raisedef _extract_tool_parts(self, messages: List[Message]) -> List:"""Extract all ToolPart from messages."""diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.pyindex dda6196bb..99d4a1215 100644--- a/openviking/session/memory_extractor.py+++ b/openviking/session/memory_extractor.py@@ -727,28 +727,6 @@ class MemoryExtractor:await viking_fs.write_file(uri=uri, content=merged_content, ctx=ctx)return self._create_tool_context(uri, candidate, ctx, abstract_override=abstract_override)- async def _enqueue_semantic_for_parent(self, file_uri: str, ctx: "RequestContext") -> None:- """Enqueue semantic generation for parent directory."""- try:- from openviking.storage.queuefs import get_queue_manager- from openviking.storage.queuefs.semantic_msg import SemanticMsg-- parent_uri = "/".join(file_uri.rsplit("/", 1)[:-1])- queue_manager = get_queue_manager()- semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)- msg = SemanticMsg(- uri=parent_uri,- context_type="memory",- account_id=ctx.account_id,- user_id=ctx.user.user_id,- agent_id=ctx.user.agent_id,- role=ctx.role.value,- )- await semantic_queue.enqueue(msg)- logger.debug(f"Enqueued semantic generation for: {parent_uri}")- except Exception as e:- logger.warning(f"Failed to enqueue semantic generation for {file_uri}: {e}")-def _compute_statistics_derived(self, stats: dict) -> dict:"""计算派生统计数据(平均值、成功率)"""if stats["total_calls"] > 0:diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex ea33b5d9b..c46f2d769 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -353,3 +353,12 @@ class TextEmbeddingHandler(DequeueHandlerBase):self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)self.report_error(str(e), data)return None+ finally:+ if embedding_msg and embedding_msg.semantic_msg_id:+ from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker++ tracker = EmbeddingTaskTracker.get_instance()+ try:+ await tracker.decrement(embedding_msg.semantic_msg_id)+ except Exception as tracker_err:+ logger.warning(f"Failed to decrement embedding tracker: {tracker_err}")diff --git a/openviking/storage/queuefs/__init__.py b/openviking/storage/queuefs/__init__.pyindex b73a01d7f..87514a830 100644--- a/openviking/storage/queuefs/__init__.py+++ b/openviking/storage/queuefs/__init__.py@@ -2,6 +2,7 @@# SPDX-License-Identifier: Apache-2.0from .embedding_msg import EmbeddingMsgfrom .embedding_queue import EmbeddingQueue+from .embedding_tracker import EmbeddingTaskTrackerfrom .named_queue import NamedQueue, QueueError, QueueStatusfrom .queue_manager import QueueManager, get_queue_manager, init_queue_managerfrom .semantic_dag import SemanticDagExecutor@@ -18,6 +19,7 @@ __all__ = ["QueueError","EmbeddingQueue","EmbeddingMsg",+ "EmbeddingTaskTracker","SemanticQueue","SemanticDagExecutor","SemanticMsg",diff --git a/openviking/storage/queuefs/embedding_msg.py b/openviking/storage/queuefs/embedding_msg.pyindex c100d569f..af0e95eea 100644--- a/openviking/storage/queuefs/embedding_msg.py+++ b/openviking/storage/queuefs/embedding_msg.py@@ -2,7 +2,7 @@# SPDX-License-Identifier: Apache-2.0import jsonfrom dataclasses import asdict, dataclass-from typing import Any, Dict, List, Union+from typing import Any, Dict, List, Optional, Unionfrom uuid import uuid4@@ -11,17 +11,20 @@ class EmbeddingMsg:message: Union[str, List[Dict[str, Any]]]context_data: Dict[str, Any]telemetry_id: str = ""+ semantic_msg_id: Optional[str] = Nonedef __init__(self,message: Union[str, List[Dict[str, Any]]],context_data: Dict[str, Any],telemetry_id: str = "",+ semantic_msg_id: Optional[str] = None,):self.id = str(uuid4())self.message = messageself.context_data = context_dataself.telemetry_id = telemetry_id+ self.semantic_msg_id = semantic_msg_iddef to_dict(self) -> Dict[str, Any]:"""Convert embedding message to dictionary format."""@@ -38,6 +41,7 @@ class EmbeddingMsg:message=data["message"],context_data=data["context_data"],telemetry_id=data.get("telemetry_id", ""),+ semantic_msg_id=data.get("semantic_msg_id"),)obj.id = data.get("id", obj.id)return objdiff --git a/openviking/storage/queuefs/embedding_queue.py b/openviking/storage/queuefs/embedding_queue.pyindex d44595e8f..5fb1f4af4 100644--- a/openviking/storage/queuefs/embedding_queue.py+++ b/openviking/storage/queuefs/embedding_queue.py@@ -21,6 +21,7 @@ class EmbeddingQueue(NamedQueue):if msg is None:logger.warning("Embedding message is None, skipping enqueuing")return ""+ logger.debug(f"Enqueued embedding message: {msg}")return await super().enqueue(msg.to_dict())async def dequeue(self) -> Optional[EmbeddingMsg]:diff --git a/openviking/storage/queuefs/embedding_tracker.py b/openviking/storage/queuefs/embedding_tracker.pynew file mode 100644index 000000000..bfee27b79--- /dev/null+++ b/openviking/storage/queuefs/embedding_tracker.py@@ -0,0 +1,179 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Embedding Task Tracker for tracking embedding task completion status."""++import asyncio+from typing import Any, Callable, Dict, Optional++from openviking_cli.utils.logger import get_logger++logger = get_logger(__name__)+++class EmbeddingTaskTracker:+ """Track embedding task completion status for each SemanticMsg.++ This tracker maintains a global registry of embedding tasks associated+ with each SemanticMsg. When all embedding tasks for a SemanticMsg are+ completed, it triggers the registered callback and removes the entry.+ """++ _instance: Optional["EmbeddingTaskTracker"] = None+ _initialized: bool = False++ def __new__(cls) -> "EmbeddingTaskTracker":+ if cls._instance is None:+ cls._instance = super().__new__(cls)+ return cls._instance++ def __init__(self):+ if self._initialized:+ return+ self._lock: asyncio.Lock = asyncio.Lock()+ self._tasks: Dict[str, Dict[str, Any]] = {}+ self._initialized = True++ @classmethod+ def get_instance(cls) -> "EmbeddingTaskTracker":+ """Get the singleton instance of EmbeddingTaskTracker."""+ if cls._instance is None:+ cls._instance = cls()+ return cls._instance++ async def register(+ self,+ semantic_msg_id: str,+ total_count: int,+ on_complete: Optional[Callable[[], Any]] = None,+ metadata: Optional[Dict[str, Any]] = None,+ ) -> None:+ """Register a SemanticMsg with its total embedding task count.++ Args:+ semantic_msg_id: The ID of the SemanticMsg+ total_count: Total number of embedding tasks for this SemanticMsg+ on_complete: Optional callback when all tasks complete+ metadata: Optional metadata to store with the task+ """+ async with self._lock:+ self._tasks[semantic_msg_id] = {+ "remaining": total_count,+ "total": total_count,+ "on_complete": on_complete,+ "metadata": metadata or {},+ }+ logger.info(+ f"Registered embedding tracker for SemanticMsg {semantic_msg_id}: "+ f"{total_count} tasks"+ )++ if total_count <= 0 and on_complete:+ del self._tasks[semantic_msg_id]+ logger.info(+ f"No embedding tasks for SemanticMsg {semantic_msg_id}, "+ f"triggering on_complete immediately"+ )++ if total_count <= 0 and on_complete:+ try:+ result = on_complete()+ if asyncio.iscoroutine(result):+ await result+ except Exception as e:+ logger.error(+ f"Error in completion callback for {semantic_msg_id}: {e}",+ exc_info=True,+ )++ async def decrement(self, semantic_msg_id: str) -> Optional[int]:+ """Decrement the remaining task count for a SemanticMsg.++ This method should be called when an embedding task is completed.+ When the count reaches zero, the registered callback is executed+ and the entry is removed from the tracker.++ Args:+ semantic_msg_id: The ID of the SemanticMsg++ Returns:+ The remaining count after decrement, or None if not found+ """+ on_complete = None++ async with self._lock:+ if semantic_msg_id not in self._tasks:+ return None++ task_info = self._tasks[semantic_msg_id]+ task_info["remaining"] -= 1+ remaining = task_info["remaining"]++ if remaining <= 0:+ on_complete = task_info.get("on_complete")++ del self._tasks[semantic_msg_id]+ logger.info(+ f"All embedding tasks({task_info['total']}) completed for SemanticMsg {semantic_msg_id}"+ )++ if on_complete:+ try:+ result = on_complete()+ if asyncio.iscoroutine(result):+ await result+ except Exception as e:+ logger.error(+ f"Error in completion callback for {semantic_msg_id}: {e}",+ exc_info=True,+ )+ return remaining++ async def get_status(self, semantic_msg_id: str) -> Optional[Dict[str, Any]]:+ """Get the current status of a SemanticMsg's embedding tasks.++ Args:+ semantic_msg_id: The ID of the SemanticMsg++ Returns:+ Dict with 'remaining', 'total', 'metadata' or None if not found+ """+ async with self._lock:+ if semantic_msg_id not in self._tasks:+ return None+ task_info = self._tasks[semantic_msg_id]+ return {+ "remaining": task_info["remaining"],+ "total": task_info["total"],+ "metadata": task_info.get("metadata", {}),+ }++ async def remove(self, semantic_msg_id: str) -> bool:+ """Remove a SemanticMsg from the tracker.++ Args:+ semantic_msg_id: The ID of the SemanticMsg++ Returns:+ True if removed, False if not found+ """+ async with self._lock:+ if semantic_msg_id in self._tasks:+ del self._tasks[semantic_msg_id]+ return True+ return False++ async def get_all_tracked(self) -> Dict[str, Dict[str, Any]]:+ """Get all currently tracked SemanticMsgs.++ Returns:+ Dict of semantic_msg_id -> task info+ """+ async with self._lock:+ return {+ msg_id: {+ "remaining": info["remaining"],+ "total": info["total"],+ "metadata": info.get("metadata", {}),+ }+ for msg_id, info in self._tasks.items()+ }diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex 212cffd99..9ed037b6d 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -4,7 +4,7 @@import asynciofrom dataclasses import dataclass, field-from typing import Dict, List, Optional+from typing import Awaitable, Callable, Dict, List, Optionalfrom openviking.server.identity import RequestContextfrom openviking.storage.viking_fs import get_viking_fs@@ -44,6 +44,24 @@ class DagStats:done_nodes: int = 0+@dataclass+class VectorizeTask:+ """Vectorize task information."""++ task_type: str # "file" or "directory"+ uri: str+ context_type: str+ ctx: "RequestContext"+ semantic_msg_id: Optional[str] = None+ # For file tasks+ file_path: Optional[str] = None+ summary_dict: Optional[Dict[str, str]] = None+ parent_uri: Optional[str] = None+ # For directory tasks+ abstract: Optional[str] = None+ overview: Optional[str] = None++class SemanticDagExecutor:"""Execute semantic generation with DAG-style, event-driven lazy dispatch."""@@ -53,11 +71,19 @@ class SemanticDagExecutor:context_type: str,max_concurrent_llm: int,ctx: RequestContext,+ incremental_update: bool = False,+ target_uri: Optional[str] = None,+ semantic_msg_id: Optional[str] = None,+ recursive: bool = True,):self._processor = processorself._context_type = context_typeself._max_concurrent_llm = max_concurrent_llmself._ctx = ctx+ self._incremental_update = incremental_update+ self._target_uri = target_uri+ self._semantic_msg_id = semantic_msg_id+ self._recursive = recursiveself._llm_sem = asyncio.Semaphore(max_concurrent_llm)self._viking_fs = get_viking_fs()self._nodes: Dict[str, DirNode] = {}@@ -65,6 +91,70 @@ class SemanticDagExecutor:self._root_uri: Optional[str] = Noneself._root_done: Optional[asyncio.Event] = Noneself._stats = DagStats()+ self._vectorize_task_count: int = 0+ self._pending_vectorize_tasks: List[VectorizeTask] = []+ self._vectorize_lock = asyncio.Lock()+ self._file_change_status: Dict[str, bool] = {}+ self._dir_change_status: Dict[str, bool] = {}+ self._overview_cache: Dict[str, Dict[str, str]] = {}+ self._overview_cache_lock = asyncio.Lock()++ def _create_on_complete_callback(self) -> Optional[Callable[[], Awaitable[None]]]:+ """Create on_complete callback for incremental update or full update."""+ if not self._target_uri or not self._root_uri:+ return None++ if self._incremental_update:++ async def sync_diff_callback() -> None:+ try:+ diff = await self._processor._sync_topdown_recursive(+ self._root_uri,+ self._target_uri,+ ctx=self._ctx,+ file_change_status=self._file_change_status,+ )+ logger.info(+ f"[SyncDiff] Diff computed: "+ f"added_files={len(diff.added_files)}, "+ f"deleted_files={len(diff.deleted_files)}, "+ f"updated_files={len(diff.updated_files)}, "+ f"added_dirs={len(diff.added_dirs)}, "+ f"deleted_dirs={len(diff.deleted_dirs)}"+ )+ except Exception as e:+ logger.error(+ f"[SyncDiff] Error in sync_diff_callback: "+ f"root_uri={self._root_uri}, target_uri={self._target_uri} "+ f"error={e}",+ exc_info=True,+ )++ return sync_diff_callback+ else:++ async def move_temp_to_target_callback() -> None:+ try:+ target_exists = await self._viking_fs.exists(self._target_uri, ctx=self._ctx)+ if target_exists:+ await self._viking_fs.rm(self._target_uri, recursive=True, ctx=self._ctx)+ parent_uri = "/".join(self._target_uri.rsplit("/", 1)[:-1])+ if parent_uri:+ await self._viking_fs.mkdir(parent_uri, exist_ok=True, ctx=self._ctx)+ await self._viking_fs.mv(self._root_uri, self._target_uri, ctx=self._ctx)+ logger.info(+ f"[MoveTemp] Moved temp uri to target uri: "+ f"root_uri={self._root_uri}, target_uri={self._target_uri}"+ )+ except Exception as e:+ logger.error(+ f"[MoveTemp] Error moving temp uri to target uri: "+ f"root_uri={self._root_uri}, target_uri={self._target_uri} "+ f"error={e}",+ exc_info=True,+ )++ return move_temp_to_target_callbackasync def run(self, root_uri: str) -> None:"""Run DAG execution starting from root_uri."""@@ -73,6 +163,52 @@ class SemanticDagExecutor:await self._dispatch_dir(root_uri, parent_uri=None)await self._root_done.wait()+ on_complete = self._create_on_complete_callback()++ async with self._vectorize_lock:+ task_count = self._vectorize_task_count+ tasks = list(self._pending_vectorize_tasks)++ if task_count > 0:+ from .embedding_tracker import EmbeddingTaskTracker++ tracker = EmbeddingTaskTracker.get_instance()+ await tracker.register(+ semantic_msg_id=self._semantic_msg_id,+ total_count=task_count,+ on_complete=on_complete,+ metadata={"uri": root_uri},+ )++ for task in tasks:+ if task.task_type == "file":+ asyncio.create_task(+ self._processor._vectorize_single_file(+ parent_uri=task.parent_uri,+ context_type=task.context_type,+ file_path=task.file_path,+ summary_dict=task.summary_dict,+ ctx=task.ctx,+ semantic_msg_id=task.semantic_msg_id,+ )+ )+ else:+ asyncio.create_task(+ self._processor._vectorize_directory(+ task.uri,+ task.context_type,+ task.abstract,+ task.overview,+ ctx=task.ctx,+ semantic_msg_id=task.semantic_msg_id,+ )+ )+ elif on_complete:+ try:+ await on_complete()+ except Exception as e:+ logger.error(f"Error in on_complete callback: {e}", exc_info=True)+async def _dispatch_dir(self, dir_uri: str, parent_uri: Optional[str]) -> None:"""Lazy-dispatch tasks for a directory when it is triggered."""if dir_uri in self._nodes:@@ -84,7 +220,10 @@ class SemanticDagExecutor:children_dirs, file_paths = await self._list_dir(dir_uri)file_index = {path: idx for idx, path in enumerate(file_paths)}child_index = {path: idx for idx, path in enumerate(children_dirs)}- pending = len(children_dirs) + len(file_paths)+ if self._recursive:+ pending = len(children_dirs) + len(file_paths)+ else:+ pending = len(file_paths)node = DirNode(uri=dir_uri,@@ -113,8 +252,10 @@ class SemanticDagExecutor:self._stats.in_progress_nodes += 1asyncio.create_task(self._file_summary_task(dir_uri, file_path))- for child_uri in children_dirs:- asyncio.create_task(self._dispatch_dir(child_uri, dir_uri))+ if children_dirs:+ if self._recursive:+ for child_uri in children_dirs:+ asyncio.create_task(self._dispatch_dir(child_uri, dir_uri))except Exception as e:logger.error(f"Failed to dispatch directory {dir_uri}: {e}", exc_info=True)if parent_uri:@@ -146,13 +287,133 @@ class SemanticDagExecutor:return children_dirs, file_paths+ def _get_target_file_path(self, current_uri: str) -> Optional[str]:+ if not self._incremental_update or not self._target_uri or not self._root_uri:+ logger.warning(+ f"Invalid target_uri or root_uri for incremental update: target_uri={self._target_uri}, root_uri={self._root_uri}"+ )+ return None+ try:+ relative_path = current_uri[len(self._root_uri) :]+ if relative_path.startswith("/"):+ relative_path = relative_path[1:]+ return f"{self._target_uri}/{relative_path}" if relative_path else self._target_uri+ except Exception:+ return None++ async def _check_file_content_changed(self, file_path: str) -> bool:+ target_path = self._get_target_file_path(file_path)+ if not target_path:+ return True+ try:+ current_content = await self._viking_fs.read_file(file_path, ctx=self._ctx)+ target_content = await self._viking_fs.read_file(target_path, ctx=self._ctx)+ return current_content != target_content+ except Exception:+ return True++ async def _read_existing_summary(self, file_path: str) -> Optional[Dict[str, str]]:+ """Read existing summary from parent directory's .overview.md.++ Args:+ file_path: Current file path++ Returns:+ Summary dict with 'name' and 'summary' keys, or None if not found+ """+ target_path = self._get_target_file_path(file_path)+ if not target_path:+ return None++ try:+ parent_uri = "/".join(target_path.rsplit("/", 1)[:-1])+ if not parent_uri:+ return None++ if parent_uri not in self._overview_cache:+ async with self._overview_cache_lock:+ if parent_uri not in self._overview_cache:+ overview_path = f"{parent_uri}/.overview.md"+ overview_content = await self._viking_fs.read_file(+ overview_path, ctx=self._ctx+ )+ if overview_content:+ self._overview_cache[parent_uri] = self._processor._parse_overview_md(+ overview_content+ )+ else:+ self._overview_cache[parent_uri] = {}++ existing_summaries = self._overview_cache.get(parent_uri, {})+ file_name = file_path.split("/")[-1]++ if file_name in existing_summaries:+ return {"name": file_name, "summary": existing_summaries[file_name]}++ except Exception as e:+ logger.debug(f"Failed to read existing summary from overview.md for {file_path}: {e}")++ return None++ async def _check_dir_children_changed(+ self, dir_uri: str, current_files: List[str], current_dirs: List[str]+ ) -> bool:+ target_path = self._get_target_file_path(dir_uri)+ if not target_path:+ return True+ try:+ target_dirs, target_files = await self._list_dir(target_path)+ current_file_names = {f.split("/")[-1] for f in current_files}+ target_file_names = {f.split("/")[-1] for f in target_files}+ if current_file_names != target_file_names:+ return True+ current_dir_names = {d.split("/")[-1] for d in current_dirs}+ target_dir_names = {d.split("/")[-1] for d in target_dirs}+ if current_dir_names != target_dir_names:+ return True+ for current_file in current_files:+ if self._file_change_status.get(current_file, True):+ return True+ for current_dir in current_dirs:+ if self._dir_change_status.get(current_dir, True):+ return True+ return False+ except Exception:+ return True++ async def _read_existing_overview_abstract(+ self, dir_uri: str+ ) -> tuple[Optional[str], Optional[str]]:+ target_path = self._get_target_file_path(dir_uri)+ if not target_path:+ return None, None+ try:+ overview = await self._viking_fs.read_file(f"{target_path}/.overview.md", ctx=self._ctx)+ abstract = await self._viking_fs.read_file(f"{target_path}/.abstract.md", ctx=self._ctx)+ return overview, abstract+ except Exception:+ return None, None+async def _file_summary_task(self, parent_uri: str, file_path: str) -> None:"""Generate file summary and notify parent completion."""+file_name = file_path.split("/")[-1]+ need_vectorize = Truetry:- summary_dict = await self._processor._generate_single_file_summary(- file_path, llm_sem=self._llm_sem, ctx=self._ctx- )+ summary_dict = None+ if self._incremental_update:+ content_changed = await self._check_file_content_changed(file_path)+ self._file_change_status[file_path] = content_changed++ if not content_changed:+ summary_dict = await self._read_existing_summary(file_path)+ need_vectorize = False+ else:+ self._file_change_status[file_path] = True+ if summary_dict is None:+ summary_dict = await self._processor._generate_single_file_summary(+ file_path, llm_sem=self._llm_sem, ctx=self._ctx+ )except Exception as e:logger.warning(f"Failed to generate summary for {file_path}: {e}")summary_dict = {"name": file_name, "summary": ""}@@ -160,21 +421,22 @@ class SemanticDagExecutor:self._stats.done_nodes += 1self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)- await self._on_file_done(parent_uri, file_path, summary_dict)-- # Vectorize file as soon as summary is ready to avoid waiting for overview.try:- asyncio.create_task(- self._processor._vectorize_single_file(- parent_uri=parent_uri,+ if need_vectorize:+ task = VectorizeTask(+ task_type="file",+ uri=file_path,context_type=self._context_type,+ ctx=self._ctx,+ semantic_msg_id=self._semantic_msg_id,file_path=file_path,summary_dict=summary_dict,- ctx=self._ctx,+ parent_uri=parent_uri,)- )+ await self._add_vectorize_task(task)except Exception as e:logger.error(f"Failed to schedule vectorization for {file_path}: {e}", exc_info=True)+ await self._on_file_done(parent_uri, file_path, summary_dict)async def _on_file_done(self, parent_uri: str, file_path: str, summary_dict: Dict[str, str]@@ -246,17 +508,28 @@ class SemanticDagExecutor:node = self._nodes.get(dir_uri)if not node:return-- async with node.lock:- file_summaries = self._finalize_file_summaries(node)- children_abstracts = self._finalize_children_abstracts(node)-+ need_vectorize = True+ children_changed = Truetry:- async with self._llm_sem:- overview = await self._processor._generate_overview(- dir_uri, file_summaries, children_abstracts+ overview = None+ abstract = None+ if self._incremental_update:+ children_changed = await self._check_dir_children_changed(+ dir_uri, node.file_paths, node.children_dirs)- abstract = self._processor._extract_abstract_from_overview(overview)++ if not children_changed:+ need_vectorize = False+ overview, abstract = await self._read_existing_overview_abstract(dir_uri)+ if overview is None or abstract is None:+ async with node.lock:+ file_summaries = self._finalize_file_summaries(node)+ children_abstracts = self._finalize_children_abstracts(node)+ async with self._llm_sem:+ overview = await self._processor._generate_overview(+ dir_uri, file_summaries, children_abstracts+ )+ abstract = self._processor._extract_abstract_from_overview(overview)try:await self._viking_fs.write_file(f"{dir_uri}/.overview.md", overview, ctx=self._ctx)@@ -265,11 +538,19 @@ class SemanticDagExecutor:logger.warning(f"Failed to write overview/abstract for {dir_uri}: {e}")try:- await self._processor._vectorize_directory_simple(- dir_uri, self._context_type, abstract, overview, ctx=self._ctx- )+ if need_vectorize:+ task = VectorizeTask(+ task_type="directory",+ uri=dir_uri,+ context_type=self._context_type,+ ctx=self._ctx,+ semantic_msg_id=self._semantic_msg_id,+ abstract=abstract,+ overview=overview,+ )+ await self._add_vectorize_task(task)except Exception as e:- logger.error(f"Failed to vectorize directory {dir_uri}: {e}", exc_info=True)+ logger.error(f"Failed to schedule vectorization for {dir_uri}: {e}", exc_info=True)except Exception as e:logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)@@ -278,6 +559,8 @@ class SemanticDagExecutor:self._stats.done_nodes += 1self._stats.in_progress_nodes = max(0, self._stats.in_progress_nodes - 1)+ self._dir_change_status[dir_uri] = children_changed+parent_uri = self._parent.get(dir_uri)if parent_uri is None:if self._root_done:@@ -286,6 +569,15 @@ class SemanticDagExecutor:await self._on_child_done(parent_uri, dir_uri, abstract)+ async def _add_vectorize_task(self, task: VectorizeTask) -> None:+ """Add a vectorize task to the pending list."""+ async with self._vectorize_lock:+ self._pending_vectorize_tasks.append(task)+ if task.task_type == "file":+ self._vectorize_task_count += 1+ else: # directory+ self._vectorize_task_count += 2+def get_stats(self) -> DagStats:return DagStats(total_nodes=self._stats.total_nodes,diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.pyindex 59c4ba94e..f6acdaf4e 100644--- a/openviking/storage/queuefs/semantic_msg.py+++ b/openviking/storage/queuefs/semantic_msg.py@@ -5,7 +5,7 @@import jsonfrom dataclasses import asdict, dataclassfrom datetime import datetime-from typing import Any, Dict+from typing import Any, Dict, List, Optionalfrom uuid import uuid4@@ -16,7 +16,7 @@ class SemanticMsg:Attributes:id: Unique identifier (UUID)uri: Directory URI to process- context_type: Type of context (resource, memory, skill)+ context_type: Type of context (resource, memory, skill, session)status: Processing status (pending/processing/completed)timestamp: Creation timestamprecursive: Whether to recursively process subdirectories.@@ -27,7 +27,7 @@ class SemanticMsg:id: str # UUIDuri: str # Directory URI- context_type: str # resource, memory, skill+ context_type: str # resource, memory, skill, sessionstatus: str = "pending" # pending/processing/completedtimestamp: int = int(datetime.now().timestamp())recursive: bool = True # Whether to recursively process subdirectories@@ -38,6 +38,10 @@ class SemanticMsg:# Additional flagsskip_vectorization: bool = Falsetelemetry_id: str = ""+ target_uri: str = ""+ changes: Optional[Dict[str, List[str]]] = (+ None # {"added": [...], "modified": [...], "deleted": [...]}+ )def __init__(self,@@ -50,6 +54,8 @@ class SemanticMsg:role: str = "root",skip_vectorization: bool = False,telemetry_id: str = "",+ target_uri: str = "",+ changes: Optional[Dict[str, List[str]]] = None,):self.id = str(uuid4())self.uri = uri@@ -61,6 +67,8 @@ class SemanticMsg:self.role = roleself.skip_vectorization = skip_vectorizationself.telemetry_id = telemetry_id+ self.target_uri = target_uri+ self.changes = changesdef to_dict(self) -> Dict[str, Any]:"""Convert object to dictionary."""@@ -97,6 +105,8 @@ class SemanticMsg:role=data.get("role", "root"),skip_vectorization=data.get("skip_vectorization", False),telemetry_id=data.get("telemetry_id", ""),+ target_uri=data.get("target_uri", ""),+ changes=data.get("changes"),)if "id" in data and data["id"]:obj.id = data["id"]diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 2e861bcc3..60dc06e64 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -5,8 +5,8 @@import asyncioimport threadingfrom contextlib import nullcontext-from dataclasses import dataclass-from typing import Any, Dict, List, Optional, Tuple+from dataclasses import dataclass, field+from typing import Any, Dict, List, Optional, Set, Tuplefrom openviking.parse.parsers.constants import (CODE_EXTENSIONS,@@ -37,6 +37,16 @@ logger = get_logger(__name__)@dataclass+class DiffResult:+ """Directory diff result for sync operations."""++ added_files: List[str] = field(default_factory=list)+ deleted_files: List[str] = field(default_factory=list)+ updated_files: List[str] = field(default_factory=list)+ added_dirs: List[str] = field(default_factory=list)+ deleted_dirs: List[str] = field(default_factory=list)++class RequestQueueStats:processed: int = 0error_count: int = 0@@ -176,52 +186,17 @@ class SemanticProcessor(DequeueHandlerBase):# Default to otherreturn FILE_TYPE_OTHER- async def _enqueue_semantic_msg(self, msg: SemanticMsg) -> None:- """Enqueue a SemanticMsg to the semantic queue for processing."""- from openviking.storage.queuefs import get_queue_manager-- queue_manager = get_queue_manager()- semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC)- # The queue manager returns SemanticQueue but method signature says NamedQueue- # We need to ignore the type error for the enqueue call- await semantic_queue.enqueue(msg) # type: ignore- logger.debug(f"Enqueued semantic message for processing: {msg.uri}")-- async def _collect_directory_info(- self,- uri: str,- result: List[Tuple[str, List[str], List[str]]],- ) -> None:- """Recursively collect directory info, post-order traversal ensures bottom-up order."""+ async def _check_file_content_changed(+ self, file_path: str, target_file: str, ctx: Optional[RequestContext] = None+ ) -> bool:+ """Check if file content has changed compared to target file."""viking_fs = get_viking_fs()-try:- entries = await viking_fs.ls(uri, ctx=self._current_ctx)- except Exception as e:- logger.warning(f"Failed to list directory {uri}: {e}")- return-- children_uris = []- file_paths = []-- for entry in entries:- name = entry.get("name", "")- if not name or name.startswith(".") or name in [".", ".."]:- continue-- item_uri = VikingURI(uri).join(name).uri-- if entry.get("isDir", False):- # Child directory- children_uris.append(item_uri)- # Recursively collect children- await self._collect_directory_info(item_uri, result)- else:- # File (not starting with .)- file_paths.append(item_uri)-- # Add current directory info- result.append((uri, children_uris, file_paths))+ current_content = await viking_fs.read_file(file_path, ctx=ctx)+ target_content = await viking_fs.read_file(target_file, ctx=ctx)+ return current_content != target_content+ except Exception:+ return Trueasync def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:"""Process dequeued SemanticMsg, recursively process all subdirectories."""@@ -236,12 +211,10 @@ class SemanticProcessor(DequeueHandlerBase):if "data" in data and isinstance(data["data"], str):data = json.loads(data["data"])- # data is guaranteed to be not None at this pointassert data is not Nonemsg = SemanticMsg.from_dict(data)collector = resolve_telemetry(msg.telemetry_id)telemetry_ctx = bind_telemetry(collector) if collector is not None else nullcontext()-with telemetry_ctx:self._current_msg = msgself._current_ctx = self._ctx_from_semantic_msg(msg)@@ -249,12 +222,32 @@ class SemanticProcessor(DequeueHandlerBase):f"Processing semantic generation for: {msg.uri} (recursive={msg.recursive})")- if msg.recursive:+ logger.info(f"Processing semantic generation for: {msg})")++ if msg.context_type == "memory":+ await self._process_memory_directory(msg)+ else:+ is_incremental = False+ viking_fs = get_viking_fs()+ if msg.target_uri:+ target_exists = await viking_fs.exists(+ msg.target_uri, ctx=self._current_ctx+ )+ if target_exists:+ is_incremental = True+ logger.info(+ f"Target URI exists, using incremental update: {msg.target_uri}"+ )+executor = SemanticDagExecutor(processor=self,context_type=msg.context_type,max_concurrent_llm=self.max_concurrent_llm,ctx=self._current_ctx,+ incremental_update=is_incremental,+ target_uri=msg.target_uri,+ semantic_msg_id=msg.id,+ recursive=msg.recursive,)self._dag_executor = executorawait executor.run(msg.uri)@@ -263,44 +256,6 @@ class SemanticProcessor(DequeueHandlerBase):msg.uri,executor.get_stats(),)- else:- children_uris = []- file_paths = []-- viking_fs = get_viking_fs()- try:- entries = await viking_fs.ls(msg.uri, ctx=self._current_ctx)- for entry in entries:- name = entry.get("name", "")- if not name or name.startswith(".") or name in [".", ".."]:- continue-- item_uri = VikingURI(msg.uri).join(name).uri-- if entry.get("isDir", False):- children_uris.append(item_uri)- else:- file_paths.append(item_uri)- except Exception as e:- logger.warning(f"Failed to list directory {msg.uri}: {e}")-- await self._process_single_directory(- uri=msg.uri,- context_type=msg.context_type,- children_uris=children_uris,- file_paths=file_paths,- )- self._cache_dag_stats(- msg.telemetry_id,- msg.uri,- DagStats(- total_nodes=1,- pending_nodes=0,- in_progress_nodes=0,- done_nodes=1,- ),- )-self._merge_request_stats(msg.telemetry_id, processed=1)logger.info(f"Completed semantic generation for: {msg.uri}")self.report_success()@@ -314,90 +269,293 @@ class SemanticProcessor(DequeueHandlerBase):return Nonefinally:self._current_msg = None+ self._current_ctx = Nonedef get_dag_stats(self) -> Optional["DagStats"]:if not self._dag_executor:return Nonereturn self._dag_executor.get_stats()- async def _process_single_directory(- self,- uri: str,- context_type: str,- children_uris: List[str],- file_paths: List[str],- ) -> None:- """Process single directory, generate .abstract.md and .overview.md."""+ async def _process_memory_directory(self, msg: SemanticMsg) -> None:+ """Process a memory directory with special handling.++ For memory directories:+ - Memory files are already vectorized via embedding queue+ - Only generate abstract.md and overview.md+ - Vectorize the generated abstract.md and overview.md++ Args:+ msg: The semantic message containing directory info and changes+ """viking_fs = get_viking_fs()+ dir_uri = msg.uri+ ctx = self._current_ctx+ llm_sem = asyncio.Semaphore(self.max_concurrent_llm)- # 1. Collect .abstract.md from subdirectories (already processed earlier)- children_abstracts = await self._collect_children_abstracts(children_uris)+ try:+ entries = await viking_fs.ls(dir_uri, ctx=ctx)+ except Exception as e:+ logger.warning(f"Failed to list memory directory {dir_uri}: {e}")+ return- # 2. Concurrently generate summaries for files in directory- file_summaries = await self._generate_file_summaries(- file_paths, context_type=context_type, parent_uri=uri, enqueue_files=True- )+ file_paths: List[str] = []+ for entry in entries:+ name = entry.get("name", "")+ if not name or name.startswith(".") or name in [".", ".."]:+ continue+ if not entry.get("isDir", False):+ item_uri = VikingURI(dir_uri).join(name).uri+ file_paths.append(item_uri)- # 3. Generate .overview.md (contains brief description)- overview = await self._generate_overview(uri, file_summaries, children_abstracts)+ if not file_paths:+ logger.info(f"No memory files found in {dir_uri}")+ return- # 4. Extract abstract from overview+ file_summaries: List[Dict[str, str]] = []+ existing_summaries: Dict[str, str] = {}++ if msg.changes:+ try:+ old_overview = await viking_fs.read_file(f"{dir_uri}/.overview.md", ctx=ctx)+ if old_overview:+ existing_summaries = self._parse_overview_md(old_overview)+ logger.info(+ f"Parsed {len(existing_summaries)} existing summaries from overview.md"+ )+ except Exception as e:+ logger.debug(f"No existing overview.md found for {dir_uri}: {e}")++ changed_files: Set[str] = set()+ if msg.changes:+ changed_files = set(msg.changes.get("added", []) + msg.changes.get("modified", []))+ deleted_files = set(msg.changes.get("deleted", []))+ logger.info(+ f"Processing memory directory {dir_uri} with changes: "+ f"added={len(msg.changes.get('added', []))}, "+ f"modified={len(msg.changes.get('modified', []))}, "+ f"deleted={len(deleted_files)}"+ )++ for file_path in file_paths:+ file_name = file_path.split("/")[-1]++ if file_path not in changed_files and file_name in existing_summaries:+ file_summaries.append({"name": file_name, "summary": existing_summaries[file_name]})+ logger.debug(f"Reused existing summary for {file_name}")+ else:+ try:+ summary_dict = await self._generate_single_file_summary(+ file_path, llm_sem=llm_sem, ctx=ctx+ )+ file_summaries.append(summary_dict)+ logger.debug(f"Generated summary for {file_name}")+ except Exception as e:+ logger.warning(f"Failed to generate summary for {file_path}: {e}")+ file_summaries.append({"name": file_name, "summary": ""})++ overview = await self._generate_overview(dir_uri, file_summaries, [])abstract = self._extract_abstract_from_overview(overview)- # 5. Write files- await viking_fs.write_file(f"{uri}/.overview.md", overview, ctx=self._current_ctx)- await viking_fs.write_file(f"{uri}/.abstract.md", abstract, ctx=self._current_ctx)+ try:+ await viking_fs.write_file(f"{dir_uri}/.overview.md", overview, ctx=ctx)+ await viking_fs.write_file(f"{dir_uri}/.abstract.md", abstract, ctx=ctx)+ logger.info(f"Generated abstract.md and overview.md for {dir_uri}")+ except Exception as e:+ logger.error(f"Failed to write abstract/overview for {dir_uri}: {e}")+ return- logger.debug(f"Generated overview and abstract for {uri}")+ await self._vectorize_directory(+ uri=dir_uri,+ context_type="memory",+ abstract=abstract,+ overview=overview,+ ctx=ctx,+ semantic_msg_id=msg.id,+ )+ logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")- # 6. Vectorize directory+ async def _sync_topdown_recursive(+ self,+ root_uri: str,+ target_uri: str,+ ctx: Optional[RequestContext] = None,+ file_change_status: Optional[Dict[str, bool]] = None,+ ) -> DiffResult:+ viking_fs = get_viking_fs()+ diff = DiffResult()++ async def list_children(dir_uri: str) -> Tuple[Dict[str, str], Dict[str, str]]:+ files: Dict[str, str] = {}+ dirs: Dict[str, str] = {}+ try:+ entries = await viking_fs.ls(dir_uri, show_all_hidden=True, ctx=ctx)+ except Exception as e:+ logger.error(f"[SyncDiff] Failed to list {dir_uri}: {e}")+ return files, dirs++ for entry in entries:+ name = entry.get("name", "")+ if not name or name in [".", ".."]:+ continue+ if name.startswith(".") and name not in [".abstract.md", ".overview.md"]:+ continue+ item_uri = VikingURI(dir_uri).join(name).uri+ if entry.get("isDir", False):+ dirs[name] = item_uri+ else:+ files[name] = item_uri+ return files, dirs++ async def sync_dir(root_dir: str, target_dir: str) -> None:+ root_files, root_dirs = await list_children(root_dir)+ target_files, target_dirs = await list_children(target_dir)++ file_names = set(root_files.keys()) | set(target_files.keys())+ for name in sorted(file_names):+ root_file = root_files.get(name)+ target_file = target_files.get(name)++ if root_file and name in target_dirs:+ target_conflict_dir = target_dirs[name]+ try:+ await viking_fs.rm(target_conflict_dir, recursive=True, ctx=ctx)+ diff.deleted_dirs.append(target_conflict_dir)+ target_dirs.pop(name, None)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to delete directory for file conflict: {target_conflict_dir}, error={e}"+ )+ target_file = None++ if target_file and name in root_dirs and not root_file:+ try:+ await viking_fs.rm(target_file, ctx=ctx)+ diff.deleted_files.append(target_file)+ target_files.pop(name, None)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to delete file for dir conflict: {target_file}, error={e}"+ )+ continue++ if target_file and not root_file:+ try:+ await viking_fs.rm(target_file, ctx=ctx)+ diff.deleted_files.append(target_file)+ except Exception as e:+ logger.error(f"[SyncDiff] Failed to delete file: {target_file}, error={e}")+ continue++ if root_file and target_file:+ changed = False+ if file_change_status and root_file in file_change_status:+ changed = file_change_status[root_file]+ else:+ try:+ changed = await self._check_file_content_changed(+ root_file, target_file, ctx=ctx+ )+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to compare file content for {root_file}: {e}, treating as unchanged"+ )+ changed = False+ if changed:+ diff.updated_files.append(root_file)+ try:+ await viking_fs.rm(target_file, ctx=ctx)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to remove old file before update: {target_file}, error={e}"+ )+ try:+ await viking_fs.mv(root_file, target_file, ctx=ctx)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to move updated file: {root_file} -> {target_file}, error={e}"+ )+ continue++ if root_file and not target_file:+ diff.added_files.append(root_file)+ target_file_uri = VikingURI(target_dir).join(name).uri+ try:+ await viking_fs.mv(root_file, target_file_uri, ctx=ctx)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to move added file: {root_file} -> {target_file_uri}, error={e}"+ )++ dir_names = set(root_dirs.keys()) | set(target_dirs.keys())+ for name in sorted(dir_names):+ root_subdir = root_dirs.get(name)+ target_subdir = target_dirs.get(name)++ if root_subdir and name in target_files:+ target_conflict_file = target_files[name]+ try:+ await viking_fs.rm(target_conflict_file, ctx=ctx)+ diff.deleted_files.append(target_conflict_file)+ target_files.pop(name, None)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to delete file for dir conflict: {target_conflict_file}, error={e}"+ )+ target_subdir = None++ if target_subdir and not root_subdir:+ try:+ await viking_fs.rm(target_subdir, recursive=True, ctx=ctx)+ diff.deleted_dirs.append(target_subdir)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to delete directory: {target_subdir}, error={e}"+ )+ continue++ if root_subdir and not target_subdir:+ diff.added_dirs.append(root_subdir)+ target_subdir_uri = VikingURI(target_dir).join(name).uri+ try:+ await viking_fs.mv(root_subdir, target_subdir_uri, ctx=ctx)+ except Exception as e:+ logger.error(+ f"[SyncDiff] Failed to move added directory: {root_subdir} -> {target_subdir_uri}, error={e}"+ )+ continue++ if root_subdir and target_subdir:+ await sync_dir(root_subdir, target_subdir)++ target_exists = await viking_fs.exists(target_uri, ctx=ctx)+ if not target_exists:+ parent_uri = VikingURI(target_uri).parent+ if parent_uri:+ await viking_fs.mkdir(parent_uri.uri, exist_ok=True, ctx=ctx)+ diff.added_dirs.append(root_uri)+ await viking_fs.mv(root_uri, target_uri, ctx=ctx)+ return diff++ await sync_dir(root_uri, target_uri)try:- await self._vectorize_directory_simple(uri, context_type, abstract, overview)+ await viking_fs.rm(root_uri, recursive=True, ctx=ctx)except Exception as e:- logger.error(f"Failed to vectorize directory {uri}: {e}", exc_info=True)+ logger.error(f"[SyncDiff] Failed to delete root directory {root_uri}: {e}")+ return diff- async def _collect_children_abstracts(self, children_uris: List[str]) -> List[Dict[str, str]]:+ async def _collect_children_abstracts(+ self, children_uris: List[str], ctx: Optional[RequestContext] = None+ ) -> List[Dict[str, str]]:"""Collect .abstract.md from subdirectories."""viking_fs = get_viking_fs()results = []for child_uri in children_uris:- abstract = await viking_fs.abstract(child_uri, ctx=self._current_ctx)+ abstract = await viking_fs.abstract(child_uri, ctx=ctx)dir_name = child_uri.split("/")[-1]results.append({"name": dir_name, "abstract": abstract})return results- async def _generate_file_summaries(- self,- file_paths: List[str],- context_type: Optional[str] = None,- parent_uri: Optional[str] = None,- enqueue_files: bool = False,- ) -> List[Dict[str, str]]:- """Concurrently generate file summaries."""- if not file_paths:- return []-- async def generate_one_summary(file_path: str) -> Dict[str, str]:- summary = await self._generate_single_file_summary(file_path, ctx=self._current_ctx)- if enqueue_files and context_type and parent_uri:- try:- await self._vectorize_single_file(- parent_uri=parent_uri,- context_type=context_type,- file_path=file_path,- summary_dict=summary,- )- except Exception as e:- logger.error(- f"Failed to vectorize file {file_path}: {e}",- exc_info=True,- )- return summary-- tasks = [generate_one_summary(fp) for fp in file_paths]- return await asyncio.gather(*tasks)-async def _generate_text_summary(self,file_path: str,@@ -410,11 +568,6 @@ class SemanticProcessor(DequeueHandlerBase):vlm = get_openviking_config().vlmactive_ctx = ctx or self._current_ctx- # Read file content (limit length)- content = await viking_fs.read_file(file_path, ctx=active_ctx)-- # Limit content length (about 10000 tokens)- max_chars = 30000content = await viking_fs.read_file(file_path, ctx=active_ctx)if isinstance(content, bytes):# Try to decode with error handling for text files@@ -533,6 +686,59 @@ class SemanticProcessor(DequeueHandlerBase):return "\n".join(content_lines).strip()+ def _parse_overview_md(self, overview_content: str) -> Dict[str, str]:+ """Parse overview.md and extract file summaries.++ Args:+ overview_content: Content of the overview.md file++ Returns:+ Dictionary mapping file names to their summaries+ """+ import re++ summaries: Dict[str, str] = {}++ if not overview_content or not overview_content.strip():+ return summaries++ lines = overview_content.split("\n")+ current_file = None+ current_summary_lines: List[str] = []++ for line in lines:+ header_match = re.match(r"^###\s+(.+?)\s*$", line)+ if header_match:+ if current_file and current_summary_lines:+ summaries[current_file] = " ".join(current_summary_lines).strip()++ file_name = header_match.group(1).strip()+ parts = file_name.split()+ if len(parts) >= 2 and parts[0] == parts[1]:+ file_name = parts[0]++ current_file = file_name+ current_summary_lines = []+ continue++ numbered_match = re.match(r"^\[(\d+)\]\s+(.+?):\s*(.+)$", line)+ if numbered_match:+ if current_file and current_summary_lines:+ summaries[current_file] = " ".join(current_summary_lines).strip()+ current_file = numbered_match.group(2).strip()+ current_summary_lines = [numbered_match.group(3).strip()]+ continue++ if current_file:+ stripped = line.strip()+ if stripped and not stripped.startswith("#"):+ current_summary_lines.append(stripped)++ if current_file and current_summary_lines:+ summaries[current_file] = " ".join(current_summary_lines).strip()++ return summaries+async def _generate_overview(self,dir_uri: str,@@ -598,13 +804,14 @@ class SemanticProcessor(DequeueHandlerBase):logger.error(f"Failed to generate overview for {dir_uri}: {e}", exc_info=True)return f"# {dir_uri.split('/')[-1]}\n\nDirectory overview"- async def _vectorize_directory_simple(+ async def _vectorize_directory(self,uri: str,context_type: str,abstract: str,overview: str,ctx: Optional[RequestContext] = None,+ semantic_msg_id: Optional[str] = None,) -> None:"""Create directory Context and enqueue to EmbeddingQueue."""@@ -621,40 +828,17 @@ class SemanticProcessor(DequeueHandlerBase):overview=overview,context_type=context_type,ctx=active_ctx,+ semantic_msg_id=semantic_msg_id,)- async def _vectorize_files(- self,- uri: str,- context_type: str,- file_paths: List[str],- file_summaries: List[Dict[str, str]],- ctx: Optional[RequestContext] = None,- ) -> None:- """Vectorize files in directory."""- from openviking.storage.queuefs import get_queue_manager-- queue_manager = get_queue_manager()- embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)-- for file_path, file_summary_dict in zip(file_paths, file_summaries):- await self._vectorize_single_file(- parent_uri=uri,- context_type=context_type,- file_path=file_path,- summary_dict=file_summary_dict,- embedding_queue=embedding_queue,- ctx=ctx,- )-async def _vectorize_single_file(self,parent_uri: str,context_type: str,file_path: str,summary_dict: Dict[str, str],- embedding_queue: Optional[Any] = None,ctx: Optional[RequestContext] = None,+ semantic_msg_id: Optional[str] = None,) -> None:"""Vectorize a single file using its content or summary."""from openviking.utils.embedding_utils import vectorize_file@@ -666,4 +850,5 @@ class SemanticProcessor(DequeueHandlerBase):parent_uri=parent_uri,context_type=context_type,ctx=active_ctx,+ semantic_msg_id=semantic_msg_id,)diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex 62cf15fa7..7c45f5441 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -361,6 +361,22 @@ class VikingFS:path = self._uri_to_path(uri, ctx=ctx)return self.agfs.stat(path)+ async def exists(self, uri: str, ctx: Optional[RequestContext] = None) -> bool:+ """Check if a URI exists.++ Args:+ uri: Viking URI+ ctx: Request context++ Returns:+ bool: True if the URI exists, False otherwise+ """+ try:+ await self.stat(uri, ctx=ctx)+ return True+ except Exception:+ return False+async def glob(self,pattern: str,@@ -1086,7 +1102,7 @@ class VikingFS:async def _collect_uris(self, path: str, recursive: bool, ctx: Optional[RequestContext] = None) -> List[str]:- """Recursively collect all URIs (for rm/mv)."""+ """Recursively collect all URIs (for rm/mv), including directories."""uris = []async def _collect(p: str):@@ -1097,6 +1113,7 @@ class VikingFS:continuefull_path = f"{p}/{name}".replace("//", "/")if entry.get("isDir"):+ uris.append(self._path_to_uri(full_path, ctx=ctx))if recursive:await _collect(full_path)else:@@ -1146,23 +1163,8 @@ class VikingFS:for uri in uris:try:- records = await vector_store.get_context_by_uri(- uri=uri,- limit=1,- ctx=self._ctx_or_default(ctx),- )-- if not records or "id" not in records[0]:- continue-- record = records[0]-new_uri = uri.replace(old_base_uri, new_base_uri, 1)-- old_parent_uri = record.get("parent_uri", "")- new_parent_uri = (- old_parent_uri.replace(old_base_uri, new_base_uri, 1) if old_parent_uri else ""- )+ new_parent_uri = VikingURI(new_uri).parent.uriawait vector_store.update_uri_mapping(ctx=self._ctx_or_default(ctx),diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex 5effb6cbc..880af30b7 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -830,15 +830,54 @@ class VikingVectorIndexBackend:new_uri: str,new_parent_uri: str,) -> bool:+ import hashlib+records = await self.filter(filter=And([Eq("uri", uri), Eq("account_id", ctx.account_id)]),- limit=1,+ limit=100,ctx=ctx,)- if not records or "id" not in records[0]:+ if not records:return False- updated = {**records[0], "uri": new_uri, "parent_uri": new_parent_uri}- return bool(await self.upsert(updated, ctx=ctx))++ def _seed_uri_for_id(uri: str, level: int) -> str:+ if level == 0:+ return uri if uri.endswith("/.abstract.md") else f"{uri}/.abstract.md"+ if level == 1:+ return uri if uri.endswith("/.overview.md") else f"{uri}/.overview.md"+ return uri++ success = False+ ids_to_delete: List[str] = []+ for record in records:+ if "id" not in record:+ continue+ raw_level = record.get("level", 2)+ try:+ level = int(raw_level)+ except (TypeError, ValueError):+ level = 2++ seed_uri = _seed_uri_for_id(new_uri, level)+ id_seed = f"{ctx.account_id}:{seed_uri}"+ new_id = hashlib.md5(id_seed.encode("utf-8")).hexdigest()++ updated = {+ **record,+ "id": new_id,+ "uri": new_uri,+ "parent_uri": new_parent_uri,+ }+ if await self.upsert(updated, ctx=ctx):+ success = True+ old_id = record.get("id")+ if old_id and old_id != new_id:+ ids_to_delete.append(old_id)++ if ids_to_delete:+ await self.delete(list(set(ids_to_delete)), ctx=ctx)++ return successasync def increment_active_count(self, ctx: RequestContext, uris: List[str]) -> int:updated = 0diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex 1dffc1c8a..9a2f9d2d5 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -20,6 +20,22 @@ from openviking_cli.utils import VikingURI, get_loggerlogger = get_logger(__name__)+async def _decrement_embedding_tracker(semantic_msg_id: Optional[str], count: int) -> None:+ if not semantic_msg_id or count <= 0:+ return+ try:+ from openviking.storage.queuefs.embedding_tracker import EmbeddingTaskTracker++ tracker = EmbeddingTaskTracker.get_instance()+ for _ in range(count):+ await tracker.decrement(semantic_msg_id)+ except Exception as e:+ logger.error(+ f"Failed to decrement embedding tracker for semantic_msg_id={semantic_msg_id}: {e}",+ exc_info=True,+ )++def _owner_space_for_uri(uri: str, ctx: RequestContext) -> str:"""Derive owner_space from a URI."""if uri.startswith("viking://agent/"):@@ -116,57 +132,78 @@ async def vectorize_directory_meta(overview: str,context_type: str = "resource",ctx: Optional[RequestContext] = None,+ semantic_msg_id: Optional[str] = None,) -> None:"""Vectorize directory metadata (.abstract.md and .overview.md).Creates Context objects for abstract and overview and enqueues them."""- if not ctx:- logger.warning("No context provided for vectorization")- return+ enqueued = 0+ try:+ if not ctx:+ logger.warning("No context provided for vectorization")+ return- queue_manager = get_queue_manager()- embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)-- parent_uri = VikingURI(uri).parent.uri- owner_space = _owner_space_for_uri(uri, ctx)-- # Vectorize L0: .abstract.md (abstract)- context_abstract = Context(- uri=uri,- parent_uri=parent_uri,- is_leaf=False,- abstract=abstract,- context_type=context_type,- level=ContextLevel.ABSTRACT,- user=ctx.user,- account_id=ctx.account_id,- owner_space=owner_space,- )- context_abstract.set_vectorize(Vectorize(text=abstract))- msg_abstract = EmbeddingMsgConverter.from_context(context_abstract)- if msg_abstract:- await embedding_queue.enqueue(msg_abstract)- logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {uri}")-- # Vectorize L1: .overview.md (overview)- context_overview = Context(- uri=uri,- parent_uri=parent_uri,- is_leaf=False,- abstract=abstract,- context_type=context_type,- level=ContextLevel.OVERVIEW,- user=ctx.user,- account_id=ctx.account_id,- owner_space=owner_space,- )- context_overview.set_vectorize(Vectorize(text=overview))- msg_overview = EmbeddingMsgConverter.from_context(context_overview)- if msg_overview:- await embedding_queue.enqueue(msg_overview)- logger.debug(f"Enqueued directory L1 (overview) for vectorization: {uri}")+ queue_manager = get_queue_manager()+ embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)++ parent_uri = VikingURI(uri).parent.uri+ owner_space = _owner_space_for_uri(uri, ctx)++ # Vectorize L0: .abstract.md (abstract)+ context_abstract = Context(+ uri=uri,+ parent_uri=parent_uri,+ is_leaf=False,+ abstract=abstract,+ context_type=context_type,+ level=ContextLevel.ABSTRACT,+ user=ctx.user,+ account_id=ctx.account_id,+ owner_space=owner_space,+ )+ context_abstract.set_vectorize(Vectorize(text=abstract))+ msg_abstract = EmbeddingMsgConverter.from_context(context_abstract)+ if msg_abstract:+ msg_abstract.semantic_msg_id = semantic_msg_id+ try:+ await embedding_queue.enqueue(msg_abstract)+ enqueued += 1+ logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {uri}")+ except Exception as e:+ logger.error(+ f"Failed to enqueue directory L0 (abstract) for vectorization: {uri}: {e}",+ exc_info=True,+ )++ # Vectorize L1: .overview.md (overview)+ context_overview = Context(+ uri=uri,+ parent_uri=parent_uri,+ is_leaf=False,+ abstract=abstract,+ context_type=context_type,+ level=ContextLevel.OVERVIEW,+ user=ctx.user,+ account_id=ctx.account_id,+ owner_space=owner_space,+ )+ context_overview.set_vectorize(Vectorize(text=overview))+ msg_overview = EmbeddingMsgConverter.from_context(context_overview)+ if msg_overview:+ msg_overview.semantic_msg_id = semantic_msg_id+ try:+ await embedding_queue.enqueue(msg_overview)+ enqueued += 1+ logger.debug(f"Enqueued directory L1 (overview) for vectorization: {uri}")+ except Exception as e:+ logger.error(+ f"Failed to enqueue directory L1 (overview) for vectorization: {uri}: {e}",+ exc_info=True,+ )+ finally:+ await _decrement_embedding_tracker(semantic_msg_id, 2 - enqueued)async def vectorize_file(@@ -175,6 +212,7 @@ async def vectorize_file(parent_uri: str,context_type: str = "resource",ctx: Optional[RequestContext] = None,+ semantic_msg_id: Optional[str] = None,) -> None:"""Vectorize a single file.@@ -182,15 +220,17 @@ async def vectorize_file(Creates Context object for the file and enqueues it.Reads content for TEXT files, otherwise uses summary."""- if not ctx:- logger.warning("No context provided for vectorization")- return-- queue_manager = get_queue_manager()- embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)- viking_fs = get_viking_fs()+ enqueued = Falsetry:+ if not ctx:+ logger.warning("No context provided for vectorization")+ return++ queue_manager = get_queue_manager()+ embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)+ viking_fs = get_viking_fs()+file_name = summary_dict.get("name") or os.path.basename(file_path)summary = summary_dict.get("summary", "")@@ -246,11 +286,16 @@ async def vectorize_file(if not embedding_msg:return+ embedding_msg.semantic_msg_id = semantic_msg_idawait embedding_queue.enqueue(embedding_msg)+ enqueued = Truelogger.debug(f"Enqueued file for vectorization: {file_path}")except Exception as e:logger.error(f"Failed to vectorize file {file_path}: {e}", exc_info=True)+ finally:+ if not enqueued:+ await _decrement_embedding_tracker(semantic_msg_id, 1)async def index_resource(diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex 668ecb0ff..bd56a2eae 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -123,7 +123,7 @@ class ResourceProcessor:}telemetry = get_current_telemetry()- # ============ Phase 1: Parse source (Parser generates L0/L1 and writes to temp) ============+ # ============ Phase 1: Parse source and writes to temp viking fs ============try:parse_start = time.perf_counter()media_processor = self._get_media_processor()@@ -187,10 +187,10 @@ class ResourceProcessor:parent_uri=parent,source_path=parse_result.source_path,source_format=parse_result.source_format,- trigger_semantic=True,)if context_tree and context_tree.root:result["root_uri"] = context_tree.root.uri+ result["temp_uri"] = context_tree.root.temp_uritelemetry.set("resource.finalize.duration_ms",round((time.perf_counter() - finalize_start) * 1000, 3),@@ -211,6 +211,7 @@ class ResourceProcessor:# ============ Phase 4: Optional Steps ============build_index = kwargs.get("build_index", True)+ temp_uri_for_summarize = result.get("temp_uri") or parse_result.temp_dir_pathif summarize:# Explicit summarization request.# If build_index is ALSO True, we want vectorization.@@ -221,6 +222,7 @@ class ResourceProcessor:resource_uris=[result["root_uri"]],ctx=ctx,skip_vectorization=skip_vec,+ temp_uris=[temp_uri_for_summarize],**kwargs,)except Exception as e:@@ -232,7 +234,11 @@ class ResourceProcessor:# We assume this means "Ingest and Index", which requires summarization.try:await self._get_summarizer().summarize(- resource_uris=[result["root_uri"]], ctx=ctx, skip_vectorization=False, **kwargs+ resource_uris=[result["root_uri"]],+ ctx=ctx,+ skip_vectorization=False,+ temp_uris=[temp_uri_for_summarize],+ **kwargs,)except Exception as e:logger.error(f"Auto-index failed: {e}")diff --git a/openviking/utils/summarizer.py b/openviking/utils/summarizer.pyindex 3c6a39c68..22076f925 100644--- a/openviking/utils/summarizer.py+++ b/openviking/utils/summarizer.py@@ -40,9 +40,21 @@ class Summarizer:queue_manager = get_queue_manager()semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)+ temp_uris = kwargs.get("temp_uris", [])+ if not temp_uris:+ temp_uris = resource_uris+ if len(temp_uris) != len(resource_uris):+ logger.error(+ f"temp_uris length ({len(temp_uris)}) must match resource_uris length ({len(resource_uris)})"+ )+ return {+ "status": "error",+ "message": "temp_uris length must match resource_uris length",+ }enqueued_count = 0+telemetry = get_current_telemetry()- for uri in resource_uris:+ for uri, temp_uri in zip(resource_uris, temp_uris):# Determine context_type based on URIcontext_type = "resource"if uri.startswith("viking://memory/"):@@ -51,7 +63,7 @@ class Summarizer:context_type = "skill"msg = SemanticMsg(- uri=uri,+ uri=temp_uri,context_type=context_type,account_id=ctx.account_id,user_id=ctx.user.user_id,@@ -59,6 +71,7 @@ class Summarizer:role=ctx.role.value,skip_vectorization=skip_vectorization,telemetry_id=telemetry.telemetry_id if telemetry.enabled else "",+ target_uri=uri,)await semantic_queue.enqueue(msg)enqueued_count += 1diff --git a/openviking_cli/exceptions.py b/openviking_cli/exceptions.pyindex 807d317e5..cd432552c 100644--- a/openviking_cli/exceptions.py+++ b/openviking_cli/exceptions.py@@ -70,6 +70,14 @@ class AlreadyExistsError(OpenVikingError):)+class ConflictError(OpenVikingError):+ """Resource conflict (e.g., locked by another operation)."""++ def __init__(self, message: str, resource: Optional[str] = None):+ details = {"resource": resource} if resource else {}+ super().__init__(message, code="CONFLICT", details=details)++# ============= Authentication Errors =============diff --git a/tests/storage/test_semantic_dag_skip_files.py b/tests/storage/test_semantic_dag_skip_files.pyindex 75b233146..6b52fa4e9 100644--- a/tests/storage/test_semantic_dag_skip_files.py+++ b/tests/storage/test_semantic_dag_skip_files.py@@ -35,19 +35,29 @@ class _FakeProcessor:def _extract_abstract_from_overview(self, overview):return "abstract"- async def _vectorize_directory_simple(self, uri, context_type, abstract, overview, ctx=None):+ async def _vectorize_directory(+ self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None+ ):pass+ async def _vectorize_directory_simple(self, uri, context_type, abstract, overview, ctx=None):+ await self._vectorize_directory(uri, context_type, abstract, overview, ctx=ctx)+async def _vectorize_single_file(- self, parent_uri, context_type, file_path, summary_dict, ctx=None+ self, parent_uri, context_type, file_path, summary_dict, ctx=None, semantic_msg_id=None):self.vectorized_files.append(file_path)+class _DummyTracker:+ async def register(self, **_kwargs):+ return None++@pytest.mark.asyncioasync def test_messages_jsonl_excluded_from_summary(monkeypatch):"""messages.jsonl should be skipped by _list_dir and never summarized."""- root_uri = "viking://sessions/test-session"+ root_uri = "viking://session/test-session"tree = {root_uri: [{"name": "messages.jsonl", "isDir": False},@@ -57,6 +67,10 @@ async def test_messages_jsonl_excluded_from_summary(monkeypatch):}fake_fs = _FakeVikingFS(tree)monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)+ monkeypatch.setattr(+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",+ lambda: _DummyTracker(),+ )processor = _FakeProcessor()ctx = RequestContext(user=UserIdentifier("acc1", "user1", "agent1"), role=Role.USER)@@ -77,7 +91,7 @@ async def test_messages_jsonl_excluded_from_summary(monkeypatch):@pytest.mark.asyncioasync def test_messages_jsonl_excluded_in_subdirectory(monkeypatch):"""messages.jsonl in a subdirectory should also be skipped."""- root_uri = "viking://sessions/test-session"+ root_uri = "viking://session/test-session"tree = {root_uri: [{"name": "subdir", "isDir": True},@@ -89,6 +103,10 @@ async def test_messages_jsonl_excluded_in_subdirectory(monkeypatch):}fake_fs = _FakeVikingFS(tree)monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)+ monkeypatch.setattr(+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",+ lambda: _DummyTracker(),+ )processor = _FakeProcessor()ctx = RequestContext(user=UserIdentifier("acc1", "user1", "agent1"), role=Role.USER)diff --git a/tests/storage/test_semantic_dag_stats.py b/tests/storage/test_semantic_dag_stats.pyindex 10f06c226..8cb8c1f8f 100644--- a/tests/storage/test_semantic_dag_stats.py+++ b/tests/storage/test_semantic_dag_stats.py@@ -1,6 +1,8 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0+import asyncio+import pytestfrom openviking.server.identity import RequestContext, Role@@ -34,14 +36,24 @@ class _FakeProcessor:def _extract_abstract_from_overview(self, overview):return "abstract"- async def _vectorize_directory_simple(self, uri, context_type, abstract, overview, ctx=None):+ async def _vectorize_directory(+ self, uri, context_type, abstract, overview, ctx=None, semantic_msg_id=None+ ):self.vectorized_dirs.append(uri)async def _vectorize_single_file(- self, parent_uri, context_type, file_path, summary_dict, ctx=None+ self, parent_uri, context_type, file_path, summary_dict, ctx=None, semantic_msg_id=None):self.vectorized_files.append(file_path)+ async def _vectorize_directory_simple(self, uri, context_type, abstract, overview, ctx=None):+ await self._vectorize_directory(uri, context_type, abstract, overview, ctx=ctx)+++class _DummyTracker:+ async def register(self, **_kwargs):+ return None+@pytest.mark.asyncioasync def test_semantic_dag_stats_collects_nodes(monkeypatch):@@ -58,6 +70,10 @@ async def test_semantic_dag_stats_collects_nodes(monkeypatch):}fake_fs = _FakeVikingFS(tree)monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: fake_fs)+ monkeypatch.setattr(+ "openviking.storage.queuefs.embedding_tracker.EmbeddingTaskTracker.get_instance",+ lambda: _DummyTracker(),+ )processor = _FakeProcessor()ctx = RequestContext(user=UserIdentifier("acc1", "user1", "agent1"), role=Role.USER)@@ -68,6 +84,7 @@ async def test_semantic_dag_stats_collects_nodes(monkeypatch):ctx=ctx,)await executor.run(root_uri)+ await asyncio.sleep(0)stats = executor.get_stats()assert isinstance(stats, DagStats)