Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(queue): 修复写入等待遗漏下游索引任务 (#5357)
hezhang-12162b7efd566f93
diff --git a/openviking/storage/queuefs/embedding_msg_converter.py b/openviking/storage/queuefs/embedding_msg_converter.pyindex a4c2a9e17..f44e4d12d 100644--- a/openviking/storage/queuefs/embedding_msg_converter.py+++ b/openviking/storage/queuefs/embedding_msg_converter.py@@ -24,6 +24,8 @@ class EmbeddingMsgConverter:def from_context(context: Context,action: IndexAction = IndexAction.MERGE,+ *,+ telemetry_id: str | None = None,) -> EmbeddingMsg | None:"""Convert a Context object to EmbeddingMsg.@@ -90,7 +92,9 @@ class EmbeddingMsgConverter:embedding_msg = EmbeddingMsg.for_embed(message=message,context_data=context_data,- telemetry_id=get_current_telemetry().telemetry_id,+ telemetry_id=(+ get_current_telemetry().telemetry_id if telemetry_id is None else telemetry_id+ ),action=action,)return embedding_msgdiff --git a/openviking/storage/queuefs/semantic_executor.py b/openviking/storage/queuefs/semantic_executor.pyindex 12cd9bbce..b3099f67d 100644--- a/openviking/storage/queuefs/semantic_executor.py+++ b/openviking/storage/queuefs/semantic_executor.py@@ -197,6 +197,7 @@ class SemanticTreeExecutor:artifact_files: Optional[List[str]] = None,file_abstracts: Optional[Dict[str, str]] = None,semantic_plan: Optional["SemanticPlan"] = None,+ telemetry_id: str | None = None,):self._processor = processorself._context_type = context_type@@ -233,6 +234,7 @@ class SemanticTreeExecutor:self._task_context = get_task_context()self._processing_index = Noneself._telemetry = get_current_telemetry()+ self._telemetry_id = telemetry_idself._stale = Falseself._changed_paths = {path for key in ("added", "modified", "deleted") for path in self._changes.get(key, [])@@ -1145,6 +1147,8 @@ class SemanticTreeExecutor:vectorize_kwargs: Dict[str, Any] = {}if file_content is not None:vectorize_kwargs["file_content"] = file_content+ if self._telemetry_id is not None:+ vectorize_kwargs["telemetry_id"] = self._telemetry_idmanifest_md5 = self._file_md5s.get(file_path.rstrip("/")) or Noneif file_content is not None and not manifest_md5:file_md5 = content_md5(file_content)@@ -1517,6 +1521,8 @@ class SemanticTreeExecutor:"include_abstract": include_abstract,"include_overview": include_overview,}+ if self._telemetry_id is not None:+ directory_vector_kwargs["telemetry_id"] = self._telemetry_idif include_abstract or include_overview:enqueued_levels = await self._processor._vectorize_directory(dir_uri,@@ -1572,6 +1578,11 @@ class SemanticTreeExecutor:}),ctx=self._ctx,+ **(+ {"telemetry_id": self._telemetry_id}+ if self._telemetry_id is not None+ else {}+ ),)if enqueued:enqueued_levels.add(level)diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 97c4fe535..71a5922b5 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -530,6 +530,7 @@ class SemanticProcessor(DequeueHandlerBase):lock=semantic_lock.lock,source=msg.plan.source_metadata,semantic_plan=msg.plan,+ telemetry_id=msg.telemetry_id,)await executor.run(run_uri)self._cache_tree_stats(@@ -650,6 +651,7 @@ class SemanticProcessor(DequeueHandlerBase):file_md5s=msg.file_md5s,artifact_files=msg.artifact_files,file_abstracts=msg.file_abstracts,+ telemetry_id=msg.telemetry_id,)await executor.run(run_uri)dag_stats = executor.get_stats()@@ -961,6 +963,7 @@ class SemanticProcessor(DequeueHandlerBase):summary_dict=summary_dict,ctx=ctx,preserve_existing_created_at=True,+ telemetry_id=msg.telemetry_id,)file_summaries[idx] = {"name": str(summary_dict.get("name") or file_name),@@ -1028,6 +1031,7 @@ class SemanticProcessor(DequeueHandlerBase):abstract=abstract,overview=overview,ctx=ctx,+ telemetry_id=msg.telemetry_id,)logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")@@ -1857,6 +1861,7 @@ class SemanticProcessor(DequeueHandlerBase):field_patches: Optional[Dict[int, FieldPatch]] = None,include_abstract: bool = True,include_overview: bool = True,+ telemetry_id: str | None = None,) -> set[int]:"""Create directory Context and enqueue to EmbeddingQueue."""@@ -1888,6 +1893,7 @@ class SemanticProcessor(DequeueHandlerBase):field_patches=field_patches,include_abstract=include_abstract,include_overview=include_overview,+ telemetry_id=telemetry_id,)async def _load_transfer_file_summaries(@@ -1913,6 +1919,7 @@ class SemanticProcessor(DequeueHandlerBase):level: int,field_patch: FieldPatch,ctx: RequestContext,+ telemetry_id: str | None = None,) -> bool:from openviking.storage.queuefs import get_queue_managerfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsg@@ -1928,7 +1935,9 @@ class SemanticProcessor(DequeueHandlerBase):"account_id": ctx.account_id,"owner_user_id": ctx.user.user_id,},- telemetry_id=get_current_telemetry().telemetry_id,+ telemetry_id=(+ get_current_telemetry().telemetry_id if telemetry_id is None else telemetry_id+ ),)queue_manager = get_queue_manager()embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING, allow_create=True)@@ -1953,6 +1962,7 @@ class SemanticProcessor(DequeueHandlerBase):scalar_override: Optional[Dict[str, Any]] = None,field_patch: FieldPatch | None = None,action: str = "merge",+ telemetry_id: str | None = None,) -> bool:"""Vectorize a single file using its content or summary."""from openviking.utils.embedding_utils import vectorize_file@@ -1972,4 +1982,5 @@ class SemanticProcessor(DequeueHandlerBase):scalar_override=scalar_override,field_patch=field_patch,action=action,+ telemetry_id=telemetry_id,)diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex cc6abd6bd..a4e2dc86d 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -411,6 +411,7 @@ async def vectorize_directory_meta(content_is_body: bool = False,actions: Optional[Dict[int, IndexAction | str]] = None,field_patches: Optional[Dict[int, FieldPatch]] = None,+ telemetry_id: str | None = None,) -> set[int]:"""Vectorize directory metadata (.abstract.md and .overview.md).@@ -472,6 +473,7 @@ async def vectorize_directory_meta(IndexAction.MERGE,)),+ telemetry_id=telemetry_id,)level_overrides = (scalar_overrides or {}).get(int(ContextLevel.ABSTRACT.value))_apply_scalar_overrides(@@ -532,6 +534,7 @@ async def vectorize_directory_meta(IndexAction.MERGE,)),+ telemetry_id=telemetry_id,)level_overrides = (scalar_overrides or {}).get(int(ContextLevel.OVERVIEW.value))_apply_scalar_overrides(@@ -588,6 +591,7 @@ async def vectorize_file(file_md5: Optional[str] = None,file_content: Optional[bytes] = None,action: str = "merge",+ telemetry_id: str | None = None,) -> bool:"""Vectorize a single file.@@ -725,6 +729,7 @@ async def vectorize_file(embedding_msg = EmbeddingMsgConverter.from_context(context,action=resolved_action,+ telemetry_id=telemetry_id,)if not embedding_msg:return Falsediff --git a/tests/storage/test_embedding_msg_converter_tenant.py b/tests/storage/test_embedding_msg_converter_tenant.pyindex 7a0a8a313..6880de097 100644--- a/tests/storage/test_embedding_msg_converter_tenant.py+++ b/tests/storage/test_embedding_msg_converter_tenant.py@@ -8,6 +8,7 @@ import pytestfrom openviking.core.context import Context, Vectorizefrom openviking.storage.index_action import IndexActionfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter+from openviking.telemetry import OperationTelemetry, bind_telemetryfrom openviking_cli.session.user_id import UserIdentifier@@ -59,3 +60,14 @@ def test_embedding_msg_converter_keeps_only_embedding_input():assert msg.message == "bounded embedding text"assert "content" not in msg.context_dataassert msg.action is IndexAction.MERGE+++def test_embedding_msg_converter_prefers_explicit_telemetry_id():+ context = Context(uri="viking://resources/doc.md", abstract="summary")+ ambient = OperationTelemetry(operation="ambient", enabled=False)++ with bind_telemetry(ambient):+ msg = EmbeddingMsgConverter.from_context(context, telemetry_id="request-telemetry-id")++ assert msg is not None+ assert msg.telemetry_id == "request-telemetry-id"