Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(queue): 修复写入等待遗漏下游索引任务 (#5357)

hezhang-12162b7efd566f93
diff --git a/openviking/storage/queuefs/embedding_msg_converter.py b/openviking/storage/queuefs/embedding_msg_converter.py
index a4c2a9e17..f44e4d12d 100644
--- a/openviking/storage/queuefs/embedding_msg_converter.py
+++ b/openviking/storage/queuefs/embedding_msg_converter.py
@@ -24,6 +24,8 @@ class EmbeddingMsgConverter:
def from_context(
context: Context,
action: IndexAction = IndexAction.MERGE,
+ *,
+ telemetry_id: str | None = None,
) -> EmbeddingMsg | None:
"""
Convert a Context object to EmbeddingMsg.
@@ -90,7 +92,9 @@ class EmbeddingMsgConverter:
embedding_msg = EmbeddingMsg.for_embed(
message=message,
context_data=context_data,
- telemetry_id=get_current_telemetry().telemetry_id,
+ telemetry_id=(
+ get_current_telemetry().telemetry_id if telemetry_id is None else telemetry_id
+ ),
action=action,
)
return embedding_msg
diff --git a/openviking/storage/queuefs/semantic_executor.py b/openviking/storage/queuefs/semantic_executor.py
index 12cd9bbce..b3099f67d 100644
--- a/openviking/storage/queuefs/semantic_executor.py
+++ b/openviking/storage/queuefs/semantic_executor.py
@@ -197,6 +197,7 @@ class SemanticTreeExecutor:
artifact_files: Optional[List[str]] = None,
file_abstracts: Optional[Dict[str, str]] = None,
semantic_plan: Optional["SemanticPlan"] = None,
+ telemetry_id: str | None = None,
):
self._processor = processor
self._context_type = context_type
@@ -233,6 +234,7 @@ class SemanticTreeExecutor:
self._task_context = get_task_context()
self._processing_index = None
self._telemetry = get_current_telemetry()
+ self._telemetry_id = telemetry_id
self._stale = False
self._changed_paths = {
path for key in ("added", "modified", "deleted") for path in self._changes.get(key, [])
@@ -1145,6 +1147,8 @@ class SemanticTreeExecutor:
vectorize_kwargs: Dict[str, Any] = {}
if file_content is not None:
vectorize_kwargs["file_content"] = file_content
+ if self._telemetry_id is not None:
+ vectorize_kwargs["telemetry_id"] = self._telemetry_id
manifest_md5 = self._file_md5s.get(file_path.rstrip("/")) or None
if file_content is not None and not manifest_md5:
file_md5 = content_md5(file_content)
@@ -1517,6 +1521,8 @@ class SemanticTreeExecutor:
"include_abstract": include_abstract,
"include_overview": include_overview,
}
+ if self._telemetry_id is not None:
+ directory_vector_kwargs["telemetry_id"] = self._telemetry_id
if include_abstract or include_overview:
enqueued_levels = await self._processor._vectorize_directory(
dir_uri,
@@ -1572,6 +1578,11 @@ class SemanticTreeExecutor:
}
),
ctx=self._ctx,
+ **(
+ {"telemetry_id": self._telemetry_id}
+ if self._telemetry_id is not None
+ else {}
+ ),
)
if enqueued:
enqueued_levels.add(level)
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 97c4fe535..71a5922b5 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -530,6 +530,7 @@ class SemanticProcessor(DequeueHandlerBase):
lock=semantic_lock.lock,
source=msg.plan.source_metadata,
semantic_plan=msg.plan,
+ telemetry_id=msg.telemetry_id,
)
await executor.run(run_uri)
self._cache_tree_stats(
@@ -650,6 +651,7 @@ class SemanticProcessor(DequeueHandlerBase):
file_md5s=msg.file_md5s,
artifact_files=msg.artifact_files,
file_abstracts=msg.file_abstracts,
+ telemetry_id=msg.telemetry_id,
)
await executor.run(run_uri)
dag_stats = executor.get_stats()
@@ -961,6 +963,7 @@ class SemanticProcessor(DequeueHandlerBase):
summary_dict=summary_dict,
ctx=ctx,
preserve_existing_created_at=True,
+ telemetry_id=msg.telemetry_id,
)
file_summaries[idx] = {
"name": str(summary_dict.get("name") or file_name),
@@ -1028,6 +1031,7 @@ class SemanticProcessor(DequeueHandlerBase):
abstract=abstract,
overview=overview,
ctx=ctx,
+ telemetry_id=msg.telemetry_id,
)
logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")
@@ -1857,6 +1861,7 @@ class SemanticProcessor(DequeueHandlerBase):
field_patches: Optional[Dict[int, FieldPatch]] = None,
include_abstract: bool = True,
include_overview: bool = True,
+ telemetry_id: str | None = None,
) -> set[int]:
"""Create directory Context and enqueue to EmbeddingQueue."""
@@ -1888,6 +1893,7 @@ class SemanticProcessor(DequeueHandlerBase):
field_patches=field_patches,
include_abstract=include_abstract,
include_overview=include_overview,
+ telemetry_id=telemetry_id,
)
async def _load_transfer_file_summaries(
@@ -1913,6 +1919,7 @@ class SemanticProcessor(DequeueHandlerBase):
level: int,
field_patch: FieldPatch,
ctx: RequestContext,
+ telemetry_id: str | None = None,
) -> bool:
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
@@ -1928,7 +1935,9 @@ class SemanticProcessor(DequeueHandlerBase):
"account_id": ctx.account_id,
"owner_user_id": ctx.user.user_id,
},
- telemetry_id=get_current_telemetry().telemetry_id,
+ telemetry_id=(
+ get_current_telemetry().telemetry_id if telemetry_id is None else telemetry_id
+ ),
)
queue_manager = get_queue_manager()
embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING, allow_create=True)
@@ -1953,6 +1962,7 @@ class SemanticProcessor(DequeueHandlerBase):
scalar_override: Optional[Dict[str, Any]] = None,
field_patch: FieldPatch | None = None,
action: str = "merge",
+ telemetry_id: str | None = None,
) -> bool:
"""Vectorize a single file using its content or summary."""
from openviking.utils.embedding_utils import vectorize_file
@@ -1972,4 +1982,5 @@ class SemanticProcessor(DequeueHandlerBase):
scalar_override=scalar_override,
field_patch=field_patch,
action=action,
+ telemetry_id=telemetry_id,
)
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index cc6abd6bd..a4e2dc86d 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -411,6 +411,7 @@ async def vectorize_directory_meta(
content_is_body: bool = False,
actions: Optional[Dict[int, IndexAction | str]] = None,
field_patches: Optional[Dict[int, FieldPatch]] = None,
+ telemetry_id: str | None = None,
) -> set[int]:
"""
Vectorize directory metadata (.abstract.md and .overview.md).
@@ -472,6 +473,7 @@ async def vectorize_directory_meta(
IndexAction.MERGE,
)
),
+ telemetry_id=telemetry_id,
)
level_overrides = (scalar_overrides or {}).get(int(ContextLevel.ABSTRACT.value))
_apply_scalar_overrides(
@@ -532,6 +534,7 @@ async def vectorize_directory_meta(
IndexAction.MERGE,
)
),
+ telemetry_id=telemetry_id,
)
level_overrides = (scalar_overrides or {}).get(int(ContextLevel.OVERVIEW.value))
_apply_scalar_overrides(
@@ -588,6 +591,7 @@ async def vectorize_file(
file_md5: Optional[str] = None,
file_content: Optional[bytes] = None,
action: str = "merge",
+ telemetry_id: str | None = None,
) -> bool:
"""
Vectorize a single file.
@@ -725,6 +729,7 @@ async def vectorize_file(
embedding_msg = EmbeddingMsgConverter.from_context(
context,
action=resolved_action,
+ telemetry_id=telemetry_id,
)
if not embedding_msg:
return False
diff --git a/tests/storage/test_embedding_msg_converter_tenant.py b/tests/storage/test_embedding_msg_converter_tenant.py
index 7a0a8a313..6880de097 100644
--- a/tests/storage/test_embedding_msg_converter_tenant.py
+++ b/tests/storage/test_embedding_msg_converter_tenant.py
@@ -8,6 +8,7 @@ import pytest
from openviking.core.context import Context, Vectorize
from openviking.storage.index_action import IndexAction
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
+from openviking.telemetry import OperationTelemetry, bind_telemetry
from openviking_cli.session.user_id import UserIdentifier
@@ -59,3 +60,14 @@ def test_embedding_msg_converter_keeps_only_embedding_input():
assert msg.message == "bounded embedding text"
assert "content" not in msg.context_data
assert msg.action is IndexAction.MERGE
+
+
+def test_embedding_msg_converter_prefers_explicit_telemetry_id():
+ context = Context(uri="viking://resources/doc.md", abstract="summary")
+ ambient = OperationTelemetry(operation="ambient", enabled=False)
+
+ with bind_telemetry(ambient):
+ msg = EmbeddingMsgConverter.from_context(context, telemetry_id="request-telemetry-id")
+
+ assert msg is not None
+ assert msg.telemetry_id == "request-telemetry-id"