Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(resources): expose context count in upload tasks

Hao Zheda138de7cea1
diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.md
index 6c175ba74..4275eb95a 100644
--- a/docs/en/api/02-resources.md
+++ b/docs/en/api/02-resources.md
@@ -575,7 +575,36 @@ task_id uuid-xxx
| `warnings` | array | (Optional) List of warnings (only when `strict=False`) |
| `queue_status` | object | (Optional, only when `wait=true`) Queue processing status with `pending`, `processing`, `completed` counts |
-For Git repository sources with `wait=false`, the background task has `task_type="add_resource"` and `resource_id` equal to the returned `root_uri`. Running task records may include `stage`; completed task results include `queue_status` with the final semantic and embedding queue summary.
+**Completed add-resource task result**
+
+For Git repository sources with `wait=false`, the background task has `task_type="add_resource"` and `resource_id` equal to the returned `root_uri`. Running task records may include `stage`. Poll `/api/v1/tasks/{task_id}` until the task completes. Its nested `result` then contains the final queue summary and `context_count`:
+
+```json
+{
+ "status": "ok",
+ "result": {
+ "task_id": "uuid-xxx",
+ "task_type": "add_resource",
+ "status": "completed",
+ "resource_id": "viking://resources/guide",
+ "result": {
+ "status": "success",
+ "root_uri": "viking://resources/guide",
+ "queue_status": {
+ "Embedding": {
+ "processed": 11,
+ "requeue_count": 0,
+ "error_count": 0,
+ "errors": []
+ }
+ },
+ "context_count": 11
+ }
+ }
+}
+```
+
+`context_count` is the number of contexts successfully produced and indexed by this upload task. A context is counted after its embedding record is written successfully. It is not the total number of contexts already stored under `root_uri`. If the server restarts before the task persists its final metrics, the field is omitted instead of reporting a partial count.
---
diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.md
index bad0f1011..83437c9d0 100644
--- a/docs/zh/api/02-resources.md
+++ b/docs/zh/api/02-resources.md
@@ -581,7 +581,36 @@ task_id uuid-xxx
| `queue_status` | object | (可选,仅当 `wait=true` 时)队列处理状态,包含 `pending`、`processing`、`completed` 计数 |
| `memory_linking` | object | (可选,仅当 `reason` 触发记忆生成时)本次资源 URI 与用户记忆的关联结果 |
-对于 `wait=false` 的 Git 仓库来源,后台任务的 `task_type="add_resource"`,`resource_id` 等于返回的 `root_uri`。运行中的任务记录可能包含 `stage`;完成后的任务 `result` 会包含带有 semantic 和 embedding 汇总的 `queue_status`。
+**完成后的资源添加任务结果**
+
+对于 `wait=false` 的 Git 仓库来源,后台任务的 `task_type="add_resource"`,`resource_id` 等于返回的 `root_uri`。运行中的任务记录可能包含 `stage`。轮询 `/api/v1/tasks/{task_id}` 直到任务完成。完成后,任务内层的 `result` 会包含最终队列汇总和 `context_count`:
+
+```json
+{
+ "status": "ok",
+ "result": {
+ "task_id": "uuid-xxx",
+ "task_type": "add_resource",
+ "status": "completed",
+ "resource_id": "viking://resources/guide",
+ "result": {
+ "status": "success",
+ "root_uri": "viking://resources/guide",
+ "queue_status": {
+ "Embedding": {
+ "processed": 11,
+ "requeue_count": 0,
+ "error_count": 0,
+ "errors": []
+ }
+ },
+ "context_count": 11
+ }
+ }
+}
+```
+
+`context_count` 是本次上传任务成功生成并完成索引的上下文数量。每条上下文对应的嵌入记录成功写入后,计数增加一次。该值不是 `root_uri` 下已有上下文的总数。如果服务器在任务持久化最终指标前重启,该字段会被省略,以避免返回不完整的计数。
---
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index 77b48caa6..879f40589 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -212,8 +212,7 @@ def _embedding_metadata_compatible(
if existing_meta is None:
return False
return all(
- existing_meta.get(key) == current_meta.get(key)
- for key in _EMBEDDING_COMPATIBILITY_KEYS
+ existing_meta.get(key) == current_meta.get(key) for key in _EMBEDDING_COMPATIBILITY_KEYS
)
@@ -848,7 +847,10 @@ class TextEmbeddingHandler(DequeueHandlerBase):
return None
self._merge_request_stats(embedding_msg.telemetry_id, processed=1)
- self._record_request_success(embedding_msg)
+ self._record_request_success(
+ embedding_msg,
+ vector_written=bool(record_id),
+ )
report_success = True
self._circuit_breaker.record_success()
return inserted_data
@@ -894,10 +896,16 @@ class TextEmbeddingHandler(DequeueHandlerBase):
return None
@staticmethod
- def _record_request_success(embedding_msg: EmbeddingMsg) -> None:
- get_request_wait_tracker().mark_embedding_done(
+ def _record_request_success(
+ embedding_msg: EmbeddingMsg,
+ *,
+ vector_written: bool = False,
+ ) -> None:
+ tracker = get_request_wait_tracker()
+ tracker.mark_embedding_done(
embedding_msg.telemetry_id,
embedding_msg.id,
+ vector_written=vector_written,
)
@staticmethod
diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.py
index 3ea345ce8..0717e6526 100644
--- a/openviking/storage/queuefs/add_resource_processor.py
+++ b/openviking/storage/queuefs/add_resource_processor.py
@@ -211,9 +211,12 @@ class AddResourceProcessor(DequeueHandlerBase):
result = deepcopy(replay_result)
await tracker.wait_for_descendants(msg.task_id, metadata.work_id)
result.setdefault(
- "queue_status",
- request_wait_tracker.build_queue_status(telemetry_id),
+ "queue_status", request_wait_tracker.build_queue_status(telemetry_id)
)
+ if replay_result is None:
+ result["context_count"] = request_wait_tracker.get_embedding_context_count(
+ telemetry_id
+ )
record_resource_queue_metrics(
telemetry=telemetry,
telemetry_id=telemetry_id,
diff --git a/openviking/telemetry/request_wait_tracker.py b/openviking/telemetry/request_wait_tracker.py
index a26b84a18..76dc9d52c 100644
--- a/openviking/telemetry/request_wait_tracker.py
+++ b/openviking/telemetry/request_wait_tracker.py
@@ -20,6 +20,7 @@ class _RequestWaitState:
semantic_error_count: int = 0
semantic_errors: List[str] = field(default_factory=list)
embedding_processed: int = 0
+ embedding_context_count: int = 0
embedding_requeue_count: int = 0
embedding_error_count: int = 0
embedding_errors: List[str] = field(default_factory=list)
@@ -82,6 +83,14 @@ class RequestWaitTracker:
return
state.embedding_requeue_count += max(delta, 0)
+ def get_embedding_context_count(self, telemetry_id: str) -> int:
+ """Return contexts successfully indexed for one request."""
+ if not telemetry_id:
+ return 0
+ with self._lock:
+ state = self._states.get(telemetry_id)
+ return state.embedding_context_count if state is not None else 0
+
def mark_semantic_done(
self,
telemetry_id: str,
@@ -123,6 +132,8 @@ class RequestWaitTracker:
telemetry_id: str,
root_id: str,
processed_delta: int = 1,
+ *,
+ vector_written: bool = False,
) -> None:
if not telemetry_id:
return
@@ -130,8 +141,11 @@ class RequestWaitTracker:
state = self._states.get(telemetry_id)
if state is None:
return
+ was_pending = root_id in state.pending_embedding_roots
state.pending_embedding_roots.discard(root_id)
state.embedding_processed += max(processed_delta, 0)
+ if vector_written and was_pending:
+ state.embedding_context_count += 1
def mark_embedding_failed(self, telemetry_id: str, root_id: str, message: str) -> None:
if not telemetry_id:
diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.py
index 15c57daaf..21282b770 100644
--- a/tests/parse/test_feishu_parser_api.py
+++ b/tests/parse/test_feishu_parser_api.py
@@ -16,6 +16,7 @@ from openviking.service.task_work_index import TASK_WORK_ID_FIELD
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg, AddResourcePhase
from openviking.storage.queuefs.add_resource_processor import AddResourceProcessor
from openviking.storage.queuefs.queue_manager import QueueManager
+from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.utils.media_processor import UnifiedResourceProcessor
from openviking_cli.exceptions import InvalidArgumentError
from openviking_cli.session.user_id import UserIdentifier
@@ -865,12 +866,7 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc
monkeypatch,
):
final_uri = "viking://resources/真实文档标题"
- service = SimpleNamespace(
- execute_add_resource_job=AsyncMock(
- return_value={"status": "success", "root_uri": final_uri}
- ),
- _link_resource_reason_memory=AsyncMock(),
- )
+ telemetry_id = "telemetry-final-resource"
task_tracker = SimpleNamespace(
create=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.PENDING)),
start=AsyncMock(),
@@ -880,6 +876,13 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc
get_task_auth=AsyncMock(return_value={}),
wait_for_descendants=AsyncMock(),
)
+
+ service = SimpleNamespace(
+ execute_add_resource_job=AsyncMock(
+ return_value={"status": "success", "root_uri": final_uri}
+ ),
+ _link_resource_reason_memory=AsyncMock(),
+ )
monkeypatch.setattr(
"openviking.storage.queuefs.add_resource_processor.get_task_tracker",
Mock(return_value=task_tracker),
@@ -906,6 +909,7 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc
account_id="account-1",
user_id="user-1",
role="user",
+ telemetry_id=telemetry_id,
defer_target_resolution=True,
staged_source={
"temp_uri": "viking://temp/account-1/task-1",
@@ -916,6 +920,12 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc
},
)
+ wait_tracker = get_request_wait_tracker()
+ wait_tracker.register_request(telemetry_id)
+ for index in range(9):
+ embedding_id = f"embedding-{index}"
+ wait_tracker.register_embedding_root(telemetry_id, embedding_id)
+ wait_tracker.mark_embedding_done(telemetry_id, embedding_id, vector_written=True)
data = msg.to_dict()
data[TASK_WORK_ID_FIELD] = "work-1"
await processor._process(msg, data)
@@ -945,6 +955,7 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc
{
"status": "success",
"root_uri": final_uri,
+ "context_count": 9,
"queue_status": {
"Semantic": {
"processed": 0,
@@ -953,7 +964,7 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc
"errors": [],
},
"Embedding": {
- "processed": 0,
+ "processed": 9,
"requeue_count": 0,
"error_count": 0,
"errors": [],
@@ -978,6 +989,7 @@ async def test_add_resource_processor_persists_final_uri_and_cleans_staged_sourc
@pytest.mark.asyncio
async def test_add_resource_processor_replay_skips_lock_adopt_when_result_exists(monkeypatch):
final_uri = "viking://resources/replayed"
+ telemetry_id = "telemetry-replayed-resource"
service = SimpleNamespace(
execute_add_resource_job=AsyncMock(
return_value={"status": "success", "root_uri": "viking://resources/duplicated"}
@@ -988,7 +1000,7 @@ async def test_add_resource_processor_replay_skips_lock_adopt_when_result_exists
create=AsyncMock(
return_value=SimpleNamespace(
status=TaskStatus.RUNNING,
- result={"status": "success", "root_uri": final_uri},
+ result={"status": "success", "root_uri": final_uri, "context_count": 4},
)
),
start=AsyncMock(),
@@ -1018,6 +1030,7 @@ async def test_add_resource_processor_replay_skips_lock_adopt_when_result_exists
account_id="account-1",
user_id="user-1",
role="user",
+ telemetry_id=telemetry_id,
lock_handoff={"owner_id": "owner-1", "lock_paths": ["/resource/.path.ovlock"]},
)
data = msg.to_dict()
@@ -1032,6 +1045,53 @@ async def test_add_resource_processor_replay_skips_lock_adopt_when_result_exists
task_tracker.complete.assert_awaited_once()
completed_result = task_tracker.complete.await_args.args[1]
assert completed_result["root_uri"] == final_uri
+ assert completed_result["context_count"] == 4
+
+
+async def test_add_resource_processor_reports_zero_vectors(monkeypatch):
+ final_uri = "viking://resources/no-vectors"
+ service = SimpleNamespace(
+ execute_add_resource_job=AsyncMock(
+ return_value={"status": "success", "root_uri": final_uri}
+ ),
+ _link_resource_reason_memory=AsyncMock(),
+ )
+ task_tracker = SimpleNamespace(
+ create=AsyncMock(return_value=SimpleNamespace(status=TaskStatus.PENDING)),
+ start=AsyncMock(),
+ update_stage=AsyncMock(),
+ complete=AsyncMock(),
+ fail=AsyncMock(),
+ wait_for_descendants=AsyncMock(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.add_resource_processor.get_task_tracker",
+ Mock(return_value=task_tracker),
+ )
+ processor = AddResourceProcessor(
+ service,
+ asyncio.get_running_loop(),
+ QueueManager.ADD_RESOURCE,
+ SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock())),
+ )
+ msg = AddResourceMsg(
+ task_id="task-1",
+ path="/tmp/demo.md",
+ root_uri=final_uri,
+ account_id="account-1",
+ user_id="user-1",
+ role="user",
+ )
+ data = msg.to_dict()
+ data[TASK_WORK_ID_FIELD] = "work-1"
+
+ await processor._process(msg, data)
+
+ task_tracker.fail.assert_not_awaited()
+ final_result = task_tracker.complete.await_args.args[1]
+ assert final_result["root_uri"] == final_uri
+ assert final_result["context_count"] == 0
def test_feishu_direct_submission_requires_configured_auth(monkeypatch):
diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py
index 22f98658c..977dbf86e 100644
--- a/tests/storage/test_collection_schemas.py
+++ b/tests/storage/test_collection_schemas.py
@@ -679,8 +679,8 @@ async def test_embedding_handler_settles_request_wait_by_message_id(monkeypatch)
monkeypatch.setattr(
"openviking.storage.collection_schemas.get_request_wait_tracker",
lambda: SimpleNamespace(
- mark_embedding_done=lambda telemetry_id, root_id: completed.append(
- (telemetry_id, root_id)
+ mark_embedding_done=lambda telemetry_id, root_id, **kwargs: completed.append(
+ (telemetry_id, root_id, kwargs)
)
),
)
@@ -693,7 +693,7 @@ async def test_embedding_handler_settles_request_wait_by_message_id(monkeypatch)
await handler.on_dequeue(payload)
- assert completed == [("request-1", queue_data["id"])]
+ assert completed == [("request-1", queue_data["id"], {"vector_written": True})]
def test_context_collection_excludes_parent_uri():
@@ -1942,9 +1942,7 @@ async def test_viking_vector_index_backend_upsert_partial_update_delegates_to_ac
)
assert result == "rec-1"
- assert calls == [
- ({"id": "rec-1", "abstract": "patched"}, UpsertOptions(True, "append"))
- ]
+ assert calls == [({"id": "rec-1", "abstract": "patched"}, UpsertOptions(True, "append"))]
@pytest.mark.asyncio
diff --git a/tests/telemetry/test_request_wait_tracker.py b/tests/telemetry/test_request_wait_tracker.py
index fffb5550f..122788fd2 100644
--- a/tests/telemetry/test_request_wait_tracker.py
+++ b/tests/telemetry/test_request_wait_tracker.py
@@ -54,6 +54,23 @@ def test_request_wait_tracker_records_requeues():
}
+def test_request_wait_tracker_counts_contexts_indexed_by_request():
+ tracker = RequestWaitTracker()
+ telemetry_id = "tm_vectors"
+
+ tracker.register_request(telemetry_id)
+ for root_id in ("embedding-1", "embedding-2", "embedding-3"):
+ tracker.register_embedding_root(telemetry_id, root_id)
+ tracker.mark_embedding_done(telemetry_id, root_id, vector_written=True)
+ tracker.mark_embedding_done(telemetry_id, "embedding-3", vector_written=True)
+
+ assert tracker.get_embedding_context_count(telemetry_id) == 3
+
+ tracker.cleanup(telemetry_id)
+
+ assert tracker.get_embedding_context_count(telemetry_id) == 0
+
+
async def test_wait_for_request_timeout_keeps_existing_error():
tracker = RequestWaitTracker()
telemetry_id = "tm_wait_timeout"