Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(queuefs): skip add-resource lock replay after persisted result (#4007)

baojun-zhang84c0895c448a
diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.py
index 551ed5991..a8b3f2c2b 100644
--- a/openviking/storage/queuefs/add_resource_processor.py
+++ b/openviking/storage/queuefs/add_resource_processor.py
@@ -6,6 +6,7 @@ import asyncio
import concurrent.futures
import json
from contextlib import suppress
+from copy import deepcopy
from typing import Any, Dict, Optional
from openviking.observability.context import bind_execution_context
@@ -108,21 +109,24 @@ class AddResourceProcessor(DequeueHandlerBase):
self.report_success()
return None
+ metadata = extract_task_metadata(data)
+ replay_result = getattr(task, "result", None)
resource_lock = None
- try:
- resource_lock = await self._load_lock(msg, ctx)
- except Exception as exc:
- if await self._requeue_lock_handoff(msg, exc):
+ if replay_result is None:
+ try:
+ resource_lock = await self._load_lock(msg, ctx)
+ except Exception as exc:
+ if await self._requeue_lock_handoff(msg, exc):
+ return None
+ await tracker.fail(
+ msg.task_id,
+ f"Invalid lock_handoff: {exc}",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error(f"Invalid lock_handoff: {exc}", data)
+ unregister_telemetry(telemetry_id)
return None
- await tracker.fail(
- msg.task_id,
- f"Invalid lock_handoff: {exc}",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
- self.report_error(f"Invalid lock_handoff: {exc}", data)
- unregister_telemetry(telemetry_id)
- return None
telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None
if telemetry is None:
@@ -148,31 +152,43 @@ class AddResourceProcessor(DequeueHandlerBase):
bind_task_context(msg.task_id, ctx.account_id, ctx.user.user_id),
):
try:
- metadata = extract_task_metadata(data)
- await tracker.start(
- msg.task_id,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- stage="queued",
- )
- result = await self._resource_service.execute_add_resource_job(
- msg,
- ctx=ctx,
- resource_lock=resource_lock,
- stage_callback=_set_stage,
- )
- if result.get("status") == "error":
- errors = result.get("errors") or ["resource processing failed"]
- await tracker.fail(
+ if replay_result is None:
+ await tracker.start(
msg.task_id,
- "; ".join(str(error) for error in errors),
account_id=ctx.account_id,
user_id=ctx.user.user_id,
+ stage="queued",
)
- self.report_error("resource processing failed", data)
- return None
+ result = await self._resource_service.execute_add_resource_job(
+ msg,
+ ctx=ctx,
+ resource_lock=resource_lock,
+ stage_callback=_set_stage,
+ )
+ if result.get("status") == "error":
+ errors = result.get("errors") or ["resource processing failed"]
+ await tracker.fail(
+ msg.task_id,
+ "; ".join(str(error) for error in errors),
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
+ self.report_error("resource processing failed", data)
+ return None
+ await tracker.complete(
+ msg.task_id,
+ deepcopy(result),
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ resource_id=result.get("root_uri"),
+ )
+ else:
+ result = deepcopy(replay_result)
await tracker.wait_for_descendants(msg.task_id, metadata.work_id)
- result["queue_status"] = request_wait_tracker.build_queue_status(telemetry_id)
+ result.setdefault(
+ "queue_status",
+ request_wait_tracker.build_queue_status(telemetry_id),
+ )
record_resource_queue_metrics(
telemetry=telemetry,
telemetry_id=telemetry_id,
diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.py
index 7d68f20b5..a15774214 100644
--- a/tests/parse/test_feishu_parser_api.py
+++ b/tests/parse/test_feishu_parser_api.py
@@ -838,7 +838,19 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
task_id="task-1",
meta={"source_path": ""},
)
- task_tracker.complete.assert_awaited_once_with(
+ assert task_tracker.complete.await_count == 2
+ first_complete = task_tracker.complete.await_args_list[0]
+ assert first_complete.args == (
+ "task-1",
+ {"status": "success", "root_uri": final_uri},
+ )
+ assert first_complete.kwargs == {
+ "account_id": "account-1",
+ "user_id": "user-1",
+ "resource_id": final_uri,
+ }
+ final_complete = task_tracker.complete.await_args_list[-1]
+ assert final_complete.args == (
"task-1",
{
"status": "success",
@@ -858,14 +870,75 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
},
},
},
- account_id="account-1",
- user_id="user-1",
- resource_id=final_uri,
)
+ assert final_complete.kwargs == {
+ "account_id": "account-1",
+ "user_id": "user-1",
+ "resource_id": final_uri,
+ }
assert await processor._requeue_lock_handoff(msg, RuntimeError("stale lock"))
assert queue_manager.enqueue.await_args.args[0] == QueueManager.ADD_RESOURCE
+async def test_add_resource_processor_replay_skips_lock_adopt_when_result_exists(monkeypatch):
+ final_uri = "viking://resources/replayed"
+ service = SimpleNamespace(
+ execute_add_resource_job=AsyncMock(
+ return_value={"status": "success", "root_uri": "viking://resources/duplicated"}
+ ),
+ _link_resource_reason_memory=AsyncMock(),
+ )
+ task_tracker = SimpleNamespace(
+ create=AsyncMock(
+ return_value=SimpleNamespace(
+ status=TaskStatus.RUNNING,
+ result={"status": "success", "root_uri": final_uri},
+ )
+ ),
+ start=AsyncMock(),
+ update_stage=AsyncMock(),
+ complete=AsyncMock(),
+ fail=AsyncMock(),
+ wait_for_descendants=AsyncMock(),
+ )
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.add_resource_processor.get_task_tracker",
+ Mock(return_value=task_tracker),
+ )
+ async_agfs = SimpleNamespace(
+ pathlock_adopt=AsyncMock(return_value={"lease": "duplicate"}),
+ pathlock_release=AsyncMock(),
+ )
+ processor = AddResourceProcessor(
+ service,
+ asyncio.get_running_loop(),
+ QueueManager.ADD_RESOURCE,
+ SimpleNamespace(_async_agfs=async_agfs),
+ )
+ msg = AddResourceMsg(
+ task_id="task-1",
+ path="/tmp/demo.md",
+ root_uri=final_uri,
+ account_id="account-1",
+ user_id="user-1",
+ role="user",
+ lock_handoff={"owner_id": "owner-1", "lock_paths": ["/resource/.path.ovlock"]},
+ )
+ data = msg.to_dict()
+ data[TASK_WORK_ID_FIELD] = "work-1"
+
+ await processor._process(msg, data)
+
+ async_agfs.pathlock_adopt.assert_not_awaited()
+ service.execute_add_resource_job.assert_not_awaited()
+ task_tracker.start.assert_not_awaited()
+ task_tracker.wait_for_descendants.assert_awaited_once_with("task-1", "work-1")
+ task_tracker.complete.assert_awaited_once()
+ completed_result = task_tracker.complete.await_args.args[1]
+ assert completed_result["root_uri"] == final_uri
+
+
def test_feishu_direct_submission_requires_configured_auth(monkeypatch):
api = _understanding_api_for_parse()
source = "https://example.larkoffice.com/docx/doxcnToken"