Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: 限制语义与向量任务的失败重入队次数 (#5934)

hezhang-121684cb68c07daa
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index 2df4d0855..a71a37eed 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -35,6 +35,7 @@ from openviking.storage.queuefs.embedding_msg import (
)
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
from openviking.storage.queuefs.process_result import ProcessResult
+from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNT
from openviking.storage.vector_ids import vector_record_id
from openviking.storage.viking_vector_index_backend import (
VIKINGDB_CONTENT_MAX_SIZE,
@@ -824,6 +825,19 @@ class TextEmbeddingHandler(DequeueHandlerBase):
self._log_embedding_error(logging.WARNING, error_msg, embedding_msg)
execute_status = "requeued"
if self._vikingdb.has_queue_manager:
+ if embedding_msg.failure_requeue_count >= MAX_FAILURE_REQUEUE_COUNT:
+ terminal_error = (
+ f"{error_msg} (failure requeue limit reached: "
+ f"{MAX_FAILURE_REQUEUE_COUNT})"
+ )
+ self._log_embedding_error(
+ logging.ERROR, terminal_error, embedding_msg
+ )
+ self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
+ execute_status = "error"
+ request_failed_message = terminal_error
+ return ProcessResult.failed(terminal_error)
+ embedding_msg.failure_requeue_count += 1
try:
await self._reenqueue_embedding_msg(embedding_msg)
self._merge_request_stats(
diff --git a/openviking/storage/queuefs/embedding_msg.py b/openviking/storage/queuefs/embedding_msg.py
index 6aff26c9f..b5f7203ca 100644
--- a/openviking/storage/queuefs/embedding_msg.py
+++ b/openviking/storage/queuefs/embedding_msg.py
@@ -101,6 +101,7 @@ class EmbeddingMsg:
id: str = field(default_factory=lambda: str(uuid4()))
telemetry_id: str = ""
queue_enqueued_at: float = 0.0
+ failure_requeue_count: int = 0
def __init__(
self,
@@ -113,12 +114,14 @@ class EmbeddingMsg:
field_modes: Optional[Dict[str, str]] = None,
initial_fields: Optional[Dict[str, Any]] = None,
queue_enqueued_at: float = 0.0,
+ failure_requeue_count: int = 0,
*,
payload: EmbeddingPayload | None = None,
) -> None:
self.id = str(uuid4())
self.telemetry_id = telemetry_id
self.queue_enqueued_at = max(float(queue_enqueued_at or 0.0), 0.0)
+ self.failure_requeue_count = max(int(failure_requeue_count or 0), 0)
self.action = IndexAction(action)
if payload is not None:
if any(
@@ -323,6 +326,7 @@ class EmbeddingMsg:
"id": self.id,
"telemetry_id": self.telemetry_id,
"queue_enqueued_at": self.queue_enqueued_at,
+ "failure_requeue_count": self.failure_requeue_count,
"action": self.action.value,
"payload": self.payload.to_dict(),
}
@@ -388,6 +392,7 @@ class EmbeddingMsg:
payload=payload,
telemetry_id=str(data.get("telemetry_id") or ""),
queue_enqueued_at=data.get("queue_enqueued_at", 0.0),
+ failure_requeue_count=data.get("failure_requeue_count", 0),
)
else:
obj = cls(
@@ -400,6 +405,7 @@ class EmbeddingMsg:
field_modes=data.get("field_modes"),
initial_fields=data.get("initial_fields"),
queue_enqueued_at=data.get("queue_enqueued_at", 0.0),
+ failure_requeue_count=data.get("failure_requeue_count", 0),
)
obj.id = str(data.get("id") or obj.id)
return obj
diff --git a/openviking/storage/queuefs/retry_policy.py b/openviking/storage/queuefs/retry_policy.py
new file mode 100644
index 000000000..921483091
--- /dev/null
+++ b/openviking/storage/queuefs/retry_policy.py
@@ -0,0 +1,5 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Shared limits for durable queue retry policies."""
+
+MAX_FAILURE_REQUEUE_COUNT = 2
diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.py
index 48c31e73c..f30876070 100644
--- a/openviking/storage/queuefs/semantic_msg.py
+++ b/openviking/storage/queuefs/semantic_msg.py
@@ -50,6 +50,7 @@ class SemanticMsg:
status: str = "pending" # pending/processing/completed
timestamp: int = field(default_factory=lambda: int(datetime.now().timestamp()))
queue_enqueued_at: float = 0.0
+ failure_requeue_count: int = 0
recursive: bool = True # Whether to recursively process subdirectories
account_id: str = "default"
user_id: str = "default"
@@ -114,10 +115,12 @@ class SemanticMsg:
file_abstracts: Optional[Dict[str, str]] = None,
plan: SemanticPlan | Dict[str, Any] | None = None,
queue_enqueued_at: float = 0.0,
+ failure_requeue_count: int = 0,
):
self.id = str(uuid4())
self.timestamp = int(datetime.now().timestamp())
self.queue_enqueued_at = max(float(queue_enqueued_at or 0.0), 0.0)
+ self.failure_requeue_count = max(int(failure_requeue_count or 0), 0)
self.uri = uri
self.context_type = context_type
self.recursive = recursive
@@ -227,6 +230,7 @@ class SemanticMsg:
),
plan=data.get("plan") if isinstance(data.get("plan"), dict) else None,
queue_enqueued_at=data.get("queue_enqueued_at", 0.0),
+ failure_requeue_count=data.get("failure_requeue_count", 0),
)
if "id" in data and data["id"]:
obj.id = data["id"]
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index e28b42d9b..53eed513c 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -52,6 +52,7 @@ from openviking.storage.errors import LockAcquisitionError
from openviking.storage.index_action import FieldPatch
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
from openviking.storage.queuefs.process_result import ProcessResult
+from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNT
from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor, SemanticTreeStats
from openviking.storage.queuefs.semantic_lock import SemanticLockScope
from openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_coalesce_key
@@ -328,7 +329,20 @@ class SemanticProcessor(DequeueHandlerBase):
error: Exception,
*,
work: SemanticMessageWork,
+ consume_failure_budget: bool = True,
) -> ProcessResult:
+ if consume_failure_budget and msg.failure_requeue_count >= MAX_FAILURE_REQUEUE_COUNT:
+ terminal_error = f"{error} (failure requeue limit reached: {MAX_FAILURE_REQUEUE_COUNT})"
+ logger.error("Semantic message reached failure requeue limit: %s", msg.uri)
+ self._merge_request_stats(msg.telemetry_id, error_count=1)
+ get_request_wait_tracker().mark_semantic_failed(
+ msg.telemetry_id, msg.id, terminal_error
+ )
+ await self._cleanup_local_artifact(msg)
+ return ProcessResult.failed(terminal_error)
+
+ if consume_failure_budget:
+ msg.failure_requeue_count += 1
try:
await work.reenqueue_after_error()
self._merge_request_stats(msg.telemetry_id, requeue_count=1)
@@ -743,11 +757,15 @@ class SemanticProcessor(DequeueHandlerBase):
exc_info=True,
)
if msg is not None and work is not None:
- return await self._requeue_semantic_msg_after_error(
+ result = await self._requeue_semantic_msg_after_error(
msg,
e,
work=work,
+ consume_failure_budget=False,
)
+ if result.error is not None:
+ execute_status = "error"
+ return result
return ProcessResult.failed(str(e))
error_class = classify_api_error(e)
@@ -794,11 +812,14 @@ class SemanticProcessor(DequeueHandlerBase):
if breaker is not None:
breaker.record_failure(e)
if msg is not None and work is not None:
- return await self._requeue_semantic_msg_after_error(
+ result = await self._requeue_semantic_msg_after_error(
msg,
e,
work=work,
)
+ if result.error is not None:
+ execute_status = "error"
+ return result
return ProcessResult.failed(str(e))
finally:
if msg is not None and execute_started_at is not None:
diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py
index 61a64378a..630fe96dc 100644
--- a/tests/storage/test_collection_schemas.py
+++ b/tests/storage/test_collection_schemas.py
@@ -32,6 +32,7 @@ from openviking.storage.expr import Eq
from openviking.storage.index_action import IndexAction
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
from openviking.storage.queuefs.process_result import ProcessOutcome
+from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNT
from openviking.storage.vector_ids import vector_record_id
from openviking.storage.vectordb import engine as vectordb_engine
from openviking.storage.vectordb.collection.collection import Collection
@@ -818,7 +819,7 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_w
self.enqueued = []
async def enqueue_embedding_msg(self, msg):
- self.enqueued.append(msg.id)
+ self.enqueued.append(msg)
return None
embedder = _DummyEmbedder()
@@ -827,7 +828,8 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_w
lambda: _DummyConfig(embedder),
)
- handler = TextEmbeddingHandler(_QueueingVikingDB())
+ vikingdb = _QueueingVikingDB()
+ handler = TextEmbeddingHandler(vikingdb)
monkeypatch.setattr(
await handler.breaker(),
"check",
@@ -855,6 +857,112 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_w
assert result.outcome is ProcessOutcome.REQUEUED
assert result.value is None
assert result.error is None
+ assert all(msg.failure_requeue_count == 0 for msg in vikingdb.enqueued)
+
+
+ ("failure_requeue_count", "expected_outcome"),
+ [
+ (0, ProcessOutcome.REQUEUED),
+ (MAX_FAILURE_REQUEUE_COUNT, ProcessOutcome.FAILED),
+ ],
+)
+async def test_embedding_failure_requeue_limit(
+ monkeypatch, failure_requeue_count, expected_outcome
+):
+ class _QueueingVikingDB:
+ is_closing = False
+ has_queue_manager = True
+
+ def __init__(self):
+ self.enqueued = []
+
+ async def enqueue_embedding_msg(self, msg):
+ self.enqueued.append(msg)
+
+ class _TransientErrorEmbedder(_DummyEmbedder):
+ async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
+ raise RuntimeError("500 Internal Server Error")
+
+ vikingdb = _QueueingVikingDB()
+ monkeypatch.setattr(
+ "openviking_cli.utils.config.get_openviking_config",
+ lambda: _DummyConfig(_TransientErrorEmbedder()),
+ )
+ handler = TextEmbeddingHandler(vikingdb)
+ msg = EmbeddingMsg(
+ message="hello",
+ context_data={
+ "id": "id-1",
+ "uri": "viking://resources/sample",
+ "account_id": "default",
+ "abstract": "sample",
+ },
+ failure_requeue_count=failure_requeue_count,
+ )
+
+ result = await handler.on_dequeue({"data": msg.to_json()})
+
+ assert result.outcome is expected_outcome
+ if expected_outcome is ProcessOutcome.REQUEUED:
+ assert len(vikingdb.enqueued) == 1
+ assert vikingdb.enqueued[0].failure_requeue_count == 1
+ else:
+ assert vikingdb.enqueued == []
+ assert result.error is not None
+ assert "failure requeue limit reached" in result.error
+
+
+async def test_embedding_failure_requeue_stops_after_three_attempts(monkeypatch):
+ class _QueueingVikingDB:
+ is_closing = False
+ has_queue_manager = True
+
+ def __init__(self):
+ self.enqueued = []
+
+ async def enqueue_embedding_msg(self, msg):
+ self.enqueued.append(msg)
+
+ class _TransientErrorEmbedder(_DummyEmbedder):
+ async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
+ self.calls += 1
+ raise RuntimeError("500 Internal Server Error")
+
+ embedder = _TransientErrorEmbedder()
+ vikingdb = _QueueingVikingDB()
+ monkeypatch.setattr(
+ "openviking_cli.utils.config.get_openviking_config",
+ lambda: _DummyConfig(embedder),
+ )
+ handler = TextEmbeddingHandler(vikingdb)
+ msg = EmbeddingMsg(
+ message="hello",
+ context_data={
+ "id": "id-1",
+ "uri": "viking://resources/sample",
+ "account_id": "default",
+ "abstract": "sample",
+ },
+ )
+
+ results = []
+ for _ in range(3):
+ results.append(await handler.on_dequeue({"data": msg.to_json()}))
+ if results[-1].outcome is ProcessOutcome.REQUEUED:
+ msg = vikingdb.enqueued[-1]
+
+ assert [result.outcome for result in results] == [
+ ProcessOutcome.REQUEUED,
+ ProcessOutcome.REQUEUED,
+ ProcessOutcome.FAILED,
+ ]
+ assert [msg.failure_requeue_count for msg in vikingdb.enqueued] == [1, 2]
+ assert embedder.calls == 3
+ assert results[-1].error is not None
+ assert "failure requeue limit reached" in results[-1].error
@pytest.mark.asyncio
diff --git a/tests/storage/test_embedding_msg.py b/tests/storage/test_embedding_msg.py
index d5d3a0537..3fb7edf01 100644
--- a/tests/storage/test_embedding_msg.py
+++ b/tests/storage/test_embedding_msg.py
@@ -48,6 +48,21 @@ def test_embedding_msg_roundtrip_preserves_queue_enqueue_time():
assert EmbeddingMsg.from_dict(msg.to_dict()).queue_enqueued_at == 123.456
+def test_embedding_msg_roundtrip_preserves_failure_requeue_count():
+ msg = EmbeddingMsg(
+ "hello",
+ {"uri": "viking://resources/demo", "account_id": "default"},
+ failure_requeue_count=1,
+ )
+
+ restored = EmbeddingMsg.from_dict(msg.to_dict())
+ legacy = msg.to_dict()
+ legacy.pop("failure_requeue_count")
+
+ assert restored.failure_requeue_count == 1
+ assert EmbeddingMsg.from_dict(legacy).failure_requeue_count == 0
+
+
def test_legacy_embedding_msg_without_account_id_is_rejected():
with pytest.raises(ValueError, match="account_id"):
EmbeddingMsg.from_dict(
@@ -305,6 +320,7 @@ def test_legacy_upsert_discards_historically_ignored_fields():
"id",
"telemetry_id",
"queue_enqueued_at",
+ "failure_requeue_count",
"action",
"payload",
}
diff --git a/tests/storage/test_semantic_msg_ingest_options.py b/tests/storage/test_semantic_msg_ingest_options.py
index 441b6b468..c53742e70 100644
--- a/tests/storage/test_semantic_msg_ingest_options.py
+++ b/tests/storage/test_semantic_msg_ingest_options.py
@@ -113,6 +113,21 @@ def test_semantic_msg_roundtrip_preserves_queue_enqueue_time():
assert SemanticMsg.from_dict(msg.to_dict()).queue_enqueued_at == 123.456
+def test_semantic_msg_roundtrip_preserves_failure_requeue_count():
+ msg = SemanticMsg(
+ uri="viking://resources/x",
+ context_type="resource",
+ failure_requeue_count=1,
+ )
+
+ restored = SemanticMsg.from_dict(msg.to_dict())
+ legacy = msg.to_dict()
+ legacy.pop("failure_requeue_count")
+
+ assert restored.failure_requeue_count == 1
+ assert SemanticMsg.from_dict(legacy).failure_requeue_count == 0
+
+
def test_semantic_msg_roundtrip_preserves_local_artifact_snapshot():
msg = SemanticMsg(
uri="viking://resources/x",
diff --git a/tests/storage/test_semantic_processor_permanent_storage_error.py b/tests/storage/test_semantic_processor_permanent_storage_error.py
index 971400f6c..999f0e12e 100644
--- a/tests/storage/test_semantic_processor_permanent_storage_error.py
+++ b/tests/storage/test_semantic_processor_permanent_storage_error.py
@@ -16,6 +16,7 @@ from openviking.service.task_work_index import TaskWorkIndex
from openviking.storage.errors import LockAcquisitionError
from openviking.storage.queuefs.named_queue import NamedQueue
from openviking.storage.queuefs.process_result import ProcessOutcome
+from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNT
from openviking.storage.queuefs.semantic_executor import SemanticTreeStats
from openviking.storage.queuefs.semantic_msg import SemanticMsg
from openviking.storage.queuefs.semantic_processor import SemanticProcessor
@@ -235,7 +236,9 @@ async def test_retryable_errors_keep_existing_behavior(
assert result.outcome is ProcessOutcome.REQUEUED
env.retry_queue.enqueue.assert_awaited_once()
- assert env.retry_queue.enqueue.call_args.args[0].id == msg.id
+ requeued = env.retry_queue.enqueue.call_args.args[0]
+ assert requeued.id == msg.id
+ assert requeued.failure_requeue_count == (0 if isinstance(error, LockAcquisitionError) else 1)
assert not env.tracker.is_complete(msg.telemetry_id)
assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"]["requeue_count"] == 1
if trips_breaker:
@@ -247,6 +250,55 @@ async def test_retryable_errors_keep_existing_behavior(
sleep.assert_not_awaited()
+async def test_semantic_failure_requeue_limit_is_terminal(semantic_env):
+ env = semantic_env
+ msg = env.message(failure_requeue_count=MAX_FAILURE_REQUEUE_COUNT)
+ error = TimeoutError("API timeout")
+ env.dag.run.side_effect = error
+
+ result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)
+
+ terminal_error = f"{error} (failure requeue limit reached: {MAX_FAILURE_REQUEUE_COUNT})"
+ assert result.outcome is ProcessOutcome.FAILED
+ assert result.error == terminal_error
+ env.retry_queue.enqueue.assert_not_awaited()
+ await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)
+ assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"] == {
+ "processed": 0,
+ "requeue_count": 0,
+ "error_count": 1,
+ "errors": [{"message": terminal_error}],
+ }
+
+
+async def test_semantic_failure_requeue_stops_after_three_attempts(semantic_env):
+ env = semantic_env
+ msg = env.message()
+ work = SimpleNamespace(reenqueue_after_error=AsyncMock())
+ error = TimeoutError("API timeout")
+
+ results = []
+ for _ in range(3):
+ msg = SemanticMsg.from_dict(msg.to_dict())
+ results.append(
+ await env.processor._requeue_semantic_msg_after_error(
+ msg,
+ error,
+ work=work,
+ )
+ )
+
+ assert [result.outcome for result in results] == [
+ ProcessOutcome.REQUEUED,
+ ProcessOutcome.REQUEUED,
+ ProcessOutcome.FAILED,
+ ]
+ assert msg.failure_requeue_count == MAX_FAILURE_REQUEUE_COUNT
+ assert work.reenqueue_after_error.await_count == MAX_FAILURE_REQUEUE_COUNT
+
+
@pytest.mark.asyncio
async def test_permanent_api_error_still_opens_breaker(semantic_env):
env = semantic_env