Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: 限制语义与向量任务的失败重入队次数 (#5934)
hezhang-121684cb68c07daa
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 2df4d0855..a71a37eed 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -35,6 +35,7 @@ from openviking.storage.queuefs.embedding_msg import ()from openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.queuefs.process_result import ProcessResult+from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNTfrom openviking.storage.vector_ids import vector_record_idfrom openviking.storage.viking_vector_index_backend import (VIKINGDB_CONTENT_MAX_SIZE,@@ -824,6 +825,19 @@ class TextEmbeddingHandler(DequeueHandlerBase):self._log_embedding_error(logging.WARNING, error_msg, embedding_msg)execute_status = "requeued"if self._vikingdb.has_queue_manager:+ if embedding_msg.failure_requeue_count >= MAX_FAILURE_REQUEUE_COUNT:+ terminal_error = (+ f"{error_msg} (failure requeue limit reached: "+ f"{MAX_FAILURE_REQUEUE_COUNT})"+ )+ self._log_embedding_error(+ logging.ERROR, terminal_error, embedding_msg+ )+ self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)+ execute_status = "error"+ request_failed_message = terminal_error+ return ProcessResult.failed(terminal_error)+ embedding_msg.failure_requeue_count += 1try:await self._reenqueue_embedding_msg(embedding_msg)self._merge_request_stats(diff --git a/openviking/storage/queuefs/embedding_msg.py b/openviking/storage/queuefs/embedding_msg.pyindex 6aff26c9f..b5f7203ca 100644--- a/openviking/storage/queuefs/embedding_msg.py+++ b/openviking/storage/queuefs/embedding_msg.py@@ -101,6 +101,7 @@ class EmbeddingMsg:id: str = field(default_factory=lambda: str(uuid4()))telemetry_id: str = ""queue_enqueued_at: float = 0.0+ failure_requeue_count: int = 0def __init__(self,@@ -113,12 +114,14 @@ class EmbeddingMsg:field_modes: Optional[Dict[str, str]] = None,initial_fields: Optional[Dict[str, Any]] = None,queue_enqueued_at: float = 0.0,+ failure_requeue_count: int = 0,*,payload: EmbeddingPayload | None = None,) -> None:self.id = str(uuid4())self.telemetry_id = telemetry_idself.queue_enqueued_at = max(float(queue_enqueued_at or 0.0), 0.0)+ self.failure_requeue_count = max(int(failure_requeue_count or 0), 0)self.action = IndexAction(action)if payload is not None:if any(@@ -323,6 +326,7 @@ class EmbeddingMsg:"id": self.id,"telemetry_id": self.telemetry_id,"queue_enqueued_at": self.queue_enqueued_at,+ "failure_requeue_count": self.failure_requeue_count,"action": self.action.value,"payload": self.payload.to_dict(),}@@ -388,6 +392,7 @@ class EmbeddingMsg:payload=payload,telemetry_id=str(data.get("telemetry_id") or ""),queue_enqueued_at=data.get("queue_enqueued_at", 0.0),+ failure_requeue_count=data.get("failure_requeue_count", 0),)else:obj = cls(@@ -400,6 +405,7 @@ class EmbeddingMsg:field_modes=data.get("field_modes"),initial_fields=data.get("initial_fields"),queue_enqueued_at=data.get("queue_enqueued_at", 0.0),+ failure_requeue_count=data.get("failure_requeue_count", 0),)obj.id = str(data.get("id") or obj.id)return objdiff --git a/openviking/storage/queuefs/retry_policy.py b/openviking/storage/queuefs/retry_policy.pynew file mode 100644index 000000000..921483091--- /dev/null+++ b/openviking/storage/queuefs/retry_policy.py@@ -0,0 +1,5 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Shared limits for durable queue retry policies."""++MAX_FAILURE_REQUEUE_COUNT = 2diff --git a/openviking/storage/queuefs/semantic_msg.py b/openviking/storage/queuefs/semantic_msg.pyindex 48c31e73c..f30876070 100644--- a/openviking/storage/queuefs/semantic_msg.py+++ b/openviking/storage/queuefs/semantic_msg.py@@ -50,6 +50,7 @@ class SemanticMsg:status: str = "pending" # pending/processing/completedtimestamp: int = field(default_factory=lambda: int(datetime.now().timestamp()))queue_enqueued_at: float = 0.0+ failure_requeue_count: int = 0recursive: bool = True # Whether to recursively process subdirectoriesaccount_id: str = "default"user_id: str = "default"@@ -114,10 +115,12 @@ class SemanticMsg:file_abstracts: Optional[Dict[str, str]] = None,plan: SemanticPlan | Dict[str, Any] | None = None,queue_enqueued_at: float = 0.0,+ failure_requeue_count: int = 0,):self.id = str(uuid4())self.timestamp = int(datetime.now().timestamp())self.queue_enqueued_at = max(float(queue_enqueued_at or 0.0), 0.0)+ self.failure_requeue_count = max(int(failure_requeue_count or 0), 0)self.uri = uriself.context_type = context_typeself.recursive = recursive@@ -227,6 +230,7 @@ class SemanticMsg:),plan=data.get("plan") if isinstance(data.get("plan"), dict) else None,queue_enqueued_at=data.get("queue_enqueued_at", 0.0),+ failure_requeue_count=data.get("failure_requeue_count", 0),)if "id" in data and data["id"]:obj.id = data["id"]diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex e28b42d9b..53eed513c 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -52,6 +52,7 @@ from openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.index_action import FieldPatchfrom openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.queuefs.process_result import ProcessResult+from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNTfrom openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor, SemanticTreeStatsfrom openviking.storage.queuefs.semantic_lock import SemanticLockScopefrom openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_coalesce_key@@ -328,7 +329,20 @@ class SemanticProcessor(DequeueHandlerBase):error: Exception,*,work: SemanticMessageWork,+ consume_failure_budget: bool = True,) -> ProcessResult:+ if consume_failure_budget and msg.failure_requeue_count >= MAX_FAILURE_REQUEUE_COUNT:+ terminal_error = f"{error} (failure requeue limit reached: {MAX_FAILURE_REQUEUE_COUNT})"+ logger.error("Semantic message reached failure requeue limit: %s", msg.uri)+ self._merge_request_stats(msg.telemetry_id, error_count=1)+ get_request_wait_tracker().mark_semantic_failed(+ msg.telemetry_id, msg.id, terminal_error+ )+ await self._cleanup_local_artifact(msg)+ return ProcessResult.failed(terminal_error)++ if consume_failure_budget:+ msg.failure_requeue_count += 1try:await work.reenqueue_after_error()self._merge_request_stats(msg.telemetry_id, requeue_count=1)@@ -743,11 +757,15 @@ class SemanticProcessor(DequeueHandlerBase):exc_info=True,)if msg is not None and work is not None:- return await self._requeue_semantic_msg_after_error(+ result = await self._requeue_semantic_msg_after_error(msg,e,work=work,+ consume_failure_budget=False,)+ if result.error is not None:+ execute_status = "error"+ return resultreturn ProcessResult.failed(str(e))error_class = classify_api_error(e)@@ -794,11 +812,14 @@ class SemanticProcessor(DequeueHandlerBase):if breaker is not None:breaker.record_failure(e)if msg is not None and work is not None:- return await self._requeue_semantic_msg_after_error(+ result = await self._requeue_semantic_msg_after_error(msg,e,work=work,)+ if result.error is not None:+ execute_status = "error"+ return resultreturn ProcessResult.failed(str(e))finally:if msg is not None and execute_started_at is not None:diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex 61a64378a..630fe96dc 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -32,6 +32,7 @@ from openviking.storage.expr import Eqfrom openviking.storage.index_action import IndexActionfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsgfrom openviking.storage.queuefs.process_result import ProcessOutcome+from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNTfrom openviking.storage.vector_ids import vector_record_idfrom openviking.storage.vectordb import engine as vectordb_enginefrom openviking.storage.vectordb.collection.collection import Collection@@ -818,7 +819,7 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_wself.enqueued = []async def enqueue_embedding_msg(self, msg):- self.enqueued.append(msg.id)+ self.enqueued.append(msg)return Noneembedder = _DummyEmbedder()@@ -827,7 +828,8 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_wlambda: _DummyConfig(embedder),)- handler = TextEmbeddingHandler(_QueueingVikingDB())+ vikingdb = _QueueingVikingDB()+ handler = TextEmbeddingHandler(vikingdb)monkeypatch.setattr(await handler.breaker(),"check",@@ -855,6 +857,112 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_wassert result.outcome is ProcessOutcome.REQUEUEDassert result.value is Noneassert result.error is None+ assert all(msg.failure_requeue_count == 0 for msg in vikingdb.enqueued)+++ ("failure_requeue_count", "expected_outcome"),+ [+ (0, ProcessOutcome.REQUEUED),+ (MAX_FAILURE_REQUEUE_COUNT, ProcessOutcome.FAILED),+ ],+)+async def test_embedding_failure_requeue_limit(+ monkeypatch, failure_requeue_count, expected_outcome+):+ class _QueueingVikingDB:+ is_closing = False+ has_queue_manager = True++ def __init__(self):+ self.enqueued = []++ async def enqueue_embedding_msg(self, msg):+ self.enqueued.append(msg)++ class _TransientErrorEmbedder(_DummyEmbedder):+ async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:+ raise RuntimeError("500 Internal Server Error")++ vikingdb = _QueueingVikingDB()+ monkeypatch.setattr(+ "openviking_cli.utils.config.get_openviking_config",+ lambda: _DummyConfig(_TransientErrorEmbedder()),+ )+ handler = TextEmbeddingHandler(vikingdb)+ msg = EmbeddingMsg(+ message="hello",+ context_data={+ "id": "id-1",+ "uri": "viking://resources/sample",+ "account_id": "default",+ "abstract": "sample",+ },+ failure_requeue_count=failure_requeue_count,+ )++ result = await handler.on_dequeue({"data": msg.to_json()})++ assert result.outcome is expected_outcome+ if expected_outcome is ProcessOutcome.REQUEUED:+ assert len(vikingdb.enqueued) == 1+ assert vikingdb.enqueued[0].failure_requeue_count == 1+ else:+ assert vikingdb.enqueued == []+ assert result.error is not None+ assert "failure requeue limit reached" in result.error+++async def test_embedding_failure_requeue_stops_after_three_attempts(monkeypatch):+ class _QueueingVikingDB:+ is_closing = False+ has_queue_manager = True++ def __init__(self):+ self.enqueued = []++ async def enqueue_embedding_msg(self, msg):+ self.enqueued.append(msg)++ class _TransientErrorEmbedder(_DummyEmbedder):+ async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:+ self.calls += 1+ raise RuntimeError("500 Internal Server Error")++ embedder = _TransientErrorEmbedder()+ vikingdb = _QueueingVikingDB()+ monkeypatch.setattr(+ "openviking_cli.utils.config.get_openviking_config",+ lambda: _DummyConfig(embedder),+ )+ handler = TextEmbeddingHandler(vikingdb)+ msg = EmbeddingMsg(+ message="hello",+ context_data={+ "id": "id-1",+ "uri": "viking://resources/sample",+ "account_id": "default",+ "abstract": "sample",+ },+ )++ results = []+ for _ in range(3):+ results.append(await handler.on_dequeue({"data": msg.to_json()}))+ if results[-1].outcome is ProcessOutcome.REQUEUED:+ msg = vikingdb.enqueued[-1]++ assert [result.outcome for result in results] == [+ ProcessOutcome.REQUEUED,+ ProcessOutcome.REQUEUED,+ ProcessOutcome.FAILED,+ ]+ assert [msg.failure_requeue_count for msg in vikingdb.enqueued] == [1, 2]+ assert embedder.calls == 3+ assert results[-1].error is not None+ assert "failure requeue limit reached" in results[-1].error@pytest.mark.asynciodiff --git a/tests/storage/test_embedding_msg.py b/tests/storage/test_embedding_msg.pyindex d5d3a0537..3fb7edf01 100644--- a/tests/storage/test_embedding_msg.py+++ b/tests/storage/test_embedding_msg.py@@ -48,6 +48,21 @@ def test_embedding_msg_roundtrip_preserves_queue_enqueue_time():assert EmbeddingMsg.from_dict(msg.to_dict()).queue_enqueued_at == 123.456+def test_embedding_msg_roundtrip_preserves_failure_requeue_count():+ msg = EmbeddingMsg(+ "hello",+ {"uri": "viking://resources/demo", "account_id": "default"},+ failure_requeue_count=1,+ )++ restored = EmbeddingMsg.from_dict(msg.to_dict())+ legacy = msg.to_dict()+ legacy.pop("failure_requeue_count")++ assert restored.failure_requeue_count == 1+ assert EmbeddingMsg.from_dict(legacy).failure_requeue_count == 0++def test_legacy_embedding_msg_without_account_id_is_rejected():with pytest.raises(ValueError, match="account_id"):EmbeddingMsg.from_dict(@@ -305,6 +320,7 @@ def test_legacy_upsert_discards_historically_ignored_fields():"id","telemetry_id","queue_enqueued_at",+ "failure_requeue_count","action","payload",}diff --git a/tests/storage/test_semantic_msg_ingest_options.py b/tests/storage/test_semantic_msg_ingest_options.pyindex 441b6b468..c53742e70 100644--- a/tests/storage/test_semantic_msg_ingest_options.py+++ b/tests/storage/test_semantic_msg_ingest_options.py@@ -113,6 +113,21 @@ def test_semantic_msg_roundtrip_preserves_queue_enqueue_time():assert SemanticMsg.from_dict(msg.to_dict()).queue_enqueued_at == 123.456+def test_semantic_msg_roundtrip_preserves_failure_requeue_count():+ msg = SemanticMsg(+ uri="viking://resources/x",+ context_type="resource",+ failure_requeue_count=1,+ )++ restored = SemanticMsg.from_dict(msg.to_dict())+ legacy = msg.to_dict()+ legacy.pop("failure_requeue_count")++ assert restored.failure_requeue_count == 1+ assert SemanticMsg.from_dict(legacy).failure_requeue_count == 0++def test_semantic_msg_roundtrip_preserves_local_artifact_snapshot():msg = SemanticMsg(uri="viking://resources/x",diff --git a/tests/storage/test_semantic_processor_permanent_storage_error.py b/tests/storage/test_semantic_processor_permanent_storage_error.pyindex 971400f6c..999f0e12e 100644--- a/tests/storage/test_semantic_processor_permanent_storage_error.py+++ b/tests/storage/test_semantic_processor_permanent_storage_error.py@@ -16,6 +16,7 @@ from openviking.service.task_work_index import TaskWorkIndexfrom openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.queuefs.named_queue import NamedQueuefrom openviking.storage.queuefs.process_result import ProcessOutcome+from openviking.storage.queuefs.retry_policy import MAX_FAILURE_REQUEUE_COUNTfrom openviking.storage.queuefs.semantic_executor import SemanticTreeStatsfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessor@@ -235,7 +236,9 @@ async def test_retryable_errors_keep_existing_behavior(assert result.outcome is ProcessOutcome.REQUEUEDenv.retry_queue.enqueue.assert_awaited_once()- assert env.retry_queue.enqueue.call_args.args[0].id == msg.id+ requeued = env.retry_queue.enqueue.call_args.args[0]+ assert requeued.id == msg.id+ assert requeued.failure_requeue_count == (0 if isinstance(error, LockAcquisitionError) else 1)assert not env.tracker.is_complete(msg.telemetry_id)assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"]["requeue_count"] == 1if trips_breaker:@@ -247,6 +250,55 @@ async def test_retryable_errors_keep_existing_behavior(sleep.assert_not_awaited()+async def test_semantic_failure_requeue_limit_is_terminal(semantic_env):+ env = semantic_env+ msg = env.message(failure_requeue_count=MAX_FAILURE_REQUEUE_COUNT)+ error = TimeoutError("API timeout")+ env.dag.run.side_effect = error++ result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)++ terminal_error = f"{error} (failure requeue limit reached: {MAX_FAILURE_REQUEUE_COUNT})"+ assert result.outcome is ProcessOutcome.FAILED+ assert result.error == terminal_error+ env.retry_queue.enqueue.assert_not_awaited()+ await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)+ assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"] == {+ "processed": 0,+ "requeue_count": 0,+ "error_count": 1,+ "errors": [{"message": terminal_error}],+ }+++async def test_semantic_failure_requeue_stops_after_three_attempts(semantic_env):+ env = semantic_env+ msg = env.message()+ work = SimpleNamespace(reenqueue_after_error=AsyncMock())+ error = TimeoutError("API timeout")++ results = []+ for _ in range(3):+ msg = SemanticMsg.from_dict(msg.to_dict())+ results.append(+ await env.processor._requeue_semantic_msg_after_error(+ msg,+ error,+ work=work,+ )+ )++ assert [result.outcome for result in results] == [+ ProcessOutcome.REQUEUED,+ ProcessOutcome.REQUEUED,+ ProcessOutcome.FAILED,+ ]+ assert msg.failure_requeue_count == MAX_FAILURE_REQUEUE_COUNT+ assert work.reenqueue_after_error.await_count == MAX_FAILURE_REQUEUE_COUNT++@pytest.mark.asyncioasync def test_permanent_api_error_still_opens_breaker(semantic_env):env = semantic_env