Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(embedding): fail terminally on auth errors instead of re-enqueueing (#2916) (#2919)

Kevin Houb2dcd74709da
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index 1b1b8d3cb..29d617401 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -33,7 +33,11 @@ from openviking.utils.circuit_breaker import (
CircuitBreakerOpen,
classify_api_error,
)
-from openviking.utils.model_retry import ERROR_CLASS_INPUT_TOO_LARGE, ERROR_CLASS_PERMANENT
+from openviking.utils.model_retry import (
+ ERROR_CLASS_AUTH,
+ ERROR_CLASS_INPUT_TOO_LARGE,
+ ERROR_CLASS_PERMANENT,
+)
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils import get_logger
from openviking_cli.utils.config.open_viking_config import OpenVikingConfig
@@ -655,6 +659,19 @@ class TextEmbeddingHandler(DequeueHandlerBase):
report_error_args = (error_msg, data)
return None
+ if error_class == ERROR_CLASS_AUTH:
+ # Bad/expired credential: retrying cannot succeed. Fail
+ # terminally instead of re-enqueueing, which would cycle
+ # forever and hold this resource's tree lock and its
+ # add-resource --wait open. Don't trip the breaker: an open
+ # breaker re-enqueues later messages and reintroduces the
+ # same leak. See #2916.
+ logger.error(error_msg)
+ self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)
+ request_failed_message = error_msg
+ report_error_args = (error_msg, data)
+ return None
+
# Transient or unknown — re-enqueue for retry
logger.warning(error_msg)
self._circuit_breaker.record_failure(embed_err)
diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py
index 95eb2772b..158165e15 100644
--- a/tests/storage/test_collection_schemas.py
+++ b/tests/storage/test_collection_schemas.py
@@ -337,6 +337,48 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_w
assert status == {"success": 2, "requeue": 2, "error": 0}
+async def test_embedding_auth_error_fails_terminally_without_reenqueue(monkeypatch):
+ """A credential (401/403) failure must fail terminally, not re-enqueue: an
+ infinite re-enqueue holds the resource's tree lock and add-resource --wait
+ open, and must not trip the circuit breaker (which re-enqueues too). #2916."""
+
+ class _QueueingVikingDB:
+ is_closing = False
+ has_queue_manager = True
+
+ def __init__(self):
+ self.enqueued = []
+
+ async def enqueue_embedding_msg(self, msg):
+ self.enqueued.append(msg.id)
+ return None
+
+ class _AuthErrorEmbedder(_DummyEmbedder):
+ async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
+ raise RuntimeError("Error code: 401 - {'code': 'AuthenticationError'} Unauthorized")
+
+ vikingdb = _QueueingVikingDB()
+ monkeypatch.setattr(
+ "openviking_cli.utils.config.get_openviking_config",
+ lambda: _DummyConfig(_AuthErrorEmbedder()),
+ )
+ handler = TextEmbeddingHandler(vikingdb)
+ status = {"success": 0, "requeue": 0, "error": 0}
+ handler.set_callbacks(
+ on_success=lambda: status.__setitem__("success", status["success"] + 1),
+ on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),
+ on_error=lambda *_: status.__setitem__("error", status["error"] + 1),
+ )
+
+ result = await handler.on_dequeue(_build_queue_payload_for_account("acct"))
+
+ assert result is None
+ assert vikingdb.enqueued == [] # terminal: not re-enqueued
+ assert status == {"success": 0, "requeue": 0, "error": 1}
+ handler._circuit_breaker.check() # breaker not tripped (would raise if open)
+
+
@pytest.mark.asyncio
async def test_embedding_handler_treats_shutdown_write_lock_as_success(monkeypatch):
class _ClosingDuringUpsertVikingDB: