Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(queue): add circuit breaker to prevent API retry storms (#772)
Deepak Dev Panwar395267a8edf4
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 859f9ed2b..f37703a78 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -16,13 +16,17 @@ from dataclasses import dataclassfrom typing import Any, Dict, List, Optionalfrom openviking.models.embedder.base import EmbedResult-from openviking.models.embedder.volcengine_embedders import is_429_errorfrom openviking.server.identity import RequestContext, Rolefrom openviking.storage.errors import CollectionNotFoundErrorfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsgfrom openviking.storage.queuefs.named_queue import DequeueHandlerBasefrom openviking.storage.viking_vector_index_backend import VikingVectorIndexBackendfrom openviking.telemetry import bind_telemetry, resolve_telemetry+from openviking.utils.circuit_breaker import (+ CircuitBreaker,+ CircuitBreakerOpen,+ classify_api_error,+)from openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils import get_loggerfrom openviking_cli.utils.config.open_viking_config import OpenVikingConfig@@ -162,6 +166,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):self._collection_name = config.storage.vectordb.nameself._vector_dim = config.embedding.dimensionself._initialize_embedder(config)+ self._circuit_breaker = CircuitBreaker()def _initialize_embedder(self, config: "OpenVikingConfig"):"""Initialize the embedder instance from config."""@@ -236,6 +241,24 @@ class TextEmbeddingHandler(DequeueHandlerBase):self.report_success()return data+ # Circuit breaker: if API is known-broken, re-enqueue and wait+ try:+ self._circuit_breaker.check()+ except CircuitBreakerOpen:+ logger.warning(+ f"Circuit breaker is open, re-enqueueing embedding: {embedding_msg.id}"+ )+ if self._vikingdb.has_queue_manager:+ wait = self._circuit_breaker.retry_after+ if wait > 0:+ await asyncio.sleep(wait)+ await self._vikingdb.enqueue_embedding_msg(embedding_msg)+ self.report_success()+ return None+ # No queue manager — cannot re-enqueue, drop with error+ self.report_error("Circuit breaker open and no queue manager", data)+ return None+# Initialize embedder if not already initializedif not self._embedder:from openviking_cli.utils.config import get_openviking_config@@ -267,13 +290,23 @@ class TextEmbeddingHandler(DequeueHandlerBase):passexcept Exception as embed_err:error_msg = f"Failed to generate embedding: {embed_err}"- logger.error(error_msg)+ error_class = classify_api_error(embed_err)++ if error_class == "permanent":+ logger.critical(error_msg)+ self._circuit_breaker.record_failure(embed_err)+ self._merge_request_stats(embedding_msg.telemetry_id, error_count=1)+ self.report_error(error_msg, data)+ return None- if is_429_error(embed_err) and self._vikingdb.has_queue_manager:+ # Transient or unknown — re-enqueue for retry+ logger.warning(error_msg)+ self._circuit_breaker.record_failure(embed_err)+ if self._vikingdb.has_queue_manager:try:await self._vikingdb.enqueue_embedding_msg(embedding_msg)logger.info(- f"Re-enqueued embedding message after rate limit: {embedding_msg.id}"+ f"Re-enqueued embedding message after transient error: {embedding_msg.id}")self.report_success()return None@@ -356,6 +389,7 @@ class TextEmbeddingHandler(DequeueHandlerBase):self._merge_request_stats(embedding_msg.telemetry_id, processed=1)self.report_success()+ self._circuit_breaker.record_success()return inserted_dataexcept Exception as e:diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex 1af1a334c..dd5e284cf 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -28,6 +28,11 @@ from openviking.storage.queuefs.semantic_dag import DagStats, SemanticDagExecutofrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.viking_fs import get_viking_fsfrom openviking.telemetry import bind_telemetry, resolve_telemetry+from openviking.utils.circuit_breaker import (+ CircuitBreaker,+ CircuitBreakerOpen,+ classify_api_error,+)from openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils import VikingURIfrom openviking_cli.utils.config import get_openviking_config@@ -82,6 +87,7 @@ class SemanticProcessor(DequeueHandlerBase):self._dag_executor: Optional[SemanticDagExecutor] = Noneself._current_ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)self._current_msg: Optional[SemanticMsg] = None+ self._circuit_breaker = CircuitBreaker()@classmethoddef _cache_dag_stats(cls, telemetry_id: str, uri: str, stats: DagStats) -> None:@@ -204,6 +210,29 @@ class SemanticProcessor(DequeueHandlerBase):except Exception:return True+ async def _reenqueue_semantic_msg(self, msg: SemanticMsg) -> None:+ """Re-enqueue a semantic message for later processing.++ Throttles with a sleep when the circuit breaker is open to prevent+ re-enqueue storms (messages cycling at 5/sec during OPEN window).+ """+ import asyncio++ from openviking.storage.queuefs import get_queue_manager++ # Throttle to prevent re-enqueue storm during OPEN window+ wait = self._circuit_breaker.retry_after+ if wait > 0:+ await asyncio.sleep(wait)++ queue_manager = get_queue_manager()+ if queue_manager is not None:+ semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC)+ await semantic_queue.enqueue(msg)+ logger.info(f"Re-enqueued semantic message: {msg.uri}")+ else:+ logger.warning(f"No queue manager available, cannot re-enqueue: {msg.uri}")+async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:"""Process dequeued SemanticMsg, recursively process all subdirectories."""msg: Optional[SemanticMsg] = None@@ -219,6 +248,16 @@ class SemanticProcessor(DequeueHandlerBase):assert data is not Nonemsg = SemanticMsg.from_dict(data)+ # Circuit breaker: if API is known-broken, re-enqueue and wait+ try:+ self._circuit_breaker.check()+ except CircuitBreakerOpen:+ logger.warning(+ f"Circuit breaker is open, re-enqueueing semantic message: {msg.uri}"+ )+ await self._reenqueue_semantic_msg(msg)+ self.report_success()+ return Nonecollector = resolve_telemetry(msg.telemetry_id)telemetry_ctx = bind_telemetry(collector) if collector is not None else nullcontext()with telemetry_ctx:@@ -276,13 +315,38 @@ class SemanticProcessor(DequeueHandlerBase):self._merge_request_stats(msg.telemetry_id, processed=1)logger.info(f"Completed semantic generation for: {msg.uri}")self.report_success()+ self._circuit_breaker.record_success()return Noneexcept Exception as e:- logger.error(f"Failed to process semantic message: {e}", exc_info=True)- if msg is not None:- self._merge_request_stats(msg.telemetry_id, error_count=1)- self.report_error(str(e), data)+ error_class = classify_api_error(e)+ if error_class == "permanent":+ logger.critical(+ f"Permanent API error processing semantic message, dropping: {e}",+ exc_info=True,+ )+ self._circuit_breaker.record_failure(e)+ if msg is not None:+ self._merge_request_stats(msg.telemetry_id, error_count=1)+ self.report_error(str(e), data)+ else:+ # Transient or unknown — re-enqueue for retry+ logger.warning(+ f"Transient API error processing semantic message, re-enqueueing: {e}",+ exc_info=True,+ )+ self._circuit_breaker.record_failure(e)+ if msg is not None:+ try:+ await self._reenqueue_semantic_msg(msg)+ except Exception as requeue_err:+ logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")+ self._merge_request_stats(msg.telemetry_id, error_count=1)+ self.report_error(str(e), data)+ return None+ self.report_success()+ else:+ self.report_error(str(e), data)return Nonefinally:# Safety net: release lifecycle lock if still held (e.g. on exceptiondiff --git a/openviking/utils/circuit_breaker.py b/openviking/utils/circuit_breaker.pynew file mode 100644index 000000000..6f9fbdc26--- /dev/null+++ b/openviking/utils/circuit_breaker.py@@ -0,0 +1,149 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Circuit breaker and error classification for API call protection."""++from __future__ import annotations++import threading+import time++from openviking_cli.utils.logger import get_logger++logger = get_logger(__name__)++# --- Error classification ---++_PERMANENT_PATTERNS = ("403", "401", "Forbidden", "Unauthorized", "AccountOverdue")+_TRANSIENT_PATTERNS = (+ "429",+ "500",+ "502",+ "503",+ "504",+ "TooManyRequests",+ "RateLimit",+ "timeout",+ "Timeout",+ "ConnectionError",+ "Connection refused",+ "Connection reset",+)+++def classify_api_error(error: Exception) -> str:+ """Classify an API error as permanent, transient, or unknown.++ Checks both str(error) and str(error.__cause__) for known patterns.++ Returns:+ "permanent" — 403/401, never retry.+ "transient" — 429/5xx/timeout, safe to retry.+ "unknown" — unrecognized, treated as transient by callers.+ """+ texts = [str(error)]+ if error.__cause__ is not None:+ texts.append(str(error.__cause__))++ for text in texts:+ for pattern in _PERMANENT_PATTERNS:+ if pattern in text:+ return "permanent"++ for text in texts:+ for pattern in _TRANSIENT_PATTERNS:+ if pattern in text:+ return "transient"++ return "unknown"+++# --- Circuit breaker ---++_STATE_CLOSED = "CLOSED"+_STATE_OPEN = "OPEN"+_STATE_HALF_OPEN = "HALF_OPEN"+++class CircuitBreakerOpen(Exception):+ """Raised when the circuit breaker is open and blocking requests."""+++class CircuitBreaker:+ """Thread-safe circuit breaker for API call protection.++ Trips after ``failure_threshold`` consecutive failures (or immediately for+ permanent errors like 403/401). After ``reset_timeout`` seconds, allows one+ probe request (HALF_OPEN). If the probe succeeds, the breaker closes; if it+ fails, the breaker reopens.+ """++ def __init__(self, failure_threshold: int = 5, reset_timeout: float = 300):+ self._failure_threshold = failure_threshold+ self._reset_timeout = reset_timeout+ self._lock = threading.Lock()+ self._state = _STATE_CLOSED+ self._failure_count = 0+ self._last_failure_time: float = 0++ def check(self) -> None:+ """Allow the request through, or raise ``CircuitBreakerOpen``."""+ with self._lock:+ if self._state == _STATE_CLOSED:+ return+ if self._state == _STATE_HALF_OPEN:+ return # allow probe request+ # OPEN — check if timeout elapsed+ elapsed = time.monotonic() - self._last_failure_time+ if elapsed >= self._reset_timeout:+ self._state = _STATE_HALF_OPEN+ logger.info("Circuit breaker transitioning OPEN -> HALF_OPEN (timeout elapsed)")+ return+ raise CircuitBreakerOpen(+ f"Circuit breaker is OPEN, retry after {self._reset_timeout - elapsed:.0f}s"+ )++ @property+ def retry_after(self) -> float:+ """Seconds until the breaker may transition to HALF_OPEN, capped at 30s.++ Returns 0 if the breaker is CLOSED or HALF_OPEN.+ """+ with self._lock:+ if self._state != _STATE_OPEN:+ return 0+ remaining = self._reset_timeout - (time.monotonic() - self._last_failure_time)+ return min(max(remaining, 0), 30)++ def record_success(self) -> None:+ """Record a successful API call. Resets failure count."""+ with self._lock:+ if self._state == _STATE_HALF_OPEN:+ logger.info("Circuit breaker transitioning HALF_OPEN -> CLOSED (probe succeeded)")+ self._failure_count = 0+ self._state = _STATE_CLOSED++ def record_failure(self, error: Exception) -> None:+ """Record a failed API call. May trip the breaker."""+ error_class = classify_api_error(error)+ with self._lock:+ self._failure_count += 1+ self._last_failure_time = time.monotonic()++ if self._state == _STATE_HALF_OPEN:+ self._state = _STATE_OPEN+ logger.info(+ f"Circuit breaker transitioning HALF_OPEN -> OPEN (probe failed: {error})"+ )+ return++ if error_class == "permanent":+ self._state = _STATE_OPEN+ logger.info(f"Circuit breaker tripped immediately on permanent error: {error}")+ return++ if self._failure_count >= self._failure_threshold:+ self._state = _STATE_OPEN+ logger.info(+ f"Circuit breaker tripped after {self._failure_count} consecutive "+ f"failures: {error}"+ )diff --git a/tests/utils/test_circuit_breaker.py b/tests/utils/test_circuit_breaker.pynew file mode 100644index 000000000..0b4874059--- /dev/null+++ b/tests/utils/test_circuit_breaker.py@@ -0,0 +1,159 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0++import threading+import time++import pytest+++def test_circuit_breaker_starts_closed():+ from openviking.utils.circuit_breaker import CircuitBreaker++ cb = CircuitBreaker(failure_threshold=3, reset_timeout=10)+ cb.check() # should not raise+++def test_circuit_breaker_opens_after_threshold():+ from openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen++ cb = CircuitBreaker(failure_threshold=3, reset_timeout=10)+ for _ in range(3):+ cb.record_failure(RuntimeError("500 Internal Server Error"))+ with pytest.raises(CircuitBreakerOpen):+ cb.check()+++def test_circuit_breaker_resets_on_success():+ from openviking.utils.circuit_breaker import CircuitBreaker++ cb = CircuitBreaker(failure_threshold=3, reset_timeout=10)+ cb.record_failure(RuntimeError("timeout"))+ cb.record_failure(RuntimeError("timeout"))+ cb.record_success() # resets count+ cb.record_failure(RuntimeError("timeout"))+ cb.record_failure(RuntimeError("timeout"))+ cb.check() # should not raise — only 2 consecutive failures+++def test_circuit_breaker_half_open_after_timeout(monkeypatch):+ from openviking.utils.circuit_breaker import CircuitBreaker++ cb = CircuitBreaker(failure_threshold=1, reset_timeout=5)+ cb.record_failure(RuntimeError("500"))+ # Simulate time passing — capture original before patching to avoid recursion+ future = time.monotonic() + 6+ monkeypatch.setattr(time, "monotonic", lambda: future)+ cb.check() # should not raise — transitions to HALF_OPEN+++def test_circuit_breaker_half_open_success_closes():+ from openviking.utils.circuit_breaker import CircuitBreaker++ cb = CircuitBreaker(failure_threshold=1, reset_timeout=0)+ cb.record_failure(RuntimeError("500"))+ # reset_timeout=0 means immediate HALF_OPEN+ cb.check() # transitions to HALF_OPEN+ cb.record_success() # transitions to CLOSED+ cb.check() # should not raise+++def test_circuit_breaker_half_open_failure_reopens(monkeypatch):+ from openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen++ cb = CircuitBreaker(failure_threshold=1, reset_timeout=5)+ cb.record_failure(RuntimeError("500"))+ # Fast-forward past reset_timeout to reach HALF_OPEN+ future = time.monotonic() + 6+ monkeypatch.setattr(time, "monotonic", lambda: future)+ cb.check() # transitions to HALF_OPEN+ cb.record_failure(RuntimeError("500 again"))+ # Now the breaker is OPEN again, and last_failure_time is `future`,+ # so elapsed is 0 which is < reset_timeout(5) — should raise.+ with pytest.raises(CircuitBreakerOpen):+ cb.check()+++def test_permanent_error_trips_immediately():+ from openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen++ cb = CircuitBreaker(failure_threshold=10, reset_timeout=10)+ cb.record_failure(RuntimeError("403 Forbidden AccountOverdueError"))+ with pytest.raises(CircuitBreakerOpen):+ cb.check()+++def test_retry_after_returns_capped_value():+ from openviking.utils.circuit_breaker import CircuitBreaker++ cb = CircuitBreaker(failure_threshold=1, reset_timeout=300)+ cb.record_failure(RuntimeError("500"))+ # retry_after should be capped at 30+ assert 0 < cb.retry_after <= 30+++def test_retry_after_zero_when_closed():+ from openviking.utils.circuit_breaker import CircuitBreaker++ cb = CircuitBreaker()+ assert cb.retry_after == 0+++def test_thread_safety():+ from openviking.utils.circuit_breaker import CircuitBreaker++ cb = CircuitBreaker(failure_threshold=100, reset_timeout=300)+ errors = []++ def record_failures():+ try:+ for _ in range(50):+ cb.record_failure(RuntimeError("500"))+ except Exception as e:+ errors.append(e)++ threads = [threading.Thread(target=record_failures) for _ in range(4)]+ for t in threads:+ t.start()+ for t in threads:+ t.join()++ assert not errors+ assert cb._failure_count == 200+++def test_classify_permanent_errors():+ from openviking.utils.circuit_breaker import classify_api_error++ assert classify_api_error(RuntimeError("403 Forbidden")) == "permanent"+ assert classify_api_error(RuntimeError("AccountOverdueError: 403")) == "permanent"+ assert classify_api_error(RuntimeError("401 Unauthorized")) == "permanent"+ assert classify_api_error(RuntimeError("Forbidden")) == "permanent"+++def test_classify_transient_errors():+ from openviking.utils.circuit_breaker import classify_api_error++ assert classify_api_error(RuntimeError("429 TooManyRequests")) == "transient"+ assert classify_api_error(RuntimeError("RateLimitError")) == "transient"+ assert classify_api_error(RuntimeError("500 Internal Server Error")) == "transient"+ assert classify_api_error(RuntimeError("502 Bad Gateway")) == "transient"+ assert classify_api_error(RuntimeError("503 Service Unavailable")) == "transient"+ assert classify_api_error(RuntimeError("Connection timeout")) == "transient"+ assert classify_api_error(RuntimeError("ConnectionError: refused")) == "transient"+++def test_classify_unknown_errors():+ from openviking.utils.circuit_breaker import classify_api_error++ assert classify_api_error(RuntimeError("something unexpected")) == "unknown"+ assert classify_api_error(ValueError("bad value")) == "unknown"+++def test_classify_chained_exception():+ from openviking.utils.circuit_breaker import classify_api_error++ cause = RuntimeError("403 Forbidden")+ wrapper = RuntimeError("API call failed")+ wrapper.__cause__ = cause+ assert classify_api_error(wrapper) == "permanent"