Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(queue): expose re-enqueue counts in queue status (#1337)
Qin Haojied72c6c984821
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.pyindex 0447a784a..54b2d52f0 100644--- a/openviking/service/resource_service.py+++ b/openviking/service/resource_service.py@@ -502,6 +502,7 @@ class ResourceService:return {name: {"processed": s.processed,+ "requeue_count": getattr(s, "requeue_count", 0),"error_count": s.error_count,"errors": [{"message": e.message} for e in s.errors],}diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 7596c8336..d63f1dcc5 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -39,6 +39,7 @@ logger = get_logger(__name__)@dataclassclass RequestQueueStats:processed: int = 0+ requeue_count: int = 0error_count: int = 0@@ -207,13 +208,18 @@ class TextEmbeddingHandler(DequeueHandlerBase):@classmethoddef _merge_request_stats(- cls, telemetry_id: str, processed: int = 0, error_count: int = 0+ cls,+ telemetry_id: str,+ processed: int = 0,+ requeue_count: int = 0,+ error_count: int = 0,) -> None:if not telemetry_id:returnwith cls._request_stats_lock:stats = cls._request_stats_by_telemetry_id.setdefault(telemetry_id, RequestQueueStats())stats.processed += processed+ stats.requeue_count += requeue_countstats.error_count += error_countcls._request_stats_order.append(telemetry_id)if len(cls._request_stats_order) > cls._max_cached_stats:@@ -291,6 +297,14 @@ class TextEmbeddingHandler(DequeueHandlerBase):if wait > 0:await asyncio.sleep(wait)await self._vikingdb.enqueue_embedding_msg(embedding_msg)+ self._merge_request_stats(+ embedding_msg.telemetry_id,+ requeue_count=1,+ )+ get_request_wait_tracker().record_embedding_requeue(+ embedding_msg.telemetry_id+ )+ self.report_requeue()report_success = Truereturn None# No queue manager — cannot re-enqueue, drop with error@@ -343,6 +357,14 @@ class TextEmbeddingHandler(DequeueHandlerBase):if self._vikingdb.has_queue_manager:try:await self._vikingdb.enqueue_embedding_msg(embedding_msg)+ self._merge_request_stats(+ embedding_msg.telemetry_id,+ requeue_count=1,+ )+ get_request_wait_tracker().record_embedding_requeue(+ embedding_msg.telemetry_id+ )+ self.report_requeue()logger.info(f"Re-enqueued embedding message after transient error: {embedding_msg.id}")diff --git a/openviking/storage/observers/queue_observer.py b/openviking/storage/observers/queue_observer.pyindex 0afd840f8..9dfc74cfe 100644--- a/openviking/storage/observers/queue_observer.py+++ b/openviking/storage/observers/queue_observer.py@@ -59,6 +59,7 @@ class QueueObserver(BaseObserver):total_pending = 0total_in_progress = 0total_processed = 0+ total_requeues = 0total_errors = 0for queue_name, status in statuses.items():@@ -69,6 +70,7 @@ class QueueObserver(BaseObserver):"Pending": status.pending,"In Progress": status.in_progress,"Processed": status.processed,+ "Requeued": status.requeue_count,"Errors": status.error_count,"Total": total,}@@ -76,6 +78,7 @@ class QueueObserver(BaseObserver):total_pending += status.pendingtotal_in_progress += status.in_progresstotal_processed += status.processed+ total_requeues += status.requeue_counttotal_errors += status.error_countdata.append(@@ -84,6 +87,7 @@ class QueueObserver(BaseObserver):"Pending": getattr(dag_stats, "pending_nodes", 0) if dag_stats else 0,"In Progress": getattr(dag_stats, "in_progress_nodes", 0) if dag_stats else 0,"Processed": getattr(dag_stats, "done_nodes", 0) if dag_stats else 0,+ "Requeued": 0,"Errors": 0,"Total": getattr(dag_stats, "total_nodes", 0) if dag_stats else 0,}@@ -97,6 +101,7 @@ class QueueObserver(BaseObserver):"Pending": total_pending,"In Progress": total_in_progress,"Processed": total_processed,+ "Requeued": total_requeues,"Errors": total_errors,"Total": total_total,}diff --git a/openviking/storage/queuefs/named_queue.py b/openviking/storage/queuefs/named_queue.pyindex 8a7fd4f40..f672c440f 100644--- a/openviking/storage/queuefs/named_queue.py+++ b/openviking/storage/queuefs/named_queue.py@@ -31,6 +31,7 @@ class QueueStatus:pending: int = 0in_progress: int = 0processed: int = 0+ requeue_count: int = 0error_count: int = 0errors: List[QueueError] = field(default_factory=list)@@ -60,15 +61,18 @@ class DequeueHandlerBase(abc.ABC):"""Dequeue handler base class, supports callback mechanism to report processing results."""_success_callback: Optional[Callable[[], None]] = None+ _requeue_callback: Optional[Callable[[], None]] = None_error_callback: Optional[Callable[[str, Optional[Dict[str, Any]]], None]] = Nonedef set_callbacks(self,on_success: Callable[[], None],+ on_requeue: Callable[[], None],on_error: Callable[[str, Optional[Dict[str, Any]]], None],) -> None:"""Set callback functions."""self._success_callback = on_success+ self._requeue_callback = on_requeueself._error_callback = on_errordef report_success(self) -> None:@@ -76,6 +80,11 @@ class DequeueHandlerBase(abc.ABC):if self._success_callback:self._success_callback()+ def report_requeue(self) -> None:+ """Report that the current message was re-enqueued for later retry."""+ if self._requeue_callback:+ self._requeue_callback()+def report_error(self, error_msg: str, data: Optional[Dict[str, Any]] = None) -> None:"""Report processing error."""if self._error_callback:@@ -113,6 +122,7 @@ class NamedQueue:self._lock = threading.Lock()self._in_progress = 0self._processed = 0+ self._requeue_count = 0self._error_count = 0self._errors: List[QueueError] = []@@ -120,6 +130,7 @@ class NamedQueue:if self._dequeue_handler:self._dequeue_handler.set_callbacks(on_success=self._on_process_success,+ on_requeue=self._on_process_requeue,on_error=self._on_process_error,)@@ -134,6 +145,11 @@ class NamedQueue:self._in_progress -= 1self._processed += 1+ def _on_process_requeue(self) -> None:+ """Called when a dequeued message is re-enqueued for later retry."""+ with self._lock:+ self._requeue_count += 1+def _on_process_error(self, error_msg: str, data: Optional[Dict[str, Any]] = None) -> None:"""Called on processing failure."""with self._lock:@@ -157,6 +173,7 @@ class NamedQueue:pending=pending,in_progress=self._in_progress,processed=self._processed,+ requeue_count=self._requeue_count,error_count=self._error_count,errors=list(self._errors),)@@ -166,6 +183,7 @@ class NamedQueue:with self._lock:self._in_progress = 0self._processed = 0+ self._requeue_count = 0self._error_count = 0self._errors = []diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex d73e7e5c8..a7b5f6d01 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -55,6 +55,7 @@ class DiffResult:class RequestQueueStats:processed: int = 0+ requeue_count: int = 0error_count: int = 0@@ -124,6 +125,7 @@ class SemanticProcessor(DequeueHandlerBase):cls,telemetry_id: str,processed: int = 0,+ requeue_count: int = 0,error_count: int = 0,) -> None:if not telemetry_id:@@ -131,6 +133,7 @@ class SemanticProcessor(DequeueHandlerBase):with cls._stats_lock:stats = cls._request_stats_by_telemetry_id.setdefault(telemetry_id, RequestQueueStats())stats.processed += processed+ stats.requeue_count += requeue_countstats.error_count += error_countcls._request_stats_order.append(telemetry_id)if len(cls._request_stats_order) > cls._max_cached_stats:@@ -258,6 +261,9 @@ class SemanticProcessor(DequeueHandlerBase):f"Circuit breaker is open, re-enqueueing semantic message: {msg.uri}")await self._reenqueue_semantic_msg(msg)+ self._merge_request_stats(msg.telemetry_id, requeue_count=1)+ get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)+ self.report_requeue()self.report_success()return Nonecollector = resolve_telemetry(msg.telemetry_id)@@ -348,6 +354,9 @@ class SemanticProcessor(DequeueHandlerBase):if msg is not None:try:await self._reenqueue_semantic_msg(msg)+ self._merge_request_stats(msg.telemetry_id, requeue_count=1)+ get_request_wait_tracker().record_semantic_requeue(msg.telemetry_id)+ self.report_requeue()except Exception as requeue_err:logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")self._merge_request_stats(msg.telemetry_id, error_count=1)diff --git a/openviking/telemetry/operation.py b/openviking/telemetry/operation.pyindex 08fdd32f3..713aa0ba5 100644--- a/openviking/telemetry/operation.py+++ b/openviking/telemetry/operation.py@@ -169,10 +169,12 @@ class TelemetrySummaryBuilder:summary["queue"] = {"semantic": {"processed": cls._i(gauges.get("queue.semantic.processed"), 0),+ "requeue_count": cls._i(gauges.get("queue.semantic.requeue_count"), 0),"error_count": cls._i(gauges.get("queue.semantic.error_count"), 0),},"embedding": {"processed": cls._i(gauges.get("queue.embedding.processed"), 0),+ "requeue_count": cls._i(gauges.get("queue.embedding.requeue_count"), 0),"error_count": cls._i(gauges.get("queue.embedding.error_count"), 0),},}diff --git a/openviking/telemetry/request_wait_tracker.py b/openviking/telemetry/request_wait_tracker.pyindex 69a122797..b22fc1527 100644--- a/openviking/telemetry/request_wait_tracker.py+++ b/openviking/telemetry/request_wait_tracker.py@@ -16,9 +16,11 @@ class _RequestWaitState:pending_semantic_roots: Set[str] = field(default_factory=set)pending_embedding_roots: Set[str] = field(default_factory=set)semantic_processed: int = 0+ semantic_requeue_count: int = 0semantic_error_count: int = 0semantic_errors: List[str] = field(default_factory=list)embedding_processed: int = 0+ embedding_requeue_count: int = 0embedding_error_count: int = 0embedding_errors: List[str] = field(default_factory=list)created_at: float = field(default_factory=time.time)@@ -80,6 +82,15 @@ class RequestWaitTracker:returnstate.embedding_processed += max(delta, 0)+ def record_embedding_requeue(self, telemetry_id: str, delta: int = 1) -> None:+ if not telemetry_id:+ return+ with self._lock:+ state = self._states.get(telemetry_id)+ if state is None:+ return+ state.embedding_requeue_count += max(delta, 0)+def record_embedding_error(self, telemetry_id: str, message: str) -> None:if not telemetry_id:return@@ -106,6 +117,15 @@ class RequestWaitTracker:state.pending_semantic_roots.discard(semantic_msg_id)state.semantic_processed += max(processed_delta, 0)+ def record_semantic_requeue(self, telemetry_id: str, delta: int = 1) -> None:+ if not telemetry_id:+ return+ with self._lock:+ state = self._states.get(telemetry_id)+ if state is None:+ return+ state.semantic_requeue_count += max(delta, 0)+def mark_semantic_failed(self, telemetry_id: str, semantic_msg_id: str, message: str) -> None:if not telemetry_id:return@@ -176,11 +196,13 @@ class RequestWaitTracker:return {"Semantic": {"processed": state.semantic_processed,+ "requeue_count": state.semantic_requeue_count,"error_count": state.semantic_error_count,"errors": [{"message": msg} for msg in state.semantic_errors],},"Embedding": {"processed": state.embedding_processed,+ "requeue_count": state.embedding_requeue_count,"error_count": state.embedding_error_count,"errors": [{"message": msg} for msg in state.embedding_errors],},diff --git a/openviking/telemetry/resource_summary.py b/openviking/telemetry/resource_summary.pyindex a43a996f9..5f090f38d 100644--- a/openviking/telemetry/resource_summary.py+++ b/openviking/telemetry/resource_summary.py@@ -58,6 +58,7 @@ def build_queue_status_payload(status: Dict[str, Any]) -> Dict[str, Dict[str, Anreturn {name: {"processed": s.processed,+ "requeue_count": getattr(s, "requeue_count", 0),"error_count": s.error_count,"errors": [{"message": e.message} for e in s.errors],}@@ -73,17 +74,20 @@ def _resolve_queue_group(if explicit_stats is not None:return {"processed": explicit_stats.processed,+ "requeue_count": getattr(explicit_stats, "requeue_count", 0),"error_count": explicit_stats.error_count,}if fallback_status is None:- return {"processed": 0, "error_count": 0}+ return {"processed": 0, "requeue_count": 0, "error_count": 0}if isinstance(fallback_status, dict):return {"processed": int(fallback_status.get("processed", 0) or 0),+ "requeue_count": int(fallback_status.get("requeue_count", 0) or 0),"error_count": int(fallback_status.get("error_count", 0) or 0),}return {"processed": fallback_status.processed,+ "requeue_count": getattr(fallback_status, "requeue_count", 0),"error_count": fallback_status.error_count,}@@ -99,8 +103,8 @@ def record_resource_wait_metrics(telemetry = telemetry or get_current_telemetry()if not telemetry.enabled:return {- "semantic": {"processed": 0, "error_count": 0},- "embedding": {"processed": 0, "error_count": 0},+ "semantic": {"processed": 0, "requeue_count": 0, "error_count": 0},+ "embedding": {"processed": 0, "requeue_count": 0, "error_count": 0},}semantic = _resolve_queue_group(@@ -113,8 +117,10 @@ def record_resource_wait_metrics()telemetry.set("queue.semantic.processed", semantic["processed"])+ telemetry.set("queue.semantic.requeue_count", semantic["requeue_count"])telemetry.set("queue.semantic.error_count", semantic["error_count"])telemetry.set("queue.embedding.processed", embedding["processed"])+ telemetry.set("queue.embedding.requeue_count", embedding["requeue_count"])telemetry.set("queue.embedding.error_count", embedding["error_count"])dag_stats = _consume_semantic_dag_stats(telemetry_id, root_uri)diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex bc636e55d..8047a9d15 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -94,9 +94,10 @@ async def test_embedding_handler_skip_all_work_when_manager_is_closing(monkeypat)handler = TextEmbeddingHandler(_ClosingVikingDB())- status = {"success": 0, "error": 0}+ status = {"success": 0, "requeue": 0, "error": 0}handler.set_callbacks(on_success=lambda: status.__setitem__("success", status["success"] + 1),+ on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),on_error=lambda *_: status.__setitem__("error", status["error"] + 1),)@@ -105,6 +106,7 @@ async def test_embedding_handler_skip_all_work_when_manager_is_closing(monkeypatassert result is Noneassert embedder.calls == 0assert status["success"] == 1+ assert status["requeue"] == 0assert status["error"] == 0@@ -132,6 +134,12 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_w)handler = TextEmbeddingHandler(_QueueingVikingDB())+ status = {"success": 0, "requeue": 0, "error": 0}+ handler.set_callbacks(+ on_success=lambda: status.__setitem__("success", status["success"] + 1),+ on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),+ on_error=lambda *_: status.__setitem__("error", status["error"] + 1),+ )monkeypatch.setattr(handler._circuit_breaker,"check",@@ -151,6 +159,7 @@ async def test_embedding_handler_open_breaker_logs_summary_instead_of_per_item_wwarnings = [record.message for record in caplog.records if record.levelno == logging.WARNING]assert warnings.count("Embedding circuit breaker is open; re-enqueueing messages") == 1+ assert status == {"success": 2, "requeue": 2, "error": 0}@pytest.mark.asyncio@@ -173,9 +182,10 @@ async def test_embedding_handler_treats_shutdown_write_lock_as_success(monkeypatvikingdb = _ClosingDuringUpsertVikingDB()handler = TextEmbeddingHandler(vikingdb)- status = {"success": 0, "error": 0}+ status = {"success": 0, "requeue": 0, "error": 0}handler.set_callbacks(on_success=lambda: status.__setitem__("success", status["success"] + 1),+ on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),on_error=lambda *_: status.__setitem__("error", status["error"] + 1),)@@ -185,6 +195,7 @@ async def test_embedding_handler_treats_shutdown_write_lock_as_success(monkeypatassert vikingdb.calls == 1assert embedder.calls == 1assert status["success"] == 1+ assert status["requeue"] == 0assert status["error"] == 0@@ -249,9 +260,10 @@ async def test_embedding_handler_marks_success_only_after_tracker_completion(mon)handler = TextEmbeddingHandler(_CapturingVikingDB())- status = {"success": 0, "error": 0}+ status = {"success": 0, "requeue": 0, "error": 0}handler.set_callbacks(on_success=lambda: status.__setitem__("success", status["success"] + 1),+ on_requeue=lambda: status.__setitem__("requeue", status["requeue"] + 1),on_error=lambda *_: status.__setitem__("error", status["error"] + 1),)@@ -264,12 +276,14 @@ async def test_embedding_handler_marks_success_only_after_tracker_completion(monawait decrement_started.wait()assert status["success"] == 0+ assert status["requeue"] == 0assert status["error"] == 0allow_decrement_finish.set()await taskassert status["success"] == 1+ assert status["requeue"] == 0assert status["error"] == 0diff --git a/tests/telemetry/test_request_wait_tracker.py b/tests/telemetry/test_request_wait_tracker.pyindex 9e5850e1e..61640e6d6 100644--- a/tests/telemetry/test_request_wait_tracker.py+++ b/tests/telemetry/test_request_wait_tracker.py@@ -16,8 +16,8 @@ def test_request_wait_tracker_cleanup_prevents_state_recreation():tracker.mark_embedding_done(telemetry_id, "embedding-1")assert tracker.build_queue_status(telemetry_id) == {- "Semantic": {"processed": 0, "error_count": 0, "errors": []},- "Embedding": {"processed": 0, "error_count": 0, "errors": []},+ "Semantic": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},+ "Embedding": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},}@@ -33,6 +33,20 @@ def test_request_wait_tracker_cleanup_prevents_root_recreation():assert tracker.is_complete(telemetry_id) is Trueassert tracker.build_queue_status(telemetry_id) == {- "Semantic": {"processed": 0, "error_count": 0, "errors": []},- "Embedding": {"processed": 0, "error_count": 0, "errors": []},+ "Semantic": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},+ "Embedding": {"processed": 0, "requeue_count": 0, "error_count": 0, "errors": []},+ }+++def test_request_wait_tracker_records_requeues():+ tracker = RequestWaitTracker()+ telemetry_id = "tm_requeue"++ tracker.register_request(telemetry_id)+ tracker.record_semantic_requeue(telemetry_id)+ tracker.record_embedding_requeue(telemetry_id, delta=2)++ assert tracker.build_queue_status(telemetry_id) == {+ "Semantic": {"processed": 0, "requeue_count": 1, "error_count": 0, "errors": []},+ "Embedding": {"processed": 0, "requeue_count": 2, "error_count": 0, "errors": []},}diff --git a/tests/test_telemetry_runtime.py b/tests/test_telemetry_runtime.pyindex 7c6dbe7fb..967ce205a 100644--- a/tests/test_telemetry_runtime.py+++ b/tests/test_telemetry_runtime.py@@ -197,6 +197,11 @@ async def test_embedding_handler_binds_registered_operation_telemetry(monkeypatcself.embedding = SimpleNamespace(dimension=2,get_embedder=lambda: _TelemetryAwareEmbedder(),+ circuit_breaker=SimpleNamespace(+ failure_threshold=5,+ reset_timeout=300.0,+ max_reset_timeout=300.0,+ ),)class _DummyVikingDB:@@ -240,6 +245,20 @@ async def test_embedding_handler_binds_registered_operation_telemetry(monkeypatc@pytest.mark.asyncioasync def test_resource_service_add_resource_reports_queue_summary(monkeypatch):telemetry = MemoryOperationTelemetry(operation="resources.add_resource", enabled=True)+ queue_status = {+ "Semantic": {+ "processed": 2,+ "requeue_count": 0,+ "error_count": 1,+ "errors": [],+ },+ "Embedding": {+ "processed": 5,+ "requeue_count": 0,+ "error_count": 0,+ "errors": [],+ },+ }class _DummyProcessor:async def process_resource(self, **kwargs):@@ -248,16 +267,24 @@ async def test_resource_service_add_resource_reports_queue_summary(monkeypatch):"root_uri": "viking://resources/demo",}- class _DummyQueueManager:- async def wait_complete(self, timeout=None):- return {- "Semantic": SimpleNamespace(processed=2, error_count=1, errors=[]),- "Embedding": SimpleNamespace(processed=5, error_count=0, errors=[]),- }+ class _DummyRequestWaitTracker:+ def register_request(self, telemetry_id: str) -> None:+ del telemetry_id++ async def wait_for_request(self, telemetry_id: str, timeout=None) -> None:+ del telemetry_id, timeout++ def build_queue_status(self, telemetry_id: str):+ del telemetry_id+ return queue_status++ def cleanup(self, telemetry_id: str) -> None:+ del telemetry_idmonkeypatch.setattr(- "openviking.service.resource_service.get_queue_manager",- lambda: _DummyQueueManager(),+ "openviking.service.resource_service.get_request_wait_tracker",+ lambda: _DummyRequestWaitTracker(),+ raising=False,)class _DagStats: