Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(retrieve): add RetrievalObserver for retrieval quality metrics (#622)
Matt Van Hornc44e6147c1d3
diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.pyindex b00087501..906795909 100644--- a/openviking/retrieve/hierarchical_retriever.py+++ b/openviking/retrieve/hierarchical_retriever.py@@ -9,11 +9,13 @@ and rerank-based relevance scoring.import heapqimport logging+import timefrom datetime import datetimefrom typing import Any, Dict, List, Optional, Tuplefrom openviking.models.embedder.base import EmbedResultfrom openviking.retrieve.memory_lifecycle import hotness_score+from openviking.retrieve.retrieval_stats import get_stats_collectorfrom openviking.server.identity import RequestContext, Rolefrom openviking.storage import VikingDBManager, VikingDBManagerProxyfrom openviking.storage.viking_fs import get_viking_fs@@ -101,6 +103,8 @@ class HierarchicalRetriever:scope_dsl: Additional scope constraints passed from public find/search filter"""+ t0 = time.monotonic()+# Use custom threshold or default thresholdeffective_threshold = score_threshold if score_threshold is not None else self.threshold@@ -193,9 +197,21 @@ class HierarchicalRetriever:# Step 6: Convert resultsmatched = await self._convert_to_matched_contexts(candidates, ctx=ctx)+ final = matched[:limit]++ # Record retrieval stats for the observer.+ elapsed_ms = (time.monotonic() - t0) * 1000+ get_stats_collector().record_query(+ context_type=query.context_type.value if query.context_type else "unknown",+ result_count=len(final),+ scores=[m.score for m in final],+ latency_ms=elapsed_ms,+ rerank_used=self._rerank_client is not None and mode == RetrieverMode.THINKING,+ )+return QueryResult(query=query,- matched_contexts=matched[:limit],+ matched_contexts=final,searched_directories=root_uris,)diff --git a/openviking/retrieve/retrieval_stats.py b/openviking/retrieve/retrieval_stats.pynew file mode 100644index 000000000..9c285dc21--- /dev/null+++ b/openviking/retrieve/retrieval_stats.py@@ -0,0 +1,165 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Thread-safe retrieval statistics accumulator.++Collects per-query metrics from the ``HierarchicalRetriever`` so that+the ``RetrievalObserver`` can report aggregate health and quality data+via the observer API.+"""++import threading+from dataclasses import dataclass, field+from typing import Dict+++@dataclass+class RetrievalStats:+ """Accumulated retrieval statistics.++ All counters are monotonically increasing within a server lifetime.+ The observer reads them to compute rates and averages.+ """++ total_queries: int = 0+ total_results: int = 0+ zero_result_queries: int = 0+ total_score_sum: float = 0.0+ max_score: float = 0.0+ min_score: float = float("inf")+ queries_by_type: Dict[str, int] = field(default_factory=dict)+ rerank_used: int = 0+ rerank_fallback: int = 0+ total_latency_ms: float = 0.0+ max_latency_ms: float = 0.0++ @property+ def avg_results_per_query(self) -> float:+ if self.total_queries == 0:+ return 0.0+ return self.total_results / self.total_queries++ @property+ def zero_result_rate(self) -> float:+ if self.total_queries == 0:+ return 0.0+ return self.zero_result_queries / self.total_queries++ @property+ def avg_score(self) -> float:+ if self.total_results == 0:+ return 0.0+ return self.total_score_sum / self.total_results++ @property+ def avg_latency_ms(self) -> float:+ if self.total_queries == 0:+ return 0.0+ return self.total_latency_ms / self.total_queries++ def to_dict(self) -> dict:+ """Serialize stats for API responses."""+ return {+ "total_queries": self.total_queries,+ "total_results": self.total_results,+ "zero_result_queries": self.zero_result_queries,+ "zero_result_rate": round(self.zero_result_rate, 4),+ "avg_results_per_query": round(self.avg_results_per_query, 2),+ "avg_score": round(self.avg_score, 4),+ "max_score": round(self.max_score, 4) if self.total_results > 0 else 0.0,+ "min_score": round(self.min_score, 4) if self.total_results > 0 else 0.0,+ "queries_by_type": dict(self.queries_by_type),+ "rerank_used": self.rerank_used,+ "rerank_fallback": self.rerank_fallback,+ "avg_latency_ms": round(self.avg_latency_ms, 1),+ "max_latency_ms": round(self.max_latency_ms, 1),+ }+++class RetrievalStatsCollector:+ """Thread-safe singleton that accumulates retrieval metrics.++ Usage in the retriever::++ from openviking.retrieve.retrieval_stats import get_stats_collector++ collector = get_stats_collector()+ collector.record_query(+ context_type="memory",+ result_count=3,+ scores=[0.82, 0.71, 0.55],+ latency_ms=42.5,+ rerank_used=True,+ )++ Usage in the observer::++ stats = get_stats_collector().snapshot()+ """++ def __init__(self) -> None:+ self._lock = threading.Lock()+ self._stats = RetrievalStats()++ def record_query(+ self,+ context_type: str,+ result_count: int,+ scores: list[float],+ latency_ms: float = 0.0,+ rerank_used: bool = False,+ rerank_fallback: bool = False,+ ) -> None:+ """Record metrics from a single retrieval query."""+ with self._lock:+ self._stats.total_queries += 1+ self._stats.total_results += result_count++ if result_count == 0:+ self._stats.zero_result_queries += 1++ for s in scores:+ self._stats.total_score_sum += s+ if s > self._stats.max_score:+ self._stats.max_score = s+ if s < self._stats.min_score:+ self._stats.min_score = s++ self._stats.queries_by_type[context_type] = (+ self._stats.queries_by_type.get(context_type, 0) + 1+ )++ if rerank_used:+ self._stats.rerank_used += 1+ if rerank_fallback:+ self._stats.rerank_fallback += 1++ self._stats.total_latency_ms += latency_ms+ if latency_ms > self._stats.max_latency_ms:+ self._stats.max_latency_ms = latency_ms++ def snapshot(self) -> RetrievalStats:+ """Return a copy of the current stats."""+ with self._lock:+ import copy++ return copy.deepcopy(self._stats)++ def reset(self) -> None:+ """Reset all counters (useful for testing)."""+ with self._lock:+ self._stats = RetrievalStats()+++# Module-level singleton.+_collector: RetrievalStatsCollector | None = None+_collector_lock = threading.Lock()+++def get_stats_collector() -> RetrievalStatsCollector:+ """Return the global stats collector singleton."""+ global _collector+ if _collector is None:+ with _collector_lock:+ if _collector is None:+ _collector = RetrievalStatsCollector()+ return _collectordiff --git a/openviking/server/routers/observer.py b/openviking/server/routers/observer.pyindex fee1c5058..4d214cbf8 100644--- a/openviking/server/routers/observer.py+++ b/openviking/server/routers/observer.py@@ -82,6 +82,16 @@ async def observer_transaction(return Response(status="ok", result=_component_to_dict(component))[email protected]("/retrieval")+async def observer_retrieval(+ _ctx: RequestContext = Depends(get_request_context),+):+ """Get retrieval quality metrics."""+ service = get_service()+ component = service.debug.observer.retrieval+ return Response(status="ok", result=_component_to_dict(component))++@router.get("/system")async def observer_system(_ctx: RequestContext = Depends(get_request_context),diff --git a/openviking/service/debug_service.py b/openviking/service/debug_service.pyindex 2e8253f98..9c3cf39b3 100644--- a/openviking/service/debug_service.py+++ b/openviking/service/debug_service.py@@ -10,6 +10,7 @@ from typing import Dict, List, Optionalfrom openviking.storage import VikingDBManagerfrom openviking.storage.observers import (QueueObserver,+ RetrievalObserver,TransactionObserver,VikingDBObserver,VLMObserver,@@ -153,6 +154,17 @@ class ObserverService:status=observer.get_status_table(),)+ @property+ def retrieval(self) -> ComponentStatus:+ """Get retrieval quality status."""+ observer = RetrievalObserver()+ return ComponentStatus(+ name="retrieval",+ is_healthy=observer.is_healthy(),+ has_errors=observer.has_errors(),+ status=observer.get_status_table(),+ )+@propertydef system(self) -> SystemStatus:"""Get system overall status."""@@ -161,6 +173,7 @@ class ObserverService:"vikingdb": self.vikingdb,"vlm": self.vlm,"transaction": self.transaction,+ "retrieval": self.retrieval,}errors = [f"{c.name} has errors" for c in components.values() if c.has_errors]return SystemStatus(diff --git a/openviking/storage/observers/__init__.py b/openviking/storage/observers/__init__.pyindex a0d3ece55..4a36700a1 100644--- a/openviking/storage/observers/__init__.py+++ b/openviking/storage/observers/__init__.py@@ -2,6 +2,7 @@# SPDX-License-Identifier: Apache-2.0from .base_observer import BaseObserverfrom .queue_observer import QueueObserver+from .retrieval_observer import RetrievalObserverfrom .transaction_observer import TransactionObserverfrom .vikingdb_observer import VikingDBObserverfrom .vlm_observer import VLMObserver@@ -9,6 +10,7 @@ from .vlm_observer import VLMObserver__all__ = ["BaseObserver","QueueObserver",+ "RetrievalObserver","TransactionObserver","VikingDBObserver","VLMObserver",diff --git a/openviking/storage/observers/retrieval_observer.py b/openviking/storage/observers/retrieval_observer.pynew file mode 100644index 000000000..a3a79a74c--- /dev/null+++ b/openviking/storage/observers/retrieval_observer.py@@ -0,0 +1,99 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""+RetrievalObserver: Retrieval system observability tool.++Provides methods to observe and report retrieval quality metrics+accumulated by the HierarchicalRetriever.+"""++from openviking.storage.observers.base_observer import BaseObserver+from openviking_cli.utils.logger import get_logger++logger = get_logger(__name__)+++class RetrievalObserver(BaseObserver):+ """+ RetrievalObserver: System observability tool for retrieval quality.++ Reads accumulated statistics from the global RetrievalStatsCollector+ and formats them for display via the observer API.+ """++ # A zero-result rate above this threshold is considered unhealthy.+ UNHEALTHY_ZERO_RESULT_RATE = 0.5++ @staticmethod+ def _get_collector():+ """Lazy import to avoid circular dependency with storage module."""+ from openviking.retrieve.retrieval_stats import get_stats_collector++ return get_stats_collector()++ def get_status_table(self) -> str:+ """Format retrieval statistics as a string table."""+ return self._format_status_as_table()++ def _format_status_as_table(self) -> str:+ """Format retrieval stats as a table using tabulate."""+ from tabulate import tabulate++ stats = self._get_collector().snapshot()++ if stats.total_queries == 0:+ return "No retrieval queries recorded."++ summary = [+ {"Metric": "Total Queries", "Value": stats.total_queries},+ {"Metric": "Total Results", "Value": stats.total_results},+ {"Metric": "Avg Results/Query", "Value": f"{stats.avg_results_per_query:.1f}"},+ {"Metric": "Zero-Result Queries", "Value": stats.zero_result_queries},+ {+ "Metric": "Zero-Result Rate",+ "Value": f"{stats.zero_result_rate:.1%}",+ },+ {"Metric": "Avg Score", "Value": f"{stats.avg_score:.4f}"},+ {+ "Metric": "Score Range",+ "Value": f"{stats.min_score:.4f} - {stats.max_score:.4f}"+ if stats.total_results > 0+ else "N/A",+ },+ {"Metric": "Rerank Used", "Value": stats.rerank_used},+ {"Metric": "Rerank Fallback", "Value": stats.rerank_fallback},+ {"Metric": "Avg Latency (ms)", "Value": f"{stats.avg_latency_ms:.1f}"},+ {"Metric": "Max Latency (ms)", "Value": f"{stats.max_latency_ms:.1f}"},+ ]++ lines = [tabulate(summary, headers="keys", tablefmt="pretty")]++ # Query breakdown by context type+ if stats.queries_by_type:+ type_data = [+ {"Context Type": ctype, "Queries": count}+ for ctype, count in sorted(+ stats.queries_by_type.items(), key=lambda x: x[1], reverse=True+ )+ ]+ lines.append("")+ lines.append(tabulate(type_data, headers="keys", tablefmt="pretty"))++ return "\n".join(lines)++ def __str__(self) -> str:+ return self.get_status_table()++ def is_healthy(self) -> bool:+ """Retrieval is healthy when the zero-result rate is acceptable."""+ stats = self._get_collector().snapshot()+ if stats.total_queries == 0:+ return True+ return stats.zero_result_rate < self.UNHEALTHY_ZERO_RESULT_RATE++ def has_errors(self) -> bool:+ """Errors are flagged when too many queries return zero results."""+ stats = self._get_collector().snapshot()+ if stats.total_queries < 5:+ return False+ return stats.zero_result_rate >= self.UNHEALTHY_ZERO_RESULT_RATEdiff --git a/tests/unit/retrieve/test_retrieval_stats.py b/tests/unit/retrieve/test_retrieval_stats.pynew file mode 100644index 000000000..e9a4af687--- /dev/null+++ b/tests/unit/retrieve/test_retrieval_stats.py@@ -0,0 +1,175 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Unit tests for retrieval statistics and observer."""++from openviking.retrieve.retrieval_stats import RetrievalStats, RetrievalStatsCollector+from openviking.storage.observers.retrieval_observer import RetrievalObserver+++class TestRetrievalStats:+ def test_defaults(self):+ stats = RetrievalStats()+ assert stats.total_queries == 0+ assert stats.avg_results_per_query == 0.0+ assert stats.zero_result_rate == 0.0+ assert stats.avg_score == 0.0+ assert stats.avg_latency_ms == 0.0++ def test_to_dict_empty(self):+ d = RetrievalStats().to_dict()+ assert d["total_queries"] == 0+ assert d["max_score"] == 0.0+ assert d["min_score"] == 0.0+++class TestRetrievalStatsCollector:+ def test_record_single_query(self):+ collector = RetrievalStatsCollector()+ collector.record_query(+ context_type="memory",+ result_count=3,+ scores=[0.9, 0.7, 0.5],+ latency_ms=42.0,+ )+ stats = collector.snapshot()+ assert stats.total_queries == 1+ assert stats.total_results == 3+ assert stats.zero_result_queries == 0+ assert stats.max_score == 0.9+ assert stats.min_score == 0.5+ assert stats.queries_by_type == {"memory": 1}+ assert stats.avg_latency_ms == 42.0++ def test_record_zero_result_query(self):+ collector = RetrievalStatsCollector()+ collector.record_query(+ context_type="resource",+ result_count=0,+ scores=[],+ latency_ms=10.0,+ )+ stats = collector.snapshot()+ assert stats.total_queries == 1+ assert stats.zero_result_queries == 1+ assert stats.zero_result_rate == 1.0++ def test_record_multiple_queries(self):+ collector = RetrievalStatsCollector()+ collector.record_query("memory", 2, [0.8, 0.6], latency_ms=30.0)+ collector.record_query("resource", 1, [0.5], latency_ms=20.0)+ collector.record_query("memory", 0, [], latency_ms=5.0)++ stats = collector.snapshot()+ assert stats.total_queries == 3+ assert stats.total_results == 3+ assert stats.zero_result_queries == 1+ assert stats.queries_by_type == {"memory": 2, "resource": 1}+ assert stats.avg_latency_ms == (30 + 20 + 5) / 3++ def test_rerank_tracking(self):+ collector = RetrievalStatsCollector()+ collector.record_query("memory", 1, [0.9], rerank_used=True)+ collector.record_query("memory", 1, [0.7], rerank_fallback=True)++ stats = collector.snapshot()+ assert stats.rerank_used == 1+ assert stats.rerank_fallback == 1++ def test_max_latency(self):+ collector = RetrievalStatsCollector()+ collector.record_query("memory", 1, [0.5], latency_ms=10.0)+ collector.record_query("memory", 1, [0.5], latency_ms=100.0)+ collector.record_query("memory", 1, [0.5], latency_ms=50.0)++ stats = collector.snapshot()+ assert stats.max_latency_ms == 100.0++ def test_reset(self):+ collector = RetrievalStatsCollector()+ collector.record_query("memory", 3, [0.9, 0.7, 0.5])+ collector.reset()+ stats = collector.snapshot()+ assert stats.total_queries == 0+ assert stats.total_results == 0++ def test_snapshot_is_copy(self):+ collector = RetrievalStatsCollector()+ collector.record_query("memory", 1, [0.9])+ snap = collector.snapshot()+ collector.record_query("memory", 1, [0.8])+ assert snap.total_queries == 1++ def test_to_dict(self):+ collector = RetrievalStatsCollector()+ collector.record_query("memory", 2, [0.9, 0.6], latency_ms=25.0)+ d = collector.snapshot().to_dict()+ assert d["total_queries"] == 1+ assert d["total_results"] == 2+ assert d["avg_results_per_query"] == 2.0+ assert d["max_score"] == 0.9+ assert d["min_score"] == 0.6+ assert d["avg_latency_ms"] == 25.0+++class TestRetrievalObserver:+ def _setup_collector(self):+ """Replace the global collector with a fresh one for testing."""+ import openviking.retrieve.retrieval_stats as mod++ collector = RetrievalStatsCollector()+ mod._collector = collector+ return collector++ def test_healthy_when_no_queries(self):+ self._setup_collector()+ observer = RetrievalObserver()+ assert observer.is_healthy() is True+ assert observer.has_errors() is False++ def test_healthy_with_good_results(self):+ collector = self._setup_collector()+ for _ in range(10):+ collector.record_query("memory", 3, [0.9, 0.7, 0.5])+ observer = RetrievalObserver()+ assert observer.is_healthy() is True+ assert observer.has_errors() is False++ def test_unhealthy_with_many_zero_results(self):+ collector = self._setup_collector()+ for _ in range(8):+ collector.record_query("memory", 0, [])+ for _ in range(2):+ collector.record_query("memory", 1, [0.5])+ observer = RetrievalObserver()+ # 80% zero-result rate > 50% threshold+ assert observer.is_healthy() is False+ assert observer.has_errors() is True++ def test_no_errors_below_min_queries(self):+ collector = self._setup_collector()+ # Only 3 queries (below the 5-query minimum for error flagging)+ for _ in range(3):+ collector.record_query("memory", 0, [])+ observer = RetrievalObserver()+ assert observer.has_errors() is False++ def test_status_table_no_data(self):+ self._setup_collector()+ observer = RetrievalObserver()+ table = observer.get_status_table()+ assert "No retrieval queries recorded" in table++ def test_status_table_with_data(self):+ collector = self._setup_collector()+ collector.record_query("memory", 2, [0.9, 0.7], latency_ms=30.0)+ collector.record_query("resource", 1, [0.5], latency_ms=20.0)+ observer = RetrievalObserver()+ table = observer.get_status_table()+ assert "Total Queries" in table+ assert "memory" in table+ assert "resource" in table++ def test_str(self):+ self._setup_collector()+ observer = RetrievalObserver()+ assert str(observer) == observer.get_status_table()