Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(retrieve): add RetrievalObserver for retrieval quality metrics (#622)

Matt Van Hornc44e6147c1d3
diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.py
index b00087501..906795909 100644
--- a/openviking/retrieve/hierarchical_retriever.py
+++ b/openviking/retrieve/hierarchical_retriever.py
@@ -9,11 +9,13 @@ and rerank-based relevance scoring.
import heapq
import logging
+import time
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
from openviking.models.embedder.base import EmbedResult
from openviking.retrieve.memory_lifecycle import hotness_score
+from openviking.retrieve.retrieval_stats import get_stats_collector
from openviking.server.identity import RequestContext, Role
from openviking.storage import VikingDBManager, VikingDBManagerProxy
from openviking.storage.viking_fs import get_viking_fs
@@ -101,6 +103,8 @@ class HierarchicalRetriever:
scope_dsl: Additional scope constraints passed from public find/search filter
"""
+ t0 = time.monotonic()
+
# Use custom threshold or default threshold
effective_threshold = score_threshold if score_threshold is not None else self.threshold
@@ -193,9 +197,21 @@ class HierarchicalRetriever:
# Step 6: Convert results
matched = await self._convert_to_matched_contexts(candidates, ctx=ctx)
+ final = matched[:limit]
+
+ # Record retrieval stats for the observer.
+ elapsed_ms = (time.monotonic() - t0) * 1000
+ get_stats_collector().record_query(
+ context_type=query.context_type.value if query.context_type else "unknown",
+ result_count=len(final),
+ scores=[m.score for m in final],
+ latency_ms=elapsed_ms,
+ rerank_used=self._rerank_client is not None and mode == RetrieverMode.THINKING,
+ )
+
return QueryResult(
query=query,
- matched_contexts=matched[:limit],
+ matched_contexts=final,
searched_directories=root_uris,
)
diff --git a/openviking/retrieve/retrieval_stats.py b/openviking/retrieve/retrieval_stats.py
new file mode 100644
index 000000000..9c285dc21
--- /dev/null
+++ b/openviking/retrieve/retrieval_stats.py
@@ -0,0 +1,165 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Thread-safe retrieval statistics accumulator.
+
+Collects per-query metrics from the ``HierarchicalRetriever`` so that
+the ``RetrievalObserver`` can report aggregate health and quality data
+via the observer API.
+"""
+
+import threading
+from dataclasses import dataclass, field
+from typing import Dict
+
+
+@dataclass
+class RetrievalStats:
+ """Accumulated retrieval statistics.
+
+ All counters are monotonically increasing within a server lifetime.
+ The observer reads them to compute rates and averages.
+ """
+
+ total_queries: int = 0
+ total_results: int = 0
+ zero_result_queries: int = 0
+ total_score_sum: float = 0.0
+ max_score: float = 0.0
+ min_score: float = float("inf")
+ queries_by_type: Dict[str, int] = field(default_factory=dict)
+ rerank_used: int = 0
+ rerank_fallback: int = 0
+ total_latency_ms: float = 0.0
+ max_latency_ms: float = 0.0
+
+ @property
+ def avg_results_per_query(self) -> float:
+ if self.total_queries == 0:
+ return 0.0
+ return self.total_results / self.total_queries
+
+ @property
+ def zero_result_rate(self) -> float:
+ if self.total_queries == 0:
+ return 0.0
+ return self.zero_result_queries / self.total_queries
+
+ @property
+ def avg_score(self) -> float:
+ if self.total_results == 0:
+ return 0.0
+ return self.total_score_sum / self.total_results
+
+ @property
+ def avg_latency_ms(self) -> float:
+ if self.total_queries == 0:
+ return 0.0
+ return self.total_latency_ms / self.total_queries
+
+ def to_dict(self) -> dict:
+ """Serialize stats for API responses."""
+ return {
+ "total_queries": self.total_queries,
+ "total_results": self.total_results,
+ "zero_result_queries": self.zero_result_queries,
+ "zero_result_rate": round(self.zero_result_rate, 4),
+ "avg_results_per_query": round(self.avg_results_per_query, 2),
+ "avg_score": round(self.avg_score, 4),
+ "max_score": round(self.max_score, 4) if self.total_results > 0 else 0.0,
+ "min_score": round(self.min_score, 4) if self.total_results > 0 else 0.0,
+ "queries_by_type": dict(self.queries_by_type),
+ "rerank_used": self.rerank_used,
+ "rerank_fallback": self.rerank_fallback,
+ "avg_latency_ms": round(self.avg_latency_ms, 1),
+ "max_latency_ms": round(self.max_latency_ms, 1),
+ }
+
+
+class RetrievalStatsCollector:
+ """Thread-safe singleton that accumulates retrieval metrics.
+
+ Usage in the retriever::
+
+ from openviking.retrieve.retrieval_stats import get_stats_collector
+
+ collector = get_stats_collector()
+ collector.record_query(
+ context_type="memory",
+ result_count=3,
+ scores=[0.82, 0.71, 0.55],
+ latency_ms=42.5,
+ rerank_used=True,
+ )
+
+ Usage in the observer::
+
+ stats = get_stats_collector().snapshot()
+ """
+
+ def __init__(self) -> None:
+ self._lock = threading.Lock()
+ self._stats = RetrievalStats()
+
+ def record_query(
+ self,
+ context_type: str,
+ result_count: int,
+ scores: list[float],
+ latency_ms: float = 0.0,
+ rerank_used: bool = False,
+ rerank_fallback: bool = False,
+ ) -> None:
+ """Record metrics from a single retrieval query."""
+ with self._lock:
+ self._stats.total_queries += 1
+ self._stats.total_results += result_count
+
+ if result_count == 0:
+ self._stats.zero_result_queries += 1
+
+ for s in scores:
+ self._stats.total_score_sum += s
+ if s > self._stats.max_score:
+ self._stats.max_score = s
+ if s < self._stats.min_score:
+ self._stats.min_score = s
+
+ self._stats.queries_by_type[context_type] = (
+ self._stats.queries_by_type.get(context_type, 0) + 1
+ )
+
+ if rerank_used:
+ self._stats.rerank_used += 1
+ if rerank_fallback:
+ self._stats.rerank_fallback += 1
+
+ self._stats.total_latency_ms += latency_ms
+ if latency_ms > self._stats.max_latency_ms:
+ self._stats.max_latency_ms = latency_ms
+
+ def snapshot(self) -> RetrievalStats:
+ """Return a copy of the current stats."""
+ with self._lock:
+ import copy
+
+ return copy.deepcopy(self._stats)
+
+ def reset(self) -> None:
+ """Reset all counters (useful for testing)."""
+ with self._lock:
+ self._stats = RetrievalStats()
+
+
+# Module-level singleton.
+_collector: RetrievalStatsCollector | None = None
+_collector_lock = threading.Lock()
+
+
+def get_stats_collector() -> RetrievalStatsCollector:
+ """Return the global stats collector singleton."""
+ global _collector
+ if _collector is None:
+ with _collector_lock:
+ if _collector is None:
+ _collector = RetrievalStatsCollector()
+ return _collector
diff --git a/openviking/server/routers/observer.py b/openviking/server/routers/observer.py
index fee1c5058..4d214cbf8 100644
--- a/openviking/server/routers/observer.py
+++ b/openviking/server/routers/observer.py
@@ -82,6 +82,16 @@ async def observer_transaction(
return Response(status="ok", result=_component_to_dict(component))
[email protected]("/retrieval")
+async def observer_retrieval(
+ _ctx: RequestContext = Depends(get_request_context),
+):
+ """Get retrieval quality metrics."""
+ service = get_service()
+ component = service.debug.observer.retrieval
+ return Response(status="ok", result=_component_to_dict(component))
+
+
@router.get("/system")
async def observer_system(
_ctx: RequestContext = Depends(get_request_context),
diff --git a/openviking/service/debug_service.py b/openviking/service/debug_service.py
index 2e8253f98..9c3cf39b3 100644
--- a/openviking/service/debug_service.py
+++ b/openviking/service/debug_service.py
@@ -10,6 +10,7 @@ from typing import Dict, List, Optional
from openviking.storage import VikingDBManager
from openviking.storage.observers import (
QueueObserver,
+ RetrievalObserver,
TransactionObserver,
VikingDBObserver,
VLMObserver,
@@ -153,6 +154,17 @@ class ObserverService:
status=observer.get_status_table(),
)
+ @property
+ def retrieval(self) -> ComponentStatus:
+ """Get retrieval quality status."""
+ observer = RetrievalObserver()
+ return ComponentStatus(
+ name="retrieval",
+ is_healthy=observer.is_healthy(),
+ has_errors=observer.has_errors(),
+ status=observer.get_status_table(),
+ )
+
@property
def system(self) -> SystemStatus:
"""Get system overall status."""
@@ -161,6 +173,7 @@ class ObserverService:
"vikingdb": self.vikingdb,
"vlm": self.vlm,
"transaction": self.transaction,
+ "retrieval": self.retrieval,
}
errors = [f"{c.name} has errors" for c in components.values() if c.has_errors]
return SystemStatus(
diff --git a/openviking/storage/observers/__init__.py b/openviking/storage/observers/__init__.py
index a0d3ece55..4a36700a1 100644
--- a/openviking/storage/observers/__init__.py
+++ b/openviking/storage/observers/__init__.py
@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
from .base_observer import BaseObserver
from .queue_observer import QueueObserver
+from .retrieval_observer import RetrievalObserver
from .transaction_observer import TransactionObserver
from .vikingdb_observer import VikingDBObserver
from .vlm_observer import VLMObserver
@@ -9,6 +10,7 @@ from .vlm_observer import VLMObserver
__all__ = [
"BaseObserver",
"QueueObserver",
+ "RetrievalObserver",
"TransactionObserver",
"VikingDBObserver",
"VLMObserver",
diff --git a/openviking/storage/observers/retrieval_observer.py b/openviking/storage/observers/retrieval_observer.py
new file mode 100644
index 000000000..a3a79a74c
--- /dev/null
+++ b/openviking/storage/observers/retrieval_observer.py
@@ -0,0 +1,99 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""
+RetrievalObserver: Retrieval system observability tool.
+
+Provides methods to observe and report retrieval quality metrics
+accumulated by the HierarchicalRetriever.
+"""
+
+from openviking.storage.observers.base_observer import BaseObserver
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class RetrievalObserver(BaseObserver):
+ """
+ RetrievalObserver: System observability tool for retrieval quality.
+
+ Reads accumulated statistics from the global RetrievalStatsCollector
+ and formats them for display via the observer API.
+ """
+
+ # A zero-result rate above this threshold is considered unhealthy.
+ UNHEALTHY_ZERO_RESULT_RATE = 0.5
+
+ @staticmethod
+ def _get_collector():
+ """Lazy import to avoid circular dependency with storage module."""
+ from openviking.retrieve.retrieval_stats import get_stats_collector
+
+ return get_stats_collector()
+
+ def get_status_table(self) -> str:
+ """Format retrieval statistics as a string table."""
+ return self._format_status_as_table()
+
+ def _format_status_as_table(self) -> str:
+ """Format retrieval stats as a table using tabulate."""
+ from tabulate import tabulate
+
+ stats = self._get_collector().snapshot()
+
+ if stats.total_queries == 0:
+ return "No retrieval queries recorded."
+
+ summary = [
+ {"Metric": "Total Queries", "Value": stats.total_queries},
+ {"Metric": "Total Results", "Value": stats.total_results},
+ {"Metric": "Avg Results/Query", "Value": f"{stats.avg_results_per_query:.1f}"},
+ {"Metric": "Zero-Result Queries", "Value": stats.zero_result_queries},
+ {
+ "Metric": "Zero-Result Rate",
+ "Value": f"{stats.zero_result_rate:.1%}",
+ },
+ {"Metric": "Avg Score", "Value": f"{stats.avg_score:.4f}"},
+ {
+ "Metric": "Score Range",
+ "Value": f"{stats.min_score:.4f} - {stats.max_score:.4f}"
+ if stats.total_results > 0
+ else "N/A",
+ },
+ {"Metric": "Rerank Used", "Value": stats.rerank_used},
+ {"Metric": "Rerank Fallback", "Value": stats.rerank_fallback},
+ {"Metric": "Avg Latency (ms)", "Value": f"{stats.avg_latency_ms:.1f}"},
+ {"Metric": "Max Latency (ms)", "Value": f"{stats.max_latency_ms:.1f}"},
+ ]
+
+ lines = [tabulate(summary, headers="keys", tablefmt="pretty")]
+
+ # Query breakdown by context type
+ if stats.queries_by_type:
+ type_data = [
+ {"Context Type": ctype, "Queries": count}
+ for ctype, count in sorted(
+ stats.queries_by_type.items(), key=lambda x: x[1], reverse=True
+ )
+ ]
+ lines.append("")
+ lines.append(tabulate(type_data, headers="keys", tablefmt="pretty"))
+
+ return "\n".join(lines)
+
+ def __str__(self) -> str:
+ return self.get_status_table()
+
+ def is_healthy(self) -> bool:
+ """Retrieval is healthy when the zero-result rate is acceptable."""
+ stats = self._get_collector().snapshot()
+ if stats.total_queries == 0:
+ return True
+ return stats.zero_result_rate < self.UNHEALTHY_ZERO_RESULT_RATE
+
+ def has_errors(self) -> bool:
+ """Errors are flagged when too many queries return zero results."""
+ stats = self._get_collector().snapshot()
+ if stats.total_queries < 5:
+ return False
+ return stats.zero_result_rate >= self.UNHEALTHY_ZERO_RESULT_RATE
diff --git a/tests/unit/retrieve/test_retrieval_stats.py b/tests/unit/retrieve/test_retrieval_stats.py
new file mode 100644
index 000000000..e9a4af687
--- /dev/null
+++ b/tests/unit/retrieve/test_retrieval_stats.py
@@ -0,0 +1,175 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Unit tests for retrieval statistics and observer."""
+
+from openviking.retrieve.retrieval_stats import RetrievalStats, RetrievalStatsCollector
+from openviking.storage.observers.retrieval_observer import RetrievalObserver
+
+
+class TestRetrievalStats:
+ def test_defaults(self):
+ stats = RetrievalStats()
+ assert stats.total_queries == 0
+ assert stats.avg_results_per_query == 0.0
+ assert stats.zero_result_rate == 0.0
+ assert stats.avg_score == 0.0
+ assert stats.avg_latency_ms == 0.0
+
+ def test_to_dict_empty(self):
+ d = RetrievalStats().to_dict()
+ assert d["total_queries"] == 0
+ assert d["max_score"] == 0.0
+ assert d["min_score"] == 0.0
+
+
+class TestRetrievalStatsCollector:
+ def test_record_single_query(self):
+ collector = RetrievalStatsCollector()
+ collector.record_query(
+ context_type="memory",
+ result_count=3,
+ scores=[0.9, 0.7, 0.5],
+ latency_ms=42.0,
+ )
+ stats = collector.snapshot()
+ assert stats.total_queries == 1
+ assert stats.total_results == 3
+ assert stats.zero_result_queries == 0
+ assert stats.max_score == 0.9
+ assert stats.min_score == 0.5
+ assert stats.queries_by_type == {"memory": 1}
+ assert stats.avg_latency_ms == 42.0
+
+ def test_record_zero_result_query(self):
+ collector = RetrievalStatsCollector()
+ collector.record_query(
+ context_type="resource",
+ result_count=0,
+ scores=[],
+ latency_ms=10.0,
+ )
+ stats = collector.snapshot()
+ assert stats.total_queries == 1
+ assert stats.zero_result_queries == 1
+ assert stats.zero_result_rate == 1.0
+
+ def test_record_multiple_queries(self):
+ collector = RetrievalStatsCollector()
+ collector.record_query("memory", 2, [0.8, 0.6], latency_ms=30.0)
+ collector.record_query("resource", 1, [0.5], latency_ms=20.0)
+ collector.record_query("memory", 0, [], latency_ms=5.0)
+
+ stats = collector.snapshot()
+ assert stats.total_queries == 3
+ assert stats.total_results == 3
+ assert stats.zero_result_queries == 1
+ assert stats.queries_by_type == {"memory": 2, "resource": 1}
+ assert stats.avg_latency_ms == (30 + 20 + 5) / 3
+
+ def test_rerank_tracking(self):
+ collector = RetrievalStatsCollector()
+ collector.record_query("memory", 1, [0.9], rerank_used=True)
+ collector.record_query("memory", 1, [0.7], rerank_fallback=True)
+
+ stats = collector.snapshot()
+ assert stats.rerank_used == 1
+ assert stats.rerank_fallback == 1
+
+ def test_max_latency(self):
+ collector = RetrievalStatsCollector()
+ collector.record_query("memory", 1, [0.5], latency_ms=10.0)
+ collector.record_query("memory", 1, [0.5], latency_ms=100.0)
+ collector.record_query("memory", 1, [0.5], latency_ms=50.0)
+
+ stats = collector.snapshot()
+ assert stats.max_latency_ms == 100.0
+
+ def test_reset(self):
+ collector = RetrievalStatsCollector()
+ collector.record_query("memory", 3, [0.9, 0.7, 0.5])
+ collector.reset()
+ stats = collector.snapshot()
+ assert stats.total_queries == 0
+ assert stats.total_results == 0
+
+ def test_snapshot_is_copy(self):
+ collector = RetrievalStatsCollector()
+ collector.record_query("memory", 1, [0.9])
+ snap = collector.snapshot()
+ collector.record_query("memory", 1, [0.8])
+ assert snap.total_queries == 1
+
+ def test_to_dict(self):
+ collector = RetrievalStatsCollector()
+ collector.record_query("memory", 2, [0.9, 0.6], latency_ms=25.0)
+ d = collector.snapshot().to_dict()
+ assert d["total_queries"] == 1
+ assert d["total_results"] == 2
+ assert d["avg_results_per_query"] == 2.0
+ assert d["max_score"] == 0.9
+ assert d["min_score"] == 0.6
+ assert d["avg_latency_ms"] == 25.0
+
+
+class TestRetrievalObserver:
+ def _setup_collector(self):
+ """Replace the global collector with a fresh one for testing."""
+ import openviking.retrieve.retrieval_stats as mod
+
+ collector = RetrievalStatsCollector()
+ mod._collector = collector
+ return collector
+
+ def test_healthy_when_no_queries(self):
+ self._setup_collector()
+ observer = RetrievalObserver()
+ assert observer.is_healthy() is True
+ assert observer.has_errors() is False
+
+ def test_healthy_with_good_results(self):
+ collector = self._setup_collector()
+ for _ in range(10):
+ collector.record_query("memory", 3, [0.9, 0.7, 0.5])
+ observer = RetrievalObserver()
+ assert observer.is_healthy() is True
+ assert observer.has_errors() is False
+
+ def test_unhealthy_with_many_zero_results(self):
+ collector = self._setup_collector()
+ for _ in range(8):
+ collector.record_query("memory", 0, [])
+ for _ in range(2):
+ collector.record_query("memory", 1, [0.5])
+ observer = RetrievalObserver()
+ # 80% zero-result rate > 50% threshold
+ assert observer.is_healthy() is False
+ assert observer.has_errors() is True
+
+ def test_no_errors_below_min_queries(self):
+ collector = self._setup_collector()
+ # Only 3 queries (below the 5-query minimum for error flagging)
+ for _ in range(3):
+ collector.record_query("memory", 0, [])
+ observer = RetrievalObserver()
+ assert observer.has_errors() is False
+
+ def test_status_table_no_data(self):
+ self._setup_collector()
+ observer = RetrievalObserver()
+ table = observer.get_status_table()
+ assert "No retrieval queries recorded" in table
+
+ def test_status_table_with_data(self):
+ collector = self._setup_collector()
+ collector.record_query("memory", 2, [0.9, 0.7], latency_ms=30.0)
+ collector.record_query("resource", 1, [0.5], latency_ms=20.0)
+ observer = RetrievalObserver()
+ table = observer.get_status_table()
+ assert "Total Queries" in table
+ assert "memory" in table
+ assert "resource" in table
+
+ def test_str(self):
+ self._setup_collector()
+ observer = RetrievalObserver()
+ assert str(observer) == observer.get_status_table()