Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(observer): treat empty retrievals as healthy (#3975)
Qin Haojieec18c1dcd841
diff --git a/crates/ov_cli/src/status_ui.rs b/crates/ov_cli/src/status_ui.rsindex 5bc21f553..8b4ecd22b 100644--- a/crates/ov_cli/src/status_ui.rs+++ b/crates/ov_cli/src/status_ui.rs@@ -646,11 +646,9 @@ fn retrieval_summary(status: Option<&str>) -> String {return "unknown".to_string();};let queries = metric_value(status, "Total Queries");- let zero_rate = metric_value(status, "Zero-Result Rate");- match (queries, zero_rate) {- (Some(queries), Some(zero_rate)) => {- format!("{queries} queries, {zero_rate} zero-result rate")- }+ let errors = metric_value(status, "Total Errors");+ match (queries, errors) {+ (Some(queries), Some(errors)) => format!("{queries} queries, {errors} errors"),_ => "unknown".to_string(),}}@@ -820,12 +818,12 @@ mod tests {"is_healthy": true,"has_errors": false,"status": "\- +---------------------+-----------------+\n\- | Metric | Value |\n\- +---------------------+-----------------+\n\- | Total Queries | 121 |\n\- | Zero-Result Rate | 28.9% |\n\- +---------------------+-----------------+"+ +---------------+-------+\n\+ | Metric | Value |\n\+ +---------------+-------+\n\+ | Total Queries | 121 |\n\+ | Total Errors | 0 |\n\+ +---------------+-------+"},"filesystem": {"name": "filesystem",diff --git a/openviking/retrieve/retrieval_stats.py b/openviking/retrieve/retrieval_stats.pyindex 5785285bd..2e8d90809 100644--- a/openviking/retrieve/retrieval_stats.py+++ b/openviking/retrieve/retrieval_stats.py@@ -2,9 +2,8 @@# SPDX-License-Identifier: AGPL-3.0"""Thread-safe retrieval statistics accumulator.-Collects per-query metrics from the ``HierarchicalRetriever`` so that-the ``RetrievalObserver`` can report aggregate health and quality data-via the observer API.+Collects per-query metrics from the ``HierarchicalRetriever`` and retrieval+errors from the service boundary for the observer API."""import threading@@ -22,7 +21,8 @@ class RetrievalStats:total_queries: int = 0total_results: int = 0- zero_result_queries: int = 0+ total_errors: int = 0+ last_error: str | None = Nonetotal_score_sum: float = 0.0max_score: float = 0.0min_score: float = float("inf")@@ -38,12 +38,6 @@ class RetrievalStats:return 0.0return self.total_results / self.total_queries- @property- def zero_result_rate(self) -> float:- if self.total_queries == 0:- return 0.0- return self.zero_result_queries / self.total_queries-@propertydef avg_score(self) -> float:if self.total_results == 0:@@ -61,8 +55,8 @@ class RetrievalStats:return {"total_queries": self.total_queries,"total_results": self.total_results,- "zero_result_queries": self.zero_result_queries,- "zero_result_rate": round(self.zero_result_rate, 4),+ "total_errors": self.total_errors,+ "last_error": self.last_error,"avg_results_per_query": round(self.avg_results_per_query, 2),"avg_score": round(self.avg_score, 4),"max_score": round(self.max_score, 4) if self.total_results > 0 else 0.0,@@ -114,9 +108,6 @@ class RetrievalStatsCollector:self._stats.total_queries += 1self._stats.total_results += result_count- if result_count == 0:- self._stats.zero_result_queries += 1-for s in scores:self._stats.total_score_sum += sif s > self._stats.max_score:@@ -150,6 +141,12 @@ class RetrievalStatsCollector:except Exception:pass+ def record_error(self, error: Exception) -> None:+ """Record a retrieval execution error observed by the service layer."""+ with self._lock:+ self._stats.total_errors += 1+ self._stats.last_error = f"{type(error).__name__}: {error}"[:200]+def snapshot(self) -> RetrievalStats:"""Return a copy of the current stats."""with self._lock:diff --git a/openviking/service/search_service.py b/openviking/service/search_service.pyindex 6a6259d09..7f4f61b99 100644--- a/openviking/service/search_service.py+++ b/openviking/service/search_service.py@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Unionfrom openviking.core.path_variables import resolve_path_variablesfrom openviking.core.uri_validation import validate_optional_viking_uris+from openviking.retrieve.retrieval_stats import get_stats_collectorfrom openviking.server.identity import RequestContextfrom openviking.storage.viking_fs import VikingFSfrom openviking.utils.image_search import (@@ -18,7 +19,11 @@ from openviking.utils.image_search import (is_http_url,is_viking_uri,)-from openviking_cli.exceptions import InvalidArgumentError, NotInitializedError+from openviking_cli.exceptions import (+ InvalidArgumentError,+ NotInitializedError,+ PermissionDeniedError,+)from openviking_cli.utils import get_loggerif TYPE_CHECKING:@@ -113,17 +118,23 @@ class SearchService:if session is not None and self.is_intent_enabled() and not resolved_image_url:session_info = await session.get_context_for_search(query)- result = await viking_fs.search(- query=query,- ctx=ctx,- target_uri=target_uri,- session_info=session_info,- limit=limit,- score_threshold=score_threshold,- filter=filter,- level=level,- image_url=resolved_image_url,- )+ try:+ result = await viking_fs.search(+ query=query,+ ctx=ctx,+ target_uri=target_uri,+ session_info=session_info,+ limit=limit,+ score_threshold=score_threshold,+ filter=filter,+ level=level,+ image_url=resolved_image_url,+ )+ except (InvalidArgumentError, PermissionDeniedError):+ raise+ except Exception as exc:+ get_stats_collector().record_error(exc)+ raisereturn resultasync def find(@@ -154,14 +165,20 @@ class SearchService:_ensure_non_empty_query(query, resolved_image_url)target_uri = validate_optional_viking_uris(target_uri, field_name="target_uri")viking_fs = self._ensure_initialized()- result = await viking_fs.find(- query=query,- ctx=ctx,- target_uri=target_uri,- limit=limit,- score_threshold=score_threshold,- filter=filter,- level=level,- image_url=resolved_image_url,- )+ try:+ result = await viking_fs.find(+ query=query,+ ctx=ctx,+ target_uri=target_uri,+ limit=limit,+ score_threshold=score_threshold,+ filter=filter,+ level=level,+ image_url=resolved_image_url,+ )+ except (InvalidArgumentError, PermissionDeniedError):+ raise+ except Exception as exc:+ get_stats_collector().record_error(exc)+ raisereturn resultdiff --git a/openviking/storage/observers/retrieval_observer.py b/openviking/storage/observers/retrieval_observer.pyindex 597e19a1e..df4bb233a 100644--- a/openviking/storage/observers/retrieval_observer.py+++ b/openviking/storage/observers/retrieval_observer.py@@ -3,8 +3,7 @@"""RetrievalObserver: Retrieval system observability tool.-Provides methods to observe and report retrieval quality metrics-accumulated by the HierarchicalRetriever.+Provides retrieval diagnostics and reports actual execution errors."""from openviking.storage.observers.base_observer import BaseObserver@@ -17,16 +16,10 @@ class RetrievalObserver(BaseObserver):"""RetrievalObserver: System observability tool for retrieval quality.- Reads accumulated statistics from the global RetrievalStatsCollector- and formats them for display via the observer API.+ Empty retrievals are valid outcomes. Health reflects execution errors,+ while result counts and scores remain diagnostics only."""- # A zero-result rate at or above this threshold is suspicious.- UNHEALTHY_ZERO_RESULT_RATE = 0.5- # Sparse fan-out may include expected empty branches. Treat it as unhealthy- # only when the aggregate workload returns fewer than one result per query.- MIN_HEALTHY_AVG_RESULTS_PER_QUERY = 1.0-@staticmethoddef _get_collector():"""Lazy import to avoid circular dependency with storage module."""@@ -44,18 +37,14 @@ class RetrievalObserver(BaseObserver):stats = self._get_collector().snapshot()- if stats.total_queries == 0:+ if stats.total_queries == 0 and stats.total_errors == 0:return "No retrieval queries recorded."summary = [{"Metric": "Total Queries", "Value": stats.total_queries},{"Metric": "Total Results", "Value": stats.total_results},{"Metric": "Avg Results/Query", "Value": f"{stats.avg_results_per_query:.1f}"},- {"Metric": "Zero-Result Queries", "Value": stats.zero_result_queries},- {- "Metric": "Zero-Result Rate",- "Value": f"{stats.zero_result_rate:.1%}",- },+ {"Metric": "Total Errors", "Value": stats.total_errors},{"Metric": "Avg Score", "Value": f"{stats.avg_score:.4f}"},{"Metric": "Score Range",@@ -68,6 +57,8 @@ class RetrievalObserver(BaseObserver):{"Metric": "Avg Latency (ms)", "Value": f"{stats.avg_latency_ms:.1f}"},{"Metric": "Max Latency (ms)", "Value": f"{stats.max_latency_ms:.1f}"},]+ if stats.last_error:+ summary.append({"Metric": "Last Error", "Value": stats.last_error})lines = [tabulate(summary, headers="keys", tablefmt="pretty")]@@ -87,24 +78,10 @@ class RetrievalObserver(BaseObserver):def __str__(self) -> str:return self.get_status_table()- @classmethod- def _has_unhealthy_yield(cls, stats) -> bool:- """Return whether empty branches coincide with low aggregate yield."""- return (- stats.zero_result_rate >= cls.UNHEALTHY_ZERO_RESULT_RATE- and stats.avg_results_per_query < cls.MIN_HEALTHY_AVG_RESULTS_PER_QUERY- )-def is_healthy(self) -> bool:- """Retrieval is healthy when empty branches still produce useful yield."""- stats = self._get_collector().snapshot()- if stats.total_queries == 0:- return True- return not self._has_unhealthy_yield(stats)+ """Retrieval is healthy when no execution errors have been recorded."""+ return not self.has_errors()def has_errors(self) -> bool:- """Errors are flagged when empty branches also have low aggregate yield."""- stats = self._get_collector().snapshot()- if stats.total_queries < 5:- return False- return self._has_unhealthy_yield(stats)+ """Return whether retrieval execution errors have been recorded."""+ return self._get_collector().snapshot().total_errors > 0diff --git a/tests/server/test_api_search_context.py b/tests/server/test_api_search_context.pyindex fb1653d6f..fd3a0f2d1 100644--- a/tests/server/test_api_search_context.py+++ b/tests/server/test_api_search_context.py@@ -5,6 +5,7 @@ import jsonimport httpx+from openviking.retrieve.retrieval_stats import get_stats_collectorfrom openviking.server.identity import RequestContext, Rolefrom openviking_cli.retrieve import ContextType, MatchedContextfrom openviking_cli.session.user_id import UserIdentifier@@ -205,7 +206,8 @@ async def test_context_mode_degrades_a_retrieval_failure(del kwargsraise RuntimeError("embedder unreachable")- monkeypatch.setattr(service.search, "find", fake_find)+ get_stats_collector().reset()+ monkeypatch.setattr(service.viking_fs, "find", fake_find)response = await client.post("/api/v1/search/search",json={"query": "still a valid request", "mode": "context"},@@ -213,6 +215,10 @@ async def test_context_mode_degrades_a_retrieval_failure(assert response.status_code == 200assert response.json()["result"]["stats"]["retrieval_errors"]+ retrieval_status = (await client.get("/api/v1/observer/retrieval")).json()["result"]+ assert retrieval_status["is_healthy"] is False+ assert retrieval_status["has_errors"] is True+ assert "RuntimeError: embedder unreachable" in retrieval_status["status"]async def test_list_mode_response_is_unchanged(diff --git a/tests/unit/retrieve/test_retrieval_stats.py b/tests/unit/retrieve/test_retrieval_stats.pyindex 85b4ee800..b76cb92cb 100644--- a/tests/unit/retrieve/test_retrieval_stats.py+++ b/tests/unit/retrieve/test_retrieval_stats.py@@ -11,7 +11,7 @@ class TestRetrievalStats:stats = RetrievalStats()assert stats.total_queries == 0assert stats.avg_results_per_query == 0.0- assert stats.zero_result_rate == 0.0+ assert stats.total_errors == 0assert stats.avg_score == 0.0assert stats.avg_latency_ms == 0.0@@ -34,13 +34,12 @@ class TestRetrievalStatsCollector:stats = collector.snapshot()assert stats.total_queries == 1assert stats.total_results == 3- assert stats.zero_result_queries == 0assert stats.max_score == 0.9assert stats.min_score == 0.5assert stats.queries_by_type == {"memory": 1}assert stats.avg_latency_ms == 42.0- def test_record_zero_result_query(self):+ def test_record_empty_query_as_success(self):collector = RetrievalStatsCollector()collector.record_query(context_type="resource",@@ -50,8 +49,8 @@ class TestRetrievalStatsCollector:)stats = collector.snapshot()assert stats.total_queries == 1- assert stats.zero_result_queries == 1- assert stats.zero_result_rate == 1.0+ assert stats.total_results == 0+ assert stats.total_errors == 0def test_record_multiple_queries(self):collector = RetrievalStatsCollector()@@ -62,7 +61,6 @@ class TestRetrievalStatsCollector:stats = collector.snapshot()assert stats.total_queries == 3assert stats.total_results == 3- assert stats.zero_result_queries == 1assert stats.queries_by_type == {"memory": 2, "resource": 1}assert stats.avg_latency_ms == (30 + 20 + 5) / 3@@ -134,35 +132,22 @@ class TestRetrievalObserver:assert observer.is_healthy() is Trueassert observer.has_errors() is False- def test_unhealthy_with_many_zero_results(self):+ def test_healthy_with_only_empty_results(self):collector = self._setup_collector()- for _ in range(8):+ for _ in range(10):collector.record_query("memory", 0, [])- for _ in range(2):- collector.record_query("memory", 1, [0.5])- observer = RetrievalObserver()- # 80% zero-result rate > 50% threshold- assert observer.is_healthy() is False- assert observer.has_errors() is True-- def test_healthy_with_sparse_fan_out_yield(self):- collector = self._setup_collector()- for _ in range(6):- collector.record_query("unknown", 0, [])- for _ in range(4):- collector.record_query("unknown", 3, [0.9, 0.8, 0.7])-observer = RetrievalObserver()assert observer.is_healthy() is Trueassert observer.has_errors() is False- def test_no_errors_below_min_queries(self):+ def test_unhealthy_with_retrieval_error(self):collector = self._setup_collector()- # Only 3 queries (below the 5-query minimum for error flagging)- for _ in range(3):- collector.record_query("memory", 0, [])+ collector.record_error(RuntimeError("backend unavailable"))+observer = RetrievalObserver()- assert observer.has_errors() is False+ assert observer.is_healthy() is False+ assert observer.has_errors() is True+ assert "RuntimeError: backend unavailable" in observer.get_status_table()def test_status_table_no_data(self):self._setup_collector()@@ -177,6 +162,8 @@ class TestRetrievalObserver:observer = RetrievalObserver()table = observer.get_status_table()assert "Total Queries" in table+ assert "Total Errors" in table+ assert "Zero-Result" not in tableassert "memory" in tableassert "resource" in table