Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(observer): keep empty retrievals diagnostic-only (#3985)
Qin Haojie9d5646169b09
diff --git a/crates/ov_cli/src/status_ui.rs b/crates/ov_cli/src/status_ui.rsindex 8b4ecd22b..5bc21f553 100644--- a/crates/ov_cli/src/status_ui.rs+++ b/crates/ov_cli/src/status_ui.rs@@ -646,9 +646,11 @@ fn retrieval_summary(status: Option<&str>) -> String {return "unknown".to_string();};let queries = metric_value(status, "Total Queries");- let errors = metric_value(status, "Total Errors");- match (queries, errors) {- (Some(queries), Some(errors)) => format!("{queries} queries, {errors} errors"),+ let zero_rate = metric_value(status, "Zero-Result Rate");+ match (queries, zero_rate) {+ (Some(queries), Some(zero_rate)) => {+ format!("{queries} queries, {zero_rate} zero-result rate")+ }_ => "unknown".to_string(),}}@@ -818,12 +820,12 @@ mod tests {"is_healthy": true,"has_errors": false,"status": "\- +---------------+-------+\n\- | Metric | Value |\n\- +---------------+-------+\n\- | Total Queries | 121 |\n\- | Total Errors | 0 |\n\- +---------------+-------+"+ +---------------------+-----------------+\n\+ | Metric | Value |\n\+ +---------------------+-----------------+\n\+ | Total Queries | 121 |\n\+ | Zero-Result Rate | 28.9% |\n\+ +---------------------+-----------------+"},"filesystem": {"name": "filesystem",diff --git a/openviking/retrieve/retrieval_stats.py b/openviking/retrieve/retrieval_stats.pyindex 2e8d90809..5785285bd 100644--- a/openviking/retrieve/retrieval_stats.py+++ b/openviking/retrieve/retrieval_stats.py@@ -2,8 +2,9 @@# SPDX-License-Identifier: AGPL-3.0"""Thread-safe retrieval statistics accumulator.-Collects per-query metrics from the ``HierarchicalRetriever`` and retrieval-errors from the service boundary for the observer API.+Collects per-query metrics from the ``HierarchicalRetriever`` so that+the ``RetrievalObserver`` can report aggregate health and quality data+via the observer API."""import threading@@ -21,8 +22,7 @@ class RetrievalStats:total_queries: int = 0total_results: int = 0- total_errors: int = 0- last_error: str | None = None+ zero_result_queries: int = 0total_score_sum: float = 0.0max_score: float = 0.0min_score: float = float("inf")@@ -38,6 +38,12 @@ class RetrievalStats:return 0.0return self.total_results / self.total_queries+ @property+ def zero_result_rate(self) -> float:+ if self.total_queries == 0:+ return 0.0+ return self.zero_result_queries / self.total_queries+@propertydef avg_score(self) -> float:if self.total_results == 0:@@ -55,8 +61,8 @@ class RetrievalStats:return {"total_queries": self.total_queries,"total_results": self.total_results,- "total_errors": self.total_errors,- "last_error": self.last_error,+ "zero_result_queries": self.zero_result_queries,+ "zero_result_rate": round(self.zero_result_rate, 4),"avg_results_per_query": round(self.avg_results_per_query, 2),"avg_score": round(self.avg_score, 4),"max_score": round(self.max_score, 4) if self.total_results > 0 else 0.0,@@ -108,6 +114,9 @@ class RetrievalStatsCollector:self._stats.total_queries += 1self._stats.total_results += result_count+ if result_count == 0:+ self._stats.zero_result_queries += 1+for s in scores:self._stats.total_score_sum += sif s > self._stats.max_score:@@ -141,12 +150,6 @@ class RetrievalStatsCollector:except Exception:pass- def record_error(self, error: Exception) -> None:- """Record a retrieval execution error observed by the service layer."""- with self._lock:- self._stats.total_errors += 1- self._stats.last_error = f"{type(error).__name__}: {error}"[:200]-def snapshot(self) -> RetrievalStats:"""Return a copy of the current stats."""with self._lock:diff --git a/openviking/service/search_service.py b/openviking/service/search_service.pyindex 7f4f61b99..6a6259d09 100644--- a/openviking/service/search_service.py+++ b/openviking/service/search_service.py@@ -10,7 +10,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Unionfrom openviking.core.path_variables import resolve_path_variablesfrom openviking.core.uri_validation import validate_optional_viking_uris-from openviking.retrieve.retrieval_stats import get_stats_collectorfrom openviking.server.identity import RequestContextfrom openviking.storage.viking_fs import VikingFSfrom openviking.utils.image_search import (@@ -19,11 +18,7 @@ from openviking.utils.image_search import (is_http_url,is_viking_uri,)-from openviking_cli.exceptions import (- InvalidArgumentError,- NotInitializedError,- PermissionDeniedError,-)+from openviking_cli.exceptions import InvalidArgumentError, NotInitializedErrorfrom openviking_cli.utils import get_loggerif TYPE_CHECKING:@@ -118,23 +113,17 @@ class SearchService:if session is not None and self.is_intent_enabled() and not resolved_image_url:session_info = await session.get_context_for_search(query)- try:- result = await viking_fs.search(- query=query,- ctx=ctx,- target_uri=target_uri,- session_info=session_info,- limit=limit,- score_threshold=score_threshold,- filter=filter,- level=level,- image_url=resolved_image_url,- )- except (InvalidArgumentError, PermissionDeniedError):- raise- except Exception as exc:- get_stats_collector().record_error(exc)- raise+ result = await viking_fs.search(+ query=query,+ ctx=ctx,+ target_uri=target_uri,+ session_info=session_info,+ limit=limit,+ score_threshold=score_threshold,+ filter=filter,+ level=level,+ image_url=resolved_image_url,+ )return resultasync def find(@@ -165,20 +154,14 @@ class SearchService:_ensure_non_empty_query(query, resolved_image_url)target_uri = validate_optional_viking_uris(target_uri, field_name="target_uri")viking_fs = self._ensure_initialized()- try:- result = await viking_fs.find(- query=query,- ctx=ctx,- target_uri=target_uri,- limit=limit,- score_threshold=score_threshold,- filter=filter,- level=level,- image_url=resolved_image_url,- )- except (InvalidArgumentError, PermissionDeniedError):- raise- except Exception as exc:- get_stats_collector().record_error(exc)- raise+ result = await viking_fs.find(+ query=query,+ ctx=ctx,+ target_uri=target_uri,+ limit=limit,+ score_threshold=score_threshold,+ filter=filter,+ level=level,+ image_url=resolved_image_url,+ )return resultdiff --git a/openviking/storage/observers/retrieval_observer.py b/openviking/storage/observers/retrieval_observer.pyindex df4bb233a..61be220c0 100644--- a/openviking/storage/observers/retrieval_observer.py+++ b/openviking/storage/observers/retrieval_observer.py@@ -3,7 +3,7 @@"""RetrievalObserver: Retrieval system observability tool.-Provides retrieval diagnostics and reports actual execution errors.+Provides retrieval diagnostics accumulated by the HierarchicalRetriever."""from openviking.storage.observers.base_observer import BaseObserver@@ -16,8 +16,8 @@ class RetrievalObserver(BaseObserver):"""RetrievalObserver: System observability tool for retrieval quality.- Empty retrievals are valid outcomes. Health reflects execution errors,- while result counts and scores remain diagnostics only.+ Empty retrievals are valid outcomes. Result counts and scores are+ diagnostics only and do not determine component health."""@staticmethod@@ -37,14 +37,18 @@ class RetrievalObserver(BaseObserver):stats = self._get_collector().snapshot()- if stats.total_queries == 0 and stats.total_errors == 0:+ if stats.total_queries == 0:return "No retrieval queries recorded."summary = [{"Metric": "Total Queries", "Value": stats.total_queries},{"Metric": "Total Results", "Value": stats.total_results},{"Metric": "Avg Results/Query", "Value": f"{stats.avg_results_per_query:.1f}"},- {"Metric": "Total Errors", "Value": stats.total_errors},+ {"Metric": "Zero-Result Queries", "Value": stats.zero_result_queries},+ {+ "Metric": "Zero-Result Rate",+ "Value": f"{stats.zero_result_rate:.1%}",+ },{"Metric": "Avg Score", "Value": f"{stats.avg_score:.4f}"},{"Metric": "Score Range",@@ -57,8 +61,6 @@ class RetrievalObserver(BaseObserver):{"Metric": "Avg Latency (ms)", "Value": f"{stats.avg_latency_ms:.1f}"},{"Metric": "Max Latency (ms)", "Value": f"{stats.max_latency_ms:.1f}"},]- if stats.last_error:- summary.append({"Metric": "Last Error", "Value": stats.last_error})lines = [tabulate(summary, headers="keys", tablefmt="pretty")]@@ -79,9 +81,9 @@ class RetrievalObserver(BaseObserver):return self.get_status_table()def is_healthy(self) -> bool:- """Retrieval is healthy when no execution errors have been recorded."""- return not self.has_errors()+ """Retrieval result diagnostics do not indicate component availability."""+ return Truedef has_errors(self) -> bool:- """Return whether retrieval execution errors have been recorded."""- return self._get_collector().snapshot().total_errors > 0+ """Empty retrieval results are valid outcomes, not errors."""+ return Falsediff --git a/tests/server/test_api_search_context.py b/tests/server/test_api_search_context.pyindex fd3a0f2d1..fb1653d6f 100644--- a/tests/server/test_api_search_context.py+++ b/tests/server/test_api_search_context.py@@ -5,7 +5,6 @@ import jsonimport httpx-from openviking.retrieve.retrieval_stats import get_stats_collectorfrom openviking.server.identity import RequestContext, Rolefrom openviking_cli.retrieve import ContextType, MatchedContextfrom openviking_cli.session.user_id import UserIdentifier@@ -206,8 +205,7 @@ async def test_context_mode_degrades_a_retrieval_failure(del kwargsraise RuntimeError("embedder unreachable")- get_stats_collector().reset()- monkeypatch.setattr(service.viking_fs, "find", fake_find)+ monkeypatch.setattr(service.search, "find", fake_find)response = await client.post("/api/v1/search/search",json={"query": "still a valid request", "mode": "context"},@@ -215,10 +213,6 @@ async def test_context_mode_degrades_a_retrieval_failure(assert response.status_code == 200assert response.json()["result"]["stats"]["retrieval_errors"]- retrieval_status = (await client.get("/api/v1/observer/retrieval")).json()["result"]- assert retrieval_status["is_healthy"] is False- assert retrieval_status["has_errors"] is True- assert "RuntimeError: embedder unreachable" in retrieval_status["status"]async def test_list_mode_response_is_unchanged(diff --git a/tests/unit/retrieve/test_retrieval_stats.py b/tests/unit/retrieve/test_retrieval_stats.pyindex b76cb92cb..47f342841 100644--- a/tests/unit/retrieve/test_retrieval_stats.py+++ b/tests/unit/retrieve/test_retrieval_stats.py@@ -11,7 +11,7 @@ class TestRetrievalStats:stats = RetrievalStats()assert stats.total_queries == 0assert stats.avg_results_per_query == 0.0- assert stats.total_errors == 0+ assert stats.zero_result_rate == 0.0assert stats.avg_score == 0.0assert stats.avg_latency_ms == 0.0@@ -34,6 +34,7 @@ class TestRetrievalStatsCollector:stats = collector.snapshot()assert stats.total_queries == 1assert stats.total_results == 3+ assert stats.zero_result_queries == 0assert stats.max_score == 0.9assert stats.min_score == 0.5assert stats.queries_by_type == {"memory": 1}@@ -50,7 +51,8 @@ class TestRetrievalStatsCollector:stats = collector.snapshot()assert stats.total_queries == 1assert stats.total_results == 0- assert stats.total_errors == 0+ assert stats.zero_result_queries == 1+ assert stats.zero_result_rate == 1.0def test_record_multiple_queries(self):collector = RetrievalStatsCollector()@@ -61,6 +63,7 @@ class TestRetrievalStatsCollector:stats = collector.snapshot()assert stats.total_queries == 3assert stats.total_results == 3+ assert stats.zero_result_queries == 1assert stats.queries_by_type == {"memory": 2, "resource": 1}assert stats.avg_latency_ms == (30 + 20 + 5) / 3@@ -139,15 +142,7 @@ class TestRetrievalObserver:observer = RetrievalObserver()assert observer.is_healthy() is Trueassert observer.has_errors() is False-- def test_unhealthy_with_retrieval_error(self):- collector = self._setup_collector()- collector.record_error(RuntimeError("backend unavailable"))-- observer = RetrievalObserver()- assert observer.is_healthy() is False- assert observer.has_errors() is True- assert "RuntimeError: backend unavailable" in observer.get_status_table()+ assert "Zero-Result Rate" in observer.get_status_table()def test_status_table_no_data(self):self._setup_collector()@@ -162,8 +157,7 @@ class TestRetrievalObserver:observer = RetrievalObserver()table = observer.get_status_table()assert "Total Queries" in table- assert "Total Errors" in table- assert "Zero-Result" not in table+ assert "Zero-Result Rate" in tableassert "memory" in tableassert "resource" in table