Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(observer): keep empty retrievals diagnostic-only (#3985)

Qin Haojie9d5646169b09
diff --git a/crates/ov_cli/src/status_ui.rs b/crates/ov_cli/src/status_ui.rs
index 8b4ecd22b..5bc21f553 100644
--- a/crates/ov_cli/src/status_ui.rs
+++ b/crates/ov_cli/src/status_ui.rs
@@ -646,9 +646,11 @@ fn retrieval_summary(status: Option<&str>) -> String {
return "unknown".to_string();
};
let queries = metric_value(status, "Total Queries");
- let errors = metric_value(status, "Total Errors");
- match (queries, errors) {
- (Some(queries), Some(errors)) => format!("{queries} queries, {errors} errors"),
+ let zero_rate = metric_value(status, "Zero-Result Rate");
+ match (queries, zero_rate) {
+ (Some(queries), Some(zero_rate)) => {
+ format!("{queries} queries, {zero_rate} zero-result rate")
+ }
_ => "unknown".to_string(),
}
}
@@ -818,12 +820,12 @@ mod tests {
"is_healthy": true,
"has_errors": false,
"status": "\
- +---------------+-------+\n\
- | Metric | Value |\n\
- +---------------+-------+\n\
- | Total Queries | 121 |\n\
- | Total Errors | 0 |\n\
- +---------------+-------+"
+ +---------------------+-----------------+\n\
+ | Metric | Value |\n\
+ +---------------------+-----------------+\n\
+ | Total Queries | 121 |\n\
+ | Zero-Result Rate | 28.9% |\n\
+ +---------------------+-----------------+"
},
"filesystem": {
"name": "filesystem",
diff --git a/openviking/retrieve/retrieval_stats.py b/openviking/retrieve/retrieval_stats.py
index 2e8d90809..5785285bd 100644
--- a/openviking/retrieve/retrieval_stats.py
+++ b/openviking/retrieve/retrieval_stats.py
@@ -2,8 +2,9 @@
# SPDX-License-Identifier: AGPL-3.0
"""Thread-safe retrieval statistics accumulator.
-Collects per-query metrics from the ``HierarchicalRetriever`` and retrieval
-errors from the service boundary for the observer API.
+Collects per-query metrics from the ``HierarchicalRetriever`` so that
+the ``RetrievalObserver`` can report aggregate health and quality data
+via the observer API.
"""
import threading
@@ -21,8 +22,7 @@ class RetrievalStats:
total_queries: int = 0
total_results: int = 0
- total_errors: int = 0
- last_error: str | None = None
+ zero_result_queries: int = 0
total_score_sum: float = 0.0
max_score: float = 0.0
min_score: float = float("inf")
@@ -38,6 +38,12 @@ class RetrievalStats:
return 0.0
return self.total_results / self.total_queries
+ @property
+ def zero_result_rate(self) -> float:
+ if self.total_queries == 0:
+ return 0.0
+ return self.zero_result_queries / self.total_queries
+
@property
def avg_score(self) -> float:
if self.total_results == 0:
@@ -55,8 +61,8 @@ class RetrievalStats:
return {
"total_queries": self.total_queries,
"total_results": self.total_results,
- "total_errors": self.total_errors,
- "last_error": self.last_error,
+ "zero_result_queries": self.zero_result_queries,
+ "zero_result_rate": round(self.zero_result_rate, 4),
"avg_results_per_query": round(self.avg_results_per_query, 2),
"avg_score": round(self.avg_score, 4),
"max_score": round(self.max_score, 4) if self.total_results > 0 else 0.0,
@@ -108,6 +114,9 @@ class RetrievalStatsCollector:
self._stats.total_queries += 1
self._stats.total_results += result_count
+ if result_count == 0:
+ self._stats.zero_result_queries += 1
+
for s in scores:
self._stats.total_score_sum += s
if s > self._stats.max_score:
@@ -141,12 +150,6 @@ class RetrievalStatsCollector:
except Exception:
pass
- def record_error(self, error: Exception) -> None:
- """Record a retrieval execution error observed by the service layer."""
- with self._lock:
- self._stats.total_errors += 1
- self._stats.last_error = f"{type(error).__name__}: {error}"[:200]
-
def snapshot(self) -> RetrievalStats:
"""Return a copy of the current stats."""
with self._lock:
diff --git a/openviking/service/search_service.py b/openviking/service/search_service.py
index 7f4f61b99..6a6259d09 100644
--- a/openviking/service/search_service.py
+++ b/openviking/service/search_service.py
@@ -10,7 +10,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from openviking.core.path_variables import resolve_path_variables
from openviking.core.uri_validation import validate_optional_viking_uris
-from openviking.retrieve.retrieval_stats import get_stats_collector
from openviking.server.identity import RequestContext
from openviking.storage.viking_fs import VikingFS
from openviking.utils.image_search import (
@@ -19,11 +18,7 @@ from openviking.utils.image_search import (
is_http_url,
is_viking_uri,
)
-from openviking_cli.exceptions import (
- InvalidArgumentError,
- NotInitializedError,
- PermissionDeniedError,
-)
+from openviking_cli.exceptions import InvalidArgumentError, NotInitializedError
from openviking_cli.utils import get_logger
if TYPE_CHECKING:
@@ -118,23 +113,17 @@ class SearchService:
if session is not None and self.is_intent_enabled() and not resolved_image_url:
session_info = await session.get_context_for_search(query)
- try:
- result = await viking_fs.search(
- query=query,
- ctx=ctx,
- target_uri=target_uri,
- session_info=session_info,
- limit=limit,
- score_threshold=score_threshold,
- filter=filter,
- level=level,
- image_url=resolved_image_url,
- )
- except (InvalidArgumentError, PermissionDeniedError):
- raise
- except Exception as exc:
- get_stats_collector().record_error(exc)
- raise
+ result = await viking_fs.search(
+ query=query,
+ ctx=ctx,
+ target_uri=target_uri,
+ session_info=session_info,
+ limit=limit,
+ score_threshold=score_threshold,
+ filter=filter,
+ level=level,
+ image_url=resolved_image_url,
+ )
return result
async def find(
@@ -165,20 +154,14 @@ class SearchService:
_ensure_non_empty_query(query, resolved_image_url)
target_uri = validate_optional_viking_uris(target_uri, field_name="target_uri")
viking_fs = self._ensure_initialized()
- try:
- result = await viking_fs.find(
- query=query,
- ctx=ctx,
- target_uri=target_uri,
- limit=limit,
- score_threshold=score_threshold,
- filter=filter,
- level=level,
- image_url=resolved_image_url,
- )
- except (InvalidArgumentError, PermissionDeniedError):
- raise
- except Exception as exc:
- get_stats_collector().record_error(exc)
- raise
+ result = await viking_fs.find(
+ query=query,
+ ctx=ctx,
+ target_uri=target_uri,
+ limit=limit,
+ score_threshold=score_threshold,
+ filter=filter,
+ level=level,
+ image_url=resolved_image_url,
+ )
return result
diff --git a/openviking/storage/observers/retrieval_observer.py b/openviking/storage/observers/retrieval_observer.py
index df4bb233a..61be220c0 100644
--- a/openviking/storage/observers/retrieval_observer.py
+++ b/openviking/storage/observers/retrieval_observer.py
@@ -3,7 +3,7 @@
"""
RetrievalObserver: Retrieval system observability tool.
-Provides retrieval diagnostics and reports actual execution errors.
+Provides retrieval diagnostics accumulated by the HierarchicalRetriever.
"""
from openviking.storage.observers.base_observer import BaseObserver
@@ -16,8 +16,8 @@ class RetrievalObserver(BaseObserver):
"""
RetrievalObserver: System observability tool for retrieval quality.
- Empty retrievals are valid outcomes. Health reflects execution errors,
- while result counts and scores remain diagnostics only.
+ Empty retrievals are valid outcomes. Result counts and scores are
+ diagnostics only and do not determine component health.
"""
@staticmethod
@@ -37,14 +37,18 @@ class RetrievalObserver(BaseObserver):
stats = self._get_collector().snapshot()
- if stats.total_queries == 0 and stats.total_errors == 0:
+ if stats.total_queries == 0:
return "No retrieval queries recorded."
summary = [
{"Metric": "Total Queries", "Value": stats.total_queries},
{"Metric": "Total Results", "Value": stats.total_results},
{"Metric": "Avg Results/Query", "Value": f"{stats.avg_results_per_query:.1f}"},
- {"Metric": "Total Errors", "Value": stats.total_errors},
+ {"Metric": "Zero-Result Queries", "Value": stats.zero_result_queries},
+ {
+ "Metric": "Zero-Result Rate",
+ "Value": f"{stats.zero_result_rate:.1%}",
+ },
{"Metric": "Avg Score", "Value": f"{stats.avg_score:.4f}"},
{
"Metric": "Score Range",
@@ -57,8 +61,6 @@ class RetrievalObserver(BaseObserver):
{"Metric": "Avg Latency (ms)", "Value": f"{stats.avg_latency_ms:.1f}"},
{"Metric": "Max Latency (ms)", "Value": f"{stats.max_latency_ms:.1f}"},
]
- if stats.last_error:
- summary.append({"Metric": "Last Error", "Value": stats.last_error})
lines = [tabulate(summary, headers="keys", tablefmt="pretty")]
@@ -79,9 +81,9 @@ class RetrievalObserver(BaseObserver):
return self.get_status_table()
def is_healthy(self) -> bool:
- """Retrieval is healthy when no execution errors have been recorded."""
- return not self.has_errors()
+ """Retrieval result diagnostics do not indicate component availability."""
+ return True
def has_errors(self) -> bool:
- """Return whether retrieval execution errors have been recorded."""
- return self._get_collector().snapshot().total_errors > 0
+ """Empty retrieval results are valid outcomes, not errors."""
+ return False
diff --git a/tests/server/test_api_search_context.py b/tests/server/test_api_search_context.py
index fd3a0f2d1..fb1653d6f 100644
--- a/tests/server/test_api_search_context.py
+++ b/tests/server/test_api_search_context.py
@@ -5,7 +5,6 @@ import json
import httpx
-from openviking.retrieve.retrieval_stats import get_stats_collector
from openviking.server.identity import RequestContext, Role
from openviking_cli.retrieve import ContextType, MatchedContext
from openviking_cli.session.user_id import UserIdentifier
@@ -206,8 +205,7 @@ async def test_context_mode_degrades_a_retrieval_failure(
del kwargs
raise RuntimeError("embedder unreachable")
- get_stats_collector().reset()
- monkeypatch.setattr(service.viking_fs, "find", fake_find)
+ monkeypatch.setattr(service.search, "find", fake_find)
response = await client.post(
"/api/v1/search/search",
json={"query": "still a valid request", "mode": "context"},
@@ -215,10 +213,6 @@ async def test_context_mode_degrades_a_retrieval_failure(
assert response.status_code == 200
assert response.json()["result"]["stats"]["retrieval_errors"]
- retrieval_status = (await client.get("/api/v1/observer/retrieval")).json()["result"]
- assert retrieval_status["is_healthy"] is False
- assert retrieval_status["has_errors"] is True
- assert "RuntimeError: embedder unreachable" in retrieval_status["status"]
async def test_list_mode_response_is_unchanged(
diff --git a/tests/unit/retrieve/test_retrieval_stats.py b/tests/unit/retrieve/test_retrieval_stats.py
index b76cb92cb..47f342841 100644
--- a/tests/unit/retrieve/test_retrieval_stats.py
+++ b/tests/unit/retrieve/test_retrieval_stats.py
@@ -11,7 +11,7 @@ class TestRetrievalStats:
stats = RetrievalStats()
assert stats.total_queries == 0
assert stats.avg_results_per_query == 0.0
- assert stats.total_errors == 0
+ assert stats.zero_result_rate == 0.0
assert stats.avg_score == 0.0
assert stats.avg_latency_ms == 0.0
@@ -34,6 +34,7 @@ class TestRetrievalStatsCollector:
stats = collector.snapshot()
assert stats.total_queries == 1
assert stats.total_results == 3
+ assert stats.zero_result_queries == 0
assert stats.max_score == 0.9
assert stats.min_score == 0.5
assert stats.queries_by_type == {"memory": 1}
@@ -50,7 +51,8 @@ class TestRetrievalStatsCollector:
stats = collector.snapshot()
assert stats.total_queries == 1
assert stats.total_results == 0
- assert stats.total_errors == 0
+ assert stats.zero_result_queries == 1
+ assert stats.zero_result_rate == 1.0
def test_record_multiple_queries(self):
collector = RetrievalStatsCollector()
@@ -61,6 +63,7 @@ class TestRetrievalStatsCollector:
stats = collector.snapshot()
assert stats.total_queries == 3
assert stats.total_results == 3
+ assert stats.zero_result_queries == 1
assert stats.queries_by_type == {"memory": 2, "resource": 1}
assert stats.avg_latency_ms == (30 + 20 + 5) / 3
@@ -139,15 +142,7 @@ class TestRetrievalObserver:
observer = RetrievalObserver()
assert observer.is_healthy() is True
assert observer.has_errors() is False
-
- def test_unhealthy_with_retrieval_error(self):
- collector = self._setup_collector()
- collector.record_error(RuntimeError("backend unavailable"))
-
- observer = RetrievalObserver()
- assert observer.is_healthy() is False
- assert observer.has_errors() is True
- assert "RuntimeError: backend unavailable" in observer.get_status_table()
+ assert "Zero-Result Rate" in observer.get_status_table()
def test_status_table_no_data(self):
self._setup_collector()
@@ -162,8 +157,7 @@ class TestRetrievalObserver:
observer = RetrievalObserver()
table = observer.get_status_table()
assert "Total Queries" in table
- assert "Total Errors" in table
- assert "Zero-Result" not in table
+ assert "Zero-Result Rate" in table
assert "memory" in table
assert "resource" in table