Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(observer): treat empty retrievals as healthy (#3975)

Qin Haojieec18c1dcd841
diff --git a/crates/ov_cli/src/status_ui.rs b/crates/ov_cli/src/status_ui.rs
index 5bc21f553..8b4ecd22b 100644
--- a/crates/ov_cli/src/status_ui.rs
+++ b/crates/ov_cli/src/status_ui.rs
@@ -646,11 +646,9 @@ fn retrieval_summary(status: Option<&str>) -> String {
return "unknown".to_string();
};
let queries = metric_value(status, "Total Queries");
- let zero_rate = metric_value(status, "Zero-Result Rate");
- match (queries, zero_rate) {
- (Some(queries), Some(zero_rate)) => {
- format!("{queries} queries, {zero_rate} zero-result rate")
- }
+ let errors = metric_value(status, "Total Errors");
+ match (queries, errors) {
+ (Some(queries), Some(errors)) => format!("{queries} queries, {errors} errors"),
_ => "unknown".to_string(),
}
}
@@ -820,12 +818,12 @@ mod tests {
"is_healthy": true,
"has_errors": false,
"status": "\
- +---------------------+-----------------+\n\
- | Metric | Value |\n\
- +---------------------+-----------------+\n\
- | Total Queries | 121 |\n\
- | Zero-Result Rate | 28.9% |\n\
- +---------------------+-----------------+"
+ +---------------+-------+\n\
+ | Metric | Value |\n\
+ +---------------+-------+\n\
+ | Total Queries | 121 |\n\
+ | Total Errors | 0 |\n\
+ +---------------+-------+"
},
"filesystem": {
"name": "filesystem",
diff --git a/openviking/retrieve/retrieval_stats.py b/openviking/retrieve/retrieval_stats.py
index 5785285bd..2e8d90809 100644
--- a/openviking/retrieve/retrieval_stats.py
+++ b/openviking/retrieve/retrieval_stats.py
@@ -2,9 +2,8 @@
# SPDX-License-Identifier: AGPL-3.0
"""Thread-safe retrieval statistics accumulator.
-Collects per-query metrics from the ``HierarchicalRetriever`` so that
-the ``RetrievalObserver`` can report aggregate health and quality data
-via the observer API.
+Collects per-query metrics from the ``HierarchicalRetriever`` and retrieval
+errors from the service boundary for the observer API.
"""
import threading
@@ -22,7 +21,8 @@ class RetrievalStats:
total_queries: int = 0
total_results: int = 0
- zero_result_queries: int = 0
+ total_errors: int = 0
+ last_error: str | None = None
total_score_sum: float = 0.0
max_score: float = 0.0
min_score: float = float("inf")
@@ -38,12 +38,6 @@ class RetrievalStats:
return 0.0
return self.total_results / self.total_queries
- @property
- def zero_result_rate(self) -> float:
- if self.total_queries == 0:
- return 0.0
- return self.zero_result_queries / self.total_queries
-
@property
def avg_score(self) -> float:
if self.total_results == 0:
@@ -61,8 +55,8 @@ class RetrievalStats:
return {
"total_queries": self.total_queries,
"total_results": self.total_results,
- "zero_result_queries": self.zero_result_queries,
- "zero_result_rate": round(self.zero_result_rate, 4),
+ "total_errors": self.total_errors,
+ "last_error": self.last_error,
"avg_results_per_query": round(self.avg_results_per_query, 2),
"avg_score": round(self.avg_score, 4),
"max_score": round(self.max_score, 4) if self.total_results > 0 else 0.0,
@@ -114,9 +108,6 @@ class RetrievalStatsCollector:
self._stats.total_queries += 1
self._stats.total_results += result_count
- if result_count == 0:
- self._stats.zero_result_queries += 1
-
for s in scores:
self._stats.total_score_sum += s
if s > self._stats.max_score:
@@ -150,6 +141,12 @@ class RetrievalStatsCollector:
except Exception:
pass
+ def record_error(self, error: Exception) -> None:
+ """Record a retrieval execution error observed by the service layer."""
+ with self._lock:
+ self._stats.total_errors += 1
+ self._stats.last_error = f"{type(error).__name__}: {error}"[:200]
+
def snapshot(self) -> RetrievalStats:
"""Return a copy of the current stats."""
with self._lock:
diff --git a/openviking/service/search_service.py b/openviking/service/search_service.py
index 6a6259d09..7f4f61b99 100644
--- a/openviking/service/search_service.py
+++ b/openviking/service/search_service.py
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from openviking.core.path_variables import resolve_path_variables
from openviking.core.uri_validation import validate_optional_viking_uris
+from openviking.retrieve.retrieval_stats import get_stats_collector
from openviking.server.identity import RequestContext
from openviking.storage.viking_fs import VikingFS
from openviking.utils.image_search import (
@@ -18,7 +19,11 @@ from openviking.utils.image_search import (
is_http_url,
is_viking_uri,
)
-from openviking_cli.exceptions import InvalidArgumentError, NotInitializedError
+from openviking_cli.exceptions import (
+ InvalidArgumentError,
+ NotInitializedError,
+ PermissionDeniedError,
+)
from openviking_cli.utils import get_logger
if TYPE_CHECKING:
@@ -113,17 +118,23 @@ class SearchService:
if session is not None and self.is_intent_enabled() and not resolved_image_url:
session_info = await session.get_context_for_search(query)
- result = await viking_fs.search(
- query=query,
- ctx=ctx,
- target_uri=target_uri,
- session_info=session_info,
- limit=limit,
- score_threshold=score_threshold,
- filter=filter,
- level=level,
- image_url=resolved_image_url,
- )
+ try:
+ result = await viking_fs.search(
+ query=query,
+ ctx=ctx,
+ target_uri=target_uri,
+ session_info=session_info,
+ limit=limit,
+ score_threshold=score_threshold,
+ filter=filter,
+ level=level,
+ image_url=resolved_image_url,
+ )
+ except (InvalidArgumentError, PermissionDeniedError):
+ raise
+ except Exception as exc:
+ get_stats_collector().record_error(exc)
+ raise
return result
async def find(
@@ -154,14 +165,20 @@ class SearchService:
_ensure_non_empty_query(query, resolved_image_url)
target_uri = validate_optional_viking_uris(target_uri, field_name="target_uri")
viking_fs = self._ensure_initialized()
- result = await viking_fs.find(
- query=query,
- ctx=ctx,
- target_uri=target_uri,
- limit=limit,
- score_threshold=score_threshold,
- filter=filter,
- level=level,
- image_url=resolved_image_url,
- )
+ try:
+ result = await viking_fs.find(
+ query=query,
+ ctx=ctx,
+ target_uri=target_uri,
+ limit=limit,
+ score_threshold=score_threshold,
+ filter=filter,
+ level=level,
+ image_url=resolved_image_url,
+ )
+ except (InvalidArgumentError, PermissionDeniedError):
+ raise
+ except Exception as exc:
+ get_stats_collector().record_error(exc)
+ raise
return result
diff --git a/openviking/storage/observers/retrieval_observer.py b/openviking/storage/observers/retrieval_observer.py
index 597e19a1e..df4bb233a 100644
--- a/openviking/storage/observers/retrieval_observer.py
+++ b/openviking/storage/observers/retrieval_observer.py
@@ -3,8 +3,7 @@
"""
RetrievalObserver: Retrieval system observability tool.
-Provides methods to observe and report retrieval quality metrics
-accumulated by the HierarchicalRetriever.
+Provides retrieval diagnostics and reports actual execution errors.
"""
from openviking.storage.observers.base_observer import BaseObserver
@@ -17,16 +16,10 @@ class RetrievalObserver(BaseObserver):
"""
RetrievalObserver: System observability tool for retrieval quality.
- Reads accumulated statistics from the global RetrievalStatsCollector
- and formats them for display via the observer API.
+ Empty retrievals are valid outcomes. Health reflects execution errors,
+ while result counts and scores remain diagnostics only.
"""
- # A zero-result rate at or above this threshold is suspicious.
- UNHEALTHY_ZERO_RESULT_RATE = 0.5
- # Sparse fan-out may include expected empty branches. Treat it as unhealthy
- # only when the aggregate workload returns fewer than one result per query.
- MIN_HEALTHY_AVG_RESULTS_PER_QUERY = 1.0
-
@staticmethod
def _get_collector():
"""Lazy import to avoid circular dependency with storage module."""
@@ -44,18 +37,14 @@ class RetrievalObserver(BaseObserver):
stats = self._get_collector().snapshot()
- if stats.total_queries == 0:
+ if stats.total_queries == 0 and stats.total_errors == 0:
return "No retrieval queries recorded."
summary = [
{"Metric": "Total Queries", "Value": stats.total_queries},
{"Metric": "Total Results", "Value": stats.total_results},
{"Metric": "Avg Results/Query", "Value": f"{stats.avg_results_per_query:.1f}"},
- {"Metric": "Zero-Result Queries", "Value": stats.zero_result_queries},
- {
- "Metric": "Zero-Result Rate",
- "Value": f"{stats.zero_result_rate:.1%}",
- },
+ {"Metric": "Total Errors", "Value": stats.total_errors},
{"Metric": "Avg Score", "Value": f"{stats.avg_score:.4f}"},
{
"Metric": "Score Range",
@@ -68,6 +57,8 @@ class RetrievalObserver(BaseObserver):
{"Metric": "Avg Latency (ms)", "Value": f"{stats.avg_latency_ms:.1f}"},
{"Metric": "Max Latency (ms)", "Value": f"{stats.max_latency_ms:.1f}"},
]
+ if stats.last_error:
+ summary.append({"Metric": "Last Error", "Value": stats.last_error})
lines = [tabulate(summary, headers="keys", tablefmt="pretty")]
@@ -87,24 +78,10 @@ class RetrievalObserver(BaseObserver):
def __str__(self) -> str:
return self.get_status_table()
- @classmethod
- def _has_unhealthy_yield(cls, stats) -> bool:
- """Return whether empty branches coincide with low aggregate yield."""
- return (
- stats.zero_result_rate >= cls.UNHEALTHY_ZERO_RESULT_RATE
- and stats.avg_results_per_query < cls.MIN_HEALTHY_AVG_RESULTS_PER_QUERY
- )
-
def is_healthy(self) -> bool:
- """Retrieval is healthy when empty branches still produce useful yield."""
- stats = self._get_collector().snapshot()
- if stats.total_queries == 0:
- return True
- return not self._has_unhealthy_yield(stats)
+ """Retrieval is healthy when no execution errors have been recorded."""
+ return not self.has_errors()
def has_errors(self) -> bool:
- """Errors are flagged when empty branches also have low aggregate yield."""
- stats = self._get_collector().snapshot()
- if stats.total_queries < 5:
- return False
- return self._has_unhealthy_yield(stats)
+ """Return whether retrieval execution errors have been recorded."""
+ return self._get_collector().snapshot().total_errors > 0
diff --git a/tests/server/test_api_search_context.py b/tests/server/test_api_search_context.py
index fb1653d6f..fd3a0f2d1 100644
--- a/tests/server/test_api_search_context.py
+++ b/tests/server/test_api_search_context.py
@@ -5,6 +5,7 @@ import json
import httpx
+from openviking.retrieve.retrieval_stats import get_stats_collector
from openviking.server.identity import RequestContext, Role
from openviking_cli.retrieve import ContextType, MatchedContext
from openviking_cli.session.user_id import UserIdentifier
@@ -205,7 +206,8 @@ async def test_context_mode_degrades_a_retrieval_failure(
del kwargs
raise RuntimeError("embedder unreachable")
- monkeypatch.setattr(service.search, "find", fake_find)
+ get_stats_collector().reset()
+ monkeypatch.setattr(service.viking_fs, "find", fake_find)
response = await client.post(
"/api/v1/search/search",
json={"query": "still a valid request", "mode": "context"},
@@ -213,6 +215,10 @@ async def test_context_mode_degrades_a_retrieval_failure(
assert response.status_code == 200
assert response.json()["result"]["stats"]["retrieval_errors"]
+ retrieval_status = (await client.get("/api/v1/observer/retrieval")).json()["result"]
+ assert retrieval_status["is_healthy"] is False
+ assert retrieval_status["has_errors"] is True
+ assert "RuntimeError: embedder unreachable" in retrieval_status["status"]
async def test_list_mode_response_is_unchanged(
diff --git a/tests/unit/retrieve/test_retrieval_stats.py b/tests/unit/retrieve/test_retrieval_stats.py
index 85b4ee800..b76cb92cb 100644
--- a/tests/unit/retrieve/test_retrieval_stats.py
+++ b/tests/unit/retrieve/test_retrieval_stats.py
@@ -11,7 +11,7 @@ class TestRetrievalStats:
stats = RetrievalStats()
assert stats.total_queries == 0
assert stats.avg_results_per_query == 0.0
- assert stats.zero_result_rate == 0.0
+ assert stats.total_errors == 0
assert stats.avg_score == 0.0
assert stats.avg_latency_ms == 0.0
@@ -34,13 +34,12 @@ class TestRetrievalStatsCollector:
stats = collector.snapshot()
assert stats.total_queries == 1
assert stats.total_results == 3
- assert stats.zero_result_queries == 0
assert stats.max_score == 0.9
assert stats.min_score == 0.5
assert stats.queries_by_type == {"memory": 1}
assert stats.avg_latency_ms == 42.0
- def test_record_zero_result_query(self):
+ def test_record_empty_query_as_success(self):
collector = RetrievalStatsCollector()
collector.record_query(
context_type="resource",
@@ -50,8 +49,8 @@ class TestRetrievalStatsCollector:
)
stats = collector.snapshot()
assert stats.total_queries == 1
- assert stats.zero_result_queries == 1
- assert stats.zero_result_rate == 1.0
+ assert stats.total_results == 0
+ assert stats.total_errors == 0
def test_record_multiple_queries(self):
collector = RetrievalStatsCollector()
@@ -62,7 +61,6 @@ class TestRetrievalStatsCollector:
stats = collector.snapshot()
assert stats.total_queries == 3
assert stats.total_results == 3
- assert stats.zero_result_queries == 1
assert stats.queries_by_type == {"memory": 2, "resource": 1}
assert stats.avg_latency_ms == (30 + 20 + 5) / 3
@@ -134,35 +132,22 @@ class TestRetrievalObserver:
assert observer.is_healthy() is True
assert observer.has_errors() is False
- def test_unhealthy_with_many_zero_results(self):
+ def test_healthy_with_only_empty_results(self):
collector = self._setup_collector()
- for _ in range(8):
+ for _ in range(10):
collector.record_query("memory", 0, [])
- for _ in range(2):
- collector.record_query("memory", 1, [0.5])
- observer = RetrievalObserver()
- # 80% zero-result rate > 50% threshold
- assert observer.is_healthy() is False
- assert observer.has_errors() is True
-
- def test_healthy_with_sparse_fan_out_yield(self):
- collector = self._setup_collector()
- for _ in range(6):
- collector.record_query("unknown", 0, [])
- for _ in range(4):
- collector.record_query("unknown", 3, [0.9, 0.8, 0.7])
-
observer = RetrievalObserver()
assert observer.is_healthy() is True
assert observer.has_errors() is False
- def test_no_errors_below_min_queries(self):
+ def test_unhealthy_with_retrieval_error(self):
collector = self._setup_collector()
- # Only 3 queries (below the 5-query minimum for error flagging)
- for _ in range(3):
- collector.record_query("memory", 0, [])
+ collector.record_error(RuntimeError("backend unavailable"))
+
observer = RetrievalObserver()
- assert observer.has_errors() is False
+ assert observer.is_healthy() is False
+ assert observer.has_errors() is True
+ assert "RuntimeError: backend unavailable" in observer.get_status_table()
def test_status_table_no_data(self):
self._setup_collector()
@@ -177,6 +162,8 @@ class TestRetrievalObserver:
observer = RetrievalObserver()
table = observer.get_status_table()
assert "Total Queries" in table
+ assert "Total Errors" in table
+ assert "Zero-Result" not in table
assert "memory" in table
assert "resource" in table