Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(metrics): preserve search outcomes and error types (#7284)

LearningCircuitdf2ad376ffe0
diff --git a/changelog.d/+search-outcome-accounting.bugfix.md b/changelog.d/+search-outcome-accounting.bugfix.md
new file mode 100644
index 0000000000..c11c25c602
--- /dev/null
+++ b/changelog.d/+search-outcome-accounting.bugfix.md
@@ -0,0 +1 @@
+Keep search metrics' success flag consistent with their status, and preserve the exception type alongside sanitized failure messages.
diff --git a/src/local_deep_research/metrics/search_tracker.py b/src/local_deep_research/metrics/search_tracker.py
index 78c064498a..1a57d8528f 100644
--- a/src/local_deep_research/metrics/search_tracker.py
+++ b/src/local_deep_research/metrics/search_tracker.py
@@ -55,12 +55,15 @@ class SearchTracker:
success: bool = True,
error_message: Optional[str] = None,
status: Optional[str] = None,
+ error_type: Optional[str] = None,
) -> None:
"""Record a completed search operation directly to database.
``status`` overrides the success/error status, e.g.
``SEARCH_STATUS_SKIPPED`` for a search skipped because its engine
was unavailable; ``error_message`` then carries the reason.
+ Pass ``error_type`` separately when the exception message has been
+ sanitized. Plain text alone cannot identify an exception class.
"""
password = None
@@ -87,10 +90,9 @@ class SearchTracker:
# Determine success status
success_status = status or ("success" if success else "error")
- error_type = None
if status == SEARCH_STATUS_SKIPPED:
error_type = "engine_unavailable"
- elif error_message:
+ elif error_message and not error_type:
error_type = (
type(error_message).__name__
if isinstance(error_message, Exception)
@@ -131,6 +133,7 @@ class SearchTracker:
query=query,
results_count=results_count,
response_time_ms=response_time_ms,
+ success=int(success_status == "success"),
success_status=success_status,
error_type=error_type,
error_message=str(error_message)
diff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.py
index 026aee9bde..e5e396433b 100644
--- a/src/local_deep_research/web_search_engines/search_engine_base.py
+++ b/src/local_deep_research/web_search_engines/search_engine_base.py
@@ -829,6 +829,7 @@ class BaseSearchEngine(ABC):
success = True
error_message = None
+ error_type = None
results_count = 0
# Define the core search function with retry logic
@@ -851,7 +852,7 @@ class BaseSearchEngine(ABC):
return _execute_search()
def _execute_search():
- nonlocal success, error_message, results_count
+ nonlocal success, error_message, error_type, results_count
previews_received = False
try:
@@ -981,6 +982,7 @@ class BaseSearchEngine(ABC):
# If rate limiting is disabled, treat as regular error
success = False
error_message = "Rate limit hit but rate limiting disabled"
+ error_type = type(exc).__name__
logger.warning(
f"Rate limit hit on {self.__class__.__name__} but rate limiting is disabled"
)
@@ -999,6 +1001,7 @@ class BaseSearchEngine(ABC):
# cause chain frequently carries the request URL or auth
# header from upstream HTTP clients (see #4131).
error_message = safe_msg = self._scrub_error(e)
+ error_type = type(e).__name__
logger.warning(
f"Search engine {self.__class__.__name__} failed: {safe_msg}"
)
@@ -1013,6 +1016,7 @@ class BaseSearchEngine(ABC):
self._record_search_failure(e)
success = False
error_message = safe_msg = self._scrub_error(e)
+ error_type = type(e).__name__
logger.warning(
f"Search engine {self.__class__.__name__} error: {safe_msg}"
)
@@ -1029,6 +1033,7 @@ class BaseSearchEngine(ABC):
response_time_ms=response_time_ms,
success=success,
error_message=error_message,
+ error_type=error_type,
)
finally:
# Clean up temporary search result storage
diff --git a/tests/metrics/test_token_counting_attribution.py b/tests/metrics/test_token_counting_attribution.py
index b8e14d9031..247e2de72c 100644
--- a/tests/metrics/test_token_counting_attribution.py
+++ b/tests/metrics/test_token_counting_attribution.py
@@ -39,6 +39,7 @@ import asyncio
import threading
import time
from contextlib import contextmanager
+from unittest.mock import Mock
import pytest
from sqlalchemy import create_engine
@@ -55,13 +56,20 @@ from local_deep_research.database.models.research import ResearchHistory
from local_deep_research.metrics.database import MetricsDatabase
from local_deep_research.metrics.pricing.cost_calculator import CostCalculator
from local_deep_research.metrics.pricing.pricing_fetcher import PricingFetcher
-from local_deep_research.metrics.search_tracker import SearchTracker
+from local_deep_research.metrics.search_tracker import (
+ SEARCH_STATUS_SKIPPED,
+ SearchTracker,
+)
from local_deep_research.metrics.token_counter import (
TokenCounter,
TokenCountingCallback,
)
from local_deep_research.utilities.request_context import request_user
from local_deep_research.utilities.thread_context import search_context
+from local_deep_research.web_search_engines.rate_limiting import RateLimitError
+from local_deep_research.web_search_engines.search_engine_base import (
+ BaseSearchEngine,
+)
# Tables the metrics layer touches. ResearchHistory / RateLimit* are
# required because ``_get_metrics_from_encrypted_db`` queries them inside
@@ -759,18 +767,6 @@ def test_retry_attempts_are_counted_once_each_not_double_incremented(user_dbs):
assert counts["total_completion_tokens"] == 50
- strict=True,
- reason=(
- "FAILED SEARCHES ARE STORED AS SUCCESSES IN THE `success` COLUMN. "
- "SearchCall carries both `success` (Integer, default=1, and the "
- "subject of the idx_search_success_timestamp index) and "
- "`success_status` (String). SearchTracker.record_search writes only "
- "success_status, so every failed search lands with success=1. Any "
- "consumer of SearchCall.success counts 100% success regardless of "
- "what actually happened."
- ),
-)
def test_failed_search_is_not_stored_as_a_success(user_dbs):
with search_context(
{
@@ -800,18 +796,8 @@ def test_failed_search_is_not_stored_as_a_success(user_dbs):
assert failed_row.success == 0
- strict=True,
- reason=(
- "error_type IS ALWAYS THE LITERAL 'unknown_error'. record_search "
- "declares error_message as Optional[str] but classifies it with "
- "`type(error_message).__name__ if isinstance(error_message, "
- "Exception) else 'unknown_error'`. For the declared type the "
- "isinstance branch is unreachable, so the error_type column carries "
- "no information for any caller that follows the signature."
- ),
-)
def test_search_error_type_reflects_the_actual_error(user_dbs):
+ """Callers supply the class separately from the sanitized message."""
with search_context(
{"research_id": "1", "username": "alice", "user_password": "pw"}
):
@@ -820,11 +806,200 @@ def test_search_error_type_reflects_the_actual_error(user_dbs):
"dogs",
success=False,
error_message="HTTP 503 from upstream",
+ error_type="HTTPError",
)
(row,) = user_dbs.rows("alice", SearchCall)
assert row.error_message == "HTTP 503 from upstream"
- assert row.error_type != "unknown_error"
+ assert row.error_type == "HTTPError"
+
+
+ "success,status,expected_success,expected_status,expected_type",
+ [
+ (True, None, 1, "success", None),
+ (False, None, 0, "error", None),
+ (True, "error", 0, "error", None),
+ (False, "success", 1, "success", None),
+ (
+ True,
+ SEARCH_STATUS_SKIPPED,
+ 0,
+ SEARCH_STATUS_SKIPPED,
+ "engine_unavailable",
+ ),
+ ],
+)
+def test_search_status_override_keeps_persisted_columns_consistent(
+ user_dbs, success, status, expected_success, expected_status, expected_type
+):
+ with search_context(
+ {"research_id": "1", "username": "alice", "user_password": "pw"}
+ ):
+ SearchTracker.record_search(
+ "searxng", "q", success=success, status=status
+ )
+
+ (row,) = user_dbs.rows("alice", SearchCall)
+ assert (row.success, row.success_status, row.error_type) == (
+ expected_success,
+ expected_status,
+ expected_type,
+ )
+
+
+def test_search_error_text_does_not_invent_an_exception_class(user_dbs):
+ with search_context(
+ {"research_id": "1", "username": "alice", "user_password": "pw"}
+ ):
+ SearchTracker.record_search(
+ "searxng", "q", success=False, error_message="TimeoutError HTTP 503"
+ )
+
+ (row,) = user_dbs.rows("alice", SearchCall)
+ assert row.error_type == "unknown_error"
+ assert row.error_message == "TimeoutError HTTP 503"
+
+
+class OutcomeSearchEngine(BaseSearchEngine):
+ """Run the real search and metrics lifecycle with a controlled provider."""
+
+ def __init__(self, outcomes, *, rate_limiting=False, content_error=None):
+ super().__init__(
+ programmatic_mode=False,
+ settings_snapshot={},
+ search_snippets_only=content_error is None,
+ )
+ self.outcomes = iter(outcomes)
+ self.attempts = 0
+ self.content_error = content_error
+ self.rate_tracker = Mock(enabled=rate_limiting)
+ self.rate_tracker.get_wait_time.return_value = 0.0
+
+ def _get_previews(self, query):
+ self.attempts += 1
+ outcome = next(self.outcomes)
+ if isinstance(outcome, Exception):
+ raise outcome
+ return outcome
+
+ def _get_full_content(self, items):
+ if self.content_error:
+ raise self.content_error
+ return items
+
+
+ "error_class,rate_limiting,expected_attempts",
+ [
+ (ValueError, False, 1),
+ (TimeoutError, False, 1),
+ (RateLimitError, False, 1),
+ (RateLimitError, True, 3),
+ ],
+)
+def test_search_engine_persists_exception_type_and_sanitized_message(
+ user_dbs, monkeypatch, error_class, rate_limiting, expected_attempts
+):
+ engine = OutcomeSearchEngine(
+ [error_class("provider detail")] * expected_attempts,
+ rate_limiting=rate_limiting,
+ )
+ monkeypatch.setattr(
+ engine, "_scrub_error", lambda error: "sanitized failure"
+ )
+ assert (
+ engine.run(
+ "q",
+ research_context={
+ "research_id": "1",
+ "username": "alice",
+ "user_password": "pw",
+ },
+ )
+ == []
+ )
+
+ assert engine.attempts == expected_attempts
+ (row,) = user_dbs.rows("alice", SearchCall)
+ assert row.error_type == error_class.__name__
+ assert row.error_message == (
+ "Rate limit hit but rate limiting disabled"
+ if error_class is RateLimitError and not rate_limiting
+ else "sanitized failure"
+ )
+ assert row.success_status == "error"
+ assert row.success == 0
+
+
+ "results", [[], [{"title": "result", "snippet": "text"}]]
+)
+def test_search_engine_records_success_after_previous_failure(
+ user_dbs, results
+):
+ engine = OutcomeSearchEngine([ValueError("provider failure"), results])
+ context = {"research_id": "1", "username": "alice", "user_password": "pw"}
+ assert engine.run("first", research_context=context) == []
+ assert engine.run("second", research_context=context) == results
+
+ failed, succeeded = user_dbs.rows("alice", SearchCall)
+ assert (failed.success, failed.error_type) == (0, "ValueError")
+ assert succeeded.success == 1
+ assert succeeded.success_status == "success"
+ assert succeeded.error_type is None
+ assert succeeded.error_message is None
+ assert succeeded.results_count == len(results)
+
+
+def test_search_engine_recovered_retry_records_one_success(user_dbs):
+ results = [{"title": "result", "snippet": "text"}]
+ engine = OutcomeSearchEngine(
+ [RateLimitError("retry"), results], rate_limiting=True
+ )
+ assert (
+ engine.run(
+ "q",
+ research_context={
+ "research_id": "1",
+ "username": "alice",
+ "user_password": "pw",
+ },
+ )
+ == results
+ )
+
+ assert engine.attempts == 2
+ (row,) = user_dbs.rows("alice", SearchCall)
+ assert row.success == 1
+ assert row.success_status == "success"
+ assert row.error_type is None
+ assert row.error_message is None
+ assert row.results_count == 1
+
+
+def test_search_engine_content_failure_preserves_exception_type(user_dbs):
+ engine = OutcomeSearchEngine(
+ [[{"title": "result", "snippet": "text"}]],
+ content_error=ValueError("content conversion failed"),
+ )
+ assert (
+ engine.run(
+ "q",
+ research_context={
+ "research_id": "1",
+ "username": "alice",
+ "user_password": "pw",
+ },
+ )
+ == []
+ )
+
+ (row,) = user_dbs.rows("alice", SearchCall)
+ assert row.success == 0
+ assert row.success_status == "error"
+ assert row.error_type == "ValueError"
+ assert row.error_message == "content conversion failed"
@pytest.mark.xfail(