~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(metrics): preserve search outcomes and error types (#7284)
LearningCircuitdf2ad376ffe0
diff --git a/changelog.d/+search-outcome-accounting.bugfix.md b/changelog.d/+search-outcome-accounting.bugfix.mdnew file mode 100644index 0000000000..c11c25c602--- /dev/null+++ b/changelog.d/+search-outcome-accounting.bugfix.md@@ -0,0 +1 @@+Keep search metrics' success flag consistent with their status, and preserve the exception type alongside sanitized failure messages.diff --git a/src/local_deep_research/metrics/search_tracker.py b/src/local_deep_research/metrics/search_tracker.pyindex 78c064498a..1a57d8528f 100644--- a/src/local_deep_research/metrics/search_tracker.py+++ b/src/local_deep_research/metrics/search_tracker.py@@ -55,12 +55,15 @@ class SearchTracker:success: bool = True,error_message: Optional[str] = None,status: Optional[str] = None,+ error_type: Optional[str] = None,) -> None:"""Record a completed search operation directly to database.``status`` overrides the success/error status, e.g.``SEARCH_STATUS_SKIPPED`` for a search skipped because its enginewas unavailable; ``error_message`` then carries the reason.+ Pass ``error_type`` separately when the exception message has been+ sanitized. Plain text alone cannot identify an exception class."""password = None@@ -87,10 +90,9 @@ class SearchTracker:# Determine success statussuccess_status = status or ("success" if success else "error")- error_type = Noneif status == SEARCH_STATUS_SKIPPED:error_type = "engine_unavailable"- elif error_message:+ elif error_message and not error_type:error_type = (type(error_message).__name__if isinstance(error_message, Exception)@@ -131,6 +133,7 @@ class SearchTracker:query=query,results_count=results_count,response_time_ms=response_time_ms,+ success=int(success_status == "success"),success_status=success_status,error_type=error_type,error_message=str(error_message)diff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.pyindex 026aee9bde..e5e396433b 100644--- a/src/local_deep_research/web_search_engines/search_engine_base.py+++ b/src/local_deep_research/web_search_engines/search_engine_base.py@@ -829,6 +829,7 @@ class BaseSearchEngine(ABC):success = Trueerror_message = None+ error_type = Noneresults_count = 0# Define the core search function with retry logic@@ -851,7 +852,7 @@ class BaseSearchEngine(ABC):return _execute_search()def _execute_search():- nonlocal success, error_message, results_count+ nonlocal success, error_message, error_type, results_countpreviews_received = Falsetry:@@ -981,6 +982,7 @@ class BaseSearchEngine(ABC):# If rate limiting is disabled, treat as regular errorsuccess = Falseerror_message = "Rate limit hit but rate limiting disabled"+ error_type = type(exc).__name__logger.warning(f"Rate limit hit on {self.__class__.__name__} but rate limiting is disabled")@@ -999,6 +1001,7 @@ class BaseSearchEngine(ABC):# cause chain frequently carries the request URL or auth# header from upstream HTTP clients (see #4131).error_message = safe_msg = self._scrub_error(e)+ error_type = type(e).__name__logger.warning(f"Search engine {self.__class__.__name__} failed: {safe_msg}")@@ -1013,6 +1016,7 @@ class BaseSearchEngine(ABC):self._record_search_failure(e)success = Falseerror_message = safe_msg = self._scrub_error(e)+ error_type = type(e).__name__logger.warning(f"Search engine {self.__class__.__name__} error: {safe_msg}")@@ -1029,6 +1033,7 @@ class BaseSearchEngine(ABC):response_time_ms=response_time_ms,success=success,error_message=error_message,+ error_type=error_type,)finally:# Clean up temporary search result storagediff --git a/tests/metrics/test_token_counting_attribution.py b/tests/metrics/test_token_counting_attribution.pyindex b8e14d9031..247e2de72c 100644--- a/tests/metrics/test_token_counting_attribution.py+++ b/tests/metrics/test_token_counting_attribution.py@@ -39,6 +39,7 @@ import asyncioimport threadingimport timefrom contextlib import contextmanager+from unittest.mock import Mockimport pytestfrom sqlalchemy import create_engine@@ -55,13 +56,20 @@ from local_deep_research.database.models.research import ResearchHistoryfrom local_deep_research.metrics.database import MetricsDatabasefrom local_deep_research.metrics.pricing.cost_calculator import CostCalculatorfrom local_deep_research.metrics.pricing.pricing_fetcher import PricingFetcher-from local_deep_research.metrics.search_tracker import SearchTracker+from local_deep_research.metrics.search_tracker import (+ SEARCH_STATUS_SKIPPED,+ SearchTracker,+)from local_deep_research.metrics.token_counter import (TokenCounter,TokenCountingCallback,)from local_deep_research.utilities.request_context import request_userfrom local_deep_research.utilities.thread_context import search_context+from local_deep_research.web_search_engines.rate_limiting import RateLimitError+from local_deep_research.web_search_engines.search_engine_base import (+ BaseSearchEngine,+)# Tables the metrics layer touches. ResearchHistory / RateLimit* are# required because ``_get_metrics_from_encrypted_db`` queries them inside@@ -759,18 +767,6 @@ def test_retry_attempts_are_counted_once_each_not_double_incremented(user_dbs):assert counts["total_completion_tokens"] == 50- strict=True,- reason=(- "FAILED SEARCHES ARE STORED AS SUCCESSES IN THE `success` COLUMN. "- "SearchCall carries both `success` (Integer, default=1, and the "- "subject of the idx_search_success_timestamp index) and "- "`success_status` (String). SearchTracker.record_search writes only "- "success_status, so every failed search lands with success=1. Any "- "consumer of SearchCall.success counts 100% success regardless of "- "what actually happened."- ),-)def test_failed_search_is_not_stored_as_a_success(user_dbs):with search_context({@@ -800,18 +796,8 @@ def test_failed_search_is_not_stored_as_a_success(user_dbs):assert failed_row.success == 0- strict=True,- reason=(- "error_type IS ALWAYS THE LITERAL 'unknown_error'. record_search "- "declares error_message as Optional[str] but classifies it with "- "`type(error_message).__name__ if isinstance(error_message, "- "Exception) else 'unknown_error'`. For the declared type the "- "isinstance branch is unreachable, so the error_type column carries "- "no information for any caller that follows the signature."- ),-)def test_search_error_type_reflects_the_actual_error(user_dbs):+ """Callers supply the class separately from the sanitized message."""with search_context({"research_id": "1", "username": "alice", "user_password": "pw"}):@@ -820,11 +806,200 @@ def test_search_error_type_reflects_the_actual_error(user_dbs):"dogs",success=False,error_message="HTTP 503 from upstream",+ error_type="HTTPError",)(row,) = user_dbs.rows("alice", SearchCall)assert row.error_message == "HTTP 503 from upstream"- assert row.error_type != "unknown_error"+ assert row.error_type == "HTTPError"+++ "success,status,expected_success,expected_status,expected_type",+ [+ (True, None, 1, "success", None),+ (False, None, 0, "error", None),+ (True, "error", 0, "error", None),+ (False, "success", 1, "success", None),+ (+ True,+ SEARCH_STATUS_SKIPPED,+ 0,+ SEARCH_STATUS_SKIPPED,+ "engine_unavailable",+ ),+ ],+)+def test_search_status_override_keeps_persisted_columns_consistent(+ user_dbs, success, status, expected_success, expected_status, expected_type+):+ with search_context(+ {"research_id": "1", "username": "alice", "user_password": "pw"}+ ):+ SearchTracker.record_search(+ "searxng", "q", success=success, status=status+ )++ (row,) = user_dbs.rows("alice", SearchCall)+ assert (row.success, row.success_status, row.error_type) == (+ expected_success,+ expected_status,+ expected_type,+ )+++def test_search_error_text_does_not_invent_an_exception_class(user_dbs):+ with search_context(+ {"research_id": "1", "username": "alice", "user_password": "pw"}+ ):+ SearchTracker.record_search(+ "searxng", "q", success=False, error_message="TimeoutError HTTP 503"+ )++ (row,) = user_dbs.rows("alice", SearchCall)+ assert row.error_type == "unknown_error"+ assert row.error_message == "TimeoutError HTTP 503"+++class OutcomeSearchEngine(BaseSearchEngine):+ """Run the real search and metrics lifecycle with a controlled provider."""++ def __init__(self, outcomes, *, rate_limiting=False, content_error=None):+ super().__init__(+ programmatic_mode=False,+ settings_snapshot={},+ search_snippets_only=content_error is None,+ )+ self.outcomes = iter(outcomes)+ self.attempts = 0+ self.content_error = content_error+ self.rate_tracker = Mock(enabled=rate_limiting)+ self.rate_tracker.get_wait_time.return_value = 0.0++ def _get_previews(self, query):+ self.attempts += 1+ outcome = next(self.outcomes)+ if isinstance(outcome, Exception):+ raise outcome+ return outcome++ def _get_full_content(self, items):+ if self.content_error:+ raise self.content_error+ return items+++ "error_class,rate_limiting,expected_attempts",+ [+ (ValueError, False, 1),+ (TimeoutError, False, 1),+ (RateLimitError, False, 1),+ (RateLimitError, True, 3),+ ],+)+def test_search_engine_persists_exception_type_and_sanitized_message(+ user_dbs, monkeypatch, error_class, rate_limiting, expected_attempts+):+ engine = OutcomeSearchEngine(+ [error_class("provider detail")] * expected_attempts,+ rate_limiting=rate_limiting,+ )+ monkeypatch.setattr(+ engine, "_scrub_error", lambda error: "sanitized failure"+ )+ assert (+ engine.run(+ "q",+ research_context={+ "research_id": "1",+ "username": "alice",+ "user_password": "pw",+ },+ )+ == []+ )++ assert engine.attempts == expected_attempts+ (row,) = user_dbs.rows("alice", SearchCall)+ assert row.error_type == error_class.__name__+ assert row.error_message == (+ "Rate limit hit but rate limiting disabled"+ if error_class is RateLimitError and not rate_limiting+ else "sanitized failure"+ )+ assert row.success_status == "error"+ assert row.success == 0+++ "results", [[], [{"title": "result", "snippet": "text"}]]+)+def test_search_engine_records_success_after_previous_failure(+ user_dbs, results+):+ engine = OutcomeSearchEngine([ValueError("provider failure"), results])+ context = {"research_id": "1", "username": "alice", "user_password": "pw"}+ assert engine.run("first", research_context=context) == []+ assert engine.run("second", research_context=context) == results++ failed, succeeded = user_dbs.rows("alice", SearchCall)+ assert (failed.success, failed.error_type) == (0, "ValueError")+ assert succeeded.success == 1+ assert succeeded.success_status == "success"+ assert succeeded.error_type is None+ assert succeeded.error_message is None+ assert succeeded.results_count == len(results)+++def test_search_engine_recovered_retry_records_one_success(user_dbs):+ results = [{"title": "result", "snippet": "text"}]+ engine = OutcomeSearchEngine(+ [RateLimitError("retry"), results], rate_limiting=True+ )+ assert (+ engine.run(+ "q",+ research_context={+ "research_id": "1",+ "username": "alice",+ "user_password": "pw",+ },+ )+ == results+ )++ assert engine.attempts == 2+ (row,) = user_dbs.rows("alice", SearchCall)+ assert row.success == 1+ assert row.success_status == "success"+ assert row.error_type is None+ assert row.error_message is None+ assert row.results_count == 1+++def test_search_engine_content_failure_preserves_exception_type(user_dbs):+ engine = OutcomeSearchEngine(+ [[{"title": "result", "snippet": "text"}]],+ content_error=ValueError("content conversion failed"),+ )+ assert (+ engine.run(+ "q",+ research_context={+ "research_id": "1",+ "username": "alice",+ "user_password": "pw",+ },+ )+ == []+ )++ (row,) = user_dbs.rows("alice", SearchCall)+ assert row.success == 0+ assert row.success_status == "error"+ assert row.error_type == "ValueError"+ assert row.error_message == "content conversion failed"@pytest.mark.xfail(