~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(relevance-filter): enhance relevance filter logging, exception handling, and policy-awareness (#5177)
S9543d9c8d475
diff --git a/src/local_deep_research/web_search_engines/relevance_filter.py b/src/local_deep_research/web_search_engines/relevance_filter.pyindex 758ca3d7e3..a66335b1d9 100644--- a/src/local_deep_research/web_search_engines/relevance_filter.py+++ b/src/local_deep_research/web_search_engines/relevance_filter.py@@ -324,6 +324,10 @@ def filter_previews_for_relevance(batch_len = min(effective_batch, len(previews) - batch_start)for li in batch_result:if not (0 <= li < batch_len):+ logger.warning(+ f"[{engine_name}] Relevance filter returned out-of-range index: "+ f"{li} for batch starting at {batch_start} (batch size {batch_len})"+ )continueglobal_idx = batch_start + liif global_idx in seen:@@ -339,13 +343,15 @@ def filter_previews_for_relevance()# Empty result is a valid LLM judgment ("none relevant"). Log a- # warning on larger batches so users can notice a misbehaving model,+ # warning/error on larger batches so users can notice a misbehaving model,# but do not override the decision.if not ranked_results and len(previews) > 2:- logger.warning(- f"[{engine_name}] LLM filter judged all {len(previews)} "- f"results irrelevant. If this is unexpected, verify your "- f"model handles structured output correctly."+ completed_non_none = [r for r in results_per_batch if r is not None]+ empty_parses_count = sum(1 for r in completed_non_none if len(r) == 0)+ logger.error(+ f"[{engine_name}] LLM relevance filter parser returned empty list for all completed batches: "+ f"kept 0 of {len(previews)} across {empty_parses_count}/{len(batches)} empty parses. "+ f"If this is unexpected, verify your model returns the index list as requested (see prior log lines at DEBUG level for raw response).")# Apply cap if set, keeping ranked_results and kept_indices aligned.@@ -399,7 +405,14 @@ def _invoke_text(llm, prompt: str, engine_name: str) -> List[int]:if isinstance(base_llm, ChatOllama):invoke_kwargs["reasoning"] = False- result = llm.invoke(prompt, **invoke_kwargs)+ try:+ result = llm.invoke(prompt, **invoke_kwargs)+ except Exception as exc:+ safe_msg = scrub_error(exc)+ logger.warning(+ f"[{engine_name}] LLM invocation failed ({type(exc).__name__}): {safe_msg}"+ )+ raise# LangChain chat models return a Message; LLMs return a string.text = getattr(result, "content", result)@@ -410,7 +423,16 @@ def _invoke_text(llm, prompt: str, engine_name: str) -> List[int]:)return []+ logger.info(f"[{engine_name}] LLM call returned {len(text)} chars")+indices = [int(m) for m in _INT_RE.findall(text)]+ if not indices:+ logger.warning(+ f"[{engine_name}] Parser found no integers in LLM response."+ )+ logger.debug(+ f"[{engine_name}] Raw response that failed parsing: {repr(text[:300])}"+ )logger.debug(f"[{engine_name}] Text output parsed {len(indices)} indices: {indices}")diff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.pyindex 03dd2d89e2..68625dc112 100644--- a/src/local_deep_research/web_search_engines/search_engine_base.py+++ b/src/local_deep_research/web_search_engines/search_engine_base.py@@ -815,7 +815,7 @@ class BaseSearchEngine(ABC):Returns:Filtered list of preview dictionaries"""- engine_name = type(self).__name__+ engine_name = self._engine_name or type(self).__name__if not self.llm or len(previews) <= 1:logger.debug(diff --git a/tests/web_search_engines/test_relevance_filter.py b/tests/web_search_engines/test_relevance_filter.pyindex 0b8a62f1bb..2266de994b 100644--- a/tests/web_search_engines/test_relevance_filter.py+++ b/tests/web_search_engines/test_relevance_filter.py@@ -196,16 +196,20 @@ def test_empty_judgment_does_not_warn_on_2_previews(loguru_caplog):def test_empty_judgment_warns_on_3_or_more_previews(loguru_caplog):- """The "all irrelevant" warning fires when the LLM rejects 3+ previews+ """The "all irrelevant" error fires when the LLM rejects 3+ previewsso the user notices a misbehaving model."""previews = _previews(3)llm = _llm_returning("none") # no integers in text- with loguru_caplog.at_level("WARNING"):+ with loguru_caplog.at_level("ERROR"):result = filter_previews_for_relevance(llm, previews, "q")assert result == []- assert "judged all 3 results irrelevant" in loguru_caplog.text+ assert (+ "LLM relevance filter parser returned empty list for all completed batches"+ in loguru_caplog.text+ )+ assert "kept 0 of 3 across 1/1 empty parses" in loguru_caplog.text# ---------- Batching ----------@@ -338,3 +342,56 @@ def test_parallel_batch_timeout_does_not_hang(monkeypatch):# All batches timed out → capped fallback slice.assert result == previews[:3]+++def test_relevance_filter_logs_out_of_range_indices(loguru_caplog):+ """Integers in the response that are out of range trigger a warning."""+ previews = _previews(3)+ llm = _llm_returning("0, 2, 99")++ with loguru_caplog.at_level("WARNING"):+ result = filter_previews_for_relevance(llm, previews, "q")++ assert result == [previews[0], previews[2]]+ assert (+ "Relevance filter returned out-of-range index: 99" in loguru_caplog.text+ )+++def test_relevance_filter_all_completed_batches_empty_logs_error(loguru_caplog):+ """When all completed batches return empty results, logger.error is triggered with counter."""+ previews = _previews(5)+ # 2 batches: first returns 'none', second returns 'none'+ llm = Mock()+ llm.invoke.side_effect = ["none", "none"]++ with loguru_caplog.at_level("ERROR"):+ result = filter_previews_for_relevance(+ llm, previews, "q", batch_size=3, max_parallel_batches=1+ )++ assert result == []+ assert (+ "LLM relevance filter parser returned empty list for all completed batches"+ in loguru_caplog.text+ )+ assert "kept 0 of 5 across 2/2 empty parses" in loguru_caplog.text+++def test_relevance_filter_logs_raw_response_on_empty_parse(loguru_caplog):+ """_invoke_text logs raw response when parser finds no integers."""+ from local_deep_research.web_search_engines.relevance_filter import (+ _invoke_text,+ )++ llm = _llm_returning("This is a response without any digits.")++ with loguru_caplog.at_level("DEBUG"):+ result = _invoke_text(llm, "prompt", "test_engine")++ assert result == []+ assert "Parser found no integers in LLM response." in loguru_caplog.text+ assert (+ "Raw response that failed parsing: 'This is a response without any digits.'"+ in loguru_caplog.text+ )