Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(relevance-filter): enhance relevance filter logging, exception handling, and policy-awareness (#5177)

S9543d9c8d475
diff --git a/src/local_deep_research/web_search_engines/relevance_filter.py b/src/local_deep_research/web_search_engines/relevance_filter.py
index 758ca3d7e3..a66335b1d9 100644
--- a/src/local_deep_research/web_search_engines/relevance_filter.py
+++ b/src/local_deep_research/web_search_engines/relevance_filter.py
@@ -324,6 +324,10 @@ def filter_previews_for_relevance(
batch_len = min(effective_batch, len(previews) - batch_start)
for li in batch_result:
if not (0 <= li < batch_len):
+ logger.warning(
+ f"[{engine_name}] Relevance filter returned out-of-range index: "
+ f"{li} for batch starting at {batch_start} (batch size {batch_len})"
+ )
continue
global_idx = batch_start + li
if global_idx in seen:
@@ -339,13 +343,15 @@ def filter_previews_for_relevance(
)
# Empty result is a valid LLM judgment ("none relevant"). Log a
- # warning on larger batches so users can notice a misbehaving model,
+ # warning/error on larger batches so users can notice a misbehaving model,
# but do not override the decision.
if not ranked_results and len(previews) > 2:
- logger.warning(
- f"[{engine_name}] LLM filter judged all {len(previews)} "
- f"results irrelevant. If this is unexpected, verify your "
- f"model handles structured output correctly."
+ completed_non_none = [r for r in results_per_batch if r is not None]
+ empty_parses_count = sum(1 for r in completed_non_none if len(r) == 0)
+ logger.error(
+ f"[{engine_name}] LLM relevance filter parser returned empty list for all completed batches: "
+ f"kept 0 of {len(previews)} across {empty_parses_count}/{len(batches)} empty parses. "
+ f"If this is unexpected, verify your model returns the index list as requested (see prior log lines at DEBUG level for raw response)."
)
# Apply cap if set, keeping ranked_results and kept_indices aligned.
@@ -399,7 +405,14 @@ def _invoke_text(llm, prompt: str, engine_name: str) -> List[int]:
if isinstance(base_llm, ChatOllama):
invoke_kwargs["reasoning"] = False
- result = llm.invoke(prompt, **invoke_kwargs)
+ try:
+ result = llm.invoke(prompt, **invoke_kwargs)
+ except Exception as exc:
+ safe_msg = scrub_error(exc)
+ logger.warning(
+ f"[{engine_name}] LLM invocation failed ({type(exc).__name__}): {safe_msg}"
+ )
+ raise
# LangChain chat models return a Message; LLMs return a string.
text = getattr(result, "content", result)
@@ -410,7 +423,16 @@ def _invoke_text(llm, prompt: str, engine_name: str) -> List[int]:
)
return []
+ logger.info(f"[{engine_name}] LLM call returned {len(text)} chars")
+
indices = [int(m) for m in _INT_RE.findall(text)]
+ if not indices:
+ logger.warning(
+ f"[{engine_name}] Parser found no integers in LLM response."
+ )
+ logger.debug(
+ f"[{engine_name}] Raw response that failed parsing: {repr(text[:300])}"
+ )
logger.debug(
f"[{engine_name}] Text output parsed {len(indices)} indices: {indices}"
)
diff --git a/src/local_deep_research/web_search_engines/search_engine_base.py b/src/local_deep_research/web_search_engines/search_engine_base.py
index 03dd2d89e2..68625dc112 100644
--- a/src/local_deep_research/web_search_engines/search_engine_base.py
+++ b/src/local_deep_research/web_search_engines/search_engine_base.py
@@ -815,7 +815,7 @@ class BaseSearchEngine(ABC):
Returns:
Filtered list of preview dictionaries
"""
- engine_name = type(self).__name__
+ engine_name = self._engine_name or type(self).__name__
if not self.llm or len(previews) <= 1:
logger.debug(
diff --git a/tests/web_search_engines/test_relevance_filter.py b/tests/web_search_engines/test_relevance_filter.py
index 0b8a62f1bb..2266de994b 100644
--- a/tests/web_search_engines/test_relevance_filter.py
+++ b/tests/web_search_engines/test_relevance_filter.py
@@ -196,16 +196,20 @@ def test_empty_judgment_does_not_warn_on_2_previews(loguru_caplog):
def test_empty_judgment_warns_on_3_or_more_previews(loguru_caplog):
- """The "all irrelevant" warning fires when the LLM rejects 3+ previews
+ """The "all irrelevant" error fires when the LLM rejects 3+ previews
so the user notices a misbehaving model."""
previews = _previews(3)
llm = _llm_returning("none") # no integers in text
- with loguru_caplog.at_level("WARNING"):
+ with loguru_caplog.at_level("ERROR"):
result = filter_previews_for_relevance(llm, previews, "q")
assert result == []
- assert "judged all 3 results irrelevant" in loguru_caplog.text
+ assert (
+ "LLM relevance filter parser returned empty list for all completed batches"
+ in loguru_caplog.text
+ )
+ assert "kept 0 of 3 across 1/1 empty parses" in loguru_caplog.text
# ---------- Batching ----------
@@ -338,3 +342,56 @@ def test_parallel_batch_timeout_does_not_hang(monkeypatch):
# All batches timed out → capped fallback slice.
assert result == previews[:3]
+
+
+def test_relevance_filter_logs_out_of_range_indices(loguru_caplog):
+ """Integers in the response that are out of range trigger a warning."""
+ previews = _previews(3)
+ llm = _llm_returning("0, 2, 99")
+
+ with loguru_caplog.at_level("WARNING"):
+ result = filter_previews_for_relevance(llm, previews, "q")
+
+ assert result == [previews[0], previews[2]]
+ assert (
+ "Relevance filter returned out-of-range index: 99" in loguru_caplog.text
+ )
+
+
+def test_relevance_filter_all_completed_batches_empty_logs_error(loguru_caplog):
+ """When all completed batches return empty results, logger.error is triggered with counter."""
+ previews = _previews(5)
+ # 2 batches: first returns 'none', second returns 'none'
+ llm = Mock()
+ llm.invoke.side_effect = ["none", "none"]
+
+ with loguru_caplog.at_level("ERROR"):
+ result = filter_previews_for_relevance(
+ llm, previews, "q", batch_size=3, max_parallel_batches=1
+ )
+
+ assert result == []
+ assert (
+ "LLM relevance filter parser returned empty list for all completed batches"
+ in loguru_caplog.text
+ )
+ assert "kept 0 of 5 across 2/2 empty parses" in loguru_caplog.text
+
+
+def test_relevance_filter_logs_raw_response_on_empty_parse(loguru_caplog):
+ """_invoke_text logs raw response when parser finds no integers."""
+ from local_deep_research.web_search_engines.relevance_filter import (
+ _invoke_text,
+ )
+
+ llm = _llm_returning("This is a response without any digits.")
+
+ with loguru_caplog.at_level("DEBUG"):
+ result = _invoke_text(llm, "prompt", "test_engine")
+
+ assert result == []
+ assert "Parser found no integers in LLM response." in loguru_caplog.text
+ assert (
+ "Raw response that failed parsing: 'This is a response without any digits.'"
+ in loguru_caplog.text
+ )