Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(pubmed): retry standalone phrases missing from the index (#7661)

Chanhyo Jungacaf8a77f488
diff --git a/changelog.d/7600.bugfix.md b/changelog.d/7600.bugfix.md
new file mode 100644
index 0000000000..b096d28205
--- /dev/null
+++ b/changelog.d/7600.bugfix.md
@@ -0,0 +1 @@
+Part of #7600: retry eligible standalone Title/Abstract or tiab phrases with adjacency-preserving, word-order-relaxing proximity after successful zero-result searches; preserve literal scientific brackets when query optimization is disabled.
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
index 854ed5336c..932db936e8 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
@@ -124,7 +124,24 @@ class PubMedSearchEngine(BaseSearchEngine):
# PMC base URL for full text
self.pmc_url = "https://www.ncbi.nlm.nih.gov/pmc/articles/"
- def _get_result_count(self, query: str) -> int:
+ @staticmethod
+ def _validate_phrase_search_response(
+ data: Dict[str, Any], *, require_ids: bool = False
+ ) -> None:
+ """Distinguish validated zeros from errors before semantic fallback."""
+ result = data["esearchresult"]
+ if int(result["count"]) < 0 or data.get("error") or result.get("ERROR"):
+ raise ValueError("Invalid PubMed count response")
+ if require_ids:
+ ids = result["idlist"]
+ if not isinstance(ids, list) or any(
+ not isinstance(pmid, str) or not pmid.isdigit() for pmid in ids
+ ):
+ raise ValueError("Invalid PubMed ID response")
+
+ def _get_result_count(
+ self, query: str, *, failures: Optional[List[Exception]] = None
+ ) -> int:
"""
Get the total number of results for a query without retrieving the results themselves.
@@ -158,11 +175,15 @@ class PubMedSearchEngine(BaseSearchEngine):
# Parse response
data = response.json()
count = int(data["esearchresult"]["count"])
+ if failures is not None:
+ self._validate_phrase_search_response(data)
logger.info("PubMed reports {} total results for the query", count)
return count
except Exception as e:
+ if failures is not None:
+ failures.append(e)
safe_msg = self._scrub_error(e)
logger.warning(
f"Error getting result count ({type(e).__name__}): {safe_msg}"
@@ -550,7 +571,9 @@ The default assumption should be that medical and scientific queries want RECENT
]
return any(term in query.lower() for term in historical_terms)
- def _adaptive_search(self, query: str) -> Tuple[List[str], str]:
+ def _adaptive_search(
+ self, query: str, *, failures: Optional[List[Exception]] = None
+ ) -> Tuple[List[str], str]:
"""
Perform an adaptive search that adjusts based on topic volume and whether
the query focuses on historical information.
@@ -561,8 +584,10 @@ The default assumption should be that medical and scientific queries want RECENT
Returns:
Tuple of (list of PMIDs, search strategy used)
"""
+ failure_args = {"failures": failures} if failures is not None else {}
+
# Estimate topic volume
- estimated_volume = self._get_result_count(query)
+ estimated_volume = self._get_result_count(query, **failure_args)
# Determine if the query is focused on historical information
is_historical_focused = self._is_historical_focused(query)
@@ -595,7 +620,7 @@ The default assumption should be that medical and scientific queries want RECENT
logger.info(
f"Using adaptive search strategy: {strategy} with filter: {time_filter}"
)
- results = self._search_pubmed(query_with_time)
+ results = self._search_pubmed(query_with_time, **failure_args)
# If too few results, gradually expand time window
if len(results) < 5 and '"last 10 years"[pdat]' not in time_filter:
@@ -604,7 +629,9 @@ The default assumption should be that medical and scientific queries want RECENT
)
expanded_time = self._expand_time_window(time_filter)
query_with_expanded_time = f"({query}) AND {expanded_time}"
- expanded_results = self._search_pubmed(query_with_expanded_time)
+ expanded_results = self._search_pubmed(
+ query_with_expanded_time, **failure_args
+ )
if len(expanded_results) > len(results):
logger.info(
@@ -617,18 +644,20 @@ The default assumption should be that medical and scientific queries want RECENT
logger.info(
"No results with time filter, trying without time restrictions"
)
- results = self._search_pubmed(query)
+ results = self._search_pubmed(query, **failure_args)
strategy = "no_time_filter"
else:
# Historical query - run without time filter
logger.info(
"Using historical search strategy without date filtering"
)
- results = self._search_pubmed(query)
+ results = self._search_pubmed(query, **failure_args)
return results, strategy
- def _search_pubmed(self, query: str) -> List[str]:
+ def _search_pubmed(
+ self, query: str, *, failures: Optional[List[Exception]] = None
+ ) -> List[str]:
"""
Search PubMed and return a list of article IDs.
@@ -685,6 +714,8 @@ The default assumption should be that medical and scientific queries want RECENT
# Parse response
data = response.json()
id_list: list[str] = data["esearchresult"]["idlist"]
+ if failures is not None:
+ self._validate_phrase_search_response(data, require_ids=True)
total_count = data["esearchresult"].get("count", "unknown")
logger.info(
@@ -696,6 +727,8 @@ The default assumption should be that medical and scientific queries want RECENT
return id_list
except Exception as e:
+ if failures is not None:
+ failures.append(e)
self._record_search_failure(e)
safe_msg = self._scrub_error(e)
logger.warning(
@@ -1352,6 +1385,30 @@ The default assumption should be that medical and scientific queries want RECENT
)
return ""
+ @staticmethod
+ def _phrase_proximity_query(query: str) -> Optional[str]:
+ """Keep adjacency but relax word order for a standalone tiab phrase."""
+ match = re.fullmatch(
+ r'\s*("([^"*]+)")\s*\[(Title/Abstract|tiab)\]\s*',
+ query,
+ flags=re.IGNORECASE,
+ )
+ if not match or len(match[2].split()) < 2:
+ return None
+ depth = 0
+ for char in match[2]:
+ if char == "[":
+ depth += 1
+ elif char == "]":
+ depth -= 1
+ if depth < 0:
+ return None
+ if depth:
+ return None
+ # Replace only the outer suffix, including its original spelling.
+ suffix_end = query.rfind("]")
+ return query[:suffix_end] + ":~0" + query[suffix_end:]
+
def _get_previews(self, query: str) -> List[Dict[str, Any]]:
"""
Get preview information for PubMed articles.
@@ -1367,8 +1424,15 @@ The default assumption should be that medical and scientific queries want RECENT
# Optimize the query for PubMed if LLM is available
optimized_query = self._optimize_query_for_pubmed(query)
+ # Retain every request failure locally: a later zero cannot erase one.
+ failures: List[Exception] = []
+ proximity_query = self._phrase_proximity_query(optimized_query)
+ failure_args = {"failures": failures} if proximity_query else {}
+
# Perform adaptive search
- pmid_list, strategy = self._adaptive_search(optimized_query)
+ pmid_list, strategy = self._adaptive_search(
+ optimized_query, **failure_args
+ )
# If no results, try a simplified query
if not pmid_list:
@@ -1378,12 +1442,20 @@ The default assumption should be that medical and scientific queries want RECENT
simplified_query = self._simplify_query(optimized_query)
if simplified_query != optimized_query:
logger.info("Trying with simplified query")
- pmid_list, strategy = self._adaptive_search(simplified_query)
+ pmid_list, strategy = self._adaptive_search(
+ simplified_query, **failure_args
+ )
if pmid_list:
logger.info(
f"Simplified query found {len(pmid_list)} results"
)
+ if not pmid_list and proximity_query and not failures:
+ # One logical adaptive stage, potentially several HTTP requests.
+ pmid_list, strategy = self._adaptive_search(
+ proximity_query, **failure_args
+ )
+
if not pmid_list:
logger.warning("No PubMed results found after query simplification")
return []
diff --git a/tests/web_search_engines/engines/test_pubmed_phrase_index_retry.py b/tests/web_search_engines/engines/test_pubmed_phrase_index_retry.py
new file mode 100644
index 0000000000..1032007c03
--- /dev/null
+++ b/tests/web_search_engines/engines/test_pubmed_phrase_index_retry.py
@@ -0,0 +1,245 @@
+"""HTTP-level regressions for the narrowly scoped phrase-index fallback."""
+
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+import pytest
+
+from local_deep_research.web_search_engines.engines import (
+ search_engine_pubmed as pubmed,
+)
+
+
+def harness():
+ engine = pubmed.PubMedSearchEngine(
+ llm=None, optimize_queries=False, max_results=150, days_limit=14
+ )
+ calls = []
+ stages = []
+ adaptive = engine._adaptive_search
+
+ def stage(query, *args, **kwargs):
+ stages.append(query)
+ return adaptive(query, *args, **kwargs)
+
+ def run(
+ query,
+ *,
+ normal=False,
+ recover=True,
+ failure=None,
+ volume=0,
+ success_term=None,
+ json_error=False,
+ rate_failure=None,
+ ):
+ def get(url, params):
+ calls.append(dict(params))
+ response = Mock(status_code=200)
+ response.text = "<PubmedArticleSet></PubmedArticleSet>"
+ if url == engine.summary_url:
+ response.json.return_value = {
+ "result": {"123": {"title": "Recovered article"}}
+ }
+ elif url == engine.search_url:
+ index = sum("term" in c for c in calls)
+ if failure and index == failure[0]:
+ if isinstance(failure[1], Exception):
+ if json_error:
+ response.json.side_effect = failure[1]
+ else:
+ response.raise_for_status.side_effect = failure[1]
+ else:
+ response.json.return_value = failure[1]
+ return response
+ ids = (
+ ["123"]
+ if normal
+ or (recover and ":~0]" in params["term"])
+ or (success_term and params["term"] == success_term)
+ else []
+ )
+ response.json.return_value = {
+ "esearchresult": {"count": str(volume), "idlist": ids}
+ }
+ return response
+
+ waits = 0
+
+ def wait(engine_type):
+ nonlocal waits
+ waits += 1
+ if waits == rate_failure:
+ raise pubmed.RateLimitError("test rate limit")
+ return 0
+
+ with (
+ patch.object(pubmed, "safe_get", side_effect=get),
+ patch.object(
+ engine.rate_tracker, "apply_rate_limit", side_effect=wait
+ ) as rate,
+ patch.object(engine, "_adaptive_search", side_effect=stage),
+ ):
+ result = engine._get_previews(query)
+ assert rate.call_count == len(calls) + (1 if rate_failure else 0)
+ for c in calls:
+ if c.get("retmax"):
+ assert c["retmax"] == 100
+ assert c["reldate"] == 14
+ assert c["datetype"] == "pdat"
+ return result
+
+ return engine, calls, stages, run
+
+
+ "query",
+ [
+ '"cardiac imaging"[Title/Abstract]',
+ '"cardiac imaging"[tiab]',
+ '"2[18F]FDG uptake"[TIAB]',
+ ],
+)
+def test_phrase_recovery(harness, query):
+ engine, calls, stages, run = harness
+ assert Path(pubmed.__file__).resolve() == (
+ Path(__file__).resolve().parents[3]
+ / "src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py"
+ )
+ result = run(query)
+ assert [p["id"] for p in result] == ["123"]
+ assert stages == [query, query[:-1] + ":~0]"]
+ assert result[0]["title"] == "Recovered article"
+
+
+def test_zero_terminates(harness):
+ _, _, stages, run = harness
+ assert run('"cardiac imaging"[tiab]', recover=False) == []
+ assert stages == ['"cardiac imaging"[tiab]', '"cardiac imaging"[tiab:~0]']
+
+
+ "query",
+ [
+ '"cardiac imaging"[Title]',
+ '"cardiac imaging"[tiab:noexp]',
+ '"cardiac imaging"[tiab:~0]',
+ '"cardiac* imaging"[tiab]',
+ '"cardiac"[tiab]',
+ "cardiac imaging",
+ '("cardiac imaging"[tiab])',
+ '"cardiac imaging"[tiab] AND cancer',
+ '"cardiac imaging[tiab]',
+ '"2[18F uptake"[tiab]',
+ '"2]18F[ uptake"[tiab]',
+ ],
+)
+def test_unsupported(harness, query):
+ _, _, stages, run = harness
+ assert run(query, recover=False) == []
+ assert not any(":~0]" in s for s in stages) or ":~0]" in query
+
+
+def test_success_bypasses(harness):
+ _, _, stages, run = harness
+ assert run('"cardiac imaging"[tiab]', normal=True)
+ assert len(stages) == 1
+
+
+def test_distinct_simplification(harness):
+ _, _, stages, run = harness
+ query = '"cardiac imaging"[tiab]'
+ assert run(query)
+ assert stages == [
+ query,
+ '"cardiac imaging"[tiab]',
+ '"cardiac imaging"[tiab:~0]',
+ ]
+
+
[email protected]("position", [1, 2, 3])
+ "error",
+ [
+ RuntimeError("HTTP 429"),
+ {},
+ {"esearchresult": {"count": "bad", "idlist": []}},
+ ],
+)
+def test_errors_block_broadening(harness, position, error):
+ _, _, stages, run = harness
+ run('"cardiac imaging"[tiab]', failure=(position, error))
+ assert len(stages) == 1
+
+
+def test_expanded_window_failure_is_sticky(harness):
+ _, _, stages, run = harness
+ run(
+ '"cardiac imaging"[tiab]',
+ volume=6000,
+ failure=(3, RuntimeError("transport")),
+ )
+ assert len(stages) == 1
+
+
[email protected]("position", [2, 3])
+def test_invalid_ids_block_broadening(harness, position):
+ _, _, stages, run = harness
+ run(
+ '"cardiac imaging"[tiab]',
+ failure=(position, {"esearchresult": {"count": "0", "idlist": "bad"}}),
+ )
+ assert len(stages) == 1
+
+
+def test_simplified_success_bypasses_proximity(harness):
+ _, _, stages, run = harness
+ assert run(
+ '"cardiac imaging"[tiab]', success_term='"cardiac imaging"[tiab]'
+ )
+ assert stages == ['"cardiac imaging"[tiab]', '"cardiac imaging"[tiab]']
+
+
[email protected]("position", [1, 2, 3])
+def test_json_failure_and_key_scrubbing(harness, position):
+ engine, _, stages, run = harness
+ engine.api_key = "phrase-retry-test-secret"
+ messages = []
+ with patch.object(pubmed, "logger") as log:
+ run(
+ '"cardiac imaging"[tiab]',
+ failure=(
+ position,
+ ValueError("bad JSON api_key=phrase-retry-test-secret"),
+ ),
+ json_error=True,
+ )
+ messages = [str(call) for call in log.mock_calls]
+ assert len(stages) == 1
+ assert "phrase-retry-test-secret" not in " ".join(messages)
+
+
+def test_failure_state_does_not_cross_invocations(harness):
+ _, calls, stages, run = harness
+ assert (
+ run('"cardiac imaging"[tiab]', failure=(1, RuntimeError("transport")))
+ == []
+ )
+ calls.clear()
+ stages.clear()
+ assert run('"cardiac imaging"[tiab]')
+ assert len(stages) == 2
+
+
[email protected]("position", [1, 2, 3])
+def test_rate_tracker_failure_blocks_proximity(harness, position):
+ _, _, stages, run = harness
+ run('"cardiac imaging"[tiab]', rate_failure=position)
+ assert stages == ['"cardiac imaging"[tiab]']
+
+
+def test_simplified_count_error_does_not_allow_proximity(harness):
+ _, _, stages, run = harness
+ run('"cardiac imaging"[tiab]', failure=(4, RuntimeError("HTTP 503")))
+ assert stages == ['"cardiac imaging"[tiab]', '"cardiac imaging"[tiab]']