~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(pubmed): retry standalone phrases missing from the index (#7661)
Chanhyo Jungacaf8a77f488
diff --git a/changelog.d/7600.bugfix.md b/changelog.d/7600.bugfix.mdnew file mode 100644index 0000000000..b096d28205--- /dev/null+++ b/changelog.d/7600.bugfix.md@@ -0,0 +1 @@+Part of #7600: retry eligible standalone Title/Abstract or tiab phrases with adjacency-preserving, word-order-relaxing proximity after successful zero-result searches; preserve literal scientific brackets when query optimization is disabled.diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.pyindex 854ed5336c..932db936e8 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py@@ -124,7 +124,24 @@ class PubMedSearchEngine(BaseSearchEngine):# PMC base URL for full textself.pmc_url = "https://www.ncbi.nlm.nih.gov/pmc/articles/"- def _get_result_count(self, query: str) -> int:+ @staticmethod+ def _validate_phrase_search_response(+ data: Dict[str, Any], *, require_ids: bool = False+ ) -> None:+ """Distinguish validated zeros from errors before semantic fallback."""+ result = data["esearchresult"]+ if int(result["count"]) < 0 or data.get("error") or result.get("ERROR"):+ raise ValueError("Invalid PubMed count response")+ if require_ids:+ ids = result["idlist"]+ if not isinstance(ids, list) or any(+ not isinstance(pmid, str) or not pmid.isdigit() for pmid in ids+ ):+ raise ValueError("Invalid PubMed ID response")++ def _get_result_count(+ self, query: str, *, failures: Optional[List[Exception]] = None+ ) -> int:"""Get the total number of results for a query without retrieving the results themselves.@@ -158,11 +175,15 @@ class PubMedSearchEngine(BaseSearchEngine):# Parse responsedata = response.json()count = int(data["esearchresult"]["count"])+ if failures is not None:+ self._validate_phrase_search_response(data)logger.info("PubMed reports {} total results for the query", count)return countexcept Exception as e:+ if failures is not None:+ failures.append(e)safe_msg = self._scrub_error(e)logger.warning(f"Error getting result count ({type(e).__name__}): {safe_msg}"@@ -550,7 +571,9 @@ The default assumption should be that medical and scientific queries want RECENT]return any(term in query.lower() for term in historical_terms)- def _adaptive_search(self, query: str) -> Tuple[List[str], str]:+ def _adaptive_search(+ self, query: str, *, failures: Optional[List[Exception]] = None+ ) -> Tuple[List[str], str]:"""Perform an adaptive search that adjusts based on topic volume and whetherthe query focuses on historical information.@@ -561,8 +584,10 @@ The default assumption should be that medical and scientific queries want RECENTReturns:Tuple of (list of PMIDs, search strategy used)"""+ failure_args = {"failures": failures} if failures is not None else {}+# Estimate topic volume- estimated_volume = self._get_result_count(query)+ estimated_volume = self._get_result_count(query, **failure_args)# Determine if the query is focused on historical informationis_historical_focused = self._is_historical_focused(query)@@ -595,7 +620,7 @@ The default assumption should be that medical and scientific queries want RECENTlogger.info(f"Using adaptive search strategy: {strategy} with filter: {time_filter}")- results = self._search_pubmed(query_with_time)+ results = self._search_pubmed(query_with_time, **failure_args)# If too few results, gradually expand time windowif len(results) < 5 and '"last 10 years"[pdat]' not in time_filter:@@ -604,7 +629,9 @@ The default assumption should be that medical and scientific queries want RECENT)expanded_time = self._expand_time_window(time_filter)query_with_expanded_time = f"({query}) AND {expanded_time}"- expanded_results = self._search_pubmed(query_with_expanded_time)+ expanded_results = self._search_pubmed(+ query_with_expanded_time, **failure_args+ )if len(expanded_results) > len(results):logger.info(@@ -617,18 +644,20 @@ The default assumption should be that medical and scientific queries want RECENTlogger.info("No results with time filter, trying without time restrictions")- results = self._search_pubmed(query)+ results = self._search_pubmed(query, **failure_args)strategy = "no_time_filter"else:# Historical query - run without time filterlogger.info("Using historical search strategy without date filtering")- results = self._search_pubmed(query)+ results = self._search_pubmed(query, **failure_args)return results, strategy- def _search_pubmed(self, query: str) -> List[str]:+ def _search_pubmed(+ self, query: str, *, failures: Optional[List[Exception]] = None+ ) -> List[str]:"""Search PubMed and return a list of article IDs.@@ -685,6 +714,8 @@ The default assumption should be that medical and scientific queries want RECENT# Parse responsedata = response.json()id_list: list[str] = data["esearchresult"]["idlist"]+ if failures is not None:+ self._validate_phrase_search_response(data, require_ids=True)total_count = data["esearchresult"].get("count", "unknown")logger.info(@@ -696,6 +727,8 @@ The default assumption should be that medical and scientific queries want RECENTreturn id_listexcept Exception as e:+ if failures is not None:+ failures.append(e)self._record_search_failure(e)safe_msg = self._scrub_error(e)logger.warning(@@ -1352,6 +1385,30 @@ The default assumption should be that medical and scientific queries want RECENT)return ""+ @staticmethod+ def _phrase_proximity_query(query: str) -> Optional[str]:+ """Keep adjacency but relax word order for a standalone tiab phrase."""+ match = re.fullmatch(+ r'\s*("([^"*]+)")\s*\[(Title/Abstract|tiab)\]\s*',+ query,+ flags=re.IGNORECASE,+ )+ if not match or len(match[2].split()) < 2:+ return None+ depth = 0+ for char in match[2]:+ if char == "[":+ depth += 1+ elif char == "]":+ depth -= 1+ if depth < 0:+ return None+ if depth:+ return None+ # Replace only the outer suffix, including its original spelling.+ suffix_end = query.rfind("]")+ return query[:suffix_end] + ":~0" + query[suffix_end:]+def _get_previews(self, query: str) -> List[Dict[str, Any]]:"""Get preview information for PubMed articles.@@ -1367,8 +1424,15 @@ The default assumption should be that medical and scientific queries want RECENT# Optimize the query for PubMed if LLM is availableoptimized_query = self._optimize_query_for_pubmed(query)+ # Retain every request failure locally: a later zero cannot erase one.+ failures: List[Exception] = []+ proximity_query = self._phrase_proximity_query(optimized_query)+ failure_args = {"failures": failures} if proximity_query else {}+# Perform adaptive search- pmid_list, strategy = self._adaptive_search(optimized_query)+ pmid_list, strategy = self._adaptive_search(+ optimized_query, **failure_args+ )# If no results, try a simplified queryif not pmid_list:@@ -1378,12 +1442,20 @@ The default assumption should be that medical and scientific queries want RECENTsimplified_query = self._simplify_query(optimized_query)if simplified_query != optimized_query:logger.info("Trying with simplified query")- pmid_list, strategy = self._adaptive_search(simplified_query)+ pmid_list, strategy = self._adaptive_search(+ simplified_query, **failure_args+ )if pmid_list:logger.info(f"Simplified query found {len(pmid_list)} results")+ if not pmid_list and proximity_query and not failures:+ # One logical adaptive stage, potentially several HTTP requests.+ pmid_list, strategy = self._adaptive_search(+ proximity_query, **failure_args+ )+if not pmid_list:logger.warning("No PubMed results found after query simplification")return []diff --git a/tests/web_search_engines/engines/test_pubmed_phrase_index_retry.py b/tests/web_search_engines/engines/test_pubmed_phrase_index_retry.pynew file mode 100644index 0000000000..1032007c03--- /dev/null+++ b/tests/web_search_engines/engines/test_pubmed_phrase_index_retry.py@@ -0,0 +1,245 @@+"""HTTP-level regressions for the narrowly scoped phrase-index fallback."""++from pathlib import Path+from unittest.mock import Mock, patch++import pytest++from local_deep_research.web_search_engines.engines import (+ search_engine_pubmed as pubmed,+)+++def harness():+ engine = pubmed.PubMedSearchEngine(+ llm=None, optimize_queries=False, max_results=150, days_limit=14+ )+ calls = []+ stages = []+ adaptive = engine._adaptive_search++ def stage(query, *args, **kwargs):+ stages.append(query)+ return adaptive(query, *args, **kwargs)++ def run(+ query,+ *,+ normal=False,+ recover=True,+ failure=None,+ volume=0,+ success_term=None,+ json_error=False,+ rate_failure=None,+ ):+ def get(url, params):+ calls.append(dict(params))+ response = Mock(status_code=200)+ response.text = "<PubmedArticleSet></PubmedArticleSet>"+ if url == engine.summary_url:+ response.json.return_value = {+ "result": {"123": {"title": "Recovered article"}}+ }+ elif url == engine.search_url:+ index = sum("term" in c for c in calls)+ if failure and index == failure[0]:+ if isinstance(failure[1], Exception):+ if json_error:+ response.json.side_effect = failure[1]+ else:+ response.raise_for_status.side_effect = failure[1]+ else:+ response.json.return_value = failure[1]+ return response+ ids = (+ ["123"]+ if normal+ or (recover and ":~0]" in params["term"])+ or (success_term and params["term"] == success_term)+ else []+ )+ response.json.return_value = {+ "esearchresult": {"count": str(volume), "idlist": ids}+ }+ return response++ waits = 0++ def wait(engine_type):+ nonlocal waits+ waits += 1+ if waits == rate_failure:+ raise pubmed.RateLimitError("test rate limit")+ return 0++ with (+ patch.object(pubmed, "safe_get", side_effect=get),+ patch.object(+ engine.rate_tracker, "apply_rate_limit", side_effect=wait+ ) as rate,+ patch.object(engine, "_adaptive_search", side_effect=stage),+ ):+ result = engine._get_previews(query)+ assert rate.call_count == len(calls) + (1 if rate_failure else 0)+ for c in calls:+ if c.get("retmax"):+ assert c["retmax"] == 100+ assert c["reldate"] == 14+ assert c["datetype"] == "pdat"+ return result++ return engine, calls, stages, run+++ "query",+ [+ '"cardiac imaging"[Title/Abstract]',+ '"cardiac imaging"[tiab]',+ '"2[18F]FDG uptake"[TIAB]',+ ],+)+def test_phrase_recovery(harness, query):+ engine, calls, stages, run = harness+ assert Path(pubmed.__file__).resolve() == (+ Path(__file__).resolve().parents[3]+ / "src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py"+ )+ result = run(query)+ assert [p["id"] for p in result] == ["123"]+ assert stages == [query, query[:-1] + ":~0]"]+ assert result[0]["title"] == "Recovered article"+++def test_zero_terminates(harness):+ _, _, stages, run = harness+ assert run('"cardiac imaging"[tiab]', recover=False) == []+ assert stages == ['"cardiac imaging"[tiab]', '"cardiac imaging"[tiab:~0]']+++ "query",+ [+ '"cardiac imaging"[Title]',+ '"cardiac imaging"[tiab:noexp]',+ '"cardiac imaging"[tiab:~0]',+ '"cardiac* imaging"[tiab]',+ '"cardiac"[tiab]',+ "cardiac imaging",+ '("cardiac imaging"[tiab])',+ '"cardiac imaging"[tiab] AND cancer',+ '"cardiac imaging[tiab]',+ '"2[18F uptake"[tiab]',+ '"2]18F[ uptake"[tiab]',+ ],+)+def test_unsupported(harness, query):+ _, _, stages, run = harness+ assert run(query, recover=False) == []+ assert not any(":~0]" in s for s in stages) or ":~0]" in query+++def test_success_bypasses(harness):+ _, _, stages, run = harness+ assert run('"cardiac imaging"[tiab]', normal=True)+ assert len(stages) == 1+++def test_distinct_simplification(harness):+ _, _, stages, run = harness+ query = '"cardiac imaging"[tiab]'+ assert run(query)+ assert stages == [+ query,+ '"cardiac imaging"[tiab]',+ '"cardiac imaging"[tiab:~0]',+ ]++[email protected]("position", [1, 2, 3])+ "error",+ [+ RuntimeError("HTTP 429"),+ {},+ {"esearchresult": {"count": "bad", "idlist": []}},+ ],+)+def test_errors_block_broadening(harness, position, error):+ _, _, stages, run = harness+ run('"cardiac imaging"[tiab]', failure=(position, error))+ assert len(stages) == 1+++def test_expanded_window_failure_is_sticky(harness):+ _, _, stages, run = harness+ run(+ '"cardiac imaging"[tiab]',+ volume=6000,+ failure=(3, RuntimeError("transport")),+ )+ assert len(stages) == 1++[email protected]("position", [2, 3])+def test_invalid_ids_block_broadening(harness, position):+ _, _, stages, run = harness+ run(+ '"cardiac imaging"[tiab]',+ failure=(position, {"esearchresult": {"count": "0", "idlist": "bad"}}),+ )+ assert len(stages) == 1+++def test_simplified_success_bypasses_proximity(harness):+ _, _, stages, run = harness+ assert run(+ '"cardiac imaging"[tiab]', success_term='"cardiac imaging"[tiab]'+ )+ assert stages == ['"cardiac imaging"[tiab]', '"cardiac imaging"[tiab]']++[email protected]("position", [1, 2, 3])+def test_json_failure_and_key_scrubbing(harness, position):+ engine, _, stages, run = harness+ engine.api_key = "phrase-retry-test-secret"+ messages = []+ with patch.object(pubmed, "logger") as log:+ run(+ '"cardiac imaging"[tiab]',+ failure=(+ position,+ ValueError("bad JSON api_key=phrase-retry-test-secret"),+ ),+ json_error=True,+ )+ messages = [str(call) for call in log.mock_calls]+ assert len(stages) == 1+ assert "phrase-retry-test-secret" not in " ".join(messages)+++def test_failure_state_does_not_cross_invocations(harness):+ _, calls, stages, run = harness+ assert (+ run('"cardiac imaging"[tiab]', failure=(1, RuntimeError("transport")))+ == []+ )+ calls.clear()+ stages.clear()+ assert run('"cardiac imaging"[tiab]')+ assert len(stages) == 2++[email protected]("position", [1, 2, 3])+def test_rate_tracker_failure_blocks_proximity(harness, position):+ _, _, stages, run = harness+ run('"cardiac imaging"[tiab]', rate_failure=position)+ assert stages == ['"cardiac imaging"[tiab]']+++def test_simplified_count_error_does_not_allow_proximity(harness):+ _, _, stages, run = harness+ run('"cardiac imaging"[tiab]', failure=(4, RuntimeError("HTTP 503")))+ assert stages == ['"cardiac imaging"[tiab]', '"cardiac imaging"[tiab]']