~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(pubmed): read PMC sections once, in order, with paragraphs outside any section (#7729)
Lukas68ef11832d18
diff --git a/changelog.d/7729.bugfix.md b/changelog.d/7729.bugfix.mdnew file mode 100644index 0000000000..98bb7d6cf8--- /dev/null+++ b/changelog.d/7729.bugfix.md@@ -0,0 +1,3 @@+PMC full text no longer repeats the paragraphs of nested sections: each subsection is read+once, under its own heading one level down. Paragraphs before the first section are kept,+and titles with inline markup such as `<italic>` keep their full text.diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.pyindex c22c7e17eb..854ed5336c 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py@@ -1291,8 +1291,13 @@ The default assumption should be that medical and scientific queries want RECENT# Extract article titletitle_elem = root.find(".//article-title")- if title_elem is not None and title_elem.text:- full_text.append(f"# {title_elem.text}")+ title_text = (+ "".join(title_elem.itertext()).strip()+ if title_elem is not None+ else ""+ )+ if title_text:+ full_text.append(f"# {title_text}")# Extract abstractabstract_paras = root.findall(".//abstract//p")@@ -1303,20 +1308,36 @@ The default assumption should be that medical and scientific queries want RECENTif text:full_text.append(text)- # Extract body content- body = root.find(".//body")- if body is not None:- for section in body.findall(".//sec"):- # Get section title- title = section.find(".//title")- if title is not None and title.text:- full_text.append(f"\n## {title.text}\n")-- # Get paragraphs- for p in section.findall(".//p"):- text = "".join(p.itertext())+ # Extract body content in document order: the paragraphs of a section,+ # then its subsections one heading level down. Searching every section+ # for `.//p` read a subsection's paragraphs again under each section+ # around it, and skipped paragraphs outside any section.+ def add_section_content(element, depth):+ for child in element:+ if child.tag == "sec":+ title = child.find("title")+ title_text = (+ "".join(title.itertext()).strip()+ if title is not None+ else ""+ )+ if title_text:+ full_text.append(+ f"\n{'#' * min(depth + 2, 6)} {title_text}\n"+ )+ add_section_content(child, depth + 1)+ elif child.tag == "p":+ text = "".join(child.itertext())if text:full_text.append(text)+ elif child.tag != "title":+ # A figure, table, list or box: its paragraphs and any+ # section inside it, at the depth of the section around it+ add_section_content(child, depth)++ body = root.find(".//body")+ if body is not None:+ add_section_content(body, 0)result_text = "\n\n".join(full_text)logger.debug(diff --git a/tests/web_search_engines/engines/test_search_engine_pubmed.py b/tests/web_search_engines/engines/test_search_engine_pubmed.pyindex 73019af9f6..e89d6f6fca 100644--- a/tests/web_search_engines/engines/test_search_engine_pubmed.py+++ b/tests/web_search_engines/engines/test_search_engine_pubmed.py@@ -1218,6 +1218,56 @@ class TestFindPmcIds:class TestGetPmcFullText:"""Tests for _get_pmc_full_text method."""+ def test_reads_nested_sections_once_in_document_order(self):+ """A subsection's paragraphs appear once, under its own heading."""+ from local_deep_research.web_search_engines.engines.search_engine_pubmed import (+ PubMedSearchEngine,+ )++ engine = PubMedSearchEngine()++ xml_response = """<pmc-articleset><article>+ <front><article-meta><title-group>+ <article-title>Study of <italic>E. coli</italic></article-title>+ </title-group></article-meta></front>+ <body>+ <p>Opening paragraph outside any section.</p>+ <sec><title>Methods</title><p>Methods intro.</p>+ <sec><title>Sampling</title><p>Sampling detail.</p></sec>+ </sec>+ <sec><title>Results</title><p>Results text.</p>+ <fig><caption><p>Figure caption.</p></caption></fig>+ <boxed-text><sec><title>Key points</title><p>Boxed point.</p></sec></boxed-text>+ </sec>+ </body>+ </article></pmc-articleset>"""++ with patch(+ "local_deep_research.web_search_engines.engines.search_engine_pubmed.safe_get"+ ) as mock_get:+ mock_response = Mock()+ mock_response.text = xml_response+ mock_response.raise_for_status = Mock()+ mock_get.return_value = mock_response++ result = engine._get_pmc_full_text("PMC12345")++ assert [+ part.strip() for part in result.split("\n\n") if part.strip()+ ] == [+ "# Study of E. coli",+ "Opening paragraph outside any section.",+ "## Methods",+ "Methods intro.",+ "### Sampling",+ "Sampling detail.",+ "## Results",+ "Results text.",+ "Figure caption.",+ "### Key points",+ "Boxed point.",+ ]+def test_extracts_full_text_from_xml(self):"""Extracts full text content from PMC XML."""from local_deep_research.web_search_engines.engines.search_engine_pubmed import (