Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(pubmed): read PMC sections once, in order, with paragraphs outside any section (#7729)

Lukas68ef11832d18
diff --git a/changelog.d/7729.bugfix.md b/changelog.d/7729.bugfix.md
new file mode 100644
index 0000000000..98bb7d6cf8
--- /dev/null
+++ b/changelog.d/7729.bugfix.md
@@ -0,0 +1,3 @@
+PMC full text no longer repeats the paragraphs of nested sections: each subsection is read
+once, under its own heading one level down. Paragraphs before the first section are kept,
+and titles with inline markup such as `<italic>` keep their full text.
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
index c22c7e17eb..854ed5336c 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
@@ -1291,8 +1291,13 @@ The default assumption should be that medical and scientific queries want RECENT
# Extract article title
title_elem = root.find(".//article-title")
- if title_elem is not None and title_elem.text:
- full_text.append(f"# {title_elem.text}")
+ title_text = (
+ "".join(title_elem.itertext()).strip()
+ if title_elem is not None
+ else ""
+ )
+ if title_text:
+ full_text.append(f"# {title_text}")
# Extract abstract
abstract_paras = root.findall(".//abstract//p")
@@ -1303,20 +1308,36 @@ The default assumption should be that medical and scientific queries want RECENT
if text:
full_text.append(text)
- # Extract body content
- body = root.find(".//body")
- if body is not None:
- for section in body.findall(".//sec"):
- # Get section title
- title = section.find(".//title")
- if title is not None and title.text:
- full_text.append(f"\n## {title.text}\n")
-
- # Get paragraphs
- for p in section.findall(".//p"):
- text = "".join(p.itertext())
+ # Extract body content in document order: the paragraphs of a section,
+ # then its subsections one heading level down. Searching every section
+ # for `.//p` read a subsection's paragraphs again under each section
+ # around it, and skipped paragraphs outside any section.
+ def add_section_content(element, depth):
+ for child in element:
+ if child.tag == "sec":
+ title = child.find("title")
+ title_text = (
+ "".join(title.itertext()).strip()
+ if title is not None
+ else ""
+ )
+ if title_text:
+ full_text.append(
+ f"\n{'#' * min(depth + 2, 6)} {title_text}\n"
+ )
+ add_section_content(child, depth + 1)
+ elif child.tag == "p":
+ text = "".join(child.itertext())
if text:
full_text.append(text)
+ elif child.tag != "title":
+ # A figure, table, list or box: its paragraphs and any
+ # section inside it, at the depth of the section around it
+ add_section_content(child, depth)
+
+ body = root.find(".//body")
+ if body is not None:
+ add_section_content(body, 0)
result_text = "\n\n".join(full_text)
logger.debug(
diff --git a/tests/web_search_engines/engines/test_search_engine_pubmed.py b/tests/web_search_engines/engines/test_search_engine_pubmed.py
index 73019af9f6..e89d6f6fca 100644
--- a/tests/web_search_engines/engines/test_search_engine_pubmed.py
+++ b/tests/web_search_engines/engines/test_search_engine_pubmed.py
@@ -1218,6 +1218,56 @@ class TestFindPmcIds:
class TestGetPmcFullText:
"""Tests for _get_pmc_full_text method."""
+ def test_reads_nested_sections_once_in_document_order(self):
+ """A subsection's paragraphs appear once, under its own heading."""
+ from local_deep_research.web_search_engines.engines.search_engine_pubmed import (
+ PubMedSearchEngine,
+ )
+
+ engine = PubMedSearchEngine()
+
+ xml_response = """<pmc-articleset><article>
+ <front><article-meta><title-group>
+ <article-title>Study of <italic>E. coli</italic></article-title>
+ </title-group></article-meta></front>
+ <body>
+ <p>Opening paragraph outside any section.</p>
+ <sec><title>Methods</title><p>Methods intro.</p>
+ <sec><title>Sampling</title><p>Sampling detail.</p></sec>
+ </sec>
+ <sec><title>Results</title><p>Results text.</p>
+ <fig><caption><p>Figure caption.</p></caption></fig>
+ <boxed-text><sec><title>Key points</title><p>Boxed point.</p></sec></boxed-text>
+ </sec>
+ </body>
+ </article></pmc-articleset>"""
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_pubmed.safe_get"
+ ) as mock_get:
+ mock_response = Mock()
+ mock_response.text = xml_response
+ mock_response.raise_for_status = Mock()
+ mock_get.return_value = mock_response
+
+ result = engine._get_pmc_full_text("PMC12345")
+
+ assert [
+ part.strip() for part in result.split("\n\n") if part.strip()
+ ] == [
+ "# Study of E. coli",
+ "Opening paragraph outside any section.",
+ "## Methods",
+ "Methods intro.",
+ "### Sampling",
+ "Sampling detail.",
+ "## Results",
+ "Results text.",
+ "Figure caption.",
+ "### Key points",
+ "Boxed point.",
+ ]
+
def test_extracts_full_text_from_xml(self):
"""Extracts full text content from PMC XML."""
from local_deep_research.web_search_engines.engines.search_engine_pubmed import (