Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(extraction): stop the HTML fallbacks breaking a sentence into lines (#6520)

Lukas3ac41ab0abb9
diff --git a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
index 45562f25bb..80dee18c1d 100644
--- a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
+++ b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py
@@ -16,7 +16,7 @@ specialized downloaders first. Generic HTML runs only when that route permits it
from importlib import import_module
from typing import Any, Dict, List, NamedTuple, Optional
-from bs4 import BeautifulSoup
+from bs4 import BeautifulSoup, NavigableString
from loguru import logger
from local_deep_research.utilities.arxiv import is_arxiv_paper_url
@@ -36,6 +36,96 @@ from ....utilities.lxml_thread_safety import install_per_thread_html_parsers
install_per_thread_html_parsers()
+# Elements a browser lays out on a line of their own, plus the few that are not
+# laid out at all but still read as a line when a document is flattened
+# (`title`, `option`). The markup is the only place that boundary exists, so the
+# flattener marks the end of each one.
+_BLOCK_LEVEL_TAGS = (
+ "address",
+ "article",
+ "aside",
+ "blockquote",
+ "caption",
+ "dd",
+ "details",
+ "div",
+ "dl",
+ "dt",
+ "fieldset",
+ "figcaption",
+ "figure",
+ "footer",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+ "h5",
+ "h6",
+ "header",
+ "hgroup",
+ "hr",
+ "legend",
+ "li",
+ "main",
+ "nav",
+ "ol",
+ "option",
+ "p",
+ "pre",
+ "section",
+ "summary",
+ "table",
+ "td",
+ "th",
+ "title",
+ "tr",
+ "ul",
+)
+
+
+_END_OF_CHILDREN = object()
+
+
+def _flatten_html(soup: BeautifulSoup) -> str:
+ """Flatten parsed HTML to text the way a reader sees it.
+
+ ``get_text(separator="\n")`` puts a newline between *every* pair of strings,
+ including the ones inside a sentence, so ``Hello <b>world</b>! See
+ <a href="#">this</a>.`` arrives as six lines. A line break belongs only at
+ the start and end of a block-level element and at a ``<br>``, so those are
+ the only places one is written.
+
+ The tree is walked once and left unchanged. Replacing each ``<br>`` in
+ place scans its siblings on every call, which is quadratic on a page made
+ of tens of thousands of them.
+ """
+ # The strings get_text() joins: script, style and template text stays out
+ # exactly as it does there.
+ wanted = {id(string) for string in soup.strings}
+ parts: List[str] = []
+ children = [iter(soup.contents)]
+ open_tags: List[Optional[str]] = [None]
+ while children:
+ node = next(children[-1], _END_OF_CHILDREN)
+ if node is _END_OF_CHILDREN:
+ children.pop()
+ if open_tags.pop() in _BLOCK_LEVEL_TAGS:
+ parts.append("\n")
+ elif isinstance(node, NavigableString):
+ if id(node) in wanted:
+ parts.append(node)
+ elif node.name == "br":
+ parts.append("\n")
+ else:
+ if node.name in _BLOCK_LEVEL_TAGS:
+ parts.append("\n")
+ children.append(iter(node.contents))
+ open_tags.append(node.name)
+
+ lines = [line.strip() for line in "".join(parts).splitlines()]
+ return "\n".join(line for line in lines if line)
+
+
# --- Pipeline thresholds ---
# Minimum extracted text length to consider extraction successful
MIN_CONTENT_LENGTH = 50
@@ -245,14 +335,12 @@ def extract_content(
# Strip remaining HTML tags (e.g. readability-only mode)
if content and "<" in content:
- content = BeautifulSoup(content, "html.parser").get_text(
- separator="\n", strip=True
- )
+ content = _flatten_html(BeautifulSoup(content, "html.parser"))
# Last resort
if not content or len(content.strip()) < min_length:
logger.debug("Pipeline: all extractors failed, using get_text()")
- content = soup.get_text(separator="\n", strip=True)
+ content = _flatten_html(soup)
if not content or len(content.strip()) < min_length:
return None
diff --git a/tests/research_library/downloaders/test_extraction_pipeline.py b/tests/research_library/downloaders/test_extraction_pipeline.py
index 1751d0f766..cd6c5e23b0 100644
--- a/tests/research_library/downloaders/test_extraction_pipeline.py
+++ b/tests/research_library/downloaders/test_extraction_pipeline.py
@@ -1090,3 +1090,85 @@ class TestFetchAndExtractJSRenderingPlumbing:
)
kwargs = mock_class.call_args.kwargs
assert kwargs.get("enable_js_rendering") is True
+
+
+# ---------------------------------------------------------------------------
+# _flatten_html
+# ---------------------------------------------------------------------------
+
+
+class TestFlattenHtml:
+ """The fallback flattener must keep a sentence together."""
+
+ def _flatten(self, html: str) -> str:
+ from bs4 import BeautifulSoup
+
+ return pipeline._flatten_html(BeautifulSoup(html, "html.parser"))
+
+ def test_a_sentence_survives_its_inline_markup(self):
+ """`get_text(separator="\\n")` would break this into six lines."""
+ html = '<p>Hello <b>world</b>! See <a href="#">this</a>.</p>'
+
+ assert self._flatten(html) == "Hello world! See this."
+
+ def test_blocks_are_kept_apart(self):
+ html = "<h1>Quarterly Report</h1><p>Revenue rose.</p><ul><li>one</li><li>two</li></ul>"
+
+ assert (
+ self._flatten(html) == "Quarterly Report\nRevenue rose.\none\ntwo"
+ )
+
+ def test_the_title_and_options_keep_their_own_lines(self):
+ """Neither is a block, but both read as a line in a flattened page."""
+ html = (
+ "<html><head><title>Test Page</title></head><body><p>Body text</p>"
+ "<select><option>alpha</option><option>beta</option></select>"
+ "</body></html>"
+ )
+
+ assert self._flatten(html) == "Test Page\nBody text\nalpha\nbeta"
+
+ def test_a_line_break_element_becomes_a_line_break(self):
+ assert (
+ self._flatten("<p>line one<br>line two</p>") == "line one\nline two"
+ )
+
+ def test_blank_lines_are_dropped(self):
+ assert (
+ self._flatten("<div></div><p>only this</p><div> </div>")
+ == "only this"
+ )
+
+ def test_text_before_a_block_keeps_its_own_line(self):
+ """Only block ends were marked, so ``x<p>y</p>z`` read as ``xy``."""
+ assert self._flatten("x<p>y</p>z") == "x\ny\nz"
+
+ def test_script_style_and_template_text_stays_out(self):
+ html = (
+ "<p>kept</p><script>var a = 1;</script>"
+ "<style>.a{color:red}</style><template>hidden</template>"
+ )
+
+ assert self._flatten(html) == "kept"
+
+ def test_the_parsed_tree_is_left_unchanged(self):
+ """The last-resort call site flattens a soup it may read again."""
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup("<p>one<br>two</p><div>three</div>", "html.parser")
+ before = str(soup)
+
+ pipeline._flatten_html(soup)
+
+ assert str(soup) == before
+
+ @pytest.mark.timeout(10)
+ def test_many_line_breaks_flatten_in_linear_time(self):
+ """Replacing each <br> in place took 6.2 s for 20,000 of them and
+ quadruples with every doubling, so 50,000 would take about 40 s."""
+ html = "<div>" + "x<br>" * 50_000 + "</div>"
+
+ lines = self._flatten(html).split("\n")
+
+ assert len(lines) == 50_000
+ assert set(lines) == {"x"}