~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(extraction): stop the HTML fallbacks breaking a sentence into lines (#6520)
Lukas3ac41ab0abb9
diff --git a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py b/src/local_deep_research/research_library/downloaders/extraction/pipeline.pyindex 45562f25bb..80dee18c1d 100644--- a/src/local_deep_research/research_library/downloaders/extraction/pipeline.py+++ b/src/local_deep_research/research_library/downloaders/extraction/pipeline.py@@ -16,7 +16,7 @@ specialized downloaders first. Generic HTML runs only when that route permits itfrom importlib import import_modulefrom typing import Any, Dict, List, NamedTuple, Optional-from bs4 import BeautifulSoup+from bs4 import BeautifulSoup, NavigableStringfrom loguru import loggerfrom local_deep_research.utilities.arxiv import is_arxiv_paper_url@@ -36,6 +36,96 @@ from ....utilities.lxml_thread_safety import install_per_thread_html_parsersinstall_per_thread_html_parsers()+# Elements a browser lays out on a line of their own, plus the few that are not+# laid out at all but still read as a line when a document is flattened+# (`title`, `option`). The markup is the only place that boundary exists, so the+# flattener marks the end of each one.+_BLOCK_LEVEL_TAGS = (+ "address",+ "article",+ "aside",+ "blockquote",+ "caption",+ "dd",+ "details",+ "div",+ "dl",+ "dt",+ "fieldset",+ "figcaption",+ "figure",+ "footer",+ "h1",+ "h2",+ "h3",+ "h4",+ "h5",+ "h6",+ "header",+ "hgroup",+ "hr",+ "legend",+ "li",+ "main",+ "nav",+ "ol",+ "option",+ "p",+ "pre",+ "section",+ "summary",+ "table",+ "td",+ "th",+ "title",+ "tr",+ "ul",+)+++_END_OF_CHILDREN = object()+++def _flatten_html(soup: BeautifulSoup) -> str:+ """Flatten parsed HTML to text the way a reader sees it.++ ``get_text(separator="\n")`` puts a newline between *every* pair of strings,+ including the ones inside a sentence, so ``Hello <b>world</b>! See+ <a href="#">this</a>.`` arrives as six lines. A line break belongs only at+ the start and end of a block-level element and at a ``<br>``, so those are+ the only places one is written.++ The tree is walked once and left unchanged. Replacing each ``<br>`` in+ place scans its siblings on every call, which is quadratic on a page made+ of tens of thousands of them.+ """+ # The strings get_text() joins: script, style and template text stays out+ # exactly as it does there.+ wanted = {id(string) for string in soup.strings}+ parts: List[str] = []+ children = [iter(soup.contents)]+ open_tags: List[Optional[str]] = [None]+ while children:+ node = next(children[-1], _END_OF_CHILDREN)+ if node is _END_OF_CHILDREN:+ children.pop()+ if open_tags.pop() in _BLOCK_LEVEL_TAGS:+ parts.append("\n")+ elif isinstance(node, NavigableString):+ if id(node) in wanted:+ parts.append(node)+ elif node.name == "br":+ parts.append("\n")+ else:+ if node.name in _BLOCK_LEVEL_TAGS:+ parts.append("\n")+ children.append(iter(node.contents))+ open_tags.append(node.name)++ lines = [line.strip() for line in "".join(parts).splitlines()]+ return "\n".join(line for line in lines if line)++# --- Pipeline thresholds ---# Minimum extracted text length to consider extraction successfulMIN_CONTENT_LENGTH = 50@@ -245,14 +335,12 @@ def extract_content(# Strip remaining HTML tags (e.g. readability-only mode)if content and "<" in content:- content = BeautifulSoup(content, "html.parser").get_text(- separator="\n", strip=True- )+ content = _flatten_html(BeautifulSoup(content, "html.parser"))# Last resortif not content or len(content.strip()) < min_length:logger.debug("Pipeline: all extractors failed, using get_text()")- content = soup.get_text(separator="\n", strip=True)+ content = _flatten_html(soup)if not content or len(content.strip()) < min_length:return Nonediff --git a/tests/research_library/downloaders/test_extraction_pipeline.py b/tests/research_library/downloaders/test_extraction_pipeline.pyindex 1751d0f766..cd6c5e23b0 100644--- a/tests/research_library/downloaders/test_extraction_pipeline.py+++ b/tests/research_library/downloaders/test_extraction_pipeline.py@@ -1090,3 +1090,85 @@ class TestFetchAndExtractJSRenderingPlumbing:)kwargs = mock_class.call_args.kwargsassert kwargs.get("enable_js_rendering") is True+++# ---------------------------------------------------------------------------+# _flatten_html+# ---------------------------------------------------------------------------+++class TestFlattenHtml:+ """The fallback flattener must keep a sentence together."""++ def _flatten(self, html: str) -> str:+ from bs4 import BeautifulSoup++ return pipeline._flatten_html(BeautifulSoup(html, "html.parser"))++ def test_a_sentence_survives_its_inline_markup(self):+ """`get_text(separator="\\n")` would break this into six lines."""+ html = '<p>Hello <b>world</b>! See <a href="#">this</a>.</p>'++ assert self._flatten(html) == "Hello world! See this."++ def test_blocks_are_kept_apart(self):+ html = "<h1>Quarterly Report</h1><p>Revenue rose.</p><ul><li>one</li><li>two</li></ul>"++ assert (+ self._flatten(html) == "Quarterly Report\nRevenue rose.\none\ntwo"+ )++ def test_the_title_and_options_keep_their_own_lines(self):+ """Neither is a block, but both read as a line in a flattened page."""+ html = (+ "<html><head><title>Test Page</title></head><body><p>Body text</p>"+ "<select><option>alpha</option><option>beta</option></select>"+ "</body></html>"+ )++ assert self._flatten(html) == "Test Page\nBody text\nalpha\nbeta"++ def test_a_line_break_element_becomes_a_line_break(self):+ assert (+ self._flatten("<p>line one<br>line two</p>") == "line one\nline two"+ )++ def test_blank_lines_are_dropped(self):+ assert (+ self._flatten("<div></div><p>only this</p><div> </div>")+ == "only this"+ )++ def test_text_before_a_block_keeps_its_own_line(self):+ """Only block ends were marked, so ``x<p>y</p>z`` read as ``xy``."""+ assert self._flatten("x<p>y</p>z") == "x\ny\nz"++ def test_script_style_and_template_text_stays_out(self):+ html = (+ "<p>kept</p><script>var a = 1;</script>"+ "<style>.a{color:red}</style><template>hidden</template>"+ )++ assert self._flatten(html) == "kept"++ def test_the_parsed_tree_is_left_unchanged(self):+ """The last-resort call site flattens a soup it may read again."""+ from bs4 import BeautifulSoup++ soup = BeautifulSoup("<p>one<br>two</p><div>three</div>", "html.parser")+ before = str(soup)++ pipeline._flatten_html(soup)++ assert str(soup) == before++ @pytest.mark.timeout(10)+ def test_many_line_breaks_flatten_in_linear_time(self):+ """Replacing each <br> in place took 6.2 s for 20,000 of them and+ quadruples with every doubling, so 50,000 would take about 40 s."""+ html = "<div>" + "x<br>" * 50_000 + "</div>"++ lines = self._flatten(html).split("\n")++ assert len(lines) == 50_000+ assert set(lines) == {"x"}