~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(search): normalize Federal Register document text (#7280)
LearningCircuit46efcc7948d4
diff --git a/changelog.d/+federal-register-plain-text.bugfix.md b/changelog.d/+federal-register-plain-text.bugfix.mdnew file mode 100644index 0000000000..4518917d1c--- /dev/null+++ b/changelog.d/+federal-register-plain-text.bugfix.md@@ -0,0 +1 @@+Clean HTML wrappers and NUL characters from Federal Register document text before applying the content limit, preserving paragraph breaks, lists, and aligned tables in research sources.diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_federal_register.py b/src/local_deep_research/web_search_engines/engines/search_engine_federal_register.pyindex 5561d3ad18..b078ddd71d 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_federal_register.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_federal_register.py@@ -1,9 +1,11 @@+import reimport timefrom dataclasses import dataclass, fieldfrom typing import Any, Dict, Final, List, Optional, castfrom urllib.parse import quoteimport requests+from bs4 import BeautifulSoupfrom langchain_core.language_models import BaseLLMfrom urllib3.exceptions import LocationParseErrorfrom urllib3.util import parse_url@@ -341,7 +343,9 @@ class FederalRegisterSearchEngine(BaseSearchEngine):continueself._raise_if_rate_limited(text_response)text_response.raise_for_status()- full_content = self._truncate_full_content(text_response.text)+ full_content = self._truncate_full_content(+ self._normalize_full_content(text_response.text)+ )result["full_content"] = full_contentfetch_cache[document_number] = (raw_text_url, full_content)except RateLimitError:@@ -550,6 +554,42 @@ class FederalRegisterSearchEngine(BaseSearchEngine):return Falsereturn (parsed.host or "").lower() in self.RAW_TEXT_HOSTS+ @staticmethod+ def _normalize_full_content(text: str) -> str:+ """Unwrap the provider's HTML/pre representation without reflowing it.++ ``raw_text_url`` can serve an HTML document as ``text/plain``.+ Its preformatted body carries paragraph breaks and aligned tables,+ so keep its whitespace and inline text. Ordinary plain text must+ not be HTML-decoded or lose literal angle-bracketed expressions.+ """+ text = text.replace("\x00", "")+ if not re.match(+ r"\s*(?:<!doctype\s+html\b[^>]*>\s*)?<html(?:\s|>)",+ text.lstrip("\ufeff"),+ re.IGNORECASE,+ ):+ return text++ soup = BeautifulSoup(text, "html.parser")+ if soup.body is None:+ return text+ pre_blocks = soup.body.find_all("pre", recursive=False)+ # An unfamiliar layout must not silently lose document sections.+ if len(pre_blocks) != 1:+ return text+ document = pre_blocks[0]+ if soup.body.get_text().strip() != document.get_text().strip():+ return text+ for line_break in document.find_all("br"):+ line_break.replace_with("\n")+ for bullet in document.find_all("bullet"):+ # GPO's nonstandard <bullet> tags are unclosed. Replacing the+ # tag itself would discard the remainder nested inside it.+ bullet.insert_before("•")+ bullet.unwrap()+ return document.get_text()+def _truncate_full_content(self, text: str) -> str:"""Bound one document's text before it reaches an LLM prompt.diff --git a/tests/web_search_engines/engines/test_search_engine_federal_register_text.py b/tests/web_search_engines/engines/test_search_engine_federal_register_text.pynew file mode 100644index 0000000000..e1fe5c7038--- /dev/null+++ b/tests/web_search_engines/engines/test_search_engine_federal_register_text.py@@ -0,0 +1,202 @@+"""Keep Federal Register document text usable through the public factory."""++import json+from unittest.mock import patch++import pytest+import requests+++MODULE = "local_deep_research.web_search_engines.engines.search_engine_federal_register"+LIST_URL = "https://www.federalregister.gov/api/v1/documents.json"+DETAIL_URL = "https://www.federalregister.gov/api/v1/documents/2026-20885.json"+RAW_URL = "https://www.federalregister.gov/documents/full_text/text/2026/10/13/2026-20885.txt"+DOCUMENT = {+ "document_number": "2026-20885",+ "title": "Airworthiness Directives; Airbus Helicopters",+ "html_url": "https://www.federalregister.gov/documents/2026/10/13/2026-20885/airworthiness-directives-airbus-helicopters",+ "abstract": "The FAA proposes a new airworthiness directive.",+ "type": "Proposed Rule",+ "agencies": [{"name": "Federal Aviation Administration"}],+ "publication_date": "2026-10-13",+}+++def provider_document():+ # Shortened from RAW_URL, retrieved 2026-10-10. The endpoint reports+ # text/plain but wraps preformatted document text in HTML, including+ # unclosed <bullet> tags and a script outside the document's <pre>.+ # Keep NULs escaped so the source fixture itself remains a text file.+ table = (+ " Estimated Costs\n"+ "----------------------------------------------------------------------------------------------------------------\n"+ " Cost per Cost on U.S.\n"+ " Action Labor cost Parts cost product operators\n"+ "----------------------------------------------------------------------------------------------------------------\n"+ "Remove, clean, and reinstall 1 work-hour x $85 per $2,179 $2,689 $45,713\n"+ " jettisonable windows (6 per hour = $85 (per window).\n"+ " helicopter).\n"+ "Perform operational test of windows 2 work-hours x $85 per 0 170 2,890\n"+ " jettison systems. hour = $170.\n"+ "----------------------------------------------------------------------------------------------------------------\n"+ )+ header = (+ "\n[Federal Register Volume 91, Number 196 (Tuesday, October 13, 2026)]\n"+ "[Proposed Rules]\n[Pages 64809-64811]\n[FR Doc No: 2026-20885]\n\n"+ )+ raw = (+ "<html>\n<head>\n<title>Federal Register, Volume 91 Issue 196</title>\n"+ "</head>\n<body><pre>" + header + "\x00DEPARTMENT OF TRANSPORTATION\n\n"+ "Airworthiness Directives; Airbus Helicopters\n\n"+ ' <bullet> Federal eRulemaking Portal: Go to <a href="https://regulations.gov">regulations.gov</a>.\n'+ " <bullet> Fax: (202) 493-2251.\n\n"+ "Costs of Compliance\n\n"+ + table+ + "\n\x00Authority for This Rulemaking\n\n"+ "Title 49 of the United States Code specifies the FAA's authority.\n"+ '</pre><script src="/cdn-cgi/scripts/email-decode.min.js"></script>'+ "</body>\n</html>"+ )+ expected = (+ header + "DEPARTMENT OF TRANSPORTATION\n\n"+ "Airworthiness Directives; Airbus Helicopters\n\n"+ " • Federal eRulemaking Portal: Go to regulations.gov.\n"+ " • Fax: (202) 493-2251.\n\n"+ "Costs of Compliance\n\n"+ + table+ + "\nAuthority for This Rulemaking\n\n"+ "Title 49 of the United States Code specifies the FAA's authority.\n"+ )+ return raw, expected+++def _response(url, payload, *, content_type):+ response = requests.Response()+ response.status_code = 200+ response.url = url+ response.encoding = "utf-8"+ response.headers["Content-Type"] = content_type+ response._content = payload.encode("utf-8")+ return response+++def _search(raw_text, *, max_content_chars=50000):+ from local_deep_research.web_search_engines.search_engine_factory import (+ create_search_engine,+ )++ engine = create_search_engine(+ "federal_register",+ settings_snapshot={+ "search.engine.web": {"federal_register": {"default_params": {}}},+ "rate_limiting.enabled": False,+ "policy.egress_scope": "public_only",+ "search.tool": "federal_register",+ },+ programmatic_mode=True,+ search_snippets_only=False,+ max_results=1,+ max_content_chars=max_content_chars,+ )+ assert engine is not None+ with patch(+ f"{MODULE}.safe_get",+ side_effect=[+ _response(+ LIST_URL,+ json.dumps({"results": [DOCUMENT]}),+ content_type="application/json",+ ),+ _response(+ DETAIL_URL,+ json.dumps({"raw_text_url": RAW_URL}),+ content_type="application/json",+ ),+ _response(RAW_URL, raw_text, content_type="text/plain"),+ ],+ ) as request:+ results = engine.run("airworthiness")+ assert [c.args[0] for c in request.call_args_list] == [+ LIST_URL,+ DETAIL_URL,+ RAW_URL,+ ]+ assert len(results) == 1+ return results[0]+++def test_provider_text_reaches_citations_without_markup_or_nuls(+ provider_document,+):+ from local_deep_research.citation_handler import CitationHandler++ raw, expected = provider_document+ result = _search(raw)++ assert result["full_content"] == expected+ assert result["title"] == DOCUMENT["title"]+ assert result["link"] == DOCUMENT["html_url"]+ assert result["snippet"] == DOCUMENT["abstract"]+ assert result["agencies"] == DOCUMENT["agencies"]+ handler = CitationHandler(None, settings_snapshot={})+ documents = handler._create_documents([result])+ assert documents[0].page_content == expected+ assert expected in handler._format_sources(documents)++[email protected]("boundary_offset", [-1, 0, 1])+def test_content_limit_applies_to_cleaned_text(+ provider_document, boundary_offset+):+ raw, expected = provider_document+ limit = len(expected) + boundary_offset+ result = _search(raw, max_content_chars=limit)++ if boundary_offset < 0:+ expected = expected[:limit] + "\n\n[... truncated ...]"+ assert result["full_content"] == expected+++ "text",+ [+ " First paragraph.\n\n Item Cost\n Check $85\n",+ "Threshold: 0 < x <= 5; literal <amount> and & stay unchanged.\n",+ "Examples: <html><body><pre>sample</pre></body></html> and <br>.",+ "<pre>This is a plain-text example of <b>markup</b>.</pre>\n",+ "\ufeffUnicode café ≤ 5\r\n\tTabbed row\r\n",+ "\x00First\n\nSecond\x00 paragraph.\n",+ "",+ ],+)+def test_plain_text_preserves_layout_and_literal_markup(text):+ assert _search(text)["full_content"] == text.replace("\x00", "")+++def test_html_entities_and_line_breaks_preserve_document_content():+ raw = (+ '\ufeff\x00 <!DOCTYPE HTML>\n<HTML lang="en"><HEAD><TITLE>Metadata</TITLE>'+ "</HEAD><BODY><PRE>Costs & benefits: 0 < x ≤ 5.\n\n"+ " A B<br> C\n"+ "Literal <em>example</em> and &lt;code&gt;."+ "</PRE></BODY></HTML>"+ )+ assert _search(raw)["full_content"] == (+ "Costs & benefits: 0 < x ≤ 5.\n\n"+ " A\xa0B\n C\n"+ "Literal <em>example</em> and <code>."+ )+++ "text",+ [+ "<html><body>Document without the provider's pre wrapper.\x00</body></html>",+ "<html><body><pre>First section.</pre><pre>Second section.</pre></body></html>",+ "<html><body><p>Introduction.</p><pre>Table.</pre><p>Conclusion.</p></body></html>",+ "<html><head><title>Metadata only.</title></head></html>",+ ],+)+def test_unrecognized_html_layout_keeps_its_content(text):+ assert _search(text)["full_content"] == text.replace("\x00", "")