Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(search): normalize Federal Register document text (#7280)

LearningCircuit46efcc7948d4
diff --git a/changelog.d/+federal-register-plain-text.bugfix.md b/changelog.d/+federal-register-plain-text.bugfix.md
new file mode 100644
index 0000000000..4518917d1c
--- /dev/null
+++ b/changelog.d/+federal-register-plain-text.bugfix.md
@@ -0,0 +1 @@
+Clean HTML wrappers and NUL characters from Federal Register document text before applying the content limit, preserving paragraph breaks, lists, and aligned tables in research sources.
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_federal_register.py b/src/local_deep_research/web_search_engines/engines/search_engine_federal_register.py
index 5561d3ad18..b078ddd71d 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_federal_register.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_federal_register.py
@@ -1,9 +1,11 @@
+import re
import time
from dataclasses import dataclass, field
from typing import Any, Dict, Final, List, Optional, cast
from urllib.parse import quote
import requests
+from bs4 import BeautifulSoup
from langchain_core.language_models import BaseLLM
from urllib3.exceptions import LocationParseError
from urllib3.util import parse_url
@@ -341,7 +343,9 @@ class FederalRegisterSearchEngine(BaseSearchEngine):
continue
self._raise_if_rate_limited(text_response)
text_response.raise_for_status()
- full_content = self._truncate_full_content(text_response.text)
+ full_content = self._truncate_full_content(
+ self._normalize_full_content(text_response.text)
+ )
result["full_content"] = full_content
fetch_cache[document_number] = (raw_text_url, full_content)
except RateLimitError:
@@ -550,6 +554,42 @@ class FederalRegisterSearchEngine(BaseSearchEngine):
return False
return (parsed.host or "").lower() in self.RAW_TEXT_HOSTS
+ @staticmethod
+ def _normalize_full_content(text: str) -> str:
+ """Unwrap the provider's HTML/pre representation without reflowing it.
+
+ ``raw_text_url`` can serve an HTML document as ``text/plain``.
+ Its preformatted body carries paragraph breaks and aligned tables,
+ so keep its whitespace and inline text. Ordinary plain text must
+ not be HTML-decoded or lose literal angle-bracketed expressions.
+ """
+ text = text.replace("\x00", "")
+ if not re.match(
+ r"\s*(?:<!doctype\s+html\b[^>]*>\s*)?<html(?:\s|>)",
+ text.lstrip("\ufeff"),
+ re.IGNORECASE,
+ ):
+ return text
+
+ soup = BeautifulSoup(text, "html.parser")
+ if soup.body is None:
+ return text
+ pre_blocks = soup.body.find_all("pre", recursive=False)
+ # An unfamiliar layout must not silently lose document sections.
+ if len(pre_blocks) != 1:
+ return text
+ document = pre_blocks[0]
+ if soup.body.get_text().strip() != document.get_text().strip():
+ return text
+ for line_break in document.find_all("br"):
+ line_break.replace_with("\n")
+ for bullet in document.find_all("bullet"):
+ # GPO's nonstandard <bullet> tags are unclosed. Replacing the
+ # tag itself would discard the remainder nested inside it.
+ bullet.insert_before("•")
+ bullet.unwrap()
+ return document.get_text()
+
def _truncate_full_content(self, text: str) -> str:
"""Bound one document's text before it reaches an LLM prompt.
diff --git a/tests/web_search_engines/engines/test_search_engine_federal_register_text.py b/tests/web_search_engines/engines/test_search_engine_federal_register_text.py
new file mode 100644
index 0000000000..e1fe5c7038
--- /dev/null
+++ b/tests/web_search_engines/engines/test_search_engine_federal_register_text.py
@@ -0,0 +1,202 @@
+"""Keep Federal Register document text usable through the public factory."""
+
+import json
+from unittest.mock import patch
+
+import pytest
+import requests
+
+
+MODULE = "local_deep_research.web_search_engines.engines.search_engine_federal_register"
+LIST_URL = "https://www.federalregister.gov/api/v1/documents.json"
+DETAIL_URL = "https://www.federalregister.gov/api/v1/documents/2026-20885.json"
+RAW_URL = "https://www.federalregister.gov/documents/full_text/text/2026/10/13/2026-20885.txt"
+DOCUMENT = {
+ "document_number": "2026-20885",
+ "title": "Airworthiness Directives; Airbus Helicopters",
+ "html_url": "https://www.federalregister.gov/documents/2026/10/13/2026-20885/airworthiness-directives-airbus-helicopters",
+ "abstract": "The FAA proposes a new airworthiness directive.",
+ "type": "Proposed Rule",
+ "agencies": [{"name": "Federal Aviation Administration"}],
+ "publication_date": "2026-10-13",
+}
+
+
+def provider_document():
+ # Shortened from RAW_URL, retrieved 2026-10-10. The endpoint reports
+ # text/plain but wraps preformatted document text in HTML, including
+ # unclosed <bullet> tags and a script outside the document's <pre>.
+ # Keep NULs escaped so the source fixture itself remains a text file.
+ table = (
+ " Estimated Costs\n"
+ "----------------------------------------------------------------------------------------------------------------\n"
+ " Cost per Cost on U.S.\n"
+ " Action Labor cost Parts cost product operators\n"
+ "----------------------------------------------------------------------------------------------------------------\n"
+ "Remove, clean, and reinstall 1 work-hour x $85 per $2,179 $2,689 $45,713\n"
+ " jettisonable windows (6 per hour = $85 (per window).\n"
+ " helicopter).\n"
+ "Perform operational test of windows 2 work-hours x $85 per 0 170 2,890\n"
+ " jettison systems. hour = $170.\n"
+ "----------------------------------------------------------------------------------------------------------------\n"
+ )
+ header = (
+ "\n[Federal Register Volume 91, Number 196 (Tuesday, October 13, 2026)]\n"
+ "[Proposed Rules]\n[Pages 64809-64811]\n[FR Doc No: 2026-20885]\n\n"
+ )
+ raw = (
+ "<html>\n<head>\n<title>Federal Register, Volume 91 Issue 196</title>\n"
+ "</head>\n<body><pre>" + header + "\x00DEPARTMENT OF TRANSPORTATION\n\n"
+ "Airworthiness Directives; Airbus Helicopters\n\n"
+ ' <bullet> Federal eRulemaking Portal: Go to <a href="https://regulations.gov">regulations.gov</a>.\n'
+ " <bullet> Fax: (202) 493-2251.\n\n"
+ "Costs of Compliance\n\n"
+ + table
+ + "\n\x00Authority for This Rulemaking\n\n"
+ "Title 49 of the United States Code specifies the FAA's authority.\n"
+ '</pre><script src="/cdn-cgi/scripts/email-decode.min.js"></script>'
+ "</body>\n</html>"
+ )
+ expected = (
+ header + "DEPARTMENT OF TRANSPORTATION\n\n"
+ "Airworthiness Directives; Airbus Helicopters\n\n"
+ " • Federal eRulemaking Portal: Go to regulations.gov.\n"
+ " • Fax: (202) 493-2251.\n\n"
+ "Costs of Compliance\n\n"
+ + table
+ + "\nAuthority for This Rulemaking\n\n"
+ "Title 49 of the United States Code specifies the FAA's authority.\n"
+ )
+ return raw, expected
+
+
+def _response(url, payload, *, content_type):
+ response = requests.Response()
+ response.status_code = 200
+ response.url = url
+ response.encoding = "utf-8"
+ response.headers["Content-Type"] = content_type
+ response._content = payload.encode("utf-8")
+ return response
+
+
+def _search(raw_text, *, max_content_chars=50000):
+ from local_deep_research.web_search_engines.search_engine_factory import (
+ create_search_engine,
+ )
+
+ engine = create_search_engine(
+ "federal_register",
+ settings_snapshot={
+ "search.engine.web": {"federal_register": {"default_params": {}}},
+ "rate_limiting.enabled": False,
+ "policy.egress_scope": "public_only",
+ "search.tool": "federal_register",
+ },
+ programmatic_mode=True,
+ search_snippets_only=False,
+ max_results=1,
+ max_content_chars=max_content_chars,
+ )
+ assert engine is not None
+ with patch(
+ f"{MODULE}.safe_get",
+ side_effect=[
+ _response(
+ LIST_URL,
+ json.dumps({"results": [DOCUMENT]}),
+ content_type="application/json",
+ ),
+ _response(
+ DETAIL_URL,
+ json.dumps({"raw_text_url": RAW_URL}),
+ content_type="application/json",
+ ),
+ _response(RAW_URL, raw_text, content_type="text/plain"),
+ ],
+ ) as request:
+ results = engine.run("airworthiness")
+ assert [c.args[0] for c in request.call_args_list] == [
+ LIST_URL,
+ DETAIL_URL,
+ RAW_URL,
+ ]
+ assert len(results) == 1
+ return results[0]
+
+
+def test_provider_text_reaches_citations_without_markup_or_nuls(
+ provider_document,
+):
+ from local_deep_research.citation_handler import CitationHandler
+
+ raw, expected = provider_document
+ result = _search(raw)
+
+ assert result["full_content"] == expected
+ assert result["title"] == DOCUMENT["title"]
+ assert result["link"] == DOCUMENT["html_url"]
+ assert result["snippet"] == DOCUMENT["abstract"]
+ assert result["agencies"] == DOCUMENT["agencies"]
+ handler = CitationHandler(None, settings_snapshot={})
+ documents = handler._create_documents([result])
+ assert documents[0].page_content == expected
+ assert expected in handler._format_sources(documents)
+
+
[email protected]("boundary_offset", [-1, 0, 1])
+def test_content_limit_applies_to_cleaned_text(
+ provider_document, boundary_offset
+):
+ raw, expected = provider_document
+ limit = len(expected) + boundary_offset
+ result = _search(raw, max_content_chars=limit)
+
+ if boundary_offset < 0:
+ expected = expected[:limit] + "\n\n[... truncated ...]"
+ assert result["full_content"] == expected
+
+
+ "text",
+ [
+ " First paragraph.\n\n Item Cost\n Check $85\n",
+ "Threshold: 0 < x <= 5; literal <amount> and &amp; stay unchanged.\n",
+ "Examples: <html><body><pre>sample</pre></body></html> and <br>.",
+ "<pre>This is a plain-text example of <b>markup</b>.</pre>\n",
+ "\ufeffUnicode café ≤ 5\r\n\tTabbed row\r\n",
+ "\x00First\n\nSecond\x00 paragraph.\n",
+ "",
+ ],
+)
+def test_plain_text_preserves_layout_and_literal_markup(text):
+ assert _search(text)["full_content"] == text.replace("\x00", "")
+
+
+def test_html_entities_and_line_breaks_preserve_document_content():
+ raw = (
+ '\ufeff\x00 <!DOCTYPE HTML>\n<HTML lang="en"><HEAD><TITLE>Metadata</TITLE>'
+ "</HEAD><BODY><PRE>Costs &amp; benefits: 0 &lt; x &le; 5.\n\n"
+ " A&nbsp;B<br> C\n"
+ "Literal &lt;em&gt;example&lt;/em&gt; and &amp;lt;code&amp;gt;."
+ "</PRE></BODY></HTML>"
+ )
+ assert _search(raw)["full_content"] == (
+ "Costs & benefits: 0 < x ≤ 5.\n\n"
+ " A\xa0B\n C\n"
+ "Literal <em>example</em> and &lt;code&gt;."
+ )
+
+
+ "text",
+ [
+ "<html><body>Document without the provider's pre wrapper.\x00</body></html>",
+ "<html><body><pre>First section.</pre><pre>Second section.</pre></body></html>",
+ "<html><body><p>Introduction.</p><pre>Table.</pre><p>Conclusion.</p></body></html>",
+ "<html><head><title>Metadata only.</title></head></html>",
+ ],
+)
+def test_unrecognized_html_layout_keeps_its_content(text):
+ assert _search(text)["full_content"] == text.replace("\x00", "")