Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix: builtin provider flattens fetched code to prose (#129)

Francois du Plessis6240eac05bf2
diff --git a/src/hyperresearch/core/scholar.py b/src/hyperresearch/core/scholar.py
index bad5052..a877214 100644
--- a/src/hyperresearch/core/scholar.py
+++ b/src/hyperresearch/core/scholar.py
@@ -44,7 +44,9 @@ META_DOI_RE = re.compile(
r"content=[\"']\s*(?:doi:)?\s*(10\.[^\"']+)[\"']",
re.IGNORECASE,
)
-BODY_DOI_RE = re.compile(r"\bDOI:?\s*(10\.\d{4,9}/[^\s\"'<>\])}]+)", re.IGNORECASE)
+# Fetched bodies carry markdown code spans (`DOI: 10.x/y`, DOI: `10.x/y`), so a
+# backtick may open the DOI and never belongs to it.
+BODY_DOI_RE = re.compile(r"\bDOI:?\s*`?(10\.\d{4,9}/[^\s\"'<>\])}`]+)", re.IGNORECASE)
# Per-host courtesy delay between UNCACHED requests, seconds.
_HOST_DELAY = {
diff --git a/src/hyperresearch/web/builtin.py b/src/hyperresearch/web/builtin.py
index cf5b63e..b981eb8 100644
--- a/src/hyperresearch/web/builtin.py
+++ b/src/hyperresearch/web/builtin.py
@@ -21,43 +21,126 @@ from hyperresearch.web.pdf import (
is_pdf_url,
)
+# Code in fetched pages must reach the vault as markdown code, not prose.
+# Flattened <pre> content passes the extractor's code-strip untouched and
+# hits WIKI_LINK_RE, where bash test syntax ([[ -n "$kernel" ]]) is lexically
+# a wiki-link; repair/graph then materialise junk stub notes named after
+# shell fragments (40 such notes in one real vault from one Arch wiki page).
+# Both extraction branches below share these helpers.
+
+
+def _fence_for(code: str) -> str:
+ """Return a backtick fence longer than any backtick run inside the code."""
+ longest = max((len(m.group(0)) for m in re.finditer(r"`+", code)), default=0)
+ return "`" * max(3, longest + 1)
+
+
+def _fenced(code: str) -> str:
+ """Wrap a code block in a fence it cannot break out of."""
+ fence = _fence_for(code)
+ body = code.strip("\n")
+ return f"\n{fence}\n{body}\n{fence}\n"
+
+
+def _inline_code(text: str) -> str:
+ """Wrap inline code in backticks (doubled when the text contains one)."""
+ if "`" in text:
+ return f"`` {text} ``"
+ return f"`{text}`"
+
class _TextExtractor(HTMLParser):
- """Minimal HTML-to-text extractor (no external deps)."""
+ """Minimal HTML-to-text extractor (no external deps).
+
+ Emits <pre> blocks as fenced code and inline <code> as backtick spans;
+ see the module comment for why that is load-bearing, not cosmetic.
+ """
def __init__(self):
super().__init__()
- self._pieces: list[str] = []
+ # (kind, text) where kind is "text" or "code"; code segments skip the
+ # per-line strip in get_text() so indentation survives.
+ self._segments: list[tuple[str, str]] = []
self._skip = False
self._skip_tags = {"script", "style", "nav", "footer", "header"}
self._title = ""
self._in_title = False
+ self._in_pre = False
+ self._pre_buf: list[str] = []
+ self._in_inline_code = False
def handle_starttag(self, tag, attrs):
if tag in self._skip_tags:
self._skip = True
+ return
+ if self._in_pre:
+ if tag == "br":
+ self._pre_buf.append("\n")
+ return
if tag == "title":
self._in_title = True
+ return
+ if tag == "pre":
+ self._in_pre = True
+ self._pre_buf = []
+ return
+ if tag == "code":
+ self._in_inline_code = True
+ return
if tag in ("p", "br", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li"):
- self._pieces.append("\n")
+ self._segments.append(("text", "\n"))
def handle_endtag(self, tag):
if tag in self._skip_tags:
self._skip = False
+ return
+ if tag == "pre" and self._in_pre:
+ self._in_pre = False
+ code = "".join(self._pre_buf)
+ self._pre_buf = []
+ if code.strip():
+ self._segments.append(("code", code))
+ return
if tag == "title":
self._in_title = False
+ return
+ if tag == "code":
+ self._in_inline_code = False
def handle_data(self, data):
if self._in_title:
self._title = data.strip()
- if not self._skip:
- self._pieces.append(data)
+ if self._skip:
+ return
+ if self._in_pre:
+ self._pre_buf.append(data)
+ return
+ if self._in_inline_code:
+ if data.strip():
+ self._segments.append(("text", _inline_code(data)))
+ return
+ self._segments.append(("text", data))
def get_text(self) -> str:
- raw = "".join(self._pieces)
- # Collapse whitespace
- lines = [line.strip() for line in raw.splitlines()]
- return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip()
+ out: list[str] = []
+ buf: list[str] = []
+
+ def flush() -> None:
+ raw = "".join(buf)
+ lines = [line.strip() for line in raw.splitlines()]
+ cleaned = re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip()
+ if cleaned:
+ out.append(cleaned)
+ buf.clear()
+
+ for kind, text in self._segments:
+ if kind == "text":
+ buf.append(text)
+ else:
+ flush()
+ out.append(_fenced(text).strip("\n"))
+ flush()
+ return "\n\n".join(out)
class BuiltinProvider:
@@ -144,13 +227,29 @@ class BuiltinProvider:
def _extract(self, html: str) -> tuple[str, str]:
"""Extract title and clean text from HTML. Tries bs4 first, falls back to stdlib."""
try:
- from bs4 import BeautifulSoup
+ from bs4 import BeautifulSoup, NavigableString
soup = BeautifulSoup(html, "html.parser")
# Remove noise
for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
tag.decompose()
title = soup.title.string.strip() if soup.title and soup.title.string else ""
+ # Preserve code as markdown BEFORE get_text() flattens the markup
+ # (module comment has the why). <pre> first, so nested <code>
+ # inside it is consumed with it; then the remaining inline <code>.
+ # Rewrite each element's contents in place: replace_with() and
+ # decompose() look the element up in its parent's child list, which
+ # is quadratic on a page with thousands of sibling <code> tags.
+ for pre in soup.find_all("pre"):
+ code_text = pre.get_text()
+ pre.clear()
+ if code_text.strip():
+ pre.append(NavigableString(_fenced(code_text)))
+ for code in soup.find_all("code"):
+ code_text = code.get_text()
+ code.clear()
+ if code_text.strip():
+ code.append(NavigableString(_inline_code(code_text)))
text = soup.get_text(separator="\n", strip=True)
# Collapse blank lines
text = re.sub(r"\n{3,}", "\n\n", text)
diff --git a/tests/test_core/test_source_ranking.py b/tests/test_core/test_source_ranking.py
index 99da5ec..126fed7 100644
--- a/tests/test_core/test_source_ranking.py
+++ b/tests/test_core/test_source_ranking.py
@@ -104,6 +104,12 @@ class TestDoiExtraction:
def test_trailing_punctuation_stripped(self):
assert extract_doi("https://x.com", content="see DOI: 10.1000/xyz123.") == "10.1000/xyz123"
+ def test_body_doi_inside_code_span(self):
+ # Extracted page text renders <code> as a backtick span; the backtick
+ # must not end up in the DOI, and must not hide it either.
+ assert extract_doi("https://x.com", content="cite `DOI: 10.1000/xyz123` here") == "10.1000/xyz123"
+ assert extract_doi("https://x.com", content="DOI: `10.1000/xyz123`") == "10.1000/xyz123"
+
class TestPageRank:
def test_hub_ranks_highest(self):
diff --git a/tests/test_web/test_builtin_fences.py b/tests/test_web/test_builtin_fences.py
new file mode 100644
index 0000000..215d55b
--- /dev/null
+++ b/tests/test_web/test_builtin_fences.py
@@ -0,0 +1,151 @@
+"""The builtin provider must not flatten fetched code to prose.
+
+Flattened <pre>/<code> content passes the note parser's code-strip untouched
+and reaches WIKI_LINK_RE, where bash test syntax is lexically a wiki-link;
+repair/graph then materialise junk stub notes named after shell fragments
+(40 such notes in one real vault, all fetched via this provider). These
+tests pin markdown-shaped code handling in BOTH extraction branches: bs4
+and the stdlib fallback.
+"""
+
+import pytest
+
+from hyperresearch.core.patterns import (
+ CODE_BLOCK_RE,
+ INLINE_CODE_RE,
+ WIKI_LINK_RE,
+ is_valid_wiki_link_target,
+)
+from hyperresearch.web.builtin import BuiltinProvider, _TextExtractor
+from hyperresearch.web.pdf import PDF_FAILURE_KEY
+from hyperresearch.web.safe_http import SafeResponse
+
+# Shaped like the real defect page (a wiki page whose body carries a shell
+# script in <pre> plus a conditional in inline <code>, both with [[ ]] test
+# syntax).
+ARCH_WIKI_SHAPED = """<html><head><title>Secure Boot - ArchWiki</title></head>
+<body><main>
+<p>Create the following file and make it executable:</p>
+<pre>#!/usr/bin/env bash
+
+kernel="$1"
+[[ -n "$kernel" ]] || exit 0
+
+# use already installed kernel if it exists
+[[ ! -f "$KERNELDESTINATION" ]] || kernel="$KERNELDESTINATION"</pre>
+<p>Then check <code>[[ -f /boot/vmlinuz ]]</code> before rebooting.</p>
+</main></body></html>"""
+
+
+def _surviving_links(text: str) -> list[str]:
+ """The exact strip-then-extract sequence core/note.py runs on a note body."""
+ cleaned = CODE_BLOCK_RE.sub("", text)
+ cleaned = INLINE_CODE_RE.sub("", cleaned)
+ raw = (m.group(1).strip().rstrip("\\") for m in WIKI_LINK_RE.finditer(cleaned))
+ return [ref for ref in raw if is_valid_wiki_link_target(ref)]
+
+
+class _BranchContract:
+ """Shared assertions; subclasses supply the extraction branch."""
+
+ def extract(self, html: str) -> tuple[str, str]:
+ raise NotImplementedError
+
+ def test_pre_becomes_fenced_block(self):
+ _, text = self.extract(ARCH_WIKI_SHAPED)
+ assert "```" in text
+ assert '[[ -n "$kernel" ]] || exit 0' in text
+
+ def test_inline_code_becomes_backtick_span(self):
+ _, text = self.extract(ARCH_WIKI_SHAPED)
+ assert "`[[ -f /boot/vmlinuz ]]`" in text
+
+ def test_no_wiki_links_survive_the_note_strip(self):
+ # The end-to-end guarantee the fix exists for: after the exact
+ # code-strip the link extractor applies, no [[ ]] fragment remains.
+ _, text = self.extract(ARCH_WIKI_SHAPED)
+ assert _surviving_links(text) == []
+
+ def test_code_indentation_survives(self):
+ html = "<html><body><p>x</p><pre>if x:\n do_thing()</pre></body></html>"
+ _, text = self.extract(html)
+ assert " do_thing()" in text
+
+ def test_fence_outgrows_backticks_inside_code(self):
+ html = "<html><body><pre>echo ```already fenced```</pre></body></html>"
+ _, text = self.extract(html)
+ assert "````" in text
+ assert _surviving_links(text) == []
+
+ def test_title_and_noise_removal_unchanged(self):
+ html = (
+ "<html><head><title>T</title></head><body>"
+ "<nav>menu here</nav><p>body text</p><footer>foot here</footer>"
+ "</body></html>"
+ )
+ title, text = self.extract(html)
+ assert title == "T"
+ assert "menu here" not in text
+ assert "foot here" not in text
+ assert "body text" in text
+
+
+class TestBs4Branch(_BranchContract):
+ def extract(self, html: str) -> tuple[str, str]:
+ pytest.importorskip("bs4")
+ return BuiltinProvider()._extract(html)
+
+
+class TestStdlibBranch(_BranchContract):
+ def extract(self, html: str) -> tuple[str, str]:
+ parser = _TextExtractor()
+ parser.feed(html)
+ return parser._title, parser.get_text()
+
+
+# fetch() is the only way content leaves the provider: HTML fetched directly,
+# and HTML reached after the PDF lane declines a PDF-shaped URL. Both must go
+# through the code-preserving extraction, and the decline reason must still
+# travel with the result. The SSRF-gated download is stubbed at its source.
+
+
+def served(monkeypatch):
+ table: dict[str, SafeResponse] = {}
+
+ def fake_safe_get(url, *, max_bytes, timeout=None, headers=None, verify=True,
+ allow_private_hosts=()):
+ return table[url]
+
+ monkeypatch.setattr("hyperresearch.web.safe_http.safe_get", fake_safe_get)
+ monkeypatch.setattr("hyperresearch.web.safe_http.check_url", lambda url, allow=(): None)
+ return table
+
+
+def _html(url: str, body: str, status: int = 200) -> SafeResponse:
+ return SafeResponse(url=url, status_code=status, headers={"content-type": "text/html"},
+ content=body.encode())
+
+
+def test_fetch_html_lane_preserves_code(served):
+ url = "https://wiki.example.org/title/Secure_Boot"
+ served[url] = _html(url, ARCH_WIKI_SHAPED)
+
+ result = BuiltinProvider().fetch(url)
+
+ assert "```" in result.content
+ assert _surviving_links(result.content) == []
+ assert result.looks_like_junk() is None
+
+
+def test_fetch_after_pdf_lane_decline_preserves_code_and_reason(served):
+ abs_url = "https://arxiv.org/abs/2005.14165"
+ pdf_url = "https://arxiv.org/pdf/2005.14165.pdf"
+ served[pdf_url] = _html(pdf_url, "", status=403)
+ served[abs_url] = _html(abs_url, ARCH_WIKI_SHAPED)
+
+ result = BuiltinProvider().fetch(abs_url)
+
+ assert result.metadata[PDF_FAILURE_KEY] == "HTTP 403"
+ assert "```" in result.content
+ assert _surviving_links(result.content) == []