Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
fix: builtin provider flattens fetched code to prose (#129)
Francois du Plessis6240eac05bf2
diff --git a/src/hyperresearch/core/scholar.py b/src/hyperresearch/core/scholar.pyindex bad5052..a877214 100644--- a/src/hyperresearch/core/scholar.py+++ b/src/hyperresearch/core/scholar.py@@ -44,7 +44,9 @@ META_DOI_RE = re.compile(r"content=[\"']\s*(?:doi:)?\s*(10\.[^\"']+)[\"']",re.IGNORECASE,)-BODY_DOI_RE = re.compile(r"\bDOI:?\s*(10\.\d{4,9}/[^\s\"'<>\])}]+)", re.IGNORECASE)+# Fetched bodies carry markdown code spans (`DOI: 10.x/y`, DOI: `10.x/y`), so a+# backtick may open the DOI and never belongs to it.+BODY_DOI_RE = re.compile(r"\bDOI:?\s*`?(10\.\d{4,9}/[^\s\"'<>\])}`]+)", re.IGNORECASE)# Per-host courtesy delay between UNCACHED requests, seconds._HOST_DELAY = {diff --git a/src/hyperresearch/web/builtin.py b/src/hyperresearch/web/builtin.pyindex cf5b63e..b981eb8 100644--- a/src/hyperresearch/web/builtin.py+++ b/src/hyperresearch/web/builtin.py@@ -21,43 +21,126 @@ from hyperresearch.web.pdf import (is_pdf_url,)+# Code in fetched pages must reach the vault as markdown code, not prose.+# Flattened <pre> content passes the extractor's code-strip untouched and+# hits WIKI_LINK_RE, where bash test syntax ([[ -n "$kernel" ]]) is lexically+# a wiki-link; repair/graph then materialise junk stub notes named after+# shell fragments (40 such notes in one real vault from one Arch wiki page).+# Both extraction branches below share these helpers.+++def _fence_for(code: str) -> str:+ """Return a backtick fence longer than any backtick run inside the code."""+ longest = max((len(m.group(0)) for m in re.finditer(r"`+", code)), default=0)+ return "`" * max(3, longest + 1)+++def _fenced(code: str) -> str:+ """Wrap a code block in a fence it cannot break out of."""+ fence = _fence_for(code)+ body = code.strip("\n")+ return f"\n{fence}\n{body}\n{fence}\n"+++def _inline_code(text: str) -> str:+ """Wrap inline code in backticks (doubled when the text contains one)."""+ if "`" in text:+ return f"`` {text} ``"+ return f"`{text}`"+class _TextExtractor(HTMLParser):- """Minimal HTML-to-text extractor (no external deps)."""+ """Minimal HTML-to-text extractor (no external deps).++ Emits <pre> blocks as fenced code and inline <code> as backtick spans;+ see the module comment for why that is load-bearing, not cosmetic.+ """def __init__(self):super().__init__()- self._pieces: list[str] = []+ # (kind, text) where kind is "text" or "code"; code segments skip the+ # per-line strip in get_text() so indentation survives.+ self._segments: list[tuple[str, str]] = []self._skip = Falseself._skip_tags = {"script", "style", "nav", "footer", "header"}self._title = ""self._in_title = False+ self._in_pre = False+ self._pre_buf: list[str] = []+ self._in_inline_code = Falsedef handle_starttag(self, tag, attrs):if tag in self._skip_tags:self._skip = True+ return+ if self._in_pre:+ if tag == "br":+ self._pre_buf.append("\n")+ returnif tag == "title":self._in_title = True+ return+ if tag == "pre":+ self._in_pre = True+ self._pre_buf = []+ return+ if tag == "code":+ self._in_inline_code = True+ returnif tag in ("p", "br", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li"):- self._pieces.append("\n")+ self._segments.append(("text", "\n"))def handle_endtag(self, tag):if tag in self._skip_tags:self._skip = False+ return+ if tag == "pre" and self._in_pre:+ self._in_pre = False+ code = "".join(self._pre_buf)+ self._pre_buf = []+ if code.strip():+ self._segments.append(("code", code))+ returnif tag == "title":self._in_title = False+ return+ if tag == "code":+ self._in_inline_code = Falsedef handle_data(self, data):if self._in_title:self._title = data.strip()- if not self._skip:- self._pieces.append(data)+ if self._skip:+ return+ if self._in_pre:+ self._pre_buf.append(data)+ return+ if self._in_inline_code:+ if data.strip():+ self._segments.append(("text", _inline_code(data)))+ return+ self._segments.append(("text", data))def get_text(self) -> str:- raw = "".join(self._pieces)- # Collapse whitespace- lines = [line.strip() for line in raw.splitlines()]- return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip()+ out: list[str] = []+ buf: list[str] = []++ def flush() -> None:+ raw = "".join(buf)+ lines = [line.strip() for line in raw.splitlines()]+ cleaned = re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip()+ if cleaned:+ out.append(cleaned)+ buf.clear()++ for kind, text in self._segments:+ if kind == "text":+ buf.append(text)+ else:+ flush()+ out.append(_fenced(text).strip("\n"))+ flush()+ return "\n\n".join(out)class BuiltinProvider:@@ -144,13 +227,29 @@ class BuiltinProvider:def _extract(self, html: str) -> tuple[str, str]:"""Extract title and clean text from HTML. Tries bs4 first, falls back to stdlib."""try:- from bs4 import BeautifulSoup+ from bs4 import BeautifulSoup, NavigableStringsoup = BeautifulSoup(html, "html.parser")# Remove noisefor tag in soup(["script", "style", "nav", "footer", "header", "aside"]):tag.decompose()title = soup.title.string.strip() if soup.title and soup.title.string else ""+ # Preserve code as markdown BEFORE get_text() flattens the markup+ # (module comment has the why). <pre> first, so nested <code>+ # inside it is consumed with it; then the remaining inline <code>.+ # Rewrite each element's contents in place: replace_with() and+ # decompose() look the element up in its parent's child list, which+ # is quadratic on a page with thousands of sibling <code> tags.+ for pre in soup.find_all("pre"):+ code_text = pre.get_text()+ pre.clear()+ if code_text.strip():+ pre.append(NavigableString(_fenced(code_text)))+ for code in soup.find_all("code"):+ code_text = code.get_text()+ code.clear()+ if code_text.strip():+ code.append(NavigableString(_inline_code(code_text)))text = soup.get_text(separator="\n", strip=True)# Collapse blank linestext = re.sub(r"\n{3,}", "\n\n", text)diff --git a/tests/test_core/test_source_ranking.py b/tests/test_core/test_source_ranking.pyindex 99da5ec..126fed7 100644--- a/tests/test_core/test_source_ranking.py+++ b/tests/test_core/test_source_ranking.py@@ -104,6 +104,12 @@ class TestDoiExtraction:def test_trailing_punctuation_stripped(self):assert extract_doi("https://x.com", content="see DOI: 10.1000/xyz123.") == "10.1000/xyz123"+ def test_body_doi_inside_code_span(self):+ # Extracted page text renders <code> as a backtick span; the backtick+ # must not end up in the DOI, and must not hide it either.+ assert extract_doi("https://x.com", content="cite `DOI: 10.1000/xyz123` here") == "10.1000/xyz123"+ assert extract_doi("https://x.com", content="DOI: `10.1000/xyz123`") == "10.1000/xyz123"+class TestPageRank:def test_hub_ranks_highest(self):diff --git a/tests/test_web/test_builtin_fences.py b/tests/test_web/test_builtin_fences.pynew file mode 100644index 0000000..215d55b--- /dev/null+++ b/tests/test_web/test_builtin_fences.py@@ -0,0 +1,151 @@+"""The builtin provider must not flatten fetched code to prose.++Flattened <pre>/<code> content passes the note parser's code-strip untouched+and reaches WIKI_LINK_RE, where bash test syntax is lexically a wiki-link;+repair/graph then materialise junk stub notes named after shell fragments+(40 such notes in one real vault, all fetched via this provider). These+tests pin markdown-shaped code handling in BOTH extraction branches: bs4+and the stdlib fallback.+"""++import pytest++from hyperresearch.core.patterns import (+ CODE_BLOCK_RE,+ INLINE_CODE_RE,+ WIKI_LINK_RE,+ is_valid_wiki_link_target,+)+from hyperresearch.web.builtin import BuiltinProvider, _TextExtractor+from hyperresearch.web.pdf import PDF_FAILURE_KEY+from hyperresearch.web.safe_http import SafeResponse++# Shaped like the real defect page (a wiki page whose body carries a shell+# script in <pre> plus a conditional in inline <code>, both with [[ ]] test+# syntax).+ARCH_WIKI_SHAPED = """<html><head><title>Secure Boot - ArchWiki</title></head>+<body><main>+<p>Create the following file and make it executable:</p>+<pre>#!/usr/bin/env bash++kernel="$1"+[[ -n "$kernel" ]] || exit 0++# use already installed kernel if it exists+[[ ! -f "$KERNELDESTINATION" ]] || kernel="$KERNELDESTINATION"</pre>+<p>Then check <code>[[ -f /boot/vmlinuz ]]</code> before rebooting.</p>+</main></body></html>"""+++def _surviving_links(text: str) -> list[str]:+ """The exact strip-then-extract sequence core/note.py runs on a note body."""+ cleaned = CODE_BLOCK_RE.sub("", text)+ cleaned = INLINE_CODE_RE.sub("", cleaned)+ raw = (m.group(1).strip().rstrip("\\") for m in WIKI_LINK_RE.finditer(cleaned))+ return [ref for ref in raw if is_valid_wiki_link_target(ref)]+++class _BranchContract:+ """Shared assertions; subclasses supply the extraction branch."""++ def extract(self, html: str) -> tuple[str, str]:+ raise NotImplementedError++ def test_pre_becomes_fenced_block(self):+ _, text = self.extract(ARCH_WIKI_SHAPED)+ assert "```" in text+ assert '[[ -n "$kernel" ]] || exit 0' in text++ def test_inline_code_becomes_backtick_span(self):+ _, text = self.extract(ARCH_WIKI_SHAPED)+ assert "`[[ -f /boot/vmlinuz ]]`" in text++ def test_no_wiki_links_survive_the_note_strip(self):+ # The end-to-end guarantee the fix exists for: after the exact+ # code-strip the link extractor applies, no [[ ]] fragment remains.+ _, text = self.extract(ARCH_WIKI_SHAPED)+ assert _surviving_links(text) == []++ def test_code_indentation_survives(self):+ html = "<html><body><p>x</p><pre>if x:\n do_thing()</pre></body></html>"+ _, text = self.extract(html)+ assert " do_thing()" in text++ def test_fence_outgrows_backticks_inside_code(self):+ html = "<html><body><pre>echo ```already fenced```</pre></body></html>"+ _, text = self.extract(html)+ assert "````" in text+ assert _surviving_links(text) == []++ def test_title_and_noise_removal_unchanged(self):+ html = (+ "<html><head><title>T</title></head><body>"+ "<nav>menu here</nav><p>body text</p><footer>foot here</footer>"+ "</body></html>"+ )+ title, text = self.extract(html)+ assert title == "T"+ assert "menu here" not in text+ assert "foot here" not in text+ assert "body text" in text+++class TestBs4Branch(_BranchContract):+ def extract(self, html: str) -> tuple[str, str]:+ pytest.importorskip("bs4")+ return BuiltinProvider()._extract(html)+++class TestStdlibBranch(_BranchContract):+ def extract(self, html: str) -> tuple[str, str]:+ parser = _TextExtractor()+ parser.feed(html)+ return parser._title, parser.get_text()+++# fetch() is the only way content leaves the provider: HTML fetched directly,+# and HTML reached after the PDF lane declines a PDF-shaped URL. Both must go+# through the code-preserving extraction, and the decline reason must still+# travel with the result. The SSRF-gated download is stubbed at its source.+++def served(monkeypatch):+ table: dict[str, SafeResponse] = {}++ def fake_safe_get(url, *, max_bytes, timeout=None, headers=None, verify=True,+ allow_private_hosts=()):+ return table[url]++ monkeypatch.setattr("hyperresearch.web.safe_http.safe_get", fake_safe_get)+ monkeypatch.setattr("hyperresearch.web.safe_http.check_url", lambda url, allow=(): None)+ return table+++def _html(url: str, body: str, status: int = 200) -> SafeResponse:+ return SafeResponse(url=url, status_code=status, headers={"content-type": "text/html"},+ content=body.encode())+++def test_fetch_html_lane_preserves_code(served):+ url = "https://wiki.example.org/title/Secure_Boot"+ served[url] = _html(url, ARCH_WIKI_SHAPED)++ result = BuiltinProvider().fetch(url)++ assert "```" in result.content+ assert _surviving_links(result.content) == []+ assert result.looks_like_junk() is None+++def test_fetch_after_pdf_lane_decline_preserves_code_and_reason(served):+ abs_url = "https://arxiv.org/abs/2005.14165"+ pdf_url = "https://arxiv.org/pdf/2005.14165.pdf"+ served[pdf_url] = _html(pdf_url, "", status=403)+ served[abs_url] = _html(abs_url, ARCH_WIKI_SHAPED)++ result = BuiltinProvider().fetch(abs_url)++ assert result.metadata[PDF_FAILURE_KEY] == "HTTP 403"+ assert "```" in result.content+ assert _surviving_links(result.content) == []