Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: web fetch tokenizes a page once and replays it to the later main-content passes (#15686)

Daniel Hanbebcc2369528
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.py
index 75cfe476815..5eb57261f2d 100644
--- a/studio/backend/core/inference/_html_to_md.py
+++ b/studio/backend/core/inference/_html_to_md.py
@@ -798,6 +798,7 @@ class _MarkdownRenderer(HTMLParser):
page_span_limit: int | None = None,
):
super().__init__(convert_charrefs = False)
+ self._record: list | None = None
# parser positions of buttons that carry their heading title, found by _TitleButtonScan
self.title_buttons: frozenset[tuple[int, int]] = frozenset()
self._title_scan: _TitleButtonScan | None = None
@@ -1566,6 +1567,8 @@ class _MarkdownRenderer(HTMLParser):
self._close_rendered_tag(section[1])
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ if self._record is not None:
+ self._record.append((0, (tag, tuple(attrs), *self.getpos())))
tag = tag.lower()
if self._title_scan is not None:
self._title_scan.starttag(tag, attrs, self.getpos())
@@ -1757,6 +1760,8 @@ class _MarkdownRenderer(HTMLParser):
return
def handle_endtag(self, tag: str) -> None:
+ if self._record is not None:
+ self._record.append((1, tag))
tag = tag.lower()
if self._title_scan is not None:
self._title_scan.endtag(tag)
@@ -1947,6 +1952,8 @@ class _MarkdownRenderer(HTMLParser):
return self._scope_tags is not None and self._scope_depth == 0
def handle_data(self, data: str) -> None:
+ if self._record is not None:
+ self._record.append((2, data))
if self._title_scan is not None:
self._title_scan.data(data)
if self._text_suppressed():
@@ -1973,6 +1980,8 @@ class _MarkdownRenderer(HTMLParser):
self._emit(text)
def handle_entityref(self, name: str) -> None:
+ if self._record is not None:
+ self._record.append((3, name))
text = html.unescape(f"&{name};")
if self._title_scan is not None:
self._title_scan.data(text)
@@ -1982,6 +1991,8 @@ class _MarkdownRenderer(HTMLParser):
self._emit(text)
def handle_charref(self, name: str) -> None:
+ if self._record is not None:
+ self._record.append((4, name))
text = html.unescape(f"&#{name};")
if self._title_scan is not None:
self._title_scan.data(text)
@@ -1990,6 +2001,22 @@ class _MarkdownRenderer(HTMLParser):
self._count_header_text(text)
self._emit(text)
+ def replay(self, events: list) -> None:
+ """Re-run recorded handler calls in order, restoring ``getpos()`` at each start tag."""
+ start, end, data = self.handle_starttag, self.handle_endtag, self.handle_data
+ for kind, arg in events:
+ if kind == 2:
+ data(arg)
+ elif kind == 0:
+ tag, attrs, self.lineno, self.offset = arg
+ start(tag, list(attrs))
+ elif kind == 1:
+ end(arg)
+ elif kind == 3:
+ self.handle_entityref(arg)
+ else:
+ self.handle_charref(arg)
+
def flush_pending(self) -> None:
"""Flush open side-buffers into ``_out`` after close(), recovering truncated HTML."""
# Headers first: a frame emits into the enclosing link or cell, which must still be open.
@@ -2134,6 +2161,19 @@ def _strip_boilerplate_lines(text: str, site_links: SiteLinks | None = None) ->
return re.sub(r"\n{3,}", "\n\n", "\n".join(out)).strip()
+class _TokenLog:
+ """Handler calls of one document, recorded by the first fed pass and replayed by later ones.
+ Sound only while the renderer overrides just these five handlers and reads parser state only
+ via ``getpos()`` in ``handle_starttag``."""
+
+ __slots__ = ("events", "armed")
+
+ def __init__(self, armed: bool) -> None:
+ self.events: list | None = None
+ # record only when html_to_markdown knows another pass follows
+ self.armed = armed
+
+
def _new_renderer(
source_html: str,
scope_tags: frozenset[str] | None,
@@ -2141,6 +2181,7 @@ def _new_renderer(
site_links: SiteLinks | None = None,
span_char_limit: int | None = None,
header_decisions: list[bool] | None = None,
+ tokens: _TokenLog | None = None,
) -> _MarkdownRenderer:
def build(
title_buttons: frozenset[tuple[int, int]], scan: _TitleButtonScan | None
@@ -2155,8 +2196,16 @@ def _new_renderer(
)
renderer.title_buttons = title_buttons
renderer._title_scan = scan
- renderer.feed(source_html)
- renderer.close()
+ if tokens is None:
+ renderer.feed(source_html)
+ renderer.close()
+ elif tokens.events is None:
+ renderer._record = [] if tokens.armed else None
+ renderer.feed(source_html)
+ renderer.close()
+ tokens.events, renderer._record = renderer._record, None
+ else:
+ renderer.replay(tokens.events)
renderer.flush_pending()
return renderer
@@ -2191,6 +2240,7 @@ def _select_main_scope_render(
tag: str,
site_links: SiteLinks | None,
span_char_limit: int | None = None,
+ tokens: _TokenLog | None = None,
) -> tuple[int, str, int, int]:
"""return the best eligible score, subtree, count, and visible length; cap header credit at retained prose."""
renderer = _new_renderer(
@@ -2199,6 +2249,7 @@ def _select_main_scope_render(
strip_header = True,
site_links = site_links,
span_char_limit = span_char_limit,
+ tokens = tokens,
)
# generated span cells must not lift a scope past the content gate or rank; only span pages pay this second pass
scoring = (
@@ -2208,6 +2259,7 @@ def _select_main_scope_render(
strip_header = True,
span_char_limit = 0,
header_decisions = renderer.header_decisions,
+ tokens = tokens,
)
if renderer._has_generated_spans
else renderer
@@ -2217,7 +2269,11 @@ def _select_main_scope_render(
best_visible = 0
for i, seg in enumerate(renderer.scope_segments):
rendered = _strip_boilerplate_lines(_cleanup(seg), site_links)
- scored = _strip_boilerplate_lines(_cleanup(scoring.scope_segments[i]), site_links)
+ scored = (
+ rendered
+ if scoring is renderer
+ else _strip_boilerplate_lines(_cleanup(scoring.scope_segments[i]), site_links)
+ )
if site_links is not None:
scored = site_links.clean(scored)
visible = _visible_chars(scored)
@@ -2238,9 +2294,12 @@ def _select_main_scope_render(
def _render_main_document(
- source_html: str, site_links: SiteLinks | None, span_char_limit: int
+ source_html: str,
+ site_links: SiteLinks | None,
+ span_char_limit: int,
+ tokens: _TokenLog | None = None,
) -> tuple[int, str]:
- renderer = _new_renderer(source_html, None, True, site_links, span_char_limit)
+ renderer = _new_renderer(source_html, None, True, site_links, span_char_limit, tokens = tokens)
rendered = _strip_boilerplate_lines(_cleanup("".join(renderer._out)), site_links)
scoring = (
_new_renderer(
@@ -2249,11 +2308,16 @@ def _render_main_document(
True,
span_char_limit = 0,
header_decisions = renderer.header_decisions,
+ tokens = tokens,
)
if renderer._has_generated_spans
else renderer
)
- scored = _strip_boilerplate_lines(_cleanup("".join(scoring._out)), site_links)
+ scored = (
+ rendered
+ if scoring is renderer
+ else _strip_boilerplate_lines(_cleanup("".join(scoring._out)), site_links)
+ )
if site_links is not None:
scored = site_links.clean(scored)
return _visible_chars(scored), rendered
@@ -2278,6 +2342,8 @@ def _visible_len(line: str) -> int:
"""Length of *line* without Markdown link destinations. A tracking URL is not
prose: a card holding one [Read](/x?<300 bytes>) otherwise scored 339 visible
characters off 4 and displaced the article. Scanned, so no backtracking."""
+ if "[" not in line and "\\" not in line:
+ return len(line)
total, i, n = 0, 0, len(line)
open_bracket = False
while i < n:
@@ -2312,6 +2378,9 @@ def _visible_len(line: str) -> int:
# Scoped conversions shorter than this are rejected as main content, e.g. an empty article stub
_MIN_MAIN_CONTENT_CHARS = 200
+# absent text = no such tag can tokenize, so that scope finds nothing
+_ARTICLE_START = re.compile(r"<article", re.IGNORECASE)
+_MAIN_START = re.compile(r"<main", re.IGNORECASE)
def html_to_markdown(
@@ -2329,16 +2398,22 @@ def html_to_markdown(
rendered = ""
full_rendered = ""
if main_content:
+ tokens = _TokenLog(armed = _ARTICLE_START.search(source_html) is None)
length, rendered, articles, article_visible = _select_main_scope_render(
- source_html, "article", site_links, span_limit
+ source_html, "article", site_links, span_limit, tokens
)
if length < _MIN_MAIN_CONTENT_CHARS or articles > 1:
+ # no <main> = main scope scores 0, so the whole-page pass below runs
+ tokens.armed = _MAIN_START.search(source_html) is None and (
+ articles > 2 or length < _MIN_MAIN_CONTENT_CHARS
+ )
main_length, main_rendered, _, main_visible = _select_main_scope_render(
- source_html, "main", site_links, span_limit
+ source_html, "main", site_links, span_limit, tokens
)
if articles > 2 and main_length < _MIN_MAIN_CONTENT_CHARS:
+ tokens.armed = False
main_length, full_rendered = _render_main_document(
- source_html, site_links, span_limit
+ source_html, site_links, span_limit, tokens
)
main_rendered = full_rendered
main_visible = main_length
@@ -2348,7 +2423,10 @@ def html_to_markdown(
length, rendered = main_length, main_rendered
if length < _MIN_MAIN_CONTENT_CHARS:
if not full_rendered:
- _, full_rendered = _render_main_document(source_html, site_links, span_limit)
+ tokens.armed = False
+ _, full_rendered = _render_main_document(
+ source_html, site_links, span_limit, tokens
+ )
rendered = full_rendered
else:
rendered = _render(source_html, None, site_links = site_links, span_char_limit = span_limit)
diff --git a/studio/backend/tests/test_html_to_md_token_replay.py b/studio/backend/tests/test_html_to_md_token_replay.py
new file mode 100644
index 00000000000..004bf9a397a
--- /dev/null
+++ b/studio/backend/tests/test_html_to_md_token_replay.py
@@ -0,0 +1,172 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""html_to_markdown(main_content=True) tokenizes a page once and replays the tokenizer's handler
+calls to its later scope / scoring passes. Output must equal feeding every pass the HTML again."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+import core.inference._html_to_md as h2m
+from core.inference._html_to_md import SiteLinks, html_to_markdown
+
+_PROSE = " ".join(
+ f"Sentence {i} of the body text, long enough to count as prose." for i in range(12)
+)
+
+PAGES = {
+ "no_article": f"<html><body><nav><a href='/a'>Home</a></nav><main><h1>T</h1><p>{_PROSE}</p></main></body></html>",
+ "one_article": f"<article><h2>Title</h2><p>{_PROSE}</p></article><footer>f</footer>",
+ "many_articles": "".join(
+ f"<article><h3>Card {i}</h3><p>{_PROSE[:120]}</p></article>" for i in range(5)
+ )
+ + f"<main><p>{_PROSE}</p></main>",
+ "many_articles_no_main": "".join(
+ f"<ARTICLE><p>card {i} &amp; more</p></ARTICLE>" for i in range(4)
+ )
+ + f"<div><p>{_PROSE}</p></div>",
+ "accordion_titles": "<main>"
+ + "".join(
+ f"<h3><button type='button' aria-expanded='false'>Question {i}?</button></h3><div><p>{_PROSE[:200]}</p></div>"
+ for i in range(4)
+ )
+ + "</main>",
+ "accordion_multiline": "<article>\n<h3>\n <button>Q\none</button>\n</h3>\n<p>"
+ + _PROSE
+ + "</p>\n<h3><button>Q two</button> extra</h3></article>",
+ "span_table": "<main><table><tr><td colspan='40'>wide</td><td rowspan='300'>tall</td></tr>"
+ "<tr><td>a</td><td>b</td></tr></table>" + f"<p>{_PROSE}</p></main>",
+ "charrefs": f"<main><p>&lt;tag&gt; &#65;&#x42; &#0; &#xD800; &#99999999; &nbsp; &notanentity; &amp</p><p>{_PROSE}</p></main>",
+ "script_style": f"<main><script>if (a < b) {{ x = '</div>' }}</script><style>p{{}}</style><p>{_PROSE}</p>"
+ "<textarea><b>raw</b></textarea><title>T &amp; t</title></main>",
+ "comments_cdata": f"<main><!-- c --><!--->x<!-- a -- b --!><![CDATA[ d ]]><!DOCTYPE x><?pi ?><p>{_PROSE}</p></main>",
+ "selfclosing": f"<main><p/>text<br/><img src='i.png' alt='A &lt; B'/><div/><a href='/x'/>{_PROSE}</main>",
+ "unclosed": f"<article><div><p>{_PROSE}<b>bold<i>both",
+ "truncated_tag": f"<main><p>{_PROSE}</p><a href='/x",
+ "svg_math": f"<main><svg><title>s</title><foreignObject><p>in</p></foreignObject></svg><math><mi>x</mi></math><p>{_PROSE}</p></main>",
+ "crlf": f"<main>\r\n<p>{_PROSE}</p>\r<pre>a\r\nb</pre></main>",
+ "div_only": f"<div><h1>T</h1><p>{_PROSE}</p><div><table><tr><td>x</td></tr></table></div></div>",
+ "empty": "",
+ "text_only": _PROSE,
+}
+
+
+class _NeverReplay:
+ """Stands in for _TokenLog: every pass feeds the HTML again, as before the replay."""
+
+ def __init__(self, armed: bool) -> None:
+ pass
+
+ events = property(lambda self: None, lambda self, value: None)
+ armed = property(lambda self: False, lambda self, value: None)
+
+
+def _reference(monkeypatch, *args, **kwargs):
+ with monkeypatch.context() as m:
+ m.setattr(h2m, "_TokenLog", _NeverReplay, raising = False)
+ h2m._TITLE_BUTTON_CACHE.clear()
+ return html_to_markdown(*args, **kwargs)
+
+
+def _feeds(monkeypatch):
+ calls = []
+ real = h2m._MarkdownRenderer.feed
+
+ def counted(self, data):
+ calls.append(1)
+ return real(self, data)
+
+ monkeypatch.setattr(h2m._MarkdownRenderer, "feed", counted)
+ return calls
+
+
[email protected]("name", sorted(PAGES))
[email protected]("span", [None, 50])
+def test_replayed_passes_render_what_fed_passes_render(monkeypatch, name, span):
+ html = PAGES[name]
+ expected = _reference(monkeypatch, html, main_content = True, max_span_chars = span)
+ h2m._TITLE_BUTTON_CACHE.clear()
+ assert html_to_markdown(html, main_content = True, max_span_chars = span) == expected
+ # warm title-button cache path
+ assert html_to_markdown(html, main_content = True, max_span_chars = span) == expected
+
+
[email protected]("name", sorted(PAGES))
+def test_site_links_record_the_same_links(monkeypatch, name):
+ html = PAGES[name]
+ ref_links = SiteLinks("https://example.com/page")
+ with monkeypatch.context() as m:
+ m.setattr(h2m, "_TokenLog", _NeverReplay, raising = False)
+ m.setattr(h2m.secrets, "token_hex", lambda n: "0" * (2 * n))
+ h2m._TITLE_BUTTON_CACHE.clear()
+ expected = html_to_markdown(html, main_content = True, site_links = ref_links)
+ with monkeypatch.context() as m:
+ m.setattr(h2m.secrets, "token_hex", lambda n: "0" * (2 * n))
+ links = SiteLinks("https://example.com/page")
+ h2m._TITLE_BUTTON_CACHE.clear()
+ got = html_to_markdown(html, main_content = True, site_links = links)
+ assert got == expected
+ assert links.strip(got) == ref_links.strip(expected)
+ assert links._found == ref_links._found
+
+
+def test_a_multi_pass_page_is_tokenized_once(monkeypatch):
+ feeds = _feeds(monkeypatch)
+ h2m._TITLE_BUTTON_CACHE.clear()
+ html_to_markdown(PAGES["div_only"], main_content = True)
+ assert len(feeds) == 1
+
+
+def test_a_whole_page_pass_known_to_follow_the_main_scope_replays(monkeypatch):
+ # four articles and no <main>: article scope and main scope are fed, the whole-page pass replays
+ feeds = _feeds(monkeypatch)
+ h2m._TITLE_BUTTON_CACHE.clear()
+ html_to_markdown(PAGES["many_articles_no_main"], main_content = True)
+ assert len(feeds) == 2
+
+
+def test_passes_that_cannot_be_foreseen_are_fed_without_recording(monkeypatch):
+ recorded = []
+ real = h2m._MarkdownRenderer.feed
+
+ def spy(self, data):
+ recorded.append(self._record is not None)
+ return real(self, data)
+
+ monkeypatch.setattr(h2m._MarkdownRenderer, "feed", spy)
+ for name in ("one_article", "many_articles"):
+ h2m._TITLE_BUTTON_CACHE.clear()
+ recorded.clear()
+ html_to_markdown(PAGES[name], main_content = True)
+ assert recorded and not any(recorded), name
+
+
+def test_plain_conversion_is_untouched(monkeypatch):
+ feeds = _feeds(monkeypatch)
+ for name in sorted(PAGES):
+ h2m._TITLE_BUTTON_CACHE.clear()
+ before = len(feeds)
+ html_to_markdown(PAGES[name])
+ assert len(feeds) - before in (1, 2) # 2 only for the accordion title rescan
+
+
+def test_a_renderer_error_surfaces_from_the_first_pass(monkeypatch):
+ real = h2m._MarkdownRenderer.handle_data
+
+ def boom(self, data):
+ if "Sentence 3" in data:
+ raise RuntimeError("renderer failed")
+ return real(self, data)
+
+ monkeypatch.setattr(h2m._MarkdownRenderer, "handle_data", boom)
+ h2m._TITLE_BUTTON_CACHE.clear()
+ with pytest.raises(RuntimeError, match = "renderer failed"):
+ html_to_markdown(PAGES["no_article"], main_content = True)