Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: web fetch tokenizes a page once and replays it to the later main-content passes (#15686)
Daniel Hanbebcc2369528
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.pyindex 75cfe476815..5eb57261f2d 100644--- a/studio/backend/core/inference/_html_to_md.py+++ b/studio/backend/core/inference/_html_to_md.py@@ -798,6 +798,7 @@ class _MarkdownRenderer(HTMLParser):page_span_limit: int | None = None,):super().__init__(convert_charrefs = False)+ self._record: list | None = None# parser positions of buttons that carry their heading title, found by _TitleButtonScanself.title_buttons: frozenset[tuple[int, int]] = frozenset()self._title_scan: _TitleButtonScan | None = None@@ -1566,6 +1567,8 @@ class _MarkdownRenderer(HTMLParser):self._close_rendered_tag(section[1])def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:+ if self._record is not None:+ self._record.append((0, (tag, tuple(attrs), *self.getpos())))tag = tag.lower()if self._title_scan is not None:self._title_scan.starttag(tag, attrs, self.getpos())@@ -1757,6 +1760,8 @@ class _MarkdownRenderer(HTMLParser):returndef handle_endtag(self, tag: str) -> None:+ if self._record is not None:+ self._record.append((1, tag))tag = tag.lower()if self._title_scan is not None:self._title_scan.endtag(tag)@@ -1947,6 +1952,8 @@ class _MarkdownRenderer(HTMLParser):return self._scope_tags is not None and self._scope_depth == 0def handle_data(self, data: str) -> None:+ if self._record is not None:+ self._record.append((2, data))if self._title_scan is not None:self._title_scan.data(data)if self._text_suppressed():@@ -1973,6 +1980,8 @@ class _MarkdownRenderer(HTMLParser):self._emit(text)def handle_entityref(self, name: str) -> None:+ if self._record is not None:+ self._record.append((3, name))text = html.unescape(f"&{name};")if self._title_scan is not None:self._title_scan.data(text)@@ -1982,6 +1991,8 @@ class _MarkdownRenderer(HTMLParser):self._emit(text)def handle_charref(self, name: str) -> None:+ if self._record is not None:+ self._record.append((4, name))text = html.unescape(f"&#{name};")if self._title_scan is not None:self._title_scan.data(text)@@ -1990,6 +2001,22 @@ class _MarkdownRenderer(HTMLParser):self._count_header_text(text)self._emit(text)+ def replay(self, events: list) -> None:+ """Re-run recorded handler calls in order, restoring ``getpos()`` at each start tag."""+ start, end, data = self.handle_starttag, self.handle_endtag, self.handle_data+ for kind, arg in events:+ if kind == 2:+ data(arg)+ elif kind == 0:+ tag, attrs, self.lineno, self.offset = arg+ start(tag, list(attrs))+ elif kind == 1:+ end(arg)+ elif kind == 3:+ self.handle_entityref(arg)+ else:+ self.handle_charref(arg)+def flush_pending(self) -> None:"""Flush open side-buffers into ``_out`` after close(), recovering truncated HTML."""# Headers first: a frame emits into the enclosing link or cell, which must still be open.@@ -2134,6 +2161,19 @@ def _strip_boilerplate_lines(text: str, site_links: SiteLinks | None = None) ->return re.sub(r"\n{3,}", "\n\n", "\n".join(out)).strip()+class _TokenLog:+ """Handler calls of one document, recorded by the first fed pass and replayed by later ones.+ Sound only while the renderer overrides just these five handlers and reads parser state only+ via ``getpos()`` in ``handle_starttag``."""++ __slots__ = ("events", "armed")++ def __init__(self, armed: bool) -> None:+ self.events: list | None = None+ # record only when html_to_markdown knows another pass follows+ self.armed = armed++def _new_renderer(source_html: str,scope_tags: frozenset[str] | None,@@ -2141,6 +2181,7 @@ def _new_renderer(site_links: SiteLinks | None = None,span_char_limit: int | None = None,header_decisions: list[bool] | None = None,+ tokens: _TokenLog | None = None,) -> _MarkdownRenderer:def build(title_buttons: frozenset[tuple[int, int]], scan: _TitleButtonScan | None@@ -2155,8 +2196,16 @@ def _new_renderer()renderer.title_buttons = title_buttonsrenderer._title_scan = scan- renderer.feed(source_html)- renderer.close()+ if tokens is None:+ renderer.feed(source_html)+ renderer.close()+ elif tokens.events is None:+ renderer._record = [] if tokens.armed else None+ renderer.feed(source_html)+ renderer.close()+ tokens.events, renderer._record = renderer._record, None+ else:+ renderer.replay(tokens.events)renderer.flush_pending()return renderer@@ -2191,6 +2240,7 @@ def _select_main_scope_render(tag: str,site_links: SiteLinks | None,span_char_limit: int | None = None,+ tokens: _TokenLog | None = None,) -> tuple[int, str, int, int]:"""return the best eligible score, subtree, count, and visible length; cap header credit at retained prose."""renderer = _new_renderer(@@ -2199,6 +2249,7 @@ def _select_main_scope_render(strip_header = True,site_links = site_links,span_char_limit = span_char_limit,+ tokens = tokens,)# generated span cells must not lift a scope past the content gate or rank; only span pages pay this second passscoring = (@@ -2208,6 +2259,7 @@ def _select_main_scope_render(strip_header = True,span_char_limit = 0,header_decisions = renderer.header_decisions,+ tokens = tokens,)if renderer._has_generated_spanselse renderer@@ -2217,7 +2269,11 @@ def _select_main_scope_render(best_visible = 0for i, seg in enumerate(renderer.scope_segments):rendered = _strip_boilerplate_lines(_cleanup(seg), site_links)- scored = _strip_boilerplate_lines(_cleanup(scoring.scope_segments[i]), site_links)+ scored = (+ rendered+ if scoring is renderer+ else _strip_boilerplate_lines(_cleanup(scoring.scope_segments[i]), site_links)+ )if site_links is not None:scored = site_links.clean(scored)visible = _visible_chars(scored)@@ -2238,9 +2294,12 @@ def _select_main_scope_render(def _render_main_document(- source_html: str, site_links: SiteLinks | None, span_char_limit: int+ source_html: str,+ site_links: SiteLinks | None,+ span_char_limit: int,+ tokens: _TokenLog | None = None,) -> tuple[int, str]:- renderer = _new_renderer(source_html, None, True, site_links, span_char_limit)+ renderer = _new_renderer(source_html, None, True, site_links, span_char_limit, tokens = tokens)rendered = _strip_boilerplate_lines(_cleanup("".join(renderer._out)), site_links)scoring = (_new_renderer(@@ -2249,11 +2308,16 @@ def _render_main_document(True,span_char_limit = 0,header_decisions = renderer.header_decisions,+ tokens = tokens,)if renderer._has_generated_spanselse renderer)- scored = _strip_boilerplate_lines(_cleanup("".join(scoring._out)), site_links)+ scored = (+ rendered+ if scoring is renderer+ else _strip_boilerplate_lines(_cleanup("".join(scoring._out)), site_links)+ )if site_links is not None:scored = site_links.clean(scored)return _visible_chars(scored), rendered@@ -2278,6 +2342,8 @@ def _visible_len(line: str) -> int:"""Length of *line* without Markdown link destinations. A tracking URL is notprose: a card holding one [Read](/x?<300 bytes>) otherwise scored 339 visiblecharacters off 4 and displaced the article. Scanned, so no backtracking."""+ if "[" not in line and "\\" not in line:+ return len(line)total, i, n = 0, 0, len(line)open_bracket = Falsewhile i < n:@@ -2312,6 +2378,9 @@ def _visible_len(line: str) -> int:# Scoped conversions shorter than this are rejected as main content, e.g. an empty article stub_MIN_MAIN_CONTENT_CHARS = 200+# absent text = no such tag can tokenize, so that scope finds nothing+_ARTICLE_START = re.compile(r"<article", re.IGNORECASE)+_MAIN_START = re.compile(r"<main", re.IGNORECASE)def html_to_markdown(@@ -2329,16 +2398,22 @@ def html_to_markdown(rendered = ""full_rendered = ""if main_content:+ tokens = _TokenLog(armed = _ARTICLE_START.search(source_html) is None)length, rendered, articles, article_visible = _select_main_scope_render(- source_html, "article", site_links, span_limit+ source_html, "article", site_links, span_limit, tokens)if length < _MIN_MAIN_CONTENT_CHARS or articles > 1:+ # no <main> = main scope scores 0, so the whole-page pass below runs+ tokens.armed = _MAIN_START.search(source_html) is None and (+ articles > 2 or length < _MIN_MAIN_CONTENT_CHARS+ )main_length, main_rendered, _, main_visible = _select_main_scope_render(- source_html, "main", site_links, span_limit+ source_html, "main", site_links, span_limit, tokens)if articles > 2 and main_length < _MIN_MAIN_CONTENT_CHARS:+ tokens.armed = Falsemain_length, full_rendered = _render_main_document(- source_html, site_links, span_limit+ source_html, site_links, span_limit, tokens)main_rendered = full_renderedmain_visible = main_length@@ -2348,7 +2423,10 @@ def html_to_markdown(length, rendered = main_length, main_renderedif length < _MIN_MAIN_CONTENT_CHARS:if not full_rendered:- _, full_rendered = _render_main_document(source_html, site_links, span_limit)+ tokens.armed = False+ _, full_rendered = _render_main_document(+ source_html, site_links, span_limit, tokens+ )rendered = full_renderedelse:rendered = _render(source_html, None, site_links = site_links, span_char_limit = span_limit)diff --git a/studio/backend/tests/test_html_to_md_token_replay.py b/studio/backend/tests/test_html_to_md_token_replay.pynew file mode 100644index 00000000000..004bf9a397a--- /dev/null+++ b/studio/backend/tests/test_html_to_md_token_replay.py@@ -0,0 +1,172 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""html_to_markdown(main_content=True) tokenizes a page once and replays the tokenizer's handler+calls to its later scope / scoring passes. Output must equal feeding every pass the HTML again."""++from __future__ import annotations++import sys+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++import core.inference._html_to_md as h2m+from core.inference._html_to_md import SiteLinks, html_to_markdown++_PROSE = " ".join(+ f"Sentence {i} of the body text, long enough to count as prose." for i in range(12)+)++PAGES = {+ "no_article": f"<html><body><nav><a href='/a'>Home</a></nav><main><h1>T</h1><p>{_PROSE}</p></main></body></html>",+ "one_article": f"<article><h2>Title</h2><p>{_PROSE}</p></article><footer>f</footer>",+ "many_articles": "".join(+ f"<article><h3>Card {i}</h3><p>{_PROSE[:120]}</p></article>" for i in range(5)+ )+ + f"<main><p>{_PROSE}</p></main>",+ "many_articles_no_main": "".join(+ f"<ARTICLE><p>card {i} & more</p></ARTICLE>" for i in range(4)+ )+ + f"<div><p>{_PROSE}</p></div>",+ "accordion_titles": "<main>"+ + "".join(+ f"<h3><button type='button' aria-expanded='false'>Question {i}?</button></h3><div><p>{_PROSE[:200]}</p></div>"+ for i in range(4)+ )+ + "</main>",+ "accordion_multiline": "<article>\n<h3>\n <button>Q\none</button>\n</h3>\n<p>"+ + _PROSE+ + "</p>\n<h3><button>Q two</button> extra</h3></article>",+ "span_table": "<main><table><tr><td colspan='40'>wide</td><td rowspan='300'>tall</td></tr>"+ "<tr><td>a</td><td>b</td></tr></table>" + f"<p>{_PROSE}</p></main>",+ "charrefs": f"<main><p><tag> AB � � � ¬anentity; &</p><p>{_PROSE}</p></main>",+ "script_style": f"<main><script>if (a < b) {{ x = '</div>' }}</script><style>p{{}}</style><p>{_PROSE}</p>"+ "<textarea><b>raw</b></textarea><title>T & t</title></main>",+ "comments_cdata": f"<main><!-- c --><!--->x<!-- a -- b --!><![CDATA[ d ]]><!DOCTYPE x><?pi ?><p>{_PROSE}</p></main>",+ "selfclosing": f"<main><p/>text<br/><img src='i.png' alt='A < B'/><div/><a href='/x'/>{_PROSE}</main>",+ "unclosed": f"<article><div><p>{_PROSE}<b>bold<i>both",+ "truncated_tag": f"<main><p>{_PROSE}</p><a href='/x",+ "svg_math": f"<main><svg><title>s</title><foreignObject><p>in</p></foreignObject></svg><math><mi>x</mi></math><p>{_PROSE}</p></main>",+ "crlf": f"<main>\r\n<p>{_PROSE}</p>\r<pre>a\r\nb</pre></main>",+ "div_only": f"<div><h1>T</h1><p>{_PROSE}</p><div><table><tr><td>x</td></tr></table></div></div>",+ "empty": "",+ "text_only": _PROSE,+}+++class _NeverReplay:+ """Stands in for _TokenLog: every pass feeds the HTML again, as before the replay."""++ def __init__(self, armed: bool) -> None:+ pass++ events = property(lambda self: None, lambda self, value: None)+ armed = property(lambda self: False, lambda self, value: None)+++def _reference(monkeypatch, *args, **kwargs):+ with monkeypatch.context() as m:+ m.setattr(h2m, "_TokenLog", _NeverReplay, raising = False)+ h2m._TITLE_BUTTON_CACHE.clear()+ return html_to_markdown(*args, **kwargs)+++def _feeds(monkeypatch):+ calls = []+ real = h2m._MarkdownRenderer.feed++ def counted(self, data):+ calls.append(1)+ return real(self, data)++ monkeypatch.setattr(h2m._MarkdownRenderer, "feed", counted)+ return calls++[email protected]("name", sorted(PAGES))[email protected]("span", [None, 50])+def test_replayed_passes_render_what_fed_passes_render(monkeypatch, name, span):+ html = PAGES[name]+ expected = _reference(monkeypatch, html, main_content = True, max_span_chars = span)+ h2m._TITLE_BUTTON_CACHE.clear()+ assert html_to_markdown(html, main_content = True, max_span_chars = span) == expected+ # warm title-button cache path+ assert html_to_markdown(html, main_content = True, max_span_chars = span) == expected++[email protected]("name", sorted(PAGES))+def test_site_links_record_the_same_links(monkeypatch, name):+ html = PAGES[name]+ ref_links = SiteLinks("https://example.com/page")+ with monkeypatch.context() as m:+ m.setattr(h2m, "_TokenLog", _NeverReplay, raising = False)+ m.setattr(h2m.secrets, "token_hex", lambda n: "0" * (2 * n))+ h2m._TITLE_BUTTON_CACHE.clear()+ expected = html_to_markdown(html, main_content = True, site_links = ref_links)+ with monkeypatch.context() as m:+ m.setattr(h2m.secrets, "token_hex", lambda n: "0" * (2 * n))+ links = SiteLinks("https://example.com/page")+ h2m._TITLE_BUTTON_CACHE.clear()+ got = html_to_markdown(html, main_content = True, site_links = links)+ assert got == expected+ assert links.strip(got) == ref_links.strip(expected)+ assert links._found == ref_links._found+++def test_a_multi_pass_page_is_tokenized_once(monkeypatch):+ feeds = _feeds(monkeypatch)+ h2m._TITLE_BUTTON_CACHE.clear()+ html_to_markdown(PAGES["div_only"], main_content = True)+ assert len(feeds) == 1+++def test_a_whole_page_pass_known_to_follow_the_main_scope_replays(monkeypatch):+ # four articles and no <main>: article scope and main scope are fed, the whole-page pass replays+ feeds = _feeds(monkeypatch)+ h2m._TITLE_BUTTON_CACHE.clear()+ html_to_markdown(PAGES["many_articles_no_main"], main_content = True)+ assert len(feeds) == 2+++def test_passes_that_cannot_be_foreseen_are_fed_without_recording(monkeypatch):+ recorded = []+ real = h2m._MarkdownRenderer.feed++ def spy(self, data):+ recorded.append(self._record is not None)+ return real(self, data)++ monkeypatch.setattr(h2m._MarkdownRenderer, "feed", spy)+ for name in ("one_article", "many_articles"):+ h2m._TITLE_BUTTON_CACHE.clear()+ recorded.clear()+ html_to_markdown(PAGES[name], main_content = True)+ assert recorded and not any(recorded), name+++def test_plain_conversion_is_untouched(monkeypatch):+ feeds = _feeds(monkeypatch)+ for name in sorted(PAGES):+ h2m._TITLE_BUTTON_CACHE.clear()+ before = len(feeds)+ html_to_markdown(PAGES[name])+ assert len(feeds) - before in (1, 2) # 2 only for the accordion title rescan+++def test_a_renderer_error_surfaces_from_the_first_pass(monkeypatch):+ real = h2m._MarkdownRenderer.handle_data++ def boom(self, data):+ if "Sentence 3" in data:+ raise RuntimeError("renderer failed")+ return real(self, data)++ monkeypatch.setattr(h2m._MarkdownRenderer, "handle_data", boom)+ h2m._TITLE_BUTTON_CACHE.clear()+ with pytest.raises(RuntimeError, match = "renderer failed"):+ html_to_markdown(PAGES["no_article"], main_content = True)