Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop link URLs from crowding out long page text (#13102)
Nilayac6be24ad597
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.pyindex 2fd8f2808d3..33ad49f01e1 100644--- a/studio/backend/core/inference/_html_to_md.py+++ b/studio/backend/core/inference/_html_to_md.py@@ -18,9 +18,11 @@ from __future__ import annotationsimport htmlimport re+import secretsfrom html.parser import HTMLParser+from urllib.parse import urlsplit-__all__ = ["html_to_markdown"]+__all__ = ["SiteLinks", "html_to_markdown"]_SKIP_TAGS = frozenset({@@ -275,6 +277,68 @@ class _HeaderFrame:return "".join(self.parts)+class SiteLinks:+ """Same-site links as rendered, so a page too long for its budget can drop their URLs."""++ def __init__(self, page_url: str):+ self._host = urlsplit(page_url).hostname+ self._nonce = secrets.token_hex(8)+ self._marker = re.compile(rf"\x00{self._nonce}:(\d+):([se])\x00")+ self._found: dict[int, tuple[str, str]] = {}+ self._full = ""+ self._stripped = ""++ def note(self, link: str, text: str, href: str) -> str:+ try:+ parts = urlsplit(href)+ except ValueError:+ return link+ if parts.scheme not in ("", "http", "https") or parts.hostname not in (+ None,+ self._host,+ ):+ return link+ index = len(self._found)+ self._found[index] = (link, text)+ return f"\x00{self._nonce}:{index}:s\x00{link}\x00{self._nonce}:{index}:e\x00"++ def clean(self, markdown: str) -> str:+ return self._marker.sub("", markdown)++ def finish(self, markdown: str) -> str:+ self._full = self.clean(markdown)+ self._stripped = self._strip_marked(markdown)+ return self._full++ def strip(self, markdown: str) -> str:+ return self._stripped if markdown == self._full else markdown++ def _strip_marked(self, markdown: str) -> str:+ out: list[str] = []+ last = 0+ while match := self._marker.search(markdown, last):+ index, edge = int(match.group(1)), match.group(2)+ if edge != "s" or index not in self._found:+ out.append(markdown[last : match.end()])+ last = match.end()+ continue+ end_marker = f"\x00{self._nonce}:{index}:e\x00"+ end = markdown.find(end_marker, match.end())+ if end < 0:+ out.append(markdown[last : match.end()])+ last = match.end()+ continue+ link, text = self._found[index]+ rendered_link = markdown[match.end() : end]+ if rendered_link == link.replace("|", "\\|"):+ text = text.replace("|", "\\|")+ out.append(markdown[last : match.start()])+ out.append(text)+ last = end + len(end_marker)+ out.append(markdown[last:])+ return self.clean("".join(out))++class _MarkdownRenderer(HTMLParser):"""HTMLParser subclass that emits Markdown tokens into a list.@@ -287,8 +351,10 @@ class _MarkdownRenderer(HTMLParser):self,scope_tags: frozenset[str] | None = None,strip_header: bool = False,+ site_links: SiteLinks | None = None,):super().__init__(convert_charrefs = False)+ self._site_links = site_linksself._out: list[str] = []self._skip_depth: int = 0@@ -381,9 +447,10 @@ class _MarkdownRenderer(HTMLParser):as_heading = (self._heading_marks and not in_nested_link and not self._replaying) or self._emit_as_heading+ measured = self._site_links.clean(text) if self._site_links is not None else textif frame is not None and as_heading:frame.heading_parts.append(text)- frame.heading_chars += len(text.strip())+ frame.heading_chars += len(measured.strip())# for the eligibility gate, frame or not; link text waits for _finish_link to count onceif not self._replaying and ((self._heading_marks and not self._in_link) or self._emit_as_heading@@ -392,7 +459,7 @@ class _MarkdownRenderer(HTMLParser):nested_open = self._nested_buffer_open(frame) if frame is not None else False# Tally once, on the emit reaching the frame; counting again on flush doubled it.if frame is not None and not nested_open:- frame.rendered_chars += len(text.strip())+ frame.rendered_chars += len(measured.strip())frame.parts.append(text)returnif self._in_link:@@ -411,7 +478,10 @@ class _MarkdownRenderer(HTMLParser):def _seg_heading_prose(self) -> int:"""Heading characters in this segment that the gate would otherwise read asbody prose. ATX headings carry their own ``#`` here and so score zero."""- return _visible_chars("".join(self._seg_heading_texts))+ text = "".join(self._seg_heading_texts)+ if self._site_links is not None:+ text = self._site_links.clean(text)+ return _visible_chars(text)def _drain_pre(self) -> None:"""Emit the open ``<pre>`` and empty it, so a late ``</pre>`` cannot replay@@ -490,7 +560,10 @@ class _MarkdownRenderer(HTMLParser):self._link_had_heading = Falseself._link_header_chars = 0if href and text:- self._emit(f"[{text}]({href})")+ link = f"[{text}]({href})"+ if self._site_links is not None and not self._inline_code_depth:+ link = self._site_links.note(link, text, href)+ self._emit(link)elif text:self._emit(text)self._emit_as_heading = False@@ -1031,7 +1104,7 @@ def _fence_state(line: str, fence: int) -> int:return 0 if len(stripped) >= fence else fence-def _strip_boilerplate_lines(text: str) -> str:+def _strip_boilerplate_lines(text: str, site_links: SiteLinks | None = None) -> str:"""Drop short lines that consist entirely of known page-furniture phrases.Fenced code blocks are preserved verbatim: boilerplate never renders@@ -1044,7 +1117,12 @@ def _strip_boilerplate_lines(text: str) -> str:fence = movedout.append(line)continue- if not fence and len(line) <= _BOILERPLATE_MAX_LINE_CHARS and _line_is_boilerplate(line):+ measured = site_links.clean(line) if site_links is not None else line+ if (+ not fence+ and len(measured) <= _BOILERPLATE_MAX_LINE_CHARS+ and _line_is_boilerplate(measured)+ ):continueout.append(line)# Collapse blank runs the dropped lines may have left behind.@@ -1052,9 +1130,14 @@ def _strip_boilerplate_lines(text: str) -> str:def _new_renderer(- source_html: str, scope_tags: frozenset[str] | None, strip_header: bool+ source_html: str,+ scope_tags: frozenset[str] | None,+ strip_header: bool,+ site_links: SiteLinks | None,) -> _MarkdownRenderer:- renderer = _MarkdownRenderer(scope_tags = scope_tags, strip_header = strip_header)+ renderer = _MarkdownRenderer(+ scope_tags = scope_tags, strip_header = strip_header, site_links = site_links+ )renderer.feed(source_html)renderer.close()renderer.flush_pending()@@ -1065,11 +1148,14 @@ def _render(source_html: str,scope_tags: frozenset[str] | None,strip_header: bool = False,+ site_links: SiteLinks | None = None,) -> str:- return _cleanup("".join(_new_renderer(source_html, scope_tags, strip_header)._out))+ return _cleanup("".join(_new_renderer(source_html, scope_tags, strip_header, site_links)._out))-def _select_main_scope_render(source_html: str, tag: str) -> tuple[int, str]:+def _select_main_scope_render(+ source_html: str, tag: str, site_links: SiteLinks | None+) -> tuple[int, str]:"""Length and boilerplate-stripped render of the largest single ``<tag>``subtree. Sizing candidates one at a time stops many tiny sibling cards fromclearing the threshold together, and returning that one subtree keeps@@ -1083,17 +1169,20 @@ def _select_main_scope_render(source_html: str, tag: str) -> tuple[int, str]:Nor may it dominate: the credit is capped at the retained render, so removedfurniture can never be the majority of a score. Uncapped, a teaser with a1000 link header outranked a sibling holding five times its real text."""- renderer = _new_renderer(source_html, frozenset({tag}), strip_header = True)+ renderer = _new_renderer(+ source_html, frozenset({tag}), strip_header = True, site_links = site_links+ )dropped = renderer.scope_droppedheading_prose = renderer.scope_heading_prosebest_len = 0best_render = ""for i, seg in enumerate(renderer.scope_segments):- rendered = _strip_boilerplate_lines(_cleanup(seg))- prose = _visible_chars(rendered) - heading_prose[i]+ rendered = _strip_boilerplate_lines(_cleanup(seg), site_links)+ measured = site_links.clean(rendered) if site_links is not None else rendered+ prose = _visible_chars(measured) - heading_prose[i]if prose < _MIN_MAIN_CONTENT_CHARS:continue- size = len(rendered) + min(dropped[i], len(rendered))+ size = len(measured) + min(dropped[i], len(measured))if size > best_len:best_len = sizebest_render = rendered@@ -1161,7 +1250,12 @@ def _visible_len(line: str) -> int:_MIN_MAIN_CONTENT_CHARS = 200-def html_to_markdown(source_html: str, *, main_content: bool = False) -> str:+def html_to_markdown(+ source_html: str,+ *,+ main_content: bool = False,+ site_links: SiteLinks | None = None,+) -> str:"""Convert HTML to Markdown (headings, links, emphasis, lists, tables, blockquotes, code, entities).``<script>``, ``<style>``, and ``<head>`` are stripped entirely, as are@@ -1172,13 +1266,22 @@ def html_to_markdown(source_html: str, *, main_content: bool = False) -> str:then ``<main>``, falling back to the whole document, reduce a link-only``<header>`` to the heading it carries, and strip known boilerplatefragments from the result.++ ``site_links`` records the links back into the page's own site; the output is unchanged."""source_html = source_html.replace("\r\n", "\n").replace("\r", "\n")+ rendered = ""if main_content:for scope_tag in ("article", "main"):# Render only the chosen subtree so sibling <article>/<main> elements do not leak in.- length, rendered = _select_main_scope_render(source_html, scope_tag)+ length, rendered = _select_main_scope_render(source_html, scope_tag, site_links)if length >= _MIN_MAIN_CONTENT_CHARS:- return rendered- return _strip_boilerplate_lines(_render(source_html, None, strip_header = True))- return _render(source_html, None)+ break+ else:+ rendered = _strip_boilerplate_lines(+ _render(source_html, None, strip_header = True, site_links = site_links),+ site_links,+ )+ else:+ rendered = _render(source_html, None, site_links = site_links)+ return site_links.finish(rendered) if site_links is not None else rendereddiff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex f64e36ddd54..df60ba2026e 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -16914,9 +16914,14 @@ def _fetch_page_text(return _truncate_page_text(body.strip(), max_chars)# Convert HTML to Markdown with the builtin converter (no external deps).- from ._html_to_md import html_to_markdown+ from ._html_to_md import SiteLinks, html_to_markdown- return _truncate_page_text(html_to_markdown(body, main_content = True), max_chars)+ site_links = SiteLinks(url)+ text = html_to_markdown(body, main_content = True, site_links = site_links)+ # a page that fits keeps same-site links so the model can follow them.+ if text and len(text) <= max_chars and len(text) <= _dense_char_limit(text, max_chars):+ return text+ return _truncate_page_text(site_links.strip(text), max_chars)def _search_failure_message(exc: BaseException, timeout: int) -> str:diff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.pyindex 805bb6cf2a0..9b4680927e5 100644--- a/studio/backend/tests/test_web_fetch_extraction.py+++ b/studio/backend/tests/test_web_fetch_extraction.py@@ -692,6 +692,121 @@ def test_fetch_page_text_propagates_fetch_errors(monkeypatch):)+_WIKI_URL = "https://en.wikipedia.org/wiki/Python_(programming_language)"+++def _wiki_article(paragraphs):+ body = "".join(+ f'<p>Paragraph {i} links <a href="https://en.wikipedia.org/wiki/Some_Long_Article_Title_{i}">'+ f'topic {i}</a>, <a href="/wiki/Another_Related_Page_{i}">a related page</a> and a note'+ f'<sup><a href="#cite_note-{i}">[{i}]</a></sup>.</p>'+ for i in range(paragraphs)+ )+ return (+ f"<html><body><main><article>{body}"+ "<p>Python 3.0 was released on 3 December 2008.</p>"+ '<p>See <a href="https://www.python.org/doc/">the docs</a>.</p>'+ "</article></main></body></html>"+ )+++def test_long_page_spends_its_budget_on_text_not_links_into_the_site(monkeypatch):+ out = _page_text(monkeypatch, _WIKI_URL, _wiki_article(150), "text/html")+ assert "Python 3.0 was released on 3 December 2008." in out+ assert "topic 149, a related page and a note[149]." in out+ assert "[the docs](https://www.python.org/doc/)" in out+ assert "wikipedia.org" not in out+ assert "/wiki/" not in out+ assert "#cite_note" not in out+++def test_page_that_fits_keeps_its_links(monkeypatch):+ out = _page_text(monkeypatch, _WIKI_URL, _wiki_article(3), "text/html")+ assert "[topic 1](https://en.wikipedia.org/wiki/Some_Long_Article_Title_1)" in out+ assert "[a related page](/wiki/Another_Related_Page_1)" in out+ assert "[[1]](#cite_note-1)" in out+++def test_page_cut_by_the_room_left_drops_its_site_link_urls(monkeypatch):+ from core.inference import tools++ context = tools._REQUEST_CONTEXT_TOKENS.set(32768)+ room = tools._REQUEST_RESULT_BUDGET.set(1200)+ try:+ out = _page_text(monkeypatch, _WIKI_URL, _wiki_article(60), "text/html")+ finally:+ tools._REQUEST_RESULT_BUDGET.reset(room)+ tools._REQUEST_CONTEXT_TOKENS.reset(context)+ assert "(truncated," in out+ assert "wikipedia.org" not in out+++def test_dropping_site_link_urls_leaves_code_samples_alone(monkeypatch):+ filler = "".join(+ f"<p>Guide paragraph {i} with enough words to run long.</p>" for i in range(400)+ )+ page = (+ '<html><body><main><p>See <a href="#install">Install</a> or <a href="#usage">Usage</a>.</p>'+ "<pre>[Install](#install)</pre><p>Inline <code>[Usage](#usage)</code>.</p>"+ f"{filler}</main></body></html>"+ )+ out = _page_text(monkeypatch, "https://docs.example.com/guide", page, "text/html")+ assert "```\n[Install](#install)\n```" in out+ assert "Inline `[Usage](#usage)`." in out+++def test_dropping_site_link_urls_only_rewrites_links_emitted_by_renderer(monkeypatch):+ filler = "".join(+ f"<p>Guide paragraph {i} with enough words to run long.</p>" for i in range(400)+ )+ page = (+ "<html><body><main>"+ "<p>Literal Markdown: [Install](#install).</p>"+ '<p>Inline code: <code><a href="#install">Install</a></code>.</p>'+ '<p>Actual link: <a href="#install">Install</a>.</p>'+ f"{filler}</main></body></html>"+ )+ out = _page_text(monkeypatch, "https://docs.example.com/guide", page, "text/html")+ assert out.startswith(+ "Literal Markdown: [Install](#install).\n\n"+ "Inline code: `[Install](#install)`.\n\n"+ "Actual link: Install."+ )+++def test_dropping_site_link_urls_keeps_the_same_article(monkeypatch):+ linked = "".join(+ f'<p>Story one fact {i} see <a href="/topics/a-very-long-topic-slug-number-{i}">topic {i}</a>.</p>'+ for i in range(300)+ )+ plain = "".join(+ f"<p>Story two paragraph {i} with plain prose and no links.</p>" for i in range(300)+ )+ page = f"<html><body><article><h1>STORY ONE</h1>{linked}</article><article><h1>STORY TWO</h1>{plain}</article></body></html>"+ out = _page_text(monkeypatch, "https://news.example.com/story-one", page, "text/html")+ assert out.startswith("# STORY ONE\n\nStory one fact 0 see topic 0.")+ assert "STORY TWO" not in out+++def test_dropping_site_link_urls_keeps_a_link_header_as_furniture(monkeypatch):+ labels = "machine learning,deep learning,pytorch,cuda,transformers,lora,gguf,llama,quantization,finetuning,inference,qlora"+ tags = "".join(+ f'<a href="/category/topics/{t}/archive/page/1/?ref=card-header">{t}</a> '+ for t in labels.split(",")+ )+ prose = "".join(+ f"<p>Main page prose the user asked about, paragraph {i}, with enough words.</p>"+ for i in range(250)+ )+ page = (+ f"<html><body><main><h1>Main Post</h1>{prose}<aside><article><header>{tags}</header>"+ "<p>A short teaser for a related post that is just long enough to read like a real sentence or two here.</p>"+ "</article></aside></main></body></html>"+ )+ out = _page_text(monkeypatch, "https://blog.example.com/main-post", page, "text/html")+ assert out.startswith("# Main Post\n\nMain page prose the user asked about, paragraph 0")++def test_looks_like_html():assert _looks_like_html("<!DOCTYPE html><html></html>")assert _looks_like_html("\n <HTML lang='en'>")