Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep heading text that sits inside a button on fetched pages (#13184)
Nilayaea22dff8b9a
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.pyindex 94fd57490dc..3c7b5c23a98 100644--- a/studio/backend/core/inference/_html_to_md.py+++ b/studio/backend/core/inference/_html_to_md.py@@ -399,6 +399,106 @@ class _TableFrame:self.parts: list[str] = []+class _TitleButtonScan:+ """Find buttons that hold their heading's whole title (``<h3><button>Question</button></h3>``, an accordion+ trigger): the heading's first visible button with text, with no visible heading text outside it. Fed the+ renderer's own tokens, so it parses exactly as the renderer does; the kept buttons render on a second pass."""++ def __init__(self) -> None:+ self.keep: set[tuple[int, int]] = set() # HTMLParser.getpos() of each kept <button>+ self._open: list[str] = []+ self._closable_open = (+ 0 # open tags with an optional end tag, as _MarkdownRenderer counts them+ )+ self._muted: list[int] = [] # open-tag indices of hidden / skipped subtrees+ self._button_at: int | None = None+ self._button_frame: list | None = None # heading whose candidate is the open button+ # visible text outside any button, counted so a heading reads "text since I opened" in O(1)+ self._text_seq = 0+ # per open heading: [open-tag index, candidate position or None, candidate has text, _text_seq at open]+ self._headings: list[list] = []++ def starttag(self, tag: str, attrs: list[tuple[str, str | None]], pos: tuple[int, int]) -> None:+ self._close_implicit(tag)+ if tag in _VOID_TAGS:+ return+ attr_dict = dict(attrs)+ self._open.append(tag)+ if tag in _IMPLICIT_CLOSERS:+ self._closable_open += 1+ index = len(self._open) - 1+ if _is_hidden_element(attr_dict) or (tag in _SKIP_TAGS and tag != "button"):+ self._muted.append(index)+ if tag in _HEADING_TAGS or _is_aria_heading(attr_dict):+ self._headings.append([index, None, False, self._text_seq])+ elif tag == "button" and self._button_at is None:+ self._button_at = index+ frame = self._headings[-1] if self._headings else None+ if frame is not None and frame[1] is None and not self._muted:+ frame[1] = pos+ self._button_frame = frame++ def endtag(self, tag: str) -> None:+ for i in range(len(self._open) - 1, -1, -1):+ if self._open[i] == tag:+ self._pop_to(i)+ return++ def data(self, data: str) -> None:+ if self._muted or not data.strip():+ return+ if self._button_at is None:+ self._text_seq += 1+ elif self._button_frame is not None:+ self._button_frame[2] = True++ def finish(self) -> None:+ # a page cut inside a heading (the fetch cap) still keeps its title+ self._close_headings(0)++ def _close_implicit(self, tag: str) -> None:+ """The renderer's optional-end-tag recovery (``_MarkdownRenderer._close_implicit``), so both see one tree."""+ if not self._closable_open:+ return+ barriers = _CLOSE_BARRIERS.get(tag, ())+ while True:+ close_at = None+ for i in range(len(self._open) - 1, -1, -1):+ name = self._open[i]+ if tag in _IMPLICIT_CLOSERS.get(name, ()):+ close_at = i+ break+ if name in barriers:+ break+ if close_at is None:+ return+ self._pop_to(close_at)++ def _pop_to(self, i: int) -> None:+ if self._button_at is not None and self._button_at >= i:+ self._button_at = None+ # an icon-only control releases the slot for the trigger after it+ if self._button_frame is not None and not self._button_frame[2]:+ self._button_frame[1] = None+ self._button_frame = None+ while self._muted and self._muted[-1] >= i:+ self._muted.pop()+ self._close_headings(i)+ self._closable_open -= sum(1 for name in self._open[i:] if name in _IMPLICIT_CLOSERS)+ del self._open[i:]++ def _close_headings(self, i: int) -> None:+ while self._headings and self._headings[-1][0] >= i:+ _, button, has_text, seq = self._headings.pop()+ if button is not None and has_text and self._text_seq == seq:+ self.keep.add(button)+++# (len, hash) of recent pages -> their title buttons, so main-content passes over one page scan it once+_TITLE_BUTTON_CACHE: dict[tuple[int, int], frozenset[tuple[int, int]]] = {}+_TITLE_BUTTON_CACHE_SIZE = 4++class _MarkdownRenderer(HTMLParser):"""HTMLParser subclass that emits Markdown tokens into a list.@@ -417,6 +517,9 @@ class _MarkdownRenderer(HTMLParser):page_span_limit: int | None = None,):super().__init__(convert_charrefs = False)+ # getpos() of buttons that carry their heading's title, found by _TitleButtonScan+ self.title_buttons: frozenset[tuple[int, int]] = frozenset()+ self._title_scan: _TitleButtonScan | None = Noneself._site_links = site_linksself._out: list[str] = []self._skip_depth: int = 0@@ -972,6 +1075,11 @@ class _MarkdownRenderer(HTMLParser):def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:tag = tag.lower()+ if self._title_scan is not None:+ self._title_scan.starttag(tag, attrs, self.getpos())+ title_button = (+ tag == "button" and bool(self.title_buttons) and self.getpos() in self.title_buttons+ )if self._skip_depth:if tag in _SKIP_TAGS:@@ -982,7 +1090,7 @@ class _MarkdownRenderer(HTMLParser):# <p>, releasing its hidden mark so following siblings render.self._close_implicit(tag)- if tag in _SKIP_TAGS:+ if tag in _SKIP_TAGS and not (title_button and self._heading_marks):self._skip_depth += 1return@@ -1083,8 +1191,11 @@ class _MarkdownRenderer(HTMLParser):def handle_endtag(self, tag: str) -> None:tag = tag.lower()+ if self._title_scan is not None:+ self._title_scan.endtag(tag)- if tag in _SKIP_TAGS:+ # a kept title button closes through _exit_tag; skipped ones always raised _skip_depth+ if tag in _SKIP_TAGS and (self._skip_depth or tag != "button"):self._skip_depth = max(0, self._skip_depth - 1)returnif self._skip_depth:@@ -1155,6 +1266,8 @@ class _MarkdownRenderer(HTMLParser):return self._scope_tags is not None and self._scope_depth == 0def handle_data(self, data: str) -> None:+ if self._title_scan is not None:+ self._title_scan.data(data)if self._text_suppressed():returnif self._in_pre:@@ -1180,16 +1293,20 @@ class _MarkdownRenderer(HTMLParser):self._emit(text)def handle_entityref(self, name: str) -> None:+ text = html.unescape(f"&{name};")+ if self._title_scan is not None:+ self._title_scan.data(text)if self._text_suppressed():return- text = html.unescape(f"&{name};")self._count_header_text(text)self._emit(text)def handle_charref(self, name: str) -> None:+ text = html.unescape(f"&#{name};")+ if self._title_scan is not None:+ self._title_scan.data(text)if self._text_suppressed():return- text = html.unescape(f"&#{name};")self._count_header_text(text)self._emit(text)@@ -1356,18 +1473,37 @@ def _new_renderer(span_char_limit: int | None = None,header_decisions: list[bool] | None = None,) -> _MarkdownRenderer:- renderer = _MarkdownRenderer(- scope_tags = scope_tags,- strip_header = strip_header,- site_links = site_links,- span_char_limit = 2 * len(source_html) if span_char_limit is None else span_char_limit,- header_decisions = header_decisions,- page_span_limit = 2 * len(source_html),- )- renderer.feed(source_html)- renderer.close()- renderer.flush_pending()- return renderer+ def build(+ title_buttons: frozenset[tuple[int, int]], scan: _TitleButtonScan | None+ ) -> _MarkdownRenderer:+ renderer = _MarkdownRenderer(+ scope_tags = scope_tags,+ strip_header = strip_header,+ site_links = site_links,+ span_char_limit = 2 * len(source_html) if span_char_limit is None else span_char_limit,+ header_decisions = header_decisions,+ page_span_limit = 2 * len(source_html),+ )+ renderer.title_buttons = title_buttons+ renderer._title_scan = scan+ renderer.feed(source_html)+ renderer.close()+ renderer.flush_pending()+ return renderer++ key = (len(source_html), hash(source_html))+ known = _TITLE_BUTTON_CACHE.get(key)+ if known is not None:+ return build(known, None)+ scan = _TitleButtonScan()+ renderer = build(frozenset(), scan)+ scan.finish()+ keep = frozenset(scan.keep)+ if len(_TITLE_BUTTON_CACHE) >= _TITLE_BUTTON_CACHE_SIZE:+ _TITLE_BUTTON_CACHE.clear() # safe under concurrent fetches, unlike evicting one by one+ _TITLE_BUTTON_CACHE[key] = keep+ # only a page that has accordion titles pays a second pass+ return build(keep, None) if keep else rendererdef _render(diff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.pyindex aa805d6e1ff..dcea7059972 100644--- a/studio/backend/tests/test_web_fetch_extraction.py+++ b/studio/backend/tests/test_web_fetch_extraction.py@@ -338,6 +338,148 @@ def test_skipped_tag_implicitly_closes_hidden_paragraph():assert "VISIBLE" in out+ "html, heading",+ [+ (+ '<h3 data-state="closed"><button type="button" aria-controls="r1" aria-expanded="false">'+ 'What is the right plan for me?<span aria-hidden="true">v</span></button></h3>'+ '<div id="r1" role="region"><p>Answer text.</p></div>',+ "### What is the right plan for me?",+ ),+ (+ '<h2 class="accordion-header">\n <button class="accordion-button" type="button" '+ 'aria-expanded="true" aria-controls="c1">\n What is the right plan for me?\n </button>\n</h2>'+ '<div id="c1"><div class="accordion-body">Answer text.</div></div>',+ "What is the right plan for me?",+ ),+ (+ "<h3><button>What is the right plan for me?</button></h3><p>Answer text.</p>",+ "### What is the right plan for me?",+ ),+ (+ '<div role="heading" aria-level="3"><button aria-expanded="false">'+ "What is the right plan for me?</button></div><p>Answer text.</p>",+ "What is the right plan for me?",+ ),+ ],+)+def test_accordion_question_in_a_heading_button_is_kept(html, heading):+ out = html_to_markdown(html)+ assert heading in out+ assert "Answer text." in out+++def test_buttons_that_are_not_a_heading_title_are_still_dropped():+ html = (+ "<h4><span>Create Artifacts</span><button aria-expanded='false'>"+ "<span class='sr-only'>More information</span></button></h4>"+ "<p>Body text.</p><button>Subscribe</button>"+ )+ out = html_to_markdown(html)+ assert "#### Create Artifacts" in out+ assert "Body text." in out+ assert "More information" not in out+ assert "Subscribe" not in out+++ "html",+ [+ "<h4>Create<button>More information</button></h4><p>Body text.</p>",+ "<h4>é<button>More information</button></h4><p>Body text.</p>",+ "<hgroup><h1>Create</h1><h2></h2><button>More information</button></hgroup><p>Body text.</p>",+ ],+)+def test_a_button_after_entity_or_nested_heading_text_is_dropped(html):+ out = html_to_markdown(html)+ assert "Body text." in out+ assert "More information" not in out+++ "between", ["", "<!-- </h4> -->", "<span hidden>" + "x" * 5000 + "</span>"]+)+def test_a_leading_control_before_the_heading_title_is_dropped(between):+ html = (+ f"<h4><button aria-expanded='false'>More information</button>{between}<span>Create Artifacts</span></h4>"+ "<p>Body text.</p>"+ )+ out = html_to_markdown(html)+ assert "#### Create Artifacts" in out+ assert "More information" not in out+++ "html, heading",+ [+ (+ "<hgroup><h1><button>Question</button></h1><p>Sub</p></hgroup><p>Answer text.</p>",+ "# Question",+ ),+ (+ "<h3><button hidden>old</button><button>Question</button></h3><p>Answer text.</p>",+ "### Question",+ ),+ (+ "<h3><span hidden><button>old</button></span><button>Question</button></h3><p>Answer text.</p>",+ "### Question",+ ),+ ("<h3><button>Question</button><br></h3><p>Answer text.</p>", "### Question"),+ ("<h3><em><button>Question</button></em></h3><p>Answer text.</p>", "### *Question*"),+ (+ "<h3><button aria-label='Settings'><svg><path/></svg></button><button>Question</button></h3>"+ "<p>Answer text.</p>",+ "### Question",+ ),+ ("<ul><li><h3><button>Question</button><li>Answer text.</ul>", "### Question"),+ (+ "<ul><li hidden>old<li><h3><button>Question</button></h3>Answer text.</ul>",+ "### Question",+ ),+ ("<p hidden>old<h3><button>Question</button></h3><p>Answer text.</p>", "### Question"),+ (+ '<script>const x="<h3><button>"</script><!-- <h2><button> -->'+ "<h3><button>Question</button></h3><p>Answer text.</p>",+ "### Question",+ ),+ ("<h3><!-- </h3> --><button>Question</button></h3><p>Answer text.</p>", "### Question"),+ (+ "<title-card>Card</title-card><h3><button>Question</button></h3><p>Answer text.</p>",+ "### Question",+ ),+ (+ '<div data-example="<script>"></div><h3 data-template="</h3>"><button>Question</button></h3>'+ "<p>Answer text.</p>",+ "### Question",+ ),+ ("<p>a\nb</p>\n<h3>\n <button>Question</button></h3><p>Answer text.</p>", "Question"),+ (+ '<div role="presentation heading" aria-level="3"><button>Question</button></div><p>Answer text.</p>',+ "Question",+ ),+ ],+)+def test_heading_button_title_edge_cases(html, heading):+ out = html_to_markdown(html)+ assert heading in out+ assert "Answer text." in out+ assert "old" not in out+++ "cut", ["<h3><button>Question", "<h3><button>Question</button>", "<h3><button>Question<svg>"]+)+def test_heading_button_cut_by_the_fetch_cap_keeps_its_title(cut):+ assert "### Question" in html_to_markdown("<p>Answer text.</p>" + cut)++[email protected]("title", ["<em>Question</em> one", "First<br>Second", "Q & A"])+def test_a_heading_button_title_renders_like_the_plain_heading(title):+ plain = html_to_markdown(f"<h3>{title}</h3><p>Answer text.</p>")+ assert html_to_markdown(f"<h3><button>{title}</button></h3><p>Answer text.</p>") == plain++def test_visible_void_hr_still_renders():# Guard: the suppression must not affect non-hidden void elements.html = "<body><p>a</p><hr><p>b</p></body>"