Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop cutting a fetched page down to one article element (#13178)
Nilay95728c9da3b6
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.pyindex d6f13257aa9..94fd57490dc 100644--- a/studio/backend/core/inference/_html_to_md.py+++ b/studio/backend/core/inference/_html_to_md.py@@ -1386,20 +1386,8 @@ def _select_main_scope_render(tag: str,site_links: SiteLinks | None,span_char_limit: int | None = None,-) -> tuple[int, str]:- """Length and boilerplate-stripped render of the largest single ``<tag>``- subtree. Sizing candidates one at a time stops many tiny sibling cards from- clearing the threshold together, and returning that one subtree keeps- unrelated siblings (related cards, comment threads) out of the output.-- A candidate earns its place on the prose it RETAINED, then gets its dropped- header furniture added back to rank against siblings. Furniture must not buy- eligibility: a card whose header was the only bulk would otherwise clear the- gate on deleted bytes and suppress the ``<main>`` holding the real page.-- Nor may it dominate: the credit is capped at the retained render, so removed- furniture can never be the majority of a score. Uncapped, a teaser with a- 1000 link header outranked a sibling holding five times its real text."""+) -> tuple[int, str, int, int]:+ """return the best eligible score, subtree, count, and visible length; cap header credit at retained prose."""renderer = _new_renderer(source_html,frozenset({tag}),@@ -1421,28 +1409,53 @@ def _select_main_scope_render()best_len = 0best_render = ""+ best_visible = 0for i, seg in enumerate(renderer.scope_segments):rendered = _strip_boilerplate_lines(_cleanup(seg), site_links)scored = _strip_boilerplate_lines(_cleanup(scoring.scope_segments[i]), site_links)if site_links is not None:scored = site_links.clean(scored)- prose = _visible_chars(scored) - scoring.scope_heading_prose[i]+ visible = _visible_chars(scored)+ prose = visible - scoring.scope_heading_prose[i]if prose < _MIN_MAIN_CONTENT_CHARS:continuesize = len(scored) + min(scoring.scope_dropped[i], len(scored))if size > best_len:best_len = sizebest_render = rendered- return best_len, best_render+ best_visible = visible+ return (+ best_len,+ best_render,+ sum(1 for seg in renderer.scope_segments if seg.strip()),+ best_visible,+ )-def _visible_chars(text: str) -> int:- """Visible characters in *text*, ignoring blank lines and link destinations.+def _render_main_document(+ source_html: str, site_links: SiteLinks | None, span_char_limit: int+) -> tuple[int, str]:+ renderer = _new_renderer(source_html, None, True, site_links, span_char_limit)+ rendered = _strip_boilerplate_lines(_cleanup("".join(renderer._out)), site_links)+ scoring = (+ _new_renderer(+ source_html,+ None,+ True,+ span_char_limit = 0,+ header_decisions = renderer.header_decisions,+ )+ if renderer._has_generated_spans+ else renderer+ )+ scored = _strip_boilerplate_lines(_cleanup("".join(scoring._out)), site_links)+ if site_links is not None:+ scored = site_links.clean(scored)+ return _visible_chars(scored), rendered- Headings are NOT discounted here. The renderer already tallies what it marked- as a heading (``_seg_heading_prose``), which sees ``role="heading"``, hgroup- and a linked ``h1``; re-deriving that from ATX syntax could not, and running- both meant two answers to one question."""++def _visible_chars(text: str) -> int:+ """count visible nonblank characters without link targets; callers subtract tracked heading prose."""return sum(_visible_len(line) for line in text.split("\n") if line.strip())@@ -1504,46 +1517,35 @@ def html_to_markdown(site_links: SiteLinks | None = None,max_span_chars: int | None = None,) -> str:- """Convert HTML to Markdown (headings, links, emphasis, lists, tables, blockquotes, code, entities).-- ``<script>``, ``<style>``, and ``<head>`` are stripped entirely, as are- subtrees hidden from rendering (``hidden`` / ``aria-hidden="true"``).-- ``main_content=True`` applies a readability-style heuristic for page- fetches: prefer the ``<article>`` subtree (GitHub renders READMEs there),- then ``<main>``, falling back to the whole document, reduce a link-only- ``<header>`` to the heading it carries, and strip known boilerplate- fragments from the result.-- ``site_links`` records the links back into the page's own site; the output is unchanged.-- ``max_span_chars`` caps the cells generated for ``rowspan``/``colspan``, so a caller with a- smaller result budget keeps room for the text after a table.- """+ """convert HTML to Markdown; main_content prefers a dominant article or main subtree, site_links records same-site links without changing output, and max_span_chars limits generated table cells."""source_html = source_html.replace("\r\n", "\n").replace("\r", "\n")span_limit = 2 * len(source_html)if max_span_chars is not None:span_limit = min(span_limit, max_span_chars)rendered = ""+ full_rendered = ""if main_content:- for scope_tag in ("article", "main"):- # Render only the chosen subtree so sibling <article>/<main> elements do not leak in.- length, rendered = _select_main_scope_render(- source_html, scope_tag, site_links, span_limit- )- if length >= _MIN_MAIN_CONTENT_CHARS:- break- else:- rendered = _strip_boilerplate_lines(- _render(- source_html,- None,- strip_header = True,- site_links = site_links,- span_char_limit = span_limit,- ),- site_links,+ length, rendered, articles, article_visible = _select_main_scope_render(+ source_html, "article", site_links, span_limit+ )+ if length < _MIN_MAIN_CONTENT_CHARS or articles > 1:+ main_length, main_rendered, _, main_visible = _select_main_scope_render(+ source_html, "main", site_links, span_limit)+ if articles > 2 and main_length < _MIN_MAIN_CONTENT_CHARS:+ main_length, full_rendered = _render_main_document(+ source_html, site_links, span_limit+ )+ main_rendered = full_rendered+ main_visible = main_length+ if main_length >= _MIN_MAIN_CONTENT_CHARS and (+ length < _MIN_MAIN_CONTENT_CHARS or main_visible > 2 * article_visible+ ):+ length, rendered = main_length, main_rendered+ if length < _MIN_MAIN_CONTENT_CHARS:+ if not full_rendered:+ _, full_rendered = _render_main_document(source_html, site_links, span_limit)+ rendered = full_renderedelse:rendered = _render(source_html, None, site_links = site_links, span_char_limit = span_limit)return site_links.finish(rendered) if site_links is not None else rendereddiff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.pyindex 0038cab68d9..aa805d6e1ff 100644--- a/studio/backend/tests/test_web_fetch_extraction.py+++ b/studio/backend/tests/test_web_fetch_extraction.py@@ -1546,8 +1546,7 @@ def test_many_tiny_articles_do_not_displace_substantial_main():def test_single_substantial_article_still_preferred_over_main():- # GitHub-README case: one substantial <article> inside <main> must still win- # over sibling <main> furniture.+ # GitHub README pages mix a substantial <article> with repository furniture.article_body = "Real README documentation body text. " * 20html = ("<body><main>"@@ -1560,16 +1559,115 @@ def test_single_substantial_article_still_preferred_over_main():assert "JavaScript 89.3%" not in out-# ── truncated (unclosed) main-content scopes must still be scored ──+def test_one_card_does_not_stand_in_for_a_listing_main():+ cards = "".join(+ f"<article><h2>Plan {i}</h2><p>{f'Plan {i} feature and price detail. ' * 8}</p></article>"+ for i in range(6)+ )+ html = f"<body><main><h1>Pricing</h1>{cards}</main></body>"+ out = html_to_markdown(html, main_content = True)+ for i in range(6):+ assert f"Plan {i} feature and price detail." in out+++def test_article_listing_without_main_uses_the_document():+ cards = "".join(+ f"<article><h2>Plan {i}</h2><p>{f'Plan {i} feature and price detail. ' * 8}</p></article>"+ for i in range(6)+ )+ out = html_to_markdown(f"<body><h1>Pricing</h1>{cards}</body>", main_content = True)+ for i in range(6):+ assert f"Plan {i} feature and price detail." in out+++def test_post_body_outside_article_beats_author_bio_card():+ post = "Main post body paragraph with the actual story. " * 30+ bio = "Author bio describing the writer and their work. " * 6+ related = "".join(+ f"<article class='related'><h3>Related {i}</h3><p>{'Teaser for another post. ' * 9}</p></article>"+ for i in range(3)+ )+ html = (+ "<body><main><h1>Post title</h1>"+ f"<div class='post-content'><p>{post}</p></div>"+ f"<article class='author-card'><p>{bio}</p></article>{related}"+ "</main></body>"+ )+ out = html_to_markdown(html, main_content = True)+ assert "Main post body paragraph" in out+++def test_lone_readme_article_is_kept_over_repo_page_chrome():+ readme = "Short README describing the library. " * 8+ rows = "".join(+ f"<tr><td><a href='/o/r/tree/main/dir{i}'>dir{i}</a></td>"+ f"<td><a href='/o/r/commit/{i}'>Update the dir{i} module and its tests</a></td><td>2 days ago</td></tr>"+ for i in range(25)+ )+ html = (+ "<body><main><h2>Repository files navigation</h2>"+ f"<table><tr><th>Name</th><th>Last commit message</th><th>Last commit date</th></tr>{rows}</table>"+ f"<article class='markdown-body'><h1>Lib</h1><p>{readme}</p></article>"+ "<div><h2>About</h2><p>A small library for doing one thing well.</p></div>"+ "</main></body>"+ )+ out = html_to_markdown(html, main_content = True)+ assert "Short README describing the library." in out+ assert "Last commit message" not in out+++def test_link_heavy_comments_do_not_pull_main_over_the_post():+ post = "The post explains the topic in full detail here. " * 50+ comments = "".join(+ f"<article class='comment'><p><a href='https://example.com/author/{i}?{'utm_source=comments&' * 20}'>Reader {i}</a> "+ f"says: {'Thanks for writing this up. ' * 9}</p></article>"+ for i in range(6)+ )+ html = (+ f"<body><main><article class='post'><h1>Post</h1><p>{post}</p></article>"+ f"<section id='comments'>{comments}</section></main></body>"+ )+ out = html_to_markdown(html, main_content = True)+ assert "The post explains the topic" in out+ assert "Thanks for writing this up." not in out+++def test_generated_table_spans_do_not_pull_main_over_the_post():+ post = "Primary article prose with real details. " * 30+ related = "Related card teaser. " * 12+ rows = "".join(f"<tr><td>row {i}</td></tr>" for i in range(1, 80))+ table = f"<table><tr><td rowspan='80'>repeated marker</td><td>row 0</td></tr>{rows}</table>"+ html = (+ "<body><main>"+ f"<article><h1>Primary</h1><p>{post}</p></article>"+ f"<article><p>{related}</p></article>{table}"+ "</main></body>"+ )+ out = html_to_markdown(html, main_content = True)+ assert "Primary article prose" in out+ assert "Related card teaser." not in out++[email protected]("hide", ["hidden", "aria-hidden='true'", "style='display:none'"])+def test_hidden_duplicate_article_is_not_a_second_card(hide):+ post = "The post explains the topic in full detail here. " * 20+ comments = "".join(+ f"<div class='comment'><p>Reader {i} says: {'Thanks for writing this up, it helped. ' * 6}</p></div>"+ for i in range(8)+ )+ html = (+ f"<body><main><article><h1>Post</h1><p>{post}</p></article>"+ f"<article {hide}><p>Duplicate</p></article><section>{comments}</section></main></body>"+ )+ out = html_to_markdown(html, main_content = True)+ assert "The post explains the topic" in out+ assert "Thanks for writing this up" not in outdef test_truncated_open_article_scope_is_scored_and_preferred():- # _fetch_url_raw caps large pages, so the download can end before the closing- # </article>. The scope is still the main content and must be preferred over the- # whole document (which re-leaks the page chrome).+ # _fetch_url_raw may truncate before </article>, so the open scope must exclude page chrome.chrome = "<nav>Skip to content</nav><div>Repository file tree and page chrome.</div>"article_body = "Real README documentation body text. " * 20- # No closing </article> / </body> -- the fetch cap truncated the page.html = f"<body>{chrome}<article><h1>Guide</h1><p>{article_body}</p>"out = html_to_markdown(html, main_content = True)assert "Real README documentation body text." in out