Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop cutting a fetched page down to one article element (#13178)

Nilay95728c9da3b6
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.py
index d6f13257aa9..94fd57490dc 100644
--- a/studio/backend/core/inference/_html_to_md.py
+++ b/studio/backend/core/inference/_html_to_md.py
@@ -1386,20 +1386,8 @@ def _select_main_scope_render(
tag: str,
site_links: SiteLinks | None,
span_char_limit: int | None = None,
-) -> tuple[int, str]:
- """Length and boilerplate-stripped render of the largest single ``<tag>``
- subtree. Sizing candidates one at a time stops many tiny sibling cards from
- clearing the threshold together, and returning that one subtree keeps
- unrelated siblings (related cards, comment threads) out of the output.
-
- A candidate earns its place on the prose it RETAINED, then gets its dropped
- header furniture added back to rank against siblings. Furniture must not buy
- eligibility: a card whose header was the only bulk would otherwise clear the
- gate on deleted bytes and suppress the ``<main>`` holding the real page.
-
- Nor may it dominate: the credit is capped at the retained render, so removed
- furniture can never be the majority of a score. Uncapped, a teaser with a
- 1000 link header outranked a sibling holding five times its real text."""
+) -> tuple[int, str, int, int]:
+ """return the best eligible score, subtree, count, and visible length; cap header credit at retained prose."""
renderer = _new_renderer(
source_html,
frozenset({tag}),
@@ -1421,28 +1409,53 @@ def _select_main_scope_render(
)
best_len = 0
best_render = ""
+ best_visible = 0
for i, seg in enumerate(renderer.scope_segments):
rendered = _strip_boilerplate_lines(_cleanup(seg), site_links)
scored = _strip_boilerplate_lines(_cleanup(scoring.scope_segments[i]), site_links)
if site_links is not None:
scored = site_links.clean(scored)
- prose = _visible_chars(scored) - scoring.scope_heading_prose[i]
+ visible = _visible_chars(scored)
+ prose = visible - scoring.scope_heading_prose[i]
if prose < _MIN_MAIN_CONTENT_CHARS:
continue
size = len(scored) + min(scoring.scope_dropped[i], len(scored))
if size > best_len:
best_len = size
best_render = rendered
- return best_len, best_render
+ best_visible = visible
+ return (
+ best_len,
+ best_render,
+ sum(1 for seg in renderer.scope_segments if seg.strip()),
+ best_visible,
+ )
-def _visible_chars(text: str) -> int:
- """Visible characters in *text*, ignoring blank lines and link destinations.
+def _render_main_document(
+ source_html: str, site_links: SiteLinks | None, span_char_limit: int
+) -> tuple[int, str]:
+ renderer = _new_renderer(source_html, None, True, site_links, span_char_limit)
+ rendered = _strip_boilerplate_lines(_cleanup("".join(renderer._out)), site_links)
+ scoring = (
+ _new_renderer(
+ source_html,
+ None,
+ True,
+ span_char_limit = 0,
+ header_decisions = renderer.header_decisions,
+ )
+ if renderer._has_generated_spans
+ else renderer
+ )
+ scored = _strip_boilerplate_lines(_cleanup("".join(scoring._out)), site_links)
+ if site_links is not None:
+ scored = site_links.clean(scored)
+ return _visible_chars(scored), rendered
- Headings are NOT discounted here. The renderer already tallies what it marked
- as a heading (``_seg_heading_prose``), which sees ``role="heading"``, hgroup
- and a linked ``h1``; re-deriving that from ATX syntax could not, and running
- both meant two answers to one question."""
+
+def _visible_chars(text: str) -> int:
+ """count visible nonblank characters without link targets; callers subtract tracked heading prose."""
return sum(_visible_len(line) for line in text.split("\n") if line.strip())
@@ -1504,46 +1517,35 @@ def html_to_markdown(
site_links: SiteLinks | None = None,
max_span_chars: int | None = None,
) -> str:
- """Convert HTML to Markdown (headings, links, emphasis, lists, tables, blockquotes, code, entities).
-
- ``<script>``, ``<style>``, and ``<head>`` are stripped entirely, as are
- subtrees hidden from rendering (``hidden`` / ``aria-hidden="true"``).
-
- ``main_content=True`` applies a readability-style heuristic for page
- fetches: prefer the ``<article>`` subtree (GitHub renders READMEs there),
- then ``<main>``, falling back to the whole document, reduce a link-only
- ``<header>`` to the heading it carries, and strip known boilerplate
- fragments from the result.
-
- ``site_links`` records the links back into the page's own site; the output is unchanged.
-
- ``max_span_chars`` caps the cells generated for ``rowspan``/``colspan``, so a caller with a
- smaller result budget keeps room for the text after a table.
- """
+ """convert HTML to Markdown; main_content prefers a dominant article or main subtree, site_links records same-site links without changing output, and max_span_chars limits generated table cells."""
source_html = source_html.replace("\r\n", "\n").replace("\r", "\n")
span_limit = 2 * len(source_html)
if max_span_chars is not None:
span_limit = min(span_limit, max_span_chars)
rendered = ""
+ full_rendered = ""
if main_content:
- for scope_tag in ("article", "main"):
- # Render only the chosen subtree so sibling <article>/<main> elements do not leak in.
- length, rendered = _select_main_scope_render(
- source_html, scope_tag, site_links, span_limit
- )
- if length >= _MIN_MAIN_CONTENT_CHARS:
- break
- else:
- rendered = _strip_boilerplate_lines(
- _render(
- source_html,
- None,
- strip_header = True,
- site_links = site_links,
- span_char_limit = span_limit,
- ),
- site_links,
+ length, rendered, articles, article_visible = _select_main_scope_render(
+ source_html, "article", site_links, span_limit
+ )
+ if length < _MIN_MAIN_CONTENT_CHARS or articles > 1:
+ main_length, main_rendered, _, main_visible = _select_main_scope_render(
+ source_html, "main", site_links, span_limit
)
+ if articles > 2 and main_length < _MIN_MAIN_CONTENT_CHARS:
+ main_length, full_rendered = _render_main_document(
+ source_html, site_links, span_limit
+ )
+ main_rendered = full_rendered
+ main_visible = main_length
+ if main_length >= _MIN_MAIN_CONTENT_CHARS and (
+ length < _MIN_MAIN_CONTENT_CHARS or main_visible > 2 * article_visible
+ ):
+ length, rendered = main_length, main_rendered
+ if length < _MIN_MAIN_CONTENT_CHARS:
+ if not full_rendered:
+ _, full_rendered = _render_main_document(source_html, site_links, span_limit)
+ rendered = full_rendered
else:
rendered = _render(source_html, None, site_links = site_links, span_char_limit = span_limit)
return site_links.finish(rendered) if site_links is not None else rendered
diff --git a/studio/backend/tests/test_web_fetch_extraction.py b/studio/backend/tests/test_web_fetch_extraction.py
index 0038cab68d9..aa805d6e1ff 100644
--- a/studio/backend/tests/test_web_fetch_extraction.py
+++ b/studio/backend/tests/test_web_fetch_extraction.py
@@ -1546,8 +1546,7 @@ def test_many_tiny_articles_do_not_displace_substantial_main():
def test_single_substantial_article_still_preferred_over_main():
- # GitHub-README case: one substantial <article> inside <main> must still win
- # over sibling <main> furniture.
+ # GitHub README pages mix a substantial <article> with repository furniture.
article_body = "Real README documentation body text. " * 20
html = (
"<body><main>"
@@ -1560,16 +1559,115 @@ def test_single_substantial_article_still_preferred_over_main():
assert "JavaScript 89.3%" not in out
-# ── truncated (unclosed) main-content scopes must still be scored ──
+def test_one_card_does_not_stand_in_for_a_listing_main():
+ cards = "".join(
+ f"<article><h2>Plan {i}</h2><p>{f'Plan {i} feature and price detail. ' * 8}</p></article>"
+ for i in range(6)
+ )
+ html = f"<body><main><h1>Pricing</h1>{cards}</main></body>"
+ out = html_to_markdown(html, main_content = True)
+ for i in range(6):
+ assert f"Plan {i} feature and price detail." in out
+
+
+def test_article_listing_without_main_uses_the_document():
+ cards = "".join(
+ f"<article><h2>Plan {i}</h2><p>{f'Plan {i} feature and price detail. ' * 8}</p></article>"
+ for i in range(6)
+ )
+ out = html_to_markdown(f"<body><h1>Pricing</h1>{cards}</body>", main_content = True)
+ for i in range(6):
+ assert f"Plan {i} feature and price detail." in out
+
+
+def test_post_body_outside_article_beats_author_bio_card():
+ post = "Main post body paragraph with the actual story. " * 30
+ bio = "Author bio describing the writer and their work. " * 6
+ related = "".join(
+ f"<article class='related'><h3>Related {i}</h3><p>{'Teaser for another post. ' * 9}</p></article>"
+ for i in range(3)
+ )
+ html = (
+ "<body><main><h1>Post title</h1>"
+ f"<div class='post-content'><p>{post}</p></div>"
+ f"<article class='author-card'><p>{bio}</p></article>{related}"
+ "</main></body>"
+ )
+ out = html_to_markdown(html, main_content = True)
+ assert "Main post body paragraph" in out
+
+
+def test_lone_readme_article_is_kept_over_repo_page_chrome():
+ readme = "Short README describing the library. " * 8
+ rows = "".join(
+ f"<tr><td><a href='/o/r/tree/main/dir{i}'>dir{i}</a></td>"
+ f"<td><a href='/o/r/commit/{i}'>Update the dir{i} module and its tests</a></td><td>2 days ago</td></tr>"
+ for i in range(25)
+ )
+ html = (
+ "<body><main><h2>Repository files navigation</h2>"
+ f"<table><tr><th>Name</th><th>Last commit message</th><th>Last commit date</th></tr>{rows}</table>"
+ f"<article class='markdown-body'><h1>Lib</h1><p>{readme}</p></article>"
+ "<div><h2>About</h2><p>A small library for doing one thing well.</p></div>"
+ "</main></body>"
+ )
+ out = html_to_markdown(html, main_content = True)
+ assert "Short README describing the library." in out
+ assert "Last commit message" not in out
+
+
+def test_link_heavy_comments_do_not_pull_main_over_the_post():
+ post = "The post explains the topic in full detail here. " * 50
+ comments = "".join(
+ f"<article class='comment'><p><a href='https://example.com/author/{i}?{'utm_source=comments&' * 20}'>Reader {i}</a> "
+ f"says: {'Thanks for writing this up. ' * 9}</p></article>"
+ for i in range(6)
+ )
+ html = (
+ f"<body><main><article class='post'><h1>Post</h1><p>{post}</p></article>"
+ f"<section id='comments'>{comments}</section></main></body>"
+ )
+ out = html_to_markdown(html, main_content = True)
+ assert "The post explains the topic" in out
+ assert "Thanks for writing this up." not in out
+
+
+def test_generated_table_spans_do_not_pull_main_over_the_post():
+ post = "Primary article prose with real details. " * 30
+ related = "Related card teaser. " * 12
+ rows = "".join(f"<tr><td>row {i}</td></tr>" for i in range(1, 80))
+ table = f"<table><tr><td rowspan='80'>repeated marker</td><td>row 0</td></tr>{rows}</table>"
+ html = (
+ "<body><main>"
+ f"<article><h1>Primary</h1><p>{post}</p></article>"
+ f"<article><p>{related}</p></article>{table}"
+ "</main></body>"
+ )
+ out = html_to_markdown(html, main_content = True)
+ assert "Primary article prose" in out
+ assert "Related card teaser." not in out
+
+
[email protected]("hide", ["hidden", "aria-hidden='true'", "style='display:none'"])
+def test_hidden_duplicate_article_is_not_a_second_card(hide):
+ post = "The post explains the topic in full detail here. " * 20
+ comments = "".join(
+ f"<div class='comment'><p>Reader {i} says: {'Thanks for writing this up, it helped. ' * 6}</p></div>"
+ for i in range(8)
+ )
+ html = (
+ f"<body><main><article><h1>Post</h1><p>{post}</p></article>"
+ f"<article {hide}><p>Duplicate</p></article><section>{comments}</section></main></body>"
+ )
+ out = html_to_markdown(html, main_content = True)
+ assert "The post explains the topic" in out
+ assert "Thanks for writing this up" not in out
def test_truncated_open_article_scope_is_scored_and_preferred():
- # _fetch_url_raw caps large pages, so the download can end before the closing
- # </article>. The scope is still the main content and must be preferred over the
- # whole document (which re-leaks the page chrome).
+ # _fetch_url_raw may truncate before </article>, so the open scope must exclude page chrome.
chrome = "<nav>Skip to content</nav><div>Repository file tree and page chrome.</div>"
article_body = "Real README documentation body text. " * 20
- # No closing </article> / </body> -- the fetch cap truncated the page.
html = f"<body>{chrome}<article><h1>Guide</h1><p>{article_body}</p>"
out = html_to_markdown(html, main_content = True)
assert "Real README documentation body text." in out