Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: web fetch reads forum threads and docs that keep their text in noscript (#17545)

Nilay2c4610c9a72f
diff --git a/studio/backend/core/inference/_html_to_md.py b/studio/backend/core/inference/_html_to_md.py
index 57fd900d772..fea88a1deb9 100644
--- a/studio/backend/core/inference/_html_to_md.py
+++ b/studio/backend/core/inference/_html_to_md.py
@@ -2469,6 +2469,7 @@ _ARTICLE_START = re.compile(r"<article", re.IGNORECASE)
_MAIN_START = re.compile(r"<main", re.IGNORECASE)
# absent text = no generated span cells, so a pass needs no span-free scoring re-render
_SPAN_ATTR = re.compile(r"(?:col|row)span", re.IGNORECASE)
+_NOSCRIPT_TAG = re.compile(r"<(/?)noscript\b", re.IGNORECASE)
# html.unescape raises once a decimal reference has more digits, leading zeros included, than
# int()'s limit, which Python never sets below 640
_LONG_DECIMAL_REF = re.compile(r"&#([0-9]{641,})")
@@ -2533,4 +2534,16 @@ def html_to_markdown(
rendered = full_rendered
else:
rendered = _render(source_html, None, site_links = site_links, span_char_limit = span_limit)
+ if not rendered.strip() and _NOSCRIPT_TAG.search(source_html):
+ # read whole: scoped, a link card's <article> inside one post would stand in for the thread
+ exposed = _NOSCRIPT_TAG.sub(r"<\1div", source_html)
+ if main_content:
+ visible, fallback = _render_main_document(exposed, site_links, span_limit)
+ else:
+ fallback = _render(exposed, None, site_links = site_links, span_char_limit = span_limit)
+ visible = _visible_chars(
+ site_links.clean(fallback) if site_links is not None else fallback
+ )
+ if visible >= _MIN_MAIN_CONTENT_CHARS:
+ rendered = fallback
return site_links.finish(rendered) if site_links is not None else rendered
diff --git a/studio/backend/tests/test_html_to_md_noscript.py b/studio/backend/tests/test_html_to_md_noscript.py
new file mode 100644
index 00000000000..ecff136cece
--- /dev/null
+++ b/studio/backend/tests/test_html_to_md_noscript.py
@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import sys
+import time
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference._html_to_md import SiteLinks, html_to_markdown
+from core.inference.tools import _render_page
+
+_POST = " ".join(f"Reply sentence {i} about loading part of a checkpoint." for i in range(8))
+_SECOND_POST = " ".join(f"Follow-up sentence {i} with the working answer." for i in range(8))
+_CARD = "Link preview card text that is long enough to pass for an article on its own. " * 4
+_SHELL = "<html><head><title>Topic</title></head><body><div id='app'></div>{}</body></html>"
+_TRACKING = (
+ '<noscript><img height="1" width="1" src="https://t.example/tr?ev=PageView"></noscript>'
+ "<noscript>You need to enable JavaScript to run this app.</noscript>"
+ '<noscript><iframe src="https://tags.example/ns.html" height="0" width="0"></iframe></noscript>'
+)
+_FORUM_TOPIC = _SHELL.format(
+ '<noscript data-path="/t/topic/1">'
+ "<header><a href='/'>Forum</a></header>"
+ "<div id='main-outlet' role='main'><h1>How to load part of a model</h1>"
+ f"<div class='post'><p>{_POST}</p>"
+ f"<aside class='onebox'><article class='onebox-body'><p>{_CARD}</p></article></aside></div>"
+ f"<div class='post'><p>{_SECOND_POST}</p></div></div>"
+ "</noscript>"
+)
+
+
[email protected]("main_content", [True, False])
+def test_page_whose_text_sits_in_noscript_is_read(main_content):
+ out = html_to_markdown(_FORUM_TOPIC, main_content = main_content)
+ assert "How to load part of a model" in out
+ assert _POST in out
+ assert _SECOND_POST in out
+
+
+def test_fetched_noscript_page_reaches_the_model_as_text():
+ out = _render_page(("html", "https://forum.example/t/topic/1", _FORUM_TOPIC), 20000)
+ assert _POST in out
+ assert "no readable text" not in out
+
+
+def test_same_site_links_in_noscript_are_kept_and_strippable():
+ links = SiteLinks("https://forum.example/t/topic/1")
+ page = _FORUM_TOPIC.replace("</h1>", "</h1><p><a href='/t/topic/2'>next topic</a></p>")
+ out = html_to_markdown(page, main_content = True, site_links = links)
+ assert "[next topic](/t/topic/2)" in out
+ assert "\x00" not in out
+ assert "(/t/topic/2)" not in links.strip(out)
+
+
[email protected]("main_content", [True, False])
+def test_noscript_is_ignored_when_the_page_has_text(main_content):
+ body = f"<main><h1>Guide</h1><p>{_POST}</p></main>"
+ plain = f"<html><body>{body}</body></html>"
+ with_noscript = (
+ f"<html><body>{body}{_TRACKING}<noscript><p>{_SECOND_POST}</p></noscript></body></html>"
+ )
+ assert html_to_markdown(with_noscript, main_content = main_content) == html_to_markdown(
+ plain, main_content = main_content
+ )
+
+
+def test_enable_javascript_notice_alone_is_not_readable_text():
+ page = _SHELL.format(_TRACKING)
+ assert html_to_markdown(page, main_content = True) == ""
+ assert _render_page(("html", "https://app.example/", page), 20000) == (
+ "(page returned no readable text)"
+ )
+
+
[email protected]("attr", ["hidden", 'aria-hidden="true"', 'style="display:none"'])
+def test_hidden_noscript_stays_hidden(attr):
+ page = _SHELL.format(f"<noscript {attr}><p>{_POST}</p></noscript>")
+ assert html_to_markdown(page, main_content = True) == ""
+
+
+def test_unterminated_noscript_tags_convert_in_linear_time():
+ page = "<head>" + "<noscript " * 40000
+ start = time.perf_counter()
+ assert html_to_markdown(page, main_content = True) == ""
+ assert time.perf_counter() - start < 3
+
+
[email protected]("extra", [_TRACKING, ""])
+def test_header_only_shell_with_noscript_stays_empty(extra):
+ nav = "".join(f"<a href='/s{i}'>Documentation section {i}</a> " for i in range(15))
+ page = _SHELL.format(f"<header>{nav}</header>{extra}")
+ assert html_to_markdown(page, main_content = True) == ""