Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: decode pages in the browser panel the way browsers do (#12983)

Nilayf7d992c91383
diff --git a/studio/backend/routes/browser.py b/studio/backend/routes/browser.py
index fa3c828d66a..47f525faf78 100644
--- a/studio/backend/routes/browser.py
+++ b/studio/backend/routes/browser.py
@@ -28,10 +28,16 @@ from fastapi.responses import Response
from pydantic import BaseModel, Field
from auth.authentication import get_current_subject
-from core.inference.tools import _USER_AGENTS, _fetch_url_raw, _normalize_url_scheme
+from core.inference.tools import (
+ _USER_AGENTS,
+ _WHATWG_CHARSET_CODECS,
+ _fetch_url_raw,
+ _normalize_url_scheme,
+ _sniff_meta_charset,
+)
from loggers import get_logger
-# Same embedders as the canvas shell.
+# same embedders as the canvas shell.
from routes.inference import _ARTIFACT_PREVIEW_FRAME_ANCESTORS as _FRAME_ANCESTORS
logger = get_logger(__name__)
@@ -68,8 +74,6 @@ _CONTENT_ATTR_RE = re.compile(
_REFRESH_RE = re.compile(
r"^\s*(\d+(?:\.\d+)?)\s*(?:[;,]\s*(?:url\s*=\s*)?['\"]?([^'\"]*)['\"]?)?", re.IGNORECASE
)
-_META_CHARSET_RE = re.compile(rb"""<meta[^<>]+charset\s*=\s*["']?([\w:.-]+)""", re.IGNORECASE)
-
# The sandbox's opaque origin fails CORS module loads: self-contained modules are inlined, ones with imports keep src.
_MODULE_SCRIPT_RE = re.compile(r"<script\b" + _TAG_BODY + r"\s*</script\s*>", re.IGNORECASE)
# One start-tag attribute; quoted values are skipped whole so a name inside a value isn't matched.
@@ -1100,7 +1104,7 @@ class BrowserFetchRequest(BaseModel):
url: str = Field(..., min_length = 1, max_length = 8192)
method: Literal["GET", "POST"] = "GET"
body: Optional[str] = Field(default = None, max_length = 1024 * 1024)
- # Smaller cap for favicons, so an icon can't be 50 MB.
+ # favicon callers can lower the 50 MB fetch cap.
max_bytes: Optional[int] = Field(default = None, ge = 1, le = _MAX_BROWSER_FETCH_BYTES)
@@ -1111,34 +1115,47 @@ _BOMS = (
)
-# Browsers read these labels as windows-1252 (WHATWG Encoding), which fills 0x80-0x9F with quotes and dashes.
-_WINDOWS_1252_LABELS = frozenset(
- "ansi_x3.4-1968 ascii cp1252 cp819 csisolatin1 ibm819 iso-8859-1 iso-ir-100 iso8859-1 iso88591 "
- "iso_8859-1 iso_8859-1:1987 l1 latin1 latin-1 us-ascii windows-1252 x-cp1252".split()
-)
+# headers may name UTF-16; meta labels map it to UTF-8 through the shared table.
+_HEADER_CODECS = {
+ **_WHATWG_CHARSET_CODECS,
+ **dict.fromkeys(
+ "csunicode iso-10646-ucs-2 ucs-2 unicode unicodefeff utf-16 utf-16le".split(), "utf-16"
+ ),
+ "unicodefffe": "utf-16-be",
+ "utf-16be": "utf-16-be",
+ "latin-1": "cp1252",
+}
+
+
+def _codec(label: Optional[str], table: dict = _HEADER_CODECS) -> Optional[str]:
+ return table.get(label.strip().lower()) if label else None
-def _codec(label: str) -> str:
- label = label.strip().lower()
- return "cp1252" if label in _WINDOWS_1252_LABELS else label
+def _raw_codec(label: Optional[str]) -> Optional[str]:
+ codec = _codec(label)
+ if codec or not label:
+ return codec
+ try:
+ return codecs.lookup(label).name
+ except (LookupError, ValueError):
+ return None
def _decode_html(raw: bytes, charset: Optional[str]) -> str:
- # A byte order mark wins over any declared charset, as in browsers.
+ # byte order marks override declared charsets, matching browsers.
for bom, codec in _BOMS:
if raw.startswith(bom):
return raw.decode(codec, errors = "replace")
- candidates = [_codec(charset)] if charset else []
- sniffed = _META_CHARSET_RE.search(raw[:4096])
- if sniffed:
- candidates.append(_codec(sniffed.group(1).decode("ascii", "ignore")))
- candidates.append("utf-8")
- for candidate in candidates:
- try:
- return raw.decode(candidate)
- except (LookupError, UnicodeDecodeError):
- continue
- # Unlabelled and not UTF-8: windows-1252, as browsers default to.
+ labelled = _codec(charset)
+ if labelled is None:
+ labelled = _sniff_meta_charset(raw[:4096], "text/html")
+ if labelled:
+ return raw.decode(labelled, errors = "replace")
+ try:
+ return raw.decode("utf-8")
+ except UnicodeDecodeError:
+ pass
+ # browsers use U+FFFD for bad bytes in labelled pages and windows-1252 for unlabelled pages.
return raw.decode("cp1252", errors = "replace")
@@ -1149,7 +1166,7 @@ def _attr(match: "re.Match[str] | None") -> Optional[str]:
def _join(base: str, href: str) -> Optional[str]:
- """``urljoin``, or None for an address it cannot parse (``http://[bad``)."""
+ """return None when ``urljoin`` cannot parse an address such as ``http://[bad``."""
try:
return urljoin(base, href.strip())
except ValueError:
@@ -1481,15 +1498,15 @@ def _build_response(
media_type = "application/json",
headers = {KIND_HEADER: "html"},
)
- charset = meta.get("charset")
+ codec = _raw_codec(meta.get("charset"))
textual = content_type.startswith("text/") or content_type.endswith(
("json", "xml", "javascript")
)
- if textual and charset and charset.lower().replace("_", "-") not in ("utf-8", "utf8"):
- # Text in another encoding goes out as UTF-8, the encoding the response is labelled with.
+ if textual and codec and codec != "utf-8":
+ # transcode text to UTF-8 to match the response label.
try:
- body = body.decode(_codec(charset), errors = "replace").encode("utf-8")
- except LookupError:
+ body = body.decode(codec, errors = "replace").encode("utf-8")
+ except (LookupError, UnicodeError, ValueError):
pass
return Response(
content = body,
@@ -1497,7 +1514,7 @@ def _build_response(
headers = {
KIND_HEADER: "raw",
URL_HEADER: quote(final_url, safe = ":/?#[]@!$&'()*+,;=%~"),
- # Never rendered on Studio's origin.
+ # never rendered on Studio's origin
"Content-Security-Policy": "sandbox",
**({NAME_HEADER: quote(name, safe = "")} if (name := _attachment_name(meta)) else {}),
},
diff --git a/studio/backend/tests/test_browser_routes.py b/studio/backend/tests/test_browser_routes.py
index 72823d19e57..eb33b8ce463 100644
--- a/studio/backend/tests/test_browser_routes.py
+++ b/studio/backend/tests/test_browser_routes.py
@@ -219,6 +219,98 @@ def test_raw_text_is_transcoded_from_its_charset(monkeypatch):
assert response.headers["content-type"] == "text/csv; charset=utf-8"
+ "text, label, encoding",
+ [("a,😀\n", "utf-32", "utf-32"), ("a,Ç\n", "ibm437", "cp437")],
+)
+def test_raw_text_is_transcoded_from_non_whatwg_charsets(monkeypatch, text, label, encoding):
+ _fetch(
+ monkeypatch,
+ (None, text.encode(encoding), "text/plain"),
+ {"url": "https://example.com/x.txt", "charset": label},
+ )
+ response = _call(url = "https://example.com/x.txt")
+ assert response.body == text.encode("utf-8")
+ assert response.headers["content-type"] == "text/plain; charset=utf-8"
+
+
+ "text, label, encoding",
+ [
+ ("<p>朱镕基 中央广播电视总台</p>", "gb2312", "gbk"),
+ ("<p>丸数字①の日本語</p>", "shift_jis", "cp932"),
+ ("<p>똠방각하 한국어</p>", "euc-kr", "cp949"),
+ ("<p>廣東話嘅中文</p>", "big5", "big5hkscs"),
+ ],
+)
+def test_a_page_decodes_as_browsers_read_its_label(monkeypatch, text, label, encoding):
+ page = f"<html><meta charset={label}>{text}</html>"
+ _fetch(
+ monkeypatch,
+ (None, page.encode(encoding), "text/html"),
+ {"url": "https://example.com/", "charset": label},
+ )
+ assert text in json.loads(_call().body)["html"]
+ assert browser_mod._decode_html(page.encode(encoding), None) == page
+
+
+def test_a_stray_byte_does_not_garble_a_labelled_page(monkeypatch):
+ body = "<p>नमस्ते दुनिया</p><p>हिन्दी ".encode("utf-8") + b"\xff" + " पाठ</p>".encode("utf-8")
+ _fetch(
+ monkeypatch, (None, body, "text/html"), {"url": "https://example.com/", "charset": "utf-8"}
+ )
+ html = json.loads(_call().body)["html"]
+ assert "<p>नमस्ते दुनिया</p><p>हिन्दी \ufffd पाठ</p>" in html
+ assert html.count("\ufffd") == 1
+
+
+def test_a_malformed_header_charset_does_not_fall_through_to_meta_or_utf8(monkeypatch):
+ body = b"<html><meta charset=utf-8><p>\xc2\x81</p></html>"
+ _fetch(
+ monkeypatch,
+ (None, body, "text/html"),
+ {"url": "https://example.com/", "charset": "shift_jis"},
+ )
+ html = json.loads(_call().body)["html"]
+ assert "<p>\uff82\ufffd</p>" in html
+ assert "\x81" not in html
+
+
+def test_a_commented_meta_charset_is_ignored(monkeypatch):
+ body = b"<!-- <meta charset=shift_jis> --><p>caf\xc3\xa9</p>"
+ _fetch(
+ monkeypatch,
+ (None, body, "text/html"),
+ {"url": "https://example.com/", "charset": None},
+ )
+ assert "<p>café</p>" in json.loads(_call().body)["html"]
+
+
+ "body, charset",
+ [
+ (b"<html><p>caf\xc3\xa9</p></html>", "undefined"),
+ (b"<html><meta charset=undefined><p>caf\xc3\xa9</p></html>", None),
+ ],
+)
+def test_a_page_with_an_undecodable_charset_still_renders(monkeypatch, body, charset):
+ _fetch(
+ monkeypatch, (None, body, "text/html"), {"url": "https://example.com/", "charset": charset}
+ )
+ assert "<p>café</p>" in json.loads(_call().body)["html"]
+
+
[email protected]("charset", ["undefined", "idna"])
+def test_text_with_an_undecodable_charset_passes_through(monkeypatch, charset):
+ body = "a,café\n".encode("utf-8")
+ _fetch(
+ monkeypatch,
+ (None, body, "text/csv"),
+ {"url": "https://example.com/c.csv", "charset": charset},
+ )
+ assert _call(url = "https://example.com/c.csv").body == body
+
+
def test_other_bodies_pass_through_untouched(monkeypatch):
pdf = b"%PDF-1.7\n..."
_fetch(monkeypatch, (None, pdf, "application/pdf"), {"url": "https://example.com/p.pdf"})