Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: decode pages in the browser panel the way browsers do (#12983)
Nilayf7d992c91383
diff --git a/studio/backend/routes/browser.py b/studio/backend/routes/browser.pyindex fa3c828d66a..47f525faf78 100644--- a/studio/backend/routes/browser.py+++ b/studio/backend/routes/browser.py@@ -28,10 +28,16 @@ from fastapi.responses import Responsefrom pydantic import BaseModel, Fieldfrom auth.authentication import get_current_subject-from core.inference.tools import _USER_AGENTS, _fetch_url_raw, _normalize_url_scheme+from core.inference.tools import (+ _USER_AGENTS,+ _WHATWG_CHARSET_CODECS,+ _fetch_url_raw,+ _normalize_url_scheme,+ _sniff_meta_charset,+)from loggers import get_logger-# Same embedders as the canvas shell.+# same embedders as the canvas shell.from routes.inference import _ARTIFACT_PREVIEW_FRAME_ANCESTORS as _FRAME_ANCESTORSlogger = get_logger(__name__)@@ -68,8 +74,6 @@ _CONTENT_ATTR_RE = re.compile(_REFRESH_RE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*(?:[;,]\s*(?:url\s*=\s*)?['\"]?([^'\"]*)['\"]?)?", re.IGNORECASE)-_META_CHARSET_RE = re.compile(rb"""<meta[^<>]+charset\s*=\s*["']?([\w:.-]+)""", re.IGNORECASE)-# The sandbox's opaque origin fails CORS module loads: self-contained modules are inlined, ones with imports keep src._MODULE_SCRIPT_RE = re.compile(r"<script\b" + _TAG_BODY + r"\s*</script\s*>", re.IGNORECASE)# One start-tag attribute; quoted values are skipped whole so a name inside a value isn't matched.@@ -1100,7 +1104,7 @@ class BrowserFetchRequest(BaseModel):url: str = Field(..., min_length = 1, max_length = 8192)method: Literal["GET", "POST"] = "GET"body: Optional[str] = Field(default = None, max_length = 1024 * 1024)- # Smaller cap for favicons, so an icon can't be 50 MB.+ # favicon callers can lower the 50 MB fetch cap.max_bytes: Optional[int] = Field(default = None, ge = 1, le = _MAX_BROWSER_FETCH_BYTES)@@ -1111,34 +1115,47 @@ _BOMS = ()-# Browsers read these labels as windows-1252 (WHATWG Encoding), which fills 0x80-0x9F with quotes and dashes.-_WINDOWS_1252_LABELS = frozenset(- "ansi_x3.4-1968 ascii cp1252 cp819 csisolatin1 ibm819 iso-8859-1 iso-ir-100 iso8859-1 iso88591 "- "iso_8859-1 iso_8859-1:1987 l1 latin1 latin-1 us-ascii windows-1252 x-cp1252".split()-)+# headers may name UTF-16; meta labels map it to UTF-8 through the shared table.+_HEADER_CODECS = {+ **_WHATWG_CHARSET_CODECS,+ **dict.fromkeys(+ "csunicode iso-10646-ucs-2 ucs-2 unicode unicodefeff utf-16 utf-16le".split(), "utf-16"+ ),+ "unicodefffe": "utf-16-be",+ "utf-16be": "utf-16-be",+ "latin-1": "cp1252",+}+++def _codec(label: Optional[str], table: dict = _HEADER_CODECS) -> Optional[str]:+ return table.get(label.strip().lower()) if label else None-def _codec(label: str) -> str:- label = label.strip().lower()- return "cp1252" if label in _WINDOWS_1252_LABELS else label+def _raw_codec(label: Optional[str]) -> Optional[str]:+ codec = _codec(label)+ if codec or not label:+ return codec+ try:+ return codecs.lookup(label).name+ except (LookupError, ValueError):+ return Nonedef _decode_html(raw: bytes, charset: Optional[str]) -> str:- # A byte order mark wins over any declared charset, as in browsers.+ # byte order marks override declared charsets, matching browsers.for bom, codec in _BOMS:if raw.startswith(bom):return raw.decode(codec, errors = "replace")- candidates = [_codec(charset)] if charset else []- sniffed = _META_CHARSET_RE.search(raw[:4096])- if sniffed:- candidates.append(_codec(sniffed.group(1).decode("ascii", "ignore")))- candidates.append("utf-8")- for candidate in candidates:- try:- return raw.decode(candidate)- except (LookupError, UnicodeDecodeError):- continue- # Unlabelled and not UTF-8: windows-1252, as browsers default to.+ labelled = _codec(charset)+ if labelled is None:+ labelled = _sniff_meta_charset(raw[:4096], "text/html")+ if labelled:+ return raw.decode(labelled, errors = "replace")+ try:+ return raw.decode("utf-8")+ except UnicodeDecodeError:+ pass+ # browsers use U+FFFD for bad bytes in labelled pages and windows-1252 for unlabelled pages.return raw.decode("cp1252", errors = "replace")@@ -1149,7 +1166,7 @@ def _attr(match: "re.Match[str] | None") -> Optional[str]:def _join(base: str, href: str) -> Optional[str]:- """``urljoin``, or None for an address it cannot parse (``http://[bad``)."""+ """return None when ``urljoin`` cannot parse an address such as ``http://[bad``."""try:return urljoin(base, href.strip())except ValueError:@@ -1481,15 +1498,15 @@ def _build_response(media_type = "application/json",headers = {KIND_HEADER: "html"},)- charset = meta.get("charset")+ codec = _raw_codec(meta.get("charset"))textual = content_type.startswith("text/") or content_type.endswith(("json", "xml", "javascript"))- if textual and charset and charset.lower().replace("_", "-") not in ("utf-8", "utf8"):- # Text in another encoding goes out as UTF-8, the encoding the response is labelled with.+ if textual and codec and codec != "utf-8":+ # transcode text to UTF-8 to match the response label.try:- body = body.decode(_codec(charset), errors = "replace").encode("utf-8")- except LookupError:+ body = body.decode(codec, errors = "replace").encode("utf-8")+ except (LookupError, UnicodeError, ValueError):passreturn Response(content = body,@@ -1497,7 +1514,7 @@ def _build_response(headers = {KIND_HEADER: "raw",URL_HEADER: quote(final_url, safe = ":/?#[]@!$&'()*+,;=%~"),- # Never rendered on Studio's origin.+ # never rendered on Studio's origin"Content-Security-Policy": "sandbox",**({NAME_HEADER: quote(name, safe = "")} if (name := _attachment_name(meta)) else {}),},diff --git a/studio/backend/tests/test_browser_routes.py b/studio/backend/tests/test_browser_routes.pyindex 72823d19e57..eb33b8ce463 100644--- a/studio/backend/tests/test_browser_routes.py+++ b/studio/backend/tests/test_browser_routes.py@@ -219,6 +219,98 @@ def test_raw_text_is_transcoded_from_its_charset(monkeypatch):assert response.headers["content-type"] == "text/csv; charset=utf-8"+ "text, label, encoding",+ [("a,😀\n", "utf-32", "utf-32"), ("a,Ç\n", "ibm437", "cp437")],+)+def test_raw_text_is_transcoded_from_non_whatwg_charsets(monkeypatch, text, label, encoding):+ _fetch(+ monkeypatch,+ (None, text.encode(encoding), "text/plain"),+ {"url": "https://example.com/x.txt", "charset": label},+ )+ response = _call(url = "https://example.com/x.txt")+ assert response.body == text.encode("utf-8")+ assert response.headers["content-type"] == "text/plain; charset=utf-8"+++ "text, label, encoding",+ [+ ("<p>朱镕基 中央广播电视总台</p>", "gb2312", "gbk"),+ ("<p>丸数字①の日本語</p>", "shift_jis", "cp932"),+ ("<p>똠방각하 한국어</p>", "euc-kr", "cp949"),+ ("<p>廣東話嘅中文</p>", "big5", "big5hkscs"),+ ],+)+def test_a_page_decodes_as_browsers_read_its_label(monkeypatch, text, label, encoding):+ page = f"<html><meta charset={label}>{text}</html>"+ _fetch(+ monkeypatch,+ (None, page.encode(encoding), "text/html"),+ {"url": "https://example.com/", "charset": label},+ )+ assert text in json.loads(_call().body)["html"]+ assert browser_mod._decode_html(page.encode(encoding), None) == page+++def test_a_stray_byte_does_not_garble_a_labelled_page(monkeypatch):+ body = "<p>नमस्ते दुनिया</p><p>हिन्दी ".encode("utf-8") + b"\xff" + " पाठ</p>".encode("utf-8")+ _fetch(+ monkeypatch, (None, body, "text/html"), {"url": "https://example.com/", "charset": "utf-8"}+ )+ html = json.loads(_call().body)["html"]+ assert "<p>नमस्ते दुनिया</p><p>हिन्दी \ufffd पाठ</p>" in html+ assert html.count("\ufffd") == 1+++def test_a_malformed_header_charset_does_not_fall_through_to_meta_or_utf8(monkeypatch):+ body = b"<html><meta charset=utf-8><p>\xc2\x81</p></html>"+ _fetch(+ monkeypatch,+ (None, body, "text/html"),+ {"url": "https://example.com/", "charset": "shift_jis"},+ )+ html = json.loads(_call().body)["html"]+ assert "<p>\uff82\ufffd</p>" in html+ assert "\x81" not in html+++def test_a_commented_meta_charset_is_ignored(monkeypatch):+ body = b"<!-- <meta charset=shift_jis> --><p>caf\xc3\xa9</p>"+ _fetch(+ monkeypatch,+ (None, body, "text/html"),+ {"url": "https://example.com/", "charset": None},+ )+ assert "<p>café</p>" in json.loads(_call().body)["html"]+++ "body, charset",+ [+ (b"<html><p>caf\xc3\xa9</p></html>", "undefined"),+ (b"<html><meta charset=undefined><p>caf\xc3\xa9</p></html>", None),+ ],+)+def test_a_page_with_an_undecodable_charset_still_renders(monkeypatch, body, charset):+ _fetch(+ monkeypatch, (None, body, "text/html"), {"url": "https://example.com/", "charset": charset}+ )+ assert "<p>café</p>" in json.loads(_call().body)["html"]++[email protected]("charset", ["undefined", "idna"])+def test_text_with_an_undecodable_charset_passes_through(monkeypatch, charset):+ body = "a,café\n".encode("utf-8")+ _fetch(+ monkeypatch,+ (None, body, "text/csv"),+ {"url": "https://example.com/c.csv", "charset": charset},+ )+ assert _call(url = "https://example.com/c.csv").body == body++def test_other_bodies_pass_through_untouched(monkeypatch):pdf = b"%PDF-1.7\n..."_fetch(monkeypatch, (None, pdf, "application/pdf"), {"url": "https://example.com/p.pdf"})