Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: read attached HTML files in the encoding they declare (#12235)
Nilay052b6e9006af
diff --git a/studio/frontend/src/features/chat/attachment-content.ts b/studio/frontend/src/features/chat/attachment-content.tsindex f8598886b1a..f0279a00b66 100644--- a/studio/frontend/src/features/chat/attachment-content.ts+++ b/studio/frontend/src/features/chat/attachment-content.ts@@ -1070,6 +1070,109 @@ export async function extractDocxAttachmentText(file: File): Promise<string> {return value;}+const HTML_PRESCAN_BYTES = 1024;+// Comment or whole tag, quotes included; an unterminated tag ends the scan, as in browsers.+const HTML_PRESCAN_TAG_RE =+ /<!--[\s\S]*?(?:-->|$)|<([a-z][^\s/>]*)((?:[\s/](?:[^>"']|"[^"]*"|'[^']*')*)?)>|<[!/?][^>]*>|<[a-z!/?][\s\S]*/gi;+const HTML_META_ATTR_RE =+ /([^\s"'/=>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]*)))?/g;+const HTML_META_CONTENT_CHARSET_RE = /charset\s*=\s*["']?\s*([^\s"';]+)/i;++function declaredHtmlEncoding(bytes: Uint8Array): string | null {+ const head = new TextDecoder("windows-1252").decode(+ bytes.subarray(0, HTML_PRESCAN_BYTES),+ );+ for (const [, tag, attributes] of head.matchAll(HTML_PRESCAN_TAG_RE)) {+ if (tag?.toLowerCase() !== "meta") {+ continue;+ }+ const attrs = new Map<string, string>();+ for (const [, name, ...values] of attributes.matchAll(HTML_META_ATTR_RE)) {+ const key = name.toLowerCase();+ if (!attrs.has(key)) {+ attrs.set(key, values.join(""));+ }+ }+ let label = attrs.get("charset");+ if (+ label === undefined &&+ attrs.get("http-equiv")?.toLowerCase() === "content-type"+ ) {+ label = attrs.get("content")?.match(HTML_META_CONTENT_CHARSET_RE)?.[1];+ }+ if (!label) {+ continue;+ }+ let encoding: string;+ try {+ encoding = new TextDecoder(label).encoding;+ } catch {+ continue;+ }+ // WHATWG prescan: a <meta> claiming UTF-16 means UTF-8.+ if (encoding.startsWith("utf-16")) {+ return "utf-8";+ }+ return encoding === "x-user-defined" ? "windows-1252" : encoding;+ }+ return null;+}++export function decodeHtmlAttachmentBytes(+ bytes: Uint8Array,+ truncated = false,+): string {+ const bom =+ bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf+ ? "utf-8"+ : bytes[0] === 0xff && bytes[1] === 0xfe+ ? "utf-16le"+ : bytes[0] === 0xfe && bytes[1] === 0xff+ ? "utf-16be"+ : null;+ if (bom) {+ return new TextDecoder(bom).decode(bytes);+ }+ const declared = declaredHtmlEncoding(bytes);+ // Stale meta after a UTF-8 re-save; short CJK can be valid in both, and ISO-2022-JP is 7-bit.+ if (declared && declared !== "utf-8") {+ const utf8 = strictDecode("utf-8", bytes, truncated);+ if (+ utf8 !== null &&+ utf8.length !== bytes.length &&+ (!MULTIBYTE_HTML_ENCODINGS.has(declared) ||+ strictDecode(declared, bytes, truncated) === null)+ ) {+ return utf8;+ }+ }+ return new TextDecoder(declared ?? "utf-8").decode(bytes);+}++const MULTIBYTE_HTML_ENCODINGS = new Set([+ "big5",+ "euc-jp",+ "euc-kr",+ "gb18030",+ "gbk",+ "iso-2022-jp",+ "shift_jis",+]);++function strictDecode(+ label: string,+ bytes: Uint8Array,+ truncated: boolean,+): string | null {+ try {+ return new TextDecoder(label, { fatal: true }).decode(bytes, {+ stream: truncated,+ });+ } catch {+ return null;+ }+}+export function extractHtmlAttachmentText(html: string): string {const doc = new DOMParser().parseFromString(html, "text/html");for (const el of doc.querySelectorAll("script, style, noscript, template")) {@@ -1137,7 +1240,7 @@ export async function readAttachmentText(}// raw markup, not the extraction; kept before opendocument to match the adaptersif (isHtmlAttachment(name, contentType)) {- return { label: null, ...(await readBoundedText(file)) };+ return { label: null, ...(await readBoundedHtml(file)) };}if (isOpenDocumentAttachment(name, contentType)) {const { label, text } = await readOpenDocumentAttachmentContent(@@ -1193,6 +1296,15 @@ async function readBoundedText(};}+async function readBoundedHtml(+ file: File,+): Promise<{ text: string; truncated: boolean }> {+ const truncated = file.size > MAX_PREVIEW_TEXT_BYTES;+ const slice = truncated ? file.slice(0, MAX_PREVIEW_TEXT_BYTES) : file;+ const bytes = new Uint8Array(await slice.arrayBuffer());+ return { text: decodeHtmlAttachmentBytes(bytes, truncated), truncated };+}+// A sent attachment keeps only the text its adapter produced, so the preview unwraps the// adapter's header rather than showing it. The stored payload has no size limit, so the// wrapper is matched on a prefix and only the capped body is copied out.diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsxindex 7bda7d3a40f..6eacab2de00 100644--- a/studio/frontend/src/features/chat/runtime-provider.tsx+++ b/studio/frontend/src/features/chat/runtime-provider.tsx@@ -63,6 +63,7 @@ import {} from "./api/chat-generation-api";import {TEXT_ATTACHMENT_ACCEPT,+ decodeHtmlAttachmentBytes,extractDocxAttachmentText,extractHtmlAttachmentText,extractOfficeAttachmentText,@@ -615,7 +616,8 @@ class HtmlAttachmentAdapter implements AttachmentAdapter {}async send(attachment: PendingAttachment): Promise<CompleteAttachment> {- const text = extractHtmlAttachmentText(await attachment.file.text());+ const bytes = new Uint8Array(await attachment.file.arrayBuffer());+ const text = extractHtmlAttachmentText(decodeHtmlAttachmentBytes(bytes));return {id: attachment.id,type: "document",diff --git a/studio/frontend/tests/attachment-preview-text.test.ts b/studio/frontend/tests/attachment-preview-text.test.tsindex 8c332e576fc..55030bf2494 100644--- a/studio/frontend/tests/attachment-preview-text.test.ts+++ b/studio/frontend/tests/attachment-preview-text.test.ts@@ -21,6 +21,7 @@ const {attachmentAudioSrc,attachmentTextLanguage,countAttachmentTextLines,+ decodeHtmlAttachmentBytes,extractHtmlAttachmentText,extractPdfAttachmentText,getDocxAttachmentError,@@ -1070,6 +1071,98 @@ test("a preview is never stricter than the adapter that took the file", async ());});+function legacyPage(head: string, body: number[]) {+ return new Uint8Array([+ ...new TextEncoder().encode(head),+ ...body,+ ...new TextEncoder().encode("</p>"),+ ]);+}++const WINDOWS_1252_BODY = [+ 0x43, 0x61, 0x66, 0xe9, 0x20, 0x80, 0x31, 0x32, 0x2c, 0x20, 0x6e, 0x61, 0xef,+ 0x76, 0x65,+];+const SHIFT_JIS_BODY = [+ 0x93, 0xfa, 0x96, 0x7b, 0x8c, 0xea, 0x82, 0xcc, 0x83, 0x79, 0x81, 0x5b, 0x83,+ 0x57,+];++test("an html attachment is read in the encoding its page declares", async () => {+ const word = legacyPage(+ '<html><head><meta http-equiv=Content-Type content="text/html; charset=windows-1252"></head><p>',+ WINDOWS_1252_BODY,+ );+ const japanese = legacyPage('<meta charset="Shift_JIS"><p>', SHIFT_JIS_BODY);+ for (const [bytes, name, expected] of [+ [word, "report.htm", "Café €12, naïve"],+ [japanese, "page.html", "日本語のページ"],+ ] as const) {+ const file = new File([bytes], name, { type: "text/html" });+ const { text } = await readAttachmentText(file, file.name, file.type);+ assert.ok(text.includes(expected), text);+ assert.ok(!text.includes("\uFFFD"), text);+ }+});++test("decodeHtmlAttachmentBytes reads the charset the way a browser does", () => {+ const utf8 = new TextEncoder().encode(+ "<meta charset=windows-1252><p>Café</p>",+ );+ assert.equal(+ decodeHtmlAttachmentBytes(new Uint8Array([0xef, 0xbb, 0xbf, ...utf8])),+ "<meta charset=windows-1252><p>Café</p>",+ );+ assert.equal(+ decodeHtmlAttachmentBytes(new TextEncoder().encode("<p>Café</p>")),+ "<p>Café</p>",+ );+ for (const head of [+ "<!-- <meta charset=Shift_JIS> --><meta charset=windows-1252><p>",+ '<div title="<meta charset=Shift_JIS>"><meta charset=windows-1252><p>',+ ]) {+ assert.equal(+ decodeHtmlAttachmentBytes(legacyPage(head, WINDOWS_1252_BODY)),+ `${head}Café €12, naïve</p>`,+ );+ }+ assert.equal(+ decodeHtmlAttachmentBytes(+ new TextEncoder().encode('<meta charset="utf-16"><p>Café</p>'),+ ),+ '<meta charset="utf-16"><p>Café</p>',+ );+});++test("a UTF-8 html page keeps its text when its meta names a legacy charset", async () => {+ const page =+ '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"><p>Café €12 日本語</p>';+ const file = new File([new TextEncoder().encode(page)], "saved.html", {+ type: "text/html",+ });+ const { text } = await readAttachmentText(file, file.name, file.type);+ assert.equal(text, page);+ assert.equal(+ decodeHtmlAttachmentBytes(new TextEncoder().encode(page).subarray(0, -6), true),+ page.slice(0, -5),+ );+ const jis = legacyPage('<meta charset="iso-2022-jp"><p>', [+ 0x1b, 0x24, 0x42, 0x46, 0x7c, 0x4b, 0x5c, 0x38, 0x6c, 0x1b, 0x28, 0x42,+ ]);+ assert.equal(+ decodeHtmlAttachmentBytes(jis),+ '<meta charset="iso-2022-jp"><p>日本語</p>',+ );+ const sjis = '<meta charset="Shift_JIS"><p>日本語のページです</p>';+ assert.equal(decodeHtmlAttachmentBytes(new TextEncoder().encode(sjis)), sjis);+ assert.equal(+ decodeHtmlAttachmentBytes(+ legacyPage('<meta charset="gbk"><p>', [0xd7, 0xa8, 0xd2, 0xb5]),+ ),+ '<meta charset="gbk"><p>专业</p>',+ );+});+test("a UTF-16 Markdown file previews as its text in the document viewer", async () => {const utf16 = new Uint8Array([0xff, 0xfe, ...Array.from("# Notes", (c) => [c.charCodeAt(0), 0]).flat()]);const file = new File([utf16], "notes.md", { type: "text/markdown" });