Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read attached HTML files in the encoding they declare (#12235)

Nilay052b6e9006af
diff --git a/studio/frontend/src/features/chat/attachment-content.ts b/studio/frontend/src/features/chat/attachment-content.ts
index f8598886b1a..f0279a00b66 100644
--- a/studio/frontend/src/features/chat/attachment-content.ts
+++ b/studio/frontend/src/features/chat/attachment-content.ts
@@ -1070,6 +1070,109 @@ export async function extractDocxAttachmentText(file: File): Promise<string> {
return value;
}
+const HTML_PRESCAN_BYTES = 1024;
+// Comment or whole tag, quotes included; an unterminated tag ends the scan, as in browsers.
+const HTML_PRESCAN_TAG_RE =
+ /<!--[\s\S]*?(?:-->|$)|<([a-z][^\s/>]*)((?:[\s/](?:[^>"']|"[^"]*"|'[^']*')*)?)>|<[!/?][^>]*>|<[a-z!/?][\s\S]*/gi;
+const HTML_META_ATTR_RE =
+ /([^\s"'/=>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]*)))?/g;
+const HTML_META_CONTENT_CHARSET_RE = /charset\s*=\s*["']?\s*([^\s"';]+)/i;
+
+function declaredHtmlEncoding(bytes: Uint8Array): string | null {
+ const head = new TextDecoder("windows-1252").decode(
+ bytes.subarray(0, HTML_PRESCAN_BYTES),
+ );
+ for (const [, tag, attributes] of head.matchAll(HTML_PRESCAN_TAG_RE)) {
+ if (tag?.toLowerCase() !== "meta") {
+ continue;
+ }
+ const attrs = new Map<string, string>();
+ for (const [, name, ...values] of attributes.matchAll(HTML_META_ATTR_RE)) {
+ const key = name.toLowerCase();
+ if (!attrs.has(key)) {
+ attrs.set(key, values.join(""));
+ }
+ }
+ let label = attrs.get("charset");
+ if (
+ label === undefined &&
+ attrs.get("http-equiv")?.toLowerCase() === "content-type"
+ ) {
+ label = attrs.get("content")?.match(HTML_META_CONTENT_CHARSET_RE)?.[1];
+ }
+ if (!label) {
+ continue;
+ }
+ let encoding: string;
+ try {
+ encoding = new TextDecoder(label).encoding;
+ } catch {
+ continue;
+ }
+ // WHATWG prescan: a <meta> claiming UTF-16 means UTF-8.
+ if (encoding.startsWith("utf-16")) {
+ return "utf-8";
+ }
+ return encoding === "x-user-defined" ? "windows-1252" : encoding;
+ }
+ return null;
+}
+
+export function decodeHtmlAttachmentBytes(
+ bytes: Uint8Array,
+ truncated = false,
+): string {
+ const bom =
+ bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf
+ ? "utf-8"
+ : bytes[0] === 0xff && bytes[1] === 0xfe
+ ? "utf-16le"
+ : bytes[0] === 0xfe && bytes[1] === 0xff
+ ? "utf-16be"
+ : null;
+ if (bom) {
+ return new TextDecoder(bom).decode(bytes);
+ }
+ const declared = declaredHtmlEncoding(bytes);
+ // Stale meta after a UTF-8 re-save; short CJK can be valid in both, and ISO-2022-JP is 7-bit.
+ if (declared && declared !== "utf-8") {
+ const utf8 = strictDecode("utf-8", bytes, truncated);
+ if (
+ utf8 !== null &&
+ utf8.length !== bytes.length &&
+ (!MULTIBYTE_HTML_ENCODINGS.has(declared) ||
+ strictDecode(declared, bytes, truncated) === null)
+ ) {
+ return utf8;
+ }
+ }
+ return new TextDecoder(declared ?? "utf-8").decode(bytes);
+}
+
+const MULTIBYTE_HTML_ENCODINGS = new Set([
+ "big5",
+ "euc-jp",
+ "euc-kr",
+ "gb18030",
+ "gbk",
+ "iso-2022-jp",
+ "shift_jis",
+]);
+
+function strictDecode(
+ label: string,
+ bytes: Uint8Array,
+ truncated: boolean,
+): string | null {
+ try {
+ return new TextDecoder(label, { fatal: true }).decode(bytes, {
+ stream: truncated,
+ });
+ } catch {
+ return null;
+ }
+}
+
export function extractHtmlAttachmentText(html: string): string {
const doc = new DOMParser().parseFromString(html, "text/html");
for (const el of doc.querySelectorAll("script, style, noscript, template")) {
@@ -1137,7 +1240,7 @@ export async function readAttachmentText(
}
// raw markup, not the extraction; kept before opendocument to match the adapters
if (isHtmlAttachment(name, contentType)) {
- return { label: null, ...(await readBoundedText(file)) };
+ return { label: null, ...(await readBoundedHtml(file)) };
}
if (isOpenDocumentAttachment(name, contentType)) {
const { label, text } = await readOpenDocumentAttachmentContent(
@@ -1193,6 +1296,15 @@ async function readBoundedText(
};
}
+async function readBoundedHtml(
+ file: File,
+): Promise<{ text: string; truncated: boolean }> {
+ const truncated = file.size > MAX_PREVIEW_TEXT_BYTES;
+ const slice = truncated ? file.slice(0, MAX_PREVIEW_TEXT_BYTES) : file;
+ const bytes = new Uint8Array(await slice.arrayBuffer());
+ return { text: decodeHtmlAttachmentBytes(bytes, truncated), truncated };
+}
+
// A sent attachment keeps only the text its adapter produced, so the preview unwraps the
// adapter's header rather than showing it. The stored payload has no size limit, so the
// wrapper is matched on a prefix and only the capped body is copied out.
diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx
index 7bda7d3a40f..6eacab2de00 100644
--- a/studio/frontend/src/features/chat/runtime-provider.tsx
+++ b/studio/frontend/src/features/chat/runtime-provider.tsx
@@ -63,6 +63,7 @@ import {
} from "./api/chat-generation-api";
import {
TEXT_ATTACHMENT_ACCEPT,
+ decodeHtmlAttachmentBytes,
extractDocxAttachmentText,
extractHtmlAttachmentText,
extractOfficeAttachmentText,
@@ -615,7 +616,8 @@ class HtmlAttachmentAdapter implements AttachmentAdapter {
}
async send(attachment: PendingAttachment): Promise<CompleteAttachment> {
- const text = extractHtmlAttachmentText(await attachment.file.text());
+ const bytes = new Uint8Array(await attachment.file.arrayBuffer());
+ const text = extractHtmlAttachmentText(decodeHtmlAttachmentBytes(bytes));
return {
id: attachment.id,
type: "document",
diff --git a/studio/frontend/tests/attachment-preview-text.test.ts b/studio/frontend/tests/attachment-preview-text.test.ts
index 8c332e576fc..55030bf2494 100644
--- a/studio/frontend/tests/attachment-preview-text.test.ts
+++ b/studio/frontend/tests/attachment-preview-text.test.ts
@@ -21,6 +21,7 @@ const {
attachmentAudioSrc,
attachmentTextLanguage,
countAttachmentTextLines,
+ decodeHtmlAttachmentBytes,
extractHtmlAttachmentText,
extractPdfAttachmentText,
getDocxAttachmentError,
@@ -1070,6 +1071,98 @@ test("a preview is never stricter than the adapter that took the file", async ()
);
});
+function legacyPage(head: string, body: number[]) {
+ return new Uint8Array([
+ ...new TextEncoder().encode(head),
+ ...body,
+ ...new TextEncoder().encode("</p>"),
+ ]);
+}
+
+const WINDOWS_1252_BODY = [
+ 0x43, 0x61, 0x66, 0xe9, 0x20, 0x80, 0x31, 0x32, 0x2c, 0x20, 0x6e, 0x61, 0xef,
+ 0x76, 0x65,
+];
+const SHIFT_JIS_BODY = [
+ 0x93, 0xfa, 0x96, 0x7b, 0x8c, 0xea, 0x82, 0xcc, 0x83, 0x79, 0x81, 0x5b, 0x83,
+ 0x57,
+];
+
+test("an html attachment is read in the encoding its page declares", async () => {
+ const word = legacyPage(
+ '<html><head><meta http-equiv=Content-Type content="text/html; charset=windows-1252"></head><p>',
+ WINDOWS_1252_BODY,
+ );
+ const japanese = legacyPage('<meta charset="Shift_JIS"><p>', SHIFT_JIS_BODY);
+ for (const [bytes, name, expected] of [
+ [word, "report.htm", "Café €12, naïve"],
+ [japanese, "page.html", "日本語のページ"],
+ ] as const) {
+ const file = new File([bytes], name, { type: "text/html" });
+ const { text } = await readAttachmentText(file, file.name, file.type);
+ assert.ok(text.includes(expected), text);
+ assert.ok(!text.includes("\uFFFD"), text);
+ }
+});
+
+test("decodeHtmlAttachmentBytes reads the charset the way a browser does", () => {
+ const utf8 = new TextEncoder().encode(
+ "<meta charset=windows-1252><p>Café</p>",
+ );
+ assert.equal(
+ decodeHtmlAttachmentBytes(new Uint8Array([0xef, 0xbb, 0xbf, ...utf8])),
+ "<meta charset=windows-1252><p>Café</p>",
+ );
+ assert.equal(
+ decodeHtmlAttachmentBytes(new TextEncoder().encode("<p>Café</p>")),
+ "<p>Café</p>",
+ );
+ for (const head of [
+ "<!-- <meta charset=Shift_JIS> --><meta charset=windows-1252><p>",
+ '<div title="<meta charset=Shift_JIS>"><meta charset=windows-1252><p>',
+ ]) {
+ assert.equal(
+ decodeHtmlAttachmentBytes(legacyPage(head, WINDOWS_1252_BODY)),
+ `${head}Café €12, naïve</p>`,
+ );
+ }
+ assert.equal(
+ decodeHtmlAttachmentBytes(
+ new TextEncoder().encode('<meta charset="utf-16"><p>Café</p>'),
+ ),
+ '<meta charset="utf-16"><p>Café</p>',
+ );
+});
+
+test("a UTF-8 html page keeps its text when its meta names a legacy charset", async () => {
+ const page =
+ '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"><p>Café €12 日本語</p>';
+ const file = new File([new TextEncoder().encode(page)], "saved.html", {
+ type: "text/html",
+ });
+ const { text } = await readAttachmentText(file, file.name, file.type);
+ assert.equal(text, page);
+ assert.equal(
+ decodeHtmlAttachmentBytes(new TextEncoder().encode(page).subarray(0, -6), true),
+ page.slice(0, -5),
+ );
+ const jis = legacyPage('<meta charset="iso-2022-jp"><p>', [
+ 0x1b, 0x24, 0x42, 0x46, 0x7c, 0x4b, 0x5c, 0x38, 0x6c, 0x1b, 0x28, 0x42,
+ ]);
+ assert.equal(
+ decodeHtmlAttachmentBytes(jis),
+ '<meta charset="iso-2022-jp"><p>日本語</p>',
+ );
+ const sjis = '<meta charset="Shift_JIS"><p>日本語のページです</p>';
+ assert.equal(decodeHtmlAttachmentBytes(new TextEncoder().encode(sjis)), sjis);
+ assert.equal(
+ decodeHtmlAttachmentBytes(
+ legacyPage('<meta charset="gbk"><p>', [0xd7, 0xa8, 0xd2, 0xb5]),
+ ),
+ '<meta charset="gbk"><p>专业</p>',
+ );
+});
+
test("a UTF-16 Markdown file previews as its text in the document viewer", async () => {
const utf16 = new Uint8Array([0xff, 0xfe, ...Array.from("# Notes", (c) => [c.charCodeAt(0), 0]).flat()]);
const file = new File([utf16], "notes.md", { type: "text/markdown" });