Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep Word footnotes and endnotes in chats and knowledge bases (#12378)
Nilayf000412f2c3f
diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.pyindex 09bd478db80..4b141b55dd0 100644--- a/studio/backend/core/rag/parsers.py+++ b/studio/backend/core/rag/parsers.py@@ -14,6 +14,7 @@ import codecsimport loggingimport osimport re+import secretsfrom dataclasses import dataclassfrom html.parser import HTMLParser@@ -583,6 +584,7 @@ def _docx(path: str) -> list[Page]:document = docx.Document(path)lines: list[str] = []_docx_unwrap_table_controls(document.element.body)+ label_notes = _docx_mark_notes(document)# Walk body content in document order: paragraphs alone drop tables entirely.for block in _docx_blocks(document.element.body, document):if isinstance(block, Paragraph):@@ -591,7 +593,100 @@ def _docx(path: str) -> list[Page]:lines.append(text)elif isinstance(block, Table):lines.extend(_docx_table_rows(block))- return [_page("\n".join(lines), None)]+ return [_page(label_notes("\n".join(lines)), None)]+++def _roman(n: int) -> str:+ out = ""+ for value, digits in zip(+ (1000, 900, 500, 400, 100, 90, 50, 40, 10, 9, 5, 4, 1),+ ("m", "cm", "d", "cd", "c", "xc", "l", "xl", "x", "ix", "v", "iv", "i"),+ ):+ count, n = divmod(n, value)+ out += digits * count+ return out+++def _docx_mark_notes(document):+ """Sentinels after body note references; the returned function labels the surviving ones (1, 2 / i, ii) and appends the notes."""+ from docx.opc.constants import RELATIONSHIP_TYPE as RT+ from docx.oxml import OxmlElement, parse_xml+ from docx.table import Table++ kinds = []+ for kind, reltype, label in (+ ("footnote", RT.FOOTNOTES, str),+ ("endnote", RT.ENDNOTES, _roman),+ ):+ part = next(+ (+ r.target_part+ for r in document.part.rels.values()+ if r.reltype == reltype and not r.is_external+ ),+ None,+ )+ if part is None:+ continue+ bodies: dict[str, str] = {}+ for note in parse_xml(part.blob).iterchildren(_W + kind):+ if note.get(_W + "type", "normal") != "normal":+ continue+ texts = []+ for block in _docx_blocks(note, document):+ if isinstance(block, Table):+ texts.extend(_docx_table_rows(block))+ else:+ texts.append(" ".join(_docx_paragraph_text(block).split()))+ bodies[note.get(_W + "id")] = " ".join(t for t in texts if t)+ if bodies:+ kinds.append((kind, label, bodies))+ if not kinds:+ return lambda text: text++ refs: list[tuple[int, str]] = []+ # Nonce: document text shaped like a sentinel stays as written.+ nonce = secrets.token_hex(4)+ sentinel = re.compile(f"\ue000{nonce}\\.(\\d+)\ue001")+ referenced: list[set[str]] = [set() for _ in kinds]+ for k, (kind, _, bodies) in enumerate(kinds):+ for ref in document.element.body.iter(_W + kind + "Reference"):+ note_id = ref.get(_W + "id")+ if note_id in bodies:+ referenced[k].add(note_id)+ marker = OxmlElement("w:t")+ marker.text = f"\ue000{nonce}.{len(refs)}\ue001"+ refs.append((k, note_id))+ ref.addnext(marker)++ def label_notes(text: str) -> str:+ numbers: list[dict[str, int]] = [{} for _ in kinds]++ def label(match) -> str:+ index = int(match.group(1))+ if index >= len(refs):+ return match.group(0)+ k, note_id = refs[index]+ number = numbers[k].setdefault(note_id, len(numbers[k]) + 1)+ return f"[{kinds[k][1](number)}]"++ text = sentinel.sub(label, text)+ lines = [text] if text else []+ for k, (kind, label_of, bodies) in enumerate(kinds):+ # Unreferenced notes stay; ones referenced only from deleted or moved text go.+ for note_id in bodies:+ if note_id not in referenced[k]:+ numbers[k].setdefault(note_id, len(numbers[k]) + 1)+ notes = [+ f"[{label_of(number)}] {bodies[note_id]}"+ for note_id, number in sorted(numbers[k].items(), key = lambda item: item[1])+ if bodies[note_id]+ ]+ if notes:+ lines += [kind.capitalize() + "s", *notes]+ return "\n".join(lines)++ return label_notesdef _declared_charset(data: bytes) -> str | None:diff --git a/studio/backend/tests/test_rag_parsing.py b/studio/backend/tests/test_rag_parsing.pyindex 6bcaa73360b..18218f82cf9 100644--- a/studio/backend/tests/test_rag_parsing.py+++ b/studio/backend/tests/test_rag_parsing.py@@ -556,6 +556,63 @@ def test_docx_skips_placeholder_rows_and_cells_but_keeps_columns(tmp_path):assert "Name | | END" in text+def test_docx_numbers_visible_note_references_and_marks_the_body(tmp_path):+ document, docx, parsers = _shared_setup_1()+ from docx.opc.constants import CONTENT_TYPE as CT, RELATIONSHIP_TYPE as RT+ from docx.opc.packuri import PackURI+ from docx.opc.part import Part+ from docx.oxml import parse_xml++ def note(kind, note_id, text):+ return f'<w:{kind} w:id="{note_id}"><w:p><w:r><w:{kind}Ref/></w:r>{_r(" " + text)}</w:p></w:{kind}>'++ def notes(kind, body):+ return (+ f"<w:{kind}s {_DOCX_XMLNS}>"+ f'<w:{kind} w:type="separator" w:id="-1"><w:p><w:r><w:separator/></w:r></w:p></w:{kind}>'+ f'<w:{kind} w:type="continuationSeparator" w:id="0"><w:p><w:r><w:continuationSeparator/></w:r></w:p></w:{kind}>'+ f"{body}</w:{kind}s>"+ ).encode()++ def ref(kind, note_id):+ return f'<w:r><w:{kind}Reference w:id="{note_id}"/></w:r>'++ section = document.element.body[-1]+ section.addprevious(+ parse_xml(+ f"<w:p {_DOCX_XMLNS}><w:del w:id=\"9\">{ref('footnote', 4)}</w:del>"+ f"{_r('First.')}{ref('footnote', 2)}"+ f"{_r(' Second.')}{ref('footnote', 1)}{ref('endnote', 1)}"+ f"{_r(' ' + chr(0xE000) + '7' + chr(0xE001))}</w:p>"+ )+ )+ for kind, content_type, reltype, body in (+ (+ "footnote",+ CT.WML_FOOTNOTES,+ RT.FOOTNOTES,+ note("footnote", 1, "Source: LATER")+ + note("footnote", 2, "Source: EARLIER")+ + note("footnote", 3, "Source: UNREFERENCED")+ + note("footnote", 4, "Source: DELETED"),+ ),+ ("endnote", CT.WML_ENDNOTES, RT.ENDNOTES, note("endnote", 1, "Source: ENDNOTEBODY")),+ ):+ part = Part(+ PackURI(f"/word/{kind}s.xml"), content_type, notes(kind, body), document.part.package+ )+ document.part.relate_to(part, reltype)+ path = tmp_path / "notes.docx"+ document.save(str(path))++ text = "\n".join(pg.text for pg in parsers.parse(str(path)))+ assert text == (+ "First.[1] Second.[2][i] \ue0007\ue001\n"+ "Footnotes\n[1] Source: EARLIER\n[2] Source: LATER\n[3] Source: UNREFERENCED\n"+ "Endnotes\n[i] Source: ENDNOTEBODY"+ )++def _parse_html(tmp_path, body):from core.rag import parsersdiff --git a/studio/frontend/src/features/chat/attachment-content.ts b/studio/frontend/src/features/chat/attachment-content.tsindex dc0cdf7b6a0..2b622a18c91 100644--- a/studio/frontend/src/features/chat/attachment-content.ts+++ b/studio/frontend/src/features/chat/attachment-content.ts@@ -757,6 +757,211 @@ export function repackDocxAttachmentArchive(return zipSync(archive.entries, { level: 0 });}+const WORDPROCESSINGML_NAMESPACES = new Set([+ "http://schemas.openxmlformats.org/wordprocessingml/2006/main",+ "http://purl.oclc.org/ooxml/wordprocessingml/main",+]);+const DOCX_NOTE_BREAKS = new Set(["p", "tab", "br", "cr"]);+const DOCX_NOTE_SKIP = new Set(["del", "moveFrom", "rt", "Fallback"]);++function childElements(node: Node, ns: string, name: string): Element[] {+ return Array.from(node.childNodes).filter(+ (child): child is Element =>+ child.nodeType === 1 &&+ (child as Element).localName === name &&+ (child as Element).namespaceURI === ns,+ );+}++/** An unfilled content control holds Word's prompt, not a value. */+function isDocxPlaceholder(element: Element, ns: string): boolean {+ const flag = childElements(element, ns, "sdtPr").flatMap((pr) =>+ childElements(pr, ns, "showingPlcHdr"),+ )[0];+ return (+ flag !== undefined &&+ !["0", "false", "off"].includes(flag.getAttributeNS(ns, "val") ?? "")+ );+}++function docxNoteText(node: Node, ns: string): string {+ let text = "";+ for (const child of Array.from(node.childNodes)) {+ if (child.nodeType !== 1) continue;+ const element = child as Element;+ const name = element.localName;+ if (DOCX_NOTE_SKIP.has(name)) continue;+ if (name === "sdt" && isDocxPlaceholder(element, ns)) continue;+ if (name === "t" && element.namespaceURI === ns) {+ text += element.textContent ?? "";+ } else if (name === "noBreakHyphen") {+ text += "-";+ } else {+ text +=+ (DOCX_NOTE_BREAKS.has(name) ? " " : "") + docxNoteText(element, ns);+ }+ }+ return text;+}++const DOCX_NOTE_REFERENCE_RE =+ /<(?:([\w.-]+):)?(footnote|endnote)Reference\b([^>]*?)(\/?)>(\s*<\/(?:[\w.-]+:)?\2Reference\s*>)?/g;+const DOCX_NOTE_ID_RE = /(?:^|\s)(?:[\w.-]+:)?id\s*=\s*["']([^"']*)["']/;++function romanNumeral(n: number): string {+ let out = "";+ for (const [value, digits] of [+ [1000, "m"],+ [900, "cm"],+ [500, "d"],+ [400, "cd"],+ [100, "c"],+ [90, "xc"],+ [50, "l"],+ [40, "xl"],+ [10, "x"],+ [9, "ix"],+ [5, "v"],+ [4, "iv"],+ [1, "i"],+ ] as const) {+ for (; n >= value; n -= value) out += digits;+ }+ return out;+}+++/** Sentinels after note references; `label` numbers those extractRawText kept (1, 2 / i, ii) and appends the notes. */+export function markDocxNotes(archive: Uint8Array): {+ archive: Uint8Array;+ label: (text: string) => string;+} {+ const names = new Set<string>();+ const read = (name: string) =>+ unzipSync(archive, {+ filter: (entry) => {+ names.add(entry.name);+ return entry.name === name;+ },+ })[name];+ const targetsOf = (path: string) =>+ readDocxXmlTargets(+ read(docxRelationshipsPath(path)),+ path.slice(0, Math.max(0, path.lastIndexOf("/"))),+ );+ const resolve = (targets: string[] | undefined, fallback: string) =>+ targets?.find((path) => names.has(path)) ?? fallback;+ const main = resolve(+ targetsOf("").get(DOCX_MAIN_DOCUMENT_TYPE),+ DOCX_MAIN_DOCUMENT_FALLBACK,+ );+ const mainTargets = targetsOf(main);+ const notesOf = (heading: string, label: (n: number) => string) => ({+ heading,+ label,+ ns: "",+ bodies: new Map<string, string>(),+ referenced: new Set<string>(),+ });+ const kinds = {+ footnote: notesOf("Footnotes", String),+ endnote: notesOf("Endnotes", romanNumeral),+ };+ for (const [kind, notes] of Object.entries(kinds)) {+ const xml = read(+ resolve(+ mainTargets.get(`${DOCX_RELATIONSHIP_NAMESPACE}${kind}s`),+ `word/${kind}s.xml`,+ ),+ );+ if (!xml) continue;+ const doc = new DOMParser().parseFromString(+ strFromU8(xml),+ "application/xml",+ );+ const ns = doc.documentElement?.namespaceURI ?? "";+ if (!WORDPROCESSINGML_NAMESPACES.has(ns)) continue;+ notes.ns = ns;+ for (const note of Array.from(doc.getElementsByTagNameNS(ns, kind))) {+ const type = note.getAttributeNS(ns, "type");+ if (type && type !== "normal") continue;+ notes.bodies.set(+ note.getAttributeNS(ns, "id") ?? "",+ docxNoteText(note, ns).replace(/\s+/g, " ").trim(),+ );+ }+ }+ if (!kinds.footnote.bodies.size && !kinds.endnote.bodies.size) {+ return { archive, label: (text) => text };+ }++ const refs: { kind: keyof typeof kinds; id: string }[] = [];+ // Nonce: document text shaped like a sentinel stays as written.+ const nonce = Math.random().toString(36).slice(2, 10);+ const sentinel = new RegExp(`\\uE000${nonce}\\.(\\d+)\\uE001`, "g");+ const mainXml = read(main);+ const markedXml = mainXml+ ? strFromU8(mainXml).replace(+ DOCX_NOTE_REFERENCE_RE,+ (+ reference,+ _prefix: string | undefined,+ kind: keyof typeof kinds,+ attributes: string,+ selfClosing: string,+ endTag: string | undefined,+ ) => {+ const id = DOCX_NOTE_ID_RE.exec(attributes)?.[1];+ const notes = kinds[kind];+ // Never write inside a reference whose end tag is not right after its start.+ if (!selfClosing && !endTag) return reference;+ if (id === undefined || !notes.bodies.has(id)) return reference;+ notes.referenced.add(id);+ refs.push({ kind, id });+ const marker = `\uE000${nonce}.${refs.length - 1}\uE001`;+ // Own xmlns: the reference's prefix may be declared on the reference alone.+ return `${reference}<t xmlns="${notes.ns}">${marker}</t>`;+ },+ )+ : "";+ const label = (text: string) => {+ const numbers = {+ footnote: new Map<string, number>(),+ endnote: new Map<string, number>(),+ };+ const body = text.replace(sentinel, (match, index: string) => {+ const ref = refs[Number(index)];+ if (!ref) return match;+ const { kind, id } = ref;+ const seen = numbers[kind];+ if (!seen.has(id)) seen.set(id, seen.size + 1);+ return `[${kinds[kind].label(seen.get(id)!)}]`;+ });+ const sections: string[] = [];+ for (const [kind, notes] of Object.entries(kinds)) {+ const seen = numbers[kind as keyof typeof kinds];+ // Unreferenced notes stay; ones referenced only from deleted or moved text go.+ for (const id of notes.bodies.keys()) {+ if (!notes.referenced.has(id) && !seen.has(id)) {+ seen.set(id, seen.size + 1);+ }+ }+ const lines = [...seen]+ .sort((a, b) => a[1] - b[1])+ .filter(([id]) => notes.bodies.get(id))+ .map(+ ([id, number]) => `[${notes.label(number)}] ${notes.bodies.get(id)}`,+ );+ if (lines.length) sections.push([notes.heading, ...lines].join("\n"));+ }+ return body + sections.join("\n\n");+ };+ if (!refs.length) return { archive, label };+ const entries = unzipSync(archive);+ entries[main] = strToU8(markedXml);+ return { archive: zipSync(entries, { level: 0 }), label };+}+const XML_TOKEN_RE =/<!--[\s\S]*?-->|<!\[CDATA\[[\s\S]*?\]\]>|<[?!][\s\S]*?>|<(\/?)([^\s/>]+)(?:\s+[^\s=/>]+\s*=\s*(?:"[^"]*"|'[^']*'))*\s*(\/?)>/g;const PARAGRAPH_RE = /<(?:[\w.-]+:)?p[\s/>]/;@@ -1124,10 +1329,11 @@ export async function extractDocxAttachmentText(file: File): Promise<string> {file.name,new Uint8Array(buffer),);+ const marked = markDocxNotes(repacked);const { value } = await mammoth.extractRawText({- arrayBuffer: toArrayBuffer(repacked),+ arrayBuffer: toArrayBuffer(marked.archive),});- return value;+ return marked.label(value);}const HTML_PRESCAN_BYTES = 1024;diff --git a/studio/frontend/tests/attachment-preview-text.test.ts b/studio/frontend/tests/attachment-preview-text.test.tsindex a683419c767..2ba2b28c91f 100644--- a/studio/frontend/tests/attachment-preview-text.test.ts+++ b/studio/frontend/tests/attachment-preview-text.test.ts@@ -4,6 +4,7 @@import assert from "node:assert/strict";import test from "node:test";+import { DOMParser as XmlDomParser } from "@xmldom/xmldom";import {Unzip,UnzipInflate,@@ -28,6 +29,7 @@ const {getPdfAttachmentTextError,isAudioAttachment,isTextAttachment,+ markDocxNotes,parseAttachmentText,readAttachmentText,repackDocxAttachmentArchive,@@ -1167,6 +1169,132 @@ test("repackDocxAttachmentArchive refuses an archive that unpacks past the ceili);});+test("markDocxNotes numbers the references extractRawText keeps and marks the body", async () => {+ const w = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"';+ const note = (kind: string, id: number, text: string) =>+ `<w:${kind} w:id="${id}"><w:p><w:r><w:${kind}Ref/></w:r><w:r><w:t xml:space="preserve"> ${text}</w:t></w:r></w:p></w:${kind}>`;+ const notes = (kind: string, body: string) =>+ strToU8(+ `<w:${kind}s ${w}>` ++ `<w:${kind} w:type="separator" w:id="-1"><w:p><w:r><w:separator/></w:r></w:p></w:${kind}>` ++ `<w:${kind} w:type="continuationSeparator" w:id="0"><w:p><w:r><w:continuationSeparator/></w:r></w:p></w:${kind}>` ++ body ++ `</w:${kind}s>`,+ );+ const ref = (kind: string, id: number) =>+ kind === "endnote"+ ? `<w:r><w:${kind}Reference w:id="${id}">\n</w:${kind}Reference >\n</w:r>`+ : `<w:r><w:${kind}Reference w:id="${id}"/></w:r>`;+ const archive = repackDocxAttachmentArchive(+ "paper.docx",+ zipSync({+ "[Content_Types].xml": strToU8("<Types/>"),+ "_rels/.rels": relationships([["officeDocument", "word/document.xml"]]),+ "word/document.xml": strToU8(+ `<w:document ${w}><w:body><w:p><w:del w:id="9">${ref("footnote", 4)}</w:del>` ++ `<w:r><w:t>First.</w:t></w:r>${ref("footnote", 2)}` ++ `<w:r><w:t> Second.</w:t></w:r>${ref("footnote", 1)}${ref("endnote", 1)}` ++ `<w:r><w:t xml:space="preserve"> \uE0007\uE001</w:t></w:r>` ++ `<w:r><x:footnoteReference xmlns:x="http://schemas.openxmlformats.org/wordprocessingml/2006/main" x:id="3"/></w:r>` ++ `</w:p></w:body></w:document>`,+ ),+ "word/_rels/document.xml.rels": relationships([+ ["footnotes", "notes/foot.xml"],+ ["endnotes", "endnotes.xml"],+ ]),+ "word/notes/foot.xml": notes(+ "footnote",+ note("footnote", 1, "Source: LATER") ++ note("footnote", 2, "Source: EARLIER") ++ note("footnote", 3, "Source: LOCALLY DECLARED") ++ note("footnote", 4, "Source: DELETED"),+ ),+ "word/endnotes.xml": notes("endnote", note("endnote", 1, "Source: ENDNOTEBODY")),+ }),+ );+ const original = (globalThis as { DOMParser?: unknown }).DOMParser;+ (globalThis as { DOMParser?: unknown }).DOMParser = XmlDomParser;+ try {+ const marked = markDocxNotes(archive);+ const { default: mammoth } = await import("mammoth");+ const { value } = await mammoth.extractRawText({+ buffer: Buffer.from(marked.archive),+ });+ assert.equal(+ marked.label(value),+ "First.[1] Second.[2][i] \uE0007\uE001[3]\n\n" ++ "Footnotes\n[1] Source: EARLIER\n[2] Source: LATER\n[3] Source: LOCALLY DECLARED\n\n" ++ "Endnotes\n[i] Source: ENDNOTEBODY",+ );+ } finally {+ (globalThis as { DOMParser?: unknown }).DOMParser = original;+ }+});++test("markDocxNotes reads Strict OOXML notes and skips unfilled content controls", () => {+ const w = 'xmlns:w="http://purl.oclc.org/ooxml/wordprocessingml/main"';+ const run = (text: string) => `<w:r><w:t xml:space="preserve">${text}</w:t></w:r>`;+ const archive = repackDocxAttachmentArchive(+ "strict.docx",+ zipSync({+ "[Content_Types].xml": strToU8("<Types/>"),+ "_rels/.rels": relationships([["officeDocument", "word/document.xml"]]),+ "word/document.xml": strToU8(`<w:document ${w}><w:body/></w:document>`),+ "word/footnotes.xml": strToU8(+ `<w:footnotes ${w}><w:footnote w:id="1"><w:p>` ++ run("Keep") ++ `<w:sdt><w:sdtPr><w:showingPlcHdr/></w:sdtPr><w:sdtContent>${run(" Click or tap here to enter text.")}</w:sdtContent></w:sdt>` ++ `<w:sdt><w:sdtPr><w:showingPlcHdr w:val="0"/></w:sdtPr><w:sdtContent>${run(" FILLED")}</w:sdtContent></w:sdt>` ++ "</w:p></w:footnote></w:footnotes>",+ ),+ }),+ );+ const original = (globalThis as { DOMParser?: unknown }).DOMParser;+ (globalThis as { DOMParser?: unknown }).DOMParser = XmlDomParser;+ try {+ assert.equal(markDocxNotes(archive).label(""), "Footnotes\n[1] Keep FILLED");+ } finally {+ (globalThis as { DOMParser?: unknown }).DOMParser = original;+ }+});++test("markDocxNotes skips move sources, deletions and text box fallbacks", () => {+ const w = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"';+ const mc = 'xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"';+ const run = (text: string) => `<w:r><w:t xml:space="preserve">${text}</w:t></w:r>`;+ const box = `<w:txbxContent><w:p>${run("BOX")}</w:p></w:txbxContent>`;+ const archive = repackDocxAttachmentArchive(+ "moved.docx",+ zipSync({+ "[Content_Types].xml": strToU8("<Types/>"),+ "_rels/.rels": relationships([["officeDocument", "word/document.xml"]]),+ "word/document.xml": strToU8(`<w:document ${w}><w:body/></w:document>`),+ "word/_rels/document.xml.rels": relationships([["footnotes", "footnotes.xml"]]),+ "word/footnotes.xml": strToU8(+ `<w:footnotes ${w} ${mc}><w:footnote w:id="1"><w:p>` ++ run("Keep") ++ `<w:moveFrom w:id="7">${run(" MOVED")}</w:moveFrom>` ++ `<w:del w:id="8"><w:r><w:delText> GONE</w:delText></w:r></w:del>` ++ run(" COVID") + "<w:r><w:noBreakHyphen/></w:r>" + run("19") ++ `<w:moveTo w:id="9">${run(" MOVED")}</w:moveTo>` ++ `<w:r><mc:AlternateContent><mc:Choice Requires="wps">${box}</mc:Choice>` ++ `<mc:Fallback>${box}</mc:Fallback></mc:AlternateContent></w:r>` ++ "</w:p></w:footnote></w:footnotes>",+ ),+ }),+ );+ const original = (globalThis as { DOMParser?: unknown }).DOMParser;+ (globalThis as { DOMParser?: unknown }).DOMParser = XmlDomParser;+ try {+ assert.equal(+ markDocxNotes(archive).label(""),+ "Footnotes\n[1] Keep COVID-19 MOVED BOX",+ );+ } finally {+ (globalThis as { DOMParser?: unknown }).DOMParser = original;+ }+});+/** A preview only colours what the filename says is source; extracted document text is prose whatever the file was called. */test("attachmentTextLanguage maps source files and leaves prose alone", () => {assert.equal(attachmentTextLanguage("train.py", null), "python");