Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: move chat with files into a panel above the composer (#13078)
Michael Han00958e6a03fc
diff --git a/studio/backend/core/rag/cfb.py b/studio/backend/core/rag/cfb.pynew file mode 100644index 00000000000..963b785b38a--- /dev/null+++ b/studio/backend/core/rag/cfb.py@@ -0,0 +1,173 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Read-only OLE compound file reader ([MS-CFB]), the container of .doc, .xls, .ppt and .msg."""++from __future__ import annotations++import struct++SIGNATURE = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"+_FREE, _END, _FAT, _DIFAT = 0xFFFFFFFF, 0xFFFFFFFE, 0xFFFFFFFD, 0xFFFFFFFC+_NO_STREAM = 0xFFFFFFFF+_STORAGE, _STREAM, _ROOT = 1, 2, 5+++class CompoundFileError(ValueError):+ pass+++class _Entry:+ __slots__ = ("name", "kind", "left", "right", "child", "start", "size")++ def __init__(self, raw: bytes, major: int):+ name_len = struct.unpack_from("<H", raw, 64)[0]+ self.name = raw[: max(0, min(name_len, 64) - 2)].decode("utf-16-le", "replace")+ self.kind = raw[66]+ self.left, self.right, self.child = struct.unpack_from("<III", raw, 68)+ self.start = struct.unpack_from("<I", raw, 116)[0]+ size = struct.unpack_from("<Q", raw, 120)[0]+ # Version 3 files may leave junk in the high half.+ self.size = size & 0xFFFFFFFF if major == 3 else size+++class CompoundFile:+ """Streams of a compound file, addressed by path: ``open("WordDocument")``,+ ``open("__attach_version1.0_#00000000", "__substg1.0_3707001F")``."""++ def __init__(self, data: bytes):+ if len(data) < 512 or not data.startswith(SIGNATURE):+ raise CompoundFileError("not an OLE compound file")+ self._data = data+ self._children_cache: dict[int, dict[str, int]] = {}+ major = struct.unpack_from("<H", data, 0x1A)[0]+ shift = struct.unpack_from("<H", data, 0x1E)[0]+ mini_shift = struct.unpack_from("<H", data, 0x20)[0]+ if (major, shift) not in ((3, 9), (4, 12)) or mini_shift != 6:+ raise CompoundFileError("unsupported compound file version")+ self._sector = 1 << shift+ self._mini_cutoff = struct.unpack_from("<I", data, 0x38)[0]+ (n_fat,) = struct.unpack_from("<I", data, 0x2C)+ (first_dir,) = struct.unpack_from("<I", data, 0x30)+ first_mini_fat, n_mini_fat = struct.unpack_from("<II", data, 0x3C)+ first_difat, n_difat = struct.unpack_from("<II", data, 0x44)++ fat_sectors = [s for s in struct.unpack_from("<109I", data, 0x4C) if s < _DIFAT]+ per = self._sector // 4+ # Enough to map the whole file; a crafted DIFAT can repeat one sector millions of times.+ max_fat = len(data) // self._sector // per + 1+ sector, seen = first_difat, set()+ for _ in range(n_difat):+ if sector >= _DIFAT or sector in seen or len(fat_sectors) >= max_fat:+ break+ seen.add(sector)+ words = struct.unpack(f"<{per}I", self._raw_sector(sector))+ fat_sectors += [s for s in words[:-1] if s < _DIFAT]+ sector = words[-1]+ fat_sectors = fat_sectors[: min(n_fat or max_fat, max_fat)]+ self._fat: list[int] = []+ for s in fat_sectors:+ self._fat.extend(struct.unpack(f"<{per}I", self._raw_sector(s)))++ dir_bytes = self._read_chain(first_dir, self._fat, self._raw_sector, self._sector)+ self._entries = [+ _Entry(dir_bytes[i : i + 128], major) for i in range(0, len(dir_bytes) - 127, 128)+ ]+ if not self._entries or self._entries[0].kind != _ROOT:+ raise CompoundFileError("missing root entry")+ root = self._entries[0]+ mini_fat_bytes = (+ self._read_chain(first_mini_fat, self._fat, self._raw_sector, self._sector)+ if n_mini_fat+ else b""+ )+ self._mini_fat = list(struct.unpack(f"<{len(mini_fat_bytes) // 4}I", mini_fat_bytes))+ self._mini_stream = self._read_chain(+ root.start, self._fat, self._raw_sector, self._sector, root.size+ )++ def _raw_sector(self, sector: int) -> bytes:+ offset = (sector + 1) * self._sector+ if offset + self._sector > len(self._data):+ # A short final sector is common; anything past the end is not.+ if offset >= len(self._data):+ raise CompoundFileError("sector out of range")+ return self._data[offset:].ljust(self._sector, b"\0")+ return self._data[offset : offset + self._sector]++ def _mini_sector(self, sector: int) -> bytes:+ offset = sector * 64+ if offset >= len(self._mini_stream):+ raise CompoundFileError("mini sector out of range")+ return self._mini_stream[offset : offset + 64]++ @staticmethod+ def _read_chain(+ start,+ table,+ read,+ size,+ limit = None,+ ) -> bytes:+ parts, seen, sector = [], set(), start+ while sector < _DIFAT:+ if sector in seen:+ raise CompoundFileError("broken sector chain")+ seen.add(sector)+ parts.append(read(sector))+ if limit is not None and len(parts) * size >= limit:+ break+ # Some writers leave the file's last sectors out of the FAT; end the chain there.+ sector = table[sector] if sector < len(table) else _END+ out = b"".join(parts)+ return out[:limit] if limit is not None else out++ def _children(self, index: int) -> dict[str, int]:+ """Children of a storage, keyed by lowercased name (the red-black tree flattened)."""+ # Cached: a .msg looks up properties per attachment, each walking the root again.+ cached = self._children_cache.get(index)+ if cached is not None:+ return cached+ found: dict[str, int] = {}+ self._children_cache[index] = found+ stack, seen = [self._entries[index].child], set()+ while stack:+ i = stack.pop()+ if i == _NO_STREAM or i >= len(self._entries) or i in seen:+ continue+ seen.add(i)+ entry = self._entries[i]+ found[entry.name.lower()] = i+ stack += [entry.left, entry.right]+ return found++ def _find(self, path: tuple[str, ...]) -> _Entry | None:+ index = 0+ for name in path:+ index = self._children(index).get(name.lower())+ if index is None:+ return None+ return self._entries[index]++ def exists(self, *path: str) -> bool:+ return self._find(path) is not None++ def listdir(self, *path: str) -> list[str]:+ entry_index = 0+ for name in path:+ entry_index = self._children(entry_index).get(name.lower())+ if entry_index is None:+ return []+ return [self._entries[i].name for i in self._children(entry_index).values()]++ def read(self, *path: str) -> bytes:+ entry = self._find(path)+ if entry is None or entry.kind != _STREAM:+ raise CompoundFileError(f"missing stream: {'/'.join(path)}")+ if entry.size == 0:+ return b""+ if entry.size > len(self._data):+ raise CompoundFileError("stream larger than the file")+ if entry.size < self._mini_cutoff:+ return self._read_chain(entry.start, self._mini_fat, self._mini_sector, 64, entry.size)+ return self._read_chain(entry.start, self._fat, self._raw_sector, self._sector, entry.size)diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.pyindex beca8d19c4b..0732e99ef89 100644--- a/studio/backend/core/rag/config.py+++ b/studio/backend/core/rag/config.py@@ -68,7 +68,13 @@ SOURCE_TEXT_EXTS = frozenset(.mmd .mermaid .puml .plantuml .dot .gv .feature .robot .http .rest .diff .patch""".split())-_PARSEABLE_EXTS = SUPPORTED_UPLOAD_EXTS | SOURCE_TEXT_EXTS+DOCUMENT_UPLOAD_EXTS = frozenset(+ """+ .doc .docm .dotx .dotm .xls .xlsx .xlsm .xltx .xltm .ppt .pptx .pptm .potx .potm .ppsx .ppsm+ .odt .ods .odp .ott .ots .otp .msg .eml .mht .mhtml .rtf .epub .xhtml .xht+ """.split()+)+_PARSEABLE_EXTS = SUPPORTED_UPLOAD_EXTS | SOURCE_TEXT_EXTS | DOCUMENT_UPLOAD_EXTS# RAG_UPLOAD_EXTS (e.g. ".md,.markdown") can only narrow: a type without a parser would fail every ingest._requested_exts = {"." + ext.strip().lstrip(".").lower()diff --git a/studio/backend/core/rag/office_formats.py b/studio/backend/core/rag/office_formats.pynew file mode 100644index 00000000000..9bb76249248--- /dev/null+++ b/studio/backend/core/rag/office_formats.py@@ -0,0 +1,1843 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Text from Office, OpenDocument, e-book, email and RTF files, using only the stdlib.++Each reader returns ``[(text, page_number), ...]``: one entry per slide (numbered), or+per sheet / chapter (unnumbered). Table rows are joined with " | ", as for .docx.+"""++from __future__ import annotations++import codecs+import datetime as dt+import decimal+import email+import email.policy+import math+import posixpath+import re+import struct+import zipfile+from urllib.parse import unquote+from xml.etree import ElementTree as ET++from .cfb import CompoundFile, CompoundFileError++Section = tuple[str, "int | None"]++_MAX_MEMBER_BYTES = 128 * 1024 * 1024+_MAX_TOTAL_BYTES = 512 * 1024 * 1024+_MAX_COLUMNS = 1024+_MAX_REPEAT = 1024+# Parsed XML costs about 100-300 bytes per element, far more than its compressed size.+_MAX_XML_ELEMENTS = 2_000_000+_MAX_UNIT_ELEMENTS = 200_000 # one streamed row or string+_MAX_XML_DEPTH = 512+# Text a document may add by repeating cells and rows.+_MAX_REPEATED_CHARS = 32 * 1024 * 1024++_NS = {+ "s": "http://schemas.openxmlformats.org/spreadsheetml/2006/main",+ "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",+ "rel": "http://schemas.openxmlformats.org/package/2006/relationships",+ "p": "http://schemas.openxmlformats.org/presentationml/2006/main",+ "a": "http://schemas.openxmlformats.org/drawingml/2006/main",+ "c": "http://schemas.openxmlformats.org/drawingml/2006/chart",+ "dgm": "http://schemas.openxmlformats.org/drawingml/2006/diagram",+ "office": "urn:oasis:names:tc:opendocument:xmlns:office:1.0",+ "text": "urn:oasis:names:tc:opendocument:xmlns:text:1.0",+ "table": "urn:oasis:names:tc:opendocument:xmlns:table:1.0",+ "draw": "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0",+ "presentation": "urn:oasis:names:tc:opendocument:xmlns:presentation:1.0",+ "container": "urn:oasis:names:tc:opendocument:xmlns:container",+ "opf": "http://www.idpf.org/2007/opf",+}+# ISO Strict OOXML names the same schemas with other URIs.+_STRICT_NS = tuple(+ (f"http://purl.oclc.org/ooxml/{strict}".encode(), _NS[prefix].encode())+ for strict, prefix in (+ ("spreadsheetml/main", "s"),+ ("presentationml/main", "p"),+ ("drawingml/main", "a"),+ ("drawingml/chart", "c"),+ ("drawingml/diagram", "dgm"),+ ("officeDocument/relationships", "r"),+ )+)+++def _q(prefix: str, tag: str) -> str:+ return f"{{{_NS[prefix]}}}{tag}"+++def _row(cells: list[str]) -> str:+ while cells and not cells[-1].strip():+ cells.pop()+ return " | ".join(cells) if any(c.strip() for c in cells) else ""+++def _number(value: float) -> str:+ if math.isfinite(value) and value == int(value) and abs(value) < 1e15:+ return str(int(value))+ return repr(value)+++class _Archive:+ def __init__(self, path: str):+ try:+ self._zip = zipfile.ZipFile(path)+ except zipfile.BadZipFile as exc:+ raise ValueError("file is not a valid archive") from exc+ self._names = {name.lower(): name for name in self._zip.namelist()}+ self._budget = _MAX_TOTAL_BYTES++ def __enter__(self):+ return self++ def __exit__(self, *_exc):+ self._zip.close()++ def has(self, name: str) -> bool:+ return name.lower() in self._names++ def read(self, name: str) -> bytes:+ real = self._names.get(name.lower())+ if real is None:+ raise ValueError(f"missing archive member: {name}")+ if self._zip.getinfo(real).flag_bits & 0x1:+ raise ValueError("file is password protected")+ limit = min(_MAX_MEMBER_BYTES, self._budget)+ with self._zip.open(real) as member:+ data = member.read(limit + 1)+ if len(data) > limit:+ raise ValueError("archive member is too large")+ self._budget -= len(data)+ return data++ def _xml_events(self, name: str, events: tuple[str, ...]):+ data = self.read(name)+ # Office XML never declares entities; refusing them rules out expansion attacks.+ if b"<!ENTITY" in data:+ raise ValueError("XML entity declarations are not supported")+ if b"purl.oclc.org/ooxml/" in data:+ for strict, transitional in _STRICT_NS:+ data = data.replace(strict, transitional)+ parser = ET.XMLPullParser(events)+ try:+ for start in range(0, len(data), 1 << 20):+ parser.feed(data[start : start + (1 << 20)])+ yield from parser.read_events()+ parser.close()+ yield from parser.read_events()+ except ET.ParseError as exc:+ raise ValueError(f"malformed XML in {name}") from exc++ def xml(self, name: str) -> ET.Element:+ root, count = None, 0+ for _event, element in self._xml_events(name, ("start",)):+ root = element if root is None else root+ count += 1+ if count > _MAX_XML_ELEMENTS:+ raise ValueError(f"{name} is too large to index")+ if root is None:+ raise ValueError(f"malformed XML in {name}")+ return root++ def stream(self, name: str, units: frozenset[str]):+ """("start", element) outside units and ("unit", element) once a unit is complete.++ Handled elements are dropped, so memory holds one unit rather than the whole part.+ """+ stack: list[ET.Element] = []+ unit, count = None, 0+ for event, element in self._xml_events(name, ("start", "end")):+ if event == "start":+ stack.append(element)+ if len(stack) > _MAX_XML_DEPTH:+ raise ValueError(f"{name} is nested too deeply")+ if unit is not None:+ count += 1+ if count > _MAX_UNIT_ELEMENTS:+ raise ValueError(f"{name} is too large to index")+ elif element.tag in units:+ unit, count = element, 0+ else:+ yield "start", element+ continue+ stack.pop()+ if unit is not None and element is not unit:+ continue+ if element is unit:+ unit = None+ yield "unit", element+ if stack:+ # Earlier siblings are gone already, so this is the parent's first child.+ stack[-1].remove(element)++ def relationships(self, part: str) -> list[tuple[str, str, str]]:+ """(id, type, member name) of an OOXML part's relationships; "" for the package."""+ folder, base = posixpath.split(part)+ rels_name = posixpath.join(folder, "_rels", base + ".rels")+ if not self.has(rels_name):+ return []+ out = []+ for rel in self.xml(rels_name).iter(_q("rel", "Relationship")):+ target = rel.get("Target", "")+ if rel.get("TargetMode") == "External" or not target:+ continue+ path = target.lstrip("/") if target.startswith("/") else posixpath.join(folder, target)+ out.append((rel.get("Id", ""), rel.get("Type", ""), posixpath.normpath(path)))+ return out++ def rels(self, part: str) -> dict[str, str]:+ """Relationship id -> member name for an OOXML part."""+ return {rel_id: target for rel_id, _type, target in self.relationships(part)}++ def related(self, part: str, kind: str, default: str) -> str | None:+ """The part's first ``kind`` relationship target, else ``default`` if present."""+ for _id, rel_type, target in self.relationships(part):+ if rel_type.endswith("/" + kind) and self.has(target):+ return target+ return default if self.has(default) else None+++# Built-in date and time format IDs, including the CJK (27-36, 50-58) and Thai (71-81) ones.+_BUILTIN_DATE_FORMATS = {+ *range(14, 23),+ *range(27, 37),+ *range(45, 48),+ *range(50, 59),+ *range(71, 82),+}+++_BUILTIN_DURATION_FORMATS = {46} # [h]:mm:ss+# Classified as openpyxl does: quoted text and locale tags aside, any date or time token.+_FORMAT_STRIP = re.compile(r'".*?"|\[(?!hh?\]|mm?\]|ss?\])[^\]]*\]')+_FORMAT_DURATION = re.compile(+ r"\[hh?\](:mm(:ss(\.0*)?)?)?|\[mm?\](:ss(\.0*)?)?|\[ss?\](\.0*)?", re.I+)+++def _format_kind(format_id: int, code: str) -> str | None:+ """ "date" for dates and times of day, "duration" for elapsed time, else None."""+ if not code:+ if format_id in _BUILTIN_DURATION_FORMATS:+ return "duration"+ return "date" if format_id in _BUILTIN_DATE_FORMATS else None+ code = code.split(";")[0]+ if _FORMAT_DURATION.search(code):+ return "duration"+ return "date" if re.search(r"(?<![_\\])[dmhysDMHYS]", _FORMAT_STRIP.sub("", code)) else None+++def _serial_text(serial: float, kind: str, date1904: bool) -> str:+ if kind == "duration":+ seconds = round(serial * 86_400)+ hours, rest = divmod(abs(seconds), 3600)+ return f"{'-' if seconds < 0 else ''}{hours}:{rest // 60:02}:{rest % 60:02}"+ day, fraction = divmod(serial, 1)+ clock = dt.timedelta(milliseconds = round(fraction * 86_400_000))+ if 0 <= serial < 1 and clock.days == 0:+ return (dt.datetime.min + clock).time().isoformat()+ if not date1904 and 0 < serial < 60:+ day += 1 # Excel counts a 29 February 1900 that never was.+ base = dt.datetime(1904, 1, 1) if date1904 else dt.datetime(1899, 12, 30)+ try:+ moment = base + dt.timedelta(days = day) + clock+ except (OverflowError, ValueError):+ return _number(serial)+ return moment.date().isoformat() if moment.time() == dt.time() else moment.isoformat(" ")+++# Built-in numeric formats ([ECMA-376] 18.8.30); fractions and text are left as stored.+_BUILTIN_NUMBER_FORMATS = {+ 1: "0",+ 2: "0.00",+ 3: "#,##0",+ 4: "#,##0.00",+ 9: "0%",+ 10: "0.00%",+ 11: "0.00E+00",+ 37: "#,##0 ;(#,##0)",+ 38: "#,##0 ;[Red](#,##0)",+ 39: "#,##0.00;(#,##0.00)",+ 40: "#,##0.00;[Red](#,##0.00)",+ 48: "##0.0E+0",+}+_MAX_FORMAT_CHARS = 1024+_FORMAT_LITERALS = frozenset(" $-+/():!^&'~{}<>=\u20ac\u00a3\u00a5")+++def _format_tokens(code: str) -> list[tuple[str, str]] | None:+ """("lit" | "ph" | "exp" | "sep", text) tokens of a number format, or None if unsupported."""+ tokens, i = [], 0+ while i < len(code):+ ch = code[i]+ if ch == '"':+ end = code.find('"', i + 1)+ end = len(code) if end < 0 else end+ tokens.append(("lit", code[i + 1 : end]))+ i = end + 1+ elif ch == "\\" and i + 1 < len(code):+ tokens.append(("lit", code[i + 1]))+ i += 2+ elif ch == "[":+ end = code.find("]", i)+ inner = code[i + 1 : end] if end > 0 else ""+ if inner.startswith("$"): # currency and locale, such as [$EUR-407]+ tokens.append(("lit", inner[1:].split("-")[0]))+ elif not inner.isalnum(): # conditions such as [>100]+ return None+ i = end + 1 # colours are dropped+ elif ch == "_": # space the width of the next character+ tokens.append(("lit", " "))+ i += 2+ elif ch == "*": # fill with the next character+ i += 2+ elif ch in "0#?.,%":+ tokens.append(("ph", ch))+ i += 1+ elif ch in "Ee" and code[i + 1 : i + 2] in ("+", "-"):+ tokens.append(("exp", code[i + 1]))+ i += 2+ elif ch == ";":+ tokens.append(("sep", ";"))+ i += 1+ elif ch in _FORMAT_LITERALS:+ tokens.append(("lit", ch))+ i += 1+ else:+ return None+ return tokens+++def _round(value: float, places: int) -> str:+ # Excel rounds halves away from zero.+ quantum = decimal.Decimal(1).scaleb(-places)+ return format(decimal.Decimal(repr(value)).quantize(quantum, decimal.ROUND_HALF_UP), "f")+++def _group(digits: str) -> str:+ head = len(digits) % 3 or 3+ return ",".join([digits[:head]] + [digits[i : i + 3] for i in range(head, len(digits), 3)])+++def _format_section(value: float, tokens: list[tuple[str, str]]) -> str | None:+ marks = [i for i, (kind, _) in enumerate(tokens) if kind in ("ph", "exp")]+ if not any(kind == "ph" and ch in "0#?" for kind, ch in tokens):+ return "".join(text for kind, text in tokens if kind == "lit")+ first, last = marks[0], marks[-1]+ if any(kind == "lit" for kind, _ in tokens[first:last]):+ return None+ prefix = "".join(text for kind, text in tokens[:first] if kind == "lit")+ suffix = "".join(text for kind, text in tokens[last + 1 :] if kind == "lit")+ pattern = "".join(ch if kind == "ph" else "E" + ch for kind, ch in tokens[first : last + 1])+ percent = "%" * pattern.count("%")+ value *= 100 ** len(percent)+ pattern = pattern.replace("%", "")+ mantissa, _, exponent = pattern.partition("E")+ sign_style, exponent = (exponent[:1], exponent[1:]) if exponent else ("", "")+ whole, dot, fraction = mantissa.partition(".")+ while whole.endswith(","): # trailing commas scale by thousands+ whole, value = whole[:-1], value / 1000+ grouped, whole = "," in whole, whole.replace(",", "")+ places, required = sum(fraction.count(c) for c in "0#?"), fraction.count("0")+ min_whole = whole.count("0")+ power = 0+ if sign_style:+ width = max(len(whole), 1)+ if value:+ power = math.floor(math.log10(value))+ power = power - power % width if "#" in whole else power - (max(min_whole, 1) - 1)+ if float(_round(value / 10**power, places)) >= 10 ** max(width, min_whole, 1):+ power += width if "#" in whole else 1+ value /= 10**power+ digits = _round(value, places)+ int_part, _, frac_part = digits.partition(".")+ while len(frac_part) > required and frac_part.endswith("0"):+ frac_part = frac_part[:-1]+ int_part = "" if int_part == "0" and min_whole == 0 else int_part.zfill(min_whole)+ if grouped and int_part:+ int_part = _group(int_part)+ text = int_part + (dot + frac_part if dot else "")+ if sign_style:+ exp_sign = "-" if power < 0 else ("+" if sign_style == "+" else "")+ text += f"E{exp_sign}{str(abs(power)).zfill(exponent.count('0'))}"+ return prefix + text + percent + suffix+++def _format_number(value: float, code: str) -> str | None:+ """``value`` as a numeric format displays it, or None for formats left as stored."""+ tokens = _format_tokens(code) if code else None+ if not tokens:+ return None+ sections: list[list[tuple[str, str]]] = [[]]+ for token in tokens:+ if token[0] == "sep":+ sections.append([])+ else:+ sections[-1].append(token)+ if value < 0 and len(sections) > 1 and sections[1]:+ return _format_section(-value, sections[1])+ if value == 0 and len(sections) > 2 and sections[2]:+ return _format_section(0.0, sections[2])+ text = _format_section(abs(value), sections[0])+ return None if text is None else ("-" + text if value < 0 else text)+++def _cell_number(value: float, format_id: int, code: str, date1904: bool) -> str:+ """A numeric cell as its format displays it, falling back to the stored number."""+ if len(code) > _MAX_FORMAT_CHARS: # tokenized per cell+ code = ""+ kind = _format_kind(format_id, code)+ if kind:+ return _serial_text(value, kind, date1904)+ try:+ text = _format_number(value, code or _BUILTIN_NUMBER_FORMATS.get(format_id, ""))+ except (ArithmeticError, ValueError, decimal.InvalidOperation):+ text = None+ return _number(value) if text is None else text+++def _column_index(ref: str) -> int:+ index = 0+ for ch in ref:+ if not ch.isalpha():+ break+ index = index * 26 + (ord(ch.upper()) - 64)+ return index - 1+++def _xlsx_text(node: ET.Element) -> str:+ # Plain <t> or rich runs <r><t>; phonetic hints (<rPh>) are skipped.+ parts = []+ for child in node:+ if child.tag == _q("s", "t"):+ parts.append(child.text or "")+ elif child.tag == _q("s", "r"):+ t = child.find(_q("s", "t"))+ parts.append((t.text or "") if t is not None else "")+ return "".join(parts)+++def xlsx(path: str) -> list[Section]:+ with _Archive(path) as zf:+ workbook_part = zf.related("", "officeDocument", "xl/workbook.xml")+ if workbook_part is None:+ raise ValueError("missing archive member: xl/workbook.xml")+ workbook = zf.xml(workbook_part)+ pr = workbook.find(_q("s", "workbookPr"))+ date1904 = pr is not None and pr.get("date1904") in ("1", "true")+ rels = zf.rels(workbook_part)+ folder = posixpath.dirname(workbook_part)++ strings: list[str] = []+ strings_part = zf.related(workbook_part, "sharedStrings", f"{folder}/sharedStrings.xml")+ if strings_part:+ strings = [+ _xlsx_text(si)+ for event, si in zf.stream(strings_part, frozenset([_q("s", "si")]))+ if event == "unit"+ ]++ style_formats: list[tuple[int, str]] = []+ styles_part = zf.related(workbook_part, "styles", f"{folder}/styles.xml")+ if styles_part:+ styles = zf.xml(styles_part)+ custom = {+ int(f.get("numFmtId", "-1")): f.get("formatCode", "")+ for f in styles.iter(_q("s", "numFmt"))+ }+ xfs = styles.find(_q("s", "cellXfs"))+ for xf in xfs.findall(_q("s", "xf")) if xfs is not None else []:+ fmt = int(xf.get("numFmtId", "0") or 0)+ style_formats.append((fmt, custom.get(fmt, "")))++ sections: list[Section] = []+ seen: set[int] = set()+ budget = _Budget()+ for sheet in workbook.iter(_q("s", "sheet")):+ target = rels.get(sheet.get(_q("r", "id"), ""))+ if not target or not zf.has(target):+ continue+ rows = []+ for event, row in zf.stream(target, frozenset([_q("s", "row")])):+ if event != "unit":+ continue+ cells: list[str] = []+ for c in row.findall(_q("s", "c")):+ col = _column_index(c.get("r", "")) if c.get("r") else len(cells)+ if col < 0 or col >= _MAX_COLUMNS:+ continue+ kind, v = c.get("t", "n"), c.find(_q("s", "v"))+ raw = v.text if v is not None and v.text is not None else ""+ if kind == "s":+ value = _shared(strings, int(raw) if raw.isdigit() else -1, seen, budget)+ elif kind == "inlineStr":+ inline = c.find(_q("s", "is"))+ value = _xlsx_text(inline) if inline is not None else ""+ elif kind == "b":+ value = "TRUE" if raw == "1" else "FALSE"+ elif kind in ("str", "e"):+ value = raw+ else:+ try:+ number = float(raw)+ except ValueError:+ value = raw+ else:+ style = int(c.get("s", "0") or 0)+ fmt, code = (+ style_formats[style] if style < len(style_formats) else (0, "")+ )+ value = _cell_number(number, fmt, code, date1904)+ cells.extend([""] * (col - len(cells)))+ if col == len(cells):+ cells.append(value)+ line = _row(cells)+ if line:+ rows.append(line)+ if rows:+ sections.append((f"Sheet: {sheet.get('name', '')}\n" + "\n".join(rows), None))+ return sections+++def _drawing_lines(root: ET.Element) -> list[str]:+ """Paragraphs of a DrawingML part in document order; table rows as one line each."""+ lines: list[str] = []+ in_table: set[ET.Element] = set()+ for node in root.iter():+ if node.tag == _q("a", "tbl"):+ for tr in node.iter(_q("a", "tr")):+ cells = []+ for tc in tr.findall(_q("a", "tc")):+ texts = (_drawing_paragraph(p) for p in tc.iter(_q("a", "p")))+ cells.append(" ".join(t for t in texts if t.strip()))+ line = _row(cells)+ if line:+ lines.append(line)+ in_table.update(node.iter(_q("a", "p")))+ elif node.tag == _q("a", "p") and node not in in_table:+ text = _drawing_paragraph(node)+ if text.strip():+ lines.append(text)+ return lines+++def _drawing_paragraph(p: ET.Element) -> str:+ parts = []+ for node in p.iter():+ if node.tag == _q("a", "t"):+ parts.append(node.text or "")+ elif node.tag == _q("a", "br"):+ parts.append("\n")+ return "".join(parts)+++def _without_slide_number(root: ET.Element) -> ET.Element:+ """Drops the notes page's slide number placeholder."""+ ph_path = f"{_q('p', 'nvSpPr')}/{_q('p', 'nvPr')}/{_q('p', 'ph')}"+ doomed = [+ (parent, sp)+ for parent in root.iter()+ for sp in parent.findall(_q("p", "sp"))+ if (ph := sp.find(ph_path)) is not None and ph.get("type") == "sldNum"+ ]+ for parent, sp in doomed:+ parent.remove(sp)+ return root+++def _chart_lines(root: ET.Element) -> list[str]:+ """Chart title and axis titles, then the cached data as rows of category and values."""+ lines = _drawing_lines(root)+ names, series_categories, values = [], [], []+ for series in root.iter(_q("c", "ser")):+ name = series.find(f"{_q('c', 'tx')}//{_q('c', 'v')}")+ names.append(name.text or "" if name is not None else "")+ categories, points = {}, {}+ # Scatter and bubble charts cache xVal/yVal instead of cat/val.+ for axes, store in ((("cat", "xVal"), categories), (("val", "yVal"), points)):+ for axis in axes:+ for point in series.iterfind(f"{_q('c', axis)}//{_q('c', 'pt')}"):+ v = point.find(_q("c", "v"))+ text = v.text or "" if v is not None else ""+ idx = int(point.get("idx", "0") or 0)+ # Multi-level categories repeat an idx once per c:lvl, innermost level first.+ store[idx] = " ".join(filter(None, (text, store.get(idx))))+ series_categories.append(categories)+ values.append(points)+ if len({tuple(sorted(c.items())) for c in series_categories if c}) > 1:+ # Each series has its own X values: one block per series.+ for name, categories, points in zip(names, series_categories, values):+ lines += [name] if name else []+ for idx in sorted(set(categories) | set(points)):+ line = _row([categories.get(idx, ""), points.get(idx, "")])+ if line:+ lines.append(line)+ return lines+ categories = next((c for c in series_categories if c), {})+ if any(names) and len(names) > 1:+ lines.append(_row(["", *names]).lstrip(" |"))+ for idx in sorted(set(categories) | {i for points in values for i in points}):+ line = _row([categories.get(idx, ""), *(points.get(idx, "") for points in values)])+ if line:+ lines.append(line)+ if any(names) and len(names) == 1:+ lines.insert(len(lines) - len(categories), names[0]) if categories else lines.append(+ names[0]+ )+ return lines+++def pptx(path: str) -> list[Section]:+ with _Archive(path) as zf:+ presentation_part = zf.related("", "officeDocument", "ppt/presentation.xml")+ if presentation_part is None:+ raise ValueError("missing archive member: ppt/presentation.xml")+ presentation = zf.xml(presentation_part)+ rels = zf.rels(presentation_part)+ slide_ids = presentation.find(_q("p", "sldIdLst"))+ sections: list[Section] = []+ for number, sld in enumerate(+ slide_ids.findall(_q("p", "sldId")) if slide_ids is not None else [], 1+ ):+ part = rels.get(sld.get(_q("r", "id"), ""))+ if not part or not zf.has(part):+ continue+ slide = zf.xml(part)+ slide_rels = zf.rels(part)+ lines = _drawing_lines(slide)+ for chart in slide.iter(_q("c", "chart")):+ target = slide_rels.get(chart.get(_q("r", "id"), ""))+ if target and zf.has(target):+ lines += _chart_lines(zf.xml(target))+ # SmartArt labels live in the diagram data part, not the slide.+ for diagram in slide.iter(_q("dgm", "relIds")):+ target = slide_rels.get(diagram.get(_q("r", "dm"), ""))+ if target and zf.has(target):+ lines += _drawing_lines(zf.xml(target))+ notes = next(+ (t for t in slide_rels.values() if "notesslide" in t.lower() and zf.has(t)),+ None,+ )+ if notes:+ note_lines = _drawing_lines(_without_slide_number(zf.xml(notes)))+ if note_lines:+ lines += ["Notes:"] + note_lines+ if lines:+ sections.append(("\n".join(lines), number))+ return sections+++class _Budget:+ """Caps the text repeated cells and rows add, which their source size does not bound."""++ def __init__(self):+ self.left = _MAX_REPEATED_CHARS++ def spend(self, chars: int) -> None:+ self.left -= chars+ if self.left < 0:+ raise ValueError("document repeats too much content")+++def _shared(strings: list[str], index: int, seen: set[int], budget: _Budget) -> str:+ """A shared string cell; reuses are charged, as one string can fill every cell."""+ if not 0 <= index < len(strings):+ return ""+ if index in seen:+ budget.spend(len(strings[index]))+ else:+ seen.add(index)+ return strings[index]+++def _odf_inline(node: ET.Element, budget: _Budget) -> str:+ parts = [node.text or ""]+ for child in node:+ tag = child.tag+ if tag == _q("text", "s"):+ parts.append(" " * min(int(child.get(_q("text", "c"), "1") or 1), 100))+ elif tag == _q("text", "tab"):+ parts.append("\t")+ elif tag == _q("text", "line-break"):+ parts.append("\n")+ elif tag == _q("text", "note"):+ body = child.find(_q("text", "note-body"))+ if body is not None:+ parts.append(" [" + " ".join(_odf_blocks(body, budget)) + "]")+ elif tag == _q("office", "annotation"):+ pass+ else:+ parts.append(_odf_inline(child, budget))+ parts.append(child.tail or "")+ return "".join(parts)+++_ODF_ROW_GROUPS = frozenset(+ _q("table", tag) for tag in ("table-header-rows", "table-rows", "table-row-group")+)+++def _odf_rows(node: ET.Element):+ """Rows of this table, through row groups but not into nested tables."""+ for child in node:+ if child.tag == _q("table", "table-row"):+ yield child+ elif child.tag in _ODF_ROW_GROUPS:+ yield from _odf_rows(child)+++def _odf_repeat(node: ET.Element, attribute: str, text: str, room: int, budget: _Budget) -> int:+ repeat = int(node.get(_q("table", attribute), "1") or 1)+ # Trailing empty cells repeat to the sheet edge; only content is expanded.+ count = max(min(repeat, _MAX_REPEAT, room) if text else min(repeat, 1), 0)+ budget.spend(len(text) * max(count - 1, 0))+ return count+++_ODF_VALUE_ATTRIBUTES = {+ "float": "value",+ "percentage": "value",+ "currency": "value",+ "date": "date-value",+ "time": "time-value",+ "boolean": "boolean-value",+ "string": "string-value",+}+++def _odf_cell_value(cell: ET.Element) -> str:+ """A cell's typed value, for cells written without a text paragraph."""+ kind = cell.get(_q("office", "value-type"), "")+ attribute = _ODF_VALUE_ATTRIBUTES.get(kind)+ value = cell.get(_q("office", attribute), "") if attribute else ""+ if kind == "boolean" and value:+ return "TRUE" if value.lower() == "true" else "FALSE"+ if attribute == "value" and value:+ try:+ return _number(float(value))+ except ValueError:+ pass+ return value+++def _odf_row(row: ET.Element, budget: _Budget) -> list[str]:+ """The row's lines: one, repeated as the row is."""+ cells, gap = [], 0+ for cell in row:+ if cell.tag not in (_q("table", "table-cell"), _q("table", "covered-table-cell")):+ continue+ text = " ".join(_odf_blocks(cell, budget)) or _odf_cell_value(cell)+ if not text:+ # Empty runs before content keep their width; trailing ones are dropped.+ gap += max(int(cell.get(_q("table", "number-columns-repeated"), "1") or 1), 0)+ continue+ cells += [""] * min(gap, _MAX_COLUMNS - len(cells))+ gap = 0+ room = _MAX_COLUMNS - len(cells)+ cells += [text] * _odf_repeat(cell, "number-columns-repeated", text, room, budget)+ if len(cells) >= _MAX_COLUMNS:+ break+ line = _row(cells)+ if not line:+ return []+ return [line] * _odf_repeat(row, "number-rows-repeated", line, _MAX_REPEAT, budget)+++def _odf_table(table: ET.Element, budget: _Budget) -> list[str]:+ return [line for row in _odf_rows(table) for line in _odf_row(row, budget)]+++def _ods_streamed(zf: "_Archive", budget: _Budget) -> list[Section]:+ """Spreadsheet sheets row by row, without holding content.xml as one tree."""+ sections: list[Section] = []+ name, rows = None, []++ def close_sheet():+ if name is not None and rows:+ sections.append((f"Sheet: {name}\n" + "\n".join(rows), None))++ for event, element in zf.stream("content.xml", frozenset([_q("table", "table-row")])):+ if event == "start" and element.tag == _q("table", "table"):+ close_sheet()+ name, rows = element.get(_q("table", "name"), ""), []+ elif event == "unit" and name is not None:+ rows += _odf_row(element, budget)+ close_sheet()+ return sections+++# Annotations and notes are read separately; tracked changes hold deleted text.+_ODF_SKIP = frozenset(+ (_q("office", "annotation"), _q("presentation", "notes"), _q("text", "tracked-changes"))+)+++def _odf_blocks(node: ET.Element, budget: _Budget) -> list[str]:+ lines: list[str] = []+ for child in node:+ if child.tag in (_q("text", "p"), _q("text", "h")):+ text = _odf_inline(child, budget)+ if text.strip():+ lines.append(text)+ elif child.tag == _q("table", "table"):+ lines += _odf_table(child, budget)+ elif child.tag in _ODF_SKIP:+ continue+ else:+ lines += _odf_blocks(child, budget)+ return lines+++def opendocument(path: str) -> list[Section]:+ with _Archive(path) as zf:+ manifest = "META-INF/manifest.xml"+ if zf.has(manifest) and b"encryption-data" in zf.read(manifest):+ raise ValueError("file is password protected")+ budget = _Budget()+ if zf.has("mimetype") and zf.read("mimetype").strip().endswith(+ (b".spreadsheet", b".spreadsheet-template")+ ):+ return _ods_streamed(zf, budget)+ body = zf.xml("content.xml").find(_q("office", "body"))+ if body is None:+ return []+ sections: list[Section] = []+ sheet_doc = body.find(_q("office", "spreadsheet"))+ slides_doc = body.find(_q("office", "presentation"))+ if sheet_doc is not None:+ for table in sheet_doc.findall(_q("table", "table")):+ rows = _odf_table(table, budget)+ if rows:+ name = table.get(_q("table", "name"), "")+ sections.append((f"Sheet: {name}\n" + "\n".join(rows), None))+ elif slides_doc is not None:+ for number, page in enumerate(slides_doc.findall(_q("draw", "page")), 1):+ lines = _odf_blocks(page, budget)+ notes = page.find(_q("presentation", "notes"))+ if notes is not None:+ note_lines = _odf_blocks(notes, budget)+ if note_lines:+ lines += ["Notes:"] + note_lines+ if lines:+ sections.append(("\n".join(lines), number))+ else:+ lines = _odf_blocks(body, budget)+ if lines:+ sections.append(("\n".join(lines), None))+ return sections+++def epub(path: str, html_text) -> list[Section]:+ """Chapters in spine order; ``html_text`` turns one XHTML document into text."""+ with _Archive(path) as zf:+ container = zf.xml("META-INF/container.xml")+ rootfile = next(container.iter(_q("container", "rootfile")), None)+ if rootfile is None or not rootfile.get("full-path"):+ raise ValueError("epub has no package document")+ # A URL like the manifest hrefs (EPUB 3.3 4.2.6.3.1), so "My%20Book.opf" names "My Book.opf".+ opf_path = posixpath.normpath(unquote(rootfile.get("full-path")).lstrip("/"))+ opf = zf.xml(opf_path)+ folder = posixpath.dirname(opf_path)+ # DRM encrypted resources; obfuscated fonts are listed too but are never in the spine.+ encrypted = set()+ if zf.has("META-INF/encryption.xml"):+ for node in zf.xml("META-INF/encryption.xml").iter():+ if node.tag.endswith("}CipherReference") and node.get("URI"):+ encrypted.add(posixpath.normpath(unquote(node.get("URI")).lstrip("/")))+ manifest = {item.get("id"): item for item in opf.iter(_q("opf", "item")) if item.get("id")}+ sections: list[Section] = []+ for ref in opf.iter(_q("opf", "itemref")):+ item = manifest.get(ref.get("idref"))+ if item is None or "html" not in (item.get("media-type") or ""):+ continue+ # The navigation document only repeats the chapter titles.+ if "nav" in (item.get("properties") or "").split():+ continue+ href = unquote(item.get("href", "").split("#", 1)[0])+ member = posixpath.normpath(posixpath.join(folder, href))+ if member in encrypted:+ raise ValueError("file is DRM protected")+ if not zf.has(member):+ continue+ text = html_text(zf.read(member)).strip()+ if text:+ sections.append((text, None))+ return sections+++def eml(path: str, html_text) -> list[Section]:+ with open(path, "rb") as f:+ message = email.message_from_binary_file(f, policy = email.policy.default)+ return [(_email_text(message, html_text), None)]+++def _email_parts(part, html_text) -> list[str]:+ """Inline body text in order: every part of a mixed message, one choice per alternative."""+ if part.get_content_disposition() == "attachment":+ return []+ if part.get_content_maintype() == "multipart":+ children = list(part.iter_parts())+ if part.get_content_subtype() != "alternative":+ return [text for child in children for text in _email_parts(child, html_text)]+ # Plain text first; some mailers leave it empty beside a full HTML part.+ for child in sorted(children, key = lambda c: c.get_content_type() != "text/plain"):+ texts = _email_parts(child, html_text)+ if any(t.strip() for t in texts):+ return texts+ return []+ if part.get_content_type() not in ("text/plain", "text/html"):+ return []+ try:+ content = part.get_content()+ except (LookupError, ValueError):+ content = part.get_payload(decode = True) or b""+ content = content.decode("utf-8", "replace") if isinstance(content, bytes) else content+ if part.get_content_subtype() == "html":+ return [html_text(content.encode("utf-8") if isinstance(content, str) else content)]+ return [content]+++def _email_text(message, html_text) -> str:+ lines = []+ for header in ("From", "To", "Cc", "Date", "Subject"):+ value = message.get(header)+ if value:+ lines.append(f"{header}: {value}")+ text = "\n\n".join(t.strip() for t in _email_parts(message, html_text) if t.strip())+ names = [part.get_filename() for part in message.iter_attachments() if part.get_filename()]+ if names:+ lines.append("Attachments: " + ", ".join(names))+ return "\n".join(lines) + "\n\n" + text.strip()+++_RTF_TOKEN = re.compile(+ r"\\([a-zA-Z]{1,32})(-?\d{1,10})? ?|\\'([0-9a-fA-F]{2})|\\(.)|([{}])|[\r\n]+|([^\\{}\r\n]+)",+ re.S,+)+# Destinations whose content is never body text.+_RTF_SKIP = frozenset(+ """+ fonttbl colortbl stylesheet info pict object header headerl headerr headerf footer footerl+ footerr footerf listtable listoverridetable rsidtbl generator filetbl revtbl themedata+ colorschememapping latentstyles datastore xmlnstbl mmathpr pgdsctbl fldinst bkmkstart+ bkmkend picprop shppict nonshppict userprops docvar xform template+ """.split()+)+_RTF_BREAKS = {+ "par": "\n",+ "line": "\n",+ "sect": "\n\n",+ "page": "\n\n",+ "row": "\n",+ "tab": "\t",+ "cell": " | ",+ "emdash": "\u2014",+ "endash": "\u2013",+ "bullet": "\u2022",+ "lquote": "\u2018",+ "rquote": "\u2019",+ "ldblquote": "\u201c",+ "rdblquote": "\u201d",+ "enspace": " ",+ "emspace": " ",+ "qmspace": " ",+}+++_RTF_MAX_DEPTH = 1000+# Font \fcharset -> code page. 0 (ANSI) and 1 (default) follow the document's \ansicpg.+_RTF_CHARSETS = {+ 77: "mac_roman",+ 128: "cp932",+ 129: "cp949",+ 130: "johab",+ 134: "cp936",+ 136: "cp950",+ 161: "cp1253",+ 162: "cp1254",+ 163: "cp1258",+ 177: "cp1255",+ 178: "cp1256",+ 186: "cp1257",+ 204: "cp1251",+ 222: "cp874",+ 238: "cp1250",+ 254: "cp437",+ 255: "cp850",+}+++_RTF_DOCUMENT_CHARSETS = {"mac": "mac_roman", "pc": "cp437", "pca": "cp850"}+++def _rtf_codec(code_page: str) -> str | None:+ code_page = "mac_roman" if code_page == "10000" else f"cp{code_page}"+ # b"".decode() never looks the codec up.+ try:+ codecs.lookup(code_page)+ except LookupError:+ return None+ return code_page+++def _rtf_font_codecs(data: str) -> dict[int, str]:+ """Font number -> codec, from each font table entry's cpg or fcharset."""+ start = data.find("{\\fonttbl")+ if start < 0:+ return {}+ depth, end = 0, start+ for end in range(start, min(len(data), start + 1_000_000)):+ if data[end] in "{}" and data[end - 1] != "\\":+ depth += 1 if data[end] == "{" else -1+ if depth == 0:+ break+ table = data[start:end]+ fonts = {}+ entries = list(re.finditer(r"\\f(\d+)", table))+ for entry, following in zip(entries, entries[1:] + [None]):+ segment = table[entry.end() : following.start() if following else len(table)]+ cpg = re.search(r"\\cpg(\d+)", segment)+ charset = re.search(r"\\fcharset(\d+)", segment)+ codec = _rtf_codec(cpg.group(1)) if cpg else None+ if codec is None and charset:+ codec = _RTF_CHARSETS.get(int(charset.group(1)))+ if codec:+ fonts[int(entry.group(1))] = codec+ return fonts+++def rtf(path: str) -> list[Section]:+ with open(path, "rb") as f:+ data = f.read().decode("latin-1")+ if not data.lstrip().startswith("{\\rtf"):+ raise ValueError("file is not RTF")+ return [(_rtf_text(data), None)]+++def _rtf_text(data: str) -> str:+ out: list[str] = []+ pending = bytearray()+ # Bytes decode in the active font's code page, else the document's.+ fonts = _rtf_font_codecs(data)+ document_codepage, default_font = "cp1252", None+ codepage = document_codepage+ stack: list[tuple[bool, int, bool, str]] = []+ skip, uc, to_skip = False, 1, 0+ deleted = False # tracked deletion+ ignorable = False+ upr = False # the next group is \upr's ANSI branch++ def flush():+ if pending:+ out.append(pending.decode(codepage, "replace"))+ pending.clear()++ pos = 0+ while pos < len(data):+ m = _RTF_TOKEN.match(data, pos)+ if m is None:+ pos += 1+ continue+ pos = m.end()+ word, arg, hex_byte, symbol, brace, text = m.groups()+ if to_skip and (hex_byte or text or symbol):+ # Fallback characters after \uN.+ if text and len(text) > to_skip:+ text, to_skip = text[to_skip:], 0+ else:+ to_skip -= len(text) if text else 1+ continue+ if brace == "{":+ flush()+ if len(stack) >= _RTF_MAX_DEPTH:+ raise ValueError("RTF groups are nested too deeply")+ stack.append((skip, uc, deleted, codepage))+ ignorable = False+ if upr:+ # \upr{ANSI}{\*\ud{Unicode}}: the Unicode branch follows.+ skip, upr = True, False+ elif brace == "}":+ flush()+ skip, uc, deleted, codepage = stack.pop() if stack else (False, 1, False, codepage)+ to_skip, upr = 0, False+ elif symbol is not None:+ if symbol == "*":+ ignorable = True+ elif not (skip or deleted):+ flush()+ out.append(+ {"~": "\u00a0", "_": "-", "-": "", "\n": "\n", "\r": "\n"}.get(symbol, symbol)+ )+ elif word is not None:+ if word == "bin":+ # Raw bytes follow; a negative length would rewind and loop forever.+ pos = min(len(data), pos + max(int(arg or 0), 0))+ continue+ if word == "ansicpg" and arg:+ document_codepage = _rtf_codec(arg) or document_codepage+ codepage = fonts.get(default_font, document_codepage)+ elif word in _RTF_DOCUMENT_CHARSETS:+ document_codepage = _RTF_DOCUMENT_CHARSETS[word]+ codepage = fonts.get(default_font, document_codepage)+ elif word == "deff" and arg:+ default_font = int(arg)+ codepage = fonts.get(default_font, document_codepage)+ elif word == "uc" and arg:+ uc = int(arg)+ elif word == "deleted":+ deleted = arg != "0"+ elif word == "upr":+ upr = True+ elif word == "ud":+ ignorable = False+ elif word == "plain":+ flush()+ deleted = False+ codepage = fonts.get(default_font, document_codepage)+ if ignorable or word in _RTF_SKIP:+ skip, ignorable = True, False+ continue+ if skip:+ continue+ if word == "f" and arg:+ flush()+ codepage = fonts.get(int(arg), document_codepage)+ elif word == "u" and arg:+ flush()+ code = int(arg)+ if not deleted:+ out.append(chr(code + 65536 if code < 0 else code))+ to_skip = uc+ elif word in _RTF_BREAKS:+ flush()+ out.append(_RTF_BREAKS[word])+ elif hex_byte is not None:+ if not (skip or deleted):+ pending.append(int(hex_byte, 16))+ elif text is not None and not (skip or deleted):+ # Literal 8-bit text is in the document code page, like \'hh escapes.+ pending += text.encode("latin-1")+ flush()+ # \uN gives astral characters as two surrogates; pair them, and replace any left alone.+ text = "".join(out).encode("utf-16-le", "surrogatepass").decode("utf-16-le", "replace")+ text = re.sub(r"( \| )+\n", "\n", text)+ return re.sub(r"[ \t]+\n", "\n", text).strip()+++def _compound(path: str) -> CompoundFile:+ with open(path, "rb") as f:+ data = f.read()+ try:+ return CompoundFile(data)+ except CompoundFileError as exc:+ raise ValueError(str(exc)) from exc+++def _clean_control(text: str) -> str:+ return re.sub(r"[\x00-\x08\x0e-\x1f]", "", text)+++def doc(path: str) -> list[Section]:+ cf = _compound(path)+ try:+ word = cf.read("WordDocument")+ except CompoundFileError as exc:+ raise ValueError("not a Word 97-2003 document") from exc+ if len(word) < 0x200 or struct.unpack_from("<H", word, 0)[0] != 0xA5EC:+ raise ValueError("not a Word 97-2003 document")+ flags = struct.unpack_from("<H", word, 0x0A)[0]+ if flags & 0x0100:+ raise ValueError("file is password protected")+ table = cf.read("1Table" if flags & 0x0200 else "0Table")++ # FibRgLw97 story lengths and FibRgFcLcb97.fcClx/lcbClx, located by the FIB's own counts.+ csw = struct.unpack_from("<H", word, 32)[0]+ lw_start = 34 + csw * 2 + 2+ cslw = struct.unpack_from("<H", word, 34 + csw * 2)[0]+ # ccpText, ccpFtn, ccpHdd, ccpMcr, ccpAtn, ccpEdn, ccpTxbx+ count = max(min(cslw, 10) - 3, 1)+ ccps = [max(n, 0) for n in struct.unpack_from(f"<{count}i", word, lw_start + 12)]+ ccps += [0] * (7 - len(ccps))+ fc_start = lw_start + cslw * 4 + 2+ fc_clx, lcb_clx = struct.unpack_from("<II", word, fc_start + 33 * 8)+ clx = table[fc_clx : fc_clx + lcb_clx]++ pos, pieces = 0, None+ while pos < len(clx):+ if clx[pos] == 0x01: # Prc: skip its property modifiers+ pos += 3 + struct.unpack_from("<H", clx, pos + 1)[0]+ elif clx[pos] == 0x02: # Pcdt: the piece table+ size = struct.unpack_from("<I", clx, pos + 1)[0]+ plc = clx[pos + 5 : pos + 5 + size]+ n = (len(plc) - 4) // 12+ cps = struct.unpack_from(f"<{n + 1}I", plc, 0)+ pieces = [+ (cps[i], cps[i + 1], struct.unpack_from("<I", plc, (n + 1) * 4 + i * 8 + 2)[0])+ for i in range(n)+ ]+ break+ else:+ break+ if pieces is None:+ raise ValueError("Word document has no piece table")++ # Pieces never overlap in a real file, so all text fits in the stream's bytes.+ room = [len(word)]++ def story(first: int, length: int) -> str:+ chars: list[str] = []+ for start, end, fc in pieces:+ lo, hi = max(start, first), min(end, first + length)+ if hi <= lo:+ continue+ room[0] -= hi - lo+ if room[0] < 0:+ raise ValueError("document repeats too much content")+ if fc & 0x40000000:+ offset = (fc & 0x3FFFFFFF) // 2 + (lo - start)+ chars.append(word[offset : offset + hi - lo].decode("cp1252", "replace"))+ else:+ offset = fc + 2 * (lo - start)+ chars.append(word[offset : offset + 2 * (hi - lo)].decode("utf-16-le", "replace"))+ return _word_text("".join(chars))++ # Stories follow the main text in this order; headers and comments are left out, as for .docx.+ starts = [sum(ccps[:i]) for i in range(len(ccps))]+ text, ftn, _hdd, _mcr, _atn, edn, txbx = range(7)+ parts = [story(0, ccps[text]), story(starts[txbx], ccps[txbx])]+ for label, kind in (("Footnotes", ftn), ("Endnotes", edn)):+ notes = story(starts[kind], ccps[kind])+ if notes:+ parts.append(f"{label}:\n{notes}")+ return [("\n\n".join(p for p in parts if p), None)]+++def _word_text(raw: str) -> str:+ # Fields: keep the result (after 0x14), drop the instruction (before it).+ out, depth_in_code = [], []+ for ch in raw:+ if ch == "\x13":+ depth_in_code.append(True)+ elif ch == "\x14":+ if depth_in_code:+ depth_in_code[-1] = False+ elif ch == "\x15":+ if depth_in_code:+ depth_in_code.pop()+ elif not any(depth_in_code):+ out.append(ch)+ # A cell ends in 0x07; a row ends in one more.+ text = "".join(out).replace("\x07\x07", "\n").replace("\x07", " | ")+ text = text.translate({0x0D: "\n", 0x0B: "\n", 0x0C: "\n", 0x1E: "-", 0x1F: None})+ text = re.sub(r"\n{3,}", "\n\n", text)+ return _clean_control(text).strip()+++def _rk(value: int) -> float:+ if value & 2:+ number = float(value >> 2 if not value & 0x80000000 else (value >> 2) - (1 << 30))+ else:+ number = struct.unpack("<d", struct.pack("<Q", (value & 0xFFFFFFFC) << 32))[0]+ return number / 100 if value & 1 else number+++class _Records:+ def __init__(+ self,+ data: bytes,+ pos: int = 0,+ ):+ self.data, self.pos = data, pos++ def __iter__(self):+ data = self.data+ while self.pos + 4 <= len(data):+ kind, size = struct.unpack_from("<HH", data, self.pos)+ body = data[self.pos + 4 : self.pos + 4 + size]+ self.pos += 4 + size+ yield kind, body+++def _xls_string(segments: list[bytes], seg: int, pos: int) -> tuple[str, int, int]:+ """One XLUnicodeRichExtendedString, which may continue across CONTINUE records."""++ def take(n):+ nonlocal seg, pos+ out = b""+ while n > 0:+ if pos >= len(segments[seg]):+ seg, pos = seg + 1, 0+ if seg >= len(segments):+ raise ValueError("truncated string table")+ chunk = segments[seg][pos : pos + n]+ out += chunk+ pos += len(chunk)+ n -= len(chunk)+ return out++ if pos >= len(segments[seg]):+ seg, pos = seg + 1, 0+ count = struct.unpack("<H", take(2))[0]+ flags = take(1)[0]+ runs = struct.unpack("<H", take(2))[0] if flags & 0x08 else 0+ ext = struct.unpack("<I", take(4))[0] if flags & 0x04 else 0+ parts, wide = [], flags & 0x01+ while count > 0:+ if pos >= len(segments[seg]):+ # A continued string restates its width in the CONTINUE record's first byte.+ seg, pos = seg + 1, 0+ if seg >= len(segments):+ raise ValueError("truncated string table")+ wide = segments[seg][0] & 0x01+ pos = 1+ width = 2 if wide else 1+ avail = (len(segments[seg]) - pos) // width+ n = min(count, max(avail, 1))+ raw = segments[seg][pos : pos + n * width]+ pos += n * width+ parts.append(raw.decode("utf-16-le" if wide else "latin-1", "replace"))+ count -= n+ take(4 * runs + ext)+ return "".join(parts), seg, pos+++def _xls_short_string(body: bytes, pos: int, length_bytes: int) -> str:+ count = body[pos] if length_bytes == 1 else struct.unpack_from("<H", body, pos)[0]+ pos += length_bytes+ wide = body[pos] & 0x01+ pos += 1+ raw = body[pos : pos + count * (2 if wide else 1)]+ return raw.decode("utf-16-le" if wide else "latin-1", "replace")+++_XLS_ERRORS = {+ 0x00: "#NULL!",+ 0x07: "#DIV/0!",+ 0x0F: "#VALUE!",+ 0x17: "#REF!",+ 0x1D: "#NAME?",+ 0x24: "#NUM!",+ 0x2A: "#N/A",+ 0x2B: "#GETTING_DATA",+}+++def xls(path: str) -> list[Section]:+ cf = _compound(path)+ stream = next((name for name in ("Workbook", "Book") if cf.exists(name)), None)+ if stream is None:+ raise ValueError("not an Excel 97-2003 workbook")+ data = cf.read(stream)+ # BIFF5/7 (Excel 95) strings are CODEPAGE bytes with no BIFF8 flags byte.+ biff8 = data[4:6] != b"\x00\x05"+ codec = "cp1252"++ def short(body: bytes, pos: int, length_bytes: int) -> str:+ if biff8:+ return _xls_short_string(body, pos, length_bytes)+ count = body[pos] if length_bytes == 1 else struct.unpack_from("<H", body, pos)[0]+ start = pos + length_bytes+ return body[start : start + count].decode(codec, "replace")++ sheets: list[tuple[str, int]] = []+ strings: list[str] = []+ seen: set[int] = set()+ budget = _Budget()+ sst: list[bytes] | None = None # SST body and its CONTINUE records+ sst_count = 0+ formats: dict[int, str] = {}+ xf_formats: list[int] = []+ date1904 = False+ for kind, body in _Records(data):+ if sst is not None and kind != 0x003C:+ seg = pos = 0+ for _ in range(sst_count):+ text, seg, pos = _xls_string(sst, seg, pos)+ strings.append(text)+ sst = None+ if kind == 0x002F:+ raise ValueError("file is password protected")+ if kind == 0x0042 and len(body) >= 2 and not biff8: # CODEPAGE+ codec = _code_page_codec(struct.unpack_from("<H", body)[0])+ elif kind == 0x0085 and len(body) >= 8 and body[5] == 0: # BOUNDSHEET, worksheet+ sheets.append((short(body, 6, 1), struct.unpack_from("<I", body, 0)[0]))+ elif kind == 0x00FC and len(body) >= 8: # SST+ sst_count, sst = struct.unpack_from("<I", body, 4)[0], [body[8:]]+ elif kind == 0x003C and sst is not None:+ sst.append(body)+ elif kind == 0x041E and len(body) >= 5: # FORMAT+ formats[struct.unpack_from("<H", body)[0]] = short(body, 2, 2 if biff8 else 1)+ elif kind == 0x00E0 and len(body) >= 4: # XF+ xf_formats.append(struct.unpack_from("<H", body, 2)[0])+ elif kind == 0x0022 and len(body) >= 2: # DATEMODE+ date1904 = struct.unpack_from("<H", body)[0] == 1+ elif kind == 0x000A: # end of the workbook globals+ break++ def number(value: float, xf: int) -> str:+ fmt = xf_formats[xf] if xf < len(xf_formats) else 0+ return _cell_number(value, fmt, formats.get(fmt, ""), date1904)++ sections: list[Section] = []+ # BIFF cell records are a few bytes each, so 200 MB of them would hold tens of millions of+ # Python strings; the same ceiling the XML formats get from _MAX_XML_ELEMENTS.+ cell_count = 0+ for name, offset in sheets:+ cells: dict[int, dict[int, str]] = {}+ last_formula: tuple[int, int] | None = None+ # A long STRING result goes on in CONTINUE records: (cell, segments).+ string: tuple[tuple[int, int], list[bytes]] | None = None++ def put(r, c, value):+ nonlocal cell_count+ if c < _MAX_COLUMNS:+ cell_count += 1+ if cell_count > _MAX_XML_ELEMENTS:+ raise ValueError("spreadsheet has too many cells")+ cells.setdefault(r, {})[c] = value++ def put_string():+ nonlocal string+ if string is not None:+ try:+ text = _xls_string(string[1], 0, 0)[0] if biff8 else short(string[1][0], 0, 2)+ put(*string[0], text)+ except (ValueError, struct.error, IndexError):+ pass+ string = None++ depth = 0+ for kind, body in _Records(data, offset):+ if string is not None:+ if kind == 0x003C:+ string[1].append(body)+ continue+ put_string()+ # Embedded charts nest their own BOF/EOF inside the sheet.+ if kind == 0x0809:+ depth += 1+ elif kind == 0x000A:+ depth -= 1+ if depth <= 0:+ break+ if len(body) < 6 and kind != 0x0207:+ continue+ if kind == 0x00FD: # LABELSST+ r, c, _xf, i = struct.unpack_from("<HHHI", body)+ put(r, c, _shared(strings, i, seen, budget))+ elif kind == 0x0203: # NUMBER+ r, c, xf, v = struct.unpack_from("<HHHd", body)+ put(r, c, number(v, xf))+ elif kind == 0x027E: # RK+ r, c, xf, v = struct.unpack_from("<HHHI", body)+ put(r, c, number(_rk(v), xf))+ elif kind == 0x00BD: # MULRK+ r, c = struct.unpack_from("<HH", body)+ for i in range((len(body) - 6) // 6):+ xf, v = struct.unpack_from("<HI", body, 4 + i * 6)+ put(r, c + i, number(_rk(v), xf))+ elif kind == 0x0204: # LABEL+ r, c, _xf = struct.unpack_from("<HHH", body)+ put(r, c, short(body, 6, 2))+ elif kind == 0x0205: # BOOLERR+ r, c, _xf, v, is_error = struct.unpack_from("<HHHBB", body)+ if not is_error:+ put(r, c, "TRUE" if v else "FALSE")+ elif v in _XLS_ERRORS:+ put(r, c, _XLS_ERRORS[v])+ elif kind == 0x0006 and len(body) >= 14: # FORMULA, cached result+ r, c, xf = struct.unpack_from("<HHH", body)+ result = body[6:14]+ last_formula = None+ if result[6:8] != b"\xff\xff":+ put(r, c, number(struct.unpack("<d", result)[0], xf))+ elif result[0] == 0:+ last_formula = (r, c)+ elif result[0] == 1:+ put(r, c, "TRUE" if result[2] else "FALSE")+ elif result[0] == 2 and result[2] in _XLS_ERRORS:+ put(r, c, _XLS_ERRORS[result[2]])+ elif kind == 0x0207 and last_formula and len(body) >= 3: # STRING after FORMULA+ string, last_formula = (last_formula, [body]), None+ put_string()+ rows = []+ for r in sorted(cells):+ row = cells[r]+ line = _row([row.get(c, "") for c in range(max(row) + 1)])+ if line:+ rows.append(line)+ if rows:+ sections.append((f"Sheet: {name}\n" + "\n".join(rows), None))+ return sections+++_PPT_TEXT_CHARS, _PPT_TEXT_BYTES = 0x0FA0, 0x0FA8+_PPT_TEXT_HEADER, _PPT_OUTLINE_REF = 0x0F9F, 0x0F9E+_PPT_DOCUMENT, _PPT_SLIDE, _PPT_NOTES, _PPT_NOTES_ATOM = 0x03E8, 0x03EE, 0x03F0, 0x03F1+_PPT_SLIDE_LIST, _PPT_SLIDE_PERSIST = 0x0FF0, 0x03F3+_PPT_USER_EDIT, _PPT_PERSIST_DIRECTORY = 0x0FF5, 0x1772+++def _ppt_records(data: bytes, start: int, end: int):+ """(type, instance, body start, body end, is container) of the records directly in a range."""+ pos = start+ while pos + 8 <= end:+ ver_inst, kind, size = struct.unpack_from("<HHI", data, pos)+ body = pos + 8+ yield kind, ver_inst >> 4, body, min(body + size, end), ver_inst & 0x000F == 0x000F+ pos = body + size+++def _ppt_walk(+ data: bytes,+ start: int,+ end: int,+ depth: int = 0,+):+ """Every record in a range, depth first, in stream order."""+ for record in _ppt_records(data, start, end):+ yield record+ if record[4] and depth < 32:+ yield from _ppt_walk(data, record[2], record[3], depth + 1)+++def _ppt_text(data: bytes, kind: int, start: int, end: int) -> str:+ text = data[start:end].decode("utf-16-le" if kind == _PPT_TEXT_CHARS else "latin-1", "replace")+ return _clean_control(text.replace("\r", "\n").replace("\x0b", "\n")).strip()+++def _ppt_record_at(data: bytes, offset: int | None, kind: int) -> tuple[int, int] | None:+ if offset is None or offset + 8 > len(data):+ return None+ _ver_inst, found, size = struct.unpack_from("<HHI", data, offset)+ return (offset + 8, min(offset + 8 + size, len(data))) if found == kind else None+++def _ppt_current(cf: CompoundFile, data: bytes) -> tuple[dict[int, int], int] | None:+ """Persist id -> offset as of the latest save, and the document's persist id.++ Incremental saves append new versions of records, so the stream also holds stale ones;+ the edit chain from "Current User" says which are live.+ """+ if not cf.exists("Current User"):+ return None+ user = cf.read("Current User")+ if len(user) < 20:+ return None+ offset = struct.unpack_from("<I", user, 16)[0]+ persist: dict[int, int] = {}+ document, seen, floor, edits = None, set(), offset, None+ while offset:+ floor = min(floor, offset)+ edit = _ppt_record_at(data, offset, _PPT_USER_EDIT)+ if edit is None or edit[1] - edit[0] < 20 or offset in seen:+ if not seen:+ return None+ # A broken link (some writers point an edit at itself): go on with the next older edit.+ if edits is None:+ edits = [+ b - 8+ for k, _i, b, _e, _c in _ppt_records(data, 0, len(data))+ if k == _PPT_USER_EDIT+ ]+ offset = max((o for o in edits if o < floor), default = 0)+ continue+ seen.add(offset)+ last_edit, directory_offset, document_ref = struct.unpack_from("<III", data, edit[0] + 8)+ document = document_ref if document is None else document+ directory = _ppt_record_at(data, directory_offset, _PPT_PERSIST_DIRECTORY)+ pos, end = directory if directory else (0, 0)+ while pos + 4 <= end:+ entry = struct.unpack_from("<I", data, pos)[0]+ pos += 4+ for i in range(entry >> 20):+ if pos + 4 > end:+ break+ # Older edits come later in the chain and never replace newer offsets.+ persist.setdefault((entry & 0xFFFFF) + i, struct.unpack_from("<I", data, pos)[0])+ pos += 4+ offset = last_edit+ return (persist, document) if document is not None else None+++def _ppt_sheet_lines(data: bytes, start: int, end: int, listed: list[str]) -> list[str]:+ """Text of a slide or notes page in shape order; placeholders point into ``listed``."""+ lines, used = [], set()+ for kind, _inst, body, stop, _container in _ppt_walk(data, start, end):+ if kind == _PPT_OUTLINE_REF and stop - body >= 4:+ index = struct.unpack_from("<I", data, body)[0]+ if index < len(listed) and index not in used:+ used.add(index)+ lines.append(listed[index])+ elif kind in (_PPT_TEXT_CHARS, _PPT_TEXT_BYTES):+ lines.append(_ppt_text(data, kind, body, stop))+ lines += [text for i, text in enumerate(listed) if i not in used]+ # "*" is the slide number field.+ return [line for line in lines if line and line != "*"]+++def ppt(path: str) -> list[Section]:+ cf = _compound(path)+ try:+ data = cf.read("PowerPoint Document")+ except CompoundFileError as exc:+ raise ValueError("not a PowerPoint 97-2003 presentation") from exc+ if cf.exists("EncryptedSummary"):+ raise ValueError("file is password protected")+ current = _ppt_current(cf, data)+ document = current and _ppt_record_at(data, current[0].get(current[1]), _PPT_DOCUMENT)+ if not document:+ return _ppt_scan(data)+ persist = current[0]++ # SlideListWithText instance 0 lists slides with their placeholder text; instance 2 lists notes.+ slides: list[tuple[int, int, list[str]]] = [] # (persist id, slide id, placeholder text)+ notes_refs: list[int] = []+ for kind, inst, body, stop, _container in _ppt_records(data, *document):+ if kind != _PPT_SLIDE_LIST or inst not in (0, 2):+ continue+ for child, _i, child_body, child_stop, _c in _ppt_records(data, body, stop):+ if child == _PPT_SLIDE_PERSIST and child_stop - child_body >= 16:+ ref, _flags, _texts, slide_id = struct.unpack_from("<IIII", data, child_body)+ if inst == 0:+ slides.append((ref, slide_id, []))+ else:+ notes_refs.append(ref)+ elif inst == 0 and slides and child == _PPT_TEXT_HEADER:+ slides[-1][2].append("")+ elif inst == 0 and slides and child in (_PPT_TEXT_CHARS, _PPT_TEXT_BYTES):+ if not slides[-1][2]:+ slides[-1][2].append("")+ slides[-1][2][-1] = _ppt_text(data, child, child_body, child_stop)++ notes: dict[int, list[str]] = {}+ for ref in notes_refs:+ page = _ppt_record_at(data, persist.get(ref), _PPT_NOTES)+ if page is None:+ continue+ atom = next(+ (r for r in _ppt_records(data, *page) if r[0] == _PPT_NOTES_ATOM and r[3] - r[2] >= 4),+ None,+ )+ lines = _ppt_sheet_lines(data, *page, [])+ if atom is not None and lines:+ notes[struct.unpack_from("<I", data, atom[2])[0]] = lines++ sections: list[Section] = []+ for number, (ref, slide_id, listed) in enumerate(slides, 1):+ slide = _ppt_record_at(data, persist.get(ref), _PPT_SLIDE)+ lines = _ppt_sheet_lines(data, *slide, listed) if slide else [t for t in listed if t]+ if slide_id in notes:+ lines += ["Notes:"] + notes[slide_id]+ if lines:+ sections.append(("\n".join(lines), number))+ return sections+++def _ppt_scan(data: bytes) -> list[Section]:+ """Slide text by a linear scan, for files whose edit history cannot be followed."""+ listed: list[list[str]] = [] # slide text from SlideListWithText (instance 0)+ drawn: list[list[str]] = [] # slide text from Slide containers, for files without it+ stack: list[tuple[int, int, int]] = [] # (end, type, instance)+ pos = 0+ while pos + 8 <= len(data):+ while stack and pos >= stack[-1][0]:+ stack.pop()+ ver_inst, kind, size = struct.unpack_from("<HHI", data, pos)+ if ver_inst & 0x000F == 0x000F: # container: descend+ stack.append((pos + 8 + size, kind, ver_inst >> 4))+ if kind == _PPT_SLIDE:+ drawn.append([])+ pos += 8+ continue+ body = data[pos + 8 : pos + 8 + size]+ pos += 8 + size+ in_list = next((inst for _end, t, inst in reversed(stack) if t == _PPT_SLIDE_LIST), None)+ if kind == _PPT_SLIDE_PERSIST and in_list == 0:+ listed.append([])+ elif kind in (_PPT_TEXT_CHARS, _PPT_TEXT_BYTES):+ text = body.decode("utf-16-le" if kind == _PPT_TEXT_CHARS else "latin-1", "replace")+ text = _clean_control(text.replace("\r", "\n").replace("\x0b", "\n")).strip()+ if not text:+ continue+ if in_list == 0 and listed:+ listed[-1].append(text)+ elif in_list is None and any(t == _PPT_SLIDE for _e, t, _i in stack) and drawn:+ drawn[-1].append(text)+ slides = listed if any(listed) else drawn+ return [("\n".join(lines), n) for n, lines in enumerate(slides, 1) if lines]+++_MSG_HEADERS = (("0C1A", "From"), ("0E04", "To"), ("0E03", "Cc"), ("0037", "Subject"))+++# Windows code pages Python names differently from cpNNN.+_CODE_PAGES = {+ 1200: "utf-16-le",+ 1201: "utf-16-be",+ 20127: "ascii",+ 20866: "koi8_r",+ 21866: "koi8_u",+ 50220: "iso2022_jp",+ 50221: "iso2022_jp",+ 50222: "iso2022_jp",+ 51932: "euc_jp",+ 51949: "euc_kr",+ 52936: "hz",+ 54936: "gb18030",+ 65000: "utf-7",+ 65001: "utf-8",+ **{28590 + n: f"iso8859_{n}" for n in range(1, 16)},+}+++def _code_page_codec(code_page: int | None) -> str:+ name = _CODE_PAGES.get(code_page, f"cp{code_page}") if code_page else "cp1252"+ try:+ codecs.lookup(name)+ except LookupError:+ return "cp1252"+ return name+++def _msg_props(cf: CompoundFile) -> dict[int, bytes]:+ """Fixed-size top-level properties: tag -> 8-byte value."""+ if not cf.exists("__properties_version1.0"):+ return {}+ props = cf.read("__properties_version1.0")+ return {+ struct.unpack_from("<I", props, off)[0]: props[off + 8 : off + 16]+ for off in range(32, len(props) - 15, 16)+ }+++def _msg_prop(cf: CompoundFile, prefix: tuple[str, ...], prop: str, codec: str) -> str | None:+ # String8 ("001E") values are in the message code page.+ for kind, kind_codec in (("001F", "utf-16-le"), ("001E", codec)):+ name = f"__substg1.0_{prop}{kind}"+ if cf.exists(*prefix, name):+ return cf.read(*prefix, name).decode(kind_codec, "replace").rstrip("\0")+ return None+++def _msg_date(props: dict[int, bytes]) -> str | None:+ for prop in (0x0039, 0x0E06): # submit / delivery time+ value = props.get((prop << 16) | 0x0040)+ if value is None:+ continue+ ticks = struct.unpack("<Q", value)[0]+ try:+ moment = dt.datetime(1601, 1, 1) + dt.timedelta(microseconds = ticks // 10)+ except OverflowError:+ return None+ return moment.isoformat(" ", "minutes") + " UTC"+ return None+++# [MS-OXRTFCP] dictionary prefill for compressed RTF.+_RTF_PREFILL = (+ b"{\\rtf1\\ansi\\mac\\deff0\\deftab720{\\fonttbl;}{\\f0\\fnil \\froman \\fswiss "+ b"\\fmodern \\fscript \\fdecor MS Sans SerifSymbolArialTimes New RomanCourier"+ b"{\\colortbl\\red0\\green0\\blue0\r\n\\par \\pard\\plain\\f0\\fs20\\b\\i\\u\\tab\\tx"+)+++def _decompress_rtf(data: bytes) -> bytes:+ """PidTagRtfCompressed ([MS-OXRTFCP]) to RTF bytes."""+ if len(data) < 16:+ raise ValueError("truncated compressed RTF")+ comp_size, raw_size, magic = struct.unpack_from("<III", data)+ if raw_size > _MAX_MEMBER_BYTES:+ raise ValueError("compressed RTF body is too large")+ if magic == 0x414C454D: # "MELA": stored uncompressed+ return data[16 : 16 + raw_size]+ if magic != 0x75465A4C: # "LZFu"+ raise ValueError("unknown compressed RTF format")+ window = bytearray(4096)+ window[: len(_RTF_PREFILL)] = _RTF_PREFILL+ write = len(_RTF_PREFILL)+ out = bytearray()+ pos, end = 16, min(len(data), comp_size + 4)+ while pos < end:+ control = data[pos]+ pos += 1+ for bit in range(8):+ # Stop at the declared size: back-references can expand far beyond it.+ if pos >= end or len(out) >= raw_size:+ return bytes(out[:raw_size])+ if control & (1 << bit):+ if pos + 2 > end:+ break+ ref = (data[pos] << 8) | data[pos + 1]+ pos += 2+ offset, length = ref >> 4, (ref & 0xF) + 2+ if offset == write: # end marker+ return bytes(out)+ for k in range(length):+ byte = window[(offset + k) % 4096]+ out.append(byte)+ window[write] = byte+ write = (write + 1) % 4096+ else:+ out.append(data[pos])+ window[write] = data[pos]+ write = (write + 1) % 4096+ pos += 1+ return bytes(out[:raw_size])+++def msg(path: str, html_text) -> list[Section]:+ cf = _compound(path)+ if not any(name.lower().startswith("__substg1.0_") for name in cf.listdir()):+ raise ValueError("not an Outlook message")+ props = _msg_props(cf)+ # PidTagMessageCodepage, else PidTagInternetCodepage.+ code_page = next(+ (+ struct.unpack_from("<I", props[tag])[0]+ for tag in (0x3FFD0003, 0x3FDE0003)+ if tag in props+ ),+ None,+ )+ codec = _code_page_codec(code_page)+ headers = {+ label: (_msg_prop(cf, (), prop, codec) or "").strip() for prop, label in _MSG_HEADERS+ }+ address = (_msg_prop(cf, (), "0C1F", codec) or "").strip()+ # Exchange senders carry an X.500 path; their SMTP address is stored separately.+ if not address or address.startswith("/"):+ address = (+ _msg_prop(cf, (), "5D01", codec) or _msg_prop(cf, (), "5D02", codec) or ""+ ).strip()+ if address and not address.startswith("/") and address != headers["From"]:+ headers["From"] = f"{headers['From']} <{address}>".strip()+ headers["Date"] = _msg_date(props) or ""+ lines = [+ f"{label}: {headers[label]}"+ for label in ("From", "To", "Cc", "Date", "Subject")+ if headers[label]+ ]+ body = (_msg_prop(cf, (), "1000", codec) or "").strip()+ if not body and cf.exists("__substg1.0_10130102"):+ body = html_text(cf.read("__substg1.0_10130102"))+ if not body and (html := _msg_prop(cf, (), "1013", codec)):+ body = html_text(codecs.BOM_UTF8 + html.encode("utf-8"))+ if not body and cf.exists("__substg1.0_10090102"):+ rtf_bytes = _decompress_rtf(cf.read("__substg1.0_10090102"))+ body = _rtf_text(rtf_bytes.decode("latin-1"))+ attachments = []+ for name in cf.listdir():+ if name.lower().startswith("__attach_version1.0_"):+ label = _msg_prop(cf, (name,), "3707", codec) or _msg_prop(cf, (name,), "3704", codec)+ if label:+ attachments.append(label)+ if attachments:+ lines.append("Attachments: " + ", ".join(sorted(attachments)))+ body = (body or "").replace("\r\n", "\n").strip()+ return [("\n".join(lines) + "\n\n" + body, None)]diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.pyindex 5d089a1882c..c8b1d775672 100644--- a/studio/backend/core/rag/parsers.py+++ b/studio/backend/core/rag/parsers.py@@ -15,6 +15,7 @@ import loggingimport osimport reimport secrets+import structfrom dataclasses import dataclassfrom html.parser import HTMLParser@@ -847,12 +848,37 @@ def _docx_table_rows(table) -> list[str]:return rows+_WORD_EXTS = frozenset((".docx", ".docm", ".dotx", ".dotm"))+_WORD_MAIN_TYPES = (+ "application/vnd.ms-word.document.macroEnabled.main+xml",+ "application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml",+ "application/vnd.ms-word.template.macroEnabledTemplate.main+xml",+)+++def _open_word(path: str):+ # docx.Document() rejects the macro and template main parts.+ import docx # noqa: F401 (registers the part types)+ from docx.opc.constants import CONTENT_TYPE as CT+ from docx.opc.part import PartFactory+ from docx.package import Package+ from docx.parts.document import DocumentPart++ for content_type in _WORD_MAIN_TYPES:+ PartFactory.part_type_for.setdefault(content_type, DocumentPart)+ part = Package.open(path).main_document_part+ if part.content_type not in (CT.WML_DOCUMENT_MAIN, *_WORD_MAIN_TYPES):+ raise ValueError(+ f"file '{os.path.basename(path)}' is not a Word file, content type is '{part.content_type}'"+ )+ return part.document++def _docx(path: str) -> list[Page]:- import docxfrom docx.table import Tablefrom docx.text.paragraph import Paragraph- document = docx.Document(path)+ document = _open_word(path)lines: list[str] = []_docx_unwrap_table_controls(document.element.body)label_notes = _docx_mark_notes(document)@@ -1051,10 +1077,14 @@ def parse(path: str, *, want_images: bool = False):pages, images, _total = _pdf(path, want_images)return (pages, images) if want_images else pages- if ext == ".docx":+ if ext in _WORD_EXTS:pages = _docx(path)return (pages, []) if want_images else pages+ if ext in config.DOCUMENT_UPLOAD_EXTS:+ pages = _office_document(path, ext)+ return (pages, []) if want_images else pages+if ext in (".html", ".htm", ".txt", ".md", ".markdown") or ext in config.SOURCE_TEXT_EXTS:is_html = ext in (".html", ".htm")with open(path, "rb") as f:@@ -1071,6 +1101,39 @@ def parse(path: str, *, want_images: bool = False):raise ValueError(f"unsupported file type: {ext}")+def _html_bytes_text(raw: bytes) -> str:+ return "\n\n".join(page.text for page in _html(_decode_text(raw, html = True)))+++def _xhtml(path: str) -> list[tuple[str, int | None]]:+ with open(path, "rb") as f:+ return [(_html_bytes_text(f.read()), None)]+++def _office_document(path: str, ext: str) -> list[Page]:+ from . import office_formats as office++ readers = {+ ".doc": office.doc,+ ".xls": office.xls,+ ".ppt": office.ppt,+ ".rtf": office.rtf,+ ".epub": lambda p: office.epub(p, _html_bytes_text),+ ".msg": lambda p: office.msg(p, _html_bytes_text),+ **dict.fromkeys((".xlsx", ".xlsm", ".xltx", ".xltm"), office.xlsx),+ **dict.fromkeys((".pptx", ".pptm", ".potx", ".potm", ".ppsx", ".ppsm"), office.pptx),+ **dict.fromkeys((".odt", ".ods", ".odp", ".ott", ".ots", ".otp"), office.opendocument),+ # .mht/.mhtml are MIME with an HTML body.+ **dict.fromkeys((".eml", ".mht", ".mhtml"), lambda p: office.eml(p, _html_bytes_text)),+ **dict.fromkeys((".xhtml", ".xht"), _xhtml),+ }+ try:+ sections = readers[ext](path)+ except (KeyError, IndexError, struct.error, UnicodeError, EOFError) as exc:+ raise ValueError(f"could not read {os.path.basename(path)}: file is damaged") from exc+ return [_page(text, number) for text, number in sections if text.strip()]++def parse_text(text: str) -> list[Page]:"""Wrap already-extracted text as a single Page (tests / in-memory ingest)."""return [_page(text, None)]diff --git a/studio/backend/tests/_compound_file.py b/studio/backend/tests/_compound_file.pynew file mode 100644index 00000000000..c2124692a0c--- /dev/null+++ b/studio/backend/tests/_compound_file.py@@ -0,0 +1,126 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Minimal OLE compound file writer (version 3) for building .doc/.xls/.ppt/.msg fixtures."""++from __future__ import annotations++import math+import struct++_SECTOR, _MINI, _CUTOFF = 512, 64, 4096+_FREE, _END, _FATSECT = 0xFFFFFFFF, 0xFFFFFFFE, 0xFFFFFFFD+++def _entry(name: str, kind: int, child: int, right: int, start: int, size: int) -> bytes:+ encoded = (name + "\0").encode("utf-16-le")+ return (+ encoded.ljust(64, b"\0")+ + struct.pack("<HBB", len(encoded), kind, 1)+ + struct.pack("<III", _FREE, right, child)+ + b"\0" * 36+ + struct.pack("<IQ", start, size)+ )+++def compound_file(streams: dict[tuple[str, ...], bytes]) -> bytes:+ """Streams keyed by path; storages are implied. Streams under 4096 bytes use the mini stream."""+ # Directory tree: each storage's children chained through right-sibling links.+ nodes = [{"name": "Root Entry", "kind": 5, "children": []}]+ index = {(): 0}+ for path in streams:+ for depth in range(1, len(path) + 1):+ key = path[:depth]+ if key not in index:+ index[key] = len(nodes)+ kind = 2 if depth == len(path) else 1+ nodes.append({"name": key[-1], "kind": kind, "children": [], "path": key})+ nodes[index[key[:-1]]]["children"].append(index[key])++ mini = b""+ big: list[tuple[int, bytes]] = []+ placement = {}+ for path, data in streams.items():+ i = index[path]+ if len(data) < _CUTOFF:+ placement[i] = ("mini", len(mini) // _MINI, len(data))+ mini += (+ data.ljust(math.ceil(len(data) / _MINI) * _MINI or _MINI, b"\0") if data else b""+ )+ else:+ big.append((i, data))++ def sectors(n_bytes):+ return math.ceil(n_bytes / _SECTOR)++ mini_fat = []+ for i, (where, start, size) in sorted(placement.items(), key = lambda kv: kv[1][1]):+ count = math.ceil(size / _MINI)+ mini_fat += [start + k + 1 for k in range(count - 1)] + ([_END] if count else [])+ mini_fat_bytes = struct.pack(f"<{len(mini_fat)}I", *mini_fat) if mini_fat else b""++ layout = [] # (label, n_sectors)+ n_dir = sectors(len(nodes) * 128)+ layout.append(("dir", n_dir))+ layout.append(("minifat", sectors(len(mini_fat_bytes))))+ layout.append(("ministream", sectors(len(mini))))+ for i, data in big:+ layout.append((i, sectors(len(data))))+ data_sectors = sum(n for _, n in layout)+ n_fat = 1+ while n_fat * 128 < data_sectors + n_fat:+ n_fat += 1++ start_of, cursor = {}, n_fat+ for label, n in layout:+ start_of[label] = cursor if n else _END+ cursor += n+ fat = [_FATSECT] * n_fat+ for label, n in layout:+ fat += [start_of[label] + k + 1 for k in range(n - 1)] + ([_END] if n else [])+ fat += [_FREE] * (n_fat * 128 - len(fat))++ entries = []+ for i, node in enumerate(nodes):+ kids = node["children"]+ child = kids[0] if kids else _FREE+ parent = next((n for n in nodes if i in n["children"]), None)+ right = _FREE+ if parent is not None:+ pos = parent["children"].index(i)+ right = parent["children"][pos + 1] if pos + 1 < len(parent["children"]) else _FREE+ if i == 0:+ start, size = start_of["ministream"], len(mini)+ elif node["kind"] == 2:+ if i in placement:+ start, size = placement[i][1], placement[i][2]+ else:+ start, size = start_of[i], len(streams[node["path"]])+ else:+ start, size = 0, 0+ entries.append(_entry(node["name"], node["kind"], child, right, start, size))+ directory = b"".join(entries)++ header = bytearray(512)+ header[:8] = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"+ struct.pack_into("<HHHHH", header, 0x18, 0x3E, 3, 0xFFFE, 9, 6)+ struct.pack_into("<I", header, 0x2C, n_fat)+ struct.pack_into("<I", header, 0x30, start_of["dir"])+ struct.pack_into("<I", header, 0x38, _CUTOFF)+ struct.pack_into("<II", header, 0x3C, start_of["minifat"], sectors(len(mini_fat_bytes)))+ struct.pack_into("<II", header, 0x44, _END, 0)+ difat = list(range(n_fat)) + [_FREE] * (109 - n_fat)+ struct.pack_into("<109I", header, 0x4C, *difat)++ body = struct.pack(f"<{len(fat)}I", *fat)+ for label, n in layout:+ if label == "dir":+ chunk = directory+ elif label == "minifat":+ chunk = mini_fat_bytes+ elif label == "ministream":+ chunk = mini+ else:+ chunk = streams[nodes[label]["path"]]+ body += chunk.ljust(n * _SECTOR, b"\0")+ return bytes(header) + bodydiff --git a/studio/backend/tests/test_rag_config_upload_exts.py b/studio/backend/tests/test_rag_config_upload_exts.pyindex 5cb82bddf01..5a901f095ae 100644--- a/studio/backend/tests/test_rag_config_upload_exts.py+++ b/studio/backend/tests/test_rag_config_upload_exts.py@@ -29,13 +29,20 @@ def _reload(monkeypatch, value):@pytest.mark.parametrize("value, expected",[- (None, config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS),- ("", config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS),+ (+ None,+ config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS | config.DOCUMENT_UPLOAD_EXTS,+ ),+ ("", config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS | config.DOCUMENT_UPLOAD_EXTS),(" .MD , .Markdown ", {".md", ".markdown"}),("md,pdf", {".md", ".pdf"}),(".md,.exe", {".md"}),(".py,.exe", {".py"}),- (".exe", config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS),+ (".xlsx,.epub", {".xlsx", ".epub"}),+ (+ ".exe",+ config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS | config.DOCUMENT_UPLOAD_EXTS,+ ),],)def test_upload_exts_env(monkeypatch, value, expected):diff --git a/studio/backend/tests/test_rag_office_formats.py b/studio/backend/tests/test_rag_office_formats.pynew file mode 100644index 00000000000..dc79266fe26--- /dev/null+++ b/studio/backend/tests/test_rag_office_formats.py@@ -0,0 +1,1272 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Office, OpenDocument, e-book, email and RTF files: parsing, upload and indexing."""++import datetime as dt+import io+import re+import struct+import zipfile+from email.message import EmailMessage+from xml.etree import ElementTree as ET+from xml.sax.saxutils import escape+from pathlib import Path++import pytest+from fastapi import UploadFile++from core.rag import cfb, config, ingestion, office_formats, parsers, store+from routes.rag import _save_upload+from storage import rag_db++from ._compound_file import compound_file++S = 'xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"'+P = 'xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"'+REL = 'xmlns="http://schemas.openxmlformats.org/package/2006/relationships"'+ODF = (+ 'xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" '+ 'xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" '+ 'xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" '+ 'xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" '+ 'xmlns:presentation="urn:oasis:names:tc:opendocument:xmlns:presentation:1.0"'+)+++def _zip(path, members):+ with zipfile.ZipFile(path, "w") as z:+ for name, data in members.items():+ z.writestr(name, data)+ return path+++def _text(path):+ return "\n".join(page.text for page in parsers.parse(str(path)))+++def build_xlsx(path):+ serial = (dt.date(2026, 3, 31) - dt.date(1899, 12, 30)).days+ return _zip(+ path,+ {+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="Revenue" sheetId="1" r:id="rId1"/></sheets></workbook>',+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',+ "xl/sharedStrings.xml": f"<sst {S}><si><t>Region</t></si>"+ "<si><r><t>Zebra</t></r><r><t>marker</t></r><rPh><t>ignored</t></rPh></si></sst>",+ "xl/styles.xml": f'<styleSheet {S}><cellXfs><xf numFmtId="0"/><xf numFmtId="14"/></cellXfs></styleSheet>',+ "xl/worksheets/sheet1.xml": f"<worksheet {S}><sheetData>"+ '<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1" t="inlineStr"><is><t>Q1</t></is></c></row>'+ f'<row r="2"><c r="A2" t="s"><v>1</v></c><c r="B2"><v>1200.5</v></c><c r="C2" s="1"><v>{serial}</v></c>'+ '<c r="D2" t="b"><v>1</v></c><c r="F2" t="str"><v>formula text</v></c></row>'+ "</sheetData></worksheet>",+ },+ )+++def build_pptx(path):+ run = lambda text: f"<a:p><a:r><a:t>{text}</a:t></a:r></a:p>"+ return _zip(+ path,+ {+ "ppt/presentation.xml": f'<p:presentation {P}><p:sldIdLst><p:sldId id="256" r:id="rId2"/></p:sldIdLst></p:presentation>',+ "ppt/_rels/presentation.xml.rels": f'<Relationships {REL}><Relationship Id="rId2" Target="slides/slide1.xml"/></Relationships>',+ "ppt/slides/slide1.xml": f"<p:sld {P}><p:cSld><p:spTree>"+ f"<p:sp><p:txBody>{run('Quarterly review')}</p:txBody></p:sp>"+ "<p:graphicFrame><a:graphic><a:graphicData><a:tbl>"+ f"<a:tr><a:tc><a:txBody>{run('Region')}</a:txBody></a:tc><a:tc><a:txBody>{run('Q1')}</a:txBody></a:tc></a:tr>"+ f"<a:tr><a:tc><a:txBody>{run('North')}</a:txBody></a:tc><a:tc><a:txBody>{run('1200')}</a:txBody></a:tc></a:tr>"+ "</a:tbl></a:graphicData></a:graphic></p:graphicFrame>"+ f"<p:sp><p:txBody>{run('Revenue increased')}</p:txBody></p:sp>"+ "</p:spTree></p:cSld></p:sld>",+ "ppt/slides/_rels/slide1.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="../notesSlides/notesSlide1.xml"/></Relationships>',+ "ppt/notesSlides/notesSlide1.xml": f"<p:notes {P}><p:cSld><p:spTree>"+ '<p:sp><p:nvSpPr><p:cNvPr id="5" name="Slide Number"/><p:cNvSpPr/><p:nvPr><p:ph type="sldNum" idx="5"/></p:nvPr></p:nvSpPr>'+ '<p:txBody><a:p><a:fld id="{1}" type="slidenum"><a:t>1</a:t></a:fld></a:p></p:txBody></p:sp>'+ f"<p:sp><p:txBody>{run('2026')}{run('Speaker notes zebramarker')}</p:txBody></p:sp>"+ "</p:spTree></p:cSld></p:notes>",+ },+ )+++def _odf(path, body):+ suffix = Path(path).suffix+ kind = {".odt": "text", ".ods": "spreadsheet", ".odp": "presentation"}.get(suffix)+ if kind is None:+ kind = {".ott": "text", ".ots": "spreadsheet", ".otp": "presentation"}[suffix] + "-template"+ return _zip(+ path,+ {+ "mimetype": f"application/vnd.oasis.opendocument.{kind}",+ "content.xml": f"<office:document-content {ODF}><office:body>{body}</office:body></office:document-content>",+ },+ )+++def build_odt(path):+ return _odf(+ path,+ "<office:text><text:h>Quarterly report</text:h>"+ '<text:p>Revenue<text:s text:c="2"/>increased<text:note><text:note-body><text:p>see appendix</text:p></text:note-body></text:note>.</text:p>'+ "<table:table><table:table-row><table:table-cell><text:p>Zebramarker</text:p></table:table-cell>"+ "<table:table-cell><text:p>1200</text:p></table:table-cell></table:table-row></table:table></office:text>",+ )+++def build_ods(path):+ return _odf(+ path,+ '<office:spreadsheet><table:table table:name="Revenue">'+ "<table:table-row><table:table-cell><text:p>Region</text:p></table:table-cell>"+ '<table:table-cell table:number-columns-repeated="2"><text:p>Q</text:p></table:table-cell>'+ '<table:table-cell table:number-columns-repeated="16000"/></table:table-row>'+ '<table:table-row table:number-rows-repeated="1048000"><table:table-cell table:number-columns-repeated="16000"/></table:table-row>'+ "<table:table-row><table:table-cell><text:p>Zebramarker</text:p></table:table-cell></table:table-row>"+ "</table:table></office:spreadsheet>",+ )+++def build_odp(path):+ return _odf(+ path,+ "<office:presentation>"+ "<draw:page><draw:frame><draw:text-box><text:p>First slide</text:p></draw:text-box></draw:frame>"+ "<presentation:notes><draw:frame><draw:text-box><text:p>zebramarker notes</text:p></draw:text-box></draw:frame></presentation:notes></draw:page>"+ "<draw:page><draw:frame><draw:text-box><text:p>Second slide revenue</text:p></draw:text-box></draw:frame></draw:page>"+ "</office:presentation>",+ )+++def build_epub(path):+ page = (+ lambda title,+ text: f"<html><head><title>{title}</title></head><body><p>{text}</p></body></html>"+ )+ return _zip(+ path,+ {+ "mimetype": "application/epub+zip",+ "META-INF/container.xml": '<container xmlns="urn:oasis:names:tc:opendocument:xmlns:container">'+ '<rootfiles><rootfile full-path="OEBPS/content.opf"/></rootfiles></container>',+ "OEBPS/content.opf": '<package xmlns="http://www.idpf.org/2007/opf"><manifest>'+ '<item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>'+ '<item id="c1" href="text/Chapter%201.xhtml#start" media-type="application/xhtml+xml"/>'+ '<item id="c2" href="text/c2.xhtml" media-type="application/xhtml+xml"/>'+ '</manifest><spine><itemref idref="nav"/><itemref idref="c2"/><itemref idref="c1"/></spine></package>',+ "OEBPS/nav.xhtml": page("Contents", "navmarker"),+ "OEBPS/text/Chapter 1.xhtml": page("One", "Zebramarker ends the book."),+ "OEBPS/text/c2.xhtml": page("Two", "Revenue increased."),+ },+ )+++def build_eml(path):+ message = EmailMessage()+ message["From"] = "Ann <[email protected]>"+ message["To"] = "[email protected]"+ message["Subject"] = "Q1 numbers résumé"+ message.set_content("Revenue increased.\nZebramarker in the body.", cte = "base64")+ message.add_alternative("<p>html copy</p>", subtype = "html")+ message.add_attachment(b"x,y\n", maintype = "text", subtype = "csv", filename = "data.csv")+ path.write_bytes(bytes(message))+ return path+++def build_mhtml(path):+ # As Chrome saves a page: multipart/related, the page plus its resources.+ message = EmailMessage()+ message["From"] = "<Saved by Blink>"+ message["Subject"] = "Quarterly report"+ message.set_content(+ "<html><body><p>Zebramarker on the page.</p><img src='logo.png'></body></html>",+ subtype = "html",+ )+ message.make_related()+ message.add_related(b"\x89PNG\r\n\x1a\n", maintype = "image", subtype = "png", cid = "<logo>")+ path.write_bytes(bytes(message))+ return path+++def build_xhtml(path):+ path.write_text(+ '<?xml version="1.0" encoding="utf-8"?><html xmlns="http://www.w3.org/1999/xhtml">'+ "<head><title>Report</title><style>p{}</style></head><body><p>Zebramarker résumé</p></body></html>",+ encoding = "utf-8",+ )+ return path+++_WORD_TYPES = {+ ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml",+ ".docm": "application/vnd.ms-word.document.macroEnabled.main+xml",+ ".dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml",+ ".dotm": "application/vnd.ms-word.template.macroEnabledTemplate.main+xml",+}+++def build_word(path):+ import docx++ document = docx.Document()+ document.add_paragraph("Quarterly report")+ document.add_table(rows = 1, cols = 2).rows[0].cells[0].text = "Zebramarker"+ buffer = io.BytesIO()+ document.save(buffer)+ with zipfile.ZipFile(buffer) as source, zipfile.ZipFile(path, "w") as target:+ for info in source.infolist():+ data = source.read(info)+ if info.filename == "[Content_Types].xml":+ data = data.replace(+ _WORD_TYPES[".docx"].encode(), _WORD_TYPES[path.suffix].encode()+ )+ target.writestr(info, data)+ return path+++def build_rtf(path):+ path.write_bytes(+ rb"{\rtf1\ansi\ansicpg1252\uc1{\fonttbl{\f0 Arial;}}{\colortbl;\red0\green0\blue0;}"+ rb"{\*\generator Writer;}\f0 Quarterly report\par Caf\'e9 \u8364? revenue increased\par"+ rb"{\field{\*\fldinst HYPERLINK hiddenmarker}{\fldrslt link text}}\par"+ rb"\trowd Zebramarker\cell 1200\cell\row}"+ )+ return path+++def build_doc(path, *, encrypted = False):+ compressed = "Quarterly report\r".encode("cp1252")+ wide = ("Café 東京 \x13 HYPERLINK x \x14link\x15 Zebramarker\x071200\x07\x07").encode(+ "utf-16-le"+ )+ word = bytearray(0x1800)+ flags = 0x0200 | (0x0100 if encrypted else 0)+ struct.pack_into("<HH", word, 0, 0xA5EC, 0xC1)+ struct.pack_into("<H", word, 0x0A, flags)+ struct.pack_into("<H", word, 32, 14) # csw+ struct.pack_into("<H", word, 62, 22) # cslw+ ccp = len(compressed) + len(wide) // 2+ footnote, endnote, text_box = "\x02 Footnote detail\r", "\x02 Endnote source\r", "Boxed text\r"+ struct.pack_into("<i", word, 64 + 3 * 4, ccp) # ccpText+ struct.pack_into("<i", word, 64 + 4 * 4, len(footnote)) # ccpFtn+ struct.pack_into("<i", word, 64 + 8 * 4, len(endnote)) # ccpEdn+ struct.pack_into("<i", word, 64 + 9 * 4, len(text_box)) # ccpTxbx+ struct.pack_into("<H", word, 152, 93) # cbRgFcLcb+ stories = (footnote + endnote + text_box).encode("utf-16-le")+ word[0x1000 : 0x1000 + len(compressed)] = compressed+ word[0x1400 : 0x1400 + len(wide)] = wide+ word[0x1600 : 0x1600 + len(stories)] = stories+ cps = [0, len(compressed), ccp, ccp + len(stories) // 2]+ pcds = [(0x2000 | 0x40000000), 0x1400, 0x1600]+ plc = struct.pack("<4I", *cps) + b"".join(struct.pack("<HIH", 0, fc, 0) for fc in pcds)+ clx = b"\x01" + struct.pack("<H", 2) + b"\0\0" + b"\x02" + struct.pack("<I", len(plc)) + plc+ struct.pack_into("<II", word, 154 + 33 * 8, 0, len(clx)) # fcClx, lcbClx+ path.write_bytes(compound_file({("WordDocument",): bytes(word), ("1Table",): clx}))+ return path+++def test_doc_refuses_pieces_that_repeat_one_range(tmp_path):+ word = bytearray(0x1800)+ struct.pack_into("<HH", word, 0, 0xA5EC, 0xC1)+ struct.pack_into("<H", word, 0x0A, 0x0200)+ struct.pack_into("<H", word, 32, 14) # csw+ struct.pack_into("<H", word, 62, 22) # cslw+ word[0x1000:0x1040] = b"z" * 64+ n = 400 # 400 pieces of 64 chars, all from the same 64 bytes+ struct.pack_into("<i", word, 64 + 3 * 4, 64 * n) # ccpText+ struct.pack_into("<H", word, 152, 93) # cbRgFcLcb+ plc = struct.pack(f"<{n + 1}I", *range(0, 64 * (n + 1), 64))+ plc += struct.pack("<HIH", 0, 0x2000 | 0x40000000, 0) * n+ clx = b"\x02" + struct.pack("<I", len(plc)) + plc+ struct.pack_into("<II", word, 154 + 33 * 8, 0, len(clx)) # fcClx, lcbClx+ path = tmp_path / "pieces.doc"+ path.write_bytes(compound_file({("WordDocument",): bytes(word), ("1Table",): clx}))+ with pytest.raises(ValueError, match = "repeats too much content"):+ parsers.parse(str(path))+++def _record(kind, body):+ return struct.pack("<HH", kind, len(body)) + body+++def build_xls(path):+ def short(text, size = 1):+ return struct.pack("<B" if size == 1 else "<H", len(text)) + b"\0" + text.encode("latin-1")++ # SST: "Region", then a string whose characters continue, wide, in a CONTINUE record.+ sst = struct.pack("<II", 2, 2) + struct.pack("<HB", 6, 0) + b"Region"+ sst += struct.pack("<HB", 12, 0) + b"Zebra"+ cont = b"\x01" + "marker€".encode("utf-16-le")+ xf = lambda fmt: _record(0x00E0, struct.pack("<HH", 0, fmt) + b"\0" * 16)+ globals_ = (+ _record(0x0809, b"\0" * 16)+ + _record(0x0085, b"\0" * 4 + b"\0\0" + short("Revenue"))+ + _record(0x00FC, sst)+ + _record(0x003C, cont)+ + _record(0x041E, struct.pack("<H", 164) + short("yyyy-mm-dd", 2))+ + xf(0)+ + xf(14)+ + xf(164)+ + _record(0x000A, b"")+ )+ serial = (dt.date(2026, 3, 31) - dt.date(1899, 12, 30)).days+ rk_int = lambda n: ((n << 2) | 2) & 0xFFFFFFFF+ sheet = (+ _record(0x0809, b"\0" * 16)+ + _record(0x00FD, struct.pack("<HHHI", 0, 0, 0, 0))+ + _record(0x0203, struct.pack("<HHHd", 0, 1, 0, 1200.5))+ + _record(0x00FD, struct.pack("<HHHI", 1, 0, 0, 1))+ + _record(0x027E, struct.pack("<HHHI", 1, 1, 0, rk_int(-7)))+ + _record(+ 0x00BD,+ struct.pack("<HH", 2, 0)+ + struct.pack("<HI", 0, rk_int(3))+ + struct.pack("<HI", 0, rk_int(4))+ + struct.pack("<H", 1),+ )+ + _record(0x0809, b"\0" * 16) # embedded chart+ + _record(0x000A, b"")+ + _record(0x0006, struct.pack("<HHH", 3, 0, 0) + b"\0" * 6 + b"\xff\xff" + b"\0" * 6)+ + _record(0x0207, short("formula text", 2))+ + _record(0x0205, struct.pack("<HHHBB", 3, 1, 0, 1, 0))+ + _record(0x0203, struct.pack("<HHHd", 4, 0, 2, serial))+ + _record(0x027E, struct.pack("<HHHI", 4, 1, 1, rk_int(serial + 1)))+ + _record(0x000A, b"")+ )+ stream = bytearray(globals_ + sheet)+ struct.pack_into("<I", stream, 4 + 16 + 4, len(globals_)) # BOUNDSHEET lbPlyPos+ path.write_bytes(compound_file({("Workbook",): bytes(stream)}))+ return path+++def _ppt_atom(+ kind,+ body,+ inst = 0,+):+ return struct.pack("<HHI", inst << 4, kind, len(body)) + body+++def _ppt_container(+ kind,+ children,+ inst = 0,+):+ body = b"".join(children)+ return struct.pack("<HHI", (inst << 4) | 0xF, kind, len(body)) + body+++def build_ppt(path, *, from_slides = False):+ chars = lambda text: _ppt_atom(0x0FA0, text.encode("utf-16-le"))+ persist = _ppt_atom(0x03F3, b"\0" * 20)+ if from_slides:+ slides = [+ _ppt_container(+ 0x03EE, [_ppt_container(0x040C, [_ppt_container(0xF00D, [chars(text)])])]+ )+ for text in ("Drawn slide", "Zebramarker drawn")+ ]+ document = _ppt_container(0x03E8, slides)+ else:+ document = _ppt_container(+ 0x03E8,+ [+ _ppt_container(+ 0x0FF0,+ [+ persist,+ chars("Quarterly review\rRevenue up"),+ persist,+ _ppt_atom(0x0FA8, b"Zebramarker slide"),+ ],+ inst = 0,+ ),+ _ppt_container(+ 0x0FF0, [persist, chars("Click to edit Master title style")], inst = 1+ ),+ ],+ )+ path.write_bytes(compound_file({("PowerPoint Document",): document}))+ return path+++# PidTagRtfCompressed of "{\\rtf1\\ansi\\ansicpg1252\\pard Zebramarker in the RTF body\\par repeat repeat repeat}".+RTF_COMPRESSED = bytes.fromhex(+ "49000000520000004c5a4675a3e8a93d03000a007263706731323502320af3205a656272612900c0726b04902"+ "00b80207444686507f054462006e064c6790aa30970706561054010cb027d1210"+)+++def build_ansi_msg(path):+ """String8 properties in code page 1251 and only a compressed RTF body."""+ props = b"\0" * 32 + struct.pack("<IIQ", 0x3FFD0003, 0, 1251)+ streams = {+ ("__substg1.0_0037001E",): "Привет".encode("cp1251"),+ ("__substg1.0_0C1A001E",): "Иван".encode("cp1251"),+ ("__substg1.0_10090102",): RTF_COMPRESSED,+ ("__properties_version1.0",): props,+ }+ path.write_bytes(compound_file(streams))+ return path+++def build_saved_ppt(path, *, broken_link = False):+ """Saved twice: the second save replaces the slide, adds notes and leaves the old slide behind."""+ chars = lambda text: _ppt_atom(0x0FA0, text.encode("utf-16-le"))+ textbox = lambda *atoms: _ppt_container(0xF00D, list(atoms))+ drawing = lambda *boxes: _ppt_container(0x040C, [_ppt_container(0xF002, list(boxes))])+ persist = lambda ref, sheet_id: _ppt_atom(0x03F3, struct.pack("<5I", ref, 0, 1, sheet_id, 0))+ stale = _ppt_container(0x03EE, [drawing(textbox(chars("Stale slide text")))])+ slide = _ppt_container(+ 0x03EE,+ [+ drawing(+ textbox(_ppt_atom(0x0F9E, struct.pack("<I", 0))),+ textbox(chars("Text box zebramarker")),+ )+ ],+ )+ notes = _ppt_container(+ 0x03F0,+ [+ _ppt_atom(0x03F1, struct.pack("<IHH", 256, 0, 0)),+ drawing(textbox(chars("Speaker notes")), textbox(chars("*"))),+ ],+ )+ document = _ppt_container(+ 0x03E8,+ [+ _ppt_container(+ 0x0FF0,+ [persist(2, 256), _ppt_atom(0x0F9F, b"\0" * 4), chars("Title from list")],+ inst = 0,+ ),+ _ppt_container(0x0FF0, [persist(3, 257)], inst = 2),+ ],+ )+ stream = bytearray()++ def put(record):+ stream.extend(record)+ return len(stream) - len(record)++ def directory(entries):+ body = b"".join(struct.pack("<II", (1 << 20) | pid, offset) for pid, offset in entries)+ return put(_ppt_atom(0x1772, body))++ def user_edit(last, directory_offset):+ offset = len(stream)+ last = offset if broken_link and last else last+ return put(+ _ppt_atom(0x0FF5, struct.pack("<IHBBIIII", 0, 0, 0, 3, last, directory_offset, 1, 4))+ )++ stale_at, doc_at = put(stale), put(document)+ first = user_edit(0, directory([(1, doc_at), (2, stale_at)]))+ slide_at, notes_at = put(slide), put(notes)+ current = user_edit(first, directory([(2, slide_at), (3, notes_at)]))+ current_user = _ppt_atom(0x0FF6, struct.pack("<III", 20, 0xE391C05F, current))+ path.write_bytes(+ compound_file({("PowerPoint Document",): bytes(stream), ("Current User",): current_user})+ )+ return path+++def build_msg(path):+ utf16 = lambda text: text.encode("utf-16-le")+ filetime = int(+ (dt.datetime(2026, 10, 6, 10, 0) - dt.datetime(1601, 1, 1)).total_seconds() * 10**7+ )+ props = b"\0" * 32 + struct.pack("<IIQ", (0x0039 << 16) | 0x0040, 0, filetime)+ streams = {+ ("__substg1.0_0037001F",): utf16("Q1 numbers"),+ ("__substg1.0_0C1A001F",): utf16("Ann"),+ ("__substg1.0_0C1F001F",): utf16("[email protected]"),+ ("__substg1.0_0E04001F",): utf16("Bob"),+ ("__substg1.0_1000001F",): utf16("Revenue increased.\r\nZebramarker in the body."),+ ("__properties_version1.0",): props,+ ("__attach_version1.0_#00000000", "__substg1.0_3707001F"): utf16("data.csv"),+ }+ path.write_bytes(compound_file(streams))+ return path+++BUILDERS = {+ ".doc": build_doc,+ ".xls": build_xls,+ ".ppt": build_ppt,+ ".msg": build_msg,+ ".eml": build_eml,+ ".rtf": build_rtf,+ ".epub": build_epub,+ **dict.fromkeys((".docm", ".dotx", ".dotm"), build_word),+ **dict.fromkeys((".xlsx", ".xlsm", ".xltx", ".xltm"), build_xlsx),+ **dict.fromkeys((".pptx", ".pptm", ".potx", ".potm", ".ppsx", ".ppsm"), build_pptx),+ **dict.fromkeys((".odt", ".ott"), build_odt),+ **dict.fromkeys((".ods", ".ots"), build_ods),+ **dict.fromkeys((".odp", ".otp"), build_odp),+ **dict.fromkeys((".mht", ".mhtml"), build_mhtml),+ **dict.fromkeys((".xhtml", ".xht"), build_xhtml),+}+++def test_every_document_type_has_a_fixture():+ assert set(BUILDERS) == config.DOCUMENT_UPLOAD_EXTS+ assert config.DOCUMENT_UPLOAD_EXTS <= config.UPLOAD_EXTS+++def test_document_exts_mirror_the_frontend_accept_list():+ source = (+ Path(__file__).resolve().parents[2] / "frontend/src/features/rag/types/rag.ts"+ ).read_text(encoding = "utf-8")+ accept = re.search(r'RAG_DOCUMENT_UPLOAD_ACCEPT =\s*"([^"]+)"', source).group(1)+ assert set(accept.split(",")) == config.DOCUMENT_UPLOAD_EXTS+++def test_xlsx_reads_shared_inline_dates_and_gaps(tmp_path):+ text = _text(build_xlsx(tmp_path / "book.xlsx"))+ assert (+ text+ == "Sheet: Revenue\nRegion | Q1\nZebramarker | 1200.5 | 2026-03-31 | TRUE | | formula text"+ )+++def test_pptx_reads_slides_in_order_with_tables_and_notes(tmp_path):+ pages = parsers.parse(str(build_pptx(tmp_path / "deck.pptx")))+ assert [(p.page_number, p.text) for p in pages] == [+ (+ 1,+ "Quarterly review\nRegion | Q1\nNorth | 1200\nRevenue increased\nNotes:\n2026\nSpeaker notes zebramarker",+ )+ ]+++def test_odt_reads_paragraphs_spaces_notes_and_tables(tmp_path):+ assert _text(build_odt(tmp_path / "memo.odt")) == (+ "Quarterly report\nRevenue increased [see appendix].\nZebramarker | 1200"+ )+++def test_ods_does_not_expand_empty_repeats(tmp_path):+ assert _text(build_ods(tmp_path / "sheet.ods")) == "Sheet: Revenue\nRegion | Q | Q\nZebramarker"+++def test_ots_template_is_streamed_like_ods(tmp_path, monkeypatch):+ calls = []+ real = office_formats._ods_streamed+ monkeypatch.setattr(office_formats, "_ods_streamed", lambda *a: calls.append(1) or real(*a))+ path = _odf(+ tmp_path / "plan.ots",+ '<office:spreadsheet><table:table table:name="S"><table:table-row>'+ "<table:table-cell><text:p>Zebramarker</text:p></table:table-cell>"+ "</table:table-row></table:table></office:spreadsheet>",+ )+ assert "Zebramarker" in _text(path)+ assert calls == [1]+++def test_ods_keeps_the_width_of_empty_runs_before_content(tmp_path):+ path = _odf(+ tmp_path / "gaps.ods",+ '<office:spreadsheet><table:table table:name="S"><table:table-row>'+ "<table:table-cell><text:p>A</text:p></table:table-cell>"+ '<table:table-cell table:number-columns-repeated="4"/>'+ "<table:table-cell><text:p>F</text:p></table:table-cell>"+ '<table:table-cell table:number-columns-repeated="16000"/>'+ "</table:table-row></table:table></office:spreadsheet>",+ )+ assert _text(path) == "Sheet: S\nA | | | | | F"+++def test_ods_reads_typed_values_without_text(tmp_path):+ path = _odf(+ tmp_path / "values.ods",+ '<office:spreadsheet><table:table table:name="S"><table:table-row>'+ '<table:table-cell office:value-type="float" office:value="1200"/>'+ '<table:table-cell office:value-type="date" office:date-value="2026-03-31"/>'+ '<table:table-cell office:value-type="boolean" office:boolean-value="true"/>'+ '<table:table-cell office:value-type="string" office:string-value="zebramarker"/>'+ '<table:table-cell office:value-type="float" office:value="7"><text:p>7.00</text:p></table:table-cell>'+ "</table:table-row></table:table></office:spreadsheet>",+ )+ assert _text(path) == "Sheet: S\n1200 | 2026-03-31 | TRUE | zebramarker | 7.00"+++# Dates, times and durations match what openpyxl reads back from the same cells.+ "serial, number_format, expected",+ [+ (0.5, "h:mm AM/PM", "12:00:00"),+ (1.5, "[h]:mm:ss", "36:00:00"),+ (1.5, "h:mm", "1900-01-01 12:00:00"),+ (1, "yyyy-mm-dd", "1900-01-01"),+ (59, "yyyy-mm-dd", "1900-02-28"),+ (61, "yyyy-mm-dd", "1900-03-01"),+ (0.25, "mm-dd-yy", "06:00:00"),+ (3, '"day" 0', "day 3"),+ (0.25, "0%", "25%"),+ (0.125, "0.0%", "12.5%"),+ (123, "000000", "000123"),+ (-1234.5, "#,##0.00", "-1,234.50"),+ (-1234.5, "#,##0.00;(#,##0.00)", "(1,234.50)"),+ (12345.678, "0.00E+00", "1.23E+04"),+ (12345.678, "##0.0E+0", "12.3E+3"),+ (1234.5, '"$"#,##0', "$1,235"),+ (1234.5, "[$\u20ac-407]#,##0.00", "\u20ac1,234.50"),+ (0.5, "#.##", ".5"),+ (1.234, "0.0#", "1.23"),+ (1500000, '#,##0,,"M"', "2M"),+ (1234.5, "General", "1234.5"),+ (5, "[>100]0;0.0", "5"),+ pytest.param(1234.5, "0" * 5000, "1234.5", id = "oversized-format-left-as-stored"),+ ],+)+def test_xlsx_shows_numbers_as_formatted(tmp_path, serial, number_format, expected):+ path = _zip(+ tmp_path / "times.xlsx",+ {+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="T" sheetId="1" r:id="rId1"/></sheets></workbook>',+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',+ "xl/styles.xml": f'<styleSheet {S}><numFmts><numFmt numFmtId="164" formatCode="{escape(number_format, {chr(34): """})}"/></numFmts>'+ '<cellXfs><xf numFmtId="0"/><xf numFmtId="164"/></cellXfs></styleSheet>',+ "xl/worksheets/sheet1.xml": f'<worksheet {S}><sheetData><row r="1"><c r="A1" s="1"><v>{serial}</v></c></row></sheetData></worksheet>',+ },+ )+ assert _text(path) == f"Sheet: T\n{expected}"+++def test_xlsx_finds_the_workbook_through_package_relationships(tmp_path):+ path = _zip(+ tmp_path / "moved.xlsx",+ {+ "_rels/.rels": f'<Relationships {REL}><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="book/main.xml"/></Relationships>',+ "book/main.xml": f'<workbook {S}><sheets><sheet name="M" sheetId="1" r:id="rId1"/></sheets></workbook>',+ "book/_rels/main.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="sheet.xml"/>'+ '<Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings" Target="strings.xml"/></Relationships>',+ "book/strings.xml": f"<sst {S}><si><t>zebramarker</t></si></sst>",+ "book/sheet.xml": f'<worksheet {S}><sheetData><row r="1"><c r="A1" t="s"><v>0</v></c></row></sheetData></worksheet>',+ },+ )+ assert _text(path) == "Sheet: M\nzebramarker"+++def test_xml_parts_are_bounded_by_element_count(tmp_path):+ row = "<c/>" * 200_001 # one streamed row+ sheet_bomb = _zip(+ tmp_path / "row.xlsx",+ {+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="B" sheetId="1" r:id="rId1"/></sheets></workbook>',+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',+ "xl/worksheets/sheet1.xml": f"<worksheet {S}><sheetData><row>{row}</row></sheetData></worksheet>",+ },+ )+ tree_bomb = _zip(+ tmp_path / "tree.xlsx",+ {"xl/workbook.xml": f"<workbook {S}>" + "<x/>" * 2_000_001 + "</workbook>"},+ )+ deep = _zip(+ tmp_path / "deep.ods",+ {+ "mimetype": "application/vnd.oasis.opendocument.spreadsheet",+ "content.xml": "<a>" * 600 + "</a>" * 600,+ },+ )+ for path in (sheet_bomb, tree_bomb, deep):+ with pytest.raises(ValueError, match = "too large to index|nested too deeply"):+ parsers.parse(str(path))++[email protected]("format_id", [14, 31, 57, 75])+def test_xlsx_builtin_date_formats_include_locale_ids(tmp_path, format_id):+ serial = (dt.date(2026, 3, 31) - dt.date(1899, 12, 30)).days+ path = _zip(+ tmp_path / "dates.xlsx",+ {+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="D" sheetId="1" r:id="rId1"/></sheets></workbook>',+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',+ "xl/styles.xml": f'<styleSheet {S}><cellXfs><xf numFmtId="0"/><xf numFmtId="{format_id}"/></cellXfs></styleSheet>',+ "xl/worksheets/sheet1.xml": f'<worksheet {S}><sheetData><row r="1"><c r="A1" s="1"><v>{serial}</v></c></row></sheetData></worksheet>',+ },+ )+ assert _text(path) == "Sheet: D\n2026-03-31"++[email protected]("cat, val", [("cat", "val"), ("xVal", "yVal")])+def test_pptx_reads_chart_titles_and_cached_data(tmp_path, cat, val):+ C = 'xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"'+ pt = lambda i, v: f'<c:pt idx="{i}"><c:v>{v}</c:v></c:pt>'+ series = lambda name, values: (+ f"<c:ser><c:tx><c:strRef><c:strCache>{pt(0, name)}</c:strCache></c:strRef></c:tx>"+ f"<c:{cat}><c:strRef><c:strCache>{pt(0, 'North')}{pt(1, 'South')}</c:strCache></c:strRef></c:{cat}>"+ f"<c:{val}><c:numRef><c:numCache>{pt(0, values[0])}{pt(1, values[1])}</c:numCache></c:numRef></c:{val}></c:ser>"+ )+ path = _zip(+ tmp_path / "chart.pptx",+ {+ "ppt/presentation.xml": f'<p:presentation {P}><p:sldIdLst><p:sldId id="256" r:id="rId2"/></p:sldIdLst></p:presentation>',+ "ppt/_rels/presentation.xml.rels": f'<Relationships {REL}><Relationship Id="rId2" Target="slides/slide1.xml"/></Relationships>',+ "ppt/slides/slide1.xml": f"<p:sld {P}><p:cSld><p:spTree><p:graphicFrame><a:graphic><a:graphicData>"+ '<c:chart xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" r:id="rId3"/>'+ "</a:graphicData></a:graphic></p:graphicFrame></p:spTree></p:cSld></p:sld>",+ "ppt/slides/_rels/slide1.xml.rels": f'<Relationships {REL}><Relationship Id="rId3" Target="../charts/chart1.xml"/></Relationships>',+ "ppt/charts/chart1.xml": f"<c:chartSpace {C}><c:chart><c:title><c:tx><c:rich><a:p><a:r><a:t>Sales zebramarker</a:t></a:r></a:p></c:rich></c:tx></c:title>"+ f"<c:plotArea><c:barChart>{series('Q1', (1200, 900))}{series('Q2', (1300, 950))}</c:barChart></c:plotArea></c:chart></c:chartSpace>",+ },+ )+ assert _text(path) == "Sales zebramarker\nQ1 | Q2\nNorth | 1200 | 1300\nSouth | 900 | 950"+++def test_scatter_series_keep_their_own_x_values():+ C = 'xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart"'+ pt = lambda i, v: f'<c:pt idx="{i}"><c:v>{v}</c:v></c:pt>'+ series = lambda name, xs, ys: (+ f"<c:ser><c:tx><c:strRef><c:strCache>{pt(0, name)}</c:strCache></c:strRef></c:tx>"+ f"<c:xVal><c:numRef><c:numCache>{pt(0, xs[0])}{pt(1, xs[1])}</c:numCache></c:numRef></c:xVal>"+ f"<c:yVal><c:numRef><c:numCache>{pt(0, ys[0])}{pt(1, ys[1])}</c:numCache></c:numRef></c:yVal></c:ser>"+ )+ root = ET.fromstring(+ f"<c:chartSpace {C}><c:chart><c:plotArea><c:scatterChart>"+ f"{series('A', (1, 2), (10, 20))}{series('B', (5, 6), (50, 60))}"+ "</c:scatterChart></c:plotArea></c:chart></c:chartSpace>"+ )+ assert office_formats._chart_lines(root) == ["A", "1 | 10", "2 | 20", "B", "5 | 50", "6 | 60"]+++def test_chart_keeps_every_level_of_multi_level_categories():+ C = 'xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart"'+ pts = lambda items: "".join(f'<c:pt idx="{i}"><c:v>{v}</c:v></c:pt>' for i, v in items)+ root = ET.fromstring(+ f"<c:chartSpace {C}><c:chart><c:plotArea><c:barChart><c:ser>"+ "<c:cat><c:multiLvlStrRef><c:multiLvlStrCache>"+ f"<c:lvl>{pts(enumerate(['Q1', 'Q2', 'Q1', 'Q2']))}</c:lvl>"+ f"<c:lvl>{pts([(0, '2023'), (2, '2024')])}</c:lvl>"+ "</c:multiLvlStrCache></c:multiLvlStrRef></c:cat>"+ f"<c:val><c:numRef><c:numCache>{pts(enumerate([10, 20, 30, 40]))}</c:numCache></c:numRef></c:val>"+ "</c:ser></c:barChart></c:plotArea></c:chart></c:chartSpace>"+ )+ assert office_formats._chart_lines(root) == [+ "2023 Q1 | 10",+ "Q2 | 20",+ "2024 Q1 | 30",+ "Q2 | 40",+ ]+++def test_odp_reads_one_page_per_slide(tmp_path):+ pages = parsers.parse(str(build_odp(tmp_path / "slides.odp")))+ assert [(p.page_number, p.text) for p in pages] == [+ (1, "First slide\nNotes:\nzebramarker notes"),+ (2, "Second slide revenue"),+ ]+++def test_epub_follows_the_spine_and_skips_navigation(tmp_path):+ text = _text(build_epub(tmp_path / "book.epub"))+ assert "navmarker" not in text+ assert text.index("Revenue increased") < text.index("Zebramarker ends the book")+++def test_epub_package_path_is_a_url(tmp_path):+ with zipfile.ZipFile(build_epub(tmp_path / "plain.epub")) as z:+ members = {name: z.read(name) for name in z.namelist()}+ members["OEBPS/My Book.opf"] = members.pop("OEBPS/content.opf")+ members["META-INF/container.xml"] = members["META-INF/container.xml"].replace(+ b"OEBPS/content.opf", b"OEBPS/My%20Book.opf"+ )+ book = tmp_path / "book.epub"+ with zipfile.ZipFile(book, "w") as z:+ for name, data in members.items():+ z.writestr(name, data)+ assert "Zebramarker ends the book" in _text(book)+++ "uri, protected", [("OEBPS/text/Chapter%201.xhtml", True), ("OEBPS/fonts/a.otf", False)]+)+def test_epub_with_encrypted_chapters_is_refused(tmp_path, uri, protected):+ book = build_epub(tmp_path / "book.epub")+ with zipfile.ZipFile(book, "a") as z:+ z.writestr(+ "META-INF/encryption.xml",+ '<encryption xmlns="urn:oasis:names:tc:opendocument:xmlns:container" '+ 'xmlns:enc="http://www.w3.org/2001/04/xmlenc#"><enc:EncryptedData>'+ f'<enc:CipherData><enc:CipherReference URI="{uri}"/></enc:CipherData>'+ "</enc:EncryptedData></encryption>",+ )+ if protected:+ with pytest.raises(ValueError, match = "DRM protected"):+ parsers.parse(str(book))+ else: # obfuscated fonts are listed too, and never block reading+ assert "Zebramarker" in _text(book)+++def test_eml_decodes_headers_and_body(tmp_path):+ text = _text(build_eml(tmp_path / "mail.eml"))+ assert "Subject: Q1 numbers résumé" in text+ assert "From: Ann <[email protected]>" in text+ assert "Attachments: data.csv" in text+ assert "Zebramarker in the body." in text+ assert "html copy" not in text++[email protected]("extension", [".docx", ".docm", ".dotx", ".dotm"])+def test_word_macro_and_template_files_read_like_docx(tmp_path, extension):+ text = _text(build_word(tmp_path / f"report{extension}"))+ assert "Quarterly report" in text and "Zebramarker" in text+++def test_word_extension_on_another_package_is_refused(tmp_path):+ workbook = build_word(tmp_path / "report.docx").read_bytes()+ with zipfile.ZipFile(io.BytesIO(workbook)) as source:+ types = source.read("[Content_Types].xml").replace(+ _WORD_TYPES[".docx"].encode(),+ b"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml",+ )+ with zipfile.ZipFile(tmp_path / "sheet.docm", "w") as target:+ for info in source.infolist():+ target.writestr(+ info, types if info.filename == "[Content_Types].xml" else source.read(info)+ )+ with pytest.raises(ValueError, match = "not a Word file"):+ parsers.parse(str(tmp_path / "sheet.docm"))+++def test_mhtml_reads_the_page_and_skips_resources(tmp_path):+ text = _text(build_mhtml(tmp_path / "page.mhtml"))+ assert "Zebramarker on the page." in text+ assert "PNG" not in text+++def test_xhtml_reads_visible_text(tmp_path):+ text = _text(build_xhtml(tmp_path / "page.xhtml"))+ assert "Zebramarker résumé" in text and "p{}" not in text+++def test_rtf_keeps_text_and_drops_control_groups(tmp_path):+ assert _text(build_rtf(tmp_path / "memo.rtf")) == (+ "Quarterly report\nCafé € revenue increased\nlink text\nZebramarker | 1200"+ )+++def test_doc_reads_compressed_and_unicode_pieces_and_note_stories(tmp_path):+ assert _text(build_doc(tmp_path / "memo.doc")) == (+ "Quarterly report\nCafé 東京 link Zebramarker | 1200\n\nBoxed text"+ "\n\nFootnotes:\nFootnote detail\n\nEndnotes:\nEndnote source"+ )+++def test_xls_reads_errors_and_continued_formula_strings(tmp_path):+ long_text = "A" * 8000 + "Ж" * 300+ string = struct.pack("<HB", len(long_text), 0) + b"A" * 8000+ sheet = (+ _record(0x0809, b"\0" * 16)+ + _record(0x0205, struct.pack("<HHHBB", 0, 0, 0, 0x07, 1)) # BOOLERR #DIV/0!+ + _record(0x0006, struct.pack("<HHH", 0, 1, 0) + b"\x02\0\x2a\0\0\0\xff\xff" + b"\0" * 6)+ + _record(0x0006, struct.pack("<HHH", 1, 0, 0) + b"\0" * 6 + b"\xff\xff" + b"\0" * 6)+ + _record(0x0207, string)+ + _record(0x003C, b"\x01" + "Ж".encode("utf-16-le") * 300)+ + _record(0x000A, b"")+ )+ globals_ = (+ _record(0x0809, b"\0" * 16)+ + _record(0x0085, b"\0" * 4 + b"\0\0" + struct.pack("<B", 1) + b"\0S")+ + _record(0x000A, b"")+ )+ stream = bytearray(globals_ + sheet)+ struct.pack_into("<I", stream, 4 + 16 + 4, len(globals_))+ path = tmp_path / "errors.xls"+ path.write_bytes(compound_file({("Workbook",): bytes(stream)}))+ assert _text(path) == f"Sheet: S\n#DIV/0! | #N/A\n{long_text}"+++def test_xls_reads_continued_strings_numbers_formulas_and_dates(tmp_path):+ assert _text(build_xls(tmp_path / "book.xls")) == (+ "Sheet: Revenue\nRegion | 1200.5\nZebramarker€ | -7\n3 | 4\nformula text | TRUE"+ "\n2026-03-31 | 2026-04-01"+ )+++def test_xls_refuses_a_workbook_past_the_cell_ceiling(tmp_path, monkeypatch):+ monkeypatch.setattr(office_formats, "_MAX_XML_ELEMENTS", 3)+ with pytest.raises(ValueError, match = "too many cells"):+ parsers.parse(str(build_xls(tmp_path / "book.xls")))+++# A .msg looks up properties per attachment; rebuilding the root map each time was quadratic.+def test_compound_file_walks_each_storage_once(tmp_path):+ reader = cfb.CompoundFile(build_msg(tmp_path / "mail.msg").read_bytes())+ first = reader._children(0)+ assert reader._children(0) is first+ assert reader.exists(*next(iter(first)).split("/"))+++def test_xls_reads_excel_95_byte_strings(tmp_path):+ def text(value, size):+ raw = value.encode("cp1251")+ return struct.pack("<B" if size == 1 else "<H", len(raw)) + raw++ bof = _record(0x0809, struct.pack("<HH", 0x0500, 0x0005) + b"\0" * 4)+ globals_ = lambda offset: (+ bof+ + _record(0x0042, struct.pack("<H", 1251)) # CODEPAGE+ + _record(0x0085, struct.pack("<IBB", offset, 0, 0) + text("Выручка", 1))+ + _record(0x000A, b"")+ )+ sheet = (+ bof+ + _record(0x0204, struct.pack("<HHH", 0, 0, 0) + text("Zebramarker", 2))+ + _record(0x0204, struct.pack("<HHH", 0, 1, 0) + text("Привет", 2))+ + _record(0x0006, struct.pack("<HHH", 1, 0, 0) + b"\0" * 6 + b"\xff\xff" + b"\0" * 6)+ + _record(0x0207, text("formula text", 2))+ + _record(0x000A, b"")+ )+ path = tmp_path / "excel95.xls"+ path.write_bytes(compound_file({("Book",): globals_(len(globals_(0))) + sheet}))+ assert _text(path) == "Sheet: Выручка\nZebramarker | Привет\nformula text"+++def test_ppt_reads_slide_text_and_skips_masters(tmp_path):+ pages = parsers.parse(str(build_ppt(tmp_path / "deck.ppt")))+ assert [(p.page_number, p.text) for p in pages] == [+ (1, "Quarterly review\nRevenue up"),+ (2, "Zebramarker slide"),+ ]++[email protected]("broken_link", [False, True])+def test_ppt_reads_the_latest_save_with_text_boxes_and_notes(tmp_path, broken_link):+ pages = parsers.parse(str(build_saved_ppt(tmp_path / "saved.ppt", broken_link = broken_link)))+ assert [(p.page_number, p.text) for p in pages] == [+ (1, "Title from list\nText box zebramarker\nNotes:\nSpeaker notes")+ ]+++def test_ppt_falls_back_to_slide_drawings(tmp_path):+ pages = parsers.parse(str(build_ppt(tmp_path / "deck.ppt", from_slides = True)))+ assert [p.text for p in pages] == ["Drawn slide", "Zebramarker drawn"]+++def test_msg_reads_headers_body_and_attachment_names(tmp_path):+ assert _text(build_msg(tmp_path / "mail.msg")) == (+ "From: Ann <[email protected]>\nTo: Bob\nDate: 2026-10-06 10:00 UTC\nSubject: Q1 numbers\n"+ "Attachments: data.csv\n\nRevenue increased.\nZebramarker in the body."+ )+++def test_msg_uses_the_smtp_address_for_exchange_senders(tmp_path):+ utf16 = lambda text: text.encode("utf-16-le")+ path = tmp_path / "exchange.msg"+ path.write_bytes(+ compound_file(+ {+ ("__substg1.0_0C1A001F",): utf16("Ann"),+ ("__substg1.0_0C1F001F",): utf16("/O=EXAMPLE/OU=EXCHANGE/CN=RECIPIENTS/CN=ANN"),+ ("__substg1.0_5D01001F",): utf16("[email protected]"),+ ("__substg1.0_1000001F",): utf16("Body zebramarker"),+ }+ )+ )+ assert _text(path) == "From: Ann <[email protected]>\n\nBody zebramarker"++[email protected]("plain", [None, "\r\n \r\n"])+def test_msg_reads_a_string_html_body(tmp_path, plain):+ path = tmp_path / "html.msg"+ html = '<html><head><meta charset="windows-1252"></head><body><p>Café zebramarker</p></body></html>'+ streams = {("__substg1.0_1013001F",): html.encode("utf-16-le")}+ if plain is not None: # a blank plain body must not hide the HTML one+ streams[("__substg1.0_1000001F",)] = plain.encode("utf-16-le")+ path.write_bytes(compound_file(streams))+ assert _text(path).strip() == "Café zebramarker"+++def test_pptx_reads_smartart_labels(tmp_path):+ members = {}+ with zipfile.ZipFile(build_pptx(tmp_path / "plain.pptx")) as z:+ members = {name: z.read(name) for name in z.namelist()}+ dgm = 'xmlns:dgm="http://schemas.openxmlformats.org/drawingml/2006/diagram"'+ members["ppt/slides/slide1.xml"] = members["ppt/slides/slide1.xml"].replace(+ b"</p:spTree>",+ f'<p:graphicFrame><a:graphic><a:graphicData><dgm:relIds {dgm} r:dm="rId2"/>'+ "</a:graphicData></a:graphic></p:graphicFrame></p:spTree>".encode(),+ )+ members["ppt/slides/_rels/slide1.xml.rels"] = members[+ "ppt/slides/_rels/slide1.xml.rels"+ ].replace(+ b"</Relationships>",+ b'<Relationship Id="rId2" Target="../diagrams/data1.xml"/></Relationships>',+ )+ members["ppt/diagrams/data1.xml"] = (+ f"<dgm:dataModel {dgm} {P}><dgm:ptLst>"+ "<dgm:pt><dgm:t><a:p><a:r><a:t>Plan</a:t></a:r></a:p></dgm:t></dgm:pt>"+ "<dgm:pt><dgm:t><a:p><a:r><a:t>Ship zebramarker</a:t></a:r></a:p></dgm:t></dgm:pt>"+ "</dgm:ptLst></dgm:dataModel>"+ )+ text = _text(_zip(tmp_path / "smartart.pptx", members))+ assert "Revenue increased\nPlan\nShip zebramarker\nNotes:" in text+++def test_xlsx_refuses_one_shared_string_filling_every_cell(tmp_path):+ cells = "".join(f'<c r="{chr(65 + i)}1" t="s"><v>0</v></c>' for i in range(26))+ rows = "".join(+ f'<row r="{n}">{cells.replace("1" + chr(34), str(n) + chr(34))}</row>' for n in range(1, 60)+ )+ path = _zip(+ tmp_path / "bomb.xlsx",+ {+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="S" sheetId="1" r:id="rId1"/></sheets></workbook>',+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',+ "xl/sharedStrings.xml": f"<sst {S}><si><t>{'x' * 25_000}</t></si></sst>",+ "xl/worksheets/sheet1.xml": f"<worksheet {S}><sheetData>{rows}</sheetData></worksheet>",+ },+ )+ with pytest.raises(ValueError, match = "repeats too much content"):+ parsers.parse(str(path))+++def test_eml_reads_every_inline_part(tmp_path):+ message = EmailMessage()+ message["Subject"] = "Two parts"+ message.set_content("first part")+ message.make_mixed()+ inline = EmailMessage()+ inline.set_content("second zebramarker")+ inline["Content-Disposition"] = "inline"+ message.attach(inline)+ message.add_attachment(b"x,y\n", maintype = "text", subtype = "csv", filename = "data.csv")+ path = tmp_path / "mixed.eml"+ path.write_bytes(bytes(message))+ text = _text(path)+ assert text.endswith("first part\n\nsecond zebramarker")+ assert "x,y" not in text+++def test_eml_falls_back_to_html_when_plain_is_empty(tmp_path):+ message = EmailMessage()+ message["Subject"] = "HTML only"+ message.set_content(" \n")+ message.add_alternative("<p>zebramarker in html</p>", subtype = "html")+ path = tmp_path / "html.eml"+ path.write_bytes(bytes(message))+ assert _text(path).endswith("zebramarker in html")+++def test_msg_decodes_ansi_properties_and_compressed_rtf_body(tmp_path):+ assert _text(build_ansi_msg(tmp_path / "ansi.msg")) == (+ "From: Иван\nSubject: Привет\n\nZebramarker in the RTF body\nrepeat repeat repeat"+ )+++def _strict(source, target):+ """Rewrite a Transitional OOXML package with the Strict namespaces."""+ pairs = [+ (+ b"http://schemas.openxmlformats.org/spreadsheetml/2006/main",+ b"http://purl.oclc.org/ooxml/spreadsheetml/main",+ ),+ (+ b"http://schemas.openxmlformats.org/presentationml/2006/main",+ b"http://purl.oclc.org/ooxml/presentationml/main",+ ),+ (+ b"http://schemas.openxmlformats.org/drawingml/2006/main",+ b"http://purl.oclc.org/ooxml/drawingml/main",+ ),+ (+ b"http://schemas.openxmlformats.org/officeDocument/2006/relationships",+ b"http://purl.oclc.org/ooxml/officeDocument/relationships",+ ),+ ]+ with zipfile.ZipFile(source) as z:+ members = {name: z.read(name) for name in z.namelist()}+ for name, data in members.items():+ for old, new in pairs:+ data = data.replace(old, new)+ members[name] = data+ return _zip(target, members)++[email protected]("builder, extension", [(build_xlsx, ".xlsx"), (build_pptx, ".pptx")])+def test_strict_ooxml_reads_like_transitional(tmp_path, builder, extension):+ transitional = builder(tmp_path / f"transitional{extension}")+ strict = _strict(transitional, tmp_path / f"strict{extension}")+ with zipfile.ZipFile(strict) as z:+ assert all(b"2006/main" not in z.read(name) for name in z.namelist())+ assert _text(strict) == _text(transitional) != ""+++def test_odt_skips_tracked_deletions_and_reads_nested_tables_once(tmp_path):+ path = _odf(+ tmp_path / "changes.odt",+ "<office:text><text:tracked-changes><text:changed-region>"+ "<text:deletion><text:p>deletedmarker</text:p></text:deletion>"+ "</text:changed-region></text:tracked-changes>"+ '<text:p>See <text:bookmark-ref text:ref-name="b">Chapter 2</text:bookmark-ref>.</text:p>'+ "<table:table><table:table-header-rows><table:table-row>"+ "<table:table-cell><text:p>Head</text:p></table:table-cell>"+ "</table:table-row></table:table-header-rows><table:table-row><table:table-cell>"+ "<table:table><table:table-row><table:table-cell><text:p>inner</text:p></table:table-cell>"+ "<table:table-cell><text:p>cell</text:p></table:table-cell></table:table-row></table:table>"+ "</table:table-cell><table:table-cell><text:p>outer</text:p></table:table-cell>"+ "</table:table-row></table:table></office:text>",+ )+ assert _text(path) == "See Chapter 2.\nHead\ninner | cell | outer"+++def test_rtf_decodes_literal_bytes_in_the_document_code_page(tmp_path):+ path = tmp_path / "ru.rtf"+ path.write_bytes(b"{\\rtf1\\ansi\\ansicpg1251 " + "Привет".encode("cp1251") + b" \\'80}")+ assert _text(path) == "Привет Ђ"+++ "header, encoded",+ [+ (b"\\mac", "Café".encode("mac_roman")),+ (b"\\pc", "Café".encode("cp437")),+ (b"\\pca", "Café".encode("cp850")),+ # Word for Mac writes Mac Roman as \ansicpg10000, which Python names mac_roman.+ (b"\\mac\\ansicpg10000", "Café".encode("mac_roman")),+ ],+)+def test_rtf_decodes_the_mac_and_pc_document_character_sets(tmp_path, header, encoded):+ path = tmp_path / "legacy.rtf"+ path.write_bytes(+ b"{\\rtf1" + header + b"\\deff0{\\fonttbl{\\f0\\fcharset256 Times;}}\\f0 " + encoded + b"}"+ )+ assert _text(path) == "Café"+++def test_compound_file_reads_mini_and_regular_streams():+ small, large = b"a" * 100, bytes(range(256)) * 40+ reader = cfb.CompoundFile(compound_file({("small",): small, ("dir", "large"): large}))+ assert reader.read("small") == small+ assert reader.read("DIR", "Large") == large+ assert sorted(reader.listdir()) == ["dir", "small"]+++def test_compound_file_reads_each_fat_sector_once_per_file_sector():+ # Every DIFAT slot names the same FAT sector: the FAT must stay the size of the file.+ data = bytearray(compound_file({("small",): b"a" * 100}))+ n_difat = 64+ first = len(data) // 512 - 1+ data += b"".join(+ struct.pack("<128I", *([0] * 127 + [first + i + 1 if i + 1 < n_difat else 0xFFFFFFFE]))+ for i in range(n_difat)+ )+ struct.pack_into("<I", data, 0x2C, 0) # no FAT sector count to trim by+ struct.pack_into("<II", data, 0x44, first, n_difat)+ reader = cfb.CompoundFile(bytes(data))+ assert len(reader._fat) <= len(data) // 4+ assert reader.read("small") == b"a" * 100+++def test_encrypted_doc_is_refused(tmp_path):+ with pytest.raises(ValueError, match = "password"):+ parsers.parse(str(build_doc(tmp_path / "secret.doc", encrypted = True)))++[email protected]("extension", [".doc", ".xls", ".ppt", ".msg"])+def test_non_compound_file_is_refused(tmp_path, extension):+ path = tmp_path / f"fake{extension}"+ path.write_bytes(b"not a compound file" * 50)+ with pytest.raises(ValueError):+ parsers.parse(str(path))++[email protected]("extension", [".xlsx", ".pptx", ".odt", ".epub"])+def test_damaged_archive_is_refused(tmp_path, extension):+ path = tmp_path / f"broken{extension}"+ path.write_bytes(b"PK\x03\x04 truncated")+ with pytest.raises(ValueError):+ parsers.parse(str(path))+++def test_rtf_pairs_surrogates_and_skips_tracked_deletions(tmp_path):+ path = tmp_path / "changes.rtf"+ # As macOS textutil writes an emoji; the deletion as Word writes a tracked change.+ path.write_bytes(+ rb"{\rtf1\ansi Party \uc0\u55356 \u57225 kept{\deleted\revauthdel1 removedmarker \u8364?}"+ rb" end\deleted gone\deleted0 back\plain}"+ )+ assert _text(path) == "Party \U0001f389 kept end back"+++def test_compressed_rtf_stops_at_the_declared_size():+ # One literal, then back-references that would expand to ~5 MB.+ body = b"\x00A" + (b"\xff" + b"\x00\x0f" * 8) * 40_000+ data = struct.pack("<IIII", len(body) + 12, 1000, 0x75465A4C, 0) + body+ assert len(office_formats._decompress_rtf(data)) == 1000+ with pytest.raises(ValueError, match = "too large"):+ office_formats._decompress_rtf(struct.pack("<IIII", 16, 2**31, 0x75465A4C, 0))+++def test_rtf_decodes_bytes_with_the_font_charset(tmp_path):+ path = tmp_path / "fonts.rtf"+ path.write_bytes(+ rb"{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0\fswiss\fcharset0 Arial;}"+ rb"{\f1\froman\fcharset204 Times New Roman Cyr;}{\f2\fnil\cpg1253 Greek;}}"+ rb"\f0 Hello {\f1 \'cf\'f0\'e8\'e2\'e5\'f2} back {\f2 \'e1} \plain\'e9\par}"+ )+ assert _text(path) == "Hello Привет back α é"+++def test_rtf_prefers_the_unicode_branch_of_upr(tmp_path):+ path = tmp_path / "upr.rtf"+ path.write_bytes(rb"{\rtf1\ansi \upr{?}{\*\ud{\u1040?}} end}")+ assert _text(path) == "\u0410 end"+++def test_rtf_refuses_runaway_nesting(tmp_path):+ path = tmp_path / "deep.rtf"+ path.write_bytes(b"{\\rtf1 " + b"{" * 5000)+ with pytest.raises(ValueError, match = "nested too deeply"):+ parsers.parse(str(path))+++def test_rtf_negative_binary_length_is_ignored(tmp_path):+ path = tmp_path / "bin.rtf"+ path.write_bytes(rb"{\rtf1\ansi before \bin-999999999 after}")+ assert _text(path) == "before after"+++def test_ods_refuses_runaway_repeats(tmp_path):+ cell = "x" * 100_000+ path = _odf(+ tmp_path / "repeats.ods",+ '<office:spreadsheet><table:table table:name="S">'+ '<table:table-row table:number-rows-repeated="1000">'+ f'<table:table-cell table:number-columns-repeated="1000"><text:p>{cell}</text:p></table:table-cell>'+ "</table:table-row></table:table></office:spreadsheet>",+ )+ with pytest.raises(ValueError, match = "repeats too much"):+ parsers.parse(str(path))+++def test_xml_entities_are_refused(tmp_path):+ path = _zip(+ tmp_path / "bomb.odt", {"content.xml": '<!DOCTYPE x [<!ENTITY a "aaaa">]><x>&a;</x>'}+ )+ with pytest.raises(ValueError):+ parsers.parse(str(path))++[email protected]("extension", sorted(BUILDERS))+def test_documents_upload_index_and_stay_searchable(rag_home, stub_embeddings, tmp_path, extension):+ if not rag_db.rag_available(): # vec0 will not load on some macOS Pythons+ pytest.skip("sqlite-vec unavailable here")+ source = BUILDERS[extension](tmp_path / f"report{extension}")+ stored, filename, _hash = _save_upload(+ UploadFile(file = io.BytesIO(source.read_bytes()), filename = source.name)+ )+ assert filename == source.name+ scope = store.thread_scope("office")+ doc_id, job = ingestion.start_ingestion(+ scope, None, "office", filename, stored, background = False+ )+ assert ingestion.get_job_status(job)["status"] == "completed"+ conn = rag_db.get_connection()+ try:+ assert store.get_document(conn, doc_id)["status"] == "completed"+ assert store.search_lexical(conn, scope, "zebramarker", 5)+ finally:+ conn.close()diff --git a/studio/frontend/src/components/assistant-ui/attachment-document-dialog.tsx b/studio/frontend/src/components/assistant-ui/attachment-document-dialog.tsxindex e3b2ae839fd..d84874eb6e2 100644--- a/studio/frontend/src/components/assistant-ui/attachment-document-dialog.tsx+++ b/studio/frontend/src/components/assistant-ui/attachment-document-dialog.tsx@@ -220,7 +220,13 @@ const DocumentDialog: FC<: truncateAttachmentPreviewText(await blob.text());next = { blob, text, truncated };} else if (textFallback && blob.type.startsWith("text/")) {- const { text, truncated } = truncateAttachmentPreviewText(await blob.text());+ // A File decodes as the composer does (BOM, UTF-16); a code page it refuses still shows raw.+ const { text, truncated } =+ blob instanceof File+ ? await readAttachmentText(blob, source.name, blob.type).catch(async () =>+ truncateAttachmentPreviewText(await blob.text()),+ )+ : truncateAttachmentPreviewText(await blob.text());next = { blob, plain: text, truncated };}if (!cancelled) setLoaded(next);@@ -281,6 +287,33 @@ const SentOriginalDialog: FC<);};+/** A file outside any message, such as a chat with files source, read on open: pages, markdown or+ * text (a text/* blob). */+export const LocalFileDialog: FC<PropsWithChildren<{ name: string; load: () => Promise<Blob> }>> = ({+ children,+ name,+ load,+}) => (+ <DocumentDialog+ source={{+ kind: "document",+ name,+ contentType: undefined,+ file: undefined,+ src: undefined,+ audio: undefined,+ video: undefined,+ text: undefined,+ hasOriginal: true,+ }}+ load={load}+ redactFromReload={false}+ textFallback={true}+ >+ {children}+ </DocumentDialog>+);+export const AttachmentDocumentDialog: FC<PropsWithChildren<{ source: AttachmentSource; redactFromReload: boolean }>> = ({ children, source, redactFromReload }) => {diff --git a/studio/frontend/src/components/assistant-ui/attachment.tsx b/studio/frontend/src/components/assistant-ui/attachment.tsxindex 0acb7310d2c..15190175443 100644--- a/studio/frontend/src/components/assistant-ui/attachment.tsx+++ b/studio/frontend/src/components/assistant-ui/attachment.tsx@@ -74,16 +74,16 @@ import {} from "react";import { ScrollPane } from "./scroll-pane";-const CARD_SLOT =+export const CARD_SLOT ="shrink-0 w-[calc((100%_-_var(--spacing)*8)/5)] min-w-[calc(7rem*var(--ui-space-scale,1))]";-const CARD_SIZE = "h-[calc(7rem*var(--ui-space-scale,1))] w-full";+export const CARD_SIZE = "h-[calc(7rem*var(--ui-space-scale,1))] w-full";const SENT_IMAGE_SIZE = "size-[calc(9rem*var(--ui-space-scale,1))]";const SENT_IMAGE_SIZE_COMPACT = "size-[calc(5rem*var(--ui-space-scale,1))]";const SENT_ROW_WIDTH = "w-[calc(18rem*var(--ui-space-scale,1))]";-const CARD_EDGE =+export const CARD_EDGE ="border border-[color-mix(in_oklab,var(--foreground)_calc(12%*var(--contrast-edge-gain,1)),transparent)]";// No fill, so cards take the composer background; hover still tints.-const CARD_SURFACE =+export const CARD_SURFACE ="hover:bg-[color-mix(in_oklab,var(--foreground)_6%,transparent)]";const useAttachmentKind = (): {@@ -795,3 +795,5 @@ export const ComposerAddAttachment: FC = () => {</ComposerPrimitive.AddAttachment>);};++export { AttachmentKindIcon, FileCardBody };diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsxindex 7f9a12ca343..32fd5c5239b 100644--- a/studio/frontend/src/components/assistant-ui/thread.tsx+++ b/studio/frontend/src/components/assistant-ui/thread.tsx@@ -329,6 +329,7 @@ import { usePublishedFrame } from "@/features/settings/hooks/use-published-frameimport { useVoiceSettingsStore } from "@/features/settings/stores/voice-settings-store";import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params";import { isTauri } from "@/lib/api-base";+import { openFilePicker } from "@/lib/open-file-picker";import { InternetGlyph } from "@/lib/internet-icon";import { copyToClipboard } from "@/lib/copy-to-clipboard";import { MenuDismissGuard } from "@/lib/menu-dismiss-guard";@@ -2569,7 +2570,8 @@ const ThreadWelcome: FC<{return (<div className="aui-thread-welcome-root mx-auto my-auto flex w-full max-w-(--thread-max-width) grow flex-col">- <div className="aui-thread-welcome-center flex w-full grow flex-col items-center justify-start pt-[27.5dvh]">+ {/* With a files panel showing (chat with files, or project sources), center the whole block. */}+ <div className="aui-thread-welcome-center flex w-full grow flex-col items-center justify-start pt-[27.5dvh] has-[.unsloth-files-panel]:justify-center has-[.unsloth-files-panel]:pt-0 has-[.unsloth-files-panel]:pb-[6dvh]">{/* No padding, so the composer here is as wide as once it docks. */}<div className="aui-thread-welcome-message flex w-full flex-col justify-center gap-9">{/* Center the greeting (sloth + title) over the composer. */}@@ -2611,6 +2613,8 @@ export const ProjectComposer: FC<{placeholder={placeholder}disableQueue/>+ {/* Its file cards open previews; the thread's mount is not on this page. */}+ <DocumentPreviewMount /></GeneratedImageOverlayProvider>);};@@ -5434,14 +5438,6 @@ const Composer: FC<{<PendingAudioChip /></>) : null}- {/* Keep indexing state subscribed while dictating, but hide its chips so- the waveform stays the composer's only status indicator. */}- <div className={isDictating ? "hidden" : "contents"}>- <ThreadDocumentsBar- threadId={referenceThreadId}- onIndexingChange={handleIndexingChange}- />- </div>{!isDictating ? <ComposerDraftPreview text={composerText} /> : null}{!isDictating ? <ToolStatusDisplay /> : null}<div@@ -5631,6 +5627,13 @@ const Composer: FC<{onSubmit={handleSubmit}><PromptQueueStack queueThreadIds={promptQueueThreadIds} />+ {/* Chat with files card. Stays mounted while dictating to keep indexing state. */}+ <div className={isDictating ? "hidden" : "contents"}>+ <ThreadDocumentsBar+ threadId={referenceThreadId}+ onIndexingChange={handleIndexingChange}+ />+ </div>{youtubeOfferUrl && !isDictating && !disabled ? (// Keyed by URL: pasting a second link while the first is still fetching// remounts the prompt, so its cleanup aborts the request that is no@@ -6539,6 +6542,30 @@ function attachmentAcceptForPicker(accept: string, audioEnabled: boolean): strinreturn pickerAcceptForTextBasenames(enabledAccept);}+function usePickComposerAttachment(): () => void {+ const aui = useAui();+ const audioAttachmentsEnabled = useChatRuntimeStore((s) => {+ const activeCheckpoint = s.params.checkpoint;+ // No model yet: offer audio too, since files attached now wait for the model loaded next.+ if (!activeCheckpoint || s.modelLoading) {+ return true;+ }+ const activeModel = s.models.find((m) => m.id === activeCheckpoint);+ return Boolean(activeModel?.hasAudioInput);+ });+ return useCallback(() => {+ const attachmentAccept = attachmentAcceptForPicker(+ aui.composer().getState().attachmentAccept,+ audioAttachmentsEnabled,+ );+ openFilePicker(attachmentAccept, (files) => {+ for (const file of files) {+ void aui.composer().addAttachment(file);+ }+ });+ }, [aui, audioAttachmentsEnabled]);+}+const ComposerToolsMenu: FC<{side?: "top" | "bottom";researchAvailable: boolean;@@ -6568,15 +6595,6 @@ const ComposerToolsMenu: FC<{const modelLoaded = useChatRuntimeStore((s) => !!s.params.checkpoint && !s.modelLoading,);- const audioAttachmentsEnabled = useChatRuntimeStore((s) => {- const activeCheckpoint = s.params.checkpoint;- // No model yet: offer audio too, since files attached now wait for the model loaded next.- if (!activeCheckpoint || s.modelLoading) {- return true;- }- const activeModel = s.models.find((m) => m.id === activeCheckpoint);- return Boolean(activeModel?.hasAudioInput);- });const checkpoint = useChatRuntimeStore((s) => s.params.checkpoint);const supportsTools = useChatRuntimeStore((s) => s.supportsTools);const supportsBuiltinWebSearch = useChatRuntimeStore(@@ -6652,37 +6670,7 @@ const ComposerToolsMenu: FC<{const composerCanAddAttachments = useAuiState(({ composer }) => composer.isEditing,);- const pickAttachment = useCallback(() => {- const input = document.createElement("input");- input.type = "file";- input.multiple = true;- input.hidden = true;-- const attachmentAccept = attachmentAcceptForPicker(- aui.composer().getState().attachmentAccept,- audioAttachmentsEnabled,- );- if (attachmentAccept !== "*") {- input.accept = attachmentAccept;- }-- document.body.appendChild(input);- input.onchange = (event) => {- const files = (event.target as HTMLInputElement).files;- if (files) {- for (const file of files) {- void aui.composer().addAttachment(file);- }- }- document.body.removeChild(input);- };- input.oncancel = () => {- if (!input.files || input.files.length === 0) {- document.body.removeChild(input);- }- };- input.click();- }, [aui, audioAttachmentsEnabled]);+ const pickAttachment = usePickComposerAttachment();// Straight to the picker, skipping the "+" menu the item lives in. Off-route// the chat pane is hidden rather than unmounted, so the chords gate on it// being the visible tab; a window listener does not care about `inert`.diff --git a/studio/frontend/src/features/native-intents/drop-paths.ts b/studio/frontend/src/features/native-intents/drop-paths.tsindex df322210d76..3cc714b694f 100644--- a/studio/frontend/src/features/native-intents/drop-paths.ts+++ b/studio/frontend/src/features/native-intents/drop-paths.ts@@ -14,7 +14,10 @@ import {TEXT_ATTACHMENT_EXTENSIONS,isTextAttachmentName,} from "../chat/text-attachment-accept.ts";-import { RAG_UPLOAD_ACCEPT } from "../rag/types/rag.ts";+import {+ RAG_DOCUMENT_UPLOAD_ACCEPT,+ RAG_UPLOAD_ACCEPT,+} from "../rag/types/rag.ts";import { MAX_AUDIO_FILES } from "../../lib/audio-utils.ts";const DOC_EXTS = RAG_UPLOAD_ACCEPT.split(",").map((ext) =>@@ -36,6 +39,13 @@ export function isComposerAttachmentName(path: string): boolean {);}+/** Documents only chat with files reads; composer-handled types stay there. */+const OFFICE_DOC_EXTS = RAG_DOCUMENT_UPLOAD_ACCEPT.split(",").filter(+ (ext) => !isComposerAttachmentName(`file${ext}`),+);++const DROP_DOC_EXTS = [...DOC_EXTS, ...OFFICE_DOC_EXTS];+function isTextDropName(path: string): boolean {const name = nativeFileName(path).toLowerCase();if (!name.includes(".") && isTextAttachmentName(name)) {@@ -71,7 +81,7 @@ const VIDEO_EXTS = CHAT_VIDEO_DROP_ACCEPT.split(",").map((ext) =>);/** What the window actually takes, for the rejection toast and the overlay. */-export const SUPPORTED_DROP_HINT = `Supported files: ${RAG_UPLOAD_ACCEPT}, ${OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS}, ${RTF_ATTACHMENT_EXTENSIONS}, source and text files, ${TOOL_ONLY_ATTACHMENT_EXTENSIONS} with Code on, ${CHAT_IMAGE_DROP_ACCEPT}, up to ${MAX_AUDIO_FILES} of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`;+export const SUPPORTED_DROP_HINT = `Supported files: ${DROP_DOC_EXTS.join(",")}, ${OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS}, ${RTF_ATTACHMENT_EXTENSIONS}, source and text files, ${TOOL_ONLY_ATTACHMENT_EXTENSIONS} with Code on, ${CHAT_IMAGE_DROP_ACCEPT}, up to ${MAX_AUDIO_FILES} of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`;/** Last path segment of a native path, for display and extension checks. */export function nativeFileName(path: string): string {@@ -111,7 +121,7 @@ export function classifyDropPaths(paths: string[]): NativeDropClass {}const docs = paths.filter((path) =>- DOC_EXTS.some((ext) => hasExt(path, ext)) ||+ DROP_DOC_EXTS.some((ext) => hasExt(path, ext)) ||isComposerAttachmentName(path),);const images = paths.filter((path) =>diff --git a/studio/frontend/src/features/rag/components/document-status-chip.tsx b/studio/frontend/src/features/rag/components/document-status-chip.tsxindex 2f18536d6b0..4cd16b95c4a 100644--- a/studio/frontend/src/features/rag/components/document-status-chip.tsx+++ b/studio/frontend/src/features/rag/components/document-status-chip.tsx@@ -9,7 +9,7 @@ import { HugeiconsIcon } from "@hugeicons/react";import { XIcon } from "lucide-react";import type { DocumentStatus } from "../types/rag";-const STAGE_LABELS: Record<string, string> = {+export const STAGE_LABELS: Record<string, string> = {parsing: "Reading document",ocr: "Reading scanned pages",captioning: "Reading charts and figures",diff --git a/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx b/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsxindex 5fbb6bc780c..e7089b56cda 100644--- a/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx+++ b/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx@@ -509,8 +509,8 @@ function KnowledgeBaseDocuments({dragging && "border-primary/60 bg-primary/5 text-foreground",)}>- No documents yet. Upload or drop a PDF, Markdown, DOCX, HTML, or text- file.+ No documents yet. Upload or drop documents, spreadsheets, slides,+ e-books, email, text or code.</div>) : (<divdiff --git a/studio/frontend/src/features/rag/components/project-source-dropzone.tsx b/studio/frontend/src/features/rag/components/project-source-dropzone.tsxindex 457c1f36adc..1d355e3f974 100644--- a/studio/frontend/src/features/rag/components/project-source-dropzone.tsx+++ b/studio/frontend/src/features/rag/components/project-source-dropzone.tsx@@ -22,6 +22,7 @@ import {} from "../api/rag-api";import {RAG_SOURCE_UPLOAD_ACCEPT,+ SUPPORTED_SOURCES_HINT,partitionSupported,} from "./source-drop-policy";import {@@ -253,7 +254,7 @@ export function ProjectSourceDropzone({unsupported.length === 1? `Can't add ${unsupported[0]}`: `Can't add ${unsupported.length} files`,- { description: "Supported types: documents and source code files" },+ { description: SUPPORTED_SOURCES_HINT },);}// Name, size and mtime can in principle match for two different files, sodiff --git a/studio/frontend/src/features/rag/components/project-sources-panel.tsx b/studio/frontend/src/features/rag/components/project-sources-panel.tsxindex 5fb0d56da83..64279e10c2b 100644--- a/studio/frontend/src/features/rag/components/project-sources-panel.tsx+++ b/studio/frontend/src/features/rag/components/project-sources-panel.tsx@@ -150,8 +150,8 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {Give this project context</p><p className="max-w-sm text-sm text-muted-foreground">- Upload PDFs, docs, or text. Every chat in this project can use- them.+ Add documents, spreadsheets, slides, e-books, email, text or+ code. Every chat in this project can use them.</p></div><Buttondiff --git a/studio/frontend/src/features/rag/components/source-drop-policy.ts b/studio/frontend/src/features/rag/components/source-drop-policy.tsindex 86dee573ce3..76dfcf697d6 100644--- a/studio/frontend/src/features/rag/components/source-drop-policy.ts+++ b/studio/frontend/src/features/rag/components/source-drop-policy.ts@@ -2,13 +2,20 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import { TEXT_ATTACHMENT_EXTENSIONS } from "../../chat/text-attachment-accept.ts";-import { RAG_UPLOAD_ACCEPT } from "../types/rag.ts";+import { RAG_DOCUMENT_UPLOAD_ACCEPT, RAG_UPLOAD_ACCEPT } from "../types/rag.ts";-// Projects and knowledge bases also index source files; backend SOURCE_TEXT_EXTS mirrors this list.+// Everything the backend indexes (config.UPLOAD_EXTS), for every scope: chat, project and knowledge base.export const RAG_SOURCE_UPLOAD_ACCEPT = [- ...new Set([...RAG_UPLOAD_ACCEPT.split(","), ...TEXT_ATTACHMENT_EXTENSIONS]),+ ...new Set([+ ...RAG_UPLOAD_ACCEPT.split(","),+ ...RAG_DOCUMENT_UPLOAD_ACCEPT.split(","),+ ...TEXT_ATTACHMENT_EXTENSIONS,+ ]),].join(",");+export const SUPPORTED_SOURCES_HINT =+ "Supported: documents, spreadsheets, slides, e-books, email, text and code files.";+const ACCEPTED_UPLOAD_EXTS = new Set(RAG_SOURCE_UPLOAD_ACCEPT.split(","));// `accept` only filters the picker, so a drop can carry anything, including andiff --git a/studio/frontend/src/features/rag/components/thread-documents-bar.tsx b/studio/frontend/src/features/rag/components/thread-documents-bar.tsxindex 845527ef12b..572615ab738 100644--- a/studio/frontend/src/features/rag/components/thread-documents-bar.tsx+++ b/studio/frontend/src/features/rag/components/thread-documents-bar.tsx@@ -2,12 +2,15 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import {- type RefObject,+ type ReactNode,useCallback,useEffect,+ useLayoutEffect,+ useMemo,useRef,useState,} from "react";+import { ChevronLeftIcon, ChevronRightIcon, PlusIcon, XIcon } from "lucide-react";import { HugeiconsIcon } from "@hugeicons/react";import {AttachmentIcon,@@ -15,6 +18,31 @@ import {FolderAttachmentIcon,Folder02Icon,} from "@hugeicons/core-free-icons";+import {+ AttachmentKindIcon,+ CARD_EDGE,+ CARD_SIZE,+ CARD_SLOT,+ CARD_SURFACE,+ FileCardBody,+} from "@/components/assistant-ui/attachment";+import {+ type AttachmentPreview,+ AttachmentCardPreview,+ attachmentPreview,+} from "@/components/assistant-ui/attachment-card-preview";+import { LocalFileDialog } from "@/components/assistant-ui/attachment-document-dialog";+import {+ MAX_DOCUMENT_PREVIEW_BYTES,+ documentKind,+ isMarkdown,+} from "@/components/file-viewer";+import { Spinner } from "@/components/ui/spinner";+import {+ Tooltip,+ TooltipContent,+ TooltipTrigger,+} from "@/components/ui/tooltip";import { Tick02Icon } from "@/lib/tick-icon";import { ChevronDownStandardIcon } from "@/lib/chevron-icons";import { useAui } from "@assistant-ui/react";@@ -31,12 +59,17 @@ import {ChatThreadDeletedError,ensureStoredChatThread,getStoredChatThread,+ annotationsOfFile,+ attachmentFileKind,+ isPastedTextFile,+ isTextAttachment,isThreadIncognito,} from "@/features/chat";import {useNativeAttachmentTargetKey,useNativeIntentStore,} from "@/features/native-intents";+import { openFilePicker } from "@/lib/open-file-picker";import { toast } from "@/lib/toast";import {DropdownMenu,@@ -47,19 +80,16 @@ import {} from "@/components/ui/dropdown-menu";import {announceProjectSourcesUpdated,+ getDocumentFileUrl,invalidateProjectSources,listKnowledgeBases,+ listLinkedFolders,subscribeKnowledgeBasesChanged,listProjectDocuments,listThreadDocuments,} from "../api/rag-api";import { useRagAvailabilityStore } from "../api/rag-availability";-import {- type DocumentStatus,- RAG_UPLOAD_ACCEPT,- type RagDocument,- isLinkedFolderManaged,-} from "../types/rag";+import { type RagDocument, isLinkedFolderManaged } from "../types/rag";import {AlertDialog,AlertDialogAction,@@ -70,13 +100,25 @@ import {AlertDialogHeader,AlertDialogTitle,} from "@/components/ui/alert-dialog";-import { DocumentStatusChip } from "./document-status-chip";+import { STAGE_LABELS } from "./document-status-chip";+import { useDocumentPreviewStore } from "./preview-store";import {type KnowledgeBaseFocus,KnowledgeBaseDialog,} from "./knowledge-base-dialog";+import {+ RAG_SOURCE_UPLOAD_ACCEPT,+ SUPPORTED_SOURCES_HINT,+ isSupportedSourceName,+} from "./source-drop-policy";import { EXPIRY_GRACE_MS } from "./staged-source";-import { uploadItemFromIntent, useRagDocuments } from "./use-rag-documents";+import {+ type RagUploadItem,+ type TrackedDocument,+ uploadItemFromIntent,+ useRagDocuments,+} from "./use-rag-documents";+import { useSourceDrop } from "./use-source-drop";// Refetched after any KB mutation so a rename shows at once.function useKnowledgeBaseName(kbId: string | null): string | null {@@ -109,38 +151,6 @@ function useKnowledgeBaseName(kbId: string | null): string | null {return known !== null && known.kbId === kbId ? known.name : null;}-// Shown when retrieval comes from a KB, so the source isn't invisible.-function KnowledgeBaseSourceChip({- name,- onOpen,- buttonRef,-}: {- name: string | null;- onOpen: () => void;- buttonRef: RefObject<HTMLButtonElement | null>;-}) {- return (- <div className="mb-2 flex w-full flex-row items-center gap-1.5 pl-0.5 pr-1.5 pt-0.5 pb-1">- <button- ref={buttonRef}- type="button"- onClick={onOpen}- className="composer-pill-btn min-w-0 max-w-full"- title="Add or remove documents"- >- <HugeiconsIcon- icon={FileDatabaseIcon}- strokeWidth={2}- className="size-3.5 shrink-0"- />- <span className="min-w-0 truncate">- {name ? `Knowledge base: ${name}` : "Knowledge base"}- </span>- </button>- </div>- );-}-/*** Confirm a thread is stored before documents are indexed against it. An id reaches this* component before its row write lands, from a cached initialize() or from activeThreadId, and@@ -164,47 +174,6 @@ async function requireStoredThread(threadId: string): Promise<void> {}}-/** shows inherited sources because Docs off does not stop retrieval. */-function InheritedProjectSources({- documents,- unused,-}: {- documents: { id: string; filename: string; status: DocumentStatus }[];- unused: boolean;-}) {- return (- <div className="mb-2 flex w-full flex-row items-center gap-1.5 pl-0.5 pr-1.5 pt-0.5 pb-1">- <span- className="composer-pill-btn shrink-0 cursor-default !text-foreground/60"- title={- unused- ? "The selected model can't search documents, so this chat doesn't use its project's sources. Pick a model with tool support to use them."- : "This chat retrieves from its project's sources. Manage them in the project's Sources tab."- }- >- <HugeiconsIcon icon={FolderAttachmentIcon} strokeWidth={2} className="size-3.5" />- <span>{unused ? "Project sources not used" : "Project sources"}</span>- </span>- {/* cap linked folders because their sources can fill the viewport. */}- <div- className={cn(- "flex max-h-24 flex-1 flex-row flex-wrap items-center gap-1.5 overflow-y-auto",- unused && "opacity-50",- )}- >- {documents.map((doc) => (- <DocumentStatusChip- key={`inherited:${doc.id}`}- filename={doc.filename}- status={doc.status}- shared={true}- />- ))}- </div>- </div>- );-}-/** row data avoids stale activeProjectId; undefined blocks attachments. *//** retries chat row reads before leaving the project unresolved. */const PROJECT_LOOKUP_RETRIES = 3;@@ -266,54 +235,580 @@ function useThreadProjectId(: undefined;}-/** The composer's attach control. Wording and glyph follow the active target, so- * a project chat says up front where the file is going. */-function AttachFilesButton({+function isRagIndexable(file: File): boolean {+ // Pasted text and annotations are not documents.+ if (isPastedTextFile(file) || annotationsOfFile(file)) return false;+ return isSupportedSourceName(file.name);+}++/** Display names of a project's linked folders, read once per set of folder ids. */+function useLinkedFolderNames(+ projectId: string | null,+ folderIds: string,+): ReadonlyMap<string, string> {+ const [names, setNames] = useState<ReadonlyMap<string, string>>(new Map());+ useEffect(() => {+ if (!projectId || !folderIds) return;+ let cancelled = false;+ listLinkedFolders({ type: "project", id: projectId })+ .then((folders) => {+ if (cancelled) return;+ setNames(new Map(folders.map((f) => [f.id, f.displayName])));+ })+ .catch(() => {+ // The card falls back to a generic label.+ });+ return () => {+ cancelled = true;+ };+ }, [projectId, folderIds]);+ return names;+}++/** Splits project documents into loose files and per-folder groups. A linked folder can+ * hold thousands of files, so it is drawn as one card rather than one card per file. */+function groupByLinkedFolder(docs: TrackedDocument[]): {+ loose: TrackedDocument[];+ folders: [string, TrackedDocument[]][];+} {+ const loose: TrackedDocument[] = [];+ const folders = new Map<string, TrackedDocument[]>();+ for (const doc of docs) {+ const folderId = doc.linkedFolderId;+ if (!folderId) {+ loose.push(doc);+ continue;+ }+ const group = folders.get(folderId);+ if (group) group.push(doc);+ else folders.set(folderId, [doc]);+ }+ return { loose, folders: [...folders] };+}++function isProcessing(doc: TrackedDocument): boolean {+ return doc.status === "pending" || doc.status === "running";+}++function ProjectBadge() {+ return (+ <span+ className="pointer-events-none absolute top-1.5 left-1.5 inline-flex items-center gap-1 rounded-full bg-muted px-1.5 py-0.5 text-ui-10 text-muted-foreground"+ aria-hidden={true}+ >+ <HugeiconsIcon icon={Folder02Icon} strokeWidth={2} className="size-3" />+ Project+ </span>+ );+}++function LinkedFolderCard({+ name,+ docs,+}: {+ name: string | undefined;+ docs: TrackedDocument[];+}) {+ const indexing = docs.filter(isProcessing).length;+ const label = name ?? "Linked folder";+ const count = `${docs.length} ${docs.length === 1 ? "file" : "files"}`;+ return (+ <div className={cn("relative", CARD_SLOT)}>+ <ProjectBadge />+ <div+ title={`${label}: ${count}, kept in sync and shared with every chat in this project. Manage it in the project's Sources tab.`}+ className={cn("flex overflow-hidden rounded-[15px]", CARD_SIZE, CARD_EDGE)}+ >+ <FileCardBody+ name={label}+ kind="document"+ icon={+ <HugeiconsIcon+ icon={Folder02Icon}+ strokeWidth={1.75}+ className="size-3.25 shrink-0 text-muted-foreground"+ />+ }+ center={+ <span className="flex flex-col items-center gap-1.5 text-center text-ui-11">+ {indexing > 0 ? (+ <Spinner className="size-4" />+ ) : (+ <HugeiconsIcon icon={Folder02Icon} strokeWidth={1.5} className="size-6" />+ )}+ <span>{indexing > 0 ? `Indexing ${indexing} of ${docs.length}` : count}</span>+ <span className="sr-only">linked folder, shared with the project</span>+ </span>+ }+ />+ </div>+ </div>+ );+}++// Same cap as composer attachment previews; bigger files keep the icon.+const MAX_THUMBNAIL_BYTES = 10 * 1024 * 1024;+const THUMBNAIL_CACHE_BYTES = 48 * 1024 * 1024;+// Source files fetched for card thumbnails, by document id, so a re-render or a return to the chat+// does not download them again. Insertion order doubles as LRU order.+const thumbnailFiles = new Map<string, Promise<File | null>>();+const thumbnailSizes = new Map<string, number>();+let thumbnailBytes = 0;++function forgetThumbnail(id: string): void {+ thumbnailFiles.delete(id);+ thumbnailBytes -= thumbnailSizes.get(id) ?? 0;+ thumbnailSizes.delete(id);+}++/** The stored source as a File, or null past `maxBytes` (0 = no cap). */+async function fetchSourceFile(+ doc: TrackedDocument,+ maxBytes: number,+): Promise<File | null> {+ const response = await fetch(await getDocumentFileUrl(doc.id));+ if (!response.ok) return null;+ if (maxBytes && Number(response.headers.get("content-length")) > maxBytes) {+ void response.body?.cancel();+ return null;+ }+ const blob = await response.blob();+ if (blob.size === 0 || (maxBytes && blob.size > maxBytes)) return null;+ // Linked-folder documents are named by their relative path.+ const name = doc.filename.split("/").pop() ?? doc.filename;+ // Most types are served as octet-stream; an empty type lets the extension decide the kind.+ const type = blob.type === "application/octet-stream" ? "" : blob.type;+ return new File([blob], name, { type });+}++const fetchThumbnailFile = (doc: TrackedDocument) =>+ fetchSourceFile(doc, MAX_THUMBNAIL_BYTES);++/** Whether the attachment viewer can show this source: pages, markdown or text. */+function isViewableSource(name: string): boolean {+ return (+ documentKind(name) !== null ||+ isMarkdown(name) ||+ isTextAttachment(name, undefined)+ );+}++function loadThumbnailFile(doc: TrackedDocument): Promise<File | null> {+ const cached = thumbnailFiles.get(doc.id);+ if (cached) {+ thumbnailFiles.delete(doc.id);+ thumbnailFiles.set(doc.id, cached);+ return cached;+ }+ const pending = fetchThumbnailFile(doc).catch(() => null);+ thumbnailFiles.set(doc.id, pending);+ void pending.then((file) => {+ if (thumbnailFiles.get(doc.id) !== pending) return;+ // A failure is not remembered, so the next mount tries again.+ if (file === null) {+ forgetThumbnail(doc.id);+ return;+ }+ thumbnailSizes.set(doc.id, file.size);+ thumbnailBytes += file.size;+ for (const oldest of thumbnailFiles.keys()) {+ if (thumbnailBytes <= THUMBNAIL_CACHE_BYTES || oldest === doc.id) break;+ forgetThumbnail(oldest);+ }+ });+ return pending;+}++type Thumbnail =+ | { kind: "image"; url: string }+ | { kind: "file"; file: File; preview: AttachmentPreview };++/** The card's thumbnail while the card is on screen; off screen it lets go of the file, which the+ * byte-capped cache may still hold for its return. */+function useDocumentThumbnail(+ doc: TrackedDocument,+ enabled: boolean,+ element: HTMLElement | null,+): Thumbnail | null {+ const [visible, setVisible] = useState(false);+ const [thumbnail, setThumbnail] = useState<Thumbnail | null>(null);+ useEffect(() => {+ if (!enabled || !element) return;+ const observer = new IntersectionObserver(+ (entries) => setVisible(entries[entries.length - 1].isIntersecting),+ { rootMargin: "200px" },+ );+ observer.observe(element);+ return () => observer.disconnect();+ }, [enabled, element]);+ useEffect(() => {+ if (!enabled || !visible) {+ setThumbnail(null);+ return;+ }+ let cancelled = false;+ let objectUrl: string | null = null;+ void loadThumbnailFile(doc).then((file) => {+ if (cancelled || !file) return;+ const kind = attachmentFileKind(file.name, file.type);+ if (kind === "image") {+ // An <img> renders an SVG only with its type; it never runs the SVG's scripts.+ const svg = /\.svg$/i.test(file.name) && !file.type;+ objectUrl = URL.createObjectURL(+ svg ? new Blob([file], { type: "image/svg+xml" }) : file,+ );+ setThumbnail({ kind: "image", url: objectUrl });+ return;+ }+ const preview = attachmentPreview(file, kind);+ if (preview) setThumbnail({ kind: "file", file, preview });+ });+ return () => {+ cancelled = true;+ if (objectUrl) URL.revokeObjectURL(objectUrl);+ };+ // The id names the stored file; the rest of the row changes with progress frames.+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [doc.id, enabled, visible]);+ return enabled ? thumbnail : null;+}++function DocumentCard({+ doc,+ shared,+ onRemove,+}: {+ doc: TrackedDocument;+ shared: boolean;+ onRemove?: () => void;+}) {+ const openPreview = useDocumentPreviewStore((s) => s.openPreview);+ const kind = attachmentFileKind(doc.filename, undefined);+ const processing = isProcessing(doc);+ const failed = doc.status === "failed";+ const pending = doc.id.startsWith("pending_");+ const percent =+ doc.progress != null+ ? Math.round(doc.progress <= 1 ? doc.progress * 100 : doc.progress)+ : null;+ const stageLabel = (doc.stage && STAGE_LABELS[doc.stage]) || "Indexing";+ const center = processing ? (+ <span className="flex flex-col items-center gap-1.5 text-center text-ui-11">+ <Spinner className="size-4" />+ <span className="line-clamp-2">+ {stageLabel}+ {percent != null ? ` ${percent}%` : null}+ </span>+ </span>+ ) : failed ? (+ <span className="line-clamp-3 text-center text-ui-11 text-destructive">+ Couldn't index+ </span>+ ) : (+ <AttachmentKindIcon kind={kind} className="size-6" />+ );+ const ready = doc.status === "completed" && !pending;+ const [slot, setSlot] = useState<HTMLDivElement | null>(null);+ const thumbnail = useDocumentThumbnail(doc, ready, slot);+ // PDFs open the source preview, which can jump to a page. Without a chunk it has no text for+ // anything else, so other files open in the attachment viewer, read on click.+ const isPdf = kind === "pdf";+ const viewable = !isPdf && isViewableSource(doc.filename);+ const thumbnailFile = thumbnail?.kind === "file" ? thumbnail.file : null;+ const loadSource = useCallback(async (): Promise<Blob> => {+ // The thumbnail's copy when there is one; past its cap, up to the viewer's own limit, the+ // same one the Library holds to. Past that the viewer says it can't preview.+ const file =+ thumbnailFile ?? (await fetchSourceFile(doc, MAX_DOCUMENT_PREVIEW_BYTES));+ if (!file) throw new Error(`Couldn't preview ${doc.filename}`);+ const textual =+ documentKind(file.name, file.type) === null && !isMarkdown(file.name, file.type);+ // Served as octet-stream: the viewer reads text only when the type says so.+ return textual ? new File([file], file.name, { type: "text/plain" }) : file;+ // The id names the stored file; the rest of the row changes with progress frames.+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [doc.id, thumbnailFile]);+ const canOpen = ready && (isPdf || viewable);+ const preview =+ thumbnail?.kind === "image" ? (+ <img+ src={thumbnail.url}+ alt=""+ className="h-full w-full object-cover"+ />+ ) : thumbnail?.kind === "file" ? (+ <AttachmentCardPreview+ file={thumbnail.file}+ preview={thumbnail.preview}+ fallback={+ <span className="flex h-full items-center justify-center">+ <AttachmentKindIcon kind={kind} className="size-6" />+ </span>+ }+ />+ ) : undefined;+ const card = (+ <button+ type="button"+ disabled={!canOpen}+ onClick={+ isPdf+ ? () => openPreview({ documentId: doc.id, filename: doc.filename })+ : undefined+ }+ title={+ doc.error ??+ (shared+ ? `${doc.filename}: shared with every chat in this project`+ : doc.filename)+ }+ aria-label={[+ canOpen ? `Preview ${doc.filename}` : doc.filename,+ processing ? `${stageLabel}${percent != null ? ` ${percent}%` : ""}` : null,+ failed ? "couldn't index" : null,+ shared ? "shared with the project" : null,+ ]+ .filter(Boolean)+ .join(", ")}+ className={cn(+ "flex overflow-hidden rounded-[15px] text-left transition-colors disabled:cursor-default",+ CARD_SIZE,+ CARD_EDGE,+ canOpen && CARD_SURFACE,+ canOpen && "cursor-pointer",+ failed && "border-destructive/40",+ )}+ >+ <FileCardBody+ name={doc.filename}+ kind={kind}+ center={center}+ preview={preview}+ />+ </button>+ );+ return (+ <div+ ref={setSlot}+ className={cn(+ "group/attachment-card relative",+ CARD_SLOT,+ )}+ >+ {viewable && ready ? (+ <LocalFileDialog name={doc.filename.split("/").pop() ?? doc.filename} load={loadSource}>+ {card}+ </LocalFileDialog>+ ) : (+ card+ )}+ {shared ? <ProjectBadge /> : null}+ {onRemove && !processing ? (+ <button+ type="button"+ onClick={onRemove}+ aria-label={`Remove ${doc.filename}`}+ className="absolute top-1.5 right-1.5 flex size-5 items-center justify-center rounded-full bg-foreground text-background opacity-0 shadow-sm transition-opacity focus-visible:opacity-100 group-hover/attachment-card:opacity-100 group-focus-within/attachment-card:opacity-100 [@media(pointer:coarse)]:opacity-100"+ >+ <XIcon className="size-3 stroke-[2.5px]" />+ </button>+ ) : null}+ </div>+ );+}++function AddFilesCard({disabled,- compact,- sharesWithProject,onClick,}: {disabled: boolean;- /** Icon-only once documents are attached, to leave the chips room. */- compact: boolean;- sharesWithProject: boolean;onClick: () => void;}) {return (- <button- type="button"- onClick={onClick}- disabled={disabled}- className={cn(- "composer-pill-btn shrink-0 -translate-y-px !text-foreground/80",- // Square button so the rounded-full hover reads as a circle.- compact && "size-8 justify-center px-0",- )}- aria-label={- sharesWithProject- ? "Attach documents to this project"- : "Attach documents to this thread"- }- title={- sharesWithProject- ? "Attach documents for retrieval, shared with every chat in this project"- : "Attach documents for retrieval in this chat"- }+ <div className={CARD_SLOT}>+ <button+ type="button"+ disabled={disabled}+ onClick={onClick}+ className={cn(+ "unsloth-files-add-card flex flex-col items-center justify-center gap-1.5 rounded-[15px] border border-dashed border-[color-mix(in_oklab,var(--foreground)_calc(20%*var(--contrast-edge-gain,1)),transparent)] text-ui-12 text-muted-foreground transition-colors hover:border-[color-mix(in_oklab,var(--foreground)_calc(35%*var(--contrast-edge-gain,1)),transparent)] hover:text-foreground",+ CARD_SIZE,+ )}+ >+ <PlusIcon className="size-5 stroke-[1.5px]" />+ Add files+ </button>+ </div>+ );+}++/** Card above the composer: a header with controls over a strip of the chat's files. */+function ChatFilesPanel({+ icon,+ title,+ titleSuffix,+ note,+ headerControls,+ onClose,+ closeLabel,+ onDropItems,+ dropDisabledReason,+ info,+ count,+ children,+}: {+ icon: typeof Folder02Icon;+ title: string;+ titleSuffix?: string;+ /** Shown in a tooltip on the file count. */+ info?: string;+ count?: string;+ note?: ReactNode;+ headerControls?: ReactNode;+ onClose?: () => void;+ closeLabel?: string;+ /** Handles browser and desktop drops on the panel. Unset, drops fall through to the composer. */+ onDropItems?: (items: RagUploadItem[]) => void;+ /** Set while the panel can't take files: a drop is still claimed and refused with this. */+ dropDisabledReason?: string;+ children?: ReactNode;+}) {+ const stripRef = useRef<HTMLDivElement>(null);+ const [scroll, setScroll] = useState({ back: false, forward: false });+ const measure = useCallback(() => {+ const el = stripRef.current;+ if (!el) return;+ const back = el.scrollLeft > 1;+ const forward = el.scrollWidth - el.clientWidth - el.scrollLeft > 1;+ setScroll((prev) =>+ prev.back === back && prev.forward === forward ? prev : { back, forward },+ );+ }, []);+ useLayoutEffect(() => {+ const el = stripRef.current;+ if (!el) return;+ measure();+ const observer = new ResizeObserver(measure);+ observer.observe(el);+ // Adding a card does not resize the strip.+ const mutations = new MutationObserver(measure);+ mutations.observe(el, { childList: true });+ return () => {+ observer.disconnect();+ mutations.disconnect();+ };+ }, [measure]);+ const page = (direction: 1 | -1) => {+ const el = stripRef.current;+ if (!el) return;+ el.scrollBy({ left: direction * el.clientWidth * 0.8, behavior: "smooth" });+ };+ const overflowing = scroll.back || scroll.forward;+ // Drops here are indexed; elsewhere they attach to the message. The page handler skips+ // prevented drops, and the desktop's window-wide handler skips registered targets.+ const ignoreDrop = useCallback(() => {}, []);+ const { dragging, dropProps, nativeDropTarget } = useSourceDrop({+ onItems: onDropItems ?? ignoreDrop,+ disabledReason: dropDisabledReason,+ });+ const acceptsDrop = Boolean(onDropItems);+ const heading = (+ <h2 className="min-w-0 truncate text-ui-14 font-medium text-foreground">+ {title}+ {titleSuffix ? (+ <span className="font-normal text-muted-foreground"> {titleSuffix}</span>+ ) : null}+ </h2>+ );+ return (+ <section+ ref={acceptsDrop ? nativeDropTarget : undefined}+ aria-label={title}+ className="unsloth-files-panel"+ data-dragging={acceptsDrop && dragging ? "true" : undefined}+ {...(acceptsDrop ? dropProps : {})}>- <HugeiconsIcon- icon={sharesWithProject ? Folder02Icon : AttachmentIcon}- strokeWidth={2}- className="size-3.5"- />- {compact ? null : (- <span>- {sharesWithProject- ? "Add files for this project"- : "Add files to chat with"}- </span>- )}- </button>+ {acceptsDrop ? (+ <div className="unsloth-files-panel-drop-overlay" aria-hidden={true}>+ <PlusIcon className="size-5" />+ Drop to chat with these files+ </div>+ ) : null}+ <div className="flex min-h-8 items-center gap-2 pl-1">+ <HugeiconsIcon+ icon={icon}+ strokeWidth={1.75}+ className="size-4 shrink-0 text-muted-foreground"+ />+ {heading}+ {count && info ? (+ // Only resting on the count opens it, and it shuts on leaving: it must not linger over the cards.+ <Tooltip delayDuration={300} disableHoverableContent={true}>+ <TooltipTrigger asChild={true}>+ <span+ // biome-ignore lint/a11y/noNoninteractiveTabindex: focus opens the tooltip for keyboard users.+ tabIndex={0}+ aria-label={`${count}. ${info}`}+ className="unsloth-files-panel-count cursor-default focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"+ >+ {count}+ </span>+ </TooltipTrigger>+ <TooltipContent className="max-w-[calc(300px*var(--ui-space-scale,1))] text-ui-11 leading-snug">+ {info}+ </TooltipContent>+ </Tooltip>+ ) : count ? (+ <span className="unsloth-files-panel-count">{count}</span>+ ) : null}+ {note}+ <div className="ml-auto flex shrink-0 items-center gap-1">+ {headerControls}+ {overflowing ? (+ <>+ <button+ type="button"+ className="unsloth-files-panel-icon-btn"+ onClick={() => page(-1)}+ disabled={!scroll.back}+ aria-label="Scroll files back"+ >+ <ChevronLeftIcon className="size-4" />+ </button>+ <button+ type="button"+ className="unsloth-files-panel-icon-btn"+ onClick={() => page(1)}+ disabled={!scroll.forward}+ aria-label="Scroll files forward"+ >+ <ChevronRightIcon className="size-4" />+ </button>+ </>+ ) : null}+ {onClose ? (+ <button+ type="button"+ className="unsloth-files-panel-icon-btn"+ onClick={onClose}+ aria-label={closeLabel ?? "Close"}+ title={closeLabel}+ >+ <XIcon className="size-4" />+ </button>+ ) : null}+ </div>+ </div>+ {children ? (+ <div+ ref={stripRef}+ onScroll={measure}+ className="unsloth-files-panel-strip"+ >+ {children}+ </div>+ ) : null}+ </section>);}@@ -358,7 +853,7 @@ function AttachmentTargetMenu({disabled={disabled}aria-label="Choose where attached files go"title="Choose where attached files go"- className="composer-pill-btn attachment-target-pill shrink-0 -translate-y-px gap-1 !text-foreground/70 pl-3 pr-1.5"+ className="composer-pill-btn attachment-target-pill shrink-0 gap-1 !text-foreground/70 pl-3 pr-1.5"><span className="text-ui-12">{sharesWithProject ? "Project" : "This chat"}@@ -371,7 +866,7 @@ function AttachmentTargetMenu({</button></DropdownMenuTrigger><DropdownMenuContent- align="start"+ align="end"className="unsloth-plus-menu w-[calc(328px*var(--ui-space-scale,1))]"><DropdownMenuLabel>New files go to</DropdownMenuLabel>@@ -433,7 +928,6 @@ export function ThreadDocumentsBar({(s) => s.setThreadProjectAttachmentTarget,);const aui = useAui();- const fileInputRef = useRef<HTMLInputElement>(null);// materialize locally; append() reuses the id without a mid-upload remount.const [materializedId, setMaterializedId] = useState<string | null>(null);@@ -616,6 +1110,27 @@ export function ThreadDocumentsBar({[ensureThreadId, projectId, sharesWithProject, upload, uploadToProject],);+ // Only files added or dropped here are indexed; composer attachments stay with the message.+ const attachIndexable = useCallback(+ (files: File[]) => {+ const indexable = files.filter(isRagIndexable);+ const skipped = files.length - indexable.length;+ if (skipped > 0) {+ toast.error(+ skipped === 1+ ? `"${files.find((file) => !isRagIndexable(file))?.name}" can't be searched`+ : `${skipped} files can't be searched`,+ { description: SUPPORTED_SOURCES_HINT },+ );+ }+ if (indexable.length > 0) attach(indexable);+ },+ [attach],+ );+ const pickFiles = useCallback(() => {+ openFilePicker(RAG_SOURCE_UPLOAD_ACCEPT, attachIndexable);+ }, [attachIndexable]);+// Desktop drops land in the native-intent store because the drop listener lives on// the chat page; only the chat that received the OS drop may drain its batch.const nativeAttachmentTargetKey = useNativeAttachmentTargetKey();@@ -703,27 +1218,28 @@ export function ThreadDocumentsBar({setRagEnabled,]);- const chipScrollRef = useRef<HTMLDivElement>(null);- const [chipsOverflow, setChipsOverflow] = useState(false);- // Removing a project source here deletes it for every chat, beside a chat chip+ // Removing a project source here deletes it for every chat, beside a chat card// whose X is undoable. Confirm, as the Sources tab and Settings do.const [removingShared, setRemovingShared] = useState<RagDocument | null>(null);- const updateChipFade = useCallback(() => {- const el = chipScrollRef.current;- if (!el) return;- setChipsOverflow(el.scrollHeight - el.scrollTop - el.clientHeight > 1);- }, []);- useEffect(() => {- updateChipFade();- }, [documents, updateChipFade]);-- // Open the picker synchronously to keep the click's user activation. Do NOT- // materialize here: setActiveThreadId while the native dialog is open can remount- // the composer and orphan this <input>. Materialize in onChange instead.- const handleAddDocs = useCallback(() => {- fileInputRef.current?.click();- }, []);+ const projectGroups = useMemo(+ () => groupByLinkedFolder(projectDocuments),+ [projectDocuments],+ );+ const folderNames = useLinkedFolderNames(+ projectId,+ projectGroups.folders.map(([id]) => id).join(","),+ );+ const folderCards = projectGroups.folders.map(([folderId, docs]) => (+ <div+ key={`folder:${folderId}`}+ className={cn("contents", ragToolDisabled && "[&>*]:opacity-50")}+ >+ <LinkedFolderCard name={folderNames.get(folderId)} docs={docs} />+ </div>+ ));+ const fileCount = documents.length + projectDocuments.length;+ const countSuffix = fileCount === 1 ? "1 file" : `${fileCount} files`;// Every branch, always the first fragment child: an "Add" must open after the source// moves, and deleting the active KB in it must not remount it.const kbDialog = (@@ -749,10 +1265,23 @@ export function ThreadDocumentsBar({return (<>{kbDialog}- <KnowledgeBaseSourceChip- name={activeKbName}- onOpen={() => setKbDialogFocus({ kbId })}- buttonRef={kbChipRef}+ {/* Keyed per mode: the strip's observers attach on mount, and this one has no strip. */}+ <ChatFilesPanel+ key="kb"+ icon={FileDatabaseIcon}+ title={activeKbName ? `Knowledge base: ${activeKbName}` : "Knowledge base"}+ onClose={() => setRagEnabled(false)}+ closeLabel="Stop chatting with files"+ headerControls={+ <button+ ref={kbChipRef}+ type="button"+ onClick={() => setKbDialogFocus({ kbId })}+ className="unsloth-files-panel-action"+ >+ Manage files+ </button>+ }/></>);@@ -763,10 +1292,36 @@ export function ThreadDocumentsBar({<>{kbDialog}{projectDocuments.length > 0 ? (- <InheritedProjectSources- documents={projectDocuments}- unused={ragToolDisabled}- />+ <ChatFilesPanel+ key="project"+ icon={FolderAttachmentIcon}+ title={ragToolDisabled ? "Project sources not used" : "Project sources"}+ count={countSuffix}+ note={+ <span+ className="hidden truncate text-ui-12 text-muted-foreground @[32rem]/files-panel:inline"+ title={+ ragToolDisabled+ ? "The selected model can't search documents, so this chat doesn't use its project's sources. Pick a model with tool support to use them."+ : "This chat retrieves from its project's sources. Manage them in the project's Sources tab."+ }+ >+ {ragToolDisabled+ ? "The selected model can't search documents"+ : "This chat retrieves from its project's sources"}+ </span>+ }+ >+ {folderCards}+ {projectGroups.loose.map((doc) => (+ <div+ key={`inherited:${doc.id}`}+ className={cn("contents", ragToolDisabled && "[&>*]:opacity-50")}+ >+ <DocumentCard doc={doc} shared={true} />+ </div>+ ))}+ </ChatFilesPanel>) : null}</>);@@ -774,71 +1329,75 @@ export function ThreadDocumentsBar({// block attachments until the chat's project is known to avoid guessing their scope.const busy = uploading || projectUploading || projectUnresolved;- const chipCount = documents.length + projectDocuments.length;+ // Still claim a drop while busy, or it falls through and attaches to the message instead.+ const busyReason = !busy+ ? undefined+ : projectUnresolved+ ? "Still loading this chat's project. Try again in a moment."+ : "Still uploading. Drop the files again once it finishes.";return (<>{kbDialog}- <div className="mb-2 flex w-full flex-row items-start gap-1.5 pl-0.5 pr-1.5 pt-0.5 pb-1">- {/* keep controls aligned with top-aligned chips. */}- <div className="flex shrink-0 items-center gap-1.5">- <AttachFilesButton- disabled={busy}- compact={chipCount > 0}- sharesWithProject={sharesWithProject}- onClick={handleAddDocs}- />- {/* Only a project chat has two scopes to choose between. */}- {projectId ? (- <AttachmentTargetMenu+ <ChatFilesPanel+ key="files"+ icon={FileDatabaseIcon}+ title="Chat with files"+ titleSuffix="(RAG)"+ count={countSuffix}+ info="Add or drop documents, spreadsheets, slides, e-books, email or code. The model searches them as you chat and cites what it uses."+ onDropItems={attach}+ dropDisabledReason={busyReason}+ onClose={() => setRagEnabled(false)}+ closeLabel="Stop chatting with files"+ note={+ ragToolDisabled && fileCount > 0 ? (+ <span+ className="shrink-0 rounded-full bg-muted px-2 py-0.5 text-ui-11 text-muted-foreground"+ title="The selected model can't search documents, so these files aren't used. Pick a model with tool support to use them."+ >+ Not used+ </span>+ ) : null+ }+ headerControls={+ <>+ {/* Only a project chat has two scopes to choose between. */}+ {projectId ? (+ <AttachmentTargetMenu+ disabled={busy}+ sharesWithProject={sharesWithProject}+ onSelect={(target) =>+ setThreadProjectAttachmentTarget(effectiveThreadId, target)+ }+ />+ ) : null}+ <button+ type="button"disabled={busy}- sharesWithProject={sharesWithProject}- onSelect={(target) =>- setThreadProjectAttachmentTarget(effectiveThreadId, target)+ onClick={pickFiles}+ className="unsloth-files-panel-action"+ title={+ busyReason ??+ (sharesWithProject+ ? "Add files for retrieval, shared with every chat in this project"+ : "Add files for retrieval in this chat")}- />- ) : null}- </div>- <input- ref={fileInputRef}- type="file"- multiple- accept={RAG_UPLOAD_ACCEPT}- className="hidden"- onChange={(e) => {- const files = Array.from(e.target.files ?? []);- e.target.value = "";- if (files.length === 0) return;- attach(files);- }}- />- {ragToolDisabled && chipCount > 0 ? (- <span- className="composer-pill-btn shrink-0 cursor-default !text-foreground/60"- title="The selected model can't search documents, so these files aren't used. Pick a model with tool support to use them."+ >+ Add files+ </button>+ </>+ }+ >+ {/* project sources precede thread sources because they outlive the chat. */}+ {folderCards}+ {projectGroups.loose.map((doc) => (+ <div+ key={`project:${doc.id}`}+ className={cn("contents", ragToolDisabled && "[&>*]:opacity-50")}>- Not used- </span>- ) : null}- {/* cap chip rows and fade overflow. */}- <div- ref={chipScrollRef}- onScroll={updateChipFade}- className={cn(- "flex max-h-24 flex-1 flex-row flex-wrap items-center gap-1.5 overflow-y-auto",- chipsOverflow && "rag-docs-bottom-fade",- ragToolDisabled && "opacity-50",- )}- >- {/* project sources precede thread sources because they outlive the chat. */}- {projectDocuments.map((doc) => (- <DocumentStatusChip- key={`project:${doc.id}`}- filename={doc.filename}- status={doc.status}- progress={doc.progress}- stage={doc.stage}- error={doc.error}+ <DocumentCard+ doc={doc}shared={true}onRemove={doc.id.startsWith("pending_") || isLinkedFolderManaged(doc)@@ -846,53 +1405,55 @@ export function ThreadDocumentsBar({: () => setRemovingShared(doc)}/>- ))}- {documents.map((doc) => (- <DocumentStatusChip- key={doc.id}- filename={doc.filename}- status={doc.status}- progress={doc.progress}- stage={doc.stage}- error={doc.error}+ </div>+ ))}+ {documents.map((doc) => (+ <div+ key={doc.id}+ className={cn("contents", ragToolDisabled && "[&>*]:opacity-50")}+ >+ <DocumentCard+ doc={doc}+ shared={false}onRemove={doc.id.startsWith("pending_")? undefined: () => void remove(doc.id)}/>- ))}- </div>- <AlertDialog- open={removingShared !== null}- onOpenChange={(open) => {- if (!open) setRemovingShared(null);- }}- >- <AlertDialogContent>- <AlertDialogHeader>- <AlertDialogTitle>Remove from project sources</AlertDialogTitle>- <AlertDialogDescription>- Remove "{removingShared?.filename}"? Every chat in this project- loses it, and the file and its indexed content are deleted. This- cannot be undone.- </AlertDialogDescription>- </AlertDialogHeader>- <AlertDialogFooter>- <AlertDialogCancel>Cancel</AlertDialogCancel>- <AlertDialogAction- onClick={() => {- const doc = removingShared;- setRemovingShared(null);- if (doc) void removeFromProject(doc.id);- }}- >- Remove- </AlertDialogAction>- </AlertDialogFooter>- </AlertDialogContent>- </AlertDialog>- </div>+ </div>+ ))}+ <AddFilesCard disabled={busy} onClick={pickFiles} />+ </ChatFilesPanel>+ <AlertDialog+ open={removingShared !== null}+ onOpenChange={(open) => {+ if (!open) setRemovingShared(null);+ }}+ >+ <AlertDialogContent>+ <AlertDialogHeader>+ <AlertDialogTitle>Remove from project sources</AlertDialogTitle>+ <AlertDialogDescription>+ Remove "{removingShared?.filename}"? Every chat in this project+ loses it, and the file and its indexed content are deleted. This+ cannot be undone.+ </AlertDialogDescription>+ </AlertDialogHeader>+ <AlertDialogFooter>+ <AlertDialogCancel>Cancel</AlertDialogCancel>+ <AlertDialogAction+ onClick={() => {+ const doc = removingShared;+ setRemovingShared(null);+ if (doc) void removeFromProject(doc.id);+ }}+ >+ Remove+ </AlertDialogAction>+ </AlertDialogFooter>+ </AlertDialogContent>+ </AlertDialog></>);}diff --git a/studio/frontend/src/features/rag/components/use-source-drop.ts b/studio/frontend/src/features/rag/components/use-source-drop.tsindex 8f1e0733dbc..22c1d171fb2 100644--- a/studio/frontend/src/features/rag/components/use-source-drop.ts+++ b/studio/frontend/src/features/rag/components/use-source-drop.ts@@ -13,7 +13,10 @@ import {useRef,useState,} from "react";-import { partitionSupported } from "./source-drop-policy";+import {+ SUPPORTED_SOURCES_HINT,+ partitionSupported,+} from "./source-drop-policy";import { type RagUploadItem, uploadItemFromIntent } from "./use-rag-documents";export interface SourceDropOptions {@@ -62,7 +65,7 @@ export function useSourceDrop({names.length === 1? `Can't add ${names[0]}`: `Can't add ${names.length} files`,- { description: "Supported types: documents and source code files" },+ { description: SUPPORTED_SOURCES_HINT },);}, []);diff --git a/studio/frontend/src/features/rag/types/rag.ts b/studio/frontend/src/features/rag/types/rag.tsindex 0ceafac9fe8..fbabc0be642 100644--- a/studio/frontend/src/features/rag/types/rag.ts+++ b/studio/frontend/src/features/rag/types/rag.ts@@ -181,3 +181,7 @@ export interface PreviewTarget {}export const RAG_UPLOAD_ACCEPT = ".pdf,.txt,.md,.markdown,.docx,.html,.htm";++/** Mirrors backend `DOCUMENT_UPLOAD_EXTS`. */+export const RAG_DOCUMENT_UPLOAD_ACCEPT =+ ".doc,.docm,.dotx,.dotm,.xls,.xlsx,.xlsm,.xltx,.xltm,.ppt,.pptx,.pptm,.potx,.potm,.ppsx,.ppsm,.odt,.ods,.odp,.ott,.ots,.otp,.msg,.eml,.mht,.mhtml,.rtf,.epub,.xhtml,.xht";diff --git a/studio/frontend/src/index.css b/studio/frontend/src/index.cssindex 1bf77438be3..a1a120cfd0a 100644--- a/studio/frontend/src/index.css+++ b/studio/frontend/src/index.css@@ -764,7 +764,7 @@/* Lift the dark composer slightly; flavor cards sit close to the page. */:root:where([data-palette="blueberry"], [data-palette="butterfly-pea"], [data-palette="cherry"], [data-palette="cinnamon"], [data-palette="cotton-candy"], [data-palette="dragon-fruit"], [data-palette="earl-grey"], [data-palette="espresso"], [data-palette="honey"], [data-palette="licorice"], [data-palette="macaron"], [data-palette="matcha"], [data-palette="mint"], [data-palette="neon-cyberpunk"], [data-palette="oat-milk"], [data-palette="peach"], [data-palette="pina-paraiso"], [data-palette="plum"], [data-palette="tangerine"], [data-palette="taro"], [data-palette="wasabi"], [data-palette="yuzu"])[data-palette].dark- :is(.chat-composer-surface, .unsloth-composer-surface) {+ :is(.chat-composer-surface, .unsloth-composer-surface, .unsloth-files-panel) {background-color: color-mix(in srgb, var(--card), var(--th-fg) 4%);}@@ -6051,3 +6051,79 @@ html:has([data-blocking-screen]) .interface-zoom-position {.browser-annotating *:not([data-annotate-ui], [data-annotate-ui] *) {cursor: none !important;}++.unsloth-files-panel {+ @apply relative mb-2 flex w-full flex-col gap-5 overflow-hidden rounded-[24px] px-4 pt-4 pb-6;+ container: files-panel / inline-size;+ font-family: var(--font-sans);+ background-color: #ffffff;+ /* Light mode: a flat 1px edge, about #dbdbdb on white, no shadow. */+ border: 1px solid rgb(0 0 0 / calc(0.14 * var(--contrast-edge-gain, 1)));+ transition: box-shadow 0.1s, border-color 0.1s;+}+.dark .unsloth-files-panel {+ background-color: var(--card);+ border-color: transparent;+ box-shadow: var(--composer-shadow);+}+.unsloth-files-panel[data-dragging="true"] {+ border-color: var(--primary);+}+.dark .unsloth-files-panel[data-dragging="true"] {+ box-shadow:+ 0 0 0 1.5px var(--primary),+ var(--composer-shadow);+}+.unsloth-files-panel-count {+ @apply inline-flex h-6 shrink-0 items-center rounded-full px-2.5 text-ui-12 font-medium tabular-nums text-foreground/75;+ background-color: color-mix(in oklab, var(--foreground) 6%, transparent);+}+.dark .unsloth-files-panel-count {+ background-color: color-mix(in oklab, var(--foreground) 10%, transparent);+}+.unsloth-files-panel-drop-overlay {+ @apply pointer-events-none absolute inset-0 z-20 flex items-center justify-center gap-2 rounded-[inherit] bg-background/90 text-sm font-medium text-primary opacity-0 backdrop-blur-sm transition-opacity duration-150 dark:bg-card/90;+}+.unsloth-files-panel[data-dragging="true"] .unsloth-files-panel-drop-overlay {+ opacity: 1;+}+/* One target at a time: over the panel, the composer's "Drop files here" would mislead. */+.aui-composer-root:has(.unsloth-files-panel[data-dragging="true"]) .aui-composer-drop-overlay {+ opacity: 0 !important;+}+/* Inset at rest; cards scroll under the panel edges. */+.unsloth-files-panel-strip {+ @apply -mx-4 flex flex-row flex-nowrap gap-2 overflow-x-auto overscroll-x-contain px-4;+ scroll-padding-inline: 1rem;+ scrollbar-width: none;+}+.unsloth-files-panel-strip::-webkit-scrollbar {+ display: none;+}+.unsloth-files-panel-action {+ @apply inline-flex h-8 shrink-0 items-center rounded-full border px-3.5 text-ui-13 text-foreground transition-colors disabled:pointer-events-none disabled:opacity-50;+ border-color: color-mix(in oklab, var(--foreground) 14%, transparent);+}+.unsloth-files-panel-action:hover {+ background-color: color-mix(in oklab, var(--foreground) 6%, transparent);+}+.unsloth-files-add-card:disabled {+ @apply pointer-events-none opacity-50;+}+/* Dark mode: Add files is a fill, not an outline. */+.dark .unsloth-files-panel-action,+.dark .unsloth-files-add-card {+ border-color: transparent;+ background-color: color-mix(in oklab, var(--foreground) 8%, transparent);+}+.dark .unsloth-files-panel-action:hover,+.dark .unsloth-files-add-card:hover {+ border-color: transparent;+ background-color: color-mix(in oklab, var(--foreground) 12%, transparent);+}+.unsloth-files-panel-icon-btn {+ @apply inline-flex size-8 shrink-0 items-center justify-center rounded-full text-muted-foreground transition-colors hover:text-foreground disabled:pointer-events-none disabled:opacity-35;+}+.unsloth-files-panel-icon-btn:hover {+ background-color: color-mix(in oklab, var(--foreground) 6%, transparent);+}diff --git a/studio/frontend/src/lib/open-file-picker.ts b/studio/frontend/src/lib/open-file-picker.tsnew file mode 100644index 00000000000..8f901975a9e--- /dev/null+++ b/studio/frontend/src/lib/open-file-picker.ts@@ -0,0 +1,31 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++/** Opens the file chooser. Call from a click handler (needs user activation). */+export function openFilePicker(+ accept: string,+ onFiles: (files: File[]) => void,+): void {+ const input = document.createElement("input");+ input.type = "file";+ input.multiple = true;+ input.hidden = true;+ if (accept !== "*") {+ input.accept = accept;+ }++ document.body.appendChild(input);+ input.onchange = (event) => {+ const files = (event.target as HTMLInputElement).files;+ if (files && files.length > 0) {+ onFiles(Array.from(files));+ }+ document.body.removeChild(input);+ };+ input.oncancel = () => {+ if (!input.files || input.files.length === 0) {+ document.body.removeChild(input);+ }+ };+ input.click();+}diff --git a/studio/frontend/tests/attachment-preview-text.test.ts b/studio/frontend/tests/attachment-preview-text.test.tsindex a8b994f471c..9a35f1b4fc1 100644--- a/studio/frontend/tests/attachment-preview-text.test.ts+++ b/studio/frontend/tests/attachment-preview-text.test.ts@@ -2196,6 +2196,18 @@ test("a UTF-16 Markdown file previews as its text in the document viewer", asyncassert.match(dialog, /blob instanceof File\s*\?\s*await readAttachmentText\(blob, source\.name, source\.contentType\)/);});+test("a UTF-16 text source opened from the files panel previews as its text", async () => {+ const utf16 = new Uint8Array([0xff, 0xfe, ...Array.from("Café notes", (c) => [c.charCodeAt(0), 0]).flat()]);+ const file = new File([utf16], "notes.txt", { type: "text/plain" });+ assert.equal((await readAttachmentText(file, file.name, file.type)).text, "Café notes");+ assert.notEqual(await file.text(), "Café notes");+ const { readFile } = await import("node:fs/promises");+ const dialog = await readFile(new URL("../src/components/assistant-ui/attachment-document-dialog.tsx", import.meta.url), "utf8");+ assert.match(dialog, /blob instanceof File\s*\?\s*await readAttachmentText\(blob, source\.name, blob\.type\)/);+ const bar = await readFile(new URL("../src/features/rag/components/thread-documents-bar.tsx", import.meta.url), "utf8");+ assert.match(bar, /new File\(\[file\], file\.name, \{ type: "text\/plain" \}\)/);+});+async function readRtf(rtf: string | Uint8Array<ArrayBuffer>): Promise<string> {const content = await readRtfAttachmentContent(new File([rtf], "doc.rtf"),diff --git a/studio/frontend/tests/native-drop-paths.test.ts b/studio/frontend/tests/native-drop-paths.test.tsindex 68c67ba456b..91d23e0e459 100644--- a/studio/frontend/tests/native-drop-paths.test.ts+++ b/studio/frontend/tests/native-drop-paths.test.ts@@ -220,6 +220,29 @@ for (const [format, extensions, adapter, rustList] of [});}+test("Office, Outlook and web page drops index in chat with files and match the native allowlist", () => {+ const office =+ ".doc .xls .xlsx .xlsm .xltx .xltm .ppt .pptx .pptm .ppsx .ott .ots .otp .msg .mht .mhtml .xhtml .xht".split(+ " ",+ );+ for (const extension of office) {+ const path = `/docs/Report${extension.toUpperCase()}`;+ assert.equal(classifyDropPaths([path]).kind, "docs", path);+ assert.ok(!isComposerAttachmentName(path), path);+ assert.ok(SUPPORTED_DROP_HINT.includes(extension));+ }+ // Types with a composer adapter keep going to the composer.+ for (const ext of [".odt", ".rtf", ".epub", ".eml", ".docm"]) {+ assert.ok(isComposerAttachmentName(`/docs/a${ext}`), ext);+ }+ const rust = [+ ...(readText("../../src-tauri/src/native_path_policy.rs")+ .match(/OFFICE_ATTACHMENT_EXTS[^=]*=\s*&\[([^\]]+)\]/)?.[1]+ .matchAll(RUST_EXTENSION_RE) ?? []),+ ].map((match) => `.${match[1]}`);+ assert.deepEqual(rust.sort(), [...office].sort());+});+test("images route to chat vision attachments, one or many", () => {const dropped = classifyDropPaths(["/photos/cat.PNG",diff --git a/studio/frontend/tests/project-sources-model-without-tools.test.ts b/studio/frontend/tests/project-sources-model-without-tools.test.tsindex 2134b5e9607..11727d66efa 100644--- a/studio/frontend/tests/project-sources-model-without-tools.test.ts+++ b/studio/frontend/tests/project-sources-model-without-tools.test.ts@@ -12,6 +12,7 @@ import type * as GateModule from "../src/features/chat/hooks/use-rag-tool-disablimport { loadWithStubs } from "./helpers/module-stubs.ts";const PROJECT_DOC = { id: "doc-1", filename: "handbook.pdf", status: "completed" };+let extraProjectDocs: Record<string, unknown>[] = [];let runtime: Record<string, unknown> = {};const useChatRuntimeStore = (select: (s: Record<string, unknown>) => unknown) =>@@ -46,7 +47,30 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>("react/jsx-runtime": jsxRuntime,"@hugeicons/react": { HugeiconsIcon: Nothing },"@hugeicons/core-free-icons": {},+ "lucide-react": new Proxy({}, { get: () => Nothing }),"@/lib/tick-icon": {},+ "@/lib/open-file-picker": { openFilePicker: () => undefined },+ "@/components/assistant-ui/attachment": {+ AttachmentKindIcon: Nothing,+ FileCardBody: ({ name }: { name: string }) =>+ React.createElement("span", null, name),+ },+ "@/components/assistant-ui/attachment-card-preview": {+ AttachmentCardPreview: Nothing,+ attachmentPreview: () => null,+ },+ "@/components/assistant-ui/attachment-document-dialog": {+ LocalFileDialog: Passthrough,+ },+ "@/components/ui/tooltip": {+ Tooltip: Passthrough,+ TooltipContent: Nothing,+ TooltipTrigger: Passthrough,+ },+ "@/components/ui/spinner": { Spinner: Nothing },+ "./preview-store": {+ useDocumentPreviewStore: selectorStore({ openPreview: () => undefined }),+ },"@/lib/chevron-icons": {},"@assistant-ui/react": { useAui: () => ({}) },"@/lib/utils": {@@ -61,6 +85,15 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>("@/features/chat": {isThreadIncognito: () => false,chatHistoryClearBoundary: { capture: () => 0 },+ attachmentFileKind: (name: string) =>+ name.endsWith(".pdf") ? "pdf" : "word",+ isTextAttachment: (name: string) => name.endsWith(".txt"),+ },+ "@/components/file-viewer": {+ MAX_DOCUMENT_PREVIEW_BYTES: 50,+ documentKind: (name: string) =>+ name.endsWith(".pdf") ? "pdf" : name.endsWith(".docx") ? "docx" : null,+ isMarkdown: () => false,},"@/features/native-intents": {useNativeAttachmentTargetKey: () => null,@@ -72,23 +105,34 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(listKnowledgeBases: async () => [],subscribeKnowledgeBasesChanged: () => () => undefined,listProjectDocuments: async () => [PROJECT_DOC],+ listLinkedFolders: async () => [],listThreadDocuments: async () => [],},"../api/rag-availability": {useRagAvailabilityStore: selectorStore({ isUnavailable: () => false }),},- "../types/rag": { RAG_UPLOAD_ACCEPT: "", isLinkedFolderManaged: () => false },- "@/components/ui/alert-dialog": new Proxy({}, { get: () => Passthrough }),- "./document-status-chip": {- DocumentStatusChip: ({ filename }: { filename: string }) =>- React.createElement("span", null, filename),+ "../types/rag": { isLinkedFolderManaged: () => false },+ "./source-drop-policy": {+ RAG_SOURCE_UPLOAD_ACCEPT: "",+ SUPPORTED_SOURCES_HINT: "",+ isSupportedSourceName: () => true,},+ "./use-source-drop": {+ useSourceDrop: () => ({+ dragging: false,+ dropProps: {},+ nativeDropTarget: () => undefined,+ }),+ },+ "@/components/ui/alert-dialog": new Proxy({}, { get: () => Passthrough }),+ "./document-status-chip": { STAGE_LABELS: {} },"./knowledge-base-dialog": { KnowledgeBaseDialog: Nothing },"./staged-source": { EXPIRY_GRACE_MS: 0 },"./use-rag-documents": {uploadItemFromIntent: () => null,useRagDocuments: (scope: { type: string } | null) => ({- documents: scope?.type === "project" ? [PROJECT_DOC] : [],+ documents:+ scope?.type === "project" ? [PROJECT_DOC, ...extraProjectDocs] : [],uploading: false,hasIndexing: false,loading: false,@@ -156,6 +200,52 @@ test("with Docs on, a model without tool calling dims the files it will not searassert.match(html, /opacity-50/);});+// A linked folder can hold thousands of files; one card per file would flood the composer.+test("a linked folder's files collapse into one folder card", () => {+ extraProjectDocs = ["a.py", "b.py", "c.md"].map((filename, i) => ({+ id: `linked-${i}`,+ filename,+ status: "completed",+ linkedFolderId: "folder-1",+ managed: true,+ }));+ try {+ for (const ragEnabled of [false, true]) {+ const html = renderProjectChat(+ { checkpoint: "unsloth/Qwen3-4B-GGUF", supportsTools: true },+ ragEnabled,+ );+ assert.match(html, /handbook\.pdf/);+ assert.match(html, /Linked folder/);+ assert.match(html, /3 files/);+ assert.doesNotMatch(html, /a\.py|b\.py|c\.md/);+ }+ } finally {+ extraProjectDocs = [];+ }+});++// The source preview has text only for PDFs when opened without a chunk; other files open in the+// attachment viewer, whatever their size, and a type it cannot draw is not offered at all.+test("non-PDF cards open only when the attachment viewer can show them", () => {+ extraProjectDocs = [+ { id: "doc-2", filename: "notes.docx", status: "completed" },+ { id: "doc-3", filename: "book.epub", status: "completed" },+ ];+ try {+ const html = renderProjectChat(+ { checkpoint: "unsloth/Qwen3-4B-GGUF", supportsTools: true },+ true,+ );+ assert.match(html, /aria-label="Preview handbook\.pdf/);+ assert.match(html, /aria-label="Preview notes\.docx/);+ assert.doesNotMatch(html, /aria-label="Preview book\.epub/);+ assert.match(html, /<button[^>]*disabled=""[^>]*aria-label="book\.epub/);+ } finally {+ extraProjectDocs = [];+ }+});+test("with Docs on, a model with tool calling keeps the files in effect", () => {const html = renderProjectChat({ checkpoint: "unsloth/Qwen3-4B-GGUF", supportsTools: true },diff --git a/studio/frontend/tests/rag-source-drop-filter.test.ts b/studio/frontend/tests/rag-source-drop-filter.test.tsindex 22128749d47..041a31449a8 100644--- a/studio/frontend/tests/rag-source-drop-filter.test.ts+++ b/studio/frontend/tests/rag-source-drop-filter.test.ts@@ -9,7 +9,10 @@ import {isSupportedSourceName,partitionSupported,} from "../src/features/rag/components/source-drop-policy.ts";-import { RAG_UPLOAD_ACCEPT } from "../src/features/rag/types/rag.ts";+import {+ RAG_DOCUMENT_UPLOAD_ACCEPT,+ RAG_UPLOAD_ACCEPT,+} from "../src/features/rag/types/rag.ts";test("every type the picker offers is accepted from a drop", () => {for (const ext of RAG_UPLOAD_ACCEPT.split(",")) {@@ -69,6 +72,14 @@ test("an empty drop yields nothing to upload and nothing to report", () => {assert.equal(unsupported.length, 0);});+// The backend parses these for every scope; a project or knowledge base drop must not refuse them.+test("office, OpenDocument, e-book and email files are accepted from a drop", () => {+ for (const ext of RAG_DOCUMENT_UPLOAD_ACCEPT.split(",")) {+ assert.equal(isSupportedSourceName(`deck${ext}`), true, ext);+ }+ assert.equal(RAG_SOURCE_UPLOAD_ACCEPT.split(",").includes(".pptx"), true);+});+test("source files the project picker offers are accepted from a drop", () => {for (const ext of RAG_SOURCE_UPLOAD_ACCEPT.split(",")) {assert.equal(isSupportedSourceName(`app${ext}`), true, ext);diff --git a/studio/frontend/tests/rag-thread-initialization.test.ts b/studio/frontend/tests/rag-thread-initialization.test.tsindex a0e9601c0fa..8ae60b3abfb 100644--- a/studio/frontend/tests/rag-thread-initialization.test.ts+++ b/studio/frontend/tests/rag-thread-initialization.test.ts@@ -91,6 +91,7 @@ function harness(];},};+ let pickedFiles: ((files: File[]) => void) | null = null;const nativeStore = Object.assign(() => nativePending, {getState: () => nativeState,});@@ -119,6 +120,7 @@ function harness(];},useCallback: (fn: unknown) => fn,+ useMemo: (fn: () => unknown) => fn(),useEffect(effect: () => void, deps: unknown[]) {const index = cursor++;const previous = slots[index] as unknown[] | undefined;@@ -131,7 +133,20 @@ function harness("react/jsx-runtime": stubJsxRuntime(),"@hugeicons/react": {},"@hugeicons/core-free-icons": {},+ "lucide-react": {},"@/lib/tick-icon": {},+ "@/lib/open-file-picker": {+ openFilePicker: (_accept: string, onFiles: (files: File[]) => void) => {+ pickedFiles = onFiles;+ },+ },+ "@/components/assistant-ui/attachment": {},+ "@/components/assistant-ui/attachment-card-preview": {},+ "@/components/assistant-ui/attachment-document-dialog": {},+ "@/components/file-viewer": {},+ "@/components/ui/tooltip": {},+ "@/components/ui/spinner": {},+ "./preview-store": { useDocumentPreviewStore: () => undefined },"@/lib/chevron-icons": {},"@assistant-ui/react": { useAui: () => ({ threadListItem: () => item }) },"@/lib/utils": { cn: () => "" },@@ -146,6 +161,8 @@ function harness(chatHistoryClearBoundary: { capture: () => 0 },ChatThreadDeletedError: class extends Error {},isThreadIncognito: () => incognito,+ isPastedTextFile: () => false,+ annotationsOfFile: () => null,getStoredChatThread: async () => undefined,ensureStoredChatThread: async (threadId: string) => {if (storedIds.has(threadId)) return { id: threadId };@@ -179,7 +196,13 @@ function harness(select: (s: { isUnavailable: () => boolean }) => unknown,) => select({ isUnavailable: () => false }),},- "../types/rag": { RAG_UPLOAD_ACCEPT: ".docx" },+ "../types/rag": {},+ "./source-drop-policy": {+ RAG_SOURCE_UPLOAD_ACCEPT: ".docx",+ SUPPORTED_SOURCES_HINT: "",+ isSupportedSourceName: (name: string) => name.endsWith(".docx"),+ },+ "./use-source-drop": {},"./document-status-chip": {},"./knowledge-base-dialog": { KnowledgeBaseDialog: "KnowledgeBaseDialog" },"./staged-source": { EXPIRY_GRACE_MS: 30_000 },@@ -220,14 +243,9 @@ function harness(});}function pick() {- const input = findElement(tree, "input")!;- const onChange = input.props.onChange as (event: unknown) => void;- onChange({- target: {- files: [new File(["document"], "report.docx")],- value: "report.docx",- },- });+ const addCard = findComponent(tree, "AddFilesCard")!;+ (addCard.props.onClick as () => void)();+ pickedFiles!([new File(["document"], "report.docx")]);}return {render,@@ -360,6 +378,25 @@ function findElement(node: unknown, type: string): StubElement | undefined {return findElement(element.props.children, type);}+function findComponent(node: unknown, name: string): StubElement | undefined {+ if (Array.isArray(node)) {+ for (const child of node) {+ const hit = findComponent(child, name);+ if (hit) return hit;+ }+ return undefined;+ }+ if (!node || typeof node !== "object" || !("props" in node)) return undefined;+ const element = node as StubElement;+ if (typeof element.type === "function" && element.type.name === name) return element;+ return findComponent(element.props.children, name);+}++function manageFiles(tree: StubElement): StubElement {+ const panel = findComponent(tree, "ChatFilesPanel")!;+ return panel.props.headerControls as StubElement;+}+function kbDialog(tree: StubElement): StubElement | undefined {return findElement(tree, "KnowledgeBaseDialog");}@@ -417,13 +454,10 @@ test("the drop's Add still opens its knowledge base after the chat's source moveassert.deepEqual(app.uploads, []);});-test("the knowledge base chip opens that knowledge base without uploading anything", () => {+test("the knowledge base panel opens that knowledge base without uploading anything", () => {const app = harness({ ragSource: { type: "kb", kbId: "kb-1" } });app.render();- const chip = (app.tree.props.children as StubElement[]).find(- (child) => typeof child?.type === "function",- )!;- (chip.props.onOpen as () => void)();+ (manageFiles(app.tree).props.onClick as () => void)();app.render();const dialog = kbDialog(app.tree)!;assert.equal(dialog.props.open, true);@@ -436,10 +470,7 @@ test("an open dialog keeps its place when deleting the active knowledge base mov// tree would remount, replaying its animation and dropping its state.const app = harness({ ragSource: { type: "kb", kbId: "kb-1" } });app.render();- const chip = (app.tree.props.children as StubElement[]).find(- (child) => typeof child?.type === "function",- )!;- (chip.props.onOpen as () => void)();+ (manageFiles(app.tree).props.onClick as () => void)();app.render();const place = (tree: StubElement) => ({root: tree.type,diff --git a/studio/src-tauri/src/native_path_policy.rs b/studio/src-tauri/src/native_path_policy.rsindex f0643613f7d..c6f66f274d3 100644--- a/studio/src-tauri/src/native_path_policy.rs+++ b/studio/src-tauri/src/native_path_policy.rs@@ -48,6 +48,11 @@ pub fn classify_native_model_path(path: &Path) -> Result<ClassifiedPath, String>/// Document types the RAG ingest accepts; keep in sync with `config.SUPPORTED_UPLOAD_EXTS`.pub const ATTACHMENT_EXTS: &[&str] = &["pdf", "txt", "md", "markdown", "docx", "html", "htm"];+/// RAG-only document types; keep in sync with `OFFICE_DOC_EXTS` in `drop-paths.ts`.+pub const OFFICE_ATTACHMENT_EXTS: &[&str] = &[+ "doc", "xls", "xlsx", "xlsm", "xltx", "xltm", "ppt", "pptx", "pptm", "ppsx", "ott", "ots",+ "otp", "msg", "mht", "mhtml", "xhtml", "xht",+];/// OpenDocument files the chat composer parses directly rather than indexing as RAG sources.pub const OPEN_DOCUMENT_ATTACHMENT_EXTS: &[&str] = &["ods", "odt"];pub const RTF_ATTACHMENT_EXTS: &[&str] = &["rtf"];@@ -680,6 +685,7 @@ pub const VIDEO_ATTACHMENT_EXTS: &[&str] = &[fn accepted_attachment_exts() -> impl Iterator<Item = &'static &'static str> {ATTACHMENT_EXTS.iter()+ .chain(OFFICE_ATTACHMENT_EXTS.iter()).chain(OPEN_DOCUMENT_ATTACHMENT_EXTS.iter()).chain(RTF_ATTACHMENT_EXTS.iter()).chain(TOOL_ONLY_ATTACHMENT_EXTS.iter())