Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: move chat with files into a panel above the composer (#13078)

Michael Han00958e6a03fc
diff --git a/studio/backend/core/rag/cfb.py b/studio/backend/core/rag/cfb.py
new file mode 100644
index 00000000000..963b785b38a
--- /dev/null
+++ b/studio/backend/core/rag/cfb.py
@@ -0,0 +1,173 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Read-only OLE compound file reader ([MS-CFB]), the container of .doc, .xls, .ppt and .msg."""
+
+from __future__ import annotations
+
+import struct
+
+SIGNATURE = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+_FREE, _END, _FAT, _DIFAT = 0xFFFFFFFF, 0xFFFFFFFE, 0xFFFFFFFD, 0xFFFFFFFC
+_NO_STREAM = 0xFFFFFFFF
+_STORAGE, _STREAM, _ROOT = 1, 2, 5
+
+
+class CompoundFileError(ValueError):
+ pass
+
+
+class _Entry:
+ __slots__ = ("name", "kind", "left", "right", "child", "start", "size")
+
+ def __init__(self, raw: bytes, major: int):
+ name_len = struct.unpack_from("<H", raw, 64)[0]
+ self.name = raw[: max(0, min(name_len, 64) - 2)].decode("utf-16-le", "replace")
+ self.kind = raw[66]
+ self.left, self.right, self.child = struct.unpack_from("<III", raw, 68)
+ self.start = struct.unpack_from("<I", raw, 116)[0]
+ size = struct.unpack_from("<Q", raw, 120)[0]
+ # Version 3 files may leave junk in the high half.
+ self.size = size & 0xFFFFFFFF if major == 3 else size
+
+
+class CompoundFile:
+ """Streams of a compound file, addressed by path: ``open("WordDocument")``,
+ ``open("__attach_version1.0_#00000000", "__substg1.0_3707001F")``."""
+
+ def __init__(self, data: bytes):
+ if len(data) < 512 or not data.startswith(SIGNATURE):
+ raise CompoundFileError("not an OLE compound file")
+ self._data = data
+ self._children_cache: dict[int, dict[str, int]] = {}
+ major = struct.unpack_from("<H", data, 0x1A)[0]
+ shift = struct.unpack_from("<H", data, 0x1E)[0]
+ mini_shift = struct.unpack_from("<H", data, 0x20)[0]
+ if (major, shift) not in ((3, 9), (4, 12)) or mini_shift != 6:
+ raise CompoundFileError("unsupported compound file version")
+ self._sector = 1 << shift
+ self._mini_cutoff = struct.unpack_from("<I", data, 0x38)[0]
+ (n_fat,) = struct.unpack_from("<I", data, 0x2C)
+ (first_dir,) = struct.unpack_from("<I", data, 0x30)
+ first_mini_fat, n_mini_fat = struct.unpack_from("<II", data, 0x3C)
+ first_difat, n_difat = struct.unpack_from("<II", data, 0x44)
+
+ fat_sectors = [s for s in struct.unpack_from("<109I", data, 0x4C) if s < _DIFAT]
+ per = self._sector // 4
+ # Enough to map the whole file; a crafted DIFAT can repeat one sector millions of times.
+ max_fat = len(data) // self._sector // per + 1
+ sector, seen = first_difat, set()
+ for _ in range(n_difat):
+ if sector >= _DIFAT or sector in seen or len(fat_sectors) >= max_fat:
+ break
+ seen.add(sector)
+ words = struct.unpack(f"<{per}I", self._raw_sector(sector))
+ fat_sectors += [s for s in words[:-1] if s < _DIFAT]
+ sector = words[-1]
+ fat_sectors = fat_sectors[: min(n_fat or max_fat, max_fat)]
+ self._fat: list[int] = []
+ for s in fat_sectors:
+ self._fat.extend(struct.unpack(f"<{per}I", self._raw_sector(s)))
+
+ dir_bytes = self._read_chain(first_dir, self._fat, self._raw_sector, self._sector)
+ self._entries = [
+ _Entry(dir_bytes[i : i + 128], major) for i in range(0, len(dir_bytes) - 127, 128)
+ ]
+ if not self._entries or self._entries[0].kind != _ROOT:
+ raise CompoundFileError("missing root entry")
+ root = self._entries[0]
+ mini_fat_bytes = (
+ self._read_chain(first_mini_fat, self._fat, self._raw_sector, self._sector)
+ if n_mini_fat
+ else b""
+ )
+ self._mini_fat = list(struct.unpack(f"<{len(mini_fat_bytes) // 4}I", mini_fat_bytes))
+ self._mini_stream = self._read_chain(
+ root.start, self._fat, self._raw_sector, self._sector, root.size
+ )
+
+ def _raw_sector(self, sector: int) -> bytes:
+ offset = (sector + 1) * self._sector
+ if offset + self._sector > len(self._data):
+ # A short final sector is common; anything past the end is not.
+ if offset >= len(self._data):
+ raise CompoundFileError("sector out of range")
+ return self._data[offset:].ljust(self._sector, b"\0")
+ return self._data[offset : offset + self._sector]
+
+ def _mini_sector(self, sector: int) -> bytes:
+ offset = sector * 64
+ if offset >= len(self._mini_stream):
+ raise CompoundFileError("mini sector out of range")
+ return self._mini_stream[offset : offset + 64]
+
+ @staticmethod
+ def _read_chain(
+ start,
+ table,
+ read,
+ size,
+ limit = None,
+ ) -> bytes:
+ parts, seen, sector = [], set(), start
+ while sector < _DIFAT:
+ if sector in seen:
+ raise CompoundFileError("broken sector chain")
+ seen.add(sector)
+ parts.append(read(sector))
+ if limit is not None and len(parts) * size >= limit:
+ break
+ # Some writers leave the file's last sectors out of the FAT; end the chain there.
+ sector = table[sector] if sector < len(table) else _END
+ out = b"".join(parts)
+ return out[:limit] if limit is not None else out
+
+ def _children(self, index: int) -> dict[str, int]:
+ """Children of a storage, keyed by lowercased name (the red-black tree flattened)."""
+ # Cached: a .msg looks up properties per attachment, each walking the root again.
+ cached = self._children_cache.get(index)
+ if cached is not None:
+ return cached
+ found: dict[str, int] = {}
+ self._children_cache[index] = found
+ stack, seen = [self._entries[index].child], set()
+ while stack:
+ i = stack.pop()
+ if i == _NO_STREAM or i >= len(self._entries) or i in seen:
+ continue
+ seen.add(i)
+ entry = self._entries[i]
+ found[entry.name.lower()] = i
+ stack += [entry.left, entry.right]
+ return found
+
+ def _find(self, path: tuple[str, ...]) -> _Entry | None:
+ index = 0
+ for name in path:
+ index = self._children(index).get(name.lower())
+ if index is None:
+ return None
+ return self._entries[index]
+
+ def exists(self, *path: str) -> bool:
+ return self._find(path) is not None
+
+ def listdir(self, *path: str) -> list[str]:
+ entry_index = 0
+ for name in path:
+ entry_index = self._children(entry_index).get(name.lower())
+ if entry_index is None:
+ return []
+ return [self._entries[i].name for i in self._children(entry_index).values()]
+
+ def read(self, *path: str) -> bytes:
+ entry = self._find(path)
+ if entry is None or entry.kind != _STREAM:
+ raise CompoundFileError(f"missing stream: {'/'.join(path)}")
+ if entry.size == 0:
+ return b""
+ if entry.size > len(self._data):
+ raise CompoundFileError("stream larger than the file")
+ if entry.size < self._mini_cutoff:
+ return self._read_chain(entry.start, self._mini_fat, self._mini_sector, 64, entry.size)
+ return self._read_chain(entry.start, self._fat, self._raw_sector, self._sector, entry.size)
diff --git a/studio/backend/core/rag/config.py b/studio/backend/core/rag/config.py
index beca8d19c4b..0732e99ef89 100644
--- a/studio/backend/core/rag/config.py
+++ b/studio/backend/core/rag/config.py
@@ -68,7 +68,13 @@ SOURCE_TEXT_EXTS = frozenset(
.mmd .mermaid .puml .plantuml .dot .gv .feature .robot .http .rest .diff .patch
""".split()
)
-_PARSEABLE_EXTS = SUPPORTED_UPLOAD_EXTS | SOURCE_TEXT_EXTS
+DOCUMENT_UPLOAD_EXTS = frozenset(
+ """
+ .doc .docm .dotx .dotm .xls .xlsx .xlsm .xltx .xltm .ppt .pptx .pptm .potx .potm .ppsx .ppsm
+ .odt .ods .odp .ott .ots .otp .msg .eml .mht .mhtml .rtf .epub .xhtml .xht
+ """.split()
+)
+_PARSEABLE_EXTS = SUPPORTED_UPLOAD_EXTS | SOURCE_TEXT_EXTS | DOCUMENT_UPLOAD_EXTS
# RAG_UPLOAD_EXTS (e.g. ".md,.markdown") can only narrow: a type without a parser would fail every ingest.
_requested_exts = {
"." + ext.strip().lstrip(".").lower()
diff --git a/studio/backend/core/rag/office_formats.py b/studio/backend/core/rag/office_formats.py
new file mode 100644
index 00000000000..9bb76249248
--- /dev/null
+++ b/studio/backend/core/rag/office_formats.py
@@ -0,0 +1,1843 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Text from Office, OpenDocument, e-book, email and RTF files, using only the stdlib.
+
+Each reader returns ``[(text, page_number), ...]``: one entry per slide (numbered), or
+per sheet / chapter (unnumbered). Table rows are joined with " | ", as for .docx.
+"""
+
+from __future__ import annotations
+
+import codecs
+import datetime as dt
+import decimal
+import email
+import email.policy
+import math
+import posixpath
+import re
+import struct
+import zipfile
+from urllib.parse import unquote
+from xml.etree import ElementTree as ET
+
+from .cfb import CompoundFile, CompoundFileError
+
+Section = tuple[str, "int | None"]
+
+_MAX_MEMBER_BYTES = 128 * 1024 * 1024
+_MAX_TOTAL_BYTES = 512 * 1024 * 1024
+_MAX_COLUMNS = 1024
+_MAX_REPEAT = 1024
+# Parsed XML costs about 100-300 bytes per element, far more than its compressed size.
+_MAX_XML_ELEMENTS = 2_000_000
+_MAX_UNIT_ELEMENTS = 200_000 # one streamed row or string
+_MAX_XML_DEPTH = 512
+# Text a document may add by repeating cells and rows.
+_MAX_REPEATED_CHARS = 32 * 1024 * 1024
+
+_NS = {
+ "s": "http://schemas.openxmlformats.org/spreadsheetml/2006/main",
+ "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
+ "rel": "http://schemas.openxmlformats.org/package/2006/relationships",
+ "p": "http://schemas.openxmlformats.org/presentationml/2006/main",
+ "a": "http://schemas.openxmlformats.org/drawingml/2006/main",
+ "c": "http://schemas.openxmlformats.org/drawingml/2006/chart",
+ "dgm": "http://schemas.openxmlformats.org/drawingml/2006/diagram",
+ "office": "urn:oasis:names:tc:opendocument:xmlns:office:1.0",
+ "text": "urn:oasis:names:tc:opendocument:xmlns:text:1.0",
+ "table": "urn:oasis:names:tc:opendocument:xmlns:table:1.0",
+ "draw": "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0",
+ "presentation": "urn:oasis:names:tc:opendocument:xmlns:presentation:1.0",
+ "container": "urn:oasis:names:tc:opendocument:xmlns:container",
+ "opf": "http://www.idpf.org/2007/opf",
+}
+# ISO Strict OOXML names the same schemas with other URIs.
+_STRICT_NS = tuple(
+ (f"http://purl.oclc.org/ooxml/{strict}".encode(), _NS[prefix].encode())
+ for strict, prefix in (
+ ("spreadsheetml/main", "s"),
+ ("presentationml/main", "p"),
+ ("drawingml/main", "a"),
+ ("drawingml/chart", "c"),
+ ("drawingml/diagram", "dgm"),
+ ("officeDocument/relationships", "r"),
+ )
+)
+
+
+def _q(prefix: str, tag: str) -> str:
+ return f"{{{_NS[prefix]}}}{tag}"
+
+
+def _row(cells: list[str]) -> str:
+ while cells and not cells[-1].strip():
+ cells.pop()
+ return " | ".join(cells) if any(c.strip() for c in cells) else ""
+
+
+def _number(value: float) -> str:
+ if math.isfinite(value) and value == int(value) and abs(value) < 1e15:
+ return str(int(value))
+ return repr(value)
+
+
+class _Archive:
+ def __init__(self, path: str):
+ try:
+ self._zip = zipfile.ZipFile(path)
+ except zipfile.BadZipFile as exc:
+ raise ValueError("file is not a valid archive") from exc
+ self._names = {name.lower(): name for name in self._zip.namelist()}
+ self._budget = _MAX_TOTAL_BYTES
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_exc):
+ self._zip.close()
+
+ def has(self, name: str) -> bool:
+ return name.lower() in self._names
+
+ def read(self, name: str) -> bytes:
+ real = self._names.get(name.lower())
+ if real is None:
+ raise ValueError(f"missing archive member: {name}")
+ if self._zip.getinfo(real).flag_bits & 0x1:
+ raise ValueError("file is password protected")
+ limit = min(_MAX_MEMBER_BYTES, self._budget)
+ with self._zip.open(real) as member:
+ data = member.read(limit + 1)
+ if len(data) > limit:
+ raise ValueError("archive member is too large")
+ self._budget -= len(data)
+ return data
+
+ def _xml_events(self, name: str, events: tuple[str, ...]):
+ data = self.read(name)
+ # Office XML never declares entities; refusing them rules out expansion attacks.
+ if b"<!ENTITY" in data:
+ raise ValueError("XML entity declarations are not supported")
+ if b"purl.oclc.org/ooxml/" in data:
+ for strict, transitional in _STRICT_NS:
+ data = data.replace(strict, transitional)
+ parser = ET.XMLPullParser(events)
+ try:
+ for start in range(0, len(data), 1 << 20):
+ parser.feed(data[start : start + (1 << 20)])
+ yield from parser.read_events()
+ parser.close()
+ yield from parser.read_events()
+ except ET.ParseError as exc:
+ raise ValueError(f"malformed XML in {name}") from exc
+
+ def xml(self, name: str) -> ET.Element:
+ root, count = None, 0
+ for _event, element in self._xml_events(name, ("start",)):
+ root = element if root is None else root
+ count += 1
+ if count > _MAX_XML_ELEMENTS:
+ raise ValueError(f"{name} is too large to index")
+ if root is None:
+ raise ValueError(f"malformed XML in {name}")
+ return root
+
+ def stream(self, name: str, units: frozenset[str]):
+ """("start", element) outside units and ("unit", element) once a unit is complete.
+
+ Handled elements are dropped, so memory holds one unit rather than the whole part.
+ """
+ stack: list[ET.Element] = []
+ unit, count = None, 0
+ for event, element in self._xml_events(name, ("start", "end")):
+ if event == "start":
+ stack.append(element)
+ if len(stack) > _MAX_XML_DEPTH:
+ raise ValueError(f"{name} is nested too deeply")
+ if unit is not None:
+ count += 1
+ if count > _MAX_UNIT_ELEMENTS:
+ raise ValueError(f"{name} is too large to index")
+ elif element.tag in units:
+ unit, count = element, 0
+ else:
+ yield "start", element
+ continue
+ stack.pop()
+ if unit is not None and element is not unit:
+ continue
+ if element is unit:
+ unit = None
+ yield "unit", element
+ if stack:
+ # Earlier siblings are gone already, so this is the parent's first child.
+ stack[-1].remove(element)
+
+ def relationships(self, part: str) -> list[tuple[str, str, str]]:
+ """(id, type, member name) of an OOXML part's relationships; "" for the package."""
+ folder, base = posixpath.split(part)
+ rels_name = posixpath.join(folder, "_rels", base + ".rels")
+ if not self.has(rels_name):
+ return []
+ out = []
+ for rel in self.xml(rels_name).iter(_q("rel", "Relationship")):
+ target = rel.get("Target", "")
+ if rel.get("TargetMode") == "External" or not target:
+ continue
+ path = target.lstrip("/") if target.startswith("/") else posixpath.join(folder, target)
+ out.append((rel.get("Id", ""), rel.get("Type", ""), posixpath.normpath(path)))
+ return out
+
+ def rels(self, part: str) -> dict[str, str]:
+ """Relationship id -> member name for an OOXML part."""
+ return {rel_id: target for rel_id, _type, target in self.relationships(part)}
+
+ def related(self, part: str, kind: str, default: str) -> str | None:
+ """The part's first ``kind`` relationship target, else ``default`` if present."""
+ for _id, rel_type, target in self.relationships(part):
+ if rel_type.endswith("/" + kind) and self.has(target):
+ return target
+ return default if self.has(default) else None
+
+
+# Built-in date and time format IDs, including the CJK (27-36, 50-58) and Thai (71-81) ones.
+_BUILTIN_DATE_FORMATS = {
+ *range(14, 23),
+ *range(27, 37),
+ *range(45, 48),
+ *range(50, 59),
+ *range(71, 82),
+}
+
+
+_BUILTIN_DURATION_FORMATS = {46} # [h]:mm:ss
+# Classified as openpyxl does: quoted text and locale tags aside, any date or time token.
+_FORMAT_STRIP = re.compile(r'".*?"|\[(?!hh?\]|mm?\]|ss?\])[^\]]*\]')
+_FORMAT_DURATION = re.compile(
+ r"\[hh?\](:mm(:ss(\.0*)?)?)?|\[mm?\](:ss(\.0*)?)?|\[ss?\](\.0*)?", re.I
+)
+
+
+def _format_kind(format_id: int, code: str) -> str | None:
+ """ "date" for dates and times of day, "duration" for elapsed time, else None."""
+ if not code:
+ if format_id in _BUILTIN_DURATION_FORMATS:
+ return "duration"
+ return "date" if format_id in _BUILTIN_DATE_FORMATS else None
+ code = code.split(";")[0]
+ if _FORMAT_DURATION.search(code):
+ return "duration"
+ return "date" if re.search(r"(?<![_\\])[dmhysDMHYS]", _FORMAT_STRIP.sub("", code)) else None
+
+
+def _serial_text(serial: float, kind: str, date1904: bool) -> str:
+ if kind == "duration":
+ seconds = round(serial * 86_400)
+ hours, rest = divmod(abs(seconds), 3600)
+ return f"{'-' if seconds < 0 else ''}{hours}:{rest // 60:02}:{rest % 60:02}"
+ day, fraction = divmod(serial, 1)
+ clock = dt.timedelta(milliseconds = round(fraction * 86_400_000))
+ if 0 <= serial < 1 and clock.days == 0:
+ return (dt.datetime.min + clock).time().isoformat()
+ if not date1904 and 0 < serial < 60:
+ day += 1 # Excel counts a 29 February 1900 that never was.
+ base = dt.datetime(1904, 1, 1) if date1904 else dt.datetime(1899, 12, 30)
+ try:
+ moment = base + dt.timedelta(days = day) + clock
+ except (OverflowError, ValueError):
+ return _number(serial)
+ return moment.date().isoformat() if moment.time() == dt.time() else moment.isoformat(" ")
+
+
+# Built-in numeric formats ([ECMA-376] 18.8.30); fractions and text are left as stored.
+_BUILTIN_NUMBER_FORMATS = {
+ 1: "0",
+ 2: "0.00",
+ 3: "#,##0",
+ 4: "#,##0.00",
+ 9: "0%",
+ 10: "0.00%",
+ 11: "0.00E+00",
+ 37: "#,##0 ;(#,##0)",
+ 38: "#,##0 ;[Red](#,##0)",
+ 39: "#,##0.00;(#,##0.00)",
+ 40: "#,##0.00;[Red](#,##0.00)",
+ 48: "##0.0E+0",
+}
+_MAX_FORMAT_CHARS = 1024
+_FORMAT_LITERALS = frozenset(" $-+/():!^&'~{}<>=\u20ac\u00a3\u00a5")
+
+
+def _format_tokens(code: str) -> list[tuple[str, str]] | None:
+ """("lit" | "ph" | "exp" | "sep", text) tokens of a number format, or None if unsupported."""
+ tokens, i = [], 0
+ while i < len(code):
+ ch = code[i]
+ if ch == '"':
+ end = code.find('"', i + 1)
+ end = len(code) if end < 0 else end
+ tokens.append(("lit", code[i + 1 : end]))
+ i = end + 1
+ elif ch == "\\" and i + 1 < len(code):
+ tokens.append(("lit", code[i + 1]))
+ i += 2
+ elif ch == "[":
+ end = code.find("]", i)
+ inner = code[i + 1 : end] if end > 0 else ""
+ if inner.startswith("$"): # currency and locale, such as [$EUR-407]
+ tokens.append(("lit", inner[1:].split("-")[0]))
+ elif not inner.isalnum(): # conditions such as [>100]
+ return None
+ i = end + 1 # colours are dropped
+ elif ch == "_": # space the width of the next character
+ tokens.append(("lit", " "))
+ i += 2
+ elif ch == "*": # fill with the next character
+ i += 2
+ elif ch in "0#?.,%":
+ tokens.append(("ph", ch))
+ i += 1
+ elif ch in "Ee" and code[i + 1 : i + 2] in ("+", "-"):
+ tokens.append(("exp", code[i + 1]))
+ i += 2
+ elif ch == ";":
+ tokens.append(("sep", ";"))
+ i += 1
+ elif ch in _FORMAT_LITERALS:
+ tokens.append(("lit", ch))
+ i += 1
+ else:
+ return None
+ return tokens
+
+
+def _round(value: float, places: int) -> str:
+ # Excel rounds halves away from zero.
+ quantum = decimal.Decimal(1).scaleb(-places)
+ return format(decimal.Decimal(repr(value)).quantize(quantum, decimal.ROUND_HALF_UP), "f")
+
+
+def _group(digits: str) -> str:
+ head = len(digits) % 3 or 3
+ return ",".join([digits[:head]] + [digits[i : i + 3] for i in range(head, len(digits), 3)])
+
+
+def _format_section(value: float, tokens: list[tuple[str, str]]) -> str | None:
+ marks = [i for i, (kind, _) in enumerate(tokens) if kind in ("ph", "exp")]
+ if not any(kind == "ph" and ch in "0#?" for kind, ch in tokens):
+ return "".join(text for kind, text in tokens if kind == "lit")
+ first, last = marks[0], marks[-1]
+ if any(kind == "lit" for kind, _ in tokens[first:last]):
+ return None
+ prefix = "".join(text for kind, text in tokens[:first] if kind == "lit")
+ suffix = "".join(text for kind, text in tokens[last + 1 :] if kind == "lit")
+ pattern = "".join(ch if kind == "ph" else "E" + ch for kind, ch in tokens[first : last + 1])
+ percent = "%" * pattern.count("%")
+ value *= 100 ** len(percent)
+ pattern = pattern.replace("%", "")
+ mantissa, _, exponent = pattern.partition("E")
+ sign_style, exponent = (exponent[:1], exponent[1:]) if exponent else ("", "")
+ whole, dot, fraction = mantissa.partition(".")
+ while whole.endswith(","): # trailing commas scale by thousands
+ whole, value = whole[:-1], value / 1000
+ grouped, whole = "," in whole, whole.replace(",", "")
+ places, required = sum(fraction.count(c) for c in "0#?"), fraction.count("0")
+ min_whole = whole.count("0")
+ power = 0
+ if sign_style:
+ width = max(len(whole), 1)
+ if value:
+ power = math.floor(math.log10(value))
+ power = power - power % width if "#" in whole else power - (max(min_whole, 1) - 1)
+ if float(_round(value / 10**power, places)) >= 10 ** max(width, min_whole, 1):
+ power += width if "#" in whole else 1
+ value /= 10**power
+ digits = _round(value, places)
+ int_part, _, frac_part = digits.partition(".")
+ while len(frac_part) > required and frac_part.endswith("0"):
+ frac_part = frac_part[:-1]
+ int_part = "" if int_part == "0" and min_whole == 0 else int_part.zfill(min_whole)
+ if grouped and int_part:
+ int_part = _group(int_part)
+ text = int_part + (dot + frac_part if dot else "")
+ if sign_style:
+ exp_sign = "-" if power < 0 else ("+" if sign_style == "+" else "")
+ text += f"E{exp_sign}{str(abs(power)).zfill(exponent.count('0'))}"
+ return prefix + text + percent + suffix
+
+
+def _format_number(value: float, code: str) -> str | None:
+ """``value`` as a numeric format displays it, or None for formats left as stored."""
+ tokens = _format_tokens(code) if code else None
+ if not tokens:
+ return None
+ sections: list[list[tuple[str, str]]] = [[]]
+ for token in tokens:
+ if token[0] == "sep":
+ sections.append([])
+ else:
+ sections[-1].append(token)
+ if value < 0 and len(sections) > 1 and sections[1]:
+ return _format_section(-value, sections[1])
+ if value == 0 and len(sections) > 2 and sections[2]:
+ return _format_section(0.0, sections[2])
+ text = _format_section(abs(value), sections[0])
+ return None if text is None else ("-" + text if value < 0 else text)
+
+
+def _cell_number(value: float, format_id: int, code: str, date1904: bool) -> str:
+ """A numeric cell as its format displays it, falling back to the stored number."""
+ if len(code) > _MAX_FORMAT_CHARS: # tokenized per cell
+ code = ""
+ kind = _format_kind(format_id, code)
+ if kind:
+ return _serial_text(value, kind, date1904)
+ try:
+ text = _format_number(value, code or _BUILTIN_NUMBER_FORMATS.get(format_id, ""))
+ except (ArithmeticError, ValueError, decimal.InvalidOperation):
+ text = None
+ return _number(value) if text is None else text
+
+
+def _column_index(ref: str) -> int:
+ index = 0
+ for ch in ref:
+ if not ch.isalpha():
+ break
+ index = index * 26 + (ord(ch.upper()) - 64)
+ return index - 1
+
+
+def _xlsx_text(node: ET.Element) -> str:
+ # Plain <t> or rich runs <r><t>; phonetic hints (<rPh>) are skipped.
+ parts = []
+ for child in node:
+ if child.tag == _q("s", "t"):
+ parts.append(child.text or "")
+ elif child.tag == _q("s", "r"):
+ t = child.find(_q("s", "t"))
+ parts.append((t.text or "") if t is not None else "")
+ return "".join(parts)
+
+
+def xlsx(path: str) -> list[Section]:
+ with _Archive(path) as zf:
+ workbook_part = zf.related("", "officeDocument", "xl/workbook.xml")
+ if workbook_part is None:
+ raise ValueError("missing archive member: xl/workbook.xml")
+ workbook = zf.xml(workbook_part)
+ pr = workbook.find(_q("s", "workbookPr"))
+ date1904 = pr is not None and pr.get("date1904") in ("1", "true")
+ rels = zf.rels(workbook_part)
+ folder = posixpath.dirname(workbook_part)
+
+ strings: list[str] = []
+ strings_part = zf.related(workbook_part, "sharedStrings", f"{folder}/sharedStrings.xml")
+ if strings_part:
+ strings = [
+ _xlsx_text(si)
+ for event, si in zf.stream(strings_part, frozenset([_q("s", "si")]))
+ if event == "unit"
+ ]
+
+ style_formats: list[tuple[int, str]] = []
+ styles_part = zf.related(workbook_part, "styles", f"{folder}/styles.xml")
+ if styles_part:
+ styles = zf.xml(styles_part)
+ custom = {
+ int(f.get("numFmtId", "-1")): f.get("formatCode", "")
+ for f in styles.iter(_q("s", "numFmt"))
+ }
+ xfs = styles.find(_q("s", "cellXfs"))
+ for xf in xfs.findall(_q("s", "xf")) if xfs is not None else []:
+ fmt = int(xf.get("numFmtId", "0") or 0)
+ style_formats.append((fmt, custom.get(fmt, "")))
+
+ sections: list[Section] = []
+ seen: set[int] = set()
+ budget = _Budget()
+ for sheet in workbook.iter(_q("s", "sheet")):
+ target = rels.get(sheet.get(_q("r", "id"), ""))
+ if not target or not zf.has(target):
+ continue
+ rows = []
+ for event, row in zf.stream(target, frozenset([_q("s", "row")])):
+ if event != "unit":
+ continue
+ cells: list[str] = []
+ for c in row.findall(_q("s", "c")):
+ col = _column_index(c.get("r", "")) if c.get("r") else len(cells)
+ if col < 0 or col >= _MAX_COLUMNS:
+ continue
+ kind, v = c.get("t", "n"), c.find(_q("s", "v"))
+ raw = v.text if v is not None and v.text is not None else ""
+ if kind == "s":
+ value = _shared(strings, int(raw) if raw.isdigit() else -1, seen, budget)
+ elif kind == "inlineStr":
+ inline = c.find(_q("s", "is"))
+ value = _xlsx_text(inline) if inline is not None else ""
+ elif kind == "b":
+ value = "TRUE" if raw == "1" else "FALSE"
+ elif kind in ("str", "e"):
+ value = raw
+ else:
+ try:
+ number = float(raw)
+ except ValueError:
+ value = raw
+ else:
+ style = int(c.get("s", "0") or 0)
+ fmt, code = (
+ style_formats[style] if style < len(style_formats) else (0, "")
+ )
+ value = _cell_number(number, fmt, code, date1904)
+ cells.extend([""] * (col - len(cells)))
+ if col == len(cells):
+ cells.append(value)
+ line = _row(cells)
+ if line:
+ rows.append(line)
+ if rows:
+ sections.append((f"Sheet: {sheet.get('name', '')}\n" + "\n".join(rows), None))
+ return sections
+
+
+def _drawing_lines(root: ET.Element) -> list[str]:
+ """Paragraphs of a DrawingML part in document order; table rows as one line each."""
+ lines: list[str] = []
+ in_table: set[ET.Element] = set()
+ for node in root.iter():
+ if node.tag == _q("a", "tbl"):
+ for tr in node.iter(_q("a", "tr")):
+ cells = []
+ for tc in tr.findall(_q("a", "tc")):
+ texts = (_drawing_paragraph(p) for p in tc.iter(_q("a", "p")))
+ cells.append(" ".join(t for t in texts if t.strip()))
+ line = _row(cells)
+ if line:
+ lines.append(line)
+ in_table.update(node.iter(_q("a", "p")))
+ elif node.tag == _q("a", "p") and node not in in_table:
+ text = _drawing_paragraph(node)
+ if text.strip():
+ lines.append(text)
+ return lines
+
+
+def _drawing_paragraph(p: ET.Element) -> str:
+ parts = []
+ for node in p.iter():
+ if node.tag == _q("a", "t"):
+ parts.append(node.text or "")
+ elif node.tag == _q("a", "br"):
+ parts.append("\n")
+ return "".join(parts)
+
+
+def _without_slide_number(root: ET.Element) -> ET.Element:
+ """Drops the notes page's slide number placeholder."""
+ ph_path = f"{_q('p', 'nvSpPr')}/{_q('p', 'nvPr')}/{_q('p', 'ph')}"
+ doomed = [
+ (parent, sp)
+ for parent in root.iter()
+ for sp in parent.findall(_q("p", "sp"))
+ if (ph := sp.find(ph_path)) is not None and ph.get("type") == "sldNum"
+ ]
+ for parent, sp in doomed:
+ parent.remove(sp)
+ return root
+
+
+def _chart_lines(root: ET.Element) -> list[str]:
+ """Chart title and axis titles, then the cached data as rows of category and values."""
+ lines = _drawing_lines(root)
+ names, series_categories, values = [], [], []
+ for series in root.iter(_q("c", "ser")):
+ name = series.find(f"{_q('c', 'tx')}//{_q('c', 'v')}")
+ names.append(name.text or "" if name is not None else "")
+ categories, points = {}, {}
+ # Scatter and bubble charts cache xVal/yVal instead of cat/val.
+ for axes, store in ((("cat", "xVal"), categories), (("val", "yVal"), points)):
+ for axis in axes:
+ for point in series.iterfind(f"{_q('c', axis)}//{_q('c', 'pt')}"):
+ v = point.find(_q("c", "v"))
+ text = v.text or "" if v is not None else ""
+ idx = int(point.get("idx", "0") or 0)
+ # Multi-level categories repeat an idx once per c:lvl, innermost level first.
+ store[idx] = " ".join(filter(None, (text, store.get(idx))))
+ series_categories.append(categories)
+ values.append(points)
+ if len({tuple(sorted(c.items())) for c in series_categories if c}) > 1:
+ # Each series has its own X values: one block per series.
+ for name, categories, points in zip(names, series_categories, values):
+ lines += [name] if name else []
+ for idx in sorted(set(categories) | set(points)):
+ line = _row([categories.get(idx, ""), points.get(idx, "")])
+ if line:
+ lines.append(line)
+ return lines
+ categories = next((c for c in series_categories if c), {})
+ if any(names) and len(names) > 1:
+ lines.append(_row(["", *names]).lstrip(" |"))
+ for idx in sorted(set(categories) | {i for points in values for i in points}):
+ line = _row([categories.get(idx, ""), *(points.get(idx, "") for points in values)])
+ if line:
+ lines.append(line)
+ if any(names) and len(names) == 1:
+ lines.insert(len(lines) - len(categories), names[0]) if categories else lines.append(
+ names[0]
+ )
+ return lines
+
+
+def pptx(path: str) -> list[Section]:
+ with _Archive(path) as zf:
+ presentation_part = zf.related("", "officeDocument", "ppt/presentation.xml")
+ if presentation_part is None:
+ raise ValueError("missing archive member: ppt/presentation.xml")
+ presentation = zf.xml(presentation_part)
+ rels = zf.rels(presentation_part)
+ slide_ids = presentation.find(_q("p", "sldIdLst"))
+ sections: list[Section] = []
+ for number, sld in enumerate(
+ slide_ids.findall(_q("p", "sldId")) if slide_ids is not None else [], 1
+ ):
+ part = rels.get(sld.get(_q("r", "id"), ""))
+ if not part or not zf.has(part):
+ continue
+ slide = zf.xml(part)
+ slide_rels = zf.rels(part)
+ lines = _drawing_lines(slide)
+ for chart in slide.iter(_q("c", "chart")):
+ target = slide_rels.get(chart.get(_q("r", "id"), ""))
+ if target and zf.has(target):
+ lines += _chart_lines(zf.xml(target))
+ # SmartArt labels live in the diagram data part, not the slide.
+ for diagram in slide.iter(_q("dgm", "relIds")):
+ target = slide_rels.get(diagram.get(_q("r", "dm"), ""))
+ if target and zf.has(target):
+ lines += _drawing_lines(zf.xml(target))
+ notes = next(
+ (t for t in slide_rels.values() if "notesslide" in t.lower() and zf.has(t)),
+ None,
+ )
+ if notes:
+ note_lines = _drawing_lines(_without_slide_number(zf.xml(notes)))
+ if note_lines:
+ lines += ["Notes:"] + note_lines
+ if lines:
+ sections.append(("\n".join(lines), number))
+ return sections
+
+
+class _Budget:
+ """Caps the text repeated cells and rows add, which their source size does not bound."""
+
+ def __init__(self):
+ self.left = _MAX_REPEATED_CHARS
+
+ def spend(self, chars: int) -> None:
+ self.left -= chars
+ if self.left < 0:
+ raise ValueError("document repeats too much content")
+
+
+def _shared(strings: list[str], index: int, seen: set[int], budget: _Budget) -> str:
+ """A shared string cell; reuses are charged, as one string can fill every cell."""
+ if not 0 <= index < len(strings):
+ return ""
+ if index in seen:
+ budget.spend(len(strings[index]))
+ else:
+ seen.add(index)
+ return strings[index]
+
+
+def _odf_inline(node: ET.Element, budget: _Budget) -> str:
+ parts = [node.text or ""]
+ for child in node:
+ tag = child.tag
+ if tag == _q("text", "s"):
+ parts.append(" " * min(int(child.get(_q("text", "c"), "1") or 1), 100))
+ elif tag == _q("text", "tab"):
+ parts.append("\t")
+ elif tag == _q("text", "line-break"):
+ parts.append("\n")
+ elif tag == _q("text", "note"):
+ body = child.find(_q("text", "note-body"))
+ if body is not None:
+ parts.append(" [" + " ".join(_odf_blocks(body, budget)) + "]")
+ elif tag == _q("office", "annotation"):
+ pass
+ else:
+ parts.append(_odf_inline(child, budget))
+ parts.append(child.tail or "")
+ return "".join(parts)
+
+
+_ODF_ROW_GROUPS = frozenset(
+ _q("table", tag) for tag in ("table-header-rows", "table-rows", "table-row-group")
+)
+
+
+def _odf_rows(node: ET.Element):
+ """Rows of this table, through row groups but not into nested tables."""
+ for child in node:
+ if child.tag == _q("table", "table-row"):
+ yield child
+ elif child.tag in _ODF_ROW_GROUPS:
+ yield from _odf_rows(child)
+
+
+def _odf_repeat(node: ET.Element, attribute: str, text: str, room: int, budget: _Budget) -> int:
+ repeat = int(node.get(_q("table", attribute), "1") or 1)
+ # Trailing empty cells repeat to the sheet edge; only content is expanded.
+ count = max(min(repeat, _MAX_REPEAT, room) if text else min(repeat, 1), 0)
+ budget.spend(len(text) * max(count - 1, 0))
+ return count
+
+
+_ODF_VALUE_ATTRIBUTES = {
+ "float": "value",
+ "percentage": "value",
+ "currency": "value",
+ "date": "date-value",
+ "time": "time-value",
+ "boolean": "boolean-value",
+ "string": "string-value",
+}
+
+
+def _odf_cell_value(cell: ET.Element) -> str:
+ """A cell's typed value, for cells written without a text paragraph."""
+ kind = cell.get(_q("office", "value-type"), "")
+ attribute = _ODF_VALUE_ATTRIBUTES.get(kind)
+ value = cell.get(_q("office", attribute), "") if attribute else ""
+ if kind == "boolean" and value:
+ return "TRUE" if value.lower() == "true" else "FALSE"
+ if attribute == "value" and value:
+ try:
+ return _number(float(value))
+ except ValueError:
+ pass
+ return value
+
+
+def _odf_row(row: ET.Element, budget: _Budget) -> list[str]:
+ """The row's lines: one, repeated as the row is."""
+ cells, gap = [], 0
+ for cell in row:
+ if cell.tag not in (_q("table", "table-cell"), _q("table", "covered-table-cell")):
+ continue
+ text = " ".join(_odf_blocks(cell, budget)) or _odf_cell_value(cell)
+ if not text:
+ # Empty runs before content keep their width; trailing ones are dropped.
+ gap += max(int(cell.get(_q("table", "number-columns-repeated"), "1") or 1), 0)
+ continue
+ cells += [""] * min(gap, _MAX_COLUMNS - len(cells))
+ gap = 0
+ room = _MAX_COLUMNS - len(cells)
+ cells += [text] * _odf_repeat(cell, "number-columns-repeated", text, room, budget)
+ if len(cells) >= _MAX_COLUMNS:
+ break
+ line = _row(cells)
+ if not line:
+ return []
+ return [line] * _odf_repeat(row, "number-rows-repeated", line, _MAX_REPEAT, budget)
+
+
+def _odf_table(table: ET.Element, budget: _Budget) -> list[str]:
+ return [line for row in _odf_rows(table) for line in _odf_row(row, budget)]
+
+
+def _ods_streamed(zf: "_Archive", budget: _Budget) -> list[Section]:
+ """Spreadsheet sheets row by row, without holding content.xml as one tree."""
+ sections: list[Section] = []
+ name, rows = None, []
+
+ def close_sheet():
+ if name is not None and rows:
+ sections.append((f"Sheet: {name}\n" + "\n".join(rows), None))
+
+ for event, element in zf.stream("content.xml", frozenset([_q("table", "table-row")])):
+ if event == "start" and element.tag == _q("table", "table"):
+ close_sheet()
+ name, rows = element.get(_q("table", "name"), ""), []
+ elif event == "unit" and name is not None:
+ rows += _odf_row(element, budget)
+ close_sheet()
+ return sections
+
+
+# Annotations and notes are read separately; tracked changes hold deleted text.
+_ODF_SKIP = frozenset(
+ (_q("office", "annotation"), _q("presentation", "notes"), _q("text", "tracked-changes"))
+)
+
+
+def _odf_blocks(node: ET.Element, budget: _Budget) -> list[str]:
+ lines: list[str] = []
+ for child in node:
+ if child.tag in (_q("text", "p"), _q("text", "h")):
+ text = _odf_inline(child, budget)
+ if text.strip():
+ lines.append(text)
+ elif child.tag == _q("table", "table"):
+ lines += _odf_table(child, budget)
+ elif child.tag in _ODF_SKIP:
+ continue
+ else:
+ lines += _odf_blocks(child, budget)
+ return lines
+
+
+def opendocument(path: str) -> list[Section]:
+ with _Archive(path) as zf:
+ manifest = "META-INF/manifest.xml"
+ if zf.has(manifest) and b"encryption-data" in zf.read(manifest):
+ raise ValueError("file is password protected")
+ budget = _Budget()
+ if zf.has("mimetype") and zf.read("mimetype").strip().endswith(
+ (b".spreadsheet", b".spreadsheet-template")
+ ):
+ return _ods_streamed(zf, budget)
+ body = zf.xml("content.xml").find(_q("office", "body"))
+ if body is None:
+ return []
+ sections: list[Section] = []
+ sheet_doc = body.find(_q("office", "spreadsheet"))
+ slides_doc = body.find(_q("office", "presentation"))
+ if sheet_doc is not None:
+ for table in sheet_doc.findall(_q("table", "table")):
+ rows = _odf_table(table, budget)
+ if rows:
+ name = table.get(_q("table", "name"), "")
+ sections.append((f"Sheet: {name}\n" + "\n".join(rows), None))
+ elif slides_doc is not None:
+ for number, page in enumerate(slides_doc.findall(_q("draw", "page")), 1):
+ lines = _odf_blocks(page, budget)
+ notes = page.find(_q("presentation", "notes"))
+ if notes is not None:
+ note_lines = _odf_blocks(notes, budget)
+ if note_lines:
+ lines += ["Notes:"] + note_lines
+ if lines:
+ sections.append(("\n".join(lines), number))
+ else:
+ lines = _odf_blocks(body, budget)
+ if lines:
+ sections.append(("\n".join(lines), None))
+ return sections
+
+
+def epub(path: str, html_text) -> list[Section]:
+ """Chapters in spine order; ``html_text`` turns one XHTML document into text."""
+ with _Archive(path) as zf:
+ container = zf.xml("META-INF/container.xml")
+ rootfile = next(container.iter(_q("container", "rootfile")), None)
+ if rootfile is None or not rootfile.get("full-path"):
+ raise ValueError("epub has no package document")
+ # A URL like the manifest hrefs (EPUB 3.3 4.2.6.3.1), so "My%20Book.opf" names "My Book.opf".
+ opf_path = posixpath.normpath(unquote(rootfile.get("full-path")).lstrip("/"))
+ opf = zf.xml(opf_path)
+ folder = posixpath.dirname(opf_path)
+ # DRM encrypted resources; obfuscated fonts are listed too but are never in the spine.
+ encrypted = set()
+ if zf.has("META-INF/encryption.xml"):
+ for node in zf.xml("META-INF/encryption.xml").iter():
+ if node.tag.endswith("}CipherReference") and node.get("URI"):
+ encrypted.add(posixpath.normpath(unquote(node.get("URI")).lstrip("/")))
+ manifest = {item.get("id"): item for item in opf.iter(_q("opf", "item")) if item.get("id")}
+ sections: list[Section] = []
+ for ref in opf.iter(_q("opf", "itemref")):
+ item = manifest.get(ref.get("idref"))
+ if item is None or "html" not in (item.get("media-type") or ""):
+ continue
+ # The navigation document only repeats the chapter titles.
+ if "nav" in (item.get("properties") or "").split():
+ continue
+ href = unquote(item.get("href", "").split("#", 1)[0])
+ member = posixpath.normpath(posixpath.join(folder, href))
+ if member in encrypted:
+ raise ValueError("file is DRM protected")
+ if not zf.has(member):
+ continue
+ text = html_text(zf.read(member)).strip()
+ if text:
+ sections.append((text, None))
+ return sections
+
+
+def eml(path: str, html_text) -> list[Section]:
+ with open(path, "rb") as f:
+ message = email.message_from_binary_file(f, policy = email.policy.default)
+ return [(_email_text(message, html_text), None)]
+
+
+def _email_parts(part, html_text) -> list[str]:
+ """Inline body text in order: every part of a mixed message, one choice per alternative."""
+ if part.get_content_disposition() == "attachment":
+ return []
+ if part.get_content_maintype() == "multipart":
+ children = list(part.iter_parts())
+ if part.get_content_subtype() != "alternative":
+ return [text for child in children for text in _email_parts(child, html_text)]
+ # Plain text first; some mailers leave it empty beside a full HTML part.
+ for child in sorted(children, key = lambda c: c.get_content_type() != "text/plain"):
+ texts = _email_parts(child, html_text)
+ if any(t.strip() for t in texts):
+ return texts
+ return []
+ if part.get_content_type() not in ("text/plain", "text/html"):
+ return []
+ try:
+ content = part.get_content()
+ except (LookupError, ValueError):
+ content = part.get_payload(decode = True) or b""
+ content = content.decode("utf-8", "replace") if isinstance(content, bytes) else content
+ if part.get_content_subtype() == "html":
+ return [html_text(content.encode("utf-8") if isinstance(content, str) else content)]
+ return [content]
+
+
+def _email_text(message, html_text) -> str:
+ lines = []
+ for header in ("From", "To", "Cc", "Date", "Subject"):
+ value = message.get(header)
+ if value:
+ lines.append(f"{header}: {value}")
+ text = "\n\n".join(t.strip() for t in _email_parts(message, html_text) if t.strip())
+ names = [part.get_filename() for part in message.iter_attachments() if part.get_filename()]
+ if names:
+ lines.append("Attachments: " + ", ".join(names))
+ return "\n".join(lines) + "\n\n" + text.strip()
+
+
+_RTF_TOKEN = re.compile(
+ r"\\([a-zA-Z]{1,32})(-?\d{1,10})? ?|\\'([0-9a-fA-F]{2})|\\(.)|([{}])|[\r\n]+|([^\\{}\r\n]+)",
+ re.S,
+)
+# Destinations whose content is never body text.
+_RTF_SKIP = frozenset(
+ """
+ fonttbl colortbl stylesheet info pict object header headerl headerr headerf footer footerl
+ footerr footerf listtable listoverridetable rsidtbl generator filetbl revtbl themedata
+ colorschememapping latentstyles datastore xmlnstbl mmathpr pgdsctbl fldinst bkmkstart
+ bkmkend picprop shppict nonshppict userprops docvar xform template
+ """.split()
+)
+_RTF_BREAKS = {
+ "par": "\n",
+ "line": "\n",
+ "sect": "\n\n",
+ "page": "\n\n",
+ "row": "\n",
+ "tab": "\t",
+ "cell": " | ",
+ "emdash": "\u2014",
+ "endash": "\u2013",
+ "bullet": "\u2022",
+ "lquote": "\u2018",
+ "rquote": "\u2019",
+ "ldblquote": "\u201c",
+ "rdblquote": "\u201d",
+ "enspace": " ",
+ "emspace": " ",
+ "qmspace": " ",
+}
+
+
+_RTF_MAX_DEPTH = 1000
+# Font \fcharset -> code page. 0 (ANSI) and 1 (default) follow the document's \ansicpg.
+_RTF_CHARSETS = {
+ 77: "mac_roman",
+ 128: "cp932",
+ 129: "cp949",
+ 130: "johab",
+ 134: "cp936",
+ 136: "cp950",
+ 161: "cp1253",
+ 162: "cp1254",
+ 163: "cp1258",
+ 177: "cp1255",
+ 178: "cp1256",
+ 186: "cp1257",
+ 204: "cp1251",
+ 222: "cp874",
+ 238: "cp1250",
+ 254: "cp437",
+ 255: "cp850",
+}
+
+
+_RTF_DOCUMENT_CHARSETS = {"mac": "mac_roman", "pc": "cp437", "pca": "cp850"}
+
+
+def _rtf_codec(code_page: str) -> str | None:
+ code_page = "mac_roman" if code_page == "10000" else f"cp{code_page}"
+ # b"".decode() never looks the codec up.
+ try:
+ codecs.lookup(code_page)
+ except LookupError:
+ return None
+ return code_page
+
+
+def _rtf_font_codecs(data: str) -> dict[int, str]:
+ """Font number -> codec, from each font table entry's cpg or fcharset."""
+ start = data.find("{\\fonttbl")
+ if start < 0:
+ return {}
+ depth, end = 0, start
+ for end in range(start, min(len(data), start + 1_000_000)):
+ if data[end] in "{}" and data[end - 1] != "\\":
+ depth += 1 if data[end] == "{" else -1
+ if depth == 0:
+ break
+ table = data[start:end]
+ fonts = {}
+ entries = list(re.finditer(r"\\f(\d+)", table))
+ for entry, following in zip(entries, entries[1:] + [None]):
+ segment = table[entry.end() : following.start() if following else len(table)]
+ cpg = re.search(r"\\cpg(\d+)", segment)
+ charset = re.search(r"\\fcharset(\d+)", segment)
+ codec = _rtf_codec(cpg.group(1)) if cpg else None
+ if codec is None and charset:
+ codec = _RTF_CHARSETS.get(int(charset.group(1)))
+ if codec:
+ fonts[int(entry.group(1))] = codec
+ return fonts
+
+
+def rtf(path: str) -> list[Section]:
+ with open(path, "rb") as f:
+ data = f.read().decode("latin-1")
+ if not data.lstrip().startswith("{\\rtf"):
+ raise ValueError("file is not RTF")
+ return [(_rtf_text(data), None)]
+
+
+def _rtf_text(data: str) -> str:
+ out: list[str] = []
+ pending = bytearray()
+ # Bytes decode in the active font's code page, else the document's.
+ fonts = _rtf_font_codecs(data)
+ document_codepage, default_font = "cp1252", None
+ codepage = document_codepage
+ stack: list[tuple[bool, int, bool, str]] = []
+ skip, uc, to_skip = False, 1, 0
+ deleted = False # tracked deletion
+ ignorable = False
+ upr = False # the next group is \upr's ANSI branch
+
+ def flush():
+ if pending:
+ out.append(pending.decode(codepage, "replace"))
+ pending.clear()
+
+ pos = 0
+ while pos < len(data):
+ m = _RTF_TOKEN.match(data, pos)
+ if m is None:
+ pos += 1
+ continue
+ pos = m.end()
+ word, arg, hex_byte, symbol, brace, text = m.groups()
+ if to_skip and (hex_byte or text or symbol):
+ # Fallback characters after \uN.
+ if text and len(text) > to_skip:
+ text, to_skip = text[to_skip:], 0
+ else:
+ to_skip -= len(text) if text else 1
+ continue
+ if brace == "{":
+ flush()
+ if len(stack) >= _RTF_MAX_DEPTH:
+ raise ValueError("RTF groups are nested too deeply")
+ stack.append((skip, uc, deleted, codepage))
+ ignorable = False
+ if upr:
+ # \upr{ANSI}{\*\ud{Unicode}}: the Unicode branch follows.
+ skip, upr = True, False
+ elif brace == "}":
+ flush()
+ skip, uc, deleted, codepage = stack.pop() if stack else (False, 1, False, codepage)
+ to_skip, upr = 0, False
+ elif symbol is not None:
+ if symbol == "*":
+ ignorable = True
+ elif not (skip or deleted):
+ flush()
+ out.append(
+ {"~": "\u00a0", "_": "-", "-": "", "\n": "\n", "\r": "\n"}.get(symbol, symbol)
+ )
+ elif word is not None:
+ if word == "bin":
+ # Raw bytes follow; a negative length would rewind and loop forever.
+ pos = min(len(data), pos + max(int(arg or 0), 0))
+ continue
+ if word == "ansicpg" and arg:
+ document_codepage = _rtf_codec(arg) or document_codepage
+ codepage = fonts.get(default_font, document_codepage)
+ elif word in _RTF_DOCUMENT_CHARSETS:
+ document_codepage = _RTF_DOCUMENT_CHARSETS[word]
+ codepage = fonts.get(default_font, document_codepage)
+ elif word == "deff" and arg:
+ default_font = int(arg)
+ codepage = fonts.get(default_font, document_codepage)
+ elif word == "uc" and arg:
+ uc = int(arg)
+ elif word == "deleted":
+ deleted = arg != "0"
+ elif word == "upr":
+ upr = True
+ elif word == "ud":
+ ignorable = False
+ elif word == "plain":
+ flush()
+ deleted = False
+ codepage = fonts.get(default_font, document_codepage)
+ if ignorable or word in _RTF_SKIP:
+ skip, ignorable = True, False
+ continue
+ if skip:
+ continue
+ if word == "f" and arg:
+ flush()
+ codepage = fonts.get(int(arg), document_codepage)
+ elif word == "u" and arg:
+ flush()
+ code = int(arg)
+ if not deleted:
+ out.append(chr(code + 65536 if code < 0 else code))
+ to_skip = uc
+ elif word in _RTF_BREAKS:
+ flush()
+ out.append(_RTF_BREAKS[word])
+ elif hex_byte is not None:
+ if not (skip or deleted):
+ pending.append(int(hex_byte, 16))
+ elif text is not None and not (skip or deleted):
+ # Literal 8-bit text is in the document code page, like \'hh escapes.
+ pending += text.encode("latin-1")
+ flush()
+ # \uN gives astral characters as two surrogates; pair them, and replace any left alone.
+ text = "".join(out).encode("utf-16-le", "surrogatepass").decode("utf-16-le", "replace")
+ text = re.sub(r"( \| )+\n", "\n", text)
+ return re.sub(r"[ \t]+\n", "\n", text).strip()
+
+
+def _compound(path: str) -> CompoundFile:
+ with open(path, "rb") as f:
+ data = f.read()
+ try:
+ return CompoundFile(data)
+ except CompoundFileError as exc:
+ raise ValueError(str(exc)) from exc
+
+
+def _clean_control(text: str) -> str:
+ return re.sub(r"[\x00-\x08\x0e-\x1f]", "", text)
+
+
+def doc(path: str) -> list[Section]:
+ cf = _compound(path)
+ try:
+ word = cf.read("WordDocument")
+ except CompoundFileError as exc:
+ raise ValueError("not a Word 97-2003 document") from exc
+ if len(word) < 0x200 or struct.unpack_from("<H", word, 0)[0] != 0xA5EC:
+ raise ValueError("not a Word 97-2003 document")
+ flags = struct.unpack_from("<H", word, 0x0A)[0]
+ if flags & 0x0100:
+ raise ValueError("file is password protected")
+ table = cf.read("1Table" if flags & 0x0200 else "0Table")
+
+ # FibRgLw97 story lengths and FibRgFcLcb97.fcClx/lcbClx, located by the FIB's own counts.
+ csw = struct.unpack_from("<H", word, 32)[0]
+ lw_start = 34 + csw * 2 + 2
+ cslw = struct.unpack_from("<H", word, 34 + csw * 2)[0]
+ # ccpText, ccpFtn, ccpHdd, ccpMcr, ccpAtn, ccpEdn, ccpTxbx
+ count = max(min(cslw, 10) - 3, 1)
+ ccps = [max(n, 0) for n in struct.unpack_from(f"<{count}i", word, lw_start + 12)]
+ ccps += [0] * (7 - len(ccps))
+ fc_start = lw_start + cslw * 4 + 2
+ fc_clx, lcb_clx = struct.unpack_from("<II", word, fc_start + 33 * 8)
+ clx = table[fc_clx : fc_clx + lcb_clx]
+
+ pos, pieces = 0, None
+ while pos < len(clx):
+ if clx[pos] == 0x01: # Prc: skip its property modifiers
+ pos += 3 + struct.unpack_from("<H", clx, pos + 1)[0]
+ elif clx[pos] == 0x02: # Pcdt: the piece table
+ size = struct.unpack_from("<I", clx, pos + 1)[0]
+ plc = clx[pos + 5 : pos + 5 + size]
+ n = (len(plc) - 4) // 12
+ cps = struct.unpack_from(f"<{n + 1}I", plc, 0)
+ pieces = [
+ (cps[i], cps[i + 1], struct.unpack_from("<I", plc, (n + 1) * 4 + i * 8 + 2)[0])
+ for i in range(n)
+ ]
+ break
+ else:
+ break
+ if pieces is None:
+ raise ValueError("Word document has no piece table")
+
+ # Pieces never overlap in a real file, so all text fits in the stream's bytes.
+ room = [len(word)]
+
+ def story(first: int, length: int) -> str:
+ chars: list[str] = []
+ for start, end, fc in pieces:
+ lo, hi = max(start, first), min(end, first + length)
+ if hi <= lo:
+ continue
+ room[0] -= hi - lo
+ if room[0] < 0:
+ raise ValueError("document repeats too much content")
+ if fc & 0x40000000:
+ offset = (fc & 0x3FFFFFFF) // 2 + (lo - start)
+ chars.append(word[offset : offset + hi - lo].decode("cp1252", "replace"))
+ else:
+ offset = fc + 2 * (lo - start)
+ chars.append(word[offset : offset + 2 * (hi - lo)].decode("utf-16-le", "replace"))
+ return _word_text("".join(chars))
+
+ # Stories follow the main text in this order; headers and comments are left out, as for .docx.
+ starts = [sum(ccps[:i]) for i in range(len(ccps))]
+ text, ftn, _hdd, _mcr, _atn, edn, txbx = range(7)
+ parts = [story(0, ccps[text]), story(starts[txbx], ccps[txbx])]
+ for label, kind in (("Footnotes", ftn), ("Endnotes", edn)):
+ notes = story(starts[kind], ccps[kind])
+ if notes:
+ parts.append(f"{label}:\n{notes}")
+ return [("\n\n".join(p for p in parts if p), None)]
+
+
+def _word_text(raw: str) -> str:
+ # Fields: keep the result (after 0x14), drop the instruction (before it).
+ out, depth_in_code = [], []
+ for ch in raw:
+ if ch == "\x13":
+ depth_in_code.append(True)
+ elif ch == "\x14":
+ if depth_in_code:
+ depth_in_code[-1] = False
+ elif ch == "\x15":
+ if depth_in_code:
+ depth_in_code.pop()
+ elif not any(depth_in_code):
+ out.append(ch)
+ # A cell ends in 0x07; a row ends in one more.
+ text = "".join(out).replace("\x07\x07", "\n").replace("\x07", " | ")
+ text = text.translate({0x0D: "\n", 0x0B: "\n", 0x0C: "\n", 0x1E: "-", 0x1F: None})
+ text = re.sub(r"\n{3,}", "\n\n", text)
+ return _clean_control(text).strip()
+
+
+def _rk(value: int) -> float:
+ if value & 2:
+ number = float(value >> 2 if not value & 0x80000000 else (value >> 2) - (1 << 30))
+ else:
+ number = struct.unpack("<d", struct.pack("<Q", (value & 0xFFFFFFFC) << 32))[0]
+ return number / 100 if value & 1 else number
+
+
+class _Records:
+ def __init__(
+ self,
+ data: bytes,
+ pos: int = 0,
+ ):
+ self.data, self.pos = data, pos
+
+ def __iter__(self):
+ data = self.data
+ while self.pos + 4 <= len(data):
+ kind, size = struct.unpack_from("<HH", data, self.pos)
+ body = data[self.pos + 4 : self.pos + 4 + size]
+ self.pos += 4 + size
+ yield kind, body
+
+
+def _xls_string(segments: list[bytes], seg: int, pos: int) -> tuple[str, int, int]:
+ """One XLUnicodeRichExtendedString, which may continue across CONTINUE records."""
+
+ def take(n):
+ nonlocal seg, pos
+ out = b""
+ while n > 0:
+ if pos >= len(segments[seg]):
+ seg, pos = seg + 1, 0
+ if seg >= len(segments):
+ raise ValueError("truncated string table")
+ chunk = segments[seg][pos : pos + n]
+ out += chunk
+ pos += len(chunk)
+ n -= len(chunk)
+ return out
+
+ if pos >= len(segments[seg]):
+ seg, pos = seg + 1, 0
+ count = struct.unpack("<H", take(2))[0]
+ flags = take(1)[0]
+ runs = struct.unpack("<H", take(2))[0] if flags & 0x08 else 0
+ ext = struct.unpack("<I", take(4))[0] if flags & 0x04 else 0
+ parts, wide = [], flags & 0x01
+ while count > 0:
+ if pos >= len(segments[seg]):
+ # A continued string restates its width in the CONTINUE record's first byte.
+ seg, pos = seg + 1, 0
+ if seg >= len(segments):
+ raise ValueError("truncated string table")
+ wide = segments[seg][0] & 0x01
+ pos = 1
+ width = 2 if wide else 1
+ avail = (len(segments[seg]) - pos) // width
+ n = min(count, max(avail, 1))
+ raw = segments[seg][pos : pos + n * width]
+ pos += n * width
+ parts.append(raw.decode("utf-16-le" if wide else "latin-1", "replace"))
+ count -= n
+ take(4 * runs + ext)
+ return "".join(parts), seg, pos
+
+
+def _xls_short_string(body: bytes, pos: int, length_bytes: int) -> str:
+ count = body[pos] if length_bytes == 1 else struct.unpack_from("<H", body, pos)[0]
+ pos += length_bytes
+ wide = body[pos] & 0x01
+ pos += 1
+ raw = body[pos : pos + count * (2 if wide else 1)]
+ return raw.decode("utf-16-le" if wide else "latin-1", "replace")
+
+
+_XLS_ERRORS = {
+ 0x00: "#NULL!",
+ 0x07: "#DIV/0!",
+ 0x0F: "#VALUE!",
+ 0x17: "#REF!",
+ 0x1D: "#NAME?",
+ 0x24: "#NUM!",
+ 0x2A: "#N/A",
+ 0x2B: "#GETTING_DATA",
+}
+
+
+def xls(path: str) -> list[Section]:
+ cf = _compound(path)
+ stream = next((name for name in ("Workbook", "Book") if cf.exists(name)), None)
+ if stream is None:
+ raise ValueError("not an Excel 97-2003 workbook")
+ data = cf.read(stream)
+ # BIFF5/7 (Excel 95) strings are CODEPAGE bytes with no BIFF8 flags byte.
+ biff8 = data[4:6] != b"\x00\x05"
+ codec = "cp1252"
+
+ def short(body: bytes, pos: int, length_bytes: int) -> str:
+ if biff8:
+ return _xls_short_string(body, pos, length_bytes)
+ count = body[pos] if length_bytes == 1 else struct.unpack_from("<H", body, pos)[0]
+ start = pos + length_bytes
+ return body[start : start + count].decode(codec, "replace")
+
+ sheets: list[tuple[str, int]] = []
+ strings: list[str] = []
+ seen: set[int] = set()
+ budget = _Budget()
+ sst: list[bytes] | None = None # SST body and its CONTINUE records
+ sst_count = 0
+ formats: dict[int, str] = {}
+ xf_formats: list[int] = []
+ date1904 = False
+ for kind, body in _Records(data):
+ if sst is not None and kind != 0x003C:
+ seg = pos = 0
+ for _ in range(sst_count):
+ text, seg, pos = _xls_string(sst, seg, pos)
+ strings.append(text)
+ sst = None
+ if kind == 0x002F:
+ raise ValueError("file is password protected")
+ if kind == 0x0042 and len(body) >= 2 and not biff8: # CODEPAGE
+ codec = _code_page_codec(struct.unpack_from("<H", body)[0])
+ elif kind == 0x0085 and len(body) >= 8 and body[5] == 0: # BOUNDSHEET, worksheet
+ sheets.append((short(body, 6, 1), struct.unpack_from("<I", body, 0)[0]))
+ elif kind == 0x00FC and len(body) >= 8: # SST
+ sst_count, sst = struct.unpack_from("<I", body, 4)[0], [body[8:]]
+ elif kind == 0x003C and sst is not None:
+ sst.append(body)
+ elif kind == 0x041E and len(body) >= 5: # FORMAT
+ formats[struct.unpack_from("<H", body)[0]] = short(body, 2, 2 if biff8 else 1)
+ elif kind == 0x00E0 and len(body) >= 4: # XF
+ xf_formats.append(struct.unpack_from("<H", body, 2)[0])
+ elif kind == 0x0022 and len(body) >= 2: # DATEMODE
+ date1904 = struct.unpack_from("<H", body)[0] == 1
+ elif kind == 0x000A: # end of the workbook globals
+ break
+
+ def number(value: float, xf: int) -> str:
+ fmt = xf_formats[xf] if xf < len(xf_formats) else 0
+ return _cell_number(value, fmt, formats.get(fmt, ""), date1904)
+
+ sections: list[Section] = []
+ # BIFF cell records are a few bytes each, so 200 MB of them would hold tens of millions of
+ # Python strings; the same ceiling the XML formats get from _MAX_XML_ELEMENTS.
+ cell_count = 0
+ for name, offset in sheets:
+ cells: dict[int, dict[int, str]] = {}
+ last_formula: tuple[int, int] | None = None
+ # A long STRING result goes on in CONTINUE records: (cell, segments).
+ string: tuple[tuple[int, int], list[bytes]] | None = None
+
+ def put(r, c, value):
+ nonlocal cell_count
+ if c < _MAX_COLUMNS:
+ cell_count += 1
+ if cell_count > _MAX_XML_ELEMENTS:
+ raise ValueError("spreadsheet has too many cells")
+ cells.setdefault(r, {})[c] = value
+
+ def put_string():
+ nonlocal string
+ if string is not None:
+ try:
+ text = _xls_string(string[1], 0, 0)[0] if biff8 else short(string[1][0], 0, 2)
+ put(*string[0], text)
+ except (ValueError, struct.error, IndexError):
+ pass
+ string = None
+
+ depth = 0
+ for kind, body in _Records(data, offset):
+ if string is not None:
+ if kind == 0x003C:
+ string[1].append(body)
+ continue
+ put_string()
+ # Embedded charts nest their own BOF/EOF inside the sheet.
+ if kind == 0x0809:
+ depth += 1
+ elif kind == 0x000A:
+ depth -= 1
+ if depth <= 0:
+ break
+ if len(body) < 6 and kind != 0x0207:
+ continue
+ if kind == 0x00FD: # LABELSST
+ r, c, _xf, i = struct.unpack_from("<HHHI", body)
+ put(r, c, _shared(strings, i, seen, budget))
+ elif kind == 0x0203: # NUMBER
+ r, c, xf, v = struct.unpack_from("<HHHd", body)
+ put(r, c, number(v, xf))
+ elif kind == 0x027E: # RK
+ r, c, xf, v = struct.unpack_from("<HHHI", body)
+ put(r, c, number(_rk(v), xf))
+ elif kind == 0x00BD: # MULRK
+ r, c = struct.unpack_from("<HH", body)
+ for i in range((len(body) - 6) // 6):
+ xf, v = struct.unpack_from("<HI", body, 4 + i * 6)
+ put(r, c + i, number(_rk(v), xf))
+ elif kind == 0x0204: # LABEL
+ r, c, _xf = struct.unpack_from("<HHH", body)
+ put(r, c, short(body, 6, 2))
+ elif kind == 0x0205: # BOOLERR
+ r, c, _xf, v, is_error = struct.unpack_from("<HHHBB", body)
+ if not is_error:
+ put(r, c, "TRUE" if v else "FALSE")
+ elif v in _XLS_ERRORS:
+ put(r, c, _XLS_ERRORS[v])
+ elif kind == 0x0006 and len(body) >= 14: # FORMULA, cached result
+ r, c, xf = struct.unpack_from("<HHH", body)
+ result = body[6:14]
+ last_formula = None
+ if result[6:8] != b"\xff\xff":
+ put(r, c, number(struct.unpack("<d", result)[0], xf))
+ elif result[0] == 0:
+ last_formula = (r, c)
+ elif result[0] == 1:
+ put(r, c, "TRUE" if result[2] else "FALSE")
+ elif result[0] == 2 and result[2] in _XLS_ERRORS:
+ put(r, c, _XLS_ERRORS[result[2]])
+ elif kind == 0x0207 and last_formula and len(body) >= 3: # STRING after FORMULA
+ string, last_formula = (last_formula, [body]), None
+ put_string()
+ rows = []
+ for r in sorted(cells):
+ row = cells[r]
+ line = _row([row.get(c, "") for c in range(max(row) + 1)])
+ if line:
+ rows.append(line)
+ if rows:
+ sections.append((f"Sheet: {name}\n" + "\n".join(rows), None))
+ return sections
+
+
+_PPT_TEXT_CHARS, _PPT_TEXT_BYTES = 0x0FA0, 0x0FA8
+_PPT_TEXT_HEADER, _PPT_OUTLINE_REF = 0x0F9F, 0x0F9E
+_PPT_DOCUMENT, _PPT_SLIDE, _PPT_NOTES, _PPT_NOTES_ATOM = 0x03E8, 0x03EE, 0x03F0, 0x03F1
+_PPT_SLIDE_LIST, _PPT_SLIDE_PERSIST = 0x0FF0, 0x03F3
+_PPT_USER_EDIT, _PPT_PERSIST_DIRECTORY = 0x0FF5, 0x1772
+
+
+def _ppt_records(data: bytes, start: int, end: int):
+ """(type, instance, body start, body end, is container) of the records directly in a range."""
+ pos = start
+ while pos + 8 <= end:
+ ver_inst, kind, size = struct.unpack_from("<HHI", data, pos)
+ body = pos + 8
+ yield kind, ver_inst >> 4, body, min(body + size, end), ver_inst & 0x000F == 0x000F
+ pos = body + size
+
+
+def _ppt_walk(
+ data: bytes,
+ start: int,
+ end: int,
+ depth: int = 0,
+):
+ """Every record in a range, depth first, in stream order."""
+ for record in _ppt_records(data, start, end):
+ yield record
+ if record[4] and depth < 32:
+ yield from _ppt_walk(data, record[2], record[3], depth + 1)
+
+
+def _ppt_text(data: bytes, kind: int, start: int, end: int) -> str:
+ text = data[start:end].decode("utf-16-le" if kind == _PPT_TEXT_CHARS else "latin-1", "replace")
+ return _clean_control(text.replace("\r", "\n").replace("\x0b", "\n")).strip()
+
+
+def _ppt_record_at(data: bytes, offset: int | None, kind: int) -> tuple[int, int] | None:
+ if offset is None or offset + 8 > len(data):
+ return None
+ _ver_inst, found, size = struct.unpack_from("<HHI", data, offset)
+ return (offset + 8, min(offset + 8 + size, len(data))) if found == kind else None
+
+
+def _ppt_current(cf: CompoundFile, data: bytes) -> tuple[dict[int, int], int] | None:
+ """Persist id -> offset as of the latest save, and the document's persist id.
+
+ Incremental saves append new versions of records, so the stream also holds stale ones;
+ the edit chain from "Current User" says which are live.
+ """
+ if not cf.exists("Current User"):
+ return None
+ user = cf.read("Current User")
+ if len(user) < 20:
+ return None
+ offset = struct.unpack_from("<I", user, 16)[0]
+ persist: dict[int, int] = {}
+ document, seen, floor, edits = None, set(), offset, None
+ while offset:
+ floor = min(floor, offset)
+ edit = _ppt_record_at(data, offset, _PPT_USER_EDIT)
+ if edit is None or edit[1] - edit[0] < 20 or offset in seen:
+ if not seen:
+ return None
+ # A broken link (some writers point an edit at itself): go on with the next older edit.
+ if edits is None:
+ edits = [
+ b - 8
+ for k, _i, b, _e, _c in _ppt_records(data, 0, len(data))
+ if k == _PPT_USER_EDIT
+ ]
+ offset = max((o for o in edits if o < floor), default = 0)
+ continue
+ seen.add(offset)
+ last_edit, directory_offset, document_ref = struct.unpack_from("<III", data, edit[0] + 8)
+ document = document_ref if document is None else document
+ directory = _ppt_record_at(data, directory_offset, _PPT_PERSIST_DIRECTORY)
+ pos, end = directory if directory else (0, 0)
+ while pos + 4 <= end:
+ entry = struct.unpack_from("<I", data, pos)[0]
+ pos += 4
+ for i in range(entry >> 20):
+ if pos + 4 > end:
+ break
+ # Older edits come later in the chain and never replace newer offsets.
+ persist.setdefault((entry & 0xFFFFF) + i, struct.unpack_from("<I", data, pos)[0])
+ pos += 4
+ offset = last_edit
+ return (persist, document) if document is not None else None
+
+
+def _ppt_sheet_lines(data: bytes, start: int, end: int, listed: list[str]) -> list[str]:
+ """Text of a slide or notes page in shape order; placeholders point into ``listed``."""
+ lines, used = [], set()
+ for kind, _inst, body, stop, _container in _ppt_walk(data, start, end):
+ if kind == _PPT_OUTLINE_REF and stop - body >= 4:
+ index = struct.unpack_from("<I", data, body)[0]
+ if index < len(listed) and index not in used:
+ used.add(index)
+ lines.append(listed[index])
+ elif kind in (_PPT_TEXT_CHARS, _PPT_TEXT_BYTES):
+ lines.append(_ppt_text(data, kind, body, stop))
+ lines += [text for i, text in enumerate(listed) if i not in used]
+ # "*" is the slide number field.
+ return [line for line in lines if line and line != "*"]
+
+
+def ppt(path: str) -> list[Section]:
+ cf = _compound(path)
+ try:
+ data = cf.read("PowerPoint Document")
+ except CompoundFileError as exc:
+ raise ValueError("not a PowerPoint 97-2003 presentation") from exc
+ if cf.exists("EncryptedSummary"):
+ raise ValueError("file is password protected")
+ current = _ppt_current(cf, data)
+ document = current and _ppt_record_at(data, current[0].get(current[1]), _PPT_DOCUMENT)
+ if not document:
+ return _ppt_scan(data)
+ persist = current[0]
+
+ # SlideListWithText instance 0 lists slides with their placeholder text; instance 2 lists notes.
+ slides: list[tuple[int, int, list[str]]] = [] # (persist id, slide id, placeholder text)
+ notes_refs: list[int] = []
+ for kind, inst, body, stop, _container in _ppt_records(data, *document):
+ if kind != _PPT_SLIDE_LIST or inst not in (0, 2):
+ continue
+ for child, _i, child_body, child_stop, _c in _ppt_records(data, body, stop):
+ if child == _PPT_SLIDE_PERSIST and child_stop - child_body >= 16:
+ ref, _flags, _texts, slide_id = struct.unpack_from("<IIII", data, child_body)
+ if inst == 0:
+ slides.append((ref, slide_id, []))
+ else:
+ notes_refs.append(ref)
+ elif inst == 0 and slides and child == _PPT_TEXT_HEADER:
+ slides[-1][2].append("")
+ elif inst == 0 and slides and child in (_PPT_TEXT_CHARS, _PPT_TEXT_BYTES):
+ if not slides[-1][2]:
+ slides[-1][2].append("")
+ slides[-1][2][-1] = _ppt_text(data, child, child_body, child_stop)
+
+ notes: dict[int, list[str]] = {}
+ for ref in notes_refs:
+ page = _ppt_record_at(data, persist.get(ref), _PPT_NOTES)
+ if page is None:
+ continue
+ atom = next(
+ (r for r in _ppt_records(data, *page) if r[0] == _PPT_NOTES_ATOM and r[3] - r[2] >= 4),
+ None,
+ )
+ lines = _ppt_sheet_lines(data, *page, [])
+ if atom is not None and lines:
+ notes[struct.unpack_from("<I", data, atom[2])[0]] = lines
+
+ sections: list[Section] = []
+ for number, (ref, slide_id, listed) in enumerate(slides, 1):
+ slide = _ppt_record_at(data, persist.get(ref), _PPT_SLIDE)
+ lines = _ppt_sheet_lines(data, *slide, listed) if slide else [t for t in listed if t]
+ if slide_id in notes:
+ lines += ["Notes:"] + notes[slide_id]
+ if lines:
+ sections.append(("\n".join(lines), number))
+ return sections
+
+
+def _ppt_scan(data: bytes) -> list[Section]:
+ """Slide text by a linear scan, for files whose edit history cannot be followed."""
+ listed: list[list[str]] = [] # slide text from SlideListWithText (instance 0)
+ drawn: list[list[str]] = [] # slide text from Slide containers, for files without it
+ stack: list[tuple[int, int, int]] = [] # (end, type, instance)
+ pos = 0
+ while pos + 8 <= len(data):
+ while stack and pos >= stack[-1][0]:
+ stack.pop()
+ ver_inst, kind, size = struct.unpack_from("<HHI", data, pos)
+ if ver_inst & 0x000F == 0x000F: # container: descend
+ stack.append((pos + 8 + size, kind, ver_inst >> 4))
+ if kind == _PPT_SLIDE:
+ drawn.append([])
+ pos += 8
+ continue
+ body = data[pos + 8 : pos + 8 + size]
+ pos += 8 + size
+ in_list = next((inst for _end, t, inst in reversed(stack) if t == _PPT_SLIDE_LIST), None)
+ if kind == _PPT_SLIDE_PERSIST and in_list == 0:
+ listed.append([])
+ elif kind in (_PPT_TEXT_CHARS, _PPT_TEXT_BYTES):
+ text = body.decode("utf-16-le" if kind == _PPT_TEXT_CHARS else "latin-1", "replace")
+ text = _clean_control(text.replace("\r", "\n").replace("\x0b", "\n")).strip()
+ if not text:
+ continue
+ if in_list == 0 and listed:
+ listed[-1].append(text)
+ elif in_list is None and any(t == _PPT_SLIDE for _e, t, _i in stack) and drawn:
+ drawn[-1].append(text)
+ slides = listed if any(listed) else drawn
+ return [("\n".join(lines), n) for n, lines in enumerate(slides, 1) if lines]
+
+
+_MSG_HEADERS = (("0C1A", "From"), ("0E04", "To"), ("0E03", "Cc"), ("0037", "Subject"))
+
+
+# Windows code pages Python names differently from cpNNN.
+_CODE_PAGES = {
+ 1200: "utf-16-le",
+ 1201: "utf-16-be",
+ 20127: "ascii",
+ 20866: "koi8_r",
+ 21866: "koi8_u",
+ 50220: "iso2022_jp",
+ 50221: "iso2022_jp",
+ 50222: "iso2022_jp",
+ 51932: "euc_jp",
+ 51949: "euc_kr",
+ 52936: "hz",
+ 54936: "gb18030",
+ 65000: "utf-7",
+ 65001: "utf-8",
+ **{28590 + n: f"iso8859_{n}" for n in range(1, 16)},
+}
+
+
+def _code_page_codec(code_page: int | None) -> str:
+ name = _CODE_PAGES.get(code_page, f"cp{code_page}") if code_page else "cp1252"
+ try:
+ codecs.lookup(name)
+ except LookupError:
+ return "cp1252"
+ return name
+
+
+def _msg_props(cf: CompoundFile) -> dict[int, bytes]:
+ """Fixed-size top-level properties: tag -> 8-byte value."""
+ if not cf.exists("__properties_version1.0"):
+ return {}
+ props = cf.read("__properties_version1.0")
+ return {
+ struct.unpack_from("<I", props, off)[0]: props[off + 8 : off + 16]
+ for off in range(32, len(props) - 15, 16)
+ }
+
+
+def _msg_prop(cf: CompoundFile, prefix: tuple[str, ...], prop: str, codec: str) -> str | None:
+ # String8 ("001E") values are in the message code page.
+ for kind, kind_codec in (("001F", "utf-16-le"), ("001E", codec)):
+ name = f"__substg1.0_{prop}{kind}"
+ if cf.exists(*prefix, name):
+ return cf.read(*prefix, name).decode(kind_codec, "replace").rstrip("\0")
+ return None
+
+
+def _msg_date(props: dict[int, bytes]) -> str | None:
+ for prop in (0x0039, 0x0E06): # submit / delivery time
+ value = props.get((prop << 16) | 0x0040)
+ if value is None:
+ continue
+ ticks = struct.unpack("<Q", value)[0]
+ try:
+ moment = dt.datetime(1601, 1, 1) + dt.timedelta(microseconds = ticks // 10)
+ except OverflowError:
+ return None
+ return moment.isoformat(" ", "minutes") + " UTC"
+ return None
+
+
+# [MS-OXRTFCP] dictionary prefill for compressed RTF.
+_RTF_PREFILL = (
+ b"{\\rtf1\\ansi\\mac\\deff0\\deftab720{\\fonttbl;}{\\f0\\fnil \\froman \\fswiss "
+ b"\\fmodern \\fscript \\fdecor MS Sans SerifSymbolArialTimes New RomanCourier"
+ b"{\\colortbl\\red0\\green0\\blue0\r\n\\par \\pard\\plain\\f0\\fs20\\b\\i\\u\\tab\\tx"
+)
+
+
+def _decompress_rtf(data: bytes) -> bytes:
+ """PidTagRtfCompressed ([MS-OXRTFCP]) to RTF bytes."""
+ if len(data) < 16:
+ raise ValueError("truncated compressed RTF")
+ comp_size, raw_size, magic = struct.unpack_from("<III", data)
+ if raw_size > _MAX_MEMBER_BYTES:
+ raise ValueError("compressed RTF body is too large")
+ if magic == 0x414C454D: # "MELA": stored uncompressed
+ return data[16 : 16 + raw_size]
+ if magic != 0x75465A4C: # "LZFu"
+ raise ValueError("unknown compressed RTF format")
+ window = bytearray(4096)
+ window[: len(_RTF_PREFILL)] = _RTF_PREFILL
+ write = len(_RTF_PREFILL)
+ out = bytearray()
+ pos, end = 16, min(len(data), comp_size + 4)
+ while pos < end:
+ control = data[pos]
+ pos += 1
+ for bit in range(8):
+ # Stop at the declared size: back-references can expand far beyond it.
+ if pos >= end or len(out) >= raw_size:
+ return bytes(out[:raw_size])
+ if control & (1 << bit):
+ if pos + 2 > end:
+ break
+ ref = (data[pos] << 8) | data[pos + 1]
+ pos += 2
+ offset, length = ref >> 4, (ref & 0xF) + 2
+ if offset == write: # end marker
+ return bytes(out)
+ for k in range(length):
+ byte = window[(offset + k) % 4096]
+ out.append(byte)
+ window[write] = byte
+ write = (write + 1) % 4096
+ else:
+ out.append(data[pos])
+ window[write] = data[pos]
+ write = (write + 1) % 4096
+ pos += 1
+ return bytes(out[:raw_size])
+
+
+def msg(path: str, html_text) -> list[Section]:
+ cf = _compound(path)
+ if not any(name.lower().startswith("__substg1.0_") for name in cf.listdir()):
+ raise ValueError("not an Outlook message")
+ props = _msg_props(cf)
+ # PidTagMessageCodepage, else PidTagInternetCodepage.
+ code_page = next(
+ (
+ struct.unpack_from("<I", props[tag])[0]
+ for tag in (0x3FFD0003, 0x3FDE0003)
+ if tag in props
+ ),
+ None,
+ )
+ codec = _code_page_codec(code_page)
+ headers = {
+ label: (_msg_prop(cf, (), prop, codec) or "").strip() for prop, label in _MSG_HEADERS
+ }
+ address = (_msg_prop(cf, (), "0C1F", codec) or "").strip()
+ # Exchange senders carry an X.500 path; their SMTP address is stored separately.
+ if not address or address.startswith("/"):
+ address = (
+ _msg_prop(cf, (), "5D01", codec) or _msg_prop(cf, (), "5D02", codec) or ""
+ ).strip()
+ if address and not address.startswith("/") and address != headers["From"]:
+ headers["From"] = f"{headers['From']} <{address}>".strip()
+ headers["Date"] = _msg_date(props) or ""
+ lines = [
+ f"{label}: {headers[label]}"
+ for label in ("From", "To", "Cc", "Date", "Subject")
+ if headers[label]
+ ]
+ body = (_msg_prop(cf, (), "1000", codec) or "").strip()
+ if not body and cf.exists("__substg1.0_10130102"):
+ body = html_text(cf.read("__substg1.0_10130102"))
+ if not body and (html := _msg_prop(cf, (), "1013", codec)):
+ body = html_text(codecs.BOM_UTF8 + html.encode("utf-8"))
+ if not body and cf.exists("__substg1.0_10090102"):
+ rtf_bytes = _decompress_rtf(cf.read("__substg1.0_10090102"))
+ body = _rtf_text(rtf_bytes.decode("latin-1"))
+ attachments = []
+ for name in cf.listdir():
+ if name.lower().startswith("__attach_version1.0_"):
+ label = _msg_prop(cf, (name,), "3707", codec) or _msg_prop(cf, (name,), "3704", codec)
+ if label:
+ attachments.append(label)
+ if attachments:
+ lines.append("Attachments: " + ", ".join(sorted(attachments)))
+ body = (body or "").replace("\r\n", "\n").strip()
+ return [("\n".join(lines) + "\n\n" + body, None)]
diff --git a/studio/backend/core/rag/parsers.py b/studio/backend/core/rag/parsers.py
index 5d089a1882c..c8b1d775672 100644
--- a/studio/backend/core/rag/parsers.py
+++ b/studio/backend/core/rag/parsers.py
@@ -15,6 +15,7 @@ import logging
import os
import re
import secrets
+import struct
from dataclasses import dataclass
from html.parser import HTMLParser
@@ -847,12 +848,37 @@ def _docx_table_rows(table) -> list[str]:
return rows
+_WORD_EXTS = frozenset((".docx", ".docm", ".dotx", ".dotm"))
+_WORD_MAIN_TYPES = (
+ "application/vnd.ms-word.document.macroEnabled.main+xml",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml",
+ "application/vnd.ms-word.template.macroEnabledTemplate.main+xml",
+)
+
+
+def _open_word(path: str):
+ # docx.Document() rejects the macro and template main parts.
+ import docx # noqa: F401 (registers the part types)
+ from docx.opc.constants import CONTENT_TYPE as CT
+ from docx.opc.part import PartFactory
+ from docx.package import Package
+ from docx.parts.document import DocumentPart
+
+ for content_type in _WORD_MAIN_TYPES:
+ PartFactory.part_type_for.setdefault(content_type, DocumentPart)
+ part = Package.open(path).main_document_part
+ if part.content_type not in (CT.WML_DOCUMENT_MAIN, *_WORD_MAIN_TYPES):
+ raise ValueError(
+ f"file '{os.path.basename(path)}' is not a Word file, content type is '{part.content_type}'"
+ )
+ return part.document
+
+
def _docx(path: str) -> list[Page]:
- import docx
from docx.table import Table
from docx.text.paragraph import Paragraph
- document = docx.Document(path)
+ document = _open_word(path)
lines: list[str] = []
_docx_unwrap_table_controls(document.element.body)
label_notes = _docx_mark_notes(document)
@@ -1051,10 +1077,14 @@ def parse(path: str, *, want_images: bool = False):
pages, images, _total = _pdf(path, want_images)
return (pages, images) if want_images else pages
- if ext == ".docx":
+ if ext in _WORD_EXTS:
pages = _docx(path)
return (pages, []) if want_images else pages
+ if ext in config.DOCUMENT_UPLOAD_EXTS:
+ pages = _office_document(path, ext)
+ return (pages, []) if want_images else pages
+
if ext in (".html", ".htm", ".txt", ".md", ".markdown") or ext in config.SOURCE_TEXT_EXTS:
is_html = ext in (".html", ".htm")
with open(path, "rb") as f:
@@ -1071,6 +1101,39 @@ def parse(path: str, *, want_images: bool = False):
raise ValueError(f"unsupported file type: {ext}")
+def _html_bytes_text(raw: bytes) -> str:
+ return "\n\n".join(page.text for page in _html(_decode_text(raw, html = True)))
+
+
+def _xhtml(path: str) -> list[tuple[str, int | None]]:
+ with open(path, "rb") as f:
+ return [(_html_bytes_text(f.read()), None)]
+
+
+def _office_document(path: str, ext: str) -> list[Page]:
+ from . import office_formats as office
+
+ readers = {
+ ".doc": office.doc,
+ ".xls": office.xls,
+ ".ppt": office.ppt,
+ ".rtf": office.rtf,
+ ".epub": lambda p: office.epub(p, _html_bytes_text),
+ ".msg": lambda p: office.msg(p, _html_bytes_text),
+ **dict.fromkeys((".xlsx", ".xlsm", ".xltx", ".xltm"), office.xlsx),
+ **dict.fromkeys((".pptx", ".pptm", ".potx", ".potm", ".ppsx", ".ppsm"), office.pptx),
+ **dict.fromkeys((".odt", ".ods", ".odp", ".ott", ".ots", ".otp"), office.opendocument),
+ # .mht/.mhtml are MIME with an HTML body.
+ **dict.fromkeys((".eml", ".mht", ".mhtml"), lambda p: office.eml(p, _html_bytes_text)),
+ **dict.fromkeys((".xhtml", ".xht"), _xhtml),
+ }
+ try:
+ sections = readers[ext](path)
+ except (KeyError, IndexError, struct.error, UnicodeError, EOFError) as exc:
+ raise ValueError(f"could not read {os.path.basename(path)}: file is damaged") from exc
+ return [_page(text, number) for text, number in sections if text.strip()]
+
+
def parse_text(text: str) -> list[Page]:
"""Wrap already-extracted text as a single Page (tests / in-memory ingest)."""
return [_page(text, None)]
diff --git a/studio/backend/tests/_compound_file.py b/studio/backend/tests/_compound_file.py
new file mode 100644
index 00000000000..c2124692a0c
--- /dev/null
+++ b/studio/backend/tests/_compound_file.py
@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Minimal OLE compound file writer (version 3) for building .doc/.xls/.ppt/.msg fixtures."""
+
+from __future__ import annotations
+
+import math
+import struct
+
+_SECTOR, _MINI, _CUTOFF = 512, 64, 4096
+_FREE, _END, _FATSECT = 0xFFFFFFFF, 0xFFFFFFFE, 0xFFFFFFFD
+
+
+def _entry(name: str, kind: int, child: int, right: int, start: int, size: int) -> bytes:
+ encoded = (name + "\0").encode("utf-16-le")
+ return (
+ encoded.ljust(64, b"\0")
+ + struct.pack("<HBB", len(encoded), kind, 1)
+ + struct.pack("<III", _FREE, right, child)
+ + b"\0" * 36
+ + struct.pack("<IQ", start, size)
+ )
+
+
+def compound_file(streams: dict[tuple[str, ...], bytes]) -> bytes:
+ """Streams keyed by path; storages are implied. Streams under 4096 bytes use the mini stream."""
+ # Directory tree: each storage's children chained through right-sibling links.
+ nodes = [{"name": "Root Entry", "kind": 5, "children": []}]
+ index = {(): 0}
+ for path in streams:
+ for depth in range(1, len(path) + 1):
+ key = path[:depth]
+ if key not in index:
+ index[key] = len(nodes)
+ kind = 2 if depth == len(path) else 1
+ nodes.append({"name": key[-1], "kind": kind, "children": [], "path": key})
+ nodes[index[key[:-1]]]["children"].append(index[key])
+
+ mini = b""
+ big: list[tuple[int, bytes]] = []
+ placement = {}
+ for path, data in streams.items():
+ i = index[path]
+ if len(data) < _CUTOFF:
+ placement[i] = ("mini", len(mini) // _MINI, len(data))
+ mini += (
+ data.ljust(math.ceil(len(data) / _MINI) * _MINI or _MINI, b"\0") if data else b""
+ )
+ else:
+ big.append((i, data))
+
+ def sectors(n_bytes):
+ return math.ceil(n_bytes / _SECTOR)
+
+ mini_fat = []
+ for i, (where, start, size) in sorted(placement.items(), key = lambda kv: kv[1][1]):
+ count = math.ceil(size / _MINI)
+ mini_fat += [start + k + 1 for k in range(count - 1)] + ([_END] if count else [])
+ mini_fat_bytes = struct.pack(f"<{len(mini_fat)}I", *mini_fat) if mini_fat else b""
+
+ layout = [] # (label, n_sectors)
+ n_dir = sectors(len(nodes) * 128)
+ layout.append(("dir", n_dir))
+ layout.append(("minifat", sectors(len(mini_fat_bytes))))
+ layout.append(("ministream", sectors(len(mini))))
+ for i, data in big:
+ layout.append((i, sectors(len(data))))
+ data_sectors = sum(n for _, n in layout)
+ n_fat = 1
+ while n_fat * 128 < data_sectors + n_fat:
+ n_fat += 1
+
+ start_of, cursor = {}, n_fat
+ for label, n in layout:
+ start_of[label] = cursor if n else _END
+ cursor += n
+ fat = [_FATSECT] * n_fat
+ for label, n in layout:
+ fat += [start_of[label] + k + 1 for k in range(n - 1)] + ([_END] if n else [])
+ fat += [_FREE] * (n_fat * 128 - len(fat))
+
+ entries = []
+ for i, node in enumerate(nodes):
+ kids = node["children"]
+ child = kids[0] if kids else _FREE
+ parent = next((n for n in nodes if i in n["children"]), None)
+ right = _FREE
+ if parent is not None:
+ pos = parent["children"].index(i)
+ right = parent["children"][pos + 1] if pos + 1 < len(parent["children"]) else _FREE
+ if i == 0:
+ start, size = start_of["ministream"], len(mini)
+ elif node["kind"] == 2:
+ if i in placement:
+ start, size = placement[i][1], placement[i][2]
+ else:
+ start, size = start_of[i], len(streams[node["path"]])
+ else:
+ start, size = 0, 0
+ entries.append(_entry(node["name"], node["kind"], child, right, start, size))
+ directory = b"".join(entries)
+
+ header = bytearray(512)
+ header[:8] = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+ struct.pack_into("<HHHHH", header, 0x18, 0x3E, 3, 0xFFFE, 9, 6)
+ struct.pack_into("<I", header, 0x2C, n_fat)
+ struct.pack_into("<I", header, 0x30, start_of["dir"])
+ struct.pack_into("<I", header, 0x38, _CUTOFF)
+ struct.pack_into("<II", header, 0x3C, start_of["minifat"], sectors(len(mini_fat_bytes)))
+ struct.pack_into("<II", header, 0x44, _END, 0)
+ difat = list(range(n_fat)) + [_FREE] * (109 - n_fat)
+ struct.pack_into("<109I", header, 0x4C, *difat)
+
+ body = struct.pack(f"<{len(fat)}I", *fat)
+ for label, n in layout:
+ if label == "dir":
+ chunk = directory
+ elif label == "minifat":
+ chunk = mini_fat_bytes
+ elif label == "ministream":
+ chunk = mini
+ else:
+ chunk = streams[nodes[label]["path"]]
+ body += chunk.ljust(n * _SECTOR, b"\0")
+ return bytes(header) + body
diff --git a/studio/backend/tests/test_rag_config_upload_exts.py b/studio/backend/tests/test_rag_config_upload_exts.py
index 5cb82bddf01..5a901f095ae 100644
--- a/studio/backend/tests/test_rag_config_upload_exts.py
+++ b/studio/backend/tests/test_rag_config_upload_exts.py
@@ -29,13 +29,20 @@ def _reload(monkeypatch, value):
@pytest.mark.parametrize(
"value, expected",
[
- (None, config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS),
- ("", config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS),
+ (
+ None,
+ config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS | config.DOCUMENT_UPLOAD_EXTS,
+ ),
+ ("", config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS | config.DOCUMENT_UPLOAD_EXTS),
(" .MD , .Markdown ", {".md", ".markdown"}),
("md,pdf", {".md", ".pdf"}),
(".md,.exe", {".md"}),
(".py,.exe", {".py"}),
- (".exe", config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS),
+ (".xlsx,.epub", {".xlsx", ".epub"}),
+ (
+ ".exe",
+ config.SUPPORTED_UPLOAD_EXTS | config.SOURCE_TEXT_EXTS | config.DOCUMENT_UPLOAD_EXTS,
+ ),
],
)
def test_upload_exts_env(monkeypatch, value, expected):
diff --git a/studio/backend/tests/test_rag_office_formats.py b/studio/backend/tests/test_rag_office_formats.py
new file mode 100644
index 00000000000..dc79266fe26
--- /dev/null
+++ b/studio/backend/tests/test_rag_office_formats.py
@@ -0,0 +1,1272 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Office, OpenDocument, e-book, email and RTF files: parsing, upload and indexing."""
+
+import datetime as dt
+import io
+import re
+import struct
+import zipfile
+from email.message import EmailMessage
+from xml.etree import ElementTree as ET
+from xml.sax.saxutils import escape
+from pathlib import Path
+
+import pytest
+from fastapi import UploadFile
+
+from core.rag import cfb, config, ingestion, office_formats, parsers, store
+from routes.rag import _save_upload
+from storage import rag_db
+
+from ._compound_file import compound_file
+
+S = 'xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"'
+P = 'xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"'
+REL = 'xmlns="http://schemas.openxmlformats.org/package/2006/relationships"'
+ODF = (
+ 'xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" '
+ 'xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" '
+ 'xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" '
+ 'xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" '
+ 'xmlns:presentation="urn:oasis:names:tc:opendocument:xmlns:presentation:1.0"'
+)
+
+
+def _zip(path, members):
+ with zipfile.ZipFile(path, "w") as z:
+ for name, data in members.items():
+ z.writestr(name, data)
+ return path
+
+
+def _text(path):
+ return "\n".join(page.text for page in parsers.parse(str(path)))
+
+
+def build_xlsx(path):
+ serial = (dt.date(2026, 3, 31) - dt.date(1899, 12, 30)).days
+ return _zip(
+ path,
+ {
+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="Revenue" sheetId="1" r:id="rId1"/></sheets></workbook>',
+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',
+ "xl/sharedStrings.xml": f"<sst {S}><si><t>Region</t></si>"
+ "<si><r><t>Zebra</t></r><r><t>marker</t></r><rPh><t>ignored</t></rPh></si></sst>",
+ "xl/styles.xml": f'<styleSheet {S}><cellXfs><xf numFmtId="0"/><xf numFmtId="14"/></cellXfs></styleSheet>',
+ "xl/worksheets/sheet1.xml": f"<worksheet {S}><sheetData>"
+ '<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1" t="inlineStr"><is><t>Q1</t></is></c></row>'
+ f'<row r="2"><c r="A2" t="s"><v>1</v></c><c r="B2"><v>1200.5</v></c><c r="C2" s="1"><v>{serial}</v></c>'
+ '<c r="D2" t="b"><v>1</v></c><c r="F2" t="str"><v>formula text</v></c></row>'
+ "</sheetData></worksheet>",
+ },
+ )
+
+
+def build_pptx(path):
+ run = lambda text: f"<a:p><a:r><a:t>{text}</a:t></a:r></a:p>"
+ return _zip(
+ path,
+ {
+ "ppt/presentation.xml": f'<p:presentation {P}><p:sldIdLst><p:sldId id="256" r:id="rId2"/></p:sldIdLst></p:presentation>',
+ "ppt/_rels/presentation.xml.rels": f'<Relationships {REL}><Relationship Id="rId2" Target="slides/slide1.xml"/></Relationships>',
+ "ppt/slides/slide1.xml": f"<p:sld {P}><p:cSld><p:spTree>"
+ f"<p:sp><p:txBody>{run('Quarterly review')}</p:txBody></p:sp>"
+ "<p:graphicFrame><a:graphic><a:graphicData><a:tbl>"
+ f"<a:tr><a:tc><a:txBody>{run('Region')}</a:txBody></a:tc><a:tc><a:txBody>{run('Q1')}</a:txBody></a:tc></a:tr>"
+ f"<a:tr><a:tc><a:txBody>{run('North')}</a:txBody></a:tc><a:tc><a:txBody>{run('1200')}</a:txBody></a:tc></a:tr>"
+ "</a:tbl></a:graphicData></a:graphic></p:graphicFrame>"
+ f"<p:sp><p:txBody>{run('Revenue increased')}</p:txBody></p:sp>"
+ "</p:spTree></p:cSld></p:sld>",
+ "ppt/slides/_rels/slide1.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="../notesSlides/notesSlide1.xml"/></Relationships>',
+ "ppt/notesSlides/notesSlide1.xml": f"<p:notes {P}><p:cSld><p:spTree>"
+ '<p:sp><p:nvSpPr><p:cNvPr id="5" name="Slide Number"/><p:cNvSpPr/><p:nvPr><p:ph type="sldNum" idx="5"/></p:nvPr></p:nvSpPr>'
+ '<p:txBody><a:p><a:fld id="{1}" type="slidenum"><a:t>1</a:t></a:fld></a:p></p:txBody></p:sp>'
+ f"<p:sp><p:txBody>{run('2026')}{run('Speaker notes zebramarker')}</p:txBody></p:sp>"
+ "</p:spTree></p:cSld></p:notes>",
+ },
+ )
+
+
+def _odf(path, body):
+ suffix = Path(path).suffix
+ kind = {".odt": "text", ".ods": "spreadsheet", ".odp": "presentation"}.get(suffix)
+ if kind is None:
+ kind = {".ott": "text", ".ots": "spreadsheet", ".otp": "presentation"}[suffix] + "-template"
+ return _zip(
+ path,
+ {
+ "mimetype": f"application/vnd.oasis.opendocument.{kind}",
+ "content.xml": f"<office:document-content {ODF}><office:body>{body}</office:body></office:document-content>",
+ },
+ )
+
+
+def build_odt(path):
+ return _odf(
+ path,
+ "<office:text><text:h>Quarterly report</text:h>"
+ '<text:p>Revenue<text:s text:c="2"/>increased<text:note><text:note-body><text:p>see appendix</text:p></text:note-body></text:note>.</text:p>'
+ "<table:table><table:table-row><table:table-cell><text:p>Zebramarker</text:p></table:table-cell>"
+ "<table:table-cell><text:p>1200</text:p></table:table-cell></table:table-row></table:table></office:text>",
+ )
+
+
+def build_ods(path):
+ return _odf(
+ path,
+ '<office:spreadsheet><table:table table:name="Revenue">'
+ "<table:table-row><table:table-cell><text:p>Region</text:p></table:table-cell>"
+ '<table:table-cell table:number-columns-repeated="2"><text:p>Q</text:p></table:table-cell>'
+ '<table:table-cell table:number-columns-repeated="16000"/></table:table-row>'
+ '<table:table-row table:number-rows-repeated="1048000"><table:table-cell table:number-columns-repeated="16000"/></table:table-row>'
+ "<table:table-row><table:table-cell><text:p>Zebramarker</text:p></table:table-cell></table:table-row>"
+ "</table:table></office:spreadsheet>",
+ )
+
+
+def build_odp(path):
+ return _odf(
+ path,
+ "<office:presentation>"
+ "<draw:page><draw:frame><draw:text-box><text:p>First slide</text:p></draw:text-box></draw:frame>"
+ "<presentation:notes><draw:frame><draw:text-box><text:p>zebramarker notes</text:p></draw:text-box></draw:frame></presentation:notes></draw:page>"
+ "<draw:page><draw:frame><draw:text-box><text:p>Second slide revenue</text:p></draw:text-box></draw:frame></draw:page>"
+ "</office:presentation>",
+ )
+
+
+def build_epub(path):
+ page = (
+ lambda title,
+ text: f"<html><head><title>{title}</title></head><body><p>{text}</p></body></html>"
+ )
+ return _zip(
+ path,
+ {
+ "mimetype": "application/epub+zip",
+ "META-INF/container.xml": '<container xmlns="urn:oasis:names:tc:opendocument:xmlns:container">'
+ '<rootfiles><rootfile full-path="OEBPS/content.opf"/></rootfiles></container>',
+ "OEBPS/content.opf": '<package xmlns="http://www.idpf.org/2007/opf"><manifest>'
+ '<item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>'
+ '<item id="c1" href="text/Chapter%201.xhtml#start" media-type="application/xhtml+xml"/>'
+ '<item id="c2" href="text/c2.xhtml" media-type="application/xhtml+xml"/>'
+ '</manifest><spine><itemref idref="nav"/><itemref idref="c2"/><itemref idref="c1"/></spine></package>',
+ "OEBPS/nav.xhtml": page("Contents", "navmarker"),
+ "OEBPS/text/Chapter 1.xhtml": page("One", "Zebramarker ends the book."),
+ "OEBPS/text/c2.xhtml": page("Two", "Revenue increased."),
+ },
+ )
+
+
+def build_eml(path):
+ message = EmailMessage()
+ message["From"] = "Ann <[email protected]>"
+ message["To"] = "[email protected]"
+ message["Subject"] = "Q1 numbers résumé"
+ message.set_content("Revenue increased.\nZebramarker in the body.", cte = "base64")
+ message.add_alternative("<p>html copy</p>", subtype = "html")
+ message.add_attachment(b"x,y\n", maintype = "text", subtype = "csv", filename = "data.csv")
+ path.write_bytes(bytes(message))
+ return path
+
+
+def build_mhtml(path):
+ # As Chrome saves a page: multipart/related, the page plus its resources.
+ message = EmailMessage()
+ message["From"] = "<Saved by Blink>"
+ message["Subject"] = "Quarterly report"
+ message.set_content(
+ "<html><body><p>Zebramarker on the page.</p><img src='logo.png'></body></html>",
+ subtype = "html",
+ )
+ message.make_related()
+ message.add_related(b"\x89PNG\r\n\x1a\n", maintype = "image", subtype = "png", cid = "<logo>")
+ path.write_bytes(bytes(message))
+ return path
+
+
+def build_xhtml(path):
+ path.write_text(
+ '<?xml version="1.0" encoding="utf-8"?><html xmlns="http://www.w3.org/1999/xhtml">'
+ "<head><title>Report</title><style>p{}</style></head><body><p>Zebramarker résumé</p></body></html>",
+ encoding = "utf-8",
+ )
+ return path
+
+
+_WORD_TYPES = {
+ ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml",
+ ".docm": "application/vnd.ms-word.document.macroEnabled.main+xml",
+ ".dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml",
+ ".dotm": "application/vnd.ms-word.template.macroEnabledTemplate.main+xml",
+}
+
+
+def build_word(path):
+ import docx
+
+ document = docx.Document()
+ document.add_paragraph("Quarterly report")
+ document.add_table(rows = 1, cols = 2).rows[0].cells[0].text = "Zebramarker"
+ buffer = io.BytesIO()
+ document.save(buffer)
+ with zipfile.ZipFile(buffer) as source, zipfile.ZipFile(path, "w") as target:
+ for info in source.infolist():
+ data = source.read(info)
+ if info.filename == "[Content_Types].xml":
+ data = data.replace(
+ _WORD_TYPES[".docx"].encode(), _WORD_TYPES[path.suffix].encode()
+ )
+ target.writestr(info, data)
+ return path
+
+
+def build_rtf(path):
+ path.write_bytes(
+ rb"{\rtf1\ansi\ansicpg1252\uc1{\fonttbl{\f0 Arial;}}{\colortbl;\red0\green0\blue0;}"
+ rb"{\*\generator Writer;}\f0 Quarterly report\par Caf\'e9 \u8364? revenue increased\par"
+ rb"{\field{\*\fldinst HYPERLINK hiddenmarker}{\fldrslt link text}}\par"
+ rb"\trowd Zebramarker\cell 1200\cell\row}"
+ )
+ return path
+
+
+def build_doc(path, *, encrypted = False):
+ compressed = "Quarterly report\r".encode("cp1252")
+ wide = ("Café 東京 \x13 HYPERLINK x \x14link\x15 Zebramarker\x071200\x07\x07").encode(
+ "utf-16-le"
+ )
+ word = bytearray(0x1800)
+ flags = 0x0200 | (0x0100 if encrypted else 0)
+ struct.pack_into("<HH", word, 0, 0xA5EC, 0xC1)
+ struct.pack_into("<H", word, 0x0A, flags)
+ struct.pack_into("<H", word, 32, 14) # csw
+ struct.pack_into("<H", word, 62, 22) # cslw
+ ccp = len(compressed) + len(wide) // 2
+ footnote, endnote, text_box = "\x02 Footnote detail\r", "\x02 Endnote source\r", "Boxed text\r"
+ struct.pack_into("<i", word, 64 + 3 * 4, ccp) # ccpText
+ struct.pack_into("<i", word, 64 + 4 * 4, len(footnote)) # ccpFtn
+ struct.pack_into("<i", word, 64 + 8 * 4, len(endnote)) # ccpEdn
+ struct.pack_into("<i", word, 64 + 9 * 4, len(text_box)) # ccpTxbx
+ struct.pack_into("<H", word, 152, 93) # cbRgFcLcb
+ stories = (footnote + endnote + text_box).encode("utf-16-le")
+ word[0x1000 : 0x1000 + len(compressed)] = compressed
+ word[0x1400 : 0x1400 + len(wide)] = wide
+ word[0x1600 : 0x1600 + len(stories)] = stories
+ cps = [0, len(compressed), ccp, ccp + len(stories) // 2]
+ pcds = [(0x2000 | 0x40000000), 0x1400, 0x1600]
+ plc = struct.pack("<4I", *cps) + b"".join(struct.pack("<HIH", 0, fc, 0) for fc in pcds)
+ clx = b"\x01" + struct.pack("<H", 2) + b"\0\0" + b"\x02" + struct.pack("<I", len(plc)) + plc
+ struct.pack_into("<II", word, 154 + 33 * 8, 0, len(clx)) # fcClx, lcbClx
+ path.write_bytes(compound_file({("WordDocument",): bytes(word), ("1Table",): clx}))
+ return path
+
+
+def test_doc_refuses_pieces_that_repeat_one_range(tmp_path):
+ word = bytearray(0x1800)
+ struct.pack_into("<HH", word, 0, 0xA5EC, 0xC1)
+ struct.pack_into("<H", word, 0x0A, 0x0200)
+ struct.pack_into("<H", word, 32, 14) # csw
+ struct.pack_into("<H", word, 62, 22) # cslw
+ word[0x1000:0x1040] = b"z" * 64
+ n = 400 # 400 pieces of 64 chars, all from the same 64 bytes
+ struct.pack_into("<i", word, 64 + 3 * 4, 64 * n) # ccpText
+ struct.pack_into("<H", word, 152, 93) # cbRgFcLcb
+ plc = struct.pack(f"<{n + 1}I", *range(0, 64 * (n + 1), 64))
+ plc += struct.pack("<HIH", 0, 0x2000 | 0x40000000, 0) * n
+ clx = b"\x02" + struct.pack("<I", len(plc)) + plc
+ struct.pack_into("<II", word, 154 + 33 * 8, 0, len(clx)) # fcClx, lcbClx
+ path = tmp_path / "pieces.doc"
+ path.write_bytes(compound_file({("WordDocument",): bytes(word), ("1Table",): clx}))
+ with pytest.raises(ValueError, match = "repeats too much content"):
+ parsers.parse(str(path))
+
+
+def _record(kind, body):
+ return struct.pack("<HH", kind, len(body)) + body
+
+
+def build_xls(path):
+ def short(text, size = 1):
+ return struct.pack("<B" if size == 1 else "<H", len(text)) + b"\0" + text.encode("latin-1")
+
+ # SST: "Region", then a string whose characters continue, wide, in a CONTINUE record.
+ sst = struct.pack("<II", 2, 2) + struct.pack("<HB", 6, 0) + b"Region"
+ sst += struct.pack("<HB", 12, 0) + b"Zebra"
+ cont = b"\x01" + "marker€".encode("utf-16-le")
+ xf = lambda fmt: _record(0x00E0, struct.pack("<HH", 0, fmt) + b"\0" * 16)
+ globals_ = (
+ _record(0x0809, b"\0" * 16)
+ + _record(0x0085, b"\0" * 4 + b"\0\0" + short("Revenue"))
+ + _record(0x00FC, sst)
+ + _record(0x003C, cont)
+ + _record(0x041E, struct.pack("<H", 164) + short("yyyy-mm-dd", 2))
+ + xf(0)
+ + xf(14)
+ + xf(164)
+ + _record(0x000A, b"")
+ )
+ serial = (dt.date(2026, 3, 31) - dt.date(1899, 12, 30)).days
+ rk_int = lambda n: ((n << 2) | 2) & 0xFFFFFFFF
+ sheet = (
+ _record(0x0809, b"\0" * 16)
+ + _record(0x00FD, struct.pack("<HHHI", 0, 0, 0, 0))
+ + _record(0x0203, struct.pack("<HHHd", 0, 1, 0, 1200.5))
+ + _record(0x00FD, struct.pack("<HHHI", 1, 0, 0, 1))
+ + _record(0x027E, struct.pack("<HHHI", 1, 1, 0, rk_int(-7)))
+ + _record(
+ 0x00BD,
+ struct.pack("<HH", 2, 0)
+ + struct.pack("<HI", 0, rk_int(3))
+ + struct.pack("<HI", 0, rk_int(4))
+ + struct.pack("<H", 1),
+ )
+ + _record(0x0809, b"\0" * 16) # embedded chart
+ + _record(0x000A, b"")
+ + _record(0x0006, struct.pack("<HHH", 3, 0, 0) + b"\0" * 6 + b"\xff\xff" + b"\0" * 6)
+ + _record(0x0207, short("formula text", 2))
+ + _record(0x0205, struct.pack("<HHHBB", 3, 1, 0, 1, 0))
+ + _record(0x0203, struct.pack("<HHHd", 4, 0, 2, serial))
+ + _record(0x027E, struct.pack("<HHHI", 4, 1, 1, rk_int(serial + 1)))
+ + _record(0x000A, b"")
+ )
+ stream = bytearray(globals_ + sheet)
+ struct.pack_into("<I", stream, 4 + 16 + 4, len(globals_)) # BOUNDSHEET lbPlyPos
+ path.write_bytes(compound_file({("Workbook",): bytes(stream)}))
+ return path
+
+
+def _ppt_atom(
+ kind,
+ body,
+ inst = 0,
+):
+ return struct.pack("<HHI", inst << 4, kind, len(body)) + body
+
+
+def _ppt_container(
+ kind,
+ children,
+ inst = 0,
+):
+ body = b"".join(children)
+ return struct.pack("<HHI", (inst << 4) | 0xF, kind, len(body)) + body
+
+
+def build_ppt(path, *, from_slides = False):
+ chars = lambda text: _ppt_atom(0x0FA0, text.encode("utf-16-le"))
+ persist = _ppt_atom(0x03F3, b"\0" * 20)
+ if from_slides:
+ slides = [
+ _ppt_container(
+ 0x03EE, [_ppt_container(0x040C, [_ppt_container(0xF00D, [chars(text)])])]
+ )
+ for text in ("Drawn slide", "Zebramarker drawn")
+ ]
+ document = _ppt_container(0x03E8, slides)
+ else:
+ document = _ppt_container(
+ 0x03E8,
+ [
+ _ppt_container(
+ 0x0FF0,
+ [
+ persist,
+ chars("Quarterly review\rRevenue up"),
+ persist,
+ _ppt_atom(0x0FA8, b"Zebramarker slide"),
+ ],
+ inst = 0,
+ ),
+ _ppt_container(
+ 0x0FF0, [persist, chars("Click to edit Master title style")], inst = 1
+ ),
+ ],
+ )
+ path.write_bytes(compound_file({("PowerPoint Document",): document}))
+ return path
+
+
+# PidTagRtfCompressed of "{\\rtf1\\ansi\\ansicpg1252\\pard Zebramarker in the RTF body\\par repeat repeat repeat}".
+RTF_COMPRESSED = bytes.fromhex(
+ "49000000520000004c5a4675a3e8a93d03000a007263706731323502320af3205a656272612900c0726b04902"
+ "00b80207444686507f054462006e064c6790aa30970706561054010cb027d1210"
+)
+
+
+def build_ansi_msg(path):
+ """String8 properties in code page 1251 and only a compressed RTF body."""
+ props = b"\0" * 32 + struct.pack("<IIQ", 0x3FFD0003, 0, 1251)
+ streams = {
+ ("__substg1.0_0037001E",): "Привет".encode("cp1251"),
+ ("__substg1.0_0C1A001E",): "Иван".encode("cp1251"),
+ ("__substg1.0_10090102",): RTF_COMPRESSED,
+ ("__properties_version1.0",): props,
+ }
+ path.write_bytes(compound_file(streams))
+ return path
+
+
+def build_saved_ppt(path, *, broken_link = False):
+ """Saved twice: the second save replaces the slide, adds notes and leaves the old slide behind."""
+ chars = lambda text: _ppt_atom(0x0FA0, text.encode("utf-16-le"))
+ textbox = lambda *atoms: _ppt_container(0xF00D, list(atoms))
+ drawing = lambda *boxes: _ppt_container(0x040C, [_ppt_container(0xF002, list(boxes))])
+ persist = lambda ref, sheet_id: _ppt_atom(0x03F3, struct.pack("<5I", ref, 0, 1, sheet_id, 0))
+ stale = _ppt_container(0x03EE, [drawing(textbox(chars("Stale slide text")))])
+ slide = _ppt_container(
+ 0x03EE,
+ [
+ drawing(
+ textbox(_ppt_atom(0x0F9E, struct.pack("<I", 0))),
+ textbox(chars("Text box zebramarker")),
+ )
+ ],
+ )
+ notes = _ppt_container(
+ 0x03F0,
+ [
+ _ppt_atom(0x03F1, struct.pack("<IHH", 256, 0, 0)),
+ drawing(textbox(chars("Speaker notes")), textbox(chars("*"))),
+ ],
+ )
+ document = _ppt_container(
+ 0x03E8,
+ [
+ _ppt_container(
+ 0x0FF0,
+ [persist(2, 256), _ppt_atom(0x0F9F, b"\0" * 4), chars("Title from list")],
+ inst = 0,
+ ),
+ _ppt_container(0x0FF0, [persist(3, 257)], inst = 2),
+ ],
+ )
+ stream = bytearray()
+
+ def put(record):
+ stream.extend(record)
+ return len(stream) - len(record)
+
+ def directory(entries):
+ body = b"".join(struct.pack("<II", (1 << 20) | pid, offset) for pid, offset in entries)
+ return put(_ppt_atom(0x1772, body))
+
+ def user_edit(last, directory_offset):
+ offset = len(stream)
+ last = offset if broken_link and last else last
+ return put(
+ _ppt_atom(0x0FF5, struct.pack("<IHBBIIII", 0, 0, 0, 3, last, directory_offset, 1, 4))
+ )
+
+ stale_at, doc_at = put(stale), put(document)
+ first = user_edit(0, directory([(1, doc_at), (2, stale_at)]))
+ slide_at, notes_at = put(slide), put(notes)
+ current = user_edit(first, directory([(2, slide_at), (3, notes_at)]))
+ current_user = _ppt_atom(0x0FF6, struct.pack("<III", 20, 0xE391C05F, current))
+ path.write_bytes(
+ compound_file({("PowerPoint Document",): bytes(stream), ("Current User",): current_user})
+ )
+ return path
+
+
+def build_msg(path):
+ utf16 = lambda text: text.encode("utf-16-le")
+ filetime = int(
+ (dt.datetime(2026, 10, 6, 10, 0) - dt.datetime(1601, 1, 1)).total_seconds() * 10**7
+ )
+ props = b"\0" * 32 + struct.pack("<IIQ", (0x0039 << 16) | 0x0040, 0, filetime)
+ streams = {
+ ("__substg1.0_0037001F",): utf16("Q1 numbers"),
+ ("__substg1.0_0C1A001F",): utf16("Ann"),
+ ("__substg1.0_0C1F001F",): utf16("[email protected]"),
+ ("__substg1.0_0E04001F",): utf16("Bob"),
+ ("__substg1.0_1000001F",): utf16("Revenue increased.\r\nZebramarker in the body."),
+ ("__properties_version1.0",): props,
+ ("__attach_version1.0_#00000000", "__substg1.0_3707001F"): utf16("data.csv"),
+ }
+ path.write_bytes(compound_file(streams))
+ return path
+
+
+BUILDERS = {
+ ".doc": build_doc,
+ ".xls": build_xls,
+ ".ppt": build_ppt,
+ ".msg": build_msg,
+ ".eml": build_eml,
+ ".rtf": build_rtf,
+ ".epub": build_epub,
+ **dict.fromkeys((".docm", ".dotx", ".dotm"), build_word),
+ **dict.fromkeys((".xlsx", ".xlsm", ".xltx", ".xltm"), build_xlsx),
+ **dict.fromkeys((".pptx", ".pptm", ".potx", ".potm", ".ppsx", ".ppsm"), build_pptx),
+ **dict.fromkeys((".odt", ".ott"), build_odt),
+ **dict.fromkeys((".ods", ".ots"), build_ods),
+ **dict.fromkeys((".odp", ".otp"), build_odp),
+ **dict.fromkeys((".mht", ".mhtml"), build_mhtml),
+ **dict.fromkeys((".xhtml", ".xht"), build_xhtml),
+}
+
+
+def test_every_document_type_has_a_fixture():
+ assert set(BUILDERS) == config.DOCUMENT_UPLOAD_EXTS
+ assert config.DOCUMENT_UPLOAD_EXTS <= config.UPLOAD_EXTS
+
+
+def test_document_exts_mirror_the_frontend_accept_list():
+ source = (
+ Path(__file__).resolve().parents[2] / "frontend/src/features/rag/types/rag.ts"
+ ).read_text(encoding = "utf-8")
+ accept = re.search(r'RAG_DOCUMENT_UPLOAD_ACCEPT =\s*"([^"]+)"', source).group(1)
+ assert set(accept.split(",")) == config.DOCUMENT_UPLOAD_EXTS
+
+
+def test_xlsx_reads_shared_inline_dates_and_gaps(tmp_path):
+ text = _text(build_xlsx(tmp_path / "book.xlsx"))
+ assert (
+ text
+ == "Sheet: Revenue\nRegion | Q1\nZebramarker | 1200.5 | 2026-03-31 | TRUE | | formula text"
+ )
+
+
+def test_pptx_reads_slides_in_order_with_tables_and_notes(tmp_path):
+ pages = parsers.parse(str(build_pptx(tmp_path / "deck.pptx")))
+ assert [(p.page_number, p.text) for p in pages] == [
+ (
+ 1,
+ "Quarterly review\nRegion | Q1\nNorth | 1200\nRevenue increased\nNotes:\n2026\nSpeaker notes zebramarker",
+ )
+ ]
+
+
+def test_odt_reads_paragraphs_spaces_notes_and_tables(tmp_path):
+ assert _text(build_odt(tmp_path / "memo.odt")) == (
+ "Quarterly report\nRevenue increased [see appendix].\nZebramarker | 1200"
+ )
+
+
+def test_ods_does_not_expand_empty_repeats(tmp_path):
+ assert _text(build_ods(tmp_path / "sheet.ods")) == "Sheet: Revenue\nRegion | Q | Q\nZebramarker"
+
+
+def test_ots_template_is_streamed_like_ods(tmp_path, monkeypatch):
+ calls = []
+ real = office_formats._ods_streamed
+ monkeypatch.setattr(office_formats, "_ods_streamed", lambda *a: calls.append(1) or real(*a))
+ path = _odf(
+ tmp_path / "plan.ots",
+ '<office:spreadsheet><table:table table:name="S"><table:table-row>'
+ "<table:table-cell><text:p>Zebramarker</text:p></table:table-cell>"
+ "</table:table-row></table:table></office:spreadsheet>",
+ )
+ assert "Zebramarker" in _text(path)
+ assert calls == [1]
+
+
+def test_ods_keeps_the_width_of_empty_runs_before_content(tmp_path):
+ path = _odf(
+ tmp_path / "gaps.ods",
+ '<office:spreadsheet><table:table table:name="S"><table:table-row>'
+ "<table:table-cell><text:p>A</text:p></table:table-cell>"
+ '<table:table-cell table:number-columns-repeated="4"/>'
+ "<table:table-cell><text:p>F</text:p></table:table-cell>"
+ '<table:table-cell table:number-columns-repeated="16000"/>'
+ "</table:table-row></table:table></office:spreadsheet>",
+ )
+ assert _text(path) == "Sheet: S\nA | | | | | F"
+
+
+def test_ods_reads_typed_values_without_text(tmp_path):
+ path = _odf(
+ tmp_path / "values.ods",
+ '<office:spreadsheet><table:table table:name="S"><table:table-row>'
+ '<table:table-cell office:value-type="float" office:value="1200"/>'
+ '<table:table-cell office:value-type="date" office:date-value="2026-03-31"/>'
+ '<table:table-cell office:value-type="boolean" office:boolean-value="true"/>'
+ '<table:table-cell office:value-type="string" office:string-value="zebramarker"/>'
+ '<table:table-cell office:value-type="float" office:value="7"><text:p>7.00</text:p></table:table-cell>'
+ "</table:table-row></table:table></office:spreadsheet>",
+ )
+ assert _text(path) == "Sheet: S\n1200 | 2026-03-31 | TRUE | zebramarker | 7.00"
+
+
+# Dates, times and durations match what openpyxl reads back from the same cells.
+ "serial, number_format, expected",
+ [
+ (0.5, "h:mm AM/PM", "12:00:00"),
+ (1.5, "[h]:mm:ss", "36:00:00"),
+ (1.5, "h:mm", "1900-01-01 12:00:00"),
+ (1, "yyyy-mm-dd", "1900-01-01"),
+ (59, "yyyy-mm-dd", "1900-02-28"),
+ (61, "yyyy-mm-dd", "1900-03-01"),
+ (0.25, "mm-dd-yy", "06:00:00"),
+ (3, '"day" 0', "day 3"),
+ (0.25, "0%", "25%"),
+ (0.125, "0.0%", "12.5%"),
+ (123, "000000", "000123"),
+ (-1234.5, "#,##0.00", "-1,234.50"),
+ (-1234.5, "#,##0.00;(#,##0.00)", "(1,234.50)"),
+ (12345.678, "0.00E+00", "1.23E+04"),
+ (12345.678, "##0.0E+0", "12.3E+3"),
+ (1234.5, '"$"#,##0', "$1,235"),
+ (1234.5, "[$\u20ac-407]#,##0.00", "\u20ac1,234.50"),
+ (0.5, "#.##", ".5"),
+ (1.234, "0.0#", "1.23"),
+ (1500000, '#,##0,,"M"', "2M"),
+ (1234.5, "General", "1234.5"),
+ (5, "[>100]0;0.0", "5"),
+ pytest.param(1234.5, "0" * 5000, "1234.5", id = "oversized-format-left-as-stored"),
+ ],
+)
+def test_xlsx_shows_numbers_as_formatted(tmp_path, serial, number_format, expected):
+ path = _zip(
+ tmp_path / "times.xlsx",
+ {
+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="T" sheetId="1" r:id="rId1"/></sheets></workbook>',
+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',
+ "xl/styles.xml": f'<styleSheet {S}><numFmts><numFmt numFmtId="164" formatCode="{escape(number_format, {chr(34): "&quot;"})}"/></numFmts>'
+ '<cellXfs><xf numFmtId="0"/><xf numFmtId="164"/></cellXfs></styleSheet>',
+ "xl/worksheets/sheet1.xml": f'<worksheet {S}><sheetData><row r="1"><c r="A1" s="1"><v>{serial}</v></c></row></sheetData></worksheet>',
+ },
+ )
+ assert _text(path) == f"Sheet: T\n{expected}"
+
+
+def test_xlsx_finds_the_workbook_through_package_relationships(tmp_path):
+ path = _zip(
+ tmp_path / "moved.xlsx",
+ {
+ "_rels/.rels": f'<Relationships {REL}><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="book/main.xml"/></Relationships>',
+ "book/main.xml": f'<workbook {S}><sheets><sheet name="M" sheetId="1" r:id="rId1"/></sheets></workbook>',
+ "book/_rels/main.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="sheet.xml"/>'
+ '<Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings" Target="strings.xml"/></Relationships>',
+ "book/strings.xml": f"<sst {S}><si><t>zebramarker</t></si></sst>",
+ "book/sheet.xml": f'<worksheet {S}><sheetData><row r="1"><c r="A1" t="s"><v>0</v></c></row></sheetData></worksheet>',
+ },
+ )
+ assert _text(path) == "Sheet: M\nzebramarker"
+
+
+def test_xml_parts_are_bounded_by_element_count(tmp_path):
+ row = "<c/>" * 200_001 # one streamed row
+ sheet_bomb = _zip(
+ tmp_path / "row.xlsx",
+ {
+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="B" sheetId="1" r:id="rId1"/></sheets></workbook>',
+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',
+ "xl/worksheets/sheet1.xml": f"<worksheet {S}><sheetData><row>{row}</row></sheetData></worksheet>",
+ },
+ )
+ tree_bomb = _zip(
+ tmp_path / "tree.xlsx",
+ {"xl/workbook.xml": f"<workbook {S}>" + "<x/>" * 2_000_001 + "</workbook>"},
+ )
+ deep = _zip(
+ tmp_path / "deep.ods",
+ {
+ "mimetype": "application/vnd.oasis.opendocument.spreadsheet",
+ "content.xml": "<a>" * 600 + "</a>" * 600,
+ },
+ )
+ for path in (sheet_bomb, tree_bomb, deep):
+ with pytest.raises(ValueError, match = "too large to index|nested too deeply"):
+ parsers.parse(str(path))
+
+
[email protected]("format_id", [14, 31, 57, 75])
+def test_xlsx_builtin_date_formats_include_locale_ids(tmp_path, format_id):
+ serial = (dt.date(2026, 3, 31) - dt.date(1899, 12, 30)).days
+ path = _zip(
+ tmp_path / "dates.xlsx",
+ {
+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="D" sheetId="1" r:id="rId1"/></sheets></workbook>',
+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',
+ "xl/styles.xml": f'<styleSheet {S}><cellXfs><xf numFmtId="0"/><xf numFmtId="{format_id}"/></cellXfs></styleSheet>',
+ "xl/worksheets/sheet1.xml": f'<worksheet {S}><sheetData><row r="1"><c r="A1" s="1"><v>{serial}</v></c></row></sheetData></worksheet>',
+ },
+ )
+ assert _text(path) == "Sheet: D\n2026-03-31"
+
+
[email protected]("cat, val", [("cat", "val"), ("xVal", "yVal")])
+def test_pptx_reads_chart_titles_and_cached_data(tmp_path, cat, val):
+ C = 'xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"'
+ pt = lambda i, v: f'<c:pt idx="{i}"><c:v>{v}</c:v></c:pt>'
+ series = lambda name, values: (
+ f"<c:ser><c:tx><c:strRef><c:strCache>{pt(0, name)}</c:strCache></c:strRef></c:tx>"
+ f"<c:{cat}><c:strRef><c:strCache>{pt(0, 'North')}{pt(1, 'South')}</c:strCache></c:strRef></c:{cat}>"
+ f"<c:{val}><c:numRef><c:numCache>{pt(0, values[0])}{pt(1, values[1])}</c:numCache></c:numRef></c:{val}></c:ser>"
+ )
+ path = _zip(
+ tmp_path / "chart.pptx",
+ {
+ "ppt/presentation.xml": f'<p:presentation {P}><p:sldIdLst><p:sldId id="256" r:id="rId2"/></p:sldIdLst></p:presentation>',
+ "ppt/_rels/presentation.xml.rels": f'<Relationships {REL}><Relationship Id="rId2" Target="slides/slide1.xml"/></Relationships>',
+ "ppt/slides/slide1.xml": f"<p:sld {P}><p:cSld><p:spTree><p:graphicFrame><a:graphic><a:graphicData>"
+ '<c:chart xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" r:id="rId3"/>'
+ "</a:graphicData></a:graphic></p:graphicFrame></p:spTree></p:cSld></p:sld>",
+ "ppt/slides/_rels/slide1.xml.rels": f'<Relationships {REL}><Relationship Id="rId3" Target="../charts/chart1.xml"/></Relationships>',
+ "ppt/charts/chart1.xml": f"<c:chartSpace {C}><c:chart><c:title><c:tx><c:rich><a:p><a:r><a:t>Sales zebramarker</a:t></a:r></a:p></c:rich></c:tx></c:title>"
+ f"<c:plotArea><c:barChart>{series('Q1', (1200, 900))}{series('Q2', (1300, 950))}</c:barChart></c:plotArea></c:chart></c:chartSpace>",
+ },
+ )
+ assert _text(path) == "Sales zebramarker\nQ1 | Q2\nNorth | 1200 | 1300\nSouth | 900 | 950"
+
+
+def test_scatter_series_keep_their_own_x_values():
+ C = 'xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart"'
+ pt = lambda i, v: f'<c:pt idx="{i}"><c:v>{v}</c:v></c:pt>'
+ series = lambda name, xs, ys: (
+ f"<c:ser><c:tx><c:strRef><c:strCache>{pt(0, name)}</c:strCache></c:strRef></c:tx>"
+ f"<c:xVal><c:numRef><c:numCache>{pt(0, xs[0])}{pt(1, xs[1])}</c:numCache></c:numRef></c:xVal>"
+ f"<c:yVal><c:numRef><c:numCache>{pt(0, ys[0])}{pt(1, ys[1])}</c:numCache></c:numRef></c:yVal></c:ser>"
+ )
+ root = ET.fromstring(
+ f"<c:chartSpace {C}><c:chart><c:plotArea><c:scatterChart>"
+ f"{series('A', (1, 2), (10, 20))}{series('B', (5, 6), (50, 60))}"
+ "</c:scatterChart></c:plotArea></c:chart></c:chartSpace>"
+ )
+ assert office_formats._chart_lines(root) == ["A", "1 | 10", "2 | 20", "B", "5 | 50", "6 | 60"]
+
+
+def test_chart_keeps_every_level_of_multi_level_categories():
+ C = 'xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart"'
+ pts = lambda items: "".join(f'<c:pt idx="{i}"><c:v>{v}</c:v></c:pt>' for i, v in items)
+ root = ET.fromstring(
+ f"<c:chartSpace {C}><c:chart><c:plotArea><c:barChart><c:ser>"
+ "<c:cat><c:multiLvlStrRef><c:multiLvlStrCache>"
+ f"<c:lvl>{pts(enumerate(['Q1', 'Q2', 'Q1', 'Q2']))}</c:lvl>"
+ f"<c:lvl>{pts([(0, '2023'), (2, '2024')])}</c:lvl>"
+ "</c:multiLvlStrCache></c:multiLvlStrRef></c:cat>"
+ f"<c:val><c:numRef><c:numCache>{pts(enumerate([10, 20, 30, 40]))}</c:numCache></c:numRef></c:val>"
+ "</c:ser></c:barChart></c:plotArea></c:chart></c:chartSpace>"
+ )
+ assert office_formats._chart_lines(root) == [
+ "2023 Q1 | 10",
+ "Q2 | 20",
+ "2024 Q1 | 30",
+ "Q2 | 40",
+ ]
+
+
+def test_odp_reads_one_page_per_slide(tmp_path):
+ pages = parsers.parse(str(build_odp(tmp_path / "slides.odp")))
+ assert [(p.page_number, p.text) for p in pages] == [
+ (1, "First slide\nNotes:\nzebramarker notes"),
+ (2, "Second slide revenue"),
+ ]
+
+
+def test_epub_follows_the_spine_and_skips_navigation(tmp_path):
+ text = _text(build_epub(tmp_path / "book.epub"))
+ assert "navmarker" not in text
+ assert text.index("Revenue increased") < text.index("Zebramarker ends the book")
+
+
+def test_epub_package_path_is_a_url(tmp_path):
+ with zipfile.ZipFile(build_epub(tmp_path / "plain.epub")) as z:
+ members = {name: z.read(name) for name in z.namelist()}
+ members["OEBPS/My Book.opf"] = members.pop("OEBPS/content.opf")
+ members["META-INF/container.xml"] = members["META-INF/container.xml"].replace(
+ b"OEBPS/content.opf", b"OEBPS/My%20Book.opf"
+ )
+ book = tmp_path / "book.epub"
+ with zipfile.ZipFile(book, "w") as z:
+ for name, data in members.items():
+ z.writestr(name, data)
+ assert "Zebramarker ends the book" in _text(book)
+
+
+ "uri, protected", [("OEBPS/text/Chapter%201.xhtml", True), ("OEBPS/fonts/a.otf", False)]
+)
+def test_epub_with_encrypted_chapters_is_refused(tmp_path, uri, protected):
+ book = build_epub(tmp_path / "book.epub")
+ with zipfile.ZipFile(book, "a") as z:
+ z.writestr(
+ "META-INF/encryption.xml",
+ '<encryption xmlns="urn:oasis:names:tc:opendocument:xmlns:container" '
+ 'xmlns:enc="http://www.w3.org/2001/04/xmlenc#"><enc:EncryptedData>'
+ f'<enc:CipherData><enc:CipherReference URI="{uri}"/></enc:CipherData>'
+ "</enc:EncryptedData></encryption>",
+ )
+ if protected:
+ with pytest.raises(ValueError, match = "DRM protected"):
+ parsers.parse(str(book))
+ else: # obfuscated fonts are listed too, and never block reading
+ assert "Zebramarker" in _text(book)
+
+
+def test_eml_decodes_headers_and_body(tmp_path):
+ text = _text(build_eml(tmp_path / "mail.eml"))
+ assert "Subject: Q1 numbers résumé" in text
+ assert "From: Ann <[email protected]>" in text
+ assert "Attachments: data.csv" in text
+ assert "Zebramarker in the body." in text
+ assert "html copy" not in text
+
+
[email protected]("extension", [".docx", ".docm", ".dotx", ".dotm"])
+def test_word_macro_and_template_files_read_like_docx(tmp_path, extension):
+ text = _text(build_word(tmp_path / f"report{extension}"))
+ assert "Quarterly report" in text and "Zebramarker" in text
+
+
+def test_word_extension_on_another_package_is_refused(tmp_path):
+ workbook = build_word(tmp_path / "report.docx").read_bytes()
+ with zipfile.ZipFile(io.BytesIO(workbook)) as source:
+ types = source.read("[Content_Types].xml").replace(
+ _WORD_TYPES[".docx"].encode(),
+ b"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml",
+ )
+ with zipfile.ZipFile(tmp_path / "sheet.docm", "w") as target:
+ for info in source.infolist():
+ target.writestr(
+ info, types if info.filename == "[Content_Types].xml" else source.read(info)
+ )
+ with pytest.raises(ValueError, match = "not a Word file"):
+ parsers.parse(str(tmp_path / "sheet.docm"))
+
+
+def test_mhtml_reads_the_page_and_skips_resources(tmp_path):
+ text = _text(build_mhtml(tmp_path / "page.mhtml"))
+ assert "Zebramarker on the page." in text
+ assert "PNG" not in text
+
+
+def test_xhtml_reads_visible_text(tmp_path):
+ text = _text(build_xhtml(tmp_path / "page.xhtml"))
+ assert "Zebramarker résumé" in text and "p{}" not in text
+
+
+def test_rtf_keeps_text_and_drops_control_groups(tmp_path):
+ assert _text(build_rtf(tmp_path / "memo.rtf")) == (
+ "Quarterly report\nCafé € revenue increased\nlink text\nZebramarker | 1200"
+ )
+
+
+def test_doc_reads_compressed_and_unicode_pieces_and_note_stories(tmp_path):
+ assert _text(build_doc(tmp_path / "memo.doc")) == (
+ "Quarterly report\nCafé 東京 link Zebramarker | 1200\n\nBoxed text"
+ "\n\nFootnotes:\nFootnote detail\n\nEndnotes:\nEndnote source"
+ )
+
+
+def test_xls_reads_errors_and_continued_formula_strings(tmp_path):
+ long_text = "A" * 8000 + "Ж" * 300
+ string = struct.pack("<HB", len(long_text), 0) + b"A" * 8000
+ sheet = (
+ _record(0x0809, b"\0" * 16)
+ + _record(0x0205, struct.pack("<HHHBB", 0, 0, 0, 0x07, 1)) # BOOLERR #DIV/0!
+ + _record(0x0006, struct.pack("<HHH", 0, 1, 0) + b"\x02\0\x2a\0\0\0\xff\xff" + b"\0" * 6)
+ + _record(0x0006, struct.pack("<HHH", 1, 0, 0) + b"\0" * 6 + b"\xff\xff" + b"\0" * 6)
+ + _record(0x0207, string)
+ + _record(0x003C, b"\x01" + "Ж".encode("utf-16-le") * 300)
+ + _record(0x000A, b"")
+ )
+ globals_ = (
+ _record(0x0809, b"\0" * 16)
+ + _record(0x0085, b"\0" * 4 + b"\0\0" + struct.pack("<B", 1) + b"\0S")
+ + _record(0x000A, b"")
+ )
+ stream = bytearray(globals_ + sheet)
+ struct.pack_into("<I", stream, 4 + 16 + 4, len(globals_))
+ path = tmp_path / "errors.xls"
+ path.write_bytes(compound_file({("Workbook",): bytes(stream)}))
+ assert _text(path) == f"Sheet: S\n#DIV/0! | #N/A\n{long_text}"
+
+
+def test_xls_reads_continued_strings_numbers_formulas_and_dates(tmp_path):
+ assert _text(build_xls(tmp_path / "book.xls")) == (
+ "Sheet: Revenue\nRegion | 1200.5\nZebramarker€ | -7\n3 | 4\nformula text | TRUE"
+ "\n2026-03-31 | 2026-04-01"
+ )
+
+
+def test_xls_refuses_a_workbook_past_the_cell_ceiling(tmp_path, monkeypatch):
+ monkeypatch.setattr(office_formats, "_MAX_XML_ELEMENTS", 3)
+ with pytest.raises(ValueError, match = "too many cells"):
+ parsers.parse(str(build_xls(tmp_path / "book.xls")))
+
+
+# A .msg looks up properties per attachment; rebuilding the root map each time was quadratic.
+def test_compound_file_walks_each_storage_once(tmp_path):
+ reader = cfb.CompoundFile(build_msg(tmp_path / "mail.msg").read_bytes())
+ first = reader._children(0)
+ assert reader._children(0) is first
+ assert reader.exists(*next(iter(first)).split("/"))
+
+
+def test_xls_reads_excel_95_byte_strings(tmp_path):
+ def text(value, size):
+ raw = value.encode("cp1251")
+ return struct.pack("<B" if size == 1 else "<H", len(raw)) + raw
+
+ bof = _record(0x0809, struct.pack("<HH", 0x0500, 0x0005) + b"\0" * 4)
+ globals_ = lambda offset: (
+ bof
+ + _record(0x0042, struct.pack("<H", 1251)) # CODEPAGE
+ + _record(0x0085, struct.pack("<IBB", offset, 0, 0) + text("Выручка", 1))
+ + _record(0x000A, b"")
+ )
+ sheet = (
+ bof
+ + _record(0x0204, struct.pack("<HHH", 0, 0, 0) + text("Zebramarker", 2))
+ + _record(0x0204, struct.pack("<HHH", 0, 1, 0) + text("Привет", 2))
+ + _record(0x0006, struct.pack("<HHH", 1, 0, 0) + b"\0" * 6 + b"\xff\xff" + b"\0" * 6)
+ + _record(0x0207, text("formula text", 2))
+ + _record(0x000A, b"")
+ )
+ path = tmp_path / "excel95.xls"
+ path.write_bytes(compound_file({("Book",): globals_(len(globals_(0))) + sheet}))
+ assert _text(path) == "Sheet: Выручка\nZebramarker | Привет\nformula text"
+
+
+def test_ppt_reads_slide_text_and_skips_masters(tmp_path):
+ pages = parsers.parse(str(build_ppt(tmp_path / "deck.ppt")))
+ assert [(p.page_number, p.text) for p in pages] == [
+ (1, "Quarterly review\nRevenue up"),
+ (2, "Zebramarker slide"),
+ ]
+
+
[email protected]("broken_link", [False, True])
+def test_ppt_reads_the_latest_save_with_text_boxes_and_notes(tmp_path, broken_link):
+ pages = parsers.parse(str(build_saved_ppt(tmp_path / "saved.ppt", broken_link = broken_link)))
+ assert [(p.page_number, p.text) for p in pages] == [
+ (1, "Title from list\nText box zebramarker\nNotes:\nSpeaker notes")
+ ]
+
+
+def test_ppt_falls_back_to_slide_drawings(tmp_path):
+ pages = parsers.parse(str(build_ppt(tmp_path / "deck.ppt", from_slides = True)))
+ assert [p.text for p in pages] == ["Drawn slide", "Zebramarker drawn"]
+
+
+def test_msg_reads_headers_body_and_attachment_names(tmp_path):
+ assert _text(build_msg(tmp_path / "mail.msg")) == (
+ "From: Ann <[email protected]>\nTo: Bob\nDate: 2026-10-06 10:00 UTC\nSubject: Q1 numbers\n"
+ "Attachments: data.csv\n\nRevenue increased.\nZebramarker in the body."
+ )
+
+
+def test_msg_uses_the_smtp_address_for_exchange_senders(tmp_path):
+ utf16 = lambda text: text.encode("utf-16-le")
+ path = tmp_path / "exchange.msg"
+ path.write_bytes(
+ compound_file(
+ {
+ ("__substg1.0_0C1A001F",): utf16("Ann"),
+ ("__substg1.0_0C1F001F",): utf16("/O=EXAMPLE/OU=EXCHANGE/CN=RECIPIENTS/CN=ANN"),
+ ("__substg1.0_5D01001F",): utf16("[email protected]"),
+ ("__substg1.0_1000001F",): utf16("Body zebramarker"),
+ }
+ )
+ )
+ assert _text(path) == "From: Ann <[email protected]>\n\nBody zebramarker"
+
+
[email protected]("plain", [None, "\r\n \r\n"])
+def test_msg_reads_a_string_html_body(tmp_path, plain):
+ path = tmp_path / "html.msg"
+ html = '<html><head><meta charset="windows-1252"></head><body><p>Café zebramarker</p></body></html>'
+ streams = {("__substg1.0_1013001F",): html.encode("utf-16-le")}
+ if plain is not None: # a blank plain body must not hide the HTML one
+ streams[("__substg1.0_1000001F",)] = plain.encode("utf-16-le")
+ path.write_bytes(compound_file(streams))
+ assert _text(path).strip() == "Café zebramarker"
+
+
+def test_pptx_reads_smartart_labels(tmp_path):
+ members = {}
+ with zipfile.ZipFile(build_pptx(tmp_path / "plain.pptx")) as z:
+ members = {name: z.read(name) for name in z.namelist()}
+ dgm = 'xmlns:dgm="http://schemas.openxmlformats.org/drawingml/2006/diagram"'
+ members["ppt/slides/slide1.xml"] = members["ppt/slides/slide1.xml"].replace(
+ b"</p:spTree>",
+ f'<p:graphicFrame><a:graphic><a:graphicData><dgm:relIds {dgm} r:dm="rId2"/>'
+ "</a:graphicData></a:graphic></p:graphicFrame></p:spTree>".encode(),
+ )
+ members["ppt/slides/_rels/slide1.xml.rels"] = members[
+ "ppt/slides/_rels/slide1.xml.rels"
+ ].replace(
+ b"</Relationships>",
+ b'<Relationship Id="rId2" Target="../diagrams/data1.xml"/></Relationships>',
+ )
+ members["ppt/diagrams/data1.xml"] = (
+ f"<dgm:dataModel {dgm} {P}><dgm:ptLst>"
+ "<dgm:pt><dgm:t><a:p><a:r><a:t>Plan</a:t></a:r></a:p></dgm:t></dgm:pt>"
+ "<dgm:pt><dgm:t><a:p><a:r><a:t>Ship zebramarker</a:t></a:r></a:p></dgm:t></dgm:pt>"
+ "</dgm:ptLst></dgm:dataModel>"
+ )
+ text = _text(_zip(tmp_path / "smartart.pptx", members))
+ assert "Revenue increased\nPlan\nShip zebramarker\nNotes:" in text
+
+
+def test_xlsx_refuses_one_shared_string_filling_every_cell(tmp_path):
+ cells = "".join(f'<c r="{chr(65 + i)}1" t="s"><v>0</v></c>' for i in range(26))
+ rows = "".join(
+ f'<row r="{n}">{cells.replace("1" + chr(34), str(n) + chr(34))}</row>' for n in range(1, 60)
+ )
+ path = _zip(
+ tmp_path / "bomb.xlsx",
+ {
+ "xl/workbook.xml": f'<workbook {S}><sheets><sheet name="S" sheetId="1" r:id="rId1"/></sheets></workbook>',
+ "xl/_rels/workbook.xml.rels": f'<Relationships {REL}><Relationship Id="rId1" Target="worksheets/sheet1.xml"/></Relationships>',
+ "xl/sharedStrings.xml": f"<sst {S}><si><t>{'x' * 25_000}</t></si></sst>",
+ "xl/worksheets/sheet1.xml": f"<worksheet {S}><sheetData>{rows}</sheetData></worksheet>",
+ },
+ )
+ with pytest.raises(ValueError, match = "repeats too much content"):
+ parsers.parse(str(path))
+
+
+def test_eml_reads_every_inline_part(tmp_path):
+ message = EmailMessage()
+ message["Subject"] = "Two parts"
+ message.set_content("first part")
+ message.make_mixed()
+ inline = EmailMessage()
+ inline.set_content("second zebramarker")
+ inline["Content-Disposition"] = "inline"
+ message.attach(inline)
+ message.add_attachment(b"x,y\n", maintype = "text", subtype = "csv", filename = "data.csv")
+ path = tmp_path / "mixed.eml"
+ path.write_bytes(bytes(message))
+ text = _text(path)
+ assert text.endswith("first part\n\nsecond zebramarker")
+ assert "x,y" not in text
+
+
+def test_eml_falls_back_to_html_when_plain_is_empty(tmp_path):
+ message = EmailMessage()
+ message["Subject"] = "HTML only"
+ message.set_content(" \n")
+ message.add_alternative("<p>zebramarker in html</p>", subtype = "html")
+ path = tmp_path / "html.eml"
+ path.write_bytes(bytes(message))
+ assert _text(path).endswith("zebramarker in html")
+
+
+def test_msg_decodes_ansi_properties_and_compressed_rtf_body(tmp_path):
+ assert _text(build_ansi_msg(tmp_path / "ansi.msg")) == (
+ "From: Иван\nSubject: Привет\n\nZebramarker in the RTF body\nrepeat repeat repeat"
+ )
+
+
+def _strict(source, target):
+ """Rewrite a Transitional OOXML package with the Strict namespaces."""
+ pairs = [
+ (
+ b"http://schemas.openxmlformats.org/spreadsheetml/2006/main",
+ b"http://purl.oclc.org/ooxml/spreadsheetml/main",
+ ),
+ (
+ b"http://schemas.openxmlformats.org/presentationml/2006/main",
+ b"http://purl.oclc.org/ooxml/presentationml/main",
+ ),
+ (
+ b"http://schemas.openxmlformats.org/drawingml/2006/main",
+ b"http://purl.oclc.org/ooxml/drawingml/main",
+ ),
+ (
+ b"http://schemas.openxmlformats.org/officeDocument/2006/relationships",
+ b"http://purl.oclc.org/ooxml/officeDocument/relationships",
+ ),
+ ]
+ with zipfile.ZipFile(source) as z:
+ members = {name: z.read(name) for name in z.namelist()}
+ for name, data in members.items():
+ for old, new in pairs:
+ data = data.replace(old, new)
+ members[name] = data
+ return _zip(target, members)
+
+
[email protected]("builder, extension", [(build_xlsx, ".xlsx"), (build_pptx, ".pptx")])
+def test_strict_ooxml_reads_like_transitional(tmp_path, builder, extension):
+ transitional = builder(tmp_path / f"transitional{extension}")
+ strict = _strict(transitional, tmp_path / f"strict{extension}")
+ with zipfile.ZipFile(strict) as z:
+ assert all(b"2006/main" not in z.read(name) for name in z.namelist())
+ assert _text(strict) == _text(transitional) != ""
+
+
+def test_odt_skips_tracked_deletions_and_reads_nested_tables_once(tmp_path):
+ path = _odf(
+ tmp_path / "changes.odt",
+ "<office:text><text:tracked-changes><text:changed-region>"
+ "<text:deletion><text:p>deletedmarker</text:p></text:deletion>"
+ "</text:changed-region></text:tracked-changes>"
+ '<text:p>See <text:bookmark-ref text:ref-name="b">Chapter 2</text:bookmark-ref>.</text:p>'
+ "<table:table><table:table-header-rows><table:table-row>"
+ "<table:table-cell><text:p>Head</text:p></table:table-cell>"
+ "</table:table-row></table:table-header-rows><table:table-row><table:table-cell>"
+ "<table:table><table:table-row><table:table-cell><text:p>inner</text:p></table:table-cell>"
+ "<table:table-cell><text:p>cell</text:p></table:table-cell></table:table-row></table:table>"
+ "</table:table-cell><table:table-cell><text:p>outer</text:p></table:table-cell>"
+ "</table:table-row></table:table></office:text>",
+ )
+ assert _text(path) == "See Chapter 2.\nHead\ninner | cell | outer"
+
+
+def test_rtf_decodes_literal_bytes_in_the_document_code_page(tmp_path):
+ path = tmp_path / "ru.rtf"
+ path.write_bytes(b"{\\rtf1\\ansi\\ansicpg1251 " + "Привет".encode("cp1251") + b" \\'80}")
+ assert _text(path) == "Привет Ђ"
+
+
+ "header, encoded",
+ [
+ (b"\\mac", "Café".encode("mac_roman")),
+ (b"\\pc", "Café".encode("cp437")),
+ (b"\\pca", "Café".encode("cp850")),
+ # Word for Mac writes Mac Roman as \ansicpg10000, which Python names mac_roman.
+ (b"\\mac\\ansicpg10000", "Café".encode("mac_roman")),
+ ],
+)
+def test_rtf_decodes_the_mac_and_pc_document_character_sets(tmp_path, header, encoded):
+ path = tmp_path / "legacy.rtf"
+ path.write_bytes(
+ b"{\\rtf1" + header + b"\\deff0{\\fonttbl{\\f0\\fcharset256 Times;}}\\f0 " + encoded + b"}"
+ )
+ assert _text(path) == "Café"
+
+
+def test_compound_file_reads_mini_and_regular_streams():
+ small, large = b"a" * 100, bytes(range(256)) * 40
+ reader = cfb.CompoundFile(compound_file({("small",): small, ("dir", "large"): large}))
+ assert reader.read("small") == small
+ assert reader.read("DIR", "Large") == large
+ assert sorted(reader.listdir()) == ["dir", "small"]
+
+
+def test_compound_file_reads_each_fat_sector_once_per_file_sector():
+ # Every DIFAT slot names the same FAT sector: the FAT must stay the size of the file.
+ data = bytearray(compound_file({("small",): b"a" * 100}))
+ n_difat = 64
+ first = len(data) // 512 - 1
+ data += b"".join(
+ struct.pack("<128I", *([0] * 127 + [first + i + 1 if i + 1 < n_difat else 0xFFFFFFFE]))
+ for i in range(n_difat)
+ )
+ struct.pack_into("<I", data, 0x2C, 0) # no FAT sector count to trim by
+ struct.pack_into("<II", data, 0x44, first, n_difat)
+ reader = cfb.CompoundFile(bytes(data))
+ assert len(reader._fat) <= len(data) // 4
+ assert reader.read("small") == b"a" * 100
+
+
+def test_encrypted_doc_is_refused(tmp_path):
+ with pytest.raises(ValueError, match = "password"):
+ parsers.parse(str(build_doc(tmp_path / "secret.doc", encrypted = True)))
+
+
[email protected]("extension", [".doc", ".xls", ".ppt", ".msg"])
+def test_non_compound_file_is_refused(tmp_path, extension):
+ path = tmp_path / f"fake{extension}"
+ path.write_bytes(b"not a compound file" * 50)
+ with pytest.raises(ValueError):
+ parsers.parse(str(path))
+
+
[email protected]("extension", [".xlsx", ".pptx", ".odt", ".epub"])
+def test_damaged_archive_is_refused(tmp_path, extension):
+ path = tmp_path / f"broken{extension}"
+ path.write_bytes(b"PK\x03\x04 truncated")
+ with pytest.raises(ValueError):
+ parsers.parse(str(path))
+
+
+def test_rtf_pairs_surrogates_and_skips_tracked_deletions(tmp_path):
+ path = tmp_path / "changes.rtf"
+ # As macOS textutil writes an emoji; the deletion as Word writes a tracked change.
+ path.write_bytes(
+ rb"{\rtf1\ansi Party \uc0\u55356 \u57225 kept{\deleted\revauthdel1 removedmarker \u8364?}"
+ rb" end\deleted gone\deleted0 back\plain}"
+ )
+ assert _text(path) == "Party \U0001f389 kept end back"
+
+
+def test_compressed_rtf_stops_at_the_declared_size():
+ # One literal, then back-references that would expand to ~5 MB.
+ body = b"\x00A" + (b"\xff" + b"\x00\x0f" * 8) * 40_000
+ data = struct.pack("<IIII", len(body) + 12, 1000, 0x75465A4C, 0) + body
+ assert len(office_formats._decompress_rtf(data)) == 1000
+ with pytest.raises(ValueError, match = "too large"):
+ office_formats._decompress_rtf(struct.pack("<IIII", 16, 2**31, 0x75465A4C, 0))
+
+
+def test_rtf_decodes_bytes_with_the_font_charset(tmp_path):
+ path = tmp_path / "fonts.rtf"
+ path.write_bytes(
+ rb"{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0\fswiss\fcharset0 Arial;}"
+ rb"{\f1\froman\fcharset204 Times New Roman Cyr;}{\f2\fnil\cpg1253 Greek;}}"
+ rb"\f0 Hello {\f1 \'cf\'f0\'e8\'e2\'e5\'f2} back {\f2 \'e1} \plain\'e9\par}"
+ )
+ assert _text(path) == "Hello Привет back α é"
+
+
+def test_rtf_prefers_the_unicode_branch_of_upr(tmp_path):
+ path = tmp_path / "upr.rtf"
+ path.write_bytes(rb"{\rtf1\ansi \upr{?}{\*\ud{\u1040?}} end}")
+ assert _text(path) == "\u0410 end"
+
+
+def test_rtf_refuses_runaway_nesting(tmp_path):
+ path = tmp_path / "deep.rtf"
+ path.write_bytes(b"{\\rtf1 " + b"{" * 5000)
+ with pytest.raises(ValueError, match = "nested too deeply"):
+ parsers.parse(str(path))
+
+
+def test_rtf_negative_binary_length_is_ignored(tmp_path):
+ path = tmp_path / "bin.rtf"
+ path.write_bytes(rb"{\rtf1\ansi before \bin-999999999 after}")
+ assert _text(path) == "before after"
+
+
+def test_ods_refuses_runaway_repeats(tmp_path):
+ cell = "x" * 100_000
+ path = _odf(
+ tmp_path / "repeats.ods",
+ '<office:spreadsheet><table:table table:name="S">'
+ '<table:table-row table:number-rows-repeated="1000">'
+ f'<table:table-cell table:number-columns-repeated="1000"><text:p>{cell}</text:p></table:table-cell>'
+ "</table:table-row></table:table></office:spreadsheet>",
+ )
+ with pytest.raises(ValueError, match = "repeats too much"):
+ parsers.parse(str(path))
+
+
+def test_xml_entities_are_refused(tmp_path):
+ path = _zip(
+ tmp_path / "bomb.odt", {"content.xml": '<!DOCTYPE x [<!ENTITY a "aaaa">]><x>&a;</x>'}
+ )
+ with pytest.raises(ValueError):
+ parsers.parse(str(path))
+
+
[email protected]("extension", sorted(BUILDERS))
+def test_documents_upload_index_and_stay_searchable(rag_home, stub_embeddings, tmp_path, extension):
+ if not rag_db.rag_available(): # vec0 will not load on some macOS Pythons
+ pytest.skip("sqlite-vec unavailable here")
+ source = BUILDERS[extension](tmp_path / f"report{extension}")
+ stored, filename, _hash = _save_upload(
+ UploadFile(file = io.BytesIO(source.read_bytes()), filename = source.name)
+ )
+ assert filename == source.name
+ scope = store.thread_scope("office")
+ doc_id, job = ingestion.start_ingestion(
+ scope, None, "office", filename, stored, background = False
+ )
+ assert ingestion.get_job_status(job)["status"] == "completed"
+ conn = rag_db.get_connection()
+ try:
+ assert store.get_document(conn, doc_id)["status"] == "completed"
+ assert store.search_lexical(conn, scope, "zebramarker", 5)
+ finally:
+ conn.close()
diff --git a/studio/frontend/src/components/assistant-ui/attachment-document-dialog.tsx b/studio/frontend/src/components/assistant-ui/attachment-document-dialog.tsx
index e3b2ae839fd..d84874eb6e2 100644
--- a/studio/frontend/src/components/assistant-ui/attachment-document-dialog.tsx
+++ b/studio/frontend/src/components/assistant-ui/attachment-document-dialog.tsx
@@ -220,7 +220,13 @@ const DocumentDialog: FC<
: truncateAttachmentPreviewText(await blob.text());
next = { blob, text, truncated };
} else if (textFallback && blob.type.startsWith("text/")) {
- const { text, truncated } = truncateAttachmentPreviewText(await blob.text());
+ // A File decodes as the composer does (BOM, UTF-16); a code page it refuses still shows raw.
+ const { text, truncated } =
+ blob instanceof File
+ ? await readAttachmentText(blob, source.name, blob.type).catch(async () =>
+ truncateAttachmentPreviewText(await blob.text()),
+ )
+ : truncateAttachmentPreviewText(await blob.text());
next = { blob, plain: text, truncated };
}
if (!cancelled) setLoaded(next);
@@ -281,6 +287,33 @@ const SentOriginalDialog: FC<
);
};
+/** A file outside any message, such as a chat with files source, read on open: pages, markdown or
+ * text (a text/* blob). */
+export const LocalFileDialog: FC<PropsWithChildren<{ name: string; load: () => Promise<Blob> }>> = ({
+ children,
+ name,
+ load,
+}) => (
+ <DocumentDialog
+ source={{
+ kind: "document",
+ name,
+ contentType: undefined,
+ file: undefined,
+ src: undefined,
+ audio: undefined,
+ video: undefined,
+ text: undefined,
+ hasOriginal: true,
+ }}
+ load={load}
+ redactFromReload={false}
+ textFallback={true}
+ >
+ {children}
+ </DocumentDialog>
+);
+
export const AttachmentDocumentDialog: FC<
PropsWithChildren<{ source: AttachmentSource; redactFromReload: boolean }>
> = ({ children, source, redactFromReload }) => {
diff --git a/studio/frontend/src/components/assistant-ui/attachment.tsx b/studio/frontend/src/components/assistant-ui/attachment.tsx
index 0acb7310d2c..15190175443 100644
--- a/studio/frontend/src/components/assistant-ui/attachment.tsx
+++ b/studio/frontend/src/components/assistant-ui/attachment.tsx
@@ -74,16 +74,16 @@ import {
} from "react";
import { ScrollPane } from "./scroll-pane";
-const CARD_SLOT =
+export const CARD_SLOT =
"shrink-0 w-[calc((100%_-_var(--spacing)*8)/5)] min-w-[calc(7rem*var(--ui-space-scale,1))]";
-const CARD_SIZE = "h-[calc(7rem*var(--ui-space-scale,1))] w-full";
+export const CARD_SIZE = "h-[calc(7rem*var(--ui-space-scale,1))] w-full";
const SENT_IMAGE_SIZE = "size-[calc(9rem*var(--ui-space-scale,1))]";
const SENT_IMAGE_SIZE_COMPACT = "size-[calc(5rem*var(--ui-space-scale,1))]";
const SENT_ROW_WIDTH = "w-[calc(18rem*var(--ui-space-scale,1))]";
-const CARD_EDGE =
+export const CARD_EDGE =
"border border-[color-mix(in_oklab,var(--foreground)_calc(12%*var(--contrast-edge-gain,1)),transparent)]";
// No fill, so cards take the composer background; hover still tints.
-const CARD_SURFACE =
+export const CARD_SURFACE =
"hover:bg-[color-mix(in_oklab,var(--foreground)_6%,transparent)]";
const useAttachmentKind = (): {
@@ -795,3 +795,5 @@ export const ComposerAddAttachment: FC = () => {
</ComposerPrimitive.AddAttachment>
);
};
+
+export { AttachmentKindIcon, FileCardBody };
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx
index 7f9a12ca343..32fd5c5239b 100644
--- a/studio/frontend/src/components/assistant-ui/thread.tsx
+++ b/studio/frontend/src/components/assistant-ui/thread.tsx
@@ -329,6 +329,7 @@ import { usePublishedFrame } from "@/features/settings/hooks/use-published-frame
import { useVoiceSettingsStore } from "@/features/settings/stores/voice-settings-store";
import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params";
import { isTauri } from "@/lib/api-base";
+import { openFilePicker } from "@/lib/open-file-picker";
import { InternetGlyph } from "@/lib/internet-icon";
import { copyToClipboard } from "@/lib/copy-to-clipboard";
import { MenuDismissGuard } from "@/lib/menu-dismiss-guard";
@@ -2569,7 +2570,8 @@ const ThreadWelcome: FC<{
return (
<div className="aui-thread-welcome-root mx-auto my-auto flex w-full max-w-(--thread-max-width) grow flex-col">
- <div className="aui-thread-welcome-center flex w-full grow flex-col items-center justify-start pt-[27.5dvh]">
+ {/* With a files panel showing (chat with files, or project sources), center the whole block. */}
+ <div className="aui-thread-welcome-center flex w-full grow flex-col items-center justify-start pt-[27.5dvh] has-[.unsloth-files-panel]:justify-center has-[.unsloth-files-panel]:pt-0 has-[.unsloth-files-panel]:pb-[6dvh]">
{/* No padding, so the composer here is as wide as once it docks. */}
<div className="aui-thread-welcome-message flex w-full flex-col justify-center gap-9">
{/* Center the greeting (sloth + title) over the composer. */}
@@ -2611,6 +2613,8 @@ export const ProjectComposer: FC<{
placeholder={placeholder}
disableQueue
/>
+ {/* Its file cards open previews; the thread's mount is not on this page. */}
+ <DocumentPreviewMount />
</GeneratedImageOverlayProvider>
);
};
@@ -5434,14 +5438,6 @@ const Composer: FC<{
<PendingAudioChip />
</>
) : null}
- {/* Keep indexing state subscribed while dictating, but hide its chips so
- the waveform stays the composer's only status indicator. */}
- <div className={isDictating ? "hidden" : "contents"}>
- <ThreadDocumentsBar
- threadId={referenceThreadId}
- onIndexingChange={handleIndexingChange}
- />
- </div>
{!isDictating ? <ComposerDraftPreview text={composerText} /> : null}
{!isDictating ? <ToolStatusDisplay /> : null}
<div
@@ -5631,6 +5627,13 @@ const Composer: FC<{
onSubmit={handleSubmit}
>
<PromptQueueStack queueThreadIds={promptQueueThreadIds} />
+ {/* Chat with files card. Stays mounted while dictating to keep indexing state. */}
+ <div className={isDictating ? "hidden" : "contents"}>
+ <ThreadDocumentsBar
+ threadId={referenceThreadId}
+ onIndexingChange={handleIndexingChange}
+ />
+ </div>
{youtubeOfferUrl && !isDictating && !disabled ? (
// Keyed by URL: pasting a second link while the first is still fetching
// remounts the prompt, so its cleanup aborts the request that is no
@@ -6539,6 +6542,30 @@ function attachmentAcceptForPicker(accept: string, audioEnabled: boolean): strin
return pickerAcceptForTextBasenames(enabledAccept);
}
+function usePickComposerAttachment(): () => void {
+ const aui = useAui();
+ const audioAttachmentsEnabled = useChatRuntimeStore((s) => {
+ const activeCheckpoint = s.params.checkpoint;
+ // No model yet: offer audio too, since files attached now wait for the model loaded next.
+ if (!activeCheckpoint || s.modelLoading) {
+ return true;
+ }
+ const activeModel = s.models.find((m) => m.id === activeCheckpoint);
+ return Boolean(activeModel?.hasAudioInput);
+ });
+ return useCallback(() => {
+ const attachmentAccept = attachmentAcceptForPicker(
+ aui.composer().getState().attachmentAccept,
+ audioAttachmentsEnabled,
+ );
+ openFilePicker(attachmentAccept, (files) => {
+ for (const file of files) {
+ void aui.composer().addAttachment(file);
+ }
+ });
+ }, [aui, audioAttachmentsEnabled]);
+}
+
const ComposerToolsMenu: FC<{
side?: "top" | "bottom";
researchAvailable: boolean;
@@ -6568,15 +6595,6 @@ const ComposerToolsMenu: FC<{
const modelLoaded = useChatRuntimeStore(
(s) => !!s.params.checkpoint && !s.modelLoading,
);
- const audioAttachmentsEnabled = useChatRuntimeStore((s) => {
- const activeCheckpoint = s.params.checkpoint;
- // No model yet: offer audio too, since files attached now wait for the model loaded next.
- if (!activeCheckpoint || s.modelLoading) {
- return true;
- }
- const activeModel = s.models.find((m) => m.id === activeCheckpoint);
- return Boolean(activeModel?.hasAudioInput);
- });
const checkpoint = useChatRuntimeStore((s) => s.params.checkpoint);
const supportsTools = useChatRuntimeStore((s) => s.supportsTools);
const supportsBuiltinWebSearch = useChatRuntimeStore(
@@ -6652,37 +6670,7 @@ const ComposerToolsMenu: FC<{
const composerCanAddAttachments = useAuiState(
({ composer }) => composer.isEditing,
);
- const pickAttachment = useCallback(() => {
- const input = document.createElement("input");
- input.type = "file";
- input.multiple = true;
- input.hidden = true;
-
- const attachmentAccept = attachmentAcceptForPicker(
- aui.composer().getState().attachmentAccept,
- audioAttachmentsEnabled,
- );
- if (attachmentAccept !== "*") {
- input.accept = attachmentAccept;
- }
-
- document.body.appendChild(input);
- input.onchange = (event) => {
- const files = (event.target as HTMLInputElement).files;
- if (files) {
- for (const file of files) {
- void aui.composer().addAttachment(file);
- }
- }
- document.body.removeChild(input);
- };
- input.oncancel = () => {
- if (!input.files || input.files.length === 0) {
- document.body.removeChild(input);
- }
- };
- input.click();
- }, [aui, audioAttachmentsEnabled]);
+ const pickAttachment = usePickComposerAttachment();
// Straight to the picker, skipping the "+" menu the item lives in. Off-route
// the chat pane is hidden rather than unmounted, so the chords gate on it
// being the visible tab; a window listener does not care about `inert`.
diff --git a/studio/frontend/src/features/native-intents/drop-paths.ts b/studio/frontend/src/features/native-intents/drop-paths.ts
index df322210d76..3cc714b694f 100644
--- a/studio/frontend/src/features/native-intents/drop-paths.ts
+++ b/studio/frontend/src/features/native-intents/drop-paths.ts
@@ -14,7 +14,10 @@ import {
TEXT_ATTACHMENT_EXTENSIONS,
isTextAttachmentName,
} from "../chat/text-attachment-accept.ts";
-import { RAG_UPLOAD_ACCEPT } from "../rag/types/rag.ts";
+import {
+ RAG_DOCUMENT_UPLOAD_ACCEPT,
+ RAG_UPLOAD_ACCEPT,
+} from "../rag/types/rag.ts";
import { MAX_AUDIO_FILES } from "../../lib/audio-utils.ts";
const DOC_EXTS = RAG_UPLOAD_ACCEPT.split(",").map((ext) =>
@@ -36,6 +39,13 @@ export function isComposerAttachmentName(path: string): boolean {
);
}
+/** Documents only chat with files reads; composer-handled types stay there. */
+const OFFICE_DOC_EXTS = RAG_DOCUMENT_UPLOAD_ACCEPT.split(",").filter(
+ (ext) => !isComposerAttachmentName(`file${ext}`),
+);
+
+const DROP_DOC_EXTS = [...DOC_EXTS, ...OFFICE_DOC_EXTS];
+
function isTextDropName(path: string): boolean {
const name = nativeFileName(path).toLowerCase();
if (!name.includes(".") && isTextAttachmentName(name)) {
@@ -71,7 +81,7 @@ const VIDEO_EXTS = CHAT_VIDEO_DROP_ACCEPT.split(",").map((ext) =>
);
/** What the window actually takes, for the rejection toast and the overlay. */
-export const SUPPORTED_DROP_HINT = `Supported files: ${RAG_UPLOAD_ACCEPT}, ${OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS}, ${RTF_ATTACHMENT_EXTENSIONS}, source and text files, ${TOOL_ONLY_ATTACHMENT_EXTENSIONS} with Code on, ${CHAT_IMAGE_DROP_ACCEPT}, up to ${MAX_AUDIO_FILES} of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`;
+export const SUPPORTED_DROP_HINT = `Supported files: ${DROP_DOC_EXTS.join(",")}, ${OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS}, ${RTF_ATTACHMENT_EXTENSIONS}, source and text files, ${TOOL_ONLY_ATTACHMENT_EXTENSIONS} with Code on, ${CHAT_IMAGE_DROP_ACCEPT}, up to ${MAX_AUDIO_FILES} of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`;
/** Last path segment of a native path, for display and extension checks. */
export function nativeFileName(path: string): string {
@@ -111,7 +121,7 @@ export function classifyDropPaths(paths: string[]): NativeDropClass {
}
const docs = paths.filter(
(path) =>
- DOC_EXTS.some((ext) => hasExt(path, ext)) ||
+ DROP_DOC_EXTS.some((ext) => hasExt(path, ext)) ||
isComposerAttachmentName(path),
);
const images = paths.filter((path) =>
diff --git a/studio/frontend/src/features/rag/components/document-status-chip.tsx b/studio/frontend/src/features/rag/components/document-status-chip.tsx
index 2f18536d6b0..4cd16b95c4a 100644
--- a/studio/frontend/src/features/rag/components/document-status-chip.tsx
+++ b/studio/frontend/src/features/rag/components/document-status-chip.tsx
@@ -9,7 +9,7 @@ import { HugeiconsIcon } from "@hugeicons/react";
import { XIcon } from "lucide-react";
import type { DocumentStatus } from "../types/rag";
-const STAGE_LABELS: Record<string, string> = {
+export const STAGE_LABELS: Record<string, string> = {
parsing: "Reading document",
ocr: "Reading scanned pages",
captioning: "Reading charts and figures",
diff --git a/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx b/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx
index 5fbb6bc780c..e7089b56cda 100644
--- a/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx
+++ b/studio/frontend/src/features/rag/components/knowledge-base-dialog.tsx
@@ -509,8 +509,8 @@ function KnowledgeBaseDocuments({
dragging && "border-primary/60 bg-primary/5 text-foreground",
)}
>
- No documents yet. Upload or drop a PDF, Markdown, DOCX, HTML, or text
- file.
+ No documents yet. Upload or drop documents, spreadsheets, slides,
+ e-books, email, text or code.
</div>
) : (
<div
diff --git a/studio/frontend/src/features/rag/components/project-source-dropzone.tsx b/studio/frontend/src/features/rag/components/project-source-dropzone.tsx
index 457c1f36adc..1d355e3f974 100644
--- a/studio/frontend/src/features/rag/components/project-source-dropzone.tsx
+++ b/studio/frontend/src/features/rag/components/project-source-dropzone.tsx
@@ -22,6 +22,7 @@ import {
} from "../api/rag-api";
import {
RAG_SOURCE_UPLOAD_ACCEPT,
+ SUPPORTED_SOURCES_HINT,
partitionSupported,
} from "./source-drop-policy";
import {
@@ -253,7 +254,7 @@ export function ProjectSourceDropzone({
unsupported.length === 1
? `Can't add ${unsupported[0]}`
: `Can't add ${unsupported.length} files`,
- { description: "Supported types: documents and source code files" },
+ { description: SUPPORTED_SOURCES_HINT },
);
}
// Name, size and mtime can in principle match for two different files, so
diff --git a/studio/frontend/src/features/rag/components/project-sources-panel.tsx b/studio/frontend/src/features/rag/components/project-sources-panel.tsx
index 5fb0d56da83..64279e10c2b 100644
--- a/studio/frontend/src/features/rag/components/project-sources-panel.tsx
+++ b/studio/frontend/src/features/rag/components/project-sources-panel.tsx
@@ -150,8 +150,8 @@ export function ProjectSourcesPanel({ projectId }: { projectId: string }) {
Give this project context
</p>
<p className="max-w-sm text-sm text-muted-foreground">
- Upload PDFs, docs, or text. Every chat in this project can use
- them.
+ Add documents, spreadsheets, slides, e-books, email, text or
+ code. Every chat in this project can use them.
</p>
</div>
<Button
diff --git a/studio/frontend/src/features/rag/components/source-drop-policy.ts b/studio/frontend/src/features/rag/components/source-drop-policy.ts
index 86dee573ce3..76dfcf697d6 100644
--- a/studio/frontend/src/features/rag/components/source-drop-policy.ts
+++ b/studio/frontend/src/features/rag/components/source-drop-policy.ts
@@ -2,13 +2,20 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { TEXT_ATTACHMENT_EXTENSIONS } from "../../chat/text-attachment-accept.ts";
-import { RAG_UPLOAD_ACCEPT } from "../types/rag.ts";
+import { RAG_DOCUMENT_UPLOAD_ACCEPT, RAG_UPLOAD_ACCEPT } from "../types/rag.ts";
-// Projects and knowledge bases also index source files; backend SOURCE_TEXT_EXTS mirrors this list.
+// Everything the backend indexes (config.UPLOAD_EXTS), for every scope: chat, project and knowledge base.
export const RAG_SOURCE_UPLOAD_ACCEPT = [
- ...new Set([...RAG_UPLOAD_ACCEPT.split(","), ...TEXT_ATTACHMENT_EXTENSIONS]),
+ ...new Set([
+ ...RAG_UPLOAD_ACCEPT.split(","),
+ ...RAG_DOCUMENT_UPLOAD_ACCEPT.split(","),
+ ...TEXT_ATTACHMENT_EXTENSIONS,
+ ]),
].join(",");
+export const SUPPORTED_SOURCES_HINT =
+ "Supported: documents, spreadsheets, slides, e-books, email, text and code files.";
+
const ACCEPTED_UPLOAD_EXTS = new Set(RAG_SOURCE_UPLOAD_ACCEPT.split(","));
// `accept` only filters the picker, so a drop can carry anything, including an
diff --git a/studio/frontend/src/features/rag/components/thread-documents-bar.tsx b/studio/frontend/src/features/rag/components/thread-documents-bar.tsx
index 845527ef12b..572615ab738 100644
--- a/studio/frontend/src/features/rag/components/thread-documents-bar.tsx
+++ b/studio/frontend/src/features/rag/components/thread-documents-bar.tsx
@@ -2,12 +2,15 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import {
- type RefObject,
+ type ReactNode,
useCallback,
useEffect,
+ useLayoutEffect,
+ useMemo,
useRef,
useState,
} from "react";
+import { ChevronLeftIcon, ChevronRightIcon, PlusIcon, XIcon } from "lucide-react";
import { HugeiconsIcon } from "@hugeicons/react";
import {
AttachmentIcon,
@@ -15,6 +18,31 @@ import {
FolderAttachmentIcon,
Folder02Icon,
} from "@hugeicons/core-free-icons";
+import {
+ AttachmentKindIcon,
+ CARD_EDGE,
+ CARD_SIZE,
+ CARD_SLOT,
+ CARD_SURFACE,
+ FileCardBody,
+} from "@/components/assistant-ui/attachment";
+import {
+ type AttachmentPreview,
+ AttachmentCardPreview,
+ attachmentPreview,
+} from "@/components/assistant-ui/attachment-card-preview";
+import { LocalFileDialog } from "@/components/assistant-ui/attachment-document-dialog";
+import {
+ MAX_DOCUMENT_PREVIEW_BYTES,
+ documentKind,
+ isMarkdown,
+} from "@/components/file-viewer";
+import { Spinner } from "@/components/ui/spinner";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "@/components/ui/tooltip";
import { Tick02Icon } from "@/lib/tick-icon";
import { ChevronDownStandardIcon } from "@/lib/chevron-icons";
import { useAui } from "@assistant-ui/react";
@@ -31,12 +59,17 @@ import {
ChatThreadDeletedError,
ensureStoredChatThread,
getStoredChatThread,
+ annotationsOfFile,
+ attachmentFileKind,
+ isPastedTextFile,
+ isTextAttachment,
isThreadIncognito,
} from "@/features/chat";
import {
useNativeAttachmentTargetKey,
useNativeIntentStore,
} from "@/features/native-intents";
+import { openFilePicker } from "@/lib/open-file-picker";
import { toast } from "@/lib/toast";
import {
DropdownMenu,
@@ -47,19 +80,16 @@ import {
} from "@/components/ui/dropdown-menu";
import {
announceProjectSourcesUpdated,
+ getDocumentFileUrl,
invalidateProjectSources,
listKnowledgeBases,
+ listLinkedFolders,
subscribeKnowledgeBasesChanged,
listProjectDocuments,
listThreadDocuments,
} from "../api/rag-api";
import { useRagAvailabilityStore } from "../api/rag-availability";
-import {
- type DocumentStatus,
- RAG_UPLOAD_ACCEPT,
- type RagDocument,
- isLinkedFolderManaged,
-} from "../types/rag";
+import { type RagDocument, isLinkedFolderManaged } from "../types/rag";
import {
AlertDialog,
AlertDialogAction,
@@ -70,13 +100,25 @@ import {
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog";
-import { DocumentStatusChip } from "./document-status-chip";
+import { STAGE_LABELS } from "./document-status-chip";
+import { useDocumentPreviewStore } from "./preview-store";
import {
type KnowledgeBaseFocus,
KnowledgeBaseDialog,
} from "./knowledge-base-dialog";
+import {
+ RAG_SOURCE_UPLOAD_ACCEPT,
+ SUPPORTED_SOURCES_HINT,
+ isSupportedSourceName,
+} from "./source-drop-policy";
import { EXPIRY_GRACE_MS } from "./staged-source";
-import { uploadItemFromIntent, useRagDocuments } from "./use-rag-documents";
+import {
+ type RagUploadItem,
+ type TrackedDocument,
+ uploadItemFromIntent,
+ useRagDocuments,
+} from "./use-rag-documents";
+import { useSourceDrop } from "./use-source-drop";
// Refetched after any KB mutation so a rename shows at once.
function useKnowledgeBaseName(kbId: string | null): string | null {
@@ -109,38 +151,6 @@ function useKnowledgeBaseName(kbId: string | null): string | null {
return known !== null && known.kbId === kbId ? known.name : null;
}
-// Shown when retrieval comes from a KB, so the source isn't invisible.
-function KnowledgeBaseSourceChip({
- name,
- onOpen,
- buttonRef,
-}: {
- name: string | null;
- onOpen: () => void;
- buttonRef: RefObject<HTMLButtonElement | null>;
-}) {
- return (
- <div className="mb-2 flex w-full flex-row items-center gap-1.5 pl-0.5 pr-1.5 pt-0.5 pb-1">
- <button
- ref={buttonRef}
- type="button"
- onClick={onOpen}
- className="composer-pill-btn min-w-0 max-w-full"
- title="Add or remove documents"
- >
- <HugeiconsIcon
- icon={FileDatabaseIcon}
- strokeWidth={2}
- className="size-3.5 shrink-0"
- />
- <span className="min-w-0 truncate">
- {name ? `Knowledge base: ${name}` : "Knowledge base"}
- </span>
- </button>
- </div>
- );
-}
-
/**
* Confirm a thread is stored before documents are indexed against it. An id reaches this
* component before its row write lands, from a cached initialize() or from activeThreadId, and
@@ -164,47 +174,6 @@ async function requireStoredThread(threadId: string): Promise<void> {
}
}
-/** shows inherited sources because Docs off does not stop retrieval. */
-function InheritedProjectSources({
- documents,
- unused,
-}: {
- documents: { id: string; filename: string; status: DocumentStatus }[];
- unused: boolean;
-}) {
- return (
- <div className="mb-2 flex w-full flex-row items-center gap-1.5 pl-0.5 pr-1.5 pt-0.5 pb-1">
- <span
- className="composer-pill-btn shrink-0 cursor-default !text-foreground/60"
- title={
- unused
- ? "The selected model can't search documents, so this chat doesn't use its project's sources. Pick a model with tool support to use them."
- : "This chat retrieves from its project's sources. Manage them in the project's Sources tab."
- }
- >
- <HugeiconsIcon icon={FolderAttachmentIcon} strokeWidth={2} className="size-3.5" />
- <span>{unused ? "Project sources not used" : "Project sources"}</span>
- </span>
- {/* cap linked folders because their sources can fill the viewport. */}
- <div
- className={cn(
- "flex max-h-24 flex-1 flex-row flex-wrap items-center gap-1.5 overflow-y-auto",
- unused && "opacity-50",
- )}
- >
- {documents.map((doc) => (
- <DocumentStatusChip
- key={`inherited:${doc.id}`}
- filename={doc.filename}
- status={doc.status}
- shared={true}
- />
- ))}
- </div>
- </div>
- );
-}
-
/** row data avoids stale activeProjectId; undefined blocks attachments. */
/** retries chat row reads before leaving the project unresolved. */
const PROJECT_LOOKUP_RETRIES = 3;
@@ -266,54 +235,580 @@ function useThreadProjectId(
: undefined;
}
-/** The composer's attach control. Wording and glyph follow the active target, so
- * a project chat says up front where the file is going. */
-function AttachFilesButton({
+function isRagIndexable(file: File): boolean {
+ // Pasted text and annotations are not documents.
+ if (isPastedTextFile(file) || annotationsOfFile(file)) return false;
+ return isSupportedSourceName(file.name);
+}
+
+/** Display names of a project's linked folders, read once per set of folder ids. */
+function useLinkedFolderNames(
+ projectId: string | null,
+ folderIds: string,
+): ReadonlyMap<string, string> {
+ const [names, setNames] = useState<ReadonlyMap<string, string>>(new Map());
+ useEffect(() => {
+ if (!projectId || !folderIds) return;
+ let cancelled = false;
+ listLinkedFolders({ type: "project", id: projectId })
+ .then((folders) => {
+ if (cancelled) return;
+ setNames(new Map(folders.map((f) => [f.id, f.displayName])));
+ })
+ .catch(() => {
+ // The card falls back to a generic label.
+ });
+ return () => {
+ cancelled = true;
+ };
+ }, [projectId, folderIds]);
+ return names;
+}
+
+/** Splits project documents into loose files and per-folder groups. A linked folder can
+ * hold thousands of files, so it is drawn as one card rather than one card per file. */
+function groupByLinkedFolder(docs: TrackedDocument[]): {
+ loose: TrackedDocument[];
+ folders: [string, TrackedDocument[]][];
+} {
+ const loose: TrackedDocument[] = [];
+ const folders = new Map<string, TrackedDocument[]>();
+ for (const doc of docs) {
+ const folderId = doc.linkedFolderId;
+ if (!folderId) {
+ loose.push(doc);
+ continue;
+ }
+ const group = folders.get(folderId);
+ if (group) group.push(doc);
+ else folders.set(folderId, [doc]);
+ }
+ return { loose, folders: [...folders] };
+}
+
+function isProcessing(doc: TrackedDocument): boolean {
+ return doc.status === "pending" || doc.status === "running";
+}
+
+function ProjectBadge() {
+ return (
+ <span
+ className="pointer-events-none absolute top-1.5 left-1.5 inline-flex items-center gap-1 rounded-full bg-muted px-1.5 py-0.5 text-ui-10 text-muted-foreground"
+ aria-hidden={true}
+ >
+ <HugeiconsIcon icon={Folder02Icon} strokeWidth={2} className="size-3" />
+ Project
+ </span>
+ );
+}
+
+function LinkedFolderCard({
+ name,
+ docs,
+}: {
+ name: string | undefined;
+ docs: TrackedDocument[];
+}) {
+ const indexing = docs.filter(isProcessing).length;
+ const label = name ?? "Linked folder";
+ const count = `${docs.length} ${docs.length === 1 ? "file" : "files"}`;
+ return (
+ <div className={cn("relative", CARD_SLOT)}>
+ <ProjectBadge />
+ <div
+ title={`${label}: ${count}, kept in sync and shared with every chat in this project. Manage it in the project's Sources tab.`}
+ className={cn("flex overflow-hidden rounded-[15px]", CARD_SIZE, CARD_EDGE)}
+ >
+ <FileCardBody
+ name={label}
+ kind="document"
+ icon={
+ <HugeiconsIcon
+ icon={Folder02Icon}
+ strokeWidth={1.75}
+ className="size-3.25 shrink-0 text-muted-foreground"
+ />
+ }
+ center={
+ <span className="flex flex-col items-center gap-1.5 text-center text-ui-11">
+ {indexing > 0 ? (
+ <Spinner className="size-4" />
+ ) : (
+ <HugeiconsIcon icon={Folder02Icon} strokeWidth={1.5} className="size-6" />
+ )}
+ <span>{indexing > 0 ? `Indexing ${indexing} of ${docs.length}` : count}</span>
+ <span className="sr-only">linked folder, shared with the project</span>
+ </span>
+ }
+ />
+ </div>
+ </div>
+ );
+}
+
+// Same cap as composer attachment previews; bigger files keep the icon.
+const MAX_THUMBNAIL_BYTES = 10 * 1024 * 1024;
+const THUMBNAIL_CACHE_BYTES = 48 * 1024 * 1024;
+// Source files fetched for card thumbnails, by document id, so a re-render or a return to the chat
+// does not download them again. Insertion order doubles as LRU order.
+const thumbnailFiles = new Map<string, Promise<File | null>>();
+const thumbnailSizes = new Map<string, number>();
+let thumbnailBytes = 0;
+
+function forgetThumbnail(id: string): void {
+ thumbnailFiles.delete(id);
+ thumbnailBytes -= thumbnailSizes.get(id) ?? 0;
+ thumbnailSizes.delete(id);
+}
+
+/** The stored source as a File, or null past `maxBytes` (0 = no cap). */
+async function fetchSourceFile(
+ doc: TrackedDocument,
+ maxBytes: number,
+): Promise<File | null> {
+ const response = await fetch(await getDocumentFileUrl(doc.id));
+ if (!response.ok) return null;
+ if (maxBytes && Number(response.headers.get("content-length")) > maxBytes) {
+ void response.body?.cancel();
+ return null;
+ }
+ const blob = await response.blob();
+ if (blob.size === 0 || (maxBytes && blob.size > maxBytes)) return null;
+ // Linked-folder documents are named by their relative path.
+ const name = doc.filename.split("/").pop() ?? doc.filename;
+ // Most types are served as octet-stream; an empty type lets the extension decide the kind.
+ const type = blob.type === "application/octet-stream" ? "" : blob.type;
+ return new File([blob], name, { type });
+}
+
+const fetchThumbnailFile = (doc: TrackedDocument) =>
+ fetchSourceFile(doc, MAX_THUMBNAIL_BYTES);
+
+/** Whether the attachment viewer can show this source: pages, markdown or text. */
+function isViewableSource(name: string): boolean {
+ return (
+ documentKind(name) !== null ||
+ isMarkdown(name) ||
+ isTextAttachment(name, undefined)
+ );
+}
+
+function loadThumbnailFile(doc: TrackedDocument): Promise<File | null> {
+ const cached = thumbnailFiles.get(doc.id);
+ if (cached) {
+ thumbnailFiles.delete(doc.id);
+ thumbnailFiles.set(doc.id, cached);
+ return cached;
+ }
+ const pending = fetchThumbnailFile(doc).catch(() => null);
+ thumbnailFiles.set(doc.id, pending);
+ void pending.then((file) => {
+ if (thumbnailFiles.get(doc.id) !== pending) return;
+ // A failure is not remembered, so the next mount tries again.
+ if (file === null) {
+ forgetThumbnail(doc.id);
+ return;
+ }
+ thumbnailSizes.set(doc.id, file.size);
+ thumbnailBytes += file.size;
+ for (const oldest of thumbnailFiles.keys()) {
+ if (thumbnailBytes <= THUMBNAIL_CACHE_BYTES || oldest === doc.id) break;
+ forgetThumbnail(oldest);
+ }
+ });
+ return pending;
+}
+
+type Thumbnail =
+ | { kind: "image"; url: string }
+ | { kind: "file"; file: File; preview: AttachmentPreview };
+
+/** The card's thumbnail while the card is on screen; off screen it lets go of the file, which the
+ * byte-capped cache may still hold for its return. */
+function useDocumentThumbnail(
+ doc: TrackedDocument,
+ enabled: boolean,
+ element: HTMLElement | null,
+): Thumbnail | null {
+ const [visible, setVisible] = useState(false);
+ const [thumbnail, setThumbnail] = useState<Thumbnail | null>(null);
+ useEffect(() => {
+ if (!enabled || !element) return;
+ const observer = new IntersectionObserver(
+ (entries) => setVisible(entries[entries.length - 1].isIntersecting),
+ { rootMargin: "200px" },
+ );
+ observer.observe(element);
+ return () => observer.disconnect();
+ }, [enabled, element]);
+ useEffect(() => {
+ if (!enabled || !visible) {
+ setThumbnail(null);
+ return;
+ }
+ let cancelled = false;
+ let objectUrl: string | null = null;
+ void loadThumbnailFile(doc).then((file) => {
+ if (cancelled || !file) return;
+ const kind = attachmentFileKind(file.name, file.type);
+ if (kind === "image") {
+ // An <img> renders an SVG only with its type; it never runs the SVG's scripts.
+ const svg = /\.svg$/i.test(file.name) && !file.type;
+ objectUrl = URL.createObjectURL(
+ svg ? new Blob([file], { type: "image/svg+xml" }) : file,
+ );
+ setThumbnail({ kind: "image", url: objectUrl });
+ return;
+ }
+ const preview = attachmentPreview(file, kind);
+ if (preview) setThumbnail({ kind: "file", file, preview });
+ });
+ return () => {
+ cancelled = true;
+ if (objectUrl) URL.revokeObjectURL(objectUrl);
+ };
+ // The id names the stored file; the rest of the row changes with progress frames.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [doc.id, enabled, visible]);
+ return enabled ? thumbnail : null;
+}
+
+function DocumentCard({
+ doc,
+ shared,
+ onRemove,
+}: {
+ doc: TrackedDocument;
+ shared: boolean;
+ onRemove?: () => void;
+}) {
+ const openPreview = useDocumentPreviewStore((s) => s.openPreview);
+ const kind = attachmentFileKind(doc.filename, undefined);
+ const processing = isProcessing(doc);
+ const failed = doc.status === "failed";
+ const pending = doc.id.startsWith("pending_");
+ const percent =
+ doc.progress != null
+ ? Math.round(doc.progress <= 1 ? doc.progress * 100 : doc.progress)
+ : null;
+ const stageLabel = (doc.stage && STAGE_LABELS[doc.stage]) || "Indexing";
+ const center = processing ? (
+ <span className="flex flex-col items-center gap-1.5 text-center text-ui-11">
+ <Spinner className="size-4" />
+ <span className="line-clamp-2">
+ {stageLabel}
+ {percent != null ? ` ${percent}%` : null}
+ </span>
+ </span>
+ ) : failed ? (
+ <span className="line-clamp-3 text-center text-ui-11 text-destructive">
+ Couldn't index
+ </span>
+ ) : (
+ <AttachmentKindIcon kind={kind} className="size-6" />
+ );
+ const ready = doc.status === "completed" && !pending;
+ const [slot, setSlot] = useState<HTMLDivElement | null>(null);
+ const thumbnail = useDocumentThumbnail(doc, ready, slot);
+ // PDFs open the source preview, which can jump to a page. Without a chunk it has no text for
+ // anything else, so other files open in the attachment viewer, read on click.
+ const isPdf = kind === "pdf";
+ const viewable = !isPdf && isViewableSource(doc.filename);
+ const thumbnailFile = thumbnail?.kind === "file" ? thumbnail.file : null;
+ const loadSource = useCallback(async (): Promise<Blob> => {
+ // The thumbnail's copy when there is one; past its cap, up to the viewer's own limit, the
+ // same one the Library holds to. Past that the viewer says it can't preview.
+ const file =
+ thumbnailFile ?? (await fetchSourceFile(doc, MAX_DOCUMENT_PREVIEW_BYTES));
+ if (!file) throw new Error(`Couldn't preview ${doc.filename}`);
+ const textual =
+ documentKind(file.name, file.type) === null && !isMarkdown(file.name, file.type);
+ // Served as octet-stream: the viewer reads text only when the type says so.
+ return textual ? new File([file], file.name, { type: "text/plain" }) : file;
+ // The id names the stored file; the rest of the row changes with progress frames.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [doc.id, thumbnailFile]);
+ const canOpen = ready && (isPdf || viewable);
+ const preview =
+ thumbnail?.kind === "image" ? (
+ <img
+ src={thumbnail.url}
+ alt=""
+ className="h-full w-full object-cover"
+ />
+ ) : thumbnail?.kind === "file" ? (
+ <AttachmentCardPreview
+ file={thumbnail.file}
+ preview={thumbnail.preview}
+ fallback={
+ <span className="flex h-full items-center justify-center">
+ <AttachmentKindIcon kind={kind} className="size-6" />
+ </span>
+ }
+ />
+ ) : undefined;
+ const card = (
+ <button
+ type="button"
+ disabled={!canOpen}
+ onClick={
+ isPdf
+ ? () => openPreview({ documentId: doc.id, filename: doc.filename })
+ : undefined
+ }
+ title={
+ doc.error ??
+ (shared
+ ? `${doc.filename}: shared with every chat in this project`
+ : doc.filename)
+ }
+ aria-label={[
+ canOpen ? `Preview ${doc.filename}` : doc.filename,
+ processing ? `${stageLabel}${percent != null ? ` ${percent}%` : ""}` : null,
+ failed ? "couldn't index" : null,
+ shared ? "shared with the project" : null,
+ ]
+ .filter(Boolean)
+ .join(", ")}
+ className={cn(
+ "flex overflow-hidden rounded-[15px] text-left transition-colors disabled:cursor-default",
+ CARD_SIZE,
+ CARD_EDGE,
+ canOpen && CARD_SURFACE,
+ canOpen && "cursor-pointer",
+ failed && "border-destructive/40",
+ )}
+ >
+ <FileCardBody
+ name={doc.filename}
+ kind={kind}
+ center={center}
+ preview={preview}
+ />
+ </button>
+ );
+ return (
+ <div
+ ref={setSlot}
+ className={cn(
+ "group/attachment-card relative",
+ CARD_SLOT,
+ )}
+ >
+ {viewable && ready ? (
+ <LocalFileDialog name={doc.filename.split("/").pop() ?? doc.filename} load={loadSource}>
+ {card}
+ </LocalFileDialog>
+ ) : (
+ card
+ )}
+ {shared ? <ProjectBadge /> : null}
+ {onRemove && !processing ? (
+ <button
+ type="button"
+ onClick={onRemove}
+ aria-label={`Remove ${doc.filename}`}
+ className="absolute top-1.5 right-1.5 flex size-5 items-center justify-center rounded-full bg-foreground text-background opacity-0 shadow-sm transition-opacity focus-visible:opacity-100 group-hover/attachment-card:opacity-100 group-focus-within/attachment-card:opacity-100 [@media(pointer:coarse)]:opacity-100"
+ >
+ <XIcon className="size-3 stroke-[2.5px]" />
+ </button>
+ ) : null}
+ </div>
+ );
+}
+
+function AddFilesCard({
disabled,
- compact,
- sharesWithProject,
onClick,
}: {
disabled: boolean;
- /** Icon-only once documents are attached, to leave the chips room. */
- compact: boolean;
- sharesWithProject: boolean;
onClick: () => void;
}) {
return (
- <button
- type="button"
- onClick={onClick}
- disabled={disabled}
- className={cn(
- "composer-pill-btn shrink-0 -translate-y-px !text-foreground/80",
- // Square button so the rounded-full hover reads as a circle.
- compact && "size-8 justify-center px-0",
- )}
- aria-label={
- sharesWithProject
- ? "Attach documents to this project"
- : "Attach documents to this thread"
- }
- title={
- sharesWithProject
- ? "Attach documents for retrieval, shared with every chat in this project"
- : "Attach documents for retrieval in this chat"
- }
+ <div className={CARD_SLOT}>
+ <button
+ type="button"
+ disabled={disabled}
+ onClick={onClick}
+ className={cn(
+ "unsloth-files-add-card flex flex-col items-center justify-center gap-1.5 rounded-[15px] border border-dashed border-[color-mix(in_oklab,var(--foreground)_calc(20%*var(--contrast-edge-gain,1)),transparent)] text-ui-12 text-muted-foreground transition-colors hover:border-[color-mix(in_oklab,var(--foreground)_calc(35%*var(--contrast-edge-gain,1)),transparent)] hover:text-foreground",
+ CARD_SIZE,
+ )}
+ >
+ <PlusIcon className="size-5 stroke-[1.5px]" />
+ Add files
+ </button>
+ </div>
+ );
+}
+
+/** Card above the composer: a header with controls over a strip of the chat's files. */
+function ChatFilesPanel({
+ icon,
+ title,
+ titleSuffix,
+ note,
+ headerControls,
+ onClose,
+ closeLabel,
+ onDropItems,
+ dropDisabledReason,
+ info,
+ count,
+ children,
+}: {
+ icon: typeof Folder02Icon;
+ title: string;
+ titleSuffix?: string;
+ /** Shown in a tooltip on the file count. */
+ info?: string;
+ count?: string;
+ note?: ReactNode;
+ headerControls?: ReactNode;
+ onClose?: () => void;
+ closeLabel?: string;
+ /** Handles browser and desktop drops on the panel. Unset, drops fall through to the composer. */
+ onDropItems?: (items: RagUploadItem[]) => void;
+ /** Set while the panel can't take files: a drop is still claimed and refused with this. */
+ dropDisabledReason?: string;
+ children?: ReactNode;
+}) {
+ const stripRef = useRef<HTMLDivElement>(null);
+ const [scroll, setScroll] = useState({ back: false, forward: false });
+ const measure = useCallback(() => {
+ const el = stripRef.current;
+ if (!el) return;
+ const back = el.scrollLeft > 1;
+ const forward = el.scrollWidth - el.clientWidth - el.scrollLeft > 1;
+ setScroll((prev) =>
+ prev.back === back && prev.forward === forward ? prev : { back, forward },
+ );
+ }, []);
+ useLayoutEffect(() => {
+ const el = stripRef.current;
+ if (!el) return;
+ measure();
+ const observer = new ResizeObserver(measure);
+ observer.observe(el);
+ // Adding a card does not resize the strip.
+ const mutations = new MutationObserver(measure);
+ mutations.observe(el, { childList: true });
+ return () => {
+ observer.disconnect();
+ mutations.disconnect();
+ };
+ }, [measure]);
+ const page = (direction: 1 | -1) => {
+ const el = stripRef.current;
+ if (!el) return;
+ el.scrollBy({ left: direction * el.clientWidth * 0.8, behavior: "smooth" });
+ };
+ const overflowing = scroll.back || scroll.forward;
+ // Drops here are indexed; elsewhere they attach to the message. The page handler skips
+ // prevented drops, and the desktop's window-wide handler skips registered targets.
+ const ignoreDrop = useCallback(() => {}, []);
+ const { dragging, dropProps, nativeDropTarget } = useSourceDrop({
+ onItems: onDropItems ?? ignoreDrop,
+ disabledReason: dropDisabledReason,
+ });
+ const acceptsDrop = Boolean(onDropItems);
+ const heading = (
+ <h2 className="min-w-0 truncate text-ui-14 font-medium text-foreground">
+ {title}
+ {titleSuffix ? (
+ <span className="font-normal text-muted-foreground"> {titleSuffix}</span>
+ ) : null}
+ </h2>
+ );
+ return (
+ <section
+ ref={acceptsDrop ? nativeDropTarget : undefined}
+ aria-label={title}
+ className="unsloth-files-panel"
+ data-dragging={acceptsDrop && dragging ? "true" : undefined}
+ {...(acceptsDrop ? dropProps : {})}
>
- <HugeiconsIcon
- icon={sharesWithProject ? Folder02Icon : AttachmentIcon}
- strokeWidth={2}
- className="size-3.5"
- />
- {compact ? null : (
- <span>
- {sharesWithProject
- ? "Add files for this project"
- : "Add files to chat with"}
- </span>
- )}
- </button>
+ {acceptsDrop ? (
+ <div className="unsloth-files-panel-drop-overlay" aria-hidden={true}>
+ <PlusIcon className="size-5" />
+ Drop to chat with these files
+ </div>
+ ) : null}
+ <div className="flex min-h-8 items-center gap-2 pl-1">
+ <HugeiconsIcon
+ icon={icon}
+ strokeWidth={1.75}
+ className="size-4 shrink-0 text-muted-foreground"
+ />
+ {heading}
+ {count && info ? (
+ // Only resting on the count opens it, and it shuts on leaving: it must not linger over the cards.
+ <Tooltip delayDuration={300} disableHoverableContent={true}>
+ <TooltipTrigger asChild={true}>
+ <span
+ // biome-ignore lint/a11y/noNoninteractiveTabindex: focus opens the tooltip for keyboard users.
+ tabIndex={0}
+ aria-label={`${count}. ${info}`}
+ className="unsloth-files-panel-count cursor-default focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
+ >
+ {count}
+ </span>
+ </TooltipTrigger>
+ <TooltipContent className="max-w-[calc(300px*var(--ui-space-scale,1))] text-ui-11 leading-snug">
+ {info}
+ </TooltipContent>
+ </Tooltip>
+ ) : count ? (
+ <span className="unsloth-files-panel-count">{count}</span>
+ ) : null}
+ {note}
+ <div className="ml-auto flex shrink-0 items-center gap-1">
+ {headerControls}
+ {overflowing ? (
+ <>
+ <button
+ type="button"
+ className="unsloth-files-panel-icon-btn"
+ onClick={() => page(-1)}
+ disabled={!scroll.back}
+ aria-label="Scroll files back"
+ >
+ <ChevronLeftIcon className="size-4" />
+ </button>
+ <button
+ type="button"
+ className="unsloth-files-panel-icon-btn"
+ onClick={() => page(1)}
+ disabled={!scroll.forward}
+ aria-label="Scroll files forward"
+ >
+ <ChevronRightIcon className="size-4" />
+ </button>
+ </>
+ ) : null}
+ {onClose ? (
+ <button
+ type="button"
+ className="unsloth-files-panel-icon-btn"
+ onClick={onClose}
+ aria-label={closeLabel ?? "Close"}
+ title={closeLabel}
+ >
+ <XIcon className="size-4" />
+ </button>
+ ) : null}
+ </div>
+ </div>
+ {children ? (
+ <div
+ ref={stripRef}
+ onScroll={measure}
+ className="unsloth-files-panel-strip"
+ >
+ {children}
+ </div>
+ ) : null}
+ </section>
);
}
@@ -358,7 +853,7 @@ function AttachmentTargetMenu({
disabled={disabled}
aria-label="Choose where attached files go"
title="Choose where attached files go"
- className="composer-pill-btn attachment-target-pill shrink-0 -translate-y-px gap-1 !text-foreground/70 pl-3 pr-1.5"
+ className="composer-pill-btn attachment-target-pill shrink-0 gap-1 !text-foreground/70 pl-3 pr-1.5"
>
<span className="text-ui-12">
{sharesWithProject ? "Project" : "This chat"}
@@ -371,7 +866,7 @@ function AttachmentTargetMenu({
</button>
</DropdownMenuTrigger>
<DropdownMenuContent
- align="start"
+ align="end"
className="unsloth-plus-menu w-[calc(328px*var(--ui-space-scale,1))]"
>
<DropdownMenuLabel>New files go to</DropdownMenuLabel>
@@ -433,7 +928,6 @@ export function ThreadDocumentsBar({
(s) => s.setThreadProjectAttachmentTarget,
);
const aui = useAui();
- const fileInputRef = useRef<HTMLInputElement>(null);
// materialize locally; append() reuses the id without a mid-upload remount.
const [materializedId, setMaterializedId] = useState<string | null>(null);
@@ -616,6 +1110,27 @@ export function ThreadDocumentsBar({
[ensureThreadId, projectId, sharesWithProject, upload, uploadToProject],
);
+ // Only files added or dropped here are indexed; composer attachments stay with the message.
+ const attachIndexable = useCallback(
+ (files: File[]) => {
+ const indexable = files.filter(isRagIndexable);
+ const skipped = files.length - indexable.length;
+ if (skipped > 0) {
+ toast.error(
+ skipped === 1
+ ? `"${files.find((file) => !isRagIndexable(file))?.name}" can't be searched`
+ : `${skipped} files can't be searched`,
+ { description: SUPPORTED_SOURCES_HINT },
+ );
+ }
+ if (indexable.length > 0) attach(indexable);
+ },
+ [attach],
+ );
+ const pickFiles = useCallback(() => {
+ openFilePicker(RAG_SOURCE_UPLOAD_ACCEPT, attachIndexable);
+ }, [attachIndexable]);
+
// Desktop drops land in the native-intent store because the drop listener lives on
// the chat page; only the chat that received the OS drop may drain its batch.
const nativeAttachmentTargetKey = useNativeAttachmentTargetKey();
@@ -703,27 +1218,28 @@ export function ThreadDocumentsBar({
setRagEnabled,
]);
- const chipScrollRef = useRef<HTMLDivElement>(null);
- const [chipsOverflow, setChipsOverflow] = useState(false);
- // Removing a project source here deletes it for every chat, beside a chat chip
+ // Removing a project source here deletes it for every chat, beside a chat card
// whose X is undoable. Confirm, as the Sources tab and Settings do.
const [removingShared, setRemovingShared] = useState<RagDocument | null>(null);
- const updateChipFade = useCallback(() => {
- const el = chipScrollRef.current;
- if (!el) return;
- setChipsOverflow(el.scrollHeight - el.scrollTop - el.clientHeight > 1);
- }, []);
- useEffect(() => {
- updateChipFade();
- }, [documents, updateChipFade]);
-
- // Open the picker synchronously to keep the click's user activation. Do NOT
- // materialize here: setActiveThreadId while the native dialog is open can remount
- // the composer and orphan this <input>. Materialize in onChange instead.
- const handleAddDocs = useCallback(() => {
- fileInputRef.current?.click();
- }, []);
+ const projectGroups = useMemo(
+ () => groupByLinkedFolder(projectDocuments),
+ [projectDocuments],
+ );
+ const folderNames = useLinkedFolderNames(
+ projectId,
+ projectGroups.folders.map(([id]) => id).join(","),
+ );
+ const folderCards = projectGroups.folders.map(([folderId, docs]) => (
+ <div
+ key={`folder:${folderId}`}
+ className={cn("contents", ragToolDisabled && "[&>*]:opacity-50")}
+ >
+ <LinkedFolderCard name={folderNames.get(folderId)} docs={docs} />
+ </div>
+ ));
+ const fileCount = documents.length + projectDocuments.length;
+ const countSuffix = fileCount === 1 ? "1 file" : `${fileCount} files`;
// Every branch, always the first fragment child: an "Add" must open after the source
// moves, and deleting the active KB in it must not remount it.
const kbDialog = (
@@ -749,10 +1265,23 @@ export function ThreadDocumentsBar({
return (
<>
{kbDialog}
- <KnowledgeBaseSourceChip
- name={activeKbName}
- onOpen={() => setKbDialogFocus({ kbId })}
- buttonRef={kbChipRef}
+ {/* Keyed per mode: the strip's observers attach on mount, and this one has no strip. */}
+ <ChatFilesPanel
+ key="kb"
+ icon={FileDatabaseIcon}
+ title={activeKbName ? `Knowledge base: ${activeKbName}` : "Knowledge base"}
+ onClose={() => setRagEnabled(false)}
+ closeLabel="Stop chatting with files"
+ headerControls={
+ <button
+ ref={kbChipRef}
+ type="button"
+ onClick={() => setKbDialogFocus({ kbId })}
+ className="unsloth-files-panel-action"
+ >
+ Manage files
+ </button>
+ }
/>
</>
);
@@ -763,10 +1292,36 @@ export function ThreadDocumentsBar({
<>
{kbDialog}
{projectDocuments.length > 0 ? (
- <InheritedProjectSources
- documents={projectDocuments}
- unused={ragToolDisabled}
- />
+ <ChatFilesPanel
+ key="project"
+ icon={FolderAttachmentIcon}
+ title={ragToolDisabled ? "Project sources not used" : "Project sources"}
+ count={countSuffix}
+ note={
+ <span
+ className="hidden truncate text-ui-12 text-muted-foreground @[32rem]/files-panel:inline"
+ title={
+ ragToolDisabled
+ ? "The selected model can't search documents, so this chat doesn't use its project's sources. Pick a model with tool support to use them."
+ : "This chat retrieves from its project's sources. Manage them in the project's Sources tab."
+ }
+ >
+ {ragToolDisabled
+ ? "The selected model can't search documents"
+ : "This chat retrieves from its project's sources"}
+ </span>
+ }
+ >
+ {folderCards}
+ {projectGroups.loose.map((doc) => (
+ <div
+ key={`inherited:${doc.id}`}
+ className={cn("contents", ragToolDisabled && "[&>*]:opacity-50")}
+ >
+ <DocumentCard doc={doc} shared={true} />
+ </div>
+ ))}
+ </ChatFilesPanel>
) : null}
</>
);
@@ -774,71 +1329,75 @@ export function ThreadDocumentsBar({
// block attachments until the chat's project is known to avoid guessing their scope.
const busy = uploading || projectUploading || projectUnresolved;
- const chipCount = documents.length + projectDocuments.length;
+ // Still claim a drop while busy, or it falls through and attaches to the message instead.
+ const busyReason = !busy
+ ? undefined
+ : projectUnresolved
+ ? "Still loading this chat's project. Try again in a moment."
+ : "Still uploading. Drop the files again once it finishes.";
return (
<>
{kbDialog}
- <div className="mb-2 flex w-full flex-row items-start gap-1.5 pl-0.5 pr-1.5 pt-0.5 pb-1">
- {/* keep controls aligned with top-aligned chips. */}
- <div className="flex shrink-0 items-center gap-1.5">
- <AttachFilesButton
- disabled={busy}
- compact={chipCount > 0}
- sharesWithProject={sharesWithProject}
- onClick={handleAddDocs}
- />
- {/* Only a project chat has two scopes to choose between. */}
- {projectId ? (
- <AttachmentTargetMenu
+ <ChatFilesPanel
+ key="files"
+ icon={FileDatabaseIcon}
+ title="Chat with files"
+ titleSuffix="(RAG)"
+ count={countSuffix}
+ info="Add or drop documents, spreadsheets, slides, e-books, email or code. The model searches them as you chat and cites what it uses."
+ onDropItems={attach}
+ dropDisabledReason={busyReason}
+ onClose={() => setRagEnabled(false)}
+ closeLabel="Stop chatting with files"
+ note={
+ ragToolDisabled && fileCount > 0 ? (
+ <span
+ className="shrink-0 rounded-full bg-muted px-2 py-0.5 text-ui-11 text-muted-foreground"
+ title="The selected model can't search documents, so these files aren't used. Pick a model with tool support to use them."
+ >
+ Not used
+ </span>
+ ) : null
+ }
+ headerControls={
+ <>
+ {/* Only a project chat has two scopes to choose between. */}
+ {projectId ? (
+ <AttachmentTargetMenu
+ disabled={busy}
+ sharesWithProject={sharesWithProject}
+ onSelect={(target) =>
+ setThreadProjectAttachmentTarget(effectiveThreadId, target)
+ }
+ />
+ ) : null}
+ <button
+ type="button"
disabled={busy}
- sharesWithProject={sharesWithProject}
- onSelect={(target) =>
- setThreadProjectAttachmentTarget(effectiveThreadId, target)
+ onClick={pickFiles}
+ className="unsloth-files-panel-action"
+ title={
+ busyReason ??
+ (sharesWithProject
+ ? "Add files for retrieval, shared with every chat in this project"
+ : "Add files for retrieval in this chat")
}
- />
- ) : null}
- </div>
- <input
- ref={fileInputRef}
- type="file"
- multiple
- accept={RAG_UPLOAD_ACCEPT}
- className="hidden"
- onChange={(e) => {
- const files = Array.from(e.target.files ?? []);
- e.target.value = "";
- if (files.length === 0) return;
- attach(files);
- }}
- />
- {ragToolDisabled && chipCount > 0 ? (
- <span
- className="composer-pill-btn shrink-0 cursor-default !text-foreground/60"
- title="The selected model can't search documents, so these files aren't used. Pick a model with tool support to use them."
+ >
+ Add files
+ </button>
+ </>
+ }
+ >
+ {/* project sources precede thread sources because they outlive the chat. */}
+ {folderCards}
+ {projectGroups.loose.map((doc) => (
+ <div
+ key={`project:${doc.id}`}
+ className={cn("contents", ragToolDisabled && "[&>*]:opacity-50")}
>
- Not used
- </span>
- ) : null}
- {/* cap chip rows and fade overflow. */}
- <div
- ref={chipScrollRef}
- onScroll={updateChipFade}
- className={cn(
- "flex max-h-24 flex-1 flex-row flex-wrap items-center gap-1.5 overflow-y-auto",
- chipsOverflow && "rag-docs-bottom-fade",
- ragToolDisabled && "opacity-50",
- )}
- >
- {/* project sources precede thread sources because they outlive the chat. */}
- {projectDocuments.map((doc) => (
- <DocumentStatusChip
- key={`project:${doc.id}`}
- filename={doc.filename}
- status={doc.status}
- progress={doc.progress}
- stage={doc.stage}
- error={doc.error}
+ <DocumentCard
+ doc={doc}
shared={true}
onRemove={
doc.id.startsWith("pending_") || isLinkedFolderManaged(doc)
@@ -846,53 +1405,55 @@ export function ThreadDocumentsBar({
: () => setRemovingShared(doc)
}
/>
- ))}
- {documents.map((doc) => (
- <DocumentStatusChip
- key={doc.id}
- filename={doc.filename}
- status={doc.status}
- progress={doc.progress}
- stage={doc.stage}
- error={doc.error}
+ </div>
+ ))}
+ {documents.map((doc) => (
+ <div
+ key={doc.id}
+ className={cn("contents", ragToolDisabled && "[&>*]:opacity-50")}
+ >
+ <DocumentCard
+ doc={doc}
+ shared={false}
onRemove={
doc.id.startsWith("pending_")
? undefined
: () => void remove(doc.id)
}
/>
- ))}
- </div>
- <AlertDialog
- open={removingShared !== null}
- onOpenChange={(open) => {
- if (!open) setRemovingShared(null);
- }}
- >
- <AlertDialogContent>
- <AlertDialogHeader>
- <AlertDialogTitle>Remove from project sources</AlertDialogTitle>
- <AlertDialogDescription>
- Remove "{removingShared?.filename}"? Every chat in this project
- loses it, and the file and its indexed content are deleted. This
- cannot be undone.
- </AlertDialogDescription>
- </AlertDialogHeader>
- <AlertDialogFooter>
- <AlertDialogCancel>Cancel</AlertDialogCancel>
- <AlertDialogAction
- onClick={() => {
- const doc = removingShared;
- setRemovingShared(null);
- if (doc) void removeFromProject(doc.id);
- }}
- >
- Remove
- </AlertDialogAction>
- </AlertDialogFooter>
- </AlertDialogContent>
- </AlertDialog>
- </div>
+ </div>
+ ))}
+ <AddFilesCard disabled={busy} onClick={pickFiles} />
+ </ChatFilesPanel>
+ <AlertDialog
+ open={removingShared !== null}
+ onOpenChange={(open) => {
+ if (!open) setRemovingShared(null);
+ }}
+ >
+ <AlertDialogContent>
+ <AlertDialogHeader>
+ <AlertDialogTitle>Remove from project sources</AlertDialogTitle>
+ <AlertDialogDescription>
+ Remove "{removingShared?.filename}"? Every chat in this project
+ loses it, and the file and its indexed content are deleted. This
+ cannot be undone.
+ </AlertDialogDescription>
+ </AlertDialogHeader>
+ <AlertDialogFooter>
+ <AlertDialogCancel>Cancel</AlertDialogCancel>
+ <AlertDialogAction
+ onClick={() => {
+ const doc = removingShared;
+ setRemovingShared(null);
+ if (doc) void removeFromProject(doc.id);
+ }}
+ >
+ Remove
+ </AlertDialogAction>
+ </AlertDialogFooter>
+ </AlertDialogContent>
+ </AlertDialog>
</>
);
}
diff --git a/studio/frontend/src/features/rag/components/use-source-drop.ts b/studio/frontend/src/features/rag/components/use-source-drop.ts
index 8f1e0733dbc..22c1d171fb2 100644
--- a/studio/frontend/src/features/rag/components/use-source-drop.ts
+++ b/studio/frontend/src/features/rag/components/use-source-drop.ts
@@ -13,7 +13,10 @@ import {
useRef,
useState,
} from "react";
-import { partitionSupported } from "./source-drop-policy";
+import {
+ SUPPORTED_SOURCES_HINT,
+ partitionSupported,
+} from "./source-drop-policy";
import { type RagUploadItem, uploadItemFromIntent } from "./use-rag-documents";
export interface SourceDropOptions {
@@ -62,7 +65,7 @@ export function useSourceDrop({
names.length === 1
? `Can't add ${names[0]}`
: `Can't add ${names.length} files`,
- { description: "Supported types: documents and source code files" },
+ { description: SUPPORTED_SOURCES_HINT },
);
}, []);
diff --git a/studio/frontend/src/features/rag/types/rag.ts b/studio/frontend/src/features/rag/types/rag.ts
index 0ceafac9fe8..fbabc0be642 100644
--- a/studio/frontend/src/features/rag/types/rag.ts
+++ b/studio/frontend/src/features/rag/types/rag.ts
@@ -181,3 +181,7 @@ export interface PreviewTarget {
}
export const RAG_UPLOAD_ACCEPT = ".pdf,.txt,.md,.markdown,.docx,.html,.htm";
+
+/** Mirrors backend `DOCUMENT_UPLOAD_EXTS`. */
+export const RAG_DOCUMENT_UPLOAD_ACCEPT =
+ ".doc,.docm,.dotx,.dotm,.xls,.xlsx,.xlsm,.xltx,.xltm,.ppt,.pptx,.pptm,.potx,.potm,.ppsx,.ppsm,.odt,.ods,.odp,.ott,.ots,.otp,.msg,.eml,.mht,.mhtml,.rtf,.epub,.xhtml,.xht";
diff --git a/studio/frontend/src/index.css b/studio/frontend/src/index.css
index 1bf77438be3..a1a120cfd0a 100644
--- a/studio/frontend/src/index.css
+++ b/studio/frontend/src/index.css
@@ -764,7 +764,7 @@
/* Lift the dark composer slightly; flavor cards sit close to the page. */
:root:where([data-palette="blueberry"], [data-palette="butterfly-pea"], [data-palette="cherry"], [data-palette="cinnamon"], [data-palette="cotton-candy"], [data-palette="dragon-fruit"], [data-palette="earl-grey"], [data-palette="espresso"], [data-palette="honey"], [data-palette="licorice"], [data-palette="macaron"], [data-palette="matcha"], [data-palette="mint"], [data-palette="neon-cyberpunk"], [data-palette="oat-milk"], [data-palette="peach"], [data-palette="pina-paraiso"], [data-palette="plum"], [data-palette="tangerine"], [data-palette="taro"], [data-palette="wasabi"], [data-palette="yuzu"])[data-palette].dark
- :is(.chat-composer-surface, .unsloth-composer-surface) {
+ :is(.chat-composer-surface, .unsloth-composer-surface, .unsloth-files-panel) {
background-color: color-mix(in srgb, var(--card), var(--th-fg) 4%);
}
@@ -6051,3 +6051,79 @@ html:has([data-blocking-screen]) .interface-zoom-position {
.browser-annotating *:not([data-annotate-ui], [data-annotate-ui] *) {
cursor: none !important;
}
+
+.unsloth-files-panel {
+ @apply relative mb-2 flex w-full flex-col gap-5 overflow-hidden rounded-[24px] px-4 pt-4 pb-6;
+ container: files-panel / inline-size;
+ font-family: var(--font-sans);
+ background-color: #ffffff;
+ /* Light mode: a flat 1px edge, about #dbdbdb on white, no shadow. */
+ border: 1px solid rgb(0 0 0 / calc(0.14 * var(--contrast-edge-gain, 1)));
+ transition: box-shadow 0.1s, border-color 0.1s;
+}
+.dark .unsloth-files-panel {
+ background-color: var(--card);
+ border-color: transparent;
+ box-shadow: var(--composer-shadow);
+}
+.unsloth-files-panel[data-dragging="true"] {
+ border-color: var(--primary);
+}
+.dark .unsloth-files-panel[data-dragging="true"] {
+ box-shadow:
+ 0 0 0 1.5px var(--primary),
+ var(--composer-shadow);
+}
+.unsloth-files-panel-count {
+ @apply inline-flex h-6 shrink-0 items-center rounded-full px-2.5 text-ui-12 font-medium tabular-nums text-foreground/75;
+ background-color: color-mix(in oklab, var(--foreground) 6%, transparent);
+}
+.dark .unsloth-files-panel-count {
+ background-color: color-mix(in oklab, var(--foreground) 10%, transparent);
+}
+.unsloth-files-panel-drop-overlay {
+ @apply pointer-events-none absolute inset-0 z-20 flex items-center justify-center gap-2 rounded-[inherit] bg-background/90 text-sm font-medium text-primary opacity-0 backdrop-blur-sm transition-opacity duration-150 dark:bg-card/90;
+}
+.unsloth-files-panel[data-dragging="true"] .unsloth-files-panel-drop-overlay {
+ opacity: 1;
+}
+/* One target at a time: over the panel, the composer's "Drop files here" would mislead. */
+.aui-composer-root:has(.unsloth-files-panel[data-dragging="true"]) .aui-composer-drop-overlay {
+ opacity: 0 !important;
+}
+/* Inset at rest; cards scroll under the panel edges. */
+.unsloth-files-panel-strip {
+ @apply -mx-4 flex flex-row flex-nowrap gap-2 overflow-x-auto overscroll-x-contain px-4;
+ scroll-padding-inline: 1rem;
+ scrollbar-width: none;
+}
+.unsloth-files-panel-strip::-webkit-scrollbar {
+ display: none;
+}
+.unsloth-files-panel-action {
+ @apply inline-flex h-8 shrink-0 items-center rounded-full border px-3.5 text-ui-13 text-foreground transition-colors disabled:pointer-events-none disabled:opacity-50;
+ border-color: color-mix(in oklab, var(--foreground) 14%, transparent);
+}
+.unsloth-files-panel-action:hover {
+ background-color: color-mix(in oklab, var(--foreground) 6%, transparent);
+}
+.unsloth-files-add-card:disabled {
+ @apply pointer-events-none opacity-50;
+}
+/* Dark mode: Add files is a fill, not an outline. */
+.dark .unsloth-files-panel-action,
+.dark .unsloth-files-add-card {
+ border-color: transparent;
+ background-color: color-mix(in oklab, var(--foreground) 8%, transparent);
+}
+.dark .unsloth-files-panel-action:hover,
+.dark .unsloth-files-add-card:hover {
+ border-color: transparent;
+ background-color: color-mix(in oklab, var(--foreground) 12%, transparent);
+}
+.unsloth-files-panel-icon-btn {
+ @apply inline-flex size-8 shrink-0 items-center justify-center rounded-full text-muted-foreground transition-colors hover:text-foreground disabled:pointer-events-none disabled:opacity-35;
+}
+.unsloth-files-panel-icon-btn:hover {
+ background-color: color-mix(in oklab, var(--foreground) 6%, transparent);
+}
diff --git a/studio/frontend/src/lib/open-file-picker.ts b/studio/frontend/src/lib/open-file-picker.ts
new file mode 100644
index 00000000000..8f901975a9e
--- /dev/null
+++ b/studio/frontend/src/lib/open-file-picker.ts
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+/** Opens the file chooser. Call from a click handler (needs user activation). */
+export function openFilePicker(
+ accept: string,
+ onFiles: (files: File[]) => void,
+): void {
+ const input = document.createElement("input");
+ input.type = "file";
+ input.multiple = true;
+ input.hidden = true;
+ if (accept !== "*") {
+ input.accept = accept;
+ }
+
+ document.body.appendChild(input);
+ input.onchange = (event) => {
+ const files = (event.target as HTMLInputElement).files;
+ if (files && files.length > 0) {
+ onFiles(Array.from(files));
+ }
+ document.body.removeChild(input);
+ };
+ input.oncancel = () => {
+ if (!input.files || input.files.length === 0) {
+ document.body.removeChild(input);
+ }
+ };
+ input.click();
+}
diff --git a/studio/frontend/tests/attachment-preview-text.test.ts b/studio/frontend/tests/attachment-preview-text.test.ts
index a8b994f471c..9a35f1b4fc1 100644
--- a/studio/frontend/tests/attachment-preview-text.test.ts
+++ b/studio/frontend/tests/attachment-preview-text.test.ts
@@ -2196,6 +2196,18 @@ test("a UTF-16 Markdown file previews as its text in the document viewer", async
assert.match(dialog, /blob instanceof File\s*\?\s*await readAttachmentText\(blob, source\.name, source\.contentType\)/);
});
+test("a UTF-16 text source opened from the files panel previews as its text", async () => {
+ const utf16 = new Uint8Array([0xff, 0xfe, ...Array.from("Café notes", (c) => [c.charCodeAt(0), 0]).flat()]);
+ const file = new File([utf16], "notes.txt", { type: "text/plain" });
+ assert.equal((await readAttachmentText(file, file.name, file.type)).text, "Café notes");
+ assert.notEqual(await file.text(), "Café notes");
+ const { readFile } = await import("node:fs/promises");
+ const dialog = await readFile(new URL("../src/components/assistant-ui/attachment-document-dialog.tsx", import.meta.url), "utf8");
+ assert.match(dialog, /blob instanceof File\s*\?\s*await readAttachmentText\(blob, source\.name, blob\.type\)/);
+ const bar = await readFile(new URL("../src/features/rag/components/thread-documents-bar.tsx", import.meta.url), "utf8");
+ assert.match(bar, /new File\(\[file\], file\.name, \{ type: "text\/plain" \}\)/);
+});
+
async function readRtf(rtf: string | Uint8Array<ArrayBuffer>): Promise<string> {
const content = await readRtfAttachmentContent(
new File([rtf], "doc.rtf"),
diff --git a/studio/frontend/tests/native-drop-paths.test.ts b/studio/frontend/tests/native-drop-paths.test.ts
index 68c67ba456b..91d23e0e459 100644
--- a/studio/frontend/tests/native-drop-paths.test.ts
+++ b/studio/frontend/tests/native-drop-paths.test.ts
@@ -220,6 +220,29 @@ for (const [format, extensions, adapter, rustList] of [
});
}
+test("Office, Outlook and web page drops index in chat with files and match the native allowlist", () => {
+ const office =
+ ".doc .xls .xlsx .xlsm .xltx .xltm .ppt .pptx .pptm .ppsx .ott .ots .otp .msg .mht .mhtml .xhtml .xht".split(
+ " ",
+ );
+ for (const extension of office) {
+ const path = `/docs/Report${extension.toUpperCase()}`;
+ assert.equal(classifyDropPaths([path]).kind, "docs", path);
+ assert.ok(!isComposerAttachmentName(path), path);
+ assert.ok(SUPPORTED_DROP_HINT.includes(extension));
+ }
+ // Types with a composer adapter keep going to the composer.
+ for (const ext of [".odt", ".rtf", ".epub", ".eml", ".docm"]) {
+ assert.ok(isComposerAttachmentName(`/docs/a${ext}`), ext);
+ }
+ const rust = [
+ ...(readText("../../src-tauri/src/native_path_policy.rs")
+ .match(/OFFICE_ATTACHMENT_EXTS[^=]*=\s*&\[([^\]]+)\]/)?.[1]
+ .matchAll(RUST_EXTENSION_RE) ?? []),
+ ].map((match) => `.${match[1]}`);
+ assert.deepEqual(rust.sort(), [...office].sort());
+});
+
test("images route to chat vision attachments, one or many", () => {
const dropped = classifyDropPaths([
"/photos/cat.PNG",
diff --git a/studio/frontend/tests/project-sources-model-without-tools.test.ts b/studio/frontend/tests/project-sources-model-without-tools.test.ts
index 2134b5e9607..11727d66efa 100644
--- a/studio/frontend/tests/project-sources-model-without-tools.test.ts
+++ b/studio/frontend/tests/project-sources-model-without-tools.test.ts
@@ -12,6 +12,7 @@ import type * as GateModule from "../src/features/chat/hooks/use-rag-tool-disabl
import { loadWithStubs } from "./helpers/module-stubs.ts";
const PROJECT_DOC = { id: "doc-1", filename: "handbook.pdf", status: "completed" };
+let extraProjectDocs: Record<string, unknown>[] = [];
let runtime: Record<string, unknown> = {};
const useChatRuntimeStore = (select: (s: Record<string, unknown>) => unknown) =>
@@ -46,7 +47,30 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(
"react/jsx-runtime": jsxRuntime,
"@hugeicons/react": { HugeiconsIcon: Nothing },
"@hugeicons/core-free-icons": {},
+ "lucide-react": new Proxy({}, { get: () => Nothing }),
"@/lib/tick-icon": {},
+ "@/lib/open-file-picker": { openFilePicker: () => undefined },
+ "@/components/assistant-ui/attachment": {
+ AttachmentKindIcon: Nothing,
+ FileCardBody: ({ name }: { name: string }) =>
+ React.createElement("span", null, name),
+ },
+ "@/components/assistant-ui/attachment-card-preview": {
+ AttachmentCardPreview: Nothing,
+ attachmentPreview: () => null,
+ },
+ "@/components/assistant-ui/attachment-document-dialog": {
+ LocalFileDialog: Passthrough,
+ },
+ "@/components/ui/tooltip": {
+ Tooltip: Passthrough,
+ TooltipContent: Nothing,
+ TooltipTrigger: Passthrough,
+ },
+ "@/components/ui/spinner": { Spinner: Nothing },
+ "./preview-store": {
+ useDocumentPreviewStore: selectorStore({ openPreview: () => undefined }),
+ },
"@/lib/chevron-icons": {},
"@assistant-ui/react": { useAui: () => ({}) },
"@/lib/utils": {
@@ -61,6 +85,15 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(
"@/features/chat": {
isThreadIncognito: () => false,
chatHistoryClearBoundary: { capture: () => 0 },
+ attachmentFileKind: (name: string) =>
+ name.endsWith(".pdf") ? "pdf" : "word",
+ isTextAttachment: (name: string) => name.endsWith(".txt"),
+ },
+ "@/components/file-viewer": {
+ MAX_DOCUMENT_PREVIEW_BYTES: 50,
+ documentKind: (name: string) =>
+ name.endsWith(".pdf") ? "pdf" : name.endsWith(".docx") ? "docx" : null,
+ isMarkdown: () => false,
},
"@/features/native-intents": {
useNativeAttachmentTargetKey: () => null,
@@ -72,23 +105,34 @@ const { ThreadDocumentsBar } = loadWithStubs<typeof BarModule>(
listKnowledgeBases: async () => [],
subscribeKnowledgeBasesChanged: () => () => undefined,
listProjectDocuments: async () => [PROJECT_DOC],
+ listLinkedFolders: async () => [],
listThreadDocuments: async () => [],
},
"../api/rag-availability": {
useRagAvailabilityStore: selectorStore({ isUnavailable: () => false }),
},
- "../types/rag": { RAG_UPLOAD_ACCEPT: "", isLinkedFolderManaged: () => false },
- "@/components/ui/alert-dialog": new Proxy({}, { get: () => Passthrough }),
- "./document-status-chip": {
- DocumentStatusChip: ({ filename }: { filename: string }) =>
- React.createElement("span", null, filename),
+ "../types/rag": { isLinkedFolderManaged: () => false },
+ "./source-drop-policy": {
+ RAG_SOURCE_UPLOAD_ACCEPT: "",
+ SUPPORTED_SOURCES_HINT: "",
+ isSupportedSourceName: () => true,
},
+ "./use-source-drop": {
+ useSourceDrop: () => ({
+ dragging: false,
+ dropProps: {},
+ nativeDropTarget: () => undefined,
+ }),
+ },
+ "@/components/ui/alert-dialog": new Proxy({}, { get: () => Passthrough }),
+ "./document-status-chip": { STAGE_LABELS: {} },
"./knowledge-base-dialog": { KnowledgeBaseDialog: Nothing },
"./staged-source": { EXPIRY_GRACE_MS: 0 },
"./use-rag-documents": {
uploadItemFromIntent: () => null,
useRagDocuments: (scope: { type: string } | null) => ({
- documents: scope?.type === "project" ? [PROJECT_DOC] : [],
+ documents:
+ scope?.type === "project" ? [PROJECT_DOC, ...extraProjectDocs] : [],
uploading: false,
hasIndexing: false,
loading: false,
@@ -156,6 +200,52 @@ test("with Docs on, a model without tool calling dims the files it will not sear
assert.match(html, /opacity-50/);
});
+// A linked folder can hold thousands of files; one card per file would flood the composer.
+test("a linked folder's files collapse into one folder card", () => {
+ extraProjectDocs = ["a.py", "b.py", "c.md"].map((filename, i) => ({
+ id: `linked-${i}`,
+ filename,
+ status: "completed",
+ linkedFolderId: "folder-1",
+ managed: true,
+ }));
+ try {
+ for (const ragEnabled of [false, true]) {
+ const html = renderProjectChat(
+ { checkpoint: "unsloth/Qwen3-4B-GGUF", supportsTools: true },
+ ragEnabled,
+ );
+ assert.match(html, /handbook\.pdf/);
+ assert.match(html, /Linked folder/);
+ assert.match(html, /3 files/);
+ assert.doesNotMatch(html, /a\.py|b\.py|c\.md/);
+ }
+ } finally {
+ extraProjectDocs = [];
+ }
+});
+
+// The source preview has text only for PDFs when opened without a chunk; other files open in the
+// attachment viewer, whatever their size, and a type it cannot draw is not offered at all.
+test("non-PDF cards open only when the attachment viewer can show them", () => {
+ extraProjectDocs = [
+ { id: "doc-2", filename: "notes.docx", status: "completed" },
+ { id: "doc-3", filename: "book.epub", status: "completed" },
+ ];
+ try {
+ const html = renderProjectChat(
+ { checkpoint: "unsloth/Qwen3-4B-GGUF", supportsTools: true },
+ true,
+ );
+ assert.match(html, /aria-label="Preview handbook\.pdf/);
+ assert.match(html, /aria-label="Preview notes\.docx/);
+ assert.doesNotMatch(html, /aria-label="Preview book\.epub/);
+ assert.match(html, /<button[^>]*disabled=""[^>]*aria-label="book\.epub/);
+ } finally {
+ extraProjectDocs = [];
+ }
+});
+
test("with Docs on, a model with tool calling keeps the files in effect", () => {
const html = renderProjectChat(
{ checkpoint: "unsloth/Qwen3-4B-GGUF", supportsTools: true },
diff --git a/studio/frontend/tests/rag-source-drop-filter.test.ts b/studio/frontend/tests/rag-source-drop-filter.test.ts
index 22128749d47..041a31449a8 100644
--- a/studio/frontend/tests/rag-source-drop-filter.test.ts
+++ b/studio/frontend/tests/rag-source-drop-filter.test.ts
@@ -9,7 +9,10 @@ import {
isSupportedSourceName,
partitionSupported,
} from "../src/features/rag/components/source-drop-policy.ts";
-import { RAG_UPLOAD_ACCEPT } from "../src/features/rag/types/rag.ts";
+import {
+ RAG_DOCUMENT_UPLOAD_ACCEPT,
+ RAG_UPLOAD_ACCEPT,
+} from "../src/features/rag/types/rag.ts";
test("every type the picker offers is accepted from a drop", () => {
for (const ext of RAG_UPLOAD_ACCEPT.split(",")) {
@@ -69,6 +72,14 @@ test("an empty drop yields nothing to upload and nothing to report", () => {
assert.equal(unsupported.length, 0);
});
+// The backend parses these for every scope; a project or knowledge base drop must not refuse them.
+test("office, OpenDocument, e-book and email files are accepted from a drop", () => {
+ for (const ext of RAG_DOCUMENT_UPLOAD_ACCEPT.split(",")) {
+ assert.equal(isSupportedSourceName(`deck${ext}`), true, ext);
+ }
+ assert.equal(RAG_SOURCE_UPLOAD_ACCEPT.split(",").includes(".pptx"), true);
+});
+
test("source files the project picker offers are accepted from a drop", () => {
for (const ext of RAG_SOURCE_UPLOAD_ACCEPT.split(",")) {
assert.equal(isSupportedSourceName(`app${ext}`), true, ext);
diff --git a/studio/frontend/tests/rag-thread-initialization.test.ts b/studio/frontend/tests/rag-thread-initialization.test.ts
index a0e9601c0fa..8ae60b3abfb 100644
--- a/studio/frontend/tests/rag-thread-initialization.test.ts
+++ b/studio/frontend/tests/rag-thread-initialization.test.ts
@@ -91,6 +91,7 @@ function harness(
];
},
};
+ let pickedFiles: ((files: File[]) => void) | null = null;
const nativeStore = Object.assign(() => nativePending, {
getState: () => nativeState,
});
@@ -119,6 +120,7 @@ function harness(
];
},
useCallback: (fn: unknown) => fn,
+ useMemo: (fn: () => unknown) => fn(),
useEffect(effect: () => void, deps: unknown[]) {
const index = cursor++;
const previous = slots[index] as unknown[] | undefined;
@@ -131,7 +133,20 @@ function harness(
"react/jsx-runtime": stubJsxRuntime(),
"@hugeicons/react": {},
"@hugeicons/core-free-icons": {},
+ "lucide-react": {},
"@/lib/tick-icon": {},
+ "@/lib/open-file-picker": {
+ openFilePicker: (_accept: string, onFiles: (files: File[]) => void) => {
+ pickedFiles = onFiles;
+ },
+ },
+ "@/components/assistant-ui/attachment": {},
+ "@/components/assistant-ui/attachment-card-preview": {},
+ "@/components/assistant-ui/attachment-document-dialog": {},
+ "@/components/file-viewer": {},
+ "@/components/ui/tooltip": {},
+ "@/components/ui/spinner": {},
+ "./preview-store": { useDocumentPreviewStore: () => undefined },
"@/lib/chevron-icons": {},
"@assistant-ui/react": { useAui: () => ({ threadListItem: () => item }) },
"@/lib/utils": { cn: () => "" },
@@ -146,6 +161,8 @@ function harness(
chatHistoryClearBoundary: { capture: () => 0 },
ChatThreadDeletedError: class extends Error {},
isThreadIncognito: () => incognito,
+ isPastedTextFile: () => false,
+ annotationsOfFile: () => null,
getStoredChatThread: async () => undefined,
ensureStoredChatThread: async (threadId: string) => {
if (storedIds.has(threadId)) return { id: threadId };
@@ -179,7 +196,13 @@ function harness(
select: (s: { isUnavailable: () => boolean }) => unknown,
) => select({ isUnavailable: () => false }),
},
- "../types/rag": { RAG_UPLOAD_ACCEPT: ".docx" },
+ "../types/rag": {},
+ "./source-drop-policy": {
+ RAG_SOURCE_UPLOAD_ACCEPT: ".docx",
+ SUPPORTED_SOURCES_HINT: "",
+ isSupportedSourceName: (name: string) => name.endsWith(".docx"),
+ },
+ "./use-source-drop": {},
"./document-status-chip": {},
"./knowledge-base-dialog": { KnowledgeBaseDialog: "KnowledgeBaseDialog" },
"./staged-source": { EXPIRY_GRACE_MS: 30_000 },
@@ -220,14 +243,9 @@ function harness(
});
}
function pick() {
- const input = findElement(tree, "input")!;
- const onChange = input.props.onChange as (event: unknown) => void;
- onChange({
- target: {
- files: [new File(["document"], "report.docx")],
- value: "report.docx",
- },
- });
+ const addCard = findComponent(tree, "AddFilesCard")!;
+ (addCard.props.onClick as () => void)();
+ pickedFiles!([new File(["document"], "report.docx")]);
}
return {
render,
@@ -360,6 +378,25 @@ function findElement(node: unknown, type: string): StubElement | undefined {
return findElement(element.props.children, type);
}
+function findComponent(node: unknown, name: string): StubElement | undefined {
+ if (Array.isArray(node)) {
+ for (const child of node) {
+ const hit = findComponent(child, name);
+ if (hit) return hit;
+ }
+ return undefined;
+ }
+ if (!node || typeof node !== "object" || !("props" in node)) return undefined;
+ const element = node as StubElement;
+ if (typeof element.type === "function" && element.type.name === name) return element;
+ return findComponent(element.props.children, name);
+}
+
+function manageFiles(tree: StubElement): StubElement {
+ const panel = findComponent(tree, "ChatFilesPanel")!;
+ return panel.props.headerControls as StubElement;
+}
+
function kbDialog(tree: StubElement): StubElement | undefined {
return findElement(tree, "KnowledgeBaseDialog");
}
@@ -417,13 +454,10 @@ test("the drop's Add still opens its knowledge base after the chat's source move
assert.deepEqual(app.uploads, []);
});
-test("the knowledge base chip opens that knowledge base without uploading anything", () => {
+test("the knowledge base panel opens that knowledge base without uploading anything", () => {
const app = harness({ ragSource: { type: "kb", kbId: "kb-1" } });
app.render();
- const chip = (app.tree.props.children as StubElement[]).find(
- (child) => typeof child?.type === "function",
- )!;
- (chip.props.onOpen as () => void)();
+ (manageFiles(app.tree).props.onClick as () => void)();
app.render();
const dialog = kbDialog(app.tree)!;
assert.equal(dialog.props.open, true);
@@ -436,10 +470,7 @@ test("an open dialog keeps its place when deleting the active knowledge base mov
// tree would remount, replaying its animation and dropping its state.
const app = harness({ ragSource: { type: "kb", kbId: "kb-1" } });
app.render();
- const chip = (app.tree.props.children as StubElement[]).find(
- (child) => typeof child?.type === "function",
- )!;
- (chip.props.onOpen as () => void)();
+ (manageFiles(app.tree).props.onClick as () => void)();
app.render();
const place = (tree: StubElement) => ({
root: tree.type,
diff --git a/studio/src-tauri/src/native_path_policy.rs b/studio/src-tauri/src/native_path_policy.rs
index f0643613f7d..c6f66f274d3 100644
--- a/studio/src-tauri/src/native_path_policy.rs
+++ b/studio/src-tauri/src/native_path_policy.rs
@@ -48,6 +48,11 @@ pub fn classify_native_model_path(path: &Path) -> Result<ClassifiedPath, String>
/// Document types the RAG ingest accepts; keep in sync with `config.SUPPORTED_UPLOAD_EXTS`.
pub const ATTACHMENT_EXTS: &[&str] = &["pdf", "txt", "md", "markdown", "docx", "html", "htm"];
+/// RAG-only document types; keep in sync with `OFFICE_DOC_EXTS` in `drop-paths.ts`.
+pub const OFFICE_ATTACHMENT_EXTS: &[&str] = &[
+ "doc", "xls", "xlsx", "xlsm", "xltx", "xltm", "ppt", "pptx", "pptm", "ppsx", "ott", "ots",
+ "otp", "msg", "mht", "mhtml", "xhtml", "xht",
+];
/// OpenDocument files the chat composer parses directly rather than indexing as RAG sources.
pub const OPEN_DOCUMENT_ATTACHMENT_EXTS: &[&str] = &["ods", "odt"];
pub const RTF_ATTACHMENT_EXTS: &[&str] = &["rtf"];
@@ -680,6 +685,7 @@ pub const VIDEO_ATTACHMENT_EXTS: &[&str] = &[
fn accepted_attachment_exts() -> impl Iterator<Item = &'static &'static str> {
ATTACHMENT_EXTS
.iter()
+ .chain(OFFICE_ATTACHMENT_EXTS.iter())
.chain(OPEN_DOCUMENT_ATTACHMENT_EXTS.iter())
.chain(RTF_ATTACHMENT_EXTS.iter())
.chain(TOOL_ONLY_ATTACHMENT_EXTS.iter())