Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: read more chat attachment formats, and hand the rest to the python tool (#11379)

Long Yixing57a884987dd0
diff --git a/studio/backend/core/chat_originals.py b/studio/backend/core/chat_originals.py
index ee85e50ce20..959398bbf50 100644
--- a/studio/backend/core/chat_originals.py
+++ b/studio/backend/core/chat_originals.py
@@ -6,7 +6,8 @@
A chat keeps a document's extracted text for the model; the file itself (a PDF, a Word document, a
workbook or a deck) is kept here so it can be opened as it looked. Each is stored once under its
SHA-256, which the attachment records as ``{"original": {"sha256", "sizeBytes"}}``: a fork or an
-import copies the reference, never the bytes, and a retried send stores nothing new.
+import copies the reference, never the bytes, and a retried send stores nothing new. While the python
+tool runs, any other attached file is kept too, so the tool can be handed a copy.
A file no attachment references any more is removed by ``sweep``, once it is older than an hour, so
one uploaded moments before its message is saved is never taken. A sweep that leaves such a file for
@@ -22,7 +23,7 @@ import re
import threading
import time
import uuid
-from pathlib import Path
+from pathlib import Path, PurePath
from typing import Iterable, Optional
from loggers import get_logger
@@ -30,8 +31,16 @@ from utils.paths.storage_roots import account_path, ensure_account_dir
logger = get_logger(__name__)
-EXTENSIONS = frozenset({".pdf", ".docx", ".xlsx", ".xlsm", ".pptx"})
MAX_BYTES = 50 * 1024 * 1024
+# Only the python tool can open these, so they get the larger cap its data files need.
+TOOL_ONLY_EXTENSIONS = frozenset(
+ ".parquet .feather .arrow .orc .dta .sas7bdat .xpt .mat .npy .npz .safetensors .sqlite .sqlite3"
+ " .db .gpkg .mbtiles .duckdb .zip .jar .whl .apk .tar .gz .tgz .bz2 .tbz2 .tbz .xz .txz .lzma"
+ " .epub .mobi .fb2 .cbz .xps .oxps .docm .dotx .dotm .potx .potm .ppsm .odp .odg .vsdx .stl .3mf"
+ " .ply .glb .kmz .ttf .otf .ttc .woff .psd .ico .icns .cur .tga .dds .pcx .ppm .pgm .pbm .pnm"
+ " .qoi .jp2 .j2k .xbm .xpm .sgi .fits".split()
+)
+TOOL_ONLY_MAX_BYTES = 200 * 1024 * 1024
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
_SWEEP_GRACE_SECONDS = 3600
_SWEEP_INTERVAL_SECONDS = 600
@@ -77,8 +86,13 @@ def path_for(attachment: object) -> Optional[Path]:
return path if path.is_file() else None
-def save(chunks: Iterable[bytes]) -> tuple[str, int]:
- """Store streamed bytes under their hash: (sha256, size). Raises TooLarge past MAX_BYTES."""
+def max_bytes(filename: str) -> int:
+ suffix = PurePath(filename).suffix.lower()
+ return TOOL_ONLY_MAX_BYTES if suffix in TOOL_ONLY_EXTENSIONS else MAX_BYTES
+
+
+def save(chunks: Iterable[bytes], limit: int = MAX_BYTES) -> tuple[str, int]:
+ """Store streamed bytes under their hash: (sha256, size). Raises TooLarge past ``limit``."""
directory = ensure_account_dir(originals_dir())
tmp_path = directory / f".{uuid.uuid4().hex}.tmp"
digest = hashlib.sha256()
@@ -87,7 +101,7 @@ def save(chunks: Iterable[bytes]) -> tuple[str, int]:
with open(tmp_path, "wb") as handle:
for chunk in chunks:
size += len(chunk)
- if size > MAX_BYTES:
+ if size > limit:
raise TooLarge()
digest.update(chunk)
handle.write(chunk)
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 2e63e783cbe..2201be4c9a2 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -11455,6 +11455,8 @@ def _holds_no_user_files(target: str, owner: "str | None" = None) -> bool:
# a real file there is the user's like any other.
if _is_spill_artifact(target, parent, name):
continue
+ if _is_attachment_copy(target, parent, name):
+ continue
return False
budget -= 1
if budget <= 0:
@@ -19800,6 +19802,114 @@ def _quiet_unlink(path: str, dir_fd = None) -> None:
pass
+# Hidden, like the spill directory: a project chat's workdir can be the user's own folder.
+_ATTACHMENTS_DIR = ".unsloth_attachments"
+_ATTACHMENT_PREFIX_LEN = 12
+_ATTACHMENT_NAME_BYTES = 80
+_UNSAFE_NAME_CHARS = re.compile(r'[\x00-\x1f\x7f/\\:*?"<>|]')
+# Windows device names stay reserved with any extension (NUL.tar.gz is NUL).
+_RESERVED_NAME = re.compile(r"(?:CON|PRN|AUX|NUL|COM\d|LPT\d)", re.IGNORECASE)
+
+
+def sandbox_attachment_path(sha256: str, name: str) -> str:
+ """Mirrored by sandboxAttachmentPath in the frontend's sandbox-attachments.ts."""
+ base = _UNSAFE_NAME_CHARS.sub("_", name or "").strip(" .") or "attachment"
+ # In bytes: filesystems cap a name at 255, and macOS stores decomposed text that can triple it.
+ if len(base.encode()) > _ATTACHMENT_NAME_BYTES:
+ stem, ext = os.path.splitext(base)
+ ext = ext if len(ext.encode()) <= 16 else ""
+ room = _ATTACHMENT_NAME_BYTES - len(ext.encode())
+ # Stripped again so the basename the frontend sends back derives this same path.
+ base = (stem.encode()[:room].decode("utf-8", "ignore").rstrip(" .") or "attachment") + ext
+ if _RESERVED_NAME.fullmatch(base.split(".", 1)[0].rstrip(" ")):
+ base = "_" + base
+ return f"{_ATTACHMENTS_DIR}/{sha256[:_ATTACHMENT_PREFIX_LEN]}/{base}"
+
+
+def materialize_sandbox_attachments(
+ session_id: "str | None", attachments: "list[tuple[str, str]]"
+) -> None:
+ """Copy chat attachment originals into the sandbox, leaving one already there so edits survive."""
+ from core import chat_originals
+ with _session_in_flight(session_id):
+ workdir = _get_workdir(session_id)
+ for sha256, name in attachments:
+ source = chat_originals.originals_dir() / sha256
+ if not source.is_file():
+ continue
+ try:
+ _install_attachment_copy(workdir, sandbox_attachment_path(sha256, name), source)
+ except (OSError, ValueError):
+ logger.warning(
+ "could not copy attachment %s into the sandbox", sha256, exc_info = True
+ )
+
+
+def _install_attachment_copy(workdir: str, relative: str, source: Path) -> None:
+ """The spill writer's discipline: no link followed, and `os.link` never replaces a name."""
+ *dirs, name = relative.split("/")
+ tmp = f".tmp-{uuid.uuid4().hex[:12]}"
+ flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+ if not _DIR_FD_WRITES:
+ target = workdir
+ for part in dirs:
+ target = os.path.join(target, part)
+ if os.path.islink(target):
+ return
+ os.makedirs(target, mode = 0o700, exist_ok = True)
+ if os.path.realpath(target) != os.path.join(os.path.realpath(workdir), *dirs):
+ return
+ if os.path.lexists(os.path.join(target, name)):
+ return
+ tmp = os.path.join(target, tmp)
+ try:
+ with (
+ open(source, "rb") as src,
+ os.fdopen(os.open(tmp, flags | getattr(os, "O_NOFOLLOW", 0), 0o600), "wb") as out,
+ ):
+ shutil.copyfileobj(src, out, 1 << 20)
+ with contextlib.suppress(FileExistsError):
+ os.link(tmp, os.path.join(target, name))
+ finally:
+ _quiet_unlink(tmp)
+ return
+ fds = [os.open(workdir, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)]
+ try:
+ for part in dirs:
+ with contextlib.suppress(FileExistsError):
+ os.mkdir(part, 0o700, dir_fd = fds[-1])
+ fds.append(os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd = fds[-1]))
+ with contextlib.suppress(FileNotFoundError):
+ os.stat(name, dir_fd = fds[-1], follow_symlinks = False)
+ return
+ try:
+ with (
+ open(source, "rb") as src,
+ os.fdopen(os.open(tmp, flags, 0o600, dir_fd = fds[-1]), "wb") as out,
+ ):
+ shutil.copyfileobj(src, out, 1 << 20)
+ with contextlib.suppress(FileExistsError):
+ os.link(tmp, name, src_dir_fd = fds[-1], dst_dir_fd = fds[-1])
+ finally:
+ _quiet_unlink(tmp, dir_fd = fds[-1])
+ finally:
+ for fd in fds:
+ os.close(fd)
+
+
+def _is_attachment_copy(sandbox: str, parent: str, name: str) -> bool:
+ prefix = os.path.basename(parent)
+ path = os.path.join(parent, name)
+ if (
+ os.path.dirname(parent) != os.path.join(sandbox, _ATTACHMENTS_DIR)
+ or not re.fullmatch(r"[0-9a-f]{12}", prefix)
+ or os.path.islink(path)
+ ):
+ return False
+ digest = _file_digest(path)
+ return digest is not None and digest.startswith(prefix)
+
+
def _forget_spill_record(path: str) -> None:
"""Drop the record for a sandbox that has been removed. See `_spill_records_dir`."""
try:
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 9053d4a3ce9..364b3b9a5f2 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -2126,6 +2126,11 @@ def _normalize_permission_mode(value: Any) -> Any:
return value
+class SandboxAttachment(BaseModel):
+ sha256: str = Field(..., pattern = r"^[0-9a-f]{64}$")
+ name: str = Field(..., max_length = 1024)
+
+
class ChatCompletionRequest(BaseModel):
"""OpenAI-compatible chat completion request.
@@ -2437,6 +2442,14 @@ class ChatCompletionRequest(BaseModel):
None,
description = "[x-unsloth] Session/thread ID for scoping tool execution sandbox.",
)
+ sandbox_attachments: Optional[list[SandboxAttachment]] = Field(
+ None,
+ max_length = 64,
+ description = (
+ "[x-unsloth] Chat attachment originals (hashes from POST /api/chat/attachment-originals) "
+ "to copy into the session sandbox before the tool loop, when the python tool is enabled."
+ ),
+ )
thread_id: Optional[str] = Field(
None,
description = "[x-unsloth] Conversation ID for scoping stateful tool sessions (e.g. stdio MCP); stays per-thread where session_id may be shared project-wide.",
diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py
index 6c73367e45b..361dd44cb48 100644
--- a/studio/backend/routes/chat_history.py
+++ b/studio/backend/routes/chat_history.py
@@ -982,7 +982,7 @@ def _decode_attachment_base64(payload: str) -> bytes:
_ATTACHMENT_TAG_RE = re.compile(r"<attachment name=[^\n]*>\n(.*)\n</attachment>", re.DOTALL)
-_ATTACHMENT_LABEL_RE = re.compile(r"\[(?:PDF|DOCX|HTML|ODS|ODT|XLSX|PPTX): [^\n]*\]\n")
+_ATTACHMENT_LABEL_RE = re.compile(r"\[(?:PDF|DOCX|HTML|ODS|ODT|XLSX|PPTX|RTF): [^\n]*\]\n")
def _attachment_body_text(text: str) -> str:
@@ -1017,14 +1017,14 @@ def _safe_image_media_type(media_type: str) -> str:
def upload_attachment_original(
file: UploadFile = File(...), current_subject: str = Depends(get_current_subject)
) -> dict:
- """Store a chat document's original file; the message records the returned hash. Sync, so
- disk writes run in the threadpool."""
- from pathlib import PurePath
-
- if PurePath(file.filename or "").suffix.lower() not in chat_originals.EXTENSIONS:
- raise HTTPException(status_code = 415, detail = "Only documents keep their original file")
+ """Store an attachment's original file; the message records the returned hash. Any name is
+ taken, since the python tool reads every text and code format. Sync, so disk writes run in
+ the threadpool."""
try:
- sha256, size = chat_originals.save(iter(lambda: file.file.read(1024 * 1024), b""))
+ sha256, size = chat_originals.save(
+ iter(lambda: file.file.read(1024 * 1024), b""),
+ chat_originals.max_bytes(file.filename or ""),
+ )
except chat_originals.TooLarge:
raise HTTPException(status_code = 413, detail = f"{file.filename} is too large")
chat_originals.sweep()
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 687b0b168aa..ee0c82474ac 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -4298,6 +4298,25 @@ def _effective_enable_tools(payload) -> Optional[bool]:
return payload.enable_tools
+async def _materialize_sandbox_attachments(payload) -> None:
+ enabled = payload.enabled_tools
+ if (
+ not payload.sandbox_attachments
+ or not _effective_enable_tools(payload)
+ or (enabled is not None and "python" not in enabled)
+ ):
+ return
+ from starlette.concurrency import run_in_threadpool
+
+ from core.inference.tools import materialize_sandbox_attachments
+
+ await run_in_threadpool(
+ materialize_sandbox_attachments,
+ payload.session_id,
+ [(item.sha256, item.name) for item in payload.sandbox_attachments],
+ )
+
+
def _tools_on_by_launcher_default_only(payload) -> bool:
"""True when tools are on ONLY because of the launcher's tools-on default:
no CLI override is installed and the request itself asked for nothing."""
@@ -25582,6 +25601,7 @@ async def produce_openai_chat_completions(
# local path flattens them, so a part neither can serve has to be refused while both are
# still reachable. Otherwise it is dropped in silence and the answer looks legitimate.
_reject_unsupported_content_parts(payload)
+ await _materialize_sandbox_attachments(payload)
# ── External provider routing ────────────────────────────────
# encrypted_api_key is optional -- local providers (llama.cpp / vLLM / Ollama) may run without auth.
diff --git a/studio/backend/tests/test_chat_attachments.py b/studio/backend/tests/test_chat_attachments.py
index 027e3057c07..fa4450e3421 100644
--- a/studio/backend/tests/test_chat_attachments.py
+++ b/studio/backend/tests/test_chat_attachments.py
@@ -695,3 +695,30 @@ def test_a_sweep_rechecks_a_stale_snapshot_before_removing(tmp_path, monkeypatch
studio_db.delete_chat_threads(["thread-1"])
assert chat_originals.sweep(force = True) == 1
assert not path.exists()
+
+
+def test_originals_upload_takes_any_name_and_caps_tool_only_files_higher(tmp_path, monkeypatch):
+ import io
+
+ from fastapi import UploadFile
+
+ from core import chat_originals
+
+ _reset_studio_db(tmp_path, monkeypatch)
+ monkeypatch.setattr(chat_originals, "MAX_BYTES", 4)
+ monkeypatch.setattr(chat_originals, "TOOL_ONLY_MAX_BYTES", 8)
+
+ def upload(name: str, data: bytes) -> dict:
+ file = UploadFile(io.BytesIO(data), filename = name)
+ return chat_history.upload_attachment_original(file, current_subject = "unsloth")
+
+ assert upload("data.csv", b"a,b")["sizeBytes"] == 3
+ assert upload("t.PARQUET", b"12345678")["sizeBytes"] == 8
+ for name, data in (("a.pdf", b"12345"), ("notes.rtf", b"12345"), ("t.parquet", b"123456789")):
+ with pytest.raises(HTTPException) as refused:
+ upload(name, data)
+ assert refused.value.status_code == 413, name
+
+
+def test_rtf_label_is_stripped_from_the_stored_text():
+ assert chat_history._attachment_body_text("[RTF: notes.rtf]\nhello") == "hello"
diff --git a/studio/backend/tests/test_sandbox_files_and_storage_roots.py b/studio/backend/tests/test_sandbox_files_and_storage_roots.py
index f92d625c995..9d29bf63d01 100644
--- a/studio/backend/tests/test_sandbox_files_and_storage_roots.py
+++ b/studio/backend/tests/test_sandbox_files_and_storage_roots.py
@@ -3271,6 +3271,101 @@ def test_an_interrupted_move_is_not_read_as_a_collision(tmp_path, monkeypatch):
assert (root / "__LOCALID_part111" / "second.csv").is_file(), "the retry never happened"
[email protected]("dir_fd_writes", [True, False])
+def test_attachments_are_copied_into_the_sandbox_once(tmp_path, monkeypatch, dir_fd_writes):
+ from core import chat_originals
+
+ tools = _shared_setup_1(monkeypatch, tmp_path)
+ monkeypatch.setattr(tools, "_DIR_FD_WRITES", tools._DIR_FD_WRITES and dir_fd_writes)
+ monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "home"))
+ sheet, _ = chat_originals.save([b"a,b\n"])
+ deck, _ = chat_originals.save([b"slides"], chat_originals.max_bytes("deck.odp"))
+ session = "__LOCALID_attach1"
+ tools.materialize_sandbox_attachments(
+ session, [(sheet, "data.csv"), (deck, "../deck.odp"), ("0" * 64, "gone.csv")]
+ )
+ long_name = "季度" * 45 + ".xlsx"
+ tools.materialize_sandbox_attachments(session, [(sheet, long_name)])
+ workdir = Path(tools.get_sandbox_workdir(session))
+ copy = workdir / tools.sandbox_attachment_path(sheet, "data.csv")
+ assert copy.read_bytes() == b"a,b\n"
+ assert tools.sandbox_attachment_path(deck, "../deck.odp").endswith(f"{deck[:12]}/_deck.odp")
+ assert (workdir / tools.sandbox_attachment_path(deck, "../deck.odp")).read_bytes() == b"slides"
+ long_copy = workdir / tools.sandbox_attachment_path(sheet, long_name)
+ assert long_copy.read_bytes() == b"a,b\n" and long_copy.suffix == ".xlsx"
+ assert 70 < len(long_copy.name.encode()) <= 80
+ assert tools.session_sandbox_has_files(session) is False
+ copy.write_bytes(b"edited")
+ tools.materialize_sandbox_attachments(session, [(sheet, "data.csv")])
+ assert copy.read_bytes() == b"edited"
+ assert tools.session_sandbox_has_files(session) is True
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ other = "__LOCALID_attach2"
+ linked = Path(tools.get_sandbox_workdir(other)) / tools._ATTACHMENTS_DIR
+ linked.symlink_to(outside, target_is_directory = True)
+ tools.materialize_sandbox_attachments(other, [(sheet, "data.csv")])
+ assert list(outside.iterdir()) == []
+
+
+def test_sandbox_attachment_paths_match_the_frontend():
+ """Same table as sandbox-attachments.test.ts: the client notes these paths to the model."""
+ from core.inference.tools import sandbox_attachment_path
+
+ sha = "ab" * 32
+ for name, base in (
+ ("data.csv", "data.csv"),
+ ("../deck?.pptx", "_deck_.pptx"),
+ (" .hidden. ", "hidden"),
+ ("季度" * 45 + ".xlsx", "季度" * 12 + "季.xlsx"),
+ ("a" * 79 + " ." + "x" * 20, "a" * 79),
+ ("b" * 10 + "." + "x" * 100, "b" * 10),
+ ("..", "attachment"),
+ ("e" * 100, "e" * 80),
+ ("c" * 78 + ". ." + "z" * 20, "c" * 78),
+ ("CON.csv", "_CON.csv"),
+ ("nul.tar.gz", "_nul.tar.gz"),
+ ("com1", "_com1"),
+ ("CONSOLE.txt", "CONSOLE.txt"),
+ ):
+ path = sandbox_attachment_path(sha, name)
+ assert path == f".unsloth_attachments/abababababab/{base}", name
+ assert sandbox_attachment_path(sha, base) == path, name
+
+
+def test_attachments_are_copied_only_for_the_python_tool(monkeypatch):
+ import asyncio
+ import inspect
+ from types import SimpleNamespace
+
+ from core.inference import tools
+ from routes import inference
+
+ calls = []
+ monkeypatch.setattr(tools, "materialize_sandbox_attachments", lambda *args: calls.append(args))
+ item = SimpleNamespace(sha256 = "a" * 64, name = "data.csv")
+ for enable_tools, enabled_tools, attachments in (
+ (True, None, [item]),
+ (True, ["python"], [item]),
+ (True, ["web_search"], [item]),
+ (False, None, [item]),
+ (True, None, None),
+ ):
+ payload = SimpleNamespace(
+ enable_tools = enable_tools,
+ enabled_tools = enabled_tools,
+ sandbox_attachments = attachments,
+ session_id = "s",
+ )
+ asyncio.run(inference._materialize_sandbox_attachments(payload))
+ assert calls == [("s", [("a" * 64, "data.csv")])] * 2
+ source = inspect.getsource(inference.produce_openai_chat_completions)
+ assert source.count("await _materialize_sandbox_attachments(payload)") == 1
+ assert source.index("_materialize_sandbox_attachments") < source.index(
+ "_proxy_to_external_provider"
+ )
+
+
def test_a_delete_without_the_switch_says_what_it_kept(tmp_path, monkeypatch):
"""Surfaces other than the sidebar never offer the choice, and after the
delete the folder is unreachable, so the route reports it."""
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx
index 7a441a5b099..ae507c0ff33 100644
--- a/studio/frontend/src/components/assistant-ui/thread.tsx
+++ b/studio/frontend/src/components/assistant-ui/thread.tsx
@@ -263,6 +263,7 @@ import {
type PlusMenuItemId,
usePlusMenuPrefsStore,
writeComposerDraft,
+ normalizeChatImage,
} from "@/features/chat";
import {
applySentTextGuard,
@@ -3027,7 +3028,9 @@ const Composer: FC<{
prev.modelLoading ||
state.params.checkpoint !== prev.params.checkpoint ||
state.residentCheckpoint !== prev.residentCheckpoint ||
- state.loadedIsMultimodal !== prev.loadedIsMultimodal
+ state.loadedIsMultimodal !== prev.loadedIsMultimodal ||
+ state.codeToolsEnabled !== prev.codeToolsEnabled ||
+ state.supportsTools !== prev.supportsTools
) {
void retry();
}
@@ -3405,7 +3408,9 @@ const Composer: FC<{
const intent = intents[index]!;
let file: File;
try {
- file = await nativeAttachmentIntentToFile(intent);
+ file = await normalizeChatImage(
+ await nativeAttachmentIntentToFile(intent),
+ );
} catch (error) {
// Report once below rather than one toast per file: a whole batch
// can go unreadable at once (volume ejected, tokens expired).
diff --git a/studio/frontend/src/components/image-dropzone.tsx b/studio/frontend/src/components/image-dropzone.tsx
index 2bcb62811c5..6dc12b9b8e0 100644
--- a/studio/frontend/src/components/image-dropzone.tsx
+++ b/studio/frontend/src/components/image-dropzone.tsx
@@ -19,8 +19,6 @@ import {
import { cn } from "@/lib/utils";
import { toast } from "@/lib/toast";
-// All registerNativeAttachmentPath takes. The picker itself accepts image/*,
-// so name the droppable formats instead of surfacing the backend's refusal.
const NATIVE_IMAGE_EXTS = ["jpg", "jpeg", "png", "webp", "gif"];
/** Shared image picker that returns a data URL. */
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 1278c45a0a9..412ed360759 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -16,6 +16,7 @@ import {
serverTuningLoadPayload,
} from "../lib/server-tuning-fields";
import { authFetch, getAuthToken } from "@/features/auth";
+import { withSandboxAttachmentPaths } from "../sandbox-attachments";
import { prepareHfTokenForUse } from "@/features/hf-auth";
import { DOWNLOAD_KIND } from "@/features/hub/download-manager/constants";
import {
@@ -4972,9 +4973,14 @@ export function createOpenAIStreamAdapter(
runtime.preserveThinking
);
const survivingMessages = pruneOutboundHistory(messages, replayReasoning);
+ const { messages: renderedMessages, sandboxAttachments } =
+ supportsStudioToolsForThisTurn &&
+ studioLocalCodeTools.includes("python")
+ ? withSandboxAttachmentPaths(survivingMessages)
+ : { messages: survivingMessages, sandboxAttachments: [] };
// toOpenAIMessages emits assistant tool_calls plus role="tool" follow-ups; the backend Gemini
// translator rebuilds the functionCall/functionResponse parts.
- let outboundMessages = survivingMessages
+ let outboundMessages = renderedMessages
.flatMap((message) => toOpenAIMessages(message, replayReasoning))
.filter((message): message is NonNullable<typeof message> =>
Boolean(message),
@@ -6357,6 +6363,9 @@ export function createOpenAIStreamAdapter(
...(sandboxSessionId
? { session_id: sandboxSessionId }
: {}),
+ ...(sandboxAttachments.length > 0
+ ? { sandbox_attachments: sandboxAttachments }
+ : {}),
...(resolvedThreadId
? { thread_id: resolvedThreadId }
: {}),
@@ -6504,6 +6513,9 @@ export function createOpenAIStreamAdapter(
video_base64: findLatestUserVideoBase64(currentTurnMessages),
cancel_id: cancelId,
...(sandboxSessionId ? { session_id: sandboxSessionId } : {}),
+ ...(sandboxAttachments.length > 0
+ ? { sandbox_attachments: sandboxAttachments }
+ : {}),
...(resolvedThreadId ? { thread_id: resolvedThreadId } : {}),
...(useAdapter === undefined ? {} : { use_adapter: useAdapter }),
...localReasoningFields,
diff --git a/studio/frontend/src/features/chat/attachment-content.ts b/studio/frontend/src/features/chat/attachment-content.ts
index ffb93e6fad1..88bf85c1a5b 100644
--- a/studio/frontend/src/features/chat/attachment-content.ts
+++ b/studio/frontend/src/features/chat/attachment-content.ts
@@ -17,7 +17,11 @@ import {
import {
OPEN_DOCUMENT_SPREADSHEET_MIME,
OPEN_DOCUMENT_TEXT_MIME,
+ RTF_MIMES,
+ isRtfAttachmentName,
+ isToolOnlyAttachmentName,
} from "./open-document-accept";
+import { readRtfAttachmentContent } from "./rtf";
export type AttachmentTextLabel =
| "PDF"
@@ -26,7 +30,8 @@ export type AttachmentTextLabel =
| "ODS"
| "ODT"
| "XLSX"
- | "PPTX";
+ | "PPTX"
+ | "RTF";
export { TEXT_ATTACHMENT_ACCEPT };
@@ -45,7 +50,7 @@ const PDF_ATTACHMENT_RE = /\.pdf$/i;
const DOCX_ATTACHMENT_RE = /\.docx$/i;
const HTML_ATTACHMENT_RE = /\.x?html?$/i;
const OPEN_DOCUMENT_ATTACHMENT_RE = /\.(ods|odt)$/i;
-const LABELLED_ATTACHMENT_TEXT_RE = /^\[(PDF|DOCX|HTML|ODS|ODT|XLSX|PPTX): [^\n]*\]\n/;
+const LABELLED_ATTACHMENT_TEXT_RE = /^\[(PDF|DOCX|HTML|ODS|ODT|XLSX|PPTX|RTF): [^\n]*\]\n/;
const ATTACHMENT_TAG_OPEN_RE = /^<attachment name=[^\n]*>\n/;
const ATTACHMENT_TAG_CLOSE = "\n</attachment>";
// Both wrappers start on the first line, so only a prefix is matched against.
@@ -480,6 +485,16 @@ export function isOpenDocumentAttachment(
);
}
+export function isRtfAttachment(
+ name: string | undefined,
+ contentType: string | undefined,
+): boolean {
+ return (
+ RTF_MIMES.includes(contentType?.toLowerCase() ?? "") ||
+ isRtfAttachmentName(name ?? "")
+ );
+}
+
// CompositeAttachmentAdapter selects the first matching accept string. Text comes before the
// document-specific adapters, so previews must apply the same MIME-or-extension match
// before looking at PDF/DOCX/HTML names.
@@ -1011,6 +1026,17 @@ export async function readAttachmentText(
);
return { label, text, truncated: false };
}
+ if (isRtfAttachment(name, contentType)) {
+ const { label, text } = await readRtfAttachmentContent(file, name);
+ return { label, text, truncated: false };
+ }
+ if (isToolOnlyAttachmentName(name)) {
+ return {
+ label: null,
+ text: `${name} has no preview: only the python tool can read it.`,
+ truncated: false,
+ };
+ }
return { label: null, ...(await readBoundedText(file)) };
}
diff --git a/studio/frontend/src/features/chat/attachment-originals.ts b/studio/frontend/src/features/chat/attachment-originals.ts
index bd6bf776979..28ebbd9bda2 100644
--- a/studio/frontend/src/features/chat/attachment-originals.ts
+++ b/studio/frontend/src/features/chat/attachment-originals.ts
@@ -37,9 +37,16 @@ export async function withAttachmentOriginal(
complete: CompleteAttachment,
temporary: boolean,
epoch: number,
+ forPythonTool: boolean,
): Promise<CompleteAttachment> {
- const upload = complete.type === "document" ? originalUpload(pending.file) : null;
+ // Any other document is kept only for the python tool, which gets a copy in its sandbox: an
+ // upload no one reads would only delay the send.
+ const upload =
+ complete.type === "document" && !attachmentOriginal(complete)
+ ? (originalUpload(pending.file) ?? (forPythonTool ? pending.file : null))
+ : null;
if (!upload) return complete;
+ // A temporary chat keeps its files in memory, python tool or not: it promises nothing is saved.
if (temporary) return { ...complete, file: pending.file } as CompleteAttachment;
try {
const original: ChatAttachmentOriginal = await uploadChatAttachmentOriginal(upload, epoch);
@@ -57,7 +64,9 @@ export async function persistAttachmentOriginals(
(attachments ?? []).map(async (attachment) => {
const { file, ...rest } = attachment as CompleteAttachment & { file?: unknown };
if (file === undefined) return attachment;
- const upload = file instanceof File && !attachmentOriginal(rest) ? originalUpload(file) : null;
+ // A file was kept only because a send wanted it stored (a python turn keeps any document).
+ const upload =
+ file instanceof File && !attachmentOriginal(rest) ? (originalUpload(file) ?? file) : null;
if (!upload) return rest as CompleteAttachment;
try {
const original: ChatAttachmentOriginal = await uploadChatAttachmentOriginal(upload, epoch);
@@ -68,3 +77,11 @@ export async function persistAttachmentOriginals(
}),
);
}
+
+// Originals no message names yet are swept an hour after upload; sending one staged half that long
+// uploads it again, which restarts its clock.
+const STAGED_UPLOAD_MAX_AGE_MS = 30 * 60 * 1000;
+
+export function reuseStagedUpload(stagedAt: number | undefined, now = Date.now()): boolean {
+ return stagedAt !== undefined && now - stagedAt < STAGED_UPLOAD_MAX_AGE_MS;
+}
diff --git a/studio/frontend/src/features/chat/image-normalize.ts b/studio/frontend/src/features/chat/image-normalize.ts
new file mode 100644
index 00000000000..a8705000245
--- /dev/null
+++ b/studio/frontend/src/features/chat/image-normalize.ts
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+export const CHAT_IMAGE_MIMES = [
+ "image/jpeg",
+ "image/png",
+ "image/webp",
+ "image/gif",
+ "image/heic",
+ "image/heif",
+ "image/avif",
+ "image/bmp",
+ "image/tiff",
+];
+
+const CONVERTED_IMAGE_TYPES: Record<string, string> = {
+ heic: "image/jpeg",
+ heif: "image/jpeg",
+ avif: "image/png",
+ bmp: "image/png",
+ tif: "image/png",
+ tiff: "image/png",
+};
+const CONVERTED_IMAGE_EXTENSIONS = Object.keys(CONVERTED_IMAGE_TYPES).map(
+ (extension) => `.${extension}`,
+);
+
+export const CHAT_IMAGE_EXTENSIONS = [
+ ".jpg,.jpeg,.png,.webp,.gif",
+ ...CONVERTED_IMAGE_EXTENSIONS,
+].join(",");
+export const CHAT_IMAGE_ACCEPT = [
+ ...CHAT_IMAGE_MIMES,
+ ...CONVERTED_IMAGE_EXTENSIONS,
+].join(",");
+
+function imageKind(file: { name: string; type: string }): string | null {
+ const mime = file.type.toLowerCase();
+ if (CHAT_IMAGE_MIMES.includes(mime)) {
+ return mime.slice("image/".length);
+ }
+ const extension = file.name.toLowerCase().split(".").pop() ?? "";
+ return Object.hasOwn(CONVERTED_IMAGE_TYPES, extension) ? extension : null;
+}
+
+export function isChatImageFile(file: { name: string; type: string }): boolean {
+ return imageKind(file) !== null;
+}
+
+export function convertedImageType(file: {
+ name: string;
+ type: string;
+}): string | null {
+ const kind = imageKind(file);
+ return kind === null ? null : (CONVERTED_IMAGE_TYPES[kind] ?? null);
+}
+
+const MAX_CONVERTED_IMAGE_BYTES = 20 * 1024 * 1024;
+// iOS Safari, the one browser that decodes HEIC, refuses canvases past 16.7 megapixels.
+const MAX_CONVERTED_IMAGE_PIXELS = 16_777_216;
+
+export async function normalizeChatImage(file: File): Promise<File> {
+ const type = convertedImageType(file);
+ if (type === null) {
+ return file;
+ }
+ const url = URL.createObjectURL(file);
+ try {
+ const image = new Image();
+ image.src = url;
+ try {
+ await image.decode();
+ } catch {
+ throw new Error(
+ `This app can't read ${file.name}. Convert it to JPEG or PNG and attach it again.`,
+ );
+ }
+ const scale = Math.min(
+ 1,
+ Math.sqrt(
+ MAX_CONVERTED_IMAGE_PIXELS / (image.naturalWidth * image.naturalHeight),
+ ),
+ );
+ const canvas = document.createElement("canvas");
+ canvas.width = Math.max(1, Math.floor(image.naturalWidth * scale));
+ canvas.height = Math.max(1, Math.floor(image.naturalHeight * scale));
+ const context = canvas.getContext("2d");
+ if (!context) {
+ throw new Error(`Could not convert ${file.name}.`);
+ }
+ if (type === "image/jpeg") {
+ context.fillStyle = "#fff";
+ context.fillRect(0, 0, canvas.width, canvas.height);
+ }
+ context.drawImage(image, 0, 0, canvas.width, canvas.height);
+ const blob = await new Promise<Blob | null>((resolve) =>
+ canvas.toBlob(resolve, type, 0.92),
+ );
+ if (!blob) {
+ throw new Error(`Could not convert ${file.name}.`);
+ }
+ if (blob.size > MAX_CONVERTED_IMAGE_BYTES) {
+ throw new Error(`${file.name} is over 20 MB once converted.`);
+ }
+ const extension = type === "image/jpeg" ? ".jpg" : ".png";
+ return new File([blob], file.name.replace(/\.[^.]*$/, "") + extension, {
+ type,
+ lastModified: file.lastModified,
+ });
+ } finally {
+ URL.revokeObjectURL(url);
+ }
+}
diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.ts
index 7b37086607e..abb9d8744ec 100644
--- a/studio/frontend/src/features/chat/index.ts
+++ b/studio/frontend/src/features/chat/index.ts
@@ -292,6 +292,7 @@ export {
repackDocxPreviewArchive,
truncateAttachmentPreviewText,
} from "./attachment-content";
+export { normalizeChatImage } from "./image-normalize";
export {
ATTACHMENT_KIND_ICON_CLASS,
ATTACHMENT_KIND_ICONS,
diff --git a/studio/frontend/src/features/chat/open-document-accept.ts b/studio/frontend/src/features/chat/open-document-accept.ts
index f3451ab790b..77b5501fa95 100644
--- a/studio/frontend/src/features/chat/open-document-accept.ts
+++ b/studio/frontend/src/features/chat/open-document-accept.ts
@@ -18,3 +18,32 @@ export function isOpenDocumentAttachmentName(filename: string): boolean {
lower.endsWith(extension),
);
}
+
+export const RTF_ATTACHMENT_EXTENSIONS = ".rtf";
+export const RTF_MIMES = ["application/rtf", "text/rtf"];
+export const RTF_ATTACHMENT_ACCEPT = [
+ RTF_ATTACHMENT_EXTENSIONS,
+ ...RTF_MIMES,
+].join(",");
+
+export function isRtfAttachmentName(filename: string): boolean {
+ return filename.toLowerCase().endsWith(RTF_ATTACHMENT_EXTENSIONS);
+}
+
+// Matched by extension only: their MIME types are missing or shared with unrelated files.
+// Keep in sync with TOOL_ONLY_ATTACHMENT_EXTS in native_path_policy.rs.
+export const TOOL_ONLY_ATTACHMENT_EXTENSIONS = [
+ ".parquet,.feather,.arrow,.orc,.dta,.sas7bdat,.xpt,.mat,.npy,.npz,.safetensors",
+ ".sqlite,.sqlite3,.db,.gpkg,.mbtiles,.duckdb",
+ ".zip,.jar,.whl,.apk,.tar,.gz,.tgz,.bz2,.tbz2,.tbz,.xz,.txz,.lzma",
+ ".epub,.mobi,.fb2,.cbz,.xps,.oxps,.docm,.dotx,.dotm,.potx,.potm,.ppsm,.odp,.odg,.vsdx",
+ ".stl,.3mf,.ply,.glb,.kmz,.ttf,.otf,.ttc,.woff",
+ ".psd,.ico,.icns,.cur,.tga,.dds,.pcx,.ppm,.pgm,.pbm,.pnm,.qoi,.jp2,.j2k,.xbm,.xpm,.sgi,.fits",
+].join(",");
+
+export function isToolOnlyAttachmentName(name: string): boolean {
+ const lower = name.toLowerCase();
+ return TOOL_ONLY_ATTACHMENT_EXTENSIONS.split(",").some((ext) =>
+ lower.endsWith(ext),
+ );
+}
diff --git a/studio/frontend/src/features/chat/rtf.ts b/studio/frontend/src/features/chat/rtf.ts
new file mode 100644
index 00000000000..97ae81e2368
--- /dev/null
+++ b/studio/frontend/src/features/chat/rtf.ts
@@ -0,0 +1,329 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import {
+ MAX_OPEN_DOCUMENT_ARCHIVE_BYTES,
+ MAX_OPEN_DOCUMENT_XML_BYTES,
+} from "./open-document";
+
+// A call, not a const: read at module scope, an import the chat barrel also reaches can still be
+// in its temporal dead zone when this module runs.
+function maxTextLength(): number {
+ return MAX_OPEN_DOCUMENT_XML_BYTES;
+}
+
+const MAX_GROUP_DEPTH = 1024;
+const LATIN1 = new TextDecoder("latin1");
+
+const SKIPPED_DESTINATIONS = new Set([
+ "fonttbl",
+ "colortbl",
+ "stylesheet",
+ "listtable",
+ "listoverridetable",
+ "revtbl",
+ "rsidtbl",
+ "filetbl",
+ "info",
+ "pict",
+ "nonshppict",
+ "object",
+ "objdata",
+ "fldinst",
+ "sn",
+ "sv",
+ "pn",
+ "header",
+ "headerl",
+ "headerr",
+ "headerf",
+ "footer",
+ "footerl",
+ "footerr",
+ "footerf",
+ "footnote",
+ "annotation",
+ "xe",
+ "tc",
+ "bkmkstart",
+ "bkmkend",
+]);
+
+const CONTROL_WORD_TEXT = new Map<string, string>([
+ ["par", "\n"],
+ ["line", "\n"],
+ ["sect", "\n"],
+ ["page", "\n"],
+ ["row", "\n"],
+ ["cell", "\t"],
+ ["nestcell", "\t"],
+ ["tab", "\t"],
+ ["emdash", "—"],
+ ["endash", "–"],
+ ["bullet", "•"],
+ ["lquote", "‘"],
+ ["rquote", "’"],
+ ["ldblquote", "“"],
+ ["rdblquote", "”"],
+ ["emspace", " "],
+ ["enspace", " "],
+ ["qmspace", " "],
+]);
+
+const CHARSET_CODEPAGES: Record<number, number> = {
+ 0: 1252,
+ 77: 10000,
+ 128: 932,
+ 129: 949,
+ 134: 936,
+ 136: 950,
+ 161: 1253,
+ 162: 1254,
+ 163: 1258,
+ 177: 1255,
+ 178: 1256,
+ 186: 1257,
+ 204: 1251,
+ 222: 874,
+ 238: 1250,
+};
+
+const CODEPAGE_ENCODINGS: Record<number, string> = {
+ 874: "windows-874",
+ 932: "shift_jis",
+ 936: "gbk",
+ 949: "euc-kr",
+ 950: "big5",
+ 10000: "macintosh",
+ 65001: "utf-8",
+};
+
+type Group = {
+ skip: boolean;
+ fontTable: boolean;
+ inTable: boolean;
+ uc: number;
+ font: number;
+};
+
+export async function readRtfAttachmentContent(
+ file: File,
+ filename: string,
+): Promise<{ label: "RTF"; text: string }> {
+ if (file.size > MAX_OPEN_DOCUMENT_ARCHIVE_BYTES) {
+ throw new Error(`RTF file is too large: ${filename}`);
+ }
+ const bytes = new Uint8Array(await file.arrayBuffer());
+ if (LATIN1.decode(bytes.subarray(0, 5)) !== "{\\rtf") {
+ throw new Error(`Not an RTF file: ${filename}`);
+ }
+ return { label: "RTF", text: rtfToText(bytes, filename) };
+}
+
+function encodingFor(codepage: number): string {
+ if (codepage >= 1250 && codepage <= 1258) {
+ return `windows-${codepage}`;
+ }
+ return CODEPAGE_ENCODINGS[codepage] ?? "windows-1252";
+}
+
+function rtfToText(bytes: Uint8Array, filename: string): string {
+ const maxLength = maxTextLength();
+ const fonts = new Map<number, number>();
+ let defaultCodepage = 1252;
+ let defaultFont = -1;
+ let group: Group = {
+ skip: false,
+ fontTable: false,
+ inTable: false,
+ uc: 1,
+ font: -1,
+ };
+ const stack: Group[] = [];
+ const parts: string[] = [];
+ let length = 0;
+ let pending: number[] = [];
+ let pendingEncoding = "";
+ let fallbackToSkip = 0;
+ let tableFont = -1;
+
+ const flush = () => {
+ if (pending.length > 0) {
+ parts.push(
+ new TextDecoder(pendingEncoding).decode(new Uint8Array(pending)),
+ );
+ pending = [];
+ }
+ };
+ const append = (text: string) => {
+ parts.push(text);
+ length += text.length;
+ };
+ const emit = (text: string) => {
+ if (!group.skip) {
+ flush();
+ append(text);
+ }
+ };
+ const emitByte = (byte: number) => {
+ if (group.skip) {
+ return;
+ }
+ const charset = fonts.get(group.font < 0 ? defaultFont : group.font);
+ const encoding = encodingFor(
+ charset === undefined || charset === 1
+ ? defaultCodepage
+ : (CHARSET_CODEPAGES[charset] ?? defaultCodepage),
+ );
+ if (encoding !== pendingEncoding) {
+ flush();
+ pendingEncoding = encoding;
+ }
+ pending.push(byte);
+ length++;
+ };
+ // Writers escape a double-byte lead byte but may leave its trail byte literal, even \\ { or }.
+ const emitLiteral = (byte: number) => {
+ if (pending.length > 0) {
+ emitByte(byte);
+ flush();
+ } else {
+ emit(String.fromCharCode(byte));
+ }
+ };
+
+ let index = 0;
+ while (index < bytes.length && length < maxLength) {
+ const byte = bytes[index++];
+ if (byte === 0x7b) {
+ if (stack.length >= MAX_GROUP_DEPTH) {
+ throw new Error(`RTF groups nest too deeply: ${filename}`);
+ }
+ stack.push(group);
+ group = { ...group };
+ fallbackToSkip = 0;
+ } else if (byte === 0x7d) {
+ group = stack.pop() ?? group;
+ fallbackToSkip = 0;
+ } else if (byte === 0x0d || byte === 0x0a) {
+ continue;
+ } else if (byte !== 0x5c) {
+ if (fallbackToSkip > 0) {
+ fallbackToSkip--;
+ } else if (byte >= 0x80) {
+ emitByte(byte);
+ } else if (pending.length > 0) {
+ emitLiteral(byte);
+ } else {
+ const start = index - 1;
+ while (index < bytes.length && isPlainText(bytes[index])) index++;
+ emit(LATIN1.decode(bytes.subarray(start, index)));
+ }
+ } else if (index < bytes.length && isLetter(bytes[index])) {
+ const start = index;
+ while (index < bytes.length && isLetter(bytes[index])) index++;
+ const word = LATIN1.decode(bytes.subarray(start, index));
+ let param: number | null = null;
+ const paramStart = index;
+ if (bytes[index] === 0x2d) index++;
+ while (index < bytes.length && isDigit(bytes[index])) index++;
+ if (index > paramStart && bytes[index - 1] !== 0x2d) {
+ param = Number(LATIN1.decode(bytes.subarray(paramStart, index)));
+ } else {
+ index = paramStart;
+ }
+ if (bytes[index] === 0x20) index++;
+
+ if (word === "bin" && param !== null) {
+ index += Math.max(0, param);
+ } else if (fallbackToSkip > 0) {
+ fallbackToSkip--;
+ } else if (word === "u" && param !== null) {
+ emit(String.fromCharCode(param < 0 ? param + 0x10000 : param));
+ fallbackToSkip = group.uc;
+ } else if (word === "uc" && param !== null) {
+ group.uc = Math.max(0, param);
+ } else if (word === "deff" && param !== null) {
+ defaultFont = param;
+ } else if (word === "ansicpg" && param !== null) {
+ defaultCodepage = param;
+ } else if (word === "f" && param !== null) {
+ if (group.fontTable) {
+ tableFont = param;
+ } else {
+ group.font = param;
+ }
+ } else if (word === "fcharset" && param !== null) {
+ fonts.set(tableFont, param);
+ } else if (word === "plain") {
+ group.font = -1;
+ } else if (word === "intbl" || word === "pard") {
+ group.inTable = word === "intbl";
+ } else if (word === "par" && group.inTable) {
+ emit(" ");
+ } else if (SKIPPED_DESTINATIONS.has(word)) {
+ group.skip = true;
+ group.fontTable ||= word === "fonttbl";
+ } else if (CONTROL_WORD_TEXT.has(word)) {
+ emit(CONTROL_WORD_TEXT.get(word) as string);
+ }
+ } else if (index < bytes.length) {
+ const symbol = bytes[index++];
+ if (symbol === 0x27) {
+ const byteValue = parseInt(
+ String.fromCharCode(bytes[index], bytes[index + 1]),
+ 16,
+ );
+ index += 2;
+ if (fallbackToSkip > 0) {
+ fallbackToSkip--;
+ } else if (!Number.isNaN(byteValue)) {
+ emitByte(byteValue);
+ }
+ } else if (symbol === 0x2a) {
+ group.skip = true;
+ } else if (fallbackToSkip > 0) {
+ fallbackToSkip--;
+ } else if (symbol === 0x0d || symbol === 0x0a) {
+ emit("\n");
+ } else if (symbol === 0x7e) {
+ emit(" ");
+ } else if (symbol === 0x5f) {
+ emit("-");
+ } else if (symbol === 0x5c || symbol === 0x7b || symbol === 0x7d) {
+ emitLiteral(symbol);
+ }
+ }
+ }
+ flush();
+
+ const text = parts
+ .join("")
+ .slice(0, maxLength)
+ .replace(/[^\S\n]+\n/g, "\n")
+ .replace(/ +\t/g, "\t")
+ .replace(/\n{3,}/g, "\n\n")
+ .trim();
+ return length >= maxLength
+ ? `${text}\n\n[Truncated: the document has more text than one attachment carries]`
+ : text;
+}
+
+function isLetter(byte: number): boolean {
+ return (byte >= 0x61 && byte <= 0x7a) || (byte >= 0x41 && byte <= 0x5a);
+}
+
+function isPlainText(byte: number): boolean {
+ return (
+ byte < 0x80 &&
+ byte !== 0x5c &&
+ byte !== 0x7b &&
+ byte !== 0x7d &&
+ byte !== 0x0d &&
+ byte !== 0x0a
+ );
+}
+
+function isDigit(byte: number): boolean {
+ return byte >= 0x30 && byte <= 0x39;
+}
diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx
index 57c3be828c0..8765d8d7295 100644
--- a/studio/frontend/src/features/chat/runtime-provider.tsx
+++ b/studio/frontend/src/features/chat/runtime-provider.tsx
@@ -45,7 +45,11 @@ import {
ThreadAutosaveHandle,
createOpenAIStreamAdapter,
} from "./api/chat-adapter";
-import { CHAT_HISTORY_UPDATED_EVENT } from "./api/chat-api";
+import {
+ CHAT_HISTORY_UPDATED_EVENT,
+ uploadChatAttachmentOriginal,
+} from "./api/chat-api";
+import { selectCodeToolNames } from "./api/code-tool-placement";
import { getResearchThreadState } from "./api/research-api";
import {
cancelChatGenerationRun,
@@ -65,7 +69,12 @@ import {
getDocumentAttachmentSizeError,
getDocxAttachmentError,
} from "./attachment-content";
-import { persistAttachmentOriginals, withAttachmentOriginal } from "./attachment-originals";
+import {
+ type ChatAttachmentOriginal,
+ persistAttachmentOriginals,
+ reuseStagedUpload,
+ withAttachmentOriginal,
+} from "./attachment-originals";
import { AudioAttachmentAdapter } from "./audio-attachment-adapter";
import {
isBinaryPropertyList,
@@ -81,18 +90,34 @@ import {
loadConnectionsEnabled,
loadExternalProviders,
parseExternalModelId,
+ providerModelSupportsStudioTools,
providerModelSupportsVision,
} from "./external-providers";
+import {
+ CHAT_IMAGE_ACCEPT,
+ convertedImageType,
+ normalizeChatImage,
+} from "./image-normalize";
import { chatModelLoaded } from "./lib/chat-model-loaded";
import {
type OpenDocumentAttachmentContent,
readActiveOpenDocumentAttachmentContent,
readOpenDocumentAttachmentContent,
} from "./open-document";
-import { OPEN_DOCUMENT_ATTACHMENT_ACCEPT } from "./open-document-accept";
+import {
+ OPEN_DOCUMENT_ATTACHMENT_ACCEPT,
+ RTF_ATTACHMENT_ACCEPT,
+ TOOL_ONLY_ATTACHMENT_EXTENSIONS,
+} from "./open-document-accept";
+import {
+ providerHostsCodeExecution,
+ providerSupportsBuiltinCodeExecution,
+} from "./provider-capabilities";
+import { readRtfAttachmentContent } from "./rtf";
import {
awaitThreadScopedSettingsWrite,
beginThreadScopedPairing,
+ codeToolsOn,
commitHeldThreadScopedEditsToTheirThread,
releaseHeldThreadScopedEdits,
useChatRuntimeStore,
@@ -235,8 +260,8 @@ class PreStreamAwareAttachmentAdapter implements AttachmentAdapter {
add(state: { file: File }) {
// A composite picks its adapter synchronously from the name and MIME type, and both say "video"
- // for an audio-only 3GP recording, so settle that from the container's own tracks first, as the
- // native readers do. Every other file goes straight through, keeping the delegate's own return.
+ // for an audio-only 3GP recording or a TypeScript .ts, so settle that from the file's own bytes
+ // first, as the native readers do. Every other file goes straight through.
if (!needsAttachmentTrackInspection(state.file)) {
return this.delegate.add(state);
}
@@ -249,9 +274,6 @@ class PreStreamAwareAttachmentAdapter implements AttachmentAdapter {
const file = await classifiedAttachmentFile(state.file);
const added = await this.delegate.add({ ...state, file });
if (Symbol.asyncIterator in added) {
- // Only the audio and video adapters claim a 3GP and both resolve to one
- // attachment, so this drains a generator to its last value rather than
- // forwarding the progress an adapter here does not report.
let last: PendingAttachment | undefined;
for await (const value of added) last = value;
if (!last) throw new Error("The attachment adapter yielded nothing.");
@@ -270,7 +292,13 @@ class PreStreamAwareAttachmentAdapter implements AttachmentAdapter {
const { incognito } = useChatRuntimeStore.getState();
const epoch = getAuthSessionEpoch();
try {
- return await withAttachmentOriginal(attachment, await this.delegate.send(attachment), incognito, epoch);
+ return await withAttachmentOriginal(
+ attachment,
+ await this.delegate.send(attachment),
+ incognito,
+ epoch,
+ pythonToolRunsInStudio(),
+ );
} catch (error) {
if (
reservationToken &&
@@ -284,9 +312,14 @@ class PreStreamAwareAttachmentAdapter implements AttachmentAdapter {
}
class VisionImageAdapter implements AttachmentAdapter {
- accept = "image/jpeg,image/png,image/webp,image/gif";
+ accept = CHAT_IMAGE_ACCEPT;
+ private readonly converted = new Map<string, Promise<File | null>>();
- async add({ file }: { file: File }): Promise<PendingAttachment> {
+ async *add({
+ file: picked,
+ }: {
+ file: File;
+ }): AsyncGenerator<PendingAttachment, void> {
const state = useChatRuntimeStore.getState();
const checkpoint = state.params.checkpoint;
const activeModel = state.models.find((m) => m.id === checkpoint);
@@ -330,38 +363,65 @@ class VisionImageAdapter implements AttachmentAdapter {
}
const maxSize = 20 * 1024 * 1024;
- if (file.size > maxSize) {
+ if (picked.size > maxSize) {
throw new Error("Image size exceeds 20MB limit");
}
-
- return {
+ const attachment = {
id: crypto.randomUUID(),
type: "image",
- name: file.name,
- contentType: file.type,
- file,
+ name: picked.name,
+ contentType: picked.type,
+ file: picked,
status: { type: "requires-action", reason: "composer-send" },
+ } satisfies PendingAttachment;
+ if (convertedImageType(picked) === null) {
+ yield attachment;
+ return;
+ }
+ yield {
+ ...attachment,
+ status: { type: "running", reason: "uploading", progress: 0 },
};
+ const conversion = normalizeChatImage(picked);
+ this.converted.set(
+ attachment.id,
+ conversion.catch(() => null),
+ );
+ let file: File;
+ try {
+ file = await conversion;
+ } catch (error) {
+ if (!this.converted.has(attachment.id)) {
+ return;
+ }
+ toast.error(error instanceof Error ? error.message : String(error));
+ throw error;
+ }
+ // Removed while converting: yielding again would put it back.
+ if (!this.converted.has(attachment.id)) {
+ return;
+ }
+ yield { ...attachment, name: file.name, contentType: file.type, file };
}
async send(attachment: PendingAttachment): Promise<CompleteAttachment> {
+ const conversion = this.converted.get(attachment.id);
+ this.converted.delete(attachment.id);
+ const file = conversion ? await conversion : attachment.file;
return {
id: attachment.id,
type: "image",
- name: attachment.name,
- contentType: attachment.contentType,
- content: [
- {
- type: "image",
- image: await this.fileToBase64DataURL(attachment.file),
- },
- ],
+ name: file?.name ?? attachment.name,
+ contentType: file?.type ?? attachment.contentType,
+ content: file
+ ? [{ type: "image", image: await this.fileToBase64DataURL(file) }]
+ : [],
status: { type: "complete" },
};
}
- async remove(): Promise<void> {
- return Promise.resolve();
+ async remove(attachment: { id: string }): Promise<void> {
+ this.converted.delete(attachment.id);
}
private async fileToBase64DataURL(file: File): Promise<string> {
@@ -646,6 +706,53 @@ class OfficeAttachmentAdapter implements AttachmentAdapter {
}
}
+class RtfAttachmentAdapter implements AttachmentAdapter {
+ accept = RTF_ATTACHMENT_ACCEPT;
+ // Read at add, like OfficeAttachmentAdapter: the composer drops the typed message before send().
+ private readonly texts = new Map<string, string>();
+
+ async add({ file }: { file: File }): Promise<PendingAttachment> {
+ let text: string;
+ try {
+ ({ text } = await readRtfAttachmentContent(file, file.name));
+ } catch (cause) {
+ const error = `RTF file could not be read: ${file.name}: ${(cause as Error).message}`;
+ toast.error(error);
+ throw new Error(error);
+ }
+ const id = crypto.randomUUID();
+ this.texts.set(id, text);
+ return {
+ id,
+ type: "document",
+ name: file.name,
+ contentType: file.type,
+ file,
+ status: { type: "requires-action", reason: "composer-send" },
+ };
+ }
+
+ async send(attachment: PendingAttachment): Promise<CompleteAttachment> {
+ const text =
+ this.texts.get(attachment.id) ??
+ (await readRtfAttachmentContent(attachment.file, attachment.name)).text;
+ this.texts.delete(attachment.id);
+ return {
+ id: attachment.id,
+ type: "document",
+ name: attachment.name,
+ contentType: attachment.contentType,
+ content: [{ type: "text", text: `[RTF: ${attachment.name}]\n${text}` }],
+ status: { type: "complete" },
+ };
+ }
+
+ remove(attachment: Attachment): Promise<void> {
+ this.texts.delete(attachment.id);
+ return Promise.resolve();
+ }
+}
+
class OpenDocumentAttachmentAdapter implements AttachmentAdapter {
private readonly active = new Set<string>();
private readonly sending = new Set<string>();
@@ -735,6 +842,124 @@ class OpenDocumentAttachmentAdapter implements AttachmentAdapter {
}
}
+const MAX_TOOL_ONLY_ATTACHMENT_BYTES = 200 * 1024 * 1024;
+
+/** Whether this turn's python tool runs in Studio's sandbox; chat-adapter.ts decides it the same way. */
+function pythonToolRunsInStudio(): boolean {
+ const state = useChatRuntimeStore.getState();
+ // The effective Code state, as the send path computes it: Full access turns it on locally.
+ const codeToolsEnabled = codeToolsOn(state);
+ const external = parseExternalModelId(state.params.checkpoint);
+ if (!external) return state.supportsTools && codeToolsEnabled;
+ const provider = (
+ loadConnectionsEnabled() ? loadExternalProviders() : []
+ ).find((p) => p.id === external.providerId);
+ if (
+ !provider ||
+ providerModelSupportsStudioTools(
+ provider.providerType,
+ external.modelId,
+ ) !== true
+ ) {
+ return false;
+ }
+ return selectCodeToolNames({
+ codeToolsEnabled,
+ hostedCodeExecutionForThisTurn: providerSupportsBuiltinCodeExecution(
+ provider.providerType,
+ external.modelId,
+ provider.baseUrl,
+ provider.apiType,
+ ),
+ providerHostsCodeExecution: providerHostsCodeExecution(
+ provider.providerType,
+ provider.baseUrl,
+ provider.apiType,
+ ),
+ }).local.includes("python");
+}
+
+class ToolOnlyAttachmentAdapter implements AttachmentAdapter {
+ accept = TOOL_ONLY_ATTACHMENT_EXTENSIONS;
+ private readonly uploads = new Map<
+ string,
+ Promise<ChatAttachmentOriginal | null>
+ >();
+ private readonly uploadedAt = new Map<string, number>();
+
+ private upload(file: File): Promise<ChatAttachmentOriginal | null> {
+ return uploadChatAttachmentOriginal(file).catch(() => null);
+ }
+
+ async *add({
+ file,
+ }: {
+ file: File;
+ }): AsyncGenerator<PendingAttachment, void> {
+ const refusal = !pythonToolRunsInStudio()
+ ? `Turn on Code with a model that runs the python tool to attach ${file.name}.`
+ : useChatRuntimeStore.getState().incognito
+ ? `Temporary chats save no files, so the python tool cannot open ${file.name}.`
+ : file.size > MAX_TOOL_ONLY_ATTACHMENT_BYTES
+ ? `File is too large: ${file.name}`
+ : null;
+ if (refusal) {
+ toast.error(refusal);
+ throw new Error(refusal);
+ }
+ const attachment = {
+ id: crypto.randomUUID(),
+ type: "document",
+ name: file.name,
+ contentType: file.type,
+ file,
+ status: { type: "running", reason: "uploading", progress: 0 },
+ } satisfies PendingAttachment;
+ yield attachment;
+ const upload = this.upload(file);
+ this.uploads.set(attachment.id, upload);
+ this.uploadedAt.set(attachment.id, Date.now());
+ const original = await upload;
+ if (this.uploads.get(attachment.id) !== upload) return;
+ if (!original) {
+ this.uploads.delete(attachment.id);
+ toast.error(`Could not upload ${file.name}`);
+ yield { ...attachment, status: { type: "incomplete", reason: "error" } };
+ return;
+ }
+ yield {
+ ...attachment,
+ status: { type: "requires-action", reason: "composer-send" },
+ };
+ }
+
+ async send(attachment: PendingAttachment): Promise<CompleteAttachment> {
+ const original =
+ (reuseStagedUpload(this.uploadedAt.get(attachment.id))
+ ? await this.uploads.get(attachment.id)
+ : null) ?? (await this.upload(attachment.file));
+ this.uploads.delete(attachment.id);
+ this.uploadedAt.delete(attachment.id);
+ const text = original
+ ? `[${attachment.name}: only the python tool can read this file]`
+ : `[${attachment.name} could not be uploaded, so it cannot be read]`;
+ const complete: CompleteAttachment = {
+ id: attachment.id,
+ type: "document",
+ name: attachment.name,
+ contentType: attachment.contentType,
+ content: [{ type: "text", text }],
+ status: { type: "complete" },
+ };
+ return original ? ({ ...complete, original } as CompleteAttachment) : complete;
+ }
+
+ async remove(attachment: { id: string }): Promise<void> {
+ this.uploads.delete(attachment.id);
+ this.uploadedAt.delete(attachment.id);
+ }
+}
+
function clip(input: string, maxLen: number): string {
const text = input.replace(/\s+/g, " ").trim();
if (text.length <= maxLen) return text;
@@ -2497,6 +2722,8 @@ function useStudioRuntimeAdapters(
new DocxAttachmentAdapter(),
new OfficeAttachmentAdapter(),
new OpenDocumentAttachmentAdapter(),
+ new RtfAttachmentAdapter(),
+ new ToolOnlyAttachmentAdapter(),
]),
() => {
const state = aui.threadListItem().getState();
diff --git a/studio/frontend/src/features/chat/sandbox-attachments.ts b/studio/frontend/src/features/chat/sandbox-attachments.ts
new file mode 100644
index 00000000000..114feacc68b
--- /dev/null
+++ b/studio/frontend/src/features/chat/sandbox-attachments.ts
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { attachmentOriginal } from "./attachment-originals";
+import { isToolOnlyAttachmentName } from "./open-document-accept";
+
+const NAME_BYTES = 80;
+const encoder = new TextEncoder();
+const utf8Length = (text: string) => encoder.encode(text).length;
+
+/** sandbox_attachment_path in core/inference/tools.py. Its result is a fixed point there, so the
+ * basename sent back derives this same path on the server. */
+export function sandboxAttachmentPath(sha256: string, name: string): string {
+ let base =
+ name
+ // eslint-disable-next-line no-control-regex
+ .replace(/[\x00-\x1f\x7f/\\:*?"<>|]/g, "_")
+ .replace(/^[ .]+|[ .]+$/g, "") || "attachment";
+ if (utf8Length(base) > NAME_BYTES) {
+ const dot = base.lastIndexOf(".");
+ const stem = dot > 0 ? base.slice(0, dot) : base;
+ let ext = dot > 0 ? base.slice(dot) : "";
+ if (utf8Length(ext) > 16) ext = "";
+ let room = NAME_BYTES - utf8Length(ext);
+ let kept = "";
+ for (const char of stem) {
+ room -= utf8Length(char);
+ if (room < 0) break;
+ kept += char;
+ }
+ base = (kept.replace(/[ .]+$/, "") || "attachment") + ext;
+ }
+ if (/^(?:CON|PRN|AUX|NUL|COM\d|LPT\d)$/i.test(base.split(".", 1)[0].trimEnd())) {
+ base = `_${base}`;
+ }
+ return `.unsloth_attachments/${sha256.slice(0, 12)}/${base}`;
+}
+
+// ChatCompletionRequest.sandbox_attachments cap: past it the whole request is refused.
+const MAX_SANDBOX_ATTACHMENTS = 64;
+
+type Attachment = { name?: string; content?: readonly unknown[] };
+
+function sandboxCopy(attachment: unknown): { sha256: string; path: string } | null {
+ const original = attachmentOriginal(attachment);
+ if (!original || !/^[0-9a-f]{64}$/.test(original.sha256)) return null;
+ const name = (attachment as Attachment).name ?? "";
+ return { sha256: original.sha256, path: sandboxAttachmentPath(original.sha256, name) };
+}
+
+/** Notes each kept file's sandbox path for the model, and lists the copies the backend makes. */
+export function withSandboxAttachmentPaths<
+ M extends { attachments?: readonly unknown[] },
+>(messages: readonly M[]) {
+ const latest = new Map<string, string>();
+ for (const message of messages) {
+ for (const attachment of message.attachments ?? []) {
+ const copy = sandboxCopy(attachment);
+ if (!copy) continue;
+ latest.delete(copy.path);
+ latest.set(copy.path, copy.sha256);
+ }
+ }
+ const carried = new Map([...latest].slice(-MAX_SANDBOX_ATTACHMENTS));
+ const sandboxAttachments = [...carried].map(([path, sha256]) => ({
+ sha256,
+ name: path.slice(path.lastIndexOf("/") + 1),
+ }));
+ const annotated = messages.map((message) => {
+ let changed = false;
+ const attachments = (message.attachments ?? []).map((attachment) => {
+ const copy = sandboxCopy(attachment);
+ if (!copy || !carried.has(copy.path)) return attachment;
+ changed = true;
+ const { name = "", content = [] } = attachment as Attachment;
+ const reader = sandboxReader(copy.path);
+ const path = JSON.stringify(copy.path);
+ const note = isToolOnlyAttachmentName(name)
+ ? `[${name} is saved at ${copy.path} in the python tool's working directory${reader ? `; open it with ${reader}, where path = ${path}` : ""}]`
+ : `[${name}: its text is below, so answer from it. For calculations, the python tool has the file at path = ${path}${reader ? `; ${reader}` : ""}]`;
+ return {
+ ...(attachment as object),
+ content: [{ type: "text", text: note }, ...content],
+ };
+ });
+ return changed ? ({ ...message, attachments } as M) : message;
+ });
+ return { messages: annotated, sandboxAttachments };
+}
+
+const READERS: ReadonlyArray<readonly [string, string]> = [
+ [".csv", "pandas.read_csv(path)"],
+ [".tsv", 'pandas.read_csv(path, sep="\\t")'],
+ [".jsonl,.ndjson", "pandas.read_json(path, lines=True)"],
+ [".json", "json.load(open(path))"],
+ [".docx", "docx.Document(path)"],
+ [
+ ".pdf,.pptx,.pptm,.ppsx,.potx,.potm,.ppsm,.docm,.dotx,.dotm,.epub,.mobi,.fb2,.cbz,.xps,.oxps",
+ "fitz.open(path)",
+ ],
+ // No reader for .ods/.odt or Excel: their text is inline, and the raw zip sends small models into XML.
+ [".odp,.odg", 'zipfile.ZipFile(path).read("content.xml")'],
+ [".zip,.jar,.whl,.apk,.kmz,.3mf,.vsdx", "zipfile.ZipFile(path)"],
+ [".tar,.tar.gz,.tgz,.tar.bz2,.tbz2,.tbz,.tar.xz,.txz", "tarfile.open(path)"],
+ [".gz", "gzip.open(path)"],
+ [".bz2", "bz2.open(path)"],
+ [".xz,.lzma", "lzma.open(path)"],
+ [".parquet", "pandas.read_parquet(path)"],
+ [".feather", "pandas.read_feather(path)"],
+ [".arrow", "pyarrow.ipc.open_file(path).read_all()"],
+ [".orc", "pyarrow.orc.read_table(path)"],
+ // `path` stays a variable: the sandbox scanner treats a literal passed to .connect() as a network host.
+ [".sqlite,.sqlite3,.db,.gpkg,.mbtiles", "sqlite3.connect(path)"],
+ [".duckdb", "duckdb.connect(path, read_only=True)"],
+ [".npy,.npz", "numpy.load(path)"],
+ [".dta", "pandas.read_stata(path)"],
+ [".sas7bdat", "pandas.read_sas(path)"],
+ [".xpt", 'pandas.read_sas(path, format="xport")'],
+ [".mat", "scipy.io.loadmat(path)"],
+ [".safetensors", 'safetensors.safe_open(path, framework="numpy")'],
+ [".stl,.ply,.glb", 'open(path, "rb").read()'],
+ [".ttf,.otf,.woff", "fontTools.ttLib.TTFont(path)"],
+ [".ttc", "fontTools.ttLib.TTCollection(path)"],
+ [
+ ".psd,.ico,.icns,.cur,.tga,.dds,.pcx,.ppm,.pgm,.pbm,.pnm,.qoi,.jp2,.j2k,.xbm,.xpm,.sgi,.fits",
+ "PIL.Image.open(path)",
+ ],
+];
+
+export function sandboxReader(name: string): string | null {
+ const lower = name.toLowerCase();
+ const reader = READERS.find(([extensions]) =>
+ extensions.split(",").some((extension) => lower.endsWith(extension)),
+ );
+ return reader ? reader[1] : null;
+}
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 4bac755dc98..8f124f4787f 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -65,6 +65,11 @@ import { classifiedAttachmentFiles, isVideoFile } from "@/lib/video-utils";
import { isDownloadCancelled } from "@/lib/native-files";
import { isMultimodalResponse } from "./types/api";
import { getImageInputUnavailableReason } from "./utils/image-input-support";
+import {
+ CHAT_IMAGE_ACCEPT,
+ isChatImageFile,
+ normalizeChatImage,
+} from "./image-normalize";
import { modelIdsMatch } from "@/features/hub/lib/model-identity";
import { CONVERSATION_MARKDOWN_LABEL } from "./utils/conversation-markdown";
import { pasteClipboardFiles } from "./utils/clipboard-files";
@@ -237,7 +242,6 @@ export interface CompareHandle {
waitForRunEnd: () => Promise<void>;
}
-const IMAGE_ACCEPT = "image/jpeg,image/png,image/webp,image/gif";
const MAX_IMAGE_SIZE = 20 * 1024 * 1024;
// Inlined to avoid a new icon dep. Kept in sync with the main composer.
@@ -592,6 +596,7 @@ export function SharedComposer({
const [running, setRunning] = useState(false);
const [comparing, setComparing] = useState(false);
const [pendingImages, setPendingImages] = useState<PendingImage[]>([]);
+ const [convertingImages, setConvertingImages] = useState(0);
const [pendingAudio, setPendingAudio] = useState<{
name: string;
base64: string;
@@ -1040,6 +1045,7 @@ export function SharedComposer({
let droppedImageForUnavailable = false;
let audioSizeError: string | null = null;
let videoUnsupported = false;
+ let conversionError: string | null = null;
for (let i = 0; i < files.length; i++) {
const file = files[i];
if (!file) continue;
@@ -1061,13 +1067,24 @@ export function SharedComposer({
videoUnsupported = true;
continue;
}
- if (!file.type.match(/^image\/(jpeg|png|webp|gif)$/i)) continue;
+ if (!isChatImageFile(file)) continue;
if (file.size > MAX_IMAGE_SIZE) continue;
if (attachUnavailableReason) {
droppedImageForUnavailable = true;
continue;
}
- next.push({ id: crypto.randomUUID(), file });
+ let image: File;
+ setConvertingImages((count) => count + 1);
+ try {
+ image = await normalizeChatImage(file);
+ } catch (error) {
+ conversionError ??=
+ error instanceof Error ? error.message : String(error);
+ continue;
+ } finally {
+ setConvertingImages((count) => count - 1);
+ }
+ next.push({ id: crypto.randomUUID(), file: image });
}
if (droppedImageForUnavailable && attachUnavailableReason) {
toast.error(attachUnavailableReason);
@@ -1075,6 +1092,9 @@ export function SharedComposer({
if (audioSizeError) {
toast.error(audioSizeError);
}
+ if (conversionError) {
+ toast.error(conversionError);
+ }
if (videoUnsupported) {
toast.error("Video can't be attached in compare mode", {
description: "Open a single chat with a video-capable model instead.",
@@ -1097,8 +1117,7 @@ export function SharedComposer({
const supported = files.some(
(file) =>
isAudioAttachmentFile(file) ||
- (file.type.match(/^image\/(jpeg|png|webp|gif)$/i) &&
- file.size <= MAX_IMAGE_SIZE),
+ (isChatImageFile(file) && file.size <= MAX_IMAGE_SIZE),
);
if (!supported) throw new Error("Unsupported compare attachment");
await addFiles(files);
@@ -1151,7 +1170,7 @@ export function SharedComposer({
useEffect(() => () => clearStuckImeTimer(), []);
async function send() {
- if (composingRef.current) {
+ if (composingRef.current || convertingImages > 0) {
resetPromptQueue();
return;
}
@@ -2039,6 +2058,7 @@ export function SharedComposer({
!busy &&
!isComposing &&
!isDictating &&
+ convertingImages === 0 &&
!sendUnavailableReason;
// Compare mode swaps this composer in for the single-chat one and only one is ever on screen, so the
@@ -2438,7 +2458,7 @@ export function SharedComposer({
<input
ref={fileInputRef}
type="file"
- accept={IMAGE_ACCEPT}
+ accept={CHAT_IMAGE_ACCEPT}
multiple
className="hidden"
onChange={(e) => {
diff --git a/studio/frontend/src/features/native-intents/drop-paths.ts b/studio/frontend/src/features/native-intents/drop-paths.ts
index 5fb7c1a96b6..eee7677e064 100644
--- a/studio/frontend/src/features/native-intents/drop-paths.ts
+++ b/studio/frontend/src/features/native-intents/drop-paths.ts
@@ -1,9 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import { CHAT_IMAGE_EXTENSIONS } from "../chat/image-normalize.ts";
import {
OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS,
isOpenDocumentAttachmentName,
+ RTF_ATTACHMENT_EXTENSIONS,
+ isRtfAttachmentName,
+ TOOL_ONLY_ATTACHMENT_EXTENSIONS,
+ isToolOnlyAttachmentName,
} from "../chat/open-document-accept.ts";
import {
TEXT_ATTACHMENT_EXTENSIONS,
@@ -22,7 +27,12 @@ const TEXT_EXTS = TEXT_ATTACHMENT_EXTENSIONS.map((ext) =>
/** Dropped paths with an inline adapter but no RAG parser. */
export function isComposerAttachmentName(path: string): boolean {
- return isOpenDocumentAttachmentName(path) || isTextDropName(path);
+ return (
+ isOpenDocumentAttachmentName(path) ||
+ isRtfAttachmentName(path) ||
+ isToolOnlyAttachmentName(path) ||
+ isTextDropName(path)
+ );
}
function isTextDropName(path: string): boolean {
@@ -35,8 +45,7 @@ function isTextDropName(path: string): boolean {
return dot > 0 && TEXT_EXTS.includes(name.slice(dot));
}
-/** Vision chat attachments; keep in sync with `shared-composer` `IMAGE_ACCEPT`. */
-export const CHAT_IMAGE_DROP_ACCEPT = ".jpg,.jpeg,.png,.webp,.gif";
+export const CHAT_IMAGE_DROP_ACCEPT = CHAT_IMAGE_EXTENSIONS;
const IMAGE_EXTS = CHAT_IMAGE_DROP_ACCEPT.split(",").map((ext) =>
ext.trim().toLowerCase(),
@@ -54,14 +63,14 @@ const AUDIO_EXTS = CHAT_AUDIO_DROP_ACCEPT.split(",").map((ext) =>
* `VIDEO_ATTACHMENT_EXTS`. llama-server decodes with ffmpeg, so this is what
* ffmpeg reads, not what the webview can play. */
export const CHAT_VIDEO_DROP_ACCEPT =
- ".mp4,.m4v,.mov,.webm,.mkv,.avi,.mpg,.mpeg,.wmv,.flv,.3gp,.ogv";
+ ".mp4,.m4v,.mov,.webm,.mkv,.avi,.mpg,.mpeg,.wmv,.flv,.3gp,.ogv,.m2ts";
const VIDEO_EXTS = CHAT_VIDEO_DROP_ACCEPT.split(",").map((ext) =>
ext.trim().toLowerCase(),
);
/** What the window actually takes, for the rejection toast and the overlay. */
-export const SUPPORTED_DROP_HINT = `Supported files: ${RAG_UPLOAD_ACCEPT}, ${OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS}, source and text files, ${CHAT_IMAGE_DROP_ACCEPT}, one of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`;
+export const SUPPORTED_DROP_HINT = `Supported files: ${RAG_UPLOAD_ACCEPT}, ${OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS}, ${RTF_ATTACHMENT_EXTENSIONS}, source and text files, ${TOOL_ONLY_ATTACHMENT_EXTENSIONS} with Code on, ${CHAT_IMAGE_DROP_ACCEPT}, one of ${CHAT_AUDIO_DROP_ACCEPT}, one of ${CHAT_VIDEO_DROP_ACCEPT}, or a single .gguf model.`;
/** Last path segment of a native path, for display and extension checks. */
export function nativeFileName(path: string): string {
@@ -102,8 +111,7 @@ export function classifyDropPaths(paths: string[]): NativeDropClass {
const docs = paths.filter(
(path) =>
DOC_EXTS.some((ext) => hasExt(path, ext)) ||
- isOpenDocumentAttachmentName(path) ||
- isTextDropName(path),
+ isComposerAttachmentName(path),
);
const images = paths.filter((path) =>
IMAGE_EXTS.some((ext) => hasExt(path, ext)),
diff --git a/studio/frontend/src/features/settings/components/uploaded-files-dialog.tsx b/studio/frontend/src/features/settings/components/uploaded-files-dialog.tsx
index 55826d1487c..d86aa9179db 100644
--- a/studio/frontend/src/features/settings/components/uploaded-files-dialog.tsx
+++ b/studio/frontend/src/features/settings/components/uploaded-files-dialog.tsx
@@ -194,9 +194,12 @@ function toSortTime(value: string | number | null | undefined): number {
// tab synchronously and point it at the URL once resolved. A blocked synchronous open is surfaced
// instead of silently losing the file after the asynchronous URL lookup. The Tauri webview has no
// window.open at all, so it goes through the OS opener.
-async function openResolvedUrl(resolve: () => Promise<string>): Promise<void> {
+async function openResolvedUrl(
+ resolve: () => Promise<string | null>,
+): Promise<void> {
if (isTauri) {
const url = await resolve();
+ if (url === null) return;
const { openUrl } = await import("@tauri-apps/plugin-opener");
await openUrl(url);
return;
@@ -208,13 +211,17 @@ async function openResolvedUrl(resolve: () => Promise<string>): Promise<void> {
);
}
win.opener = null;
- let url: string;
+ let url: string | null;
try {
url = await resolve();
} catch (err) {
win.close();
throw err;
}
+ if (url === null) {
+ win.close();
+ return;
+ }
win.location.replace(url);
}
@@ -312,6 +319,11 @@ function chatAttachmentRow(att: ChatAttachmentRecord): UploadedFileRow {
}
await openResolvedUrl(async () => {
const blob = await fetchChatAttachmentBlob(att.messageId, att.id);
+ // A kept original is served as octet-stream: a tab would save it nameless.
+ if (blob.type === "application/octet-stream") {
+ await downloadFile(blob, att.name, blob.type);
+ return null;
+ }
const url = URL.createObjectURL(blob);
// Give the new tab time to load the blob before revoking.
setTimeout(() => URL.revokeObjectURL(url), 60_000);
diff --git a/studio/frontend/src/features/video/reference-budget.ts b/studio/frontend/src/features/video/reference-budget.ts
index 3a9f41baf83..eeeebca2b53 100644
--- a/studio/frontend/src/features/video/reference-budget.ts
+++ b/studio/frontend/src/features/video/reference-budget.ts
@@ -43,7 +43,7 @@ export const MAX_REFERENCE_BYTES: Record<ReferenceKind, number> = {
* is the picker one, .3gp included, because the picker reads a recording's
* tracks once it has the file and a clip is refused then. */
export const REFERENCE_PICKER_ACCEPT: Record<ReferenceKind, string> = {
- video: VIDEO_ACCEPT,
+ video: `${VIDEO_ACCEPT},.ts,.mts`,
audio: AUDIO_PICKER_ACCEPT,
};
diff --git a/studio/frontend/src/lib/video-utils.ts b/studio/frontend/src/lib/video-utils.ts
index 1320b37f5c5..3c15347b18d 100644
--- a/studio/frontend/src/lib/video-utils.ts
+++ b/studio/frontend/src/lib/video-utils.ts
@@ -5,7 +5,7 @@
* what ffmpeg reads, not what the webview can play. Extensions ride along
* because MIME is unreliable for mkv and some mov files. */
export const VIDEO_ACCEPT =
- "video/mp4,video/x-m4v,video/quicktime,video/webm,video/x-matroska,video/x-msvideo,video/mpeg,video/x-ms-wmv,video/x-flv,video/3gpp,video/ogg,.mp4,.m4v,.mov,.webm,.mkv,.avi,.mpg,.mpeg,.wmv,.flv,.3gp,.ogv";
+ "video/mp4,video/x-m4v,video/quicktime,video/webm,video/x-matroska,video/x-msvideo,video/mpeg,video/x-ms-wmv,video/x-flv,video/3gpp,video/ogg,video/mp2t,.mp4,.m4v,.mov,.webm,.mkv,.avi,.mpg,.mpeg,.wmv,.flv,.3gp,.ogv,.m2ts";
// Matches _MAX_VIDEO_B64_CHARS in the backend, so the composer does not accept
// a clip the route refuses. The native reader's cap is a higher backstop.
@@ -35,6 +35,7 @@ const VIDEO_MIME_BY_EXTENSION: Record<string, string> = {
".flv": "video/x-flv",
".3gp": "video/3gpp",
".ogv": "video/ogg",
+ ".m2ts": "video/mp2t",
};
const VIDEO_EXTENSIONS = Object.keys(VIDEO_MIME_BY_EXTENSION);
@@ -219,12 +220,22 @@ async function read3gpTracks(file: File): Promise<BmffTracks> {
* by box count and by the size of the track table rather than by the file.
*/
export function needsAttachmentTrackInspection(file: File): boolean {
- return /\.3gp$/i.test(file.name);
+ return /\.(3gp|m?ts)$/i.test(file.name);
+}
+
+/** Mirrors `is_mpeg_transport_stream` in native_path_policy.rs. */
+export function isMpegTransportStreamBytes(head: Uint8Array): boolean {
+ return [
+ [0, 188],
+ [4, 192],
+ ].some(([start, packet]) =>
+ [0, 1, 2].every((index) => head[start! + index * packet!] === 0x47),
+ );
}
/**
* The file an attachment surface should classify, with an audio-only 3GP
- * restamped as audio.
+ * restamped as audio and a .ts or .mts file as a transport stream or text.
*
* A voice recording and a clip share the .3gp extension, and the browser
* answers "" or video/3gpp for both, so the name alone sends the recording to
@@ -237,6 +248,23 @@ export async function classifiedAttachmentFile(file: File): Promise<File> {
if (!needsAttachmentTrackInspection(file)) {
return file;
}
+ if (!/\.3gp$/i.test(file.name)) {
+ let head: Uint8Array;
+ try {
+ head = new Uint8Array(await file.slice(0, 4 + 3 * 192).arrayBuffer());
+ } catch {
+ return file;
+ }
+ const corrected = isMpegTransportStreamBytes(head)
+ ? "video/mp2t"
+ : "text/plain";
+ return corrected === file.type
+ ? file
+ : new File([file], file.name, {
+ type: corrected,
+ lastModified: file.lastModified,
+ });
+ }
let tracks: BmffTracks;
try {
tracks = await read3gpTracks(file);
diff --git a/studio/frontend/tests/attachment-preview-text.test.ts b/studio/frontend/tests/attachment-preview-text.test.ts
index ae64af17423..f40a51fbb6f 100644
--- a/studio/frontend/tests/attachment-preview-text.test.ts
+++ b/studio/frontend/tests/attachment-preview-text.test.ts
@@ -31,6 +31,8 @@ const {
truncateAttachmentPreviewText,
} = await import("../src/features/chat/attachment-content.ts");
const { definePDFJSModule } = await import("unpdf");
+const { readRtfAttachmentContent } =
+ await import("../src/features/chat/rtf.ts");
type StubNode = {
nodeType: number;
@@ -250,6 +252,15 @@ test("readAttachmentText reads a bounded slice of a large html file", async () =
assert.equal(text.length, 1_000_000);
});
+test("readAttachmentText does not decode a file only the python tool reads", async () => {
+ const file = new File([new Uint8Array([0x50, 0x41, 0x52, 0x31])], "t.PARQUET");
+ assert.deepEqual(await readAttachmentText(file, file.name, file.type), {
+ label: null,
+ text: "t.PARQUET has no preview: only the python tool can read it.",
+ truncated: false,
+ });
+});
+
// the adapter sends the extraction; the preview shows the markup unextracted
test("readAttachmentText previews an html file as its markup", async () => {
const markup = "<p>Drag to rotate<br>Scroll to zoom</p>";
@@ -998,3 +1009,72 @@ test("a UTF-16 Markdown file previews as its text in the document viewer", async
const dialog = await readFile(new URL("../src/components/assistant-ui/attachment-document-dialog.tsx", import.meta.url), "utf8");
assert.match(dialog, /blob instanceof File\s*\?\s*await readAttachmentText\(blob, source\.name, source\.contentType\)/);
});
+
+async function readRtf(rtf: string | Uint8Array<ArrayBuffer>): Promise<string> {
+ const content = await readRtfAttachmentContent(
+ new File([rtf], "doc.rtf"),
+ "doc.rtf",
+ );
+ assert.equal(content.label, "RTF");
+ return content.text;
+}
+
+test("TextEdit output decodes bytes by the font charset, not the ANSI code page", async () => {
+ const text = await readRtf(
+ [
+ "{\\rtf1\\ansi\\ansicpg936\\cocoartf2870",
+ "{\\fonttbl\\f0\\fswiss\\fcharset0 Helvetica;}",
+ "{\\colortbl;\\red255\\green255\\blue255;}",
+ "{\\*\\expandedcolortbl;;}",
+ "\\f0\\fs24 \\cf0 H\\'e9llo \\'93world\\'94\\",
+ "\\",
+ "Tab\there \\uc0\\u26085 \\u26412 \\",
+ "}",
+ ].join("\n"),
+ );
+ assert.equal(text, "Héllo “world”\n\nTab\there 日本");
+});
+
+test("Word-style output reads fields, tables and double-byte fonts", async () => {
+ const text = await readRtf(
+ new Uint8Array([
+ ...new TextEncoder().encode(
+ [
+ "{\\rtf1\\ansi\\ansicpg1252\\deff0\\uc1",
+ "{\\fonttbl{\\f0\\fnil\\fcharset134 SimSun;}{\\f1\\fcharset0 Arial;}{\\f2\\fcharset128 MS Mincho;}{\\f3\\fcharset204 Arial;}}",
+ "{\\header Page header\\par}",
+ "{\\info{\\title Secret title}}",
+ "\\pard \\'c4\\'e3\\'ba\\'c3 \\f1 caf\\'e9\\emdash\\u8364?\\u-10179?\\u-8704?\\{x\\}\\par",
+ '{\\field{\\*\\fldinst{HYPERLINK "https://x.test"}}{\\fldrslt link}}\\par',
+ "{\\pict\\bin4 ",
+ ].join("\n"),
+ ),
+ 0x7d,
+ 0x7b,
+ 0x5c,
+ 0x7d,
+ ...new TextEncoder().encode(
+ [
+ "}",
+ "\\trowd\\cellx100\\cellx200",
+ "\\pard\\intbl first\\par second\\par\\cell b\\cell\\row",
+ "\\pard after\\par more \\f2\\'83e\\'83X\\'83g\\'83\\\\ \\f3\\'cf\\plain\\'c4\\'e3\\par}",
+ ].join("\n"),
+ ),
+ ]),
+ );
+ assert.equal(
+ text,
+ "你好 café—€😀{x}\nlink\nfirst second\tb\nafter\nmore テストソ П你",
+ );
+});
+
+test("an RTF reader stays bounded", async () => {
+ const long = await readRtf(
+ `{\\rtf1 ${"x".repeat(11 * 1024 * 1024)}${"{".repeat(2000)}`,
+ );
+ assert.ok(long.length < 11 * 1024 * 1024);
+ assert.match(long, /^x+\n\n\[Truncated: [^\n]*\]$/);
+ await assert.rejects(readRtf(`{\\rtf1 ${"{".repeat(2000)}`), /nest too deeply/);
+ await assert.rejects(readRtf("plain text"), /Not an RTF file/);
+});
diff --git a/studio/frontend/tests/native-drop-paths.test.ts b/studio/frontend/tests/native-drop-paths.test.ts
index 9f69899779a..27c715077ff 100644
--- a/studio/frontend/tests/native-drop-paths.test.ts
+++ b/studio/frontend/tests/native-drop-paths.test.ts
@@ -5,9 +5,17 @@ import assert from "node:assert/strict";
import test from "node:test";
import {
+ RTF_ATTACHMENT_EXTENSIONS,
+ TOOL_ONLY_ATTACHMENT_EXTENSIONS,
OPEN_DOCUMENT_ATTACHMENT_ACCEPT,
OPEN_DOCUMENT_ATTACHMENT_EXTENSIONS,
} from "../src/features/chat/open-document-accept.ts";
+import {
+ CHAT_IMAGE_MIMES,
+ CHAT_IMAGE_ACCEPT,
+ convertedImageType,
+ isChatImageFile,
+} from "../src/features/chat/image-normalize.ts";
import {
TEXT_ATTACHMENT_ACCEPT,
TEXT_ATTACHMENT_BASENAMES,
@@ -17,13 +25,6 @@ import {
MAX_TEXT_ATTACHMENT_BYTES,
decodeTextAttachmentBytes,
isTextAttachmentName,
-} from "../src/features/chat/text-attachment-accept.ts";
-import {
- AUDIO_ATTACHMENT_ACCEPT,
- AUDIO_PICKER_ACCEPT,
- isAudioAttachmentFile,
-} from "../src/lib/audio-utils.ts";
-import {
isBinaryOfficeTemplate,
isBinaryVobSubSubtitle,
isCompiledFortranModule,
@@ -32,6 +33,12 @@ import {
readTextAttachmentOnce,
UndecodableTextError,
} from "../src/features/chat/text-attachment-accept.ts";
+import {
+ AUDIO_ATTACHMENT_ACCEPT,
+ AUDIO_PICKER_ACCEPT,
+ isAudioAttachmentFile,
+ AUDIO_ACCEPT,
+} from "../src/lib/audio-utils.ts";
import {
dequeueNativeAttachments,
enqueueNativeAttachments,
@@ -47,7 +54,6 @@ import {
import type { NativeIntent } from "../src/features/native-intents/types.ts";
import { RAG_UPLOAD_ACCEPT } from "../src/features/rag/types/rag.ts";
import { MAX_REFERENCE_BYTES } from "../src/features/video/reference-budget.ts";
-import { AUDIO_ACCEPT } from "../src/lib/audio-utils.ts";
import {
VIDEO_ACCEPT,
classifiedAttachmentFile,
@@ -94,10 +100,16 @@ const MIME_MATCH_BODY_RE =
/fn attachment_mime_type[\s\S]*?match ext\.as_str\(\) \{([\s\S]*?)\n {4}\}/;
const MIME_ARM_EXTENSION_RE =
/^\s*((?:"[^"]+"\s*\|?\s*)+)=>\s*Some\("image\//gm;
-const COMPOSER_IMAGE_ACCEPT_RE = /const IMAGE_ACCEPT\s*=\s*"([^"]+)"/;
+const COMPOSER_IMAGE_ACCEPT_RE = /accept=\{CHAT_IMAGE_ACCEPT\}/;
+const IMAGE_DRAIN_CONVERTS_PER_FILE_RE =
+ /const drainPendingImages[\s\S]*?try \{[^}]*?file = await normalizeChatImage\(\s*await nativeAttachmentIntentToFile\(intent\),?\s*\);\s*\} catch \(error\) \{\s*\/\/[^\n]*\n[^\n]*\n\s*readFailures \+= 1;\s*lastReadError = error;\s*continue;/;
const VISION_ADAPTER_ACCEPT_RE =
- /class VisionImageAdapter[^{]*\{\s*accept\s*=\s*"([^"]+)"/s;
+ /class VisionImageAdapter[^{]*\{\s*accept = CHAT_IMAGE_ACCEPT;/;
const OPEN_DOCUMENT_EXTENSION_RE = /\.ods/;
+const RTF_ADAPTER_RE =
+ /class RtfAttachmentAdapter[^{]*\{\s*accept = RTF_ATTACHMENT_ACCEPT;[\s\S]*?readRtfAttachmentContent\(file, file\.name\)[\s\S]*?new CompositeAttachmentAdapter\(\[[\s\S]*?new RtfAttachmentAdapter\(\),/;
+const TOOL_ONLY_ADAPTER_RE =
+ /function pythonToolRunsInStudio\(\)[\s\S]*?const codeToolsEnabled = codeToolsOn\(state\);[\s\S]*?if \(!external\) return state\.supportsTools && codeToolsEnabled;[\s\S]*?\}\)\.local\.includes\("python"\);[\s\S]*?class ToolOnlyAttachmentAdapter[^{]*\{\s*accept = TOOL_ONLY_ATTACHMENT_EXTENSIONS;[\s\S]*?!pythonToolRunsInStudio\(\)[\s\S]*?\?\?\s*\(await this\.upload\(attachment\.file\)\);[\s\S]*?return original \? \(\{ \.\.\.complete, original \}[\s\S]*?new RtfAttachmentAdapter\(\),\s*new ToolOnlyAttachmentAdapter\(\),\s*\]\)/;
const OPEN_DOCUMENT_ADAPTER_ACCEPT_RE =
/class OpenDocumentAttachmentAdapter[^{]*\{[\s\S]*?accept = OPEN_DOCUMENT_ATTACHMENT_ACCEPT;/;
const OPEN_DOCUMENT_DROP_TO_COMPOSER_RE =
@@ -181,6 +193,32 @@ test("OpenDocument picker types are accepted by native drops", () => {
assert.deepEqual(rust, frontend);
});
+for (const [format, extensions, adapter, rustList] of [
+ ["RTF", RTF_ATTACHMENT_EXTENSIONS, RTF_ADAPTER_RE, "RTF_ATTACHMENT_EXTS"],
+ [
+ "Tool-only",
+ TOOL_ONLY_ATTACHMENT_EXTENSIONS,
+ TOOL_ONLY_ADAPTER_RE,
+ "TOOL_ONLY_ATTACHMENT_EXTS",
+ ],
+] as const) {
+ test(`${format} drops route to the composer and match the native allowlist`, () => {
+ for (const extension of extensions.split(",")) {
+ const path = `/docs/Book${extension.toUpperCase()}`;
+ assert.equal(classifyDropPaths([path]).kind, "docs", path);
+ assert.ok(isComposerAttachmentName(path), path);
+ assert.ok(SUPPORTED_DROP_HINT.includes(extension));
+ }
+ assert.match(RUNTIME_PROVIDER, adapter);
+ const rust = [
+ ...(readText("../../src-tauri/src/native_path_policy.rs")
+ .match(new RegExp(`${rustList}[^=]*=\\s*&\\[([^\\]]+)\\]`))?.[1]
+ .matchAll(RUST_EXTENSION_RE) ?? []),
+ ].map((match) => `.${match[1]}`);
+ assert.deepEqual(rust.sort(), extensions.split(",").sort());
+ });
+}
+
test("images route to chat vision attachments, one or many", () => {
const dropped = classifyDropPaths([
"/photos/cat.PNG",
@@ -211,10 +249,10 @@ test("a mixed or unsupported drop is rejected", () => {
"unsupported",
);
assert.equal(
- classifyDropPaths(["/docs/a.pdf", "/docs/b.zip"]).kind,
+ classifyDropPaths(["/docs/a.pdf", "/docs/b.dmg"]).kind,
"unsupported",
);
- assert.equal(classifyDropPaths(["/docs/notes.zip"]).kind, "unsupported");
+ assert.equal(classifyDropPaths(["/docs/notes.dmg"]).kind, "unsupported");
});
test("an empty payload is not a drop target", () => {
@@ -321,11 +359,8 @@ test("every MIME type Rust stamps is one the vision adapter claims", () => {
),
].sort();
- const accepted = RUNTIME_PROVIDER
- .match(VISION_ADAPTER_ACCEPT_RE)?.[1]
- .split(",")
- .map((type) => type.trim())
- .sort();
+ assert.match(RUNTIME_PROVIDER, VISION_ADAPTER_ACCEPT_RE);
+ const accepted = [...CHAT_IMAGE_MIMES].sort();
assert.ok(
stamped.length > 0,
@@ -359,20 +394,33 @@ test("every accepted image extension has a Rust MIME arm", () => {
assert.deepEqual(mapped, accepted);
});
-// The one constant drop-paths.ts names in its own "keep in sync" comment.
-test("the drop image list matches the composer's file picker", () => {
- const picker = SHARED_COMPOSER
- .match(COMPOSER_IMAGE_ACCEPT_RE)?.[1]
- .split(",")
- .map((type) => type.trim().replace("image/", ""))
- .sort();
+test("no composer sends while an image is being converted", () => {
+ assert.match(
+ SHARED_COMPOSER,
+ /setConvertingImages\(\(count\) => count \+ 1\);\s*try \{\s*image = await normalizeChatImage\(file\);[\s\S]*?\} finally \{\s*setConvertingImages\(\(count\) => count - 1\);/,
+ );
+ assert.match(SHARED_COMPOSER, /const canSend =[^;]*convertingImages === 0/);
+ assert.match(
+ RUNTIME_PROVIDER,
+ /class VisionImageAdapter[\s\S]*?if \(!this\.converted\.has\(attachment\.id\)\) \{\s*return;\s*\}\s*toast\.error[\s\S]*?if \(!this\.converted\.has\(attachment\.id\)\) \{\s*return;\s*\}\s*yield \{ \.\.\.attachment, name: file\.name/,
+ );
+ assert.match(
+ RUNTIME_PROVIDER,
+ /class VisionImageAdapter[\s\S]*?this\.converted\.set\(\s*attachment\.id,\s*conversion\.catch\(\(\) => null\),?\s*\);[\s\S]*?const file = conversion \? await conversion : attachment\.file;[\s\S]*?content: file\s*\?[\s\S]*?: \[\],[\s\S]*?async remove\(attachment: \{ id: string \}\): Promise<void> \{\s*this\.converted\.delete\(attachment\.id\);/,
+ );
+});
+
+test("dropped images are converted as part of their per-file read", () => {
+ assert.match(THREAD, IMAGE_DRAIN_CONVERTS_PER_FILE_RE);
+});
+
+test("every dropped image extension is a type the composers accept", () => {
+ assert.match(SHARED_COMPOSER, COMPOSER_IMAGE_ACCEPT_RE);
const dropped = CHAT_IMAGE_DROP_ACCEPT.split(",")
.map((ext) => ext.trim().toLowerCase().replace(".", ""))
- .map((ext) => (ext === "jpg" ? "jpeg" : ext))
- .sort();
-
- assert.ok(picker, "IMAGE_ACCEPT not found in shared-composer.tsx");
- assert.deepEqual([...new Set(dropped)], picker);
+ .map((ext) => ({ jpg: "jpeg", tif: "tiff" })[ext] ?? ext)
+ .map((ext) => `image/${ext}`);
+ assert.deepEqual([...new Set(dropped)].sort(), [...CHAT_IMAGE_MIMES].sort());
});
// A remount means the instance that queued the batch cannot hand it over.
@@ -583,6 +631,7 @@ test("every video MIME Rust stamps is one the video adapter claims", () => {
assert.ok(stamped.length > 0, "Rust stamps no video MIME types");
for (const mime of stamped) {
+ if (mime === "video/mp2t") continue;
assert.ok(claimed.has(mime), `the video adapter does not claim ${mime}`);
}
});
@@ -1485,7 +1534,7 @@ test("a dotfile is not mistaken for a source file", () => {
});
test("an unreadable type is still refused", () => {
- assert.equal(classifyDropPaths(["/docs/archive.zip"]).kind, "unsupported");
+ assert.equal(classifyDropPaths(["/docs/disk.dmg"]).kind, "unsupported");
assert.equal(classifyDropPaths(["/bin/tool.exe"]).kind, "unsupported");
});
@@ -1617,6 +1666,56 @@ test("the restamped recording routes to the audio adapter", async () => {
assert.equal(fileMatchesAccept(classified, AUDIO_ATTACHMENT_ACCEPT), true);
});
+test("a transport stream named .m2ts, .ts or .mts routes to video, TypeScript to text", async () => {
+ const { fileMatchesAccept } = (await import(
+ new URL(
+ "../node_modules/@assistant-ui/core/dist/adapters/attachment.js",
+ import.meta.url,
+ ).href
+ )) as { fileMatchesAccept: (file: File, accept: string) => boolean };
+ const stream = (packet: number, start: number) => {
+ const bytes = new Uint8Array(packet * 4);
+ for (let offset = start; offset < bytes.length; offset += packet) {
+ bytes[offset] = 0x47;
+ }
+ return bytes;
+ };
+ assert.ok(
+ fileMatchesAccept(
+ new File([], "clip.M2TS", { type: "video/mp2t" }),
+ VIDEO_ACCEPT,
+ ),
+ );
+ for (const file of [
+ new File([stream(192, 4)], "camcorder.MTS", { type: "" }),
+ new File([stream(188, 0)], "broadcast.ts", { type: "video/mp2t" }),
+ ]) {
+ const classified = await classifiedAttachmentFile(file);
+ assert.equal(classified.type, "video/mp2t", file.name);
+ assert.ok(fileMatchesAccept(classified, VIDEO_ACCEPT), file.name);
+ }
+ const typescript = new File(
+ ["GENERATED\n" + "export const value = 1;\n".repeat(40)],
+ "index.ts",
+ { type: "video/mp2t" },
+ );
+ const classified = await classifiedAttachmentFile(typescript);
+ assert.equal(classified.type, "text/plain");
+ assert.equal(fileMatchesAccept(classified, VIDEO_ACCEPT), false);
+ const { REFERENCE_DROP_ACCEPT, referenceFileRejection } = await import(
+ "../src/features/video/reference-budget.ts"
+ );
+ assert.ok(REFERENCE_DROP_ACCEPT.video.split(",").includes(".mts"));
+ const clip = await classifiedAttachmentFile(
+ new File([stream(192, 4)], "camcorder.mts", { type: "" }),
+ );
+ assert.equal(referenceFileRejection("video", clip), null);
+ assert.equal(
+ referenceFileRejection("video", classified),
+ "Please choose a video file",
+ );
+});
+
test("a file dialog offers 3GP recordings, and routing still does not", () => {
// A dialog decides what is selectable, so it has to name .3gp: a platform
// that maps it to video/3gpp, or to nothing, greys the recording out. Routing
@@ -2127,7 +2226,7 @@ test("a 3GP is inspected however large the surface taking it allows", async () =
const source = readSrc("lib/video-utils.ts");
assert.match(
source,
- /export function needsAttachmentTrackInspection[^)]*\)[^{]*\{\s*return \/\\\.3gp\$\/i\.test\(file\.name\);\s*\}/,
+ /export function needsAttachmentTrackInspection[^)]*\)[^{]*\{\s*return \/\\\.\(3gp\|m\?ts\)\$\/i\.test\(file\.name\);\s*\}/,
);
});
@@ -2473,3 +2572,23 @@ test("escapes in a header entry resolve before the charset is read", async () =>
'msgid "c"\nmsgstr "café"\n';
assert.match(await readTextAttachment(new File([po], "m.po")), /café/);
});
+
+test("formats the backends cannot take are converted, the rest sent as is", () => {
+ for (const [name, type, converted] of [
+ ["photo.HEIC", "image/heic", "image/jpeg"],
+ ["photo.heic", "", "image/jpeg"],
+ ["photo.heif", "application/octet-stream", "image/jpeg"],
+ ["scan.tif", "image/tiff", "image/png"],
+ ["icon.bmp", "", "image/png"],
+ ["pic.avif", "image/avif", "image/png"],
+ ["pic.png", "image/png", null],
+ ] as const) {
+ assert.ok(isChatImageFile({ name, type }), name);
+ assert.equal(convertedImageType({ name, type }), converted, name);
+ }
+ for (const extension of [".heic", ".heif", ".avif", ".bmp", ".tif", ".tiff"]) {
+ assert.ok(CHAT_IMAGE_ACCEPT.split(",").includes(extension), extension);
+ }
+ assert.ok(!isChatImageFile({ name: "pic.png", type: "" }));
+ assert.ok(!isChatImageFile({ name: "x.constructor", type: "" }));
+});
diff --git a/studio/frontend/tests/native-drop-targets.test.ts b/studio/frontend/tests/native-drop-targets.test.ts
index e1dba24f557..2305ac89f2f 100644
--- a/studio/frontend/tests/native-drop-targets.test.ts
+++ b/studio/frontend/tests/native-drop-targets.test.ts
@@ -215,17 +215,17 @@ test("the shared image picker owns native drops and ignores stale reads", async
// The picker rejects a format the native side would refuse anyway, so the two
// lists have to stay in step or a droppable image starts being turned away.
-test("the picker's droppable formats match the native path policy", async () => {
+test("the picker's droppable formats are ones the native path policy admits", async () => {
const rust = readText("../../src-tauri/src/native_path_policy.rs");
const listed = (source: string, pattern: RegExp) =>
[...(source.match(pattern)?.[1].matchAll(/"([a-z0-9]+)"/g) ?? [])]
.map((match) => match[1])
.sort();
- assert.deepEqual(
- listed(IMAGE_DROPZONE, /NATIVE_IMAGE_EXTS\s*=\s*\[([^\]]+)\]/),
- listed(rust, /IMAGE_ATTACHMENT_EXTS:\s*&\[&str\]\s*=\s*&\[([^\]]+)\]/),
- );
+ const admitted = listed(rust, /IMAGE_ATTACHMENT_EXTS:\s*&\[&str\]\s*=\s*&\[([^\]]+)\]/);
+ const droppable = listed(IMAGE_DROPZONE, /NATIVE_IMAGE_EXTS\s*=\s*\[([^\]]+)\]/);
+ assert.ok(droppable.length > 0);
+ assert.deepEqual(droppable.filter((ext) => !admitted.includes(ext)), []);
});
// Tauri repeats "over" for every cursor move, and useNativeModelDrop sits in
diff --git a/studio/frontend/tests/sandbox-attachments.test.ts b/studio/frontend/tests/sandbox-attachments.test.ts
new file mode 100644
index 00000000000..d35deb4cdf8
--- /dev/null
+++ b/studio/frontend/tests/sandbox-attachments.test.ts
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+
+import { loadWithStubs } from "./helpers/module-stubs.ts";
+
+const accept = await import("../src/features/chat/open-document-accept.ts");
+const { TOOL_ONLY_ATTACHMENT_EXTENSIONS } = accept;
+const uploads: string[] = [];
+const originals = loadWithStubs<typeof import("../src/features/chat/attachment-originals.ts")>(
+ new URL("../src/features/chat/attachment-originals.ts", import.meta.url),
+ {
+ "./api/chat-api": {
+ uploadChatAttachmentOriginal: async (file: File) => {
+ uploads.push(file.name);
+ return { sha256: SHA, sizeBytes: 3 };
+ },
+ },
+ },
+);
+const { persistAttachmentOriginals, reuseStagedUpload, withAttachmentOriginal } = originals;
+const { sandboxAttachmentPath, sandboxReader, withSandboxAttachmentPaths } =
+ loadWithStubs<typeof import("../src/features/chat/sandbox-attachments.ts")>(
+ new URL("../src/features/chat/sandbox-attachments.ts", import.meta.url),
+ { "./attachment-originals": originals, "./open-document-accept": accept },
+ );
+
+const SHA = "ab".repeat(32);
+
+// Same table as test_sandbox_attachment_paths_match_the_frontend in test_sandbox_files_and_storage_roots.py.
+const PATHS: [string, string][] = [
+ ["data.csv", "data.csv"],
+ ["../deck?.pptx", "_deck_.pptx"],
+ [" .hidden. ", "hidden"],
+ ["季度".repeat(45) + ".xlsx", "季度".repeat(12) + "季.xlsx"],
+ ["a".repeat(79) + " ." + "x".repeat(20), "a".repeat(79)],
+ ["b".repeat(10) + "." + "x".repeat(100), "b".repeat(10)],
+ ["..", "attachment"],
+ ["e".repeat(100), "e".repeat(80)],
+ ["CON.csv", "_CON.csv"],
+ ["nul.tar.gz", "_nul.tar.gz"],
+ ["com1", "_com1"],
+ ["CONSOLE.txt", "CONSOLE.txt"],
+];
+
+test("the sandbox path is the server's, and its basename derives it again", () => {
+ for (const [name, base] of PATHS) {
+ const path = sandboxAttachmentPath(SHA, name);
+ assert.equal(path, `.unsloth_attachments/abababababab/${base}`, name);
+ assert.equal(sandboxAttachmentPath(SHA, base), path, name);
+ }
+});
+
+type Part = { type: string; text: string };
+const kept = (name: string, sha256: string) => ({
+ name,
+ content: [{ type: "text", text: name }] as Part[],
+ original: { sha256, sizeBytes: 1 },
+});
+
+test("kept files across the history name their sandbox copy once each", () => {
+ const [a, b, c] = ["a", "b", "c"].map((x) => x.repeat(64));
+ const book = kept("book.csv", a!);
+ const deck = kept("deck?.pptx", b!);
+ const table = kept("t.parquet", c!);
+ const notes = { name: "notes.txt", content: [] as Part[] };
+ const messages = [
+ { attachments: [notes, book] },
+ { attachments: [] },
+ { attachments: [deck, book, table] },
+ ];
+ const { messages: rendered, sandboxAttachments } =
+ withSandboxAttachmentPaths(messages);
+ assert.deepEqual(sandboxAttachments, [
+ { sha256: b, name: "deck_.pptx" },
+ { sha256: a, name: "book.csv" },
+ { sha256: c, name: "t.parquet" },
+ ]);
+ assert.equal(rendered[0]!.attachments[0], notes);
+ assert.equal(rendered[1], messages[1]);
+ const text = (m: number, i: number) =>
+ (rendered[m]!.attachments[i] as { content: Part[] }).content[0]!.text;
+ assert.equal(
+ text(2, 0),
+ '[deck?.pptx: its text is below, so answer from it. For calculations, the python tool has the file at path = ".unsloth_attachments/bbbbbbbbbbbb/deck_.pptx"; fitz.open(path)]',
+ );
+ assert.equal(
+ text(2, 2),
+ '[t.parquet is saved at .unsloth_attachments/cccccccccccc/t.parquet in the python tool\'s working directory; open it with pandas.read_parquet(path), where path = ".unsloth_attachments/cccccccccccc/t.parquet"]',
+ );
+ assert.equal(book.content.length, 1);
+});
+
+test("a thread past the request's limit carries its most recent files, not none", () => {
+ const one = (i: number) => kept(`f${i}.parquet`, i.toString(16).padStart(64, "0"));
+ const all = Array.from({ length: 70 }, (_, i) => ({ attachments: [one(i)] }));
+ all.push({ attachments: [one(0)] }); // Attached again late, so it counts as recent.
+ const { messages, sandboxAttachments } = withSandboxAttachmentPaths(all);
+ assert.equal(sandboxAttachments.length, 64);
+ assert.deepEqual(
+ [sandboxAttachments[0]!.name, sandboxAttachments.at(-1)!.name],
+ ["f7.parquet", "f0.parquet"],
+ );
+ assert.equal(messages[6]!.attachments[0], all[6]!.attachments[0]);
+ assert.notEqual(messages[7]!.attachments[0], all[7]!.attachments[0]);
+});
+
+test("a document outside the kept types is uploaded only for the python tool", async () => {
+ const send = (name: string, python: boolean, temporary = false, extra = {}) => {
+ const file = new File(["a,b"], name);
+ const pending = { id: "1", type: "document", name, file } as never;
+ const complete = { id: "1", type: "document", name, content: [], ...extra } as never;
+ return withAttachmentOriginal(pending, complete, temporary, 0, python);
+ };
+ const original = { sha256: SHA, sizeBytes: 3 };
+ assert.deepEqual(await send("data.csv", false), { id: "1", type: "document", name: "data.csv", content: [] });
+ assert.deepEqual((await send("data.csv", true) as { original?: unknown }).original, original);
+ assert.ok("file" in (await send("a.pdf", false, true)));
+ assert.ok("file" in (await send("a.pdf", true, true)));
+ await send("t.parquet", true, false, { original });
+ assert.deepEqual(uploads, ["data.csv"]);
+});
+
+test("only a python turn asks for copies, and every tool-only file has a reader", () => {
+ const read = (path: string) =>
+ readFileSync(new URL(`../src/features/chat/${path}`, import.meta.url), "utf8");
+ assert.match(
+ read("runtime-provider.tsx"),
+ /withAttachmentOriginal\(\s*attachment,\s*await this\.delegate\.send\(attachment\),\s*incognito,\s*epoch,\s*pythonToolRunsInStudio\(\),\s*\)/,
+ );
+ const adapter = read("api/chat-adapter.ts");
+ assert.match(
+ adapter,
+ /supportsStudioToolsForThisTurn &&\s*studioLocalCodeTools\.includes\("python"\)\s*\? withSandboxAttachmentPaths\(survivingMessages\)[^;]*;\s*(\/\/[^\n]*\n\s*)*(?:const|let) outboundMessages = renderedMessages/,
+ );
+ assert.equal(adapter.split("{ sandbox_attachments: sandboxAttachments }").length, 3);
+ for (const extension of TOOL_ONLY_ATTACHMENT_EXTENSIONS.split(",")) {
+ assert.ok(sandboxReader(`Data${extension.toUpperCase()}`), extension);
+ }
+ assert.equal(sandboxReader("logs.tar.gz"), "tarfile.open(path)");
+ assert.equal(sandboxReader("report.odt"), null);
+});
+
+test("a malformed stored hash is left out instead of failing the turn", () => {
+ const message = { attachments: [{ name: "a.csv", original: { sha256: "NOT-HEX", sizeBytes: 1 } }] };
+ assert.deepEqual(withSandboxAttachmentPaths([message]).sandboxAttachments, []);
+});
+
+test("saving a temporary chat stores a document it kept for the python tool", async () => {
+ const file = new File(["a,b"], "data.csv");
+ const [saved] = await persistAttachmentOriginals([{ id: "1", type: "document", name: "data.csv", content: [], file } as never], 0);
+ assert.deepEqual((saved as { original?: unknown }).original, { sha256: SHA, sizeBytes: 3 });
+ assert.ok(!("file" in (saved as object)));
+});
+
+test("a staged upload is sent again once it nears the originals sweep", () => {
+ const now = 10 * 60 * 60 * 1000;
+ assert.equal(reuseStagedUpload(now - 60_000, now), true);
+ assert.equal(reuseStagedUpload(now - 31 * 60_000, now), false);
+ assert.equal(reuseStagedUpload(undefined, now), false);
+});
diff --git a/studio/src-tauri/src/native_clipboard.rs b/studio/src-tauri/src/native_clipboard.rs
index b2e2d588edd..42a1fbef55f 100644
--- a/studio/src-tauri/src/native_clipboard.rs
+++ b/studio/src-tauri/src/native_clipboard.rs
@@ -2,8 +2,9 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
use crate::native_path_policy::{
- is_audio_only_3gp, is_binary_property_list, is_binary_tracker_mod, is_binary_vobsub,
- is_binary_office_template, is_compiled_fortran_mod,
+ has_transport_stream_extension, is_audio_only_3gp, is_binary_office_template,
+ is_binary_property_list, is_binary_tracker_mod, is_binary_vobsub, is_compiled_fortran_mod,
+ is_mpeg_transport_stream,
};
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use serde::Serialize;
@@ -88,6 +89,11 @@ fn clipboard_file_mime_type(path: &Path) -> Option<&'static str> {
"png" => "image/png",
"webp" => "image/webp",
"gif" => "image/gif",
+ "heic" => "image/heic",
+ "heif" => "image/heif",
+ "avif" => "image/avif",
+ "bmp" => "image/bmp",
+ "tif" | "tiff" => "image/tiff",
"mp4" => "video/mp4",
"m4v" => "video/x-m4v",
"mov" => "video/quicktime",
@@ -98,10 +104,12 @@ fn clipboard_file_mime_type(path: &Path) -> Option<&'static str> {
"wmv" => "video/x-ms-wmv",
"flv" => "video/x-flv",
"ogv" => "video/ogg",
+ "m2ts" => "video/mp2t",
"pdf" => "application/pdf",
"docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"odt" => "application/vnd.oasis.opendocument.text",
"ods" => "application/vnd.oasis.opendocument.spreadsheet",
+ "rtf" => "application/rtf",
"mp3" | "mp2" => "audio/mpeg",
"wav" => "audio/wav",
"m4a" => "audio/mp4",
@@ -122,6 +130,9 @@ fn clipboard_file_mime_type(path: &Path) -> Option<&'static str> {
// list the composer and the drop path already share.
"txt" => "text/plain",
other if crate::native_path_policy::TEXT_ATTACHMENT_EXTS.contains(&other) => "text/plain",
+ other if crate::native_path_policy::TOOL_ONLY_ATTACHMENT_EXTS.contains(&other) => {
+ "application/octet-stream"
+ }
_ => return None,
};
Some(mime_type)
@@ -187,8 +198,9 @@ fn read_clipboard_files(paths: Vec<PathBuf>) -> Result<Vec<NativeClipboardFile>,
// A 3GP recording cannot use its final size limit until its BMFF
// handlers have been read and classified as audio-only or video. Read it
// under the larger of the two, as the drop path does; the audio cap is
- // reapplied below once the track handlers say it is audio-only.
- let provisional_limit = if is_3gp {
+ // reapplied below once the track handlers say it is audio-only. A .ts or
+ // .mts path is likewise video or TypeScript only once its packets are read.
+ let provisional_limit = if is_3gp || has_transport_stream_extension(&path) {
MAX_CLIPBOARD_VIDEO_BYTES
} else {
clipboard_file_max_bytes(mime_type)
@@ -214,6 +226,8 @@ fn read_clipboard_files(paths: Vec<PathBuf>) -> Result<Vec<NativeClipboardFile>,
}
let mime_type = if is_3gp && is_audio_only_3gp(&bytes) {
"audio/3gpp"
+ } else if is_mpeg_transport_stream(&path, &bytes) {
+ "video/mp2t"
} else {
mime_type
};
@@ -463,6 +477,15 @@ mod tests {
assert_eq!(clipboard_file_mime_type(Path::new("unknown.bin")), None);
}
+ #[test]
+ fn clipboard_file_mime_types_cover_composer_documents() {
+ assert_eq!(clipboard_file_mime_type(Path::new("notes.rtf")), Some("application/rtf"));
+ assert_eq!(
+ clipboard_file_mime_type(Path::new("archive.tar.gz")),
+ Some("application/octet-stream")
+ );
+ }
+
#[test]
fn clipboard_file_mime_types_cover_chat_video_attachments() {
for (name, mime_type) in [
@@ -477,6 +500,7 @@ mod tests {
("clip.wmv", "video/x-ms-wmv"),
("clip.flv", "video/x-flv"),
("clip.ogv", "video/ogg"),
+ ("clip.m2ts", "video/mp2t"),
("clip.3gp", "video/3gpp"),
] {
assert_eq!(clipboard_file_mime_type(Path::new(name)), Some(mime_type));
@@ -591,6 +615,26 @@ mod tests {
assert_eq!(files[1].mime_type, "video/3gpp");
}
+ #[test]
+ fn clipboard_transport_stream_is_video_and_typescript_is_text() {
+ let directory = tempfile::tempdir().unwrap();
+ let stream = directory.path().join("broadcast.ts");
+ let mut raw = vec![0; 188 * 4];
+ for offset in (0..raw.len()).step_by(188) {
+ raw[offset] = 0x47;
+ }
+ std::fs::write(&stream, raw).unwrap();
+ let typescript = directory.path().join("index.ts");
+ // Opens with the sync byte's "G", so one matching packet start is not enough.
+ let source = format!("GENERATED\n{}", "export const value = 1;\n".repeat(40));
+ std::fs::write(&typescript, source).unwrap();
+
+ let files = read_clipboard_files(vec![stream, typescript]).unwrap();
+ assert_eq!(files.len(), 2);
+ assert_eq!(files[0].mime_type, "video/mp2t");
+ assert_eq!(files[1].mime_type, "text/plain");
+ }
+
#[test]
fn clipboard_audio_uses_the_chat_upload_boundary() {
assert_eq!(
diff --git a/studio/src-tauri/src/native_intents.rs b/studio/src-tauri/src/native_intents.rs
index 10a861bbc95..525ab58e5b8 100644
--- a/studio/src-tauri/src/native_intents.rs
+++ b/studio/src-tauri/src/native_intents.rs
@@ -5,8 +5,9 @@ use crate::native_backend_lease::{
};
use crate::native_path_policy::{
classify_artifact_path, classify_native_attachment_path, classify_native_dataset_path,
- classify_native_document_folder, classify_native_model_path, is_audio_only_3gp,
- is_binary_property_list, is_binary_tracker_mod, is_binary_vobsub, is_binary_office_template, is_compiled_fortran_mod, is_text_attachment_name,
+ classify_native_document_folder, classify_native_model_path, has_transport_stream_extension,
+ is_audio_only_3gp, is_binary_office_template, is_binary_property_list, is_binary_tracker_mod,
+ is_binary_vobsub, is_compiled_fortran_mod, is_mpeg_transport_stream, is_text_attachment_name,
reveal_target, ClassifiedPath, NativeArtifactKind,
};
use serde::Serialize;
@@ -638,6 +639,11 @@ fn attachment_mime_type(path: &Path) -> Option<&'static str> {
"png" => Some("image/png"),
"webp" => Some("image/webp"),
"gif" => Some("image/gif"),
+ "heic" => Some("image/heic"),
+ "heif" => Some("image/heif"),
+ "avif" => Some("image/avif"),
+ "bmp" => Some("image/bmp"),
+ "tif" | "tiff" => Some("image/tiff"),
"wav" => Some("audio/wav"),
"mp3" | "mp2" => Some("audio/mpeg"),
"m4a" => Some("audio/mp4"),
@@ -660,8 +666,10 @@ fn attachment_mime_type(path: &Path) -> Option<&'static str> {
"flv" => Some("video/x-flv"),
"3gp" => Some("video/3gpp"),
"ogv" => Some("video/ogg"),
+ "m2ts" => Some("video/mp2t"),
"ods" => Some("application/vnd.oasis.opendocument.spreadsheet"),
"odt" => Some("application/vnd.oasis.opendocument.text"),
+ "rtf" => Some("application/rtf"),
// Stamped like native_clipboard.rs.
"json" | "jsonl" | "ndjson" | "jsonc" | "json5" | "geojson" | "har" | "avsc"
| "tfstate" => Some("application/json"),
@@ -678,11 +686,17 @@ fn attachment_mime_type(path: &Path) -> Option<&'static str> {
other if crate::native_path_policy::TEXT_ATTACHMENT_EXTS.contains(&other) => {
Some("text/plain")
}
+ other if crate::native_path_policy::TOOL_ONLY_ATTACHMENT_EXTS.contains(&other) => {
+ Some("application/octet-stream")
+ }
_ => None,
}
}
fn attachment_payload_mime_type(path: &Path, raw: &[u8]) -> Option<&'static str> {
+ if is_mpeg_transport_stream(path, raw) {
+ return Some("video/mp2t");
+ }
if path
.extension()
.and_then(|value| value.to_str())
@@ -747,7 +761,11 @@ fn read_attachment_payload(entry: &NativePathEntry) -> Result<NativeAttachmentFi
.is_some_and(|ext| {
crate::native_path_policy::TEXT_ATTACHMENT_EXTS.contains(&ext.as_str())
});
- let max_bytes = if is_text_attachment {
+ // A .ts or .mts path is provisionally video until its packets are read; the text cap is
+ // reapplied below once the bytes say it is TypeScript.
+ let max_bytes = if has_transport_stream_extension(path) {
+ MAX_NATIVE_VIDEO_BYTES
+ } else if is_text_attachment {
MAX_NATIVE_TEXT_BYTES
} else if mime_type.starts_with("image/") {
MAX_NATIVE_IMAGE_BYTES
@@ -808,6 +826,9 @@ fn read_attachment_payload(entry: &NativePathEntry) -> Result<NativeAttachmentFi
}
let mime_type = attachment_payload_mime_type(path, &bytes)
.ok_or_else(|| "Only chat attachments can be read inline.".to_string())?;
+ if mime_type.starts_with("text/") && bytes.len() as u64 > MAX_NATIVE_TEXT_BYTES {
+ return Err("Attachment is unavailable or too large.".to_string());
+ }
// A 3GP path is provisionally video until its track handlers are available.
// Reapply the audio cap after an audio-only recording is identified.
if mime_type.starts_with("audio/") && bytes.len() as u64 > MAX_NATIVE_ATTACHMENT_BYTES {
@@ -927,6 +948,30 @@ mod tests {
let _ = fs::remove_file(path);
}
+ #[test]
+ fn transport_stream_reads_as_video_past_the_text_cap_and_typescript_does_not() {
+ let stream = temp_path("camcorder").with_extension("MTS");
+ let mut raw = vec![0; MAX_NATIVE_TEXT_BYTES as usize + 192];
+ for offset in (4..raw.len()).step_by(192) {
+ raw[offset] = 0x47;
+ }
+ fs::write(&stream, &raw).unwrap();
+ let (_state, entry) = attachment_entry(&stream);
+ assert_eq!(
+ read_attachment_payload(&entry).unwrap().mime_type,
+ "video/mp2t"
+ );
+ let typescript = temp_path("module").with_extension("ts");
+ fs::write(&typescript, vec![b' '; MAX_NATIVE_TEXT_BYTES as usize + 1]).unwrap();
+ let (_state, entry) = attachment_entry(&typescript);
+ let Err(error) = read_attachment_payload(&entry) else {
+ panic!("expected oversized TypeScript read to fail");
+ };
+ assert!(error.contains("too large"), "unexpected error: {error}");
+ let _ = fs::remove_file(stream);
+ let _ = fs::remove_file(typescript);
+ }
+
#[test]
fn audio_only_3gp_read_reapplies_the_audio_cap() {
let path = temp_path("oversized-recording").with_extension("3gp");
@@ -1025,10 +1070,12 @@ mod tests {
}
#[test]
- fn open_document_read_round_trips_with_its_mime_type() {
+ fn composer_document_read_round_trips_with_its_mime_type() {
for (ext, mime) in [
("ods", "application/vnd.oasis.opendocument.spreadsheet"),
("odt", "application/vnd.oasis.opendocument.text"),
+ ("rtf", "application/rtf"),
+ ("Parquet", "application/octet-stream"),
] {
let path = temp_path("open-document").with_extension(ext);
fs::write(&path, b"open-document").unwrap();
@@ -1078,8 +1125,15 @@ mod tests {
}
#[test]
- fn every_text_extension_the_drop_accepts_has_a_mime_type() {
- for ext in crate::native_path_policy::TEXT_ATTACHMENT_EXTS {
+ fn every_text_video_and_tool_only_extension_the_drop_accepts_has_a_mime_type() {
+ use crate::native_path_policy::{
+ TEXT_ATTACHMENT_EXTS, TOOL_ONLY_ATTACHMENT_EXTS, VIDEO_ATTACHMENT_EXTS,
+ };
+ for ext in TEXT_ATTACHMENT_EXTS
+ .iter()
+ .chain(VIDEO_ATTACHMENT_EXTS)
+ .chain(TOOL_ONLY_ATTACHMENT_EXTS)
+ {
let path = PathBuf::from(format!("sample.{ext}"));
assert!(attachment_mime_type(&path).is_some(), "{ext}");
}
diff --git a/studio/src-tauri/src/native_path_policy.rs b/studio/src-tauri/src/native_path_policy.rs
index d143317bb82..3ab291662fb 100644
--- a/studio/src-tauri/src/native_path_policy.rs
+++ b/studio/src-tauri/src/native_path_policy.rs
@@ -50,6 +50,82 @@ pub fn classify_native_model_path(path: &Path) -> Result<ClassifiedPath, String>
pub const ATTACHMENT_EXTS: &[&str] = &["pdf", "txt", "md", "markdown", "docx", "html", "htm"];
/// OpenDocument files the chat composer parses directly rather than indexing as RAG sources.
pub const OPEN_DOCUMENT_ATTACHMENT_EXTS: &[&str] = &["ods", "odt"];
+pub const RTF_ATTACHMENT_EXTS: &[&str] = &["rtf"];
+/// Files only the python tool reads; keep in sync with `open-document-accept.ts`.
+pub const TOOL_ONLY_ATTACHMENT_EXTS: &[&str] = &[
+ "parquet",
+ "feather",
+ "arrow",
+ "orc",
+ "dta",
+ "sas7bdat",
+ "xpt",
+ "mat",
+ "npy",
+ "npz",
+ "safetensors",
+ "sqlite",
+ "sqlite3",
+ "db",
+ "gpkg",
+ "mbtiles",
+ "duckdb",
+ "zip",
+ "jar",
+ "whl",
+ "apk",
+ "tar",
+ "gz",
+ "tgz",
+ "bz2",
+ "tbz2",
+ "tbz",
+ "xz",
+ "txz",
+ "lzma",
+ "epub",
+ "mobi",
+ "fb2",
+ "cbz",
+ "xps",
+ "oxps",
+ "docm",
+ "dotx",
+ "dotm",
+ "potx",
+ "potm",
+ "ppsm",
+ "odp",
+ "odg",
+ "vsdx",
+ "stl",
+ "3mf",
+ "ply",
+ "glb",
+ "kmz",
+ "ttf",
+ "otf",
+ "ttc",
+ "woff",
+ "psd",
+ "ico",
+ "icns",
+ "cur",
+ "tga",
+ "dds",
+ "pcx",
+ "ppm",
+ "pgm",
+ "pbm",
+ "pnm",
+ "qoi",
+ "jp2",
+ "j2k",
+ "xbm",
+ "xpm",
+ "sgi",
+ "fits",
+];
pub const TRAINING_DATASET_EXTS: &[&str] = &["csv", "json", "jsonl", "parquet"];
/// Keep in sync with `text-attachment-accept.ts`. RAG types are absent so a
@@ -571,8 +647,22 @@ pub fn is_audio_only_3gp(raw: &[u8]) -> bool {
has_audio && !has_video
}
+pub fn has_transport_stream_extension(path: &Path) -> bool {
+ has_extension(path, "ts") || has_extension(path, "mts")
+}
+
+/// 0x47 sync byte every 188 bytes, or 192 behind M2TS's 4-byte timestamp.
+pub fn is_mpeg_transport_stream(path: &Path, bytes: &[u8]) -> bool {
+ has_transport_stream_extension(path)
+ && [(0, 188), (4, 192)].iter().any(|&(start, packet)| {
+ (0..3).all(|index| bytes.get(start + index * packet) == Some(&0x47))
+ })
+}
+
/// Vision chat image attachments; keep in sync with `drop-paths.ts` `CHAT_IMAGE_DROP_ACCEPT`.
-pub const IMAGE_ATTACHMENT_EXTS: &[&str] = &["jpg", "jpeg", "png", "webp", "gif"];
+pub const IMAGE_ATTACHMENT_EXTS: &[&str] = &[
+ "jpg", "jpeg", "png", "webp", "gif", "heic", "heif", "avif", "bmp", "tif", "tiff",
+];
/// Chat audio attachments; keep in sync with `audio-attachment-adapter.ts` `accept`.
pub const AUDIO_ATTACHMENT_EXTS: &[&str] = &[
@@ -584,13 +674,15 @@ pub const AUDIO_ATTACHMENT_EXTS: &[&str] = &[
/// `CHAT_VIDEO_DROP_ACCEPT`. llama-server decodes with ffmpeg, so this is what
/// ffmpeg reads, not what the webview can play.
pub const VIDEO_ATTACHMENT_EXTS: &[&str] = &[
- "mp4", "m4v", "mov", "webm", "mkv", "avi", "mpg", "mpeg", "wmv", "flv", "3gp", "ogv",
+ "mp4", "m4v", "mov", "webm", "mkv", "avi", "mpg", "mpeg", "wmv", "flv", "3gp", "ogv", "m2ts",
];
fn accepted_attachment_exts() -> impl Iterator<Item = &'static &'static str> {
ATTACHMENT_EXTS
.iter()
.chain(OPEN_DOCUMENT_ATTACHMENT_EXTS.iter())
+ .chain(RTF_ATTACHMENT_EXTS.iter())
+ .chain(TOOL_ONLY_ATTACHMENT_EXTS.iter())
.chain(TEXT_ATTACHMENT_EXTS.iter())
.chain(IMAGE_ATTACHMENT_EXTS.iter())
.chain(AUDIO_ATTACHMENT_EXTS.iter())
diff --git a/tests/studio/test_cancelled_turn_history_prune.py b/tests/studio/test_cancelled_turn_history_prune.py
index c2ce5e46d48..f04a1c47dce 100644
--- a/tests/studio/test_cancelled_turn_history_prune.py
+++ b/tests/studio/test_cancelled_turn_history_prune.py
@@ -97,6 +97,7 @@ def _send_path_slice() -> str:
)
return (
"export function buildSendPathOutbound(messages: any, isExternalRequest: boolean) {\n"
+ " const supportsStudioToolsForThisTurn = false, studioLocalCodeTools: string[] = [];\n"
# The provider-dependent flag is covered by external-preserve-thinking.test.ts.
+ " const replayReasoning = !isExternalRequest;\n"
+ body