Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep the start of an attached file larger than the context window (#13620)

Daniel Han97ed9415f7cc
diff --git a/studio/backend/core/inference/context_window.py b/studio/backend/core/inference/context_window.py
index c7b77a93cc1..8ddbee5ef56 100644
--- a/studio/backend/core/inference/context_window.py
+++ b/studio/backend/core/inference/context_window.py
@@ -930,3 +930,151 @@ def fit_rolling_context(
"context_length": context_length,
"fits": True,
}
+
+
+# withSandboxAttachmentPaths' note (sandbox-attachments.ts): only these attachments have a copy to point at.
+_SANDBOX_ATTACHMENT_NOTE_RE = re.compile(
+ r"\[(?P<name>[^\n]*?): its text is below, so answer from it\. For calculations, the python tool has the "
+ r'file at path = (?P<path>"(?:[^"\\\n]|\\.)*")(?:;[^\n]*)?\]\n'
+)
+# The frontend's wrapper tag (attachment, pasted_text, document_annotations); the cut goes inside it.
+_ATTACHMENT_OPEN_RE = re.compile(r"<(?P<tag>[A-Za-z_][\w-]*)(?:\s[^\n]*)?>\n")
+# Room left for the fit's own additions (carried block, recall), so a cut turn is not refused one step later.
+_ATTACHMENT_FIT_MARGIN = 0.125
+# No tokenizer packs more characters than this into one token, so a longer body cannot fit and is never counted.
+_MAX_CHARS_PER_TOKEN = 16
+
+
+def _attachment_spans(text: str):
+ """(note, open tag, inner text, close tag, trailing text) per sandbox-backed attachment."""
+ notes = list(_SANDBOX_ATTACHMENT_NOTE_RE.finditer(text))
+ for index, note in enumerate(notes):
+ end = notes[index + 1].start() if index + 1 < len(notes) else len(text)
+ body = text[note.end() : end]
+ opened = _ATTACHMENT_OPEN_RE.match(body)
+ closing = f"\n</{opened.group('tag')}>" if opened else ""
+ close = body.rfind(closing) if opened else -1
+ if opened and close >= opened.end():
+ yield (
+ note,
+ opened.group(0),
+ body[opened.end() : close],
+ closing,
+ body[close + len(closing) :],
+ )
+ else:
+ yield note, "", body, "", ""
+
+
+def _cut_attachment_bodies(text: str, keep_chars: int, *, excerpt: bool) -> str:
+ """Attachments longer than ``keep_chars`` keep their first whole lines (``excerpt``) or only their path."""
+ first = _SANDBOX_ATTACHMENT_NOTE_RE.search(text)
+ if not first:
+ return text
+ out = [text[: first.start()]]
+ for note, opened, inner, close, tail in _attachment_spans(text):
+ name, path = note.group("name"), note.group("path")
+ if len(inner) <= keep_chars:
+ out.append(note.group(0) + opened + inner + close + tail)
+ continue
+ total = inner.count("\n") + (0 if inner.endswith("\n") else 1)
+ kept = inner[:keep_chars] if excerpt else ""
+ kept = kept[: kept.rindex("\n") + 1] if "\n" in kept else ""
+ where = (
+ f"The whole file is at path = {path} in the python tool's working directory, so read the part you "
+ "need with python"
+ )
+ if kept:
+ out.append(
+ f"{note.group(0)}{opened}{kept[:-1]}{close}\n[{name} is too long for the context window: "
+ f"only its first {kept.count(chr(10))} of {total} lines are above. {where} instead of answering "
+ f"from this excerpt.]{tail}"
+ )
+ else:
+ out.append(
+ f"[{name} ({total} lines) is too long for the context window to show. {where}.]{tail}"
+ )
+ return "".join(out)
+
+
+def _cut_content(content: Any, keep_chars: int, *, excerpt: bool) -> Any:
+ if isinstance(content, str):
+ return _cut_attachment_bodies(content, keep_chars, excerpt = excerpt)
+ if isinstance(content, list):
+ return [
+ {**part, "text": _cut_attachment_bodies(part["text"], keep_chars, excerpt = excerpt)}
+ if isinstance(part, dict)
+ and part.get("type") == "text"
+ and isinstance(part.get("text"), str)
+ else part
+ for part in content
+ ]
+ return content
+
+
+def _attachment_body_chars(message: dict) -> int:
+ content = message.get("content")
+ if isinstance(content, str):
+ texts = [content]
+ elif isinstance(content, list):
+ texts = [
+ part["text"]
+ for part in content
+ if isinstance(part, dict) and isinstance(part.get("text"), str)
+ ]
+ else:
+ return 0
+ return max((len(span[2]) for text in texts for span in _attachment_spans(text)), default = 0)
+
+
+def fit_oversized_attachments(
+ messages: list[dict],
+ *,
+ context_length: int,
+ max_tokens: Optional[int],
+ count_tokens: Callable[[list[dict]], int],
+) -> list[dict]:
+ """Cut sandbox-backed attachments too big for the window: the newest turn keeps the start, older turns the path.
+
+ Decided per message, so a turn renders the same on every later request. Rewrites ``content`` in place:
+ eviction matches messages by identity, and a copy of the newest turn was archived as evicted.
+ """
+ if not context_length or context_length <= 1:
+ return messages
+ attached = [
+ index
+ for index, message in enumerate(messages)
+ if message.get("role") == "user" and _attachment_body_chars(message)
+ ]
+ if not attached:
+ return messages
+ budget = prompt_budget(context_length, max_tokens)
+ # The margin sizes a cut; it never decides one, or a file that fits would lose its end.
+ target = int(budget * (1 - _ATTACHMENT_FIT_MARGIN))
+ if count_tokens(messages) <= budget:
+ return messages
+ latest = max(index for index, message in enumerate(messages) if message.get("role") == "user")
+ base = [message for message in messages if message.get("role") == "system"]
+ for index in attached:
+ message = messages[index]
+ excerpt = index == latest
+ longest = _attachment_body_chars(message)
+
+ def fits(keep_chars: int, limit: int = target) -> bool:
+ content = _cut_content(message["content"], keep_chars, excerpt = excerpt)
+ return count_tokens([*base, {**message, "content": content}]) <= limit
+
+ cap = target * _MAX_CHARS_PER_TOKEN
+ if (longest <= budget * _MAX_CHARS_PER_TOKEN and fits(longest, budget)) or (
+ excerpt and not fits(0, budget)
+ ):
+ continue
+ low, high = 0, min(longest, cap) if fits(0) else 0
+ while high - low > 64:
+ middle = (low + high) // 2
+ if fits(middle):
+ low = middle
+ else:
+ high = middle
+ message["content"] = _cut_content(message["content"], low, excerpt = excerpt)
+ return messages
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 8c2c672b46b..4408ec8270b 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -74,6 +74,7 @@ from core.inference.context_window import (
fit_rolling_context,
messages_without_unpriced_media,
retrieval_budget,
+ fit_oversized_attachments,
tool_result_budget,
turn_is_servable,
)
@@ -297,6 +298,16 @@ def _fit_context(messages, **kwargs):
# /compact: only a checkpoint reset can honour it, since only its turns are archived.
force_reset = bool(kwargs.pop("force_reset", False))
sticky_manual = bool(kwargs.pop("sticky_manual", False))
+ if kwargs.get("count_tokens") is not None:
+ try:
+ messages = fit_oversized_attachments(
+ messages,
+ context_length = kwargs.get("context_length") or 0,
+ max_tokens = kwargs.get("max_tokens"),
+ count_tokens = kwargs["count_tokens"],
+ )
+ except Exception: # noqa: BLE001 -- the fit below still refuses what cannot fit
+ logger.debug("attachment fit failed", exc_info = True)
try:
from core.inference import checkpoint
diff --git a/studio/backend/tests/test_oversized_attachment_fit.py b/studio/backend/tests/test_oversized_attachment_fit.py
new file mode 100644
index 00000000000..47ea4616421
--- /dev/null
+++ b/studio/backend/tests/test_oversized_attachment_fit.py
@@ -0,0 +1,265 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An attached file larger than the window: keep its start, point at the sandbox copy."""
+
+from __future__ import annotations
+
+import copy
+
+from core.inference import checkpoint, llama_cpp
+from core.inference.context_window import evicted_messages, fit_oversized_attachments
+
+PATH = '".unsloth_attachments/abc123/notes.txt"'
+
+
+def note(name = "notes.txt", path = PATH):
+ return (
+ f"[{name}: its text is below, so answer from it. For calculations, the python tool has the "
+ f"file at path = {path}]\n"
+ )
+
+
+def attachment(text, name = "notes.txt"):
+ return f"<attachment name={name}>\n{text}\n</attachment>"
+
+
+LINES = "".join(
+ f"line {i}: the quick brown fox jumps over the lazy dog\n" for i in range(1, 2001)
+).rstrip("\n")
+FILE = attachment(LINES)
+
+
+def count(messages):
+ total = 0
+ for message in messages:
+ content = message.get("content")
+ if isinstance(content, list):
+ content = "".join(part.get("text", "") for part in content if isinstance(part, dict))
+ total += max(1, len(str(content or "")) // 4)
+ return total
+
+
+def _messages(body = FILE, marker = None):
+ marker = note() if marker is None else marker
+ return [
+ {"role": "system", "content": "you are helpful"},
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "content": "hello"},
+ {"role": "user", "content": "What does line 1000 say?\n" + marker + body},
+ ]
+
+
+def _fit(messages, context_length = 2048):
+ return fit_oversized_attachments(
+ messages, context_length = context_length, max_tokens = 512, count_tokens = count
+ )
+
+
+def test_an_attachment_too_big_for_the_window_keeps_its_start_and_names_the_copy():
+ fitted = _fit(_messages())
+ text = fitted[-1]["content"]
+
+ assert text.startswith(
+ "What does line 1000 say?\n" + note() + "<attachment name=notes.txt>\nline 1: "
+ )
+ assert "line 2000:" not in text
+ assert text.count("<attachment name=") == text.count("</attachment>") == 1
+ excerpt = text.split("<attachment name=notes.txt>\n", 1)[1].split("\n</attachment>", 1)[0]
+ shown = len(excerpt.split("\n"))
+ assert f"only its first {shown} of 2000 lines are above" in text
+ assert text.index("</attachment>") < text.index("is too long for the context window")
+ assert f"path = {PATH}" in text.split(note(), 1)[1]
+ assert count([fitted[0], fitted[-1]]) <= 2048 - 512
+ assert fitted[:-1] == _messages()[:-1]
+
+
+def test_the_newest_turn_keeps_its_identity_so_it_is_not_archived_as_evicted():
+ messages = _messages()
+ before = list(messages)
+
+ fitted = _fit(messages)
+
+ assert evicted_messages(before, fitted) == []
+
+
+def test_cutting_twice_changes_nothing():
+ once = copy.deepcopy(_fit(_messages()))
+
+ assert _fit(copy.deepcopy(once)) == once
+
+
+def test_an_attachment_that_fits_is_untouched():
+ messages = _messages(body = attachment("line 1: short"))
+
+ assert _fit(copy.deepcopy(messages)) == messages
+
+
+def test_an_attachment_with_no_sandbox_copy_is_left_to_the_refusal():
+ messages = _messages(marker = "")
+
+ assert _fit(copy.deepcopy(messages)) == messages
+
+
+def test_a_turn_too_big_without_its_attachment_is_left_to_the_refusal():
+ messages = _messages()
+ messages[-1]["content"] = "x" * 20_000 + "\n" + note() + FILE
+
+ assert _fit(copy.deepcopy(messages)) == messages
+
+
+def test_a_name_with_brackets_is_still_recognised():
+ name = "report [final].txt"
+ messages = _messages(body = attachment(LINES, name), marker = note(name))
+
+ assert "of 2000 lines are above" in _fit(messages)[-1]["content"]
+
+
+def test_content_parts_are_cut_in_place():
+ messages = _messages()
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
+ messages[-1] = {
+ "role": "user",
+ "content": [{"type": "text", "text": messages[-1]["content"]}, image],
+ }
+
+ fitted = _fit(messages)
+
+ assert "of 2000 lines are above" in fitted[-1]["content"][0]["text"]
+ assert fitted[-1]["content"][1] == image
+
+
+def test_the_compaction_fit_serves_the_turn_instead_of_refusing_it(monkeypatch):
+ monkeypatch.setattr(checkpoint, "CONTEXT_POLICY", "rolling")
+
+ fitted, truncation = llama_cpp._fit_context(
+ _messages(), context_length = 2048, max_tokens = 512, count_tokens = count
+ )
+
+ assert "of 2000 lines are above" in fitted[-1]["content"]
+ assert truncation is None or truncation.get("fits") is not False
+
+
+def _follow_up(first = None):
+ first = first or _messages()
+ return first + [
+ {
+ "role": "assistant",
+ "content": "Line 1000 reads: line 1000: the quick brown fox jumps over the lazy dog",
+ },
+ {"role": "user", "content": "And line 1500?"},
+ ]
+
+
+def test_an_earlier_oversized_attachment_keeps_only_its_path():
+ fitted = _fit(_follow_up())
+ earlier = fitted[3]["content"]
+
+ assert earlier == (
+ "What does line 1000 say?\n[notes.txt (2000 lines) is too long for the context window to show. "
+ f"The whole file is at path = {PATH} in the python tool's working directory, so read the part you "
+ "need with python.]"
+ )
+ assert fitted[-1] == _follow_up()[-1]
+ assert count(fitted) <= 2048 - 512
+
+
+def test_a_small_file_beside_an_earlier_oversized_one_stays_whole():
+ small_path = '".unsloth_attachments/def456/small.txt"'
+ small = note("small.txt", small_path) + attachment("one short line", "small.txt")
+ first = _messages()
+ first[-1]["content"] += "\n" + small
+
+ earlier = _fit(_follow_up(first))[3]["content"]
+
+ assert "(2000 lines) is too long" in earlier
+ assert small in earlier
+
+
+def test_an_earlier_turn_renders_the_same_on_every_later_request():
+ later = _follow_up() + [
+ {
+ "role": "assistant",
+ "content": "Line 1500 reads: line 1500: the quick brown fox jumps over the lazy dog",
+ },
+ {"role": "user", "content": "And line 2000?"},
+ ]
+
+ assert _fit(later)[3] == _fit(_follow_up())[3]
+
+
+def test_a_conversation_that_fits_skips_the_attachment_checks():
+ calls = []
+
+ def counting(messages):
+ calls.append(len(messages))
+ return count(messages)
+
+ messages = _messages(body = attachment("line 1: short"))
+ fit_oversized_attachments(messages, context_length = 2048, max_tokens = 512, count_tokens = counting)
+
+ assert calls == [len(messages)]
+
+
+def test_a_long_paste_is_cut_inside_its_own_wrapper():
+ pasted = f"<pasted_text name=paste.txt bytes=110000>\n{LINES}\n</pasted_text>"
+ text = _fit(_messages(body = pasted, marker = note("paste.txt")))[-1]["content"]
+
+ assert text.count("<pasted_text name=") == text.count("</pasted_text>") == 1
+ excerpt = text.split("bytes=110000>\n", 1)[1].split("\n</pasted_text>", 1)[0]
+ assert f"only its first {len(excerpt.split(chr(10)))} of 2000 lines are above" in text
+
+
+def test_document_annotations_are_cut_inside_their_wrapper():
+ wrapped = f'<document_annotations file="notes.pdf">\n{LINES}\n</document_annotations>'
+ text = _fit(_messages(body = wrapped))[-1]["content"]
+
+ assert text.count("<document_annotations") == text.count("</document_annotations>") == 1
+
+
+def test_a_file_that_fits_the_budget_is_not_trimmed_by_the_margin():
+ # Between 87.5% and 100% of the prompt budget (1536): fits as sent.
+ lines = "".join(f"row {i:04d} ok\n" for i in range(1, 450)).rstrip("\n")
+ messages = _messages(body = attachment(lines))
+ assert 0.875 * 1536 < count(messages) <= 1536
+
+ assert _fit(copy.deepcopy(messages)) == messages
+
+
+def test_a_path_only_turn_that_fits_the_budget_is_served():
+ # The system prompt fills 90% of the prompt budget (1536): no room for an excerpt, room for the path.
+ messages = _messages()
+ messages[0] = {"role": "system", "content": "s" * (4 * 1380)}
+
+ text = _fit(messages)[-1]["content"]
+
+ assert "line 1: " not in text
+ assert "(2000 lines) is too long for the context window to show" in text
+ assert count([messages[0], messages[-1]]) <= 1536
+
+
+def test_an_excerpt_ending_in_blank_lines_keeps_them_and_counts_right():
+ lines = "\n".join(
+ ("" if i % 3 == 0 else f"line {i}: the quick brown fox") for i in range(1, 3002)
+ )
+ text = _fit(_messages(body = attachment(lines)))[-1]["content"]
+
+ excerpt = text.split("<attachment name=notes.txt>\n", 1)[1].split("\n</attachment>", 1)[0]
+ assert excerpt.endswith("\n")
+ assert f"only its first {len(excerpt.split(chr(10)))} of 3001 lines are above" in text
+
+
+def test_a_huge_file_is_sized_without_counting_all_of_it():
+ sizes = []
+
+ def counting(messages):
+ sizes.append(sum(len(str(m.get("content", ""))) for m in messages))
+ return count(messages)
+
+ body = attachment(LINES * 200)
+ fit_oversized_attachments(
+ _messages(body = body), context_length = 2048, max_tokens = 512, count_tokens = counting
+ )
+
+ # Only the first gate sees the whole turn; the search never counts more than the window could hold.
+ assert max(sizes[1:]) < 1536 * 16 + 1000