Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: replay a web_search result the way the client sent it (#10131)
Long Yixingcb2d2a0f6707
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 6a2a1189f8b..e1e7f330a70 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -1496,10 +1496,16 @@ def _archive_branch_chain(def _archive_as_wire(messages: Optional[list[dict]]) -> list[dict]:- """Project persisted rows and wire messages into boundary-counting wire units."""+ """Project persisted rows and wire messages into boundary-counting wire units.++ Unsanitised: `_branch_boundary_anchor` writes the anchor off the request itself, so a+ projection that rewrote assistant text here would look for something nobody wrote. A+ client sending the tokens raw, which the Studio one never does, then found no anchor+ and replayed the stale count instead of rebasing it.+ """try:from core.rag import conversation_archive- return conversation_archive._as_wire(list(messages or ()))+ return conversation_archive._as_wire(list(messages or ()), sanitise_assistant = False)except Exception:return list(messages or ())diff --git a/studio/backend/core/rag/conversation_archive.py b/studio/backend/core/rag/conversation_archive.pyindex 8f5c32bc2f9..d6b77215fa9 100644--- a/studio/backend/core/rag/conversation_archive.py+++ b/studio/backend/core/rag/conversation_archive.py@@ -899,6 +899,17 @@ _SERVER_BUILTIN_NAMES = frozenset({"web_search", "web_fetch", "code_execution",# `SANDBOX_FILE_TOOLS`, and `tool_loop_controller._SANDBOX_TOOLS`. Only these two wrap._SANDBOX_TOOL_NAMES = frozenset({"python", "terminal"})+# `search-images.ts`. `re.ASCII` on the scheme because Python's IGNORECASE folds Unicode,+# so `httpſ://` (U+017F) would match where JavaScript's `/i` does not.+_SEARCH_IMAGE_ID = re.compile(r"[0-9a-f]{12}")+_SEARCH_IMAGE_SOURCE = re.compile(r"https?://", re.IGNORECASE | re.ASCII)+_SEARCH_IMAGE_TOKEN = re.compile(+ r"\n\n[ \t]*\[\[img:[0-9a-f]{12}\]\][ \t]*(?=\n\n|\n?\Z)|\[\[img:[0-9a-f]{12}\]\]"+)+# `sanitizeAssistantReplayText`. An audio model answers with an `<audio-player src=...>`+# tag holding the whole wav inline, and the serializer sends `[audio]` in its place.+_REPLAY_AUDIO_DATA_URI = re.compile(r"data:audio/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+")+def _server_builtin(part: dict) -> tuple[bool, bool]:"""Whether a persisted call is a provider-side builtin, and whether it has a native part.@@ -918,43 +929,128 @@ def _server_builtin(part: dict) -> tuple[bool, bool]:return bool(args.get("_server_tool") is True or native), native-def _unwrapped(result, tool_name: str):- """A sandbox or MCP-image wrapper reduced to the text the model actually saw.-- `python` and `terminal` results are wrapped in `{text, images, sessionId, files}` on- EVERY call, and the replay adapter sends `result.text` alone rather than feeding the- model a session id and file metadata. Serialising the whole wrapper reconstructed a- tool message that can never equal the archived one.-- Both gates are the frontend's: the name, because a third-party tool answering with- `{text, sessionId, images}` is someone else's and unwrapping it would drop every other- field, and the shape.- """+# One predicate per frontend predicate of the same name. `sessionId` and `subject` are+# tested for ABSENCE, not null, because the frontend tests them against `undefined`.+def _mcp_image_result(result) -> bool:if not isinstance(result, dict) or not isinstance(result.get("text"), str):- return None+ return Falseimages = result.get("images")- if not isinstance(images, list):- return None- if tool_name in _SANDBOX_TOOL_NAMES and isinstance(result.get("sessionId"), str):- files = result.get("files")- if files is None or (- isinstance(files, list)- and all(isinstance(f, dict) and isinstance(f.get("name"), str) for f in files)- ):- return result["text"]- # The MCP image shape carries no session and always has at least one image.- if (- result.get("sessionId") is None- and images+ return (+ "sessionId" not in result+ and isinstance(images, list)+ and bool(images)and all(isinstance(image, dict)and isinstance(image.get("data"), str)and isinstance(image.get("mimeType"), str)for image in images)+ )+++def _search_image_entry(entry) -> bool:+ if not isinstance(entry, dict):+ return False+ return (+ isinstance(entry.get("id"), str)+ and _SEARCH_IMAGE_ID.fullmatch(entry["id"]) is not None+ and isinstance(entry.get("title"), str)+ and isinstance(entry.get("domain"), str)+ and isinstance(entry.get("source"), str)+ and _SEARCH_IMAGE_SOURCE.match(entry["source"]) is not None+ and ("subject" not in entry or isinstance(entry["subject"], str))+ )+++def _search_images_result(result) -> bool:+ if not isinstance(result, dict) or not isinstance(result.get("text"), str):+ return False+ entries = result.get("webImages")+ return (+ isinstance(entries, list)+ and bool(entries)+ and all(_search_image_entry(entry) for entry in entries)+ )+++def _sandbox_wrapper(result, tool_name: str) -> bool:+ # The name gates it as well as the shape: another tool answering with+ # `{text, sessionId, images}` is someone else's, and unwrapping it drops its rest.+ if not isinstance(result, dict) or not isinstance(result.get("text"), str):+ return False+ if tool_name not in _SANDBOX_TOOL_NAMES or not isinstance(result.get("sessionId"), str):+ return False+ if not isinstance(result.get("images"), list):+ return False+ files = result.get("files")+ return files is None or (+ isinstance(files, list)+ and all(isinstance(f, dict) and isinstance(f.get("name"), str) for f in files)+ )+++def _strip_search_image_tokens(text: str) -> str:+ """`stripSearchImageTokens`. A token resolves only against the message that produced+ it, so the frontend drops them rather than replay an unresolvable id.++ Its code-block carve-out is NOT mirrored: deciding it takes the frontend's whole+ markdown scanner, and nothing that reaches this writes a token into code. The tool+ cannot open a region (`_web_search` collapses every title and snippet behind a label,+ and the url branch returns before an envelope is appended); a reply that fences one+ reconstructs exactly as it did before anything here stripped.+ """+ if "[[img:" not in text:+ return text+ return _SEARCH_IMAGE_TOKEN.sub("", text)+++def _sanitised_assistant_text(text: str) -> str:+ """`sanitizeAssistantReplayText`: an assistant reply as the serializer replays it.++ The same two substitutions the tool result gets, for the same reason. A reply that+ shows a picture carries the token that placed it, and an audio turn carries its whole+ wav, so a stored reply reconstructed verbatim described a message the request never+ sent and the turn matched no transcript seat.+ """+ return _REPLAY_AUDIO_DATA_URI.sub("[audio]", _strip_search_image_tokens(text))+++def _sanitised_assistant_content(content):+ """`content` with every part `_probe_text` reads as text sanitised, others untouched.++ Keyed on the `text` field rather than on the type, as `_probe_text` is, so a part+ shape it renders cannot slip past this one.+ """+ if isinstance(content, str):+ return _sanitised_assistant_text(content)+ if not isinstance(content, list):+ return content+ return [+ {**part, "text": _sanitised_assistant_text(part["text"])}+ if isinstance(part, dict)+ and part.get("type") not in ("reasoning", "tool-call")+ and isinstance(part.get("text"), str)+ else part+ for part in content+ ]+++def _unwrapped(result, tool_name: str):+ """A wrapper this app put around a result, reduced to the text the model actually saw.++ The adapter replays that text alone, so serialising the wrapper reconstructed a tool+ message that can never equal the archived one. Being a wrapper and losing the tokens+ are two questions, as they are in the serializer: a result carrying both `images` and+ `webImages` is unwrapped by the first and stripped by the second.+ """+ if not (+ _mcp_image_result(result)+ or _search_images_result(result)+ or _sandbox_wrapper(result, tool_name)):- return result["text"]- return None+ return None+ text = result["text"]+ return _strip_search_image_tokens(text) if _search_images_result(result) else textdef _tool_result_content(result, tool_name: str = "") -> str:@@ -1023,7 +1119,7 @@ def _flushes_local_pair(part: dict) -> bool:return part.get("result") is not None-def _as_wire(messages: list[dict]) -> list[dict]:+def _as_wire(messages: list[dict], sanitise_assistant: bool = True) -> list[dict]:"""Persisted chat rows in the shape the inference layer sends them.The store keeps a tool call as a ``tool-call`` CONTENT PART carrying its own result,@@ -1043,6 +1139,11 @@ def _as_wire(messages: list[dict]) -> list[dict]:it. Only the id goes into `tool_calls`: the arguments stay on the content part, where`_probe_text` already offers both JSON spellings, and `_is_injected` still sees the idit filters our own injections by.++ `sanitise_assistant` is the STORED side of that comparison. A caller projecting the+ request's own messages against something written WITHOUT this projection must pass+ False: `_branch_boundary_anchor` records an anchor straight off the request, so+ `_archive_as_wire` has to hand the same bytes back or the rebase stops matching."""wire: list[dict] = []for message in messages:@@ -1061,6 +1162,9 @@ def _as_wire(messages: list[dict]) -> list[dict]:if not (isinstance(part, dict) and part.get("type") == "reasoning")]content = parts+ if sanitise_assistant and str(message.get("role") or "") == "assistant":+ content = _sanitised_assistant_content(content)+ parts = content if isinstance(content, list) else None# A provider-side builtin with no native part is not replayed, so it is not a call# here either: counting it invented an exchange the request never carried. Any# other tool-call part takes the replay loop even when the serializer drops it,diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.pyindex ce48304d7f7..00a1f5131ac 100644--- a/studio/backend/tests/test_anthropic_messages.py+++ b/studio/backend/tests/test_anthropic_messages.py@@ -2674,7 +2674,9 @@ class TestAnthropicMessagesToolRouting:)monitor = ApiMonitor(max_entries = 3)monkeypatch.setattr(inf_mod, "api_monitor", monitor)- fields = {"tools": [{"name": "x", "input_schema": {"type": "object"}}]} if with_tools else {}+ fields = (+ {"tools": [{"name": "x", "input_schema": {"type": "object"}}]} if with_tools else {}+ )response = _drive(anthropic_messages(diff --git a/studio/backend/tests/test_checkpoint_compaction.py b/studio/backend/tests/test_checkpoint_compaction.pyindex e6ffb68ad1e..73dc51e04ae 100644--- a/studio/backend/tests/test_checkpoint_compaction.py+++ b/studio/backend/tests/test_checkpoint_compaction.py@@ -2721,6 +2721,35 @@ def test_a_non_prefix_eviction_survives_being_persisted_and_replayed(monkeypatch)+def test_the_boundary_projection_hands_the_anchor_back_unchanged():+ """One side writes the anchor, the other reads it, and only one goes through `_as_wire`.++ `_branch_boundary_anchor` records it straight off the request, so the archive+ projection has to return the same bytes. A client that sends the image tokens or+ inline audio raw, which the Studio one never does, otherwise found no anchor at all+ and replayed the stale count instead of rebasing it against its own branch.+ """+ from core.inference import llama_cpp+ for text in (+ '<audio-player src="data:audio/wav;base64,QUJDRA==" />',+ "here [[img:aabbccddeeff]]",+ "ordinary reply",+ ):+ branch = [+ {"role": "user", "content": "q"},+ {"role": "assistant", "content": text},+ {"role": "user", "content": "next"},+ ]++ written = llama_cpp._branch_boundary_anchor(branch[1:], branch)+ read_back = [+ llama_cpp._anchor_text(message)+ for message in llama_cpp._branch_non_system(llama_cpp._archive_as_wire(branch))+ ]++ assert written and written in read_back, text++def test_a_caller_owned_carried_forward_tag_is_left_alone():"""The delimiter is prompt text, and prompt text belongs to whoever wrote it.diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.pyindex 60422eb5cb4..31c5e4d71aa 100644--- a/studio/backend/tests/test_conversation_archive.py+++ b/studio/backend/tests/test_conversation_archive.py@@ -4,6 +4,7 @@"""The archive behind rolling-context compaction: what it keeps, and what it must not touch."""import copy+import jsonimport osimport sys@@ -3661,6 +3662,271 @@ def test_a_sandbox_result_is_replayed_as_the_text_the_model_saw():]+# A `web_search` that found pictures, as the frontend leaves it on the card.+_IMAGE_ID = "aabbccddeeff"+_IMAGE_ENTRY = {+ "id": _IMAGE_ID,+ "title": "Ragdoll ZQXVARA123",+ "domain": "example.com",+ "source": "https://example.com/ragdoll.jpg",+ "subject": "ragdoll",+}+# `register_images` writes `subject` only when given one -- the ordinary shape has none.+_IMAGE_ENTRY_NO_SUBJECT = {key: value for key, value in _IMAGE_ENTRY.items() if key != "subject"}+_SEARCH_TEXT = (+ "The ZQXVARA123 ragdoll weighs 6 kg.\n\n---\n\n"+ "ragdoll:\n- [[img:%s]] Ragdoll ZQXVARA123 \u2014 example.com" % _IMAGE_ID+)+_SEARCH_TEXT_REPLAYED = (+ "The ZQXVARA123 ragdoll weighs 6 kg.\n\n---\n\n"+ "ragdoll:\n- Ragdoll ZQXVARA123 \u2014 example.com"+)+++_ANSWER = "A ZQXVARA123 ragdoll weighs 6 kg."+++def _image_search_row(+ result,+ tool_name = "web_search",+ answer = _ANSWER,+):+ return [+ {+ "role": "assistant",+ "content": [+ {+ "type": "tool-call",+ "toolCallId": "w1",+ "toolName": tool_name,+ "args": {"query": "ragdoll ZQXVARA123"},+ "result": result,+ },+ {"type": "text", "text": answer},+ ],+ }+ ]+++def _wire_tool_content(rows):+ return [m["content"] for m in conversation_archive._as_wire(rows) if m["role"] == "tool"]+++def test_a_web_search_result_is_replayed_without_its_image_tokens():+ """`{text, webImages}` is a wrapper too, and the tokens do not go back out.++ The `images` key gated both existing wrappers, so a search result carried its whole+ envelope instead and no turn that returned a picture matched what was sent.+ """+ result = {"text": _SEARCH_TEXT, "webImages": [_IMAGE_ENTRY]}++ assert _wire_tool_content(_image_search_row(result)) == [_SEARCH_TEXT_REPLAYED]+ assert _wire_tool_content(+ _image_search_row({"text": _SEARCH_TEXT, "webImages": [_IMAGE_ENTRY_NO_SUBJECT]})+ ) == [_SEARCH_TEXT_REPLAYED]+ assert _wire_tool_content(+ _image_search_row({"text": "[[img:%s]]" % _IMAGE_ID, "webImages": [_IMAGE_ENTRY]})+ ) == ['{"result":""}']+ assert _wire_tool_content(_image_search_row(result, "lookup")) == [_SEARCH_TEXT_REPLAYED]+ # A token alone in a block takes its introducing blank line too.+ assert _wire_tool_content(+ _image_search_row(+ {+ "text": "The ragdoll:\n\n[[img:%s]]\n\nIt weighs 6 kg." % _IMAGE_ID,+ "webImages": [_IMAGE_ENTRY],+ }+ )+ ) == ["The ragdoll:\n\nIt weighs 6 kg."]+ assert _wire_tool_content(+ _image_search_row(+ {+ "text": _SEARCH_TEXT,+ "webImages": [{**_IMAGE_ENTRY, "source": "HTTPS://example.com/cat.jpg"}],+ }+ )+ ) == [_SEARCH_TEXT_REPLAYED]+ assert _wire_tool_content(+ _image_search_row(+ {+ "text": "see [[img:%s]] and [[img:not-an-id]]" % _IMAGE_ID,+ "webImages": [_IMAGE_ENTRY],+ }+ )+ ) == ["see and [[img:not-an-id]]"]+++def test_an_envelope_that_is_not_the_search_shape_is_still_serialised_whole():+ """Unwrapping on `text` alone would drop every other field a tool returned.++ Every entry field is re-checked; a result failing any of them goes out as JSON.+ """+ rejected = [+ [],+ "not a list",+ [{**_IMAGE_ENTRY, "source": "javascript:alert(1)"}],+ # `httpſ` is `https` to a Unicode case fold and not to JavaScript's `/i`.+ [{**_IMAGE_ENTRY, "source": "httpſ://example.com/cat.jpg"}],+ [{**_IMAGE_ENTRY, "id": "nope"}],+ [{**_IMAGE_ENTRY, "title": None}],+ [{**_IMAGE_ENTRY, "domain": 7}],+ # The frontend accepts an absent `subject`, not a null one.+ [{**_IMAGE_ENTRY, "subject": None}],+ [_IMAGE_ENTRY, {**_IMAGE_ENTRY, "id": "nope"}],+ ]+ for entries in rejected:+ result = {"text": _SEARCH_TEXT, "webImages": entries}+ assert _wire_tool_content(_image_search_row(result)) == [+ json.dumps(result, ensure_ascii = False, separators = (",", ":"))+ ], entries+++def test_a_result_that_is_two_wrappers_at_once_is_still_stripped():+ """Being a wrapper and losing the tokens are two questions in the serializer."""+ both = {+ "text": _SEARCH_TEXT,+ "images": [{"data": "AAAA", "mimeType": "image/png"}],+ "webImages": [_IMAGE_ENTRY],+ }+ assert _wire_tool_content(_image_search_row(both)) == [_SEARCH_TEXT_REPLAYED]+ sandboxed = {+ "text": _SEARCH_TEXT,+ "images": [],+ "sessionId": "project-7",+ "webImages": [_IMAGE_ENTRY],+ }+ assert _wire_tool_content(_image_search_row(sandboxed, "terminal")) == [_SEARCH_TEXT_REPLAYED]+ assert _wire_tool_content(+ _image_search_row({"text": _SEARCH_TEXT, "images": both["images"]})+ ) == [_SEARCH_TEXT]+ # A null session is not an absent one: nobody's wrapper.+ nulled = {"text": _SEARCH_TEXT, "images": both["images"], "sessionId": None}+ assert _wire_tool_content(_image_search_row(nulled)) == [+ json.dumps(nulled, ensure_ascii = False, separators = (",", ":"))+ ]+++def _persist_image_search_turn(answer = _ANSWER):+ """The stored rows for one web_search turn, in the shape assistant-ui saves."""+ from storage import studio_db++ studio_db.upsert_chat_thread(+ {"id": THREAD, "title": "t", "modelType": "base", "modelId": "local-model", "createdAt": 1}+ )+ row = _image_search_row({"text": _SEARCH_TEXT, "webImages": [_IMAGE_ENTRY]}, answer = answer)+ rows = [+ ("u0", None, "user", [{"type": "text", "text": "how heavy is a ZQXVARA123 ragdoll"}]),+ ("a0", "u0", "assistant", row[0]["content"]),+ ]+ for index, (identifier, parent, role, content) in enumerate(rows):+ studio_db.upsert_chat_message(+ {+ "id": identifier,+ "threadId": THREAD,+ "parentId": parent,+ "role": role,+ "content": content,+ "createdAt": index + 2,+ }+ )+++# The request the client sent for that turn: one `tool` message, already stripped.+_IMAGE_SEARCH_WIRE = [+ {"role": "user", "content": "how heavy is a ZQXVARA123 ragdoll"},+ {+ "role": "assistant",+ "content": "",+ "tool_calls": [+ {+ "id": "w1",+ "function": {"name": "web_search", "arguments": '{"query":"ragdoll ZQXVARA123"}'},+ }+ ],+ },+ {"role": "tool", "tool_call_id": "w1", "content": _SEARCH_TEXT_REPLAYED},+ {"role": "assistant", "content": "A ZQXVARA123 ragdoll weighs 6 kg."},+]+++def test_a_turn_that_returned_pictures_still_finds_its_transcript_seat(conn):+ """The seat is matched against the stored rows, which is where the envelope lived.++ `_transcript_positions` reads them through `_as_wire`, so a turn serialised whole+ described a message the request never sent and matched no position at all.+ """+ _persist_image_search_turn()++ positions = conversation_archive._transcript_positions(THREAD)++ assert len(positions) == 2, positions+ assert conversation_archive._occurrences(positions, _IMAGE_SEARCH_WIRE[1:]) == [1]+++def test_a_recalled_turn_that_returned_pictures_survives_the_branch_filter(conn):+ """The other half: recall falls back to the stored rows when it has no branch.++ `_live_transcript` rebuilds them the same way, so the archived turn matched nothing.+ """+ _persist_image_search_turn()+ conversation_archive.archive_turns(THREAD, _IMAGE_SEARCH_WIRE)++ with_branch = conversation_archive.recall(+ THREAD, "ZQXVARA123 ragdoll", branch_messages = _IMAGE_SEARCH_WIRE+ )+ without_branch = conversation_archive.recall(THREAD, "ZQXVARA123 ragdoll")++ assert with_branch is not None and "6 kg" in with_branch[0]+ assert without_branch is not None, "the stored rows rejected a turn that is on branch"+ assert "6 kg" in without_branch[0]+++def test_a_reply_that_shows_the_picture_still_finds_its_transcript_seat(conn):+ """The other half of the same turn: the reply carries the token that placed the image.++ Showing a picture IS writing the token -- the tool result says to -- and the serializer+ strips it from the replayed reply, so a turn whose picture actually rendered still+ matched no seat while only the `tool` message was mirrored.+ """+ answer = "%s\n\n[[img:%s]]" % (_ANSWER, _IMAGE_ID)+ _persist_image_search_turn(answer = answer)++ positions = conversation_archive._transcript_positions(THREAD)++ assert conversation_archive._occurrences(positions, _IMAGE_SEARCH_WIRE[1:]) == [1], positions+++def test_an_audio_reply_is_replayed_as_the_sentinel_the_request_carried():+ """`sanitizeAssistantReplayText` also substitutes inline audio, and for the same reason.++ An audio model answers with the whole wav in an `<audio-player>` tag, so every such+ turn reconstructed as a message megabytes longer than the one that was sent.+ """+ row = [+ {+ "role": "assistant",+ "content": [+ {"type": "text", "text": '<audio-player src="data:audio/wav;base64,QUJD" />'}+ ],+ }+ ]++ assert conversation_archive._probe_text(conversation_archive._as_wire(row)[0]) == (+ '<audio-player src="[audio]" />'+ )+ # Off for a caller comparing the request against something written without it.+ assert (+ conversation_archive._probe_text(+ conversation_archive._as_wire(row, sanitise_assistant = False)[0]+ )+ == '<audio-player src="data:audio/wav;base64,QUJD" />'+ )+ # The assistant side only: the serializer sanitises replies, not what the user typed.+ user = [{"role": "user", "content": [{"type": "text", "text": "[[img:%s]]" % _IMAGE_ID}]}]+ assert conversation_archive._probe_text(conversation_archive._as_wire(user)[0]) == (+ "[[img:%s]]" % _IMAGE_ID+ )++def test_the_branch_seed_scores_an_in_order_run_not_a_set(conn):"""Sets lose repetition and ordering, and leaves are tried newest-first.