Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run coding agents on a Connection model (#13481)

Nilayd218abfc24dd
diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py
index 8750e124a53..691228a4e3e 100644
--- a/studio/backend/auth/storage.py
+++ b/studio/backend/auth/storage.py
@@ -1630,6 +1630,9 @@ API_KEY_PREFIX = "sk-unsloth-"
# The name is the only thing distinguishing internal keys by authority: Deep Research keys must
# reach the saved connection, data-recipe keys only local /v1.
DEEP_RESEARCH_WORKFLOW_KEY_NAME = "deep-research workflow"
+# What `unsloth start` mints for its coding agents (unsloth_cli/commands/start.py). Only a desktop
+# session can create a key with this name, and only the owner's may spend saved Connections.
+UNSLOTH_START_API_KEY_NAME = "Coding agents (unsloth start)"
def create_api_key(
@@ -1861,7 +1864,7 @@ def validate_api_key_account(raw_key: str, *, touch: bool = True) -> Optional[Tu
conn.execute("BEGIN IMMEDIATE")
cur = conn.execute(
f"""
- SELECT k.id, k.username, k.is_active, k.expires_at,
+ SELECT k.id, k.username, k.name, k.is_active, k.expires_at,
u.account_id, u.role, u.is_active AS account_active,
COALESCE(u.account_jwt_secret, u.jwt_secret) AS jwt_secret
FROM api_keys k JOIN auth_user u ON u.username = k.username
@@ -1905,6 +1908,7 @@ def validate_api_key_account(raw_key: str, *, touch: bool = True) -> Optional[Tu
"role": row["role"],
"is_active": int(row["account_active"]),
"expires_at": row["expires_at"],
+ "key_name": row["name"],
}
return record, secret
finally:
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py
index 466bc63744e..b70741288d8 100644
--- a/studio/backend/core/inference/external_provider.py
+++ b/studio/backend/core/inference/external_provider.py
@@ -1719,6 +1719,7 @@ class ExternalProviderClient:
tool_choice,
fast_mode = fast_mode,
tools = tools,
+ response_format = response_format,
):
yield line
return
@@ -2562,6 +2563,7 @@ class ExternalProviderClient:
*,
fast_mode: Optional[bool] = None,
tools: Optional[list[dict[str, Any]]] = None,
+ response_format: Optional[dict[str, Any]] = None,
) -> AsyncGenerator[str, None]:
"""call Anthropic Messages and translate its SSE into OpenAI stream events."""
import json as _json
@@ -2985,6 +2987,19 @@ class ExternalProviderClient:
if body.get("max_tokens", 0) <= budget_tokens:
body["max_tokens"] = budget_tokens + 1024
+ # Structured output lives beside effort under output_config (GA, no beta header):
+ # platform.claude.com/docs/en/build-with-claude/structured-outputs.
+ _rf_schema = (
+ (response_format.get("json_schema") or {}).get("schema")
+ if isinstance(response_format, dict) and response_format.get("type") == "json_schema"
+ else None
+ )
+ if isinstance(_rf_schema, dict):
+ body.setdefault("output_config", {})["format"] = {
+ "type": "json_schema",
+ "schema": _rf_schema,
+ }
+
# tool_choice="none" or pinned-function suppresses hosted tools so a stale UI toggle cannot fire server-side
# search/code-exec.
_anthropic_tool_choice_disabled = (
diff --git a/studio/backend/routes/auth.py b/studio/backend/routes/auth.py
index cf38319fea0..5668569b967 100644
--- a/studio/backend/routes/auth.py
+++ b/studio/backend/routes/auth.py
@@ -810,9 +810,15 @@ async def create_api_key(
credential: tuple = Depends(get_current_credential),
credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
_own_credential: None = Depends(_require_a_credential_of_its_own("Managing API keys")),
+ is_desktop: bool = Depends(authenticated_via_desktop_jwt),
) -> CreateApiKeyResponse:
"""Create a new API key. The raw key is returned once and cannot be retrieved later."""
current_subject, generation = credential
+ if payload.name == storage.UNSLOTH_START_API_KEY_NAME and not is_desktop:
+ raise HTTPException(
+ status_code = status.HTTP_403_FORBIDDEN,
+ detail = "This key name is reserved for `unsloth start`.",
+ )
expires_at = None
if payload.expires_in_days is not None:
expires_at = (
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 35fbb0c6f64..ba45e82f849 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -63,6 +63,7 @@ from loggers.media_progress import (
reset_media_load_progress,
)
import asyncio
+import collections
import contextvars
import threading
import weakref
@@ -73,13 +74,18 @@ from dataclasses import dataclass, fields as dataclass_fields, replace
import re as _re
from pydantic import BaseModel
-from urllib.parse import quote as _urlquote
+from urllib.parse import quote as _urlquote, unquote as _urlunquote
# Model size extraction (shared with core/inference/llama_cpp.py)
from utils.models import extract_model_size_b as _extract_model_size_b
from utils.models.checkpoints import is_full_finetune_output
-from utils.api_errors import openai_error_body, anthropic_error_body, error_body_for_path
+from utils.api_errors import (
+ ANTHROPIC_TYPE_BY_STATUS,
+ anthropic_error_body,
+ error_body_for_path,
+ openai_error_body,
+)
from utils.audio_tokens import GGUF_TTS_AUDIO_TYPES as _GGUF_TTS_AUDIO_TYPES
from utils.first_token_timeout import (
finish_first_token_wait,
@@ -3974,6 +3980,26 @@ def _request_is_saved_credential_workflow(request: Any) -> bool:
return name == auth_storage.DEEP_RESEARCH_WORKFLOW_KEY_NAME
+# A UI session, the Deep Research key, or the key `unsloth start` minted for the owner's coding
+# agents. Every other API key, keyless caller and data-recipe key still gets nothing.
+def _request_may_use_saved_provider_key(request: Any) -> bool:
+ if not _request_has_api_key(request) or _request_is_saved_credential_workflow(request):
+ return True
+ token = _request_api_key_token(request)
+ if token is None:
+ return False
+ try:
+ verified = auth_storage.validate_api_key_account(token, touch = False)
+ if verified is None or verified[0].get("role") not in (None, "owner"):
+ return False
+ if verified[0].get("key_name") != auth_storage.UNSLOTH_START_API_KEY_NAME:
+ return False
+ return not auth_storage.is_internal_api_key(token)
+ except Exception:
+ logger.debug("external_provider.start_key_probe_failed", exc_info = True)
+ return False
+
+
def _request_used_api_key(request: Any) -> bool:
"""True when this request authenticated with a third party's sk-unsloth key.
@@ -28141,6 +28167,9 @@ def _build_external_messages(
provider_type == "llama_cpp"
or (provider_type == "vllm" and preserve_thinking is True)
or _is_kimi_k3(provider_type, model)
+ # Thinking mode with tools answers 400 without it, and ignores it elsewhere
+ # (api-docs.deepseek.com/guides/thinking_mode).
+ or provider_type == "deepseek"
)
result = []
for msg in messages:
@@ -28160,6 +28189,17 @@ def _build_external_messages(
and native["content"]
):
replay["extra_content"] = {"anthropic": {"content": native["content"]}}
+ if (
+ provider_type == "openai"
+ and not emit_message_extra_content
+ and msg.role == "assistant"
+ and isinstance(msg.extra_content, dict)
+ and msg.extra_content.get("openai_responses_reasoning")
+ ):
+ # OpenAI wants a tool call's reasoning items back with its outputs; only that key.
+ replay["extra_content"] = {
+ "openai_responses_reasoning": msg.extra_content["openai_responses_reasoning"]
+ }
# Drop role=tool messages whose matching server-builtin tool_call was
# filtered above. An orphan tool_result with no matching tool_call is
# rejected by OpenAI Responses and Anthropic.
@@ -28204,6 +28244,7 @@ def _build_external_messages(
out["name"] = msg.name
if emit_message_extra_content and msg.role == "assistant" and msg.extra_content:
out["extra_content"] = msg.extra_content
+
result.append(out)
continue
# Assistant messages with content=None but populated tool_calls are
@@ -29285,10 +29326,7 @@ async def _proxy_to_external_provider(
resolve_provider_api_key_or_400,
payload.provider_id,
payload.encrypted_api_key,
- allow_saved_key = (
- not _request_has_api_key(request)
- or _request_is_saved_credential_workflow(request)
- ),
+ allow_saved_key = _request_may_use_saved_provider_key(request),
)
else:
api_key = resolve_provider_api_key_or_400(
@@ -30619,6 +30657,253 @@ class _DroppedFrameKeepalive:
return True
+_CONNECTION_MODEL_PREFIX = "external::"
+
+
+# The id the chat picker gives a Connection model: external::<connection id>::<url-encoded model>.
+def _connection_model_ref(model: Any) -> Optional[tuple[str, str]]:
+ if not isinstance(model, str) or not model.startswith(_CONNECTION_MODEL_PREFIX):
+ return None
+ provider_id, _, encoded = model[len(_CONNECTION_MODEL_PREFIX) :].partition("::")
+ if not provider_id or not encoded:
+ return None
+ return provider_id, _urlunquote(encoded)
+
+
+# Provider replay metadata agent clients drop: Gemini 3 answers 400 to a functionCall without its
+# thoughtSignature (tool call extra_content), and Anthropic to a tool continuation without its signed
+# thinking (message extra_content). Keep both by the tool ids the client echoes back.
+_CONNECTION_TOOL_EXTRAS: collections.OrderedDict = collections.OrderedDict()
+_CONNECTION_TOOL_EXTRAS_MAX = 4096
+_CONNECTION_TOOL_EXTRAS_LOCK = threading.Lock()
+
+
+def _remember_connection_extra(subject: Optional[str], kind: str, tool_id: Any, extra: Any) -> None:
+ if not isinstance(tool_id, str) or not tool_id or not isinstance(extra, dict) or not extra:
+ return
+ key = (subject or "", kind, tool_id)
+ with _CONNECTION_TOOL_EXTRAS_LOCK:
+ _CONNECTION_TOOL_EXTRAS[key] = extra
+ _CONNECTION_TOOL_EXTRAS.move_to_end(key)
+ while len(_CONNECTION_TOOL_EXTRAS) > _CONNECTION_TOOL_EXTRAS_MAX:
+ _CONNECTION_TOOL_EXTRAS.popitem(last = False)
+
+
+def _connection_extra(subject: Optional[str], kind: str, tool_id: Any) -> Optional[dict]:
+ with _CONNECTION_TOOL_EXTRAS_LOCK:
+ return _CONNECTION_TOOL_EXTRAS.get((subject or "", kind, tool_id))
+
+
+class _ConnectionExtrasRecorder:
+ """One reply's tool ids (as the client sees them) and its message-level extra_content."""
+
+ def __init__(self, subject: Optional[str]) -> None:
+ self.subject = subject
+ self.tool_ids: list[str] = []
+ self.message_extra: Optional[dict] = None
+
+ def call(self, tool_id: Any, extra: Any) -> None:
+ if isinstance(tool_id, str) and tool_id:
+ self.tool_ids.append(tool_id)
+ _remember_connection_extra(self.subject, "call", tool_id, extra)
+ self._file_message_extra()
+
+ def message(self, extra: Any) -> None:
+ if isinstance(extra, dict) and extra:
+ self.message_extra = extra
+ self._file_message_extra()
+
+ def chunk(
+ self,
+ chunk: Any,
+ *,
+ calls: bool = True,
+ ) -> None:
+ for choice in (chunk.get("choices") if isinstance(chunk, dict) else None) or []:
+ delta = choice.get("delta") if isinstance(choice, dict) else None
+ if not isinstance(delta, dict):
+ continue
+ self.message(delta.get("extra_content"))
+ for call in (delta.get("tool_calls") if calls else None) or []:
+ if isinstance(call, dict):
+ self.call(call.get("id"), call.get("extra_content"))
+
+ def _file_message_extra(self) -> None:
+ if self.message_extra:
+ for tool_id in self.tool_ids:
+ _remember_connection_extra(self.subject, "message", tool_id, self.message_extra)
+
+
+def _restore_connection_tool_extras(subject: Optional[str], messages: Any) -> None:
+ for message in messages or []:
+ is_dict = isinstance(message, dict)
+ tool_calls = message.get("tool_calls") if is_dict else getattr(message, "tool_calls", None)
+ message_extra = None
+ for call in tool_calls or []:
+ if not isinstance(call, dict):
+ continue
+ message_extra = message_extra or _connection_extra(subject, "message", call.get("id"))
+ if not call.get("extra_content"):
+ extra = _connection_extra(subject, "call", call.get("id"))
+ if extra:
+ call["extra_content"] = extra
+ has_extra = (
+ message.get("extra_content") if is_dict else getattr(message, "extra_content", None)
+ )
+ if message_extra and not has_extra:
+ if is_dict:
+ message["extra_content"] = message_extra
+ else:
+ message.extra_content = message_extra
+
+
+async def _relay_recording_tool_extras(chunks, subject: Optional[str]):
+ recorder = _ConnectionExtrasRecorder(subject)
+ pending = ""
+ try:
+ async for chunk in chunks:
+ yield chunk
+ pending += chunk.decode("utf-8", "replace") if isinstance(chunk, bytes) else chunk
+ *complete, pending = pending.split("\n")
+ for line in complete:
+ if line.startswith("data:") and (
+ '"tool_calls"' in line or '"extra_content"' in line
+ ):
+ try:
+ recorder.chunk(json.loads(line[5:]))
+ except ValueError:
+ pass
+ finally:
+ await chunks.aclose()
+
+
+async def _relay_split_thinking(
+ chunks,
+ stops: Any = None,
+ parse_thinking: bool = True,
+):
+ """Native thinking arrives as a leading <think> block in content (the Studio chat's form); OpenAI
+ clients read it from reasoning_content. The proxy forwards no stop sequences, so cut here."""
+ think = _ResponsesReasoningExtractor(parse_think_markers = parse_thinking, single_block = True)
+ stop = _StopSequenceFilter([stops] if isinstance(stops, str) else stops)
+ lines = _sse_data_lines(chunks)
+ try:
+ async for line in lines:
+ data = line[len("data:") :].strip() if line.startswith("data:") else ""
+ if data == "[DONE]":
+ reasoning, text = think.finish()
+ text = stop.feed(text) + stop.flush()
+ if reasoning or text:
+ delta = {
+ k: v for k, v in (("reasoning_content", reasoning), ("content", text)) if v
+ }
+ yield (
+ "data: " + json.dumps({"choices": [{"index": 0, "delta": delta}]}) + "\n\n"
+ )
+ elif data and '"choices"' in data:
+ try:
+ chunk = json.loads(data)
+ except ValueError:
+ chunk = None
+ if isinstance(chunk, dict):
+ for choice in chunk.get("choices") or []:
+ delta = choice.get("delta") if isinstance(choice, dict) else None
+ if not isinstance(delta, dict):
+ continue
+ reasoning, text = think.feed(
+ delta.get("content") if isinstance(delta.get("content"), str) else "",
+ delta.get("reasoning_content"),
+ )
+ if choice.get("finish_reason"):
+ tail_reasoning, tail_text = think.finish()
+ reasoning, text = reasoning + tail_reasoning, text + tail_text
+ text = stop.feed(text)
+ if choice.get("finish_reason") and stop.hit is None:
+ text += stop.flush()
+ if stop.hit is not None:
+ choice["finish_reason"] = "stop"
+ delta.pop("reasoning_content", None)
+ delta.pop("content", None)
+ if text:
+ delta["content"] = text
+ if reasoning:
+ delta["reasoning_content"] = reasoning
+ line = "data: " + json.dumps(chunk)
+ yield line + "\n"
+ if stop.hit is not None:
+ yield "\ndata: [DONE]\n\n"
+ return
+ finally:
+ await lines.aclose()
+
+
+async def _chat_completion_from_connection_stream(response: StreamingResponse, model: str):
+ lines = _sse_data_lines(response.body_iterator)
+ try:
+ return await _collect_chat_completion(lines, model)
+ finally:
+ await lines.aclose()
+
+
+async def _collect_chat_completion(lines, model: str):
+ completion_id, created, finish, usage = None, int(time.time()), None, None
+ content, reasoning = "", ""
+ calls: dict[int, dict] = {}
+ async for line in lines:
+ data = line[len("data:") :].strip() if line.startswith("data:") else ""
+ if data == "[DONE]":
+ break
+ if not data:
+ continue
+ chunk = json.loads(data)
+ error = chunk.get("error")
+ if error:
+ message = error.get("message") if isinstance(error, dict) else str(error)
+ try:
+ status_code = int(error.get("code"))
+ except (AttributeError, TypeError, ValueError):
+ status_code = 502
+ if not 400 <= status_code <= 599:
+ status_code = 502
+ return JSONResponse(
+ status_code = status_code,
+ content = openai_error_body(message or "The connection failed.", status = status_code),
+ )
+ completion_id = completion_id or chunk.get("id")
+ usage = chunk.get("usage") or usage
+ for choice in chunk.get("choices") or []:
+ delta = choice.get("delta") or {}
+ content += delta.get("content") or ""
+ reasoning += delta.get("reasoning_content") or ""
+ for part in delta.get("tool_calls") or []:
+ call = calls.setdefault(
+ part.get("index", len(calls)),
+ {"id": None, "type": "function", "function": {"name": "", "arguments": ""}},
+ )
+ call["id"] = call["id"] or part.get("id")
+ function = part.get("function") or {}
+ call["function"]["name"] += function.get("name") or ""
+ call["function"]["arguments"] += function.get("arguments") or ""
+ if part.get("extra_content"):
+ call["extra_content"] = part["extra_content"]
+ finish = choice.get("finish_reason") or finish
+ message: dict = {"role": "assistant", "content": content or None}
+ if reasoning:
+ message["reasoning_content"] = reasoning
+ if calls:
+ message["tool_calls"] = [calls[index] for index in sorted(calls)]
+ body = {
+ "id": completion_id or f"chatcmpl-{uuid.uuid4().hex}",
+ "object": "chat.completion",
+ "created": created,
+ "model": model,
+ "choices": [{"index": 0, "message": message, "finish_reason": finish or "stop"}],
+ }
+ if usage:
+ body["usage"] = usage
+ return JSONResponse(body)
+
+
@router.post("/chat/completions")
@account_access.gpu_busy_route
async def openai_chat_completions(
@@ -30627,6 +30912,18 @@ async def openai_chat_completions(
current_subject: str = Depends(get_current_subject),
):
_admit_tool_access(payload)
+ connection = _connection_model_ref(payload.model)
+ if connection is not None and (payload.provider_id or payload.provider_type):
+ connection = None
+ collect = False
+ if connection is not None:
+ payload.provider_id, payload.external_model = connection
+ _restore_connection_tool_extras(current_subject, payload.messages)
+ # The proxy relays a stream:false provider body as is, so stream and answer from the chunks.
+ collect = payload.stream is not True
+ if collect:
+ payload.stream = True
+ payload.stream_options = {"include_usage": True}
from auth.authentication import request_admitted_without_credential
if (payload.provider_id or payload.provider_type) and request_admitted_without_credential(
@@ -30636,12 +30933,23 @@ async def openai_chat_completions(
status_code = 403,
detail = "External providers can only be used from the Unsloth UI or with an API key.",
)
- return await produce_openai_chat_completions(
+ response = await produce_openai_chat_completions(
payload,
request,
current_subject,
cancel_on_disconnect = True,
)
+ if connection is None or not isinstance(response, StreamingResponse):
+ return response
+ response.body_iterator = _relay_split_thinking(
+ _relay_recording_tool_extras(response.body_iterator, current_subject),
+ payload.stop,
+ # Thinking switched off: a leading tag is the answer's own text.
+ parse_thinking = payload.enable_thinking is not False,
+ )
+ if collect:
+ return await _chat_completion_from_connection_stream(response, payload.model)
+ return response
async def produce_openai_chat_completions(
@@ -39290,11 +39598,27 @@ def _responses_error_code(exc: Exception) -> str:
return "server_error"
+async def _sse_data_lines(chunks):
+ pending = ""
+ try:
+ async for chunk in chunks:
+ pending += chunk.decode("utf-8") if isinstance(chunk, bytes) else chunk
+ *complete, pending = pending.split("\n")
+ for line in complete:
+ yield line
+ if pending:
+ yield pending
+ finally:
+ await chunks.aclose()
+
+
async def _responses_stream(
payload: ResponsesRequest,
messages: list[ChatMessage],
request: Request,
monitor_id: Optional[str] = None,
+ current_subject: Optional[str] = None,
+ connection: Optional[tuple[str, str]] = None,
):
"""Handle a streaming Responses API call, emitting named SSE events.
@@ -39326,7 +39650,25 @@ async def _responses_stream(
namespaced_tool_names = _responses_namespaced_tool_names(payload.tools)
llama_backend = get_llama_cpp_backend()
- if not llama_backend.is_loaded:
+ upstream = None
+ if connection is not None:
+ # A Connection model: the chat proxy serves it and the converter below relays its stream.
+ chat_req.provider_id, chat_req.external_model = connection
+ _restore_connection_tool_extras(current_subject, chat_req.messages)
+ chat_req.stream_options = {"include_usage": True}
+ request.state.skip_api_monitor = True
+ upstream = await produce_openai_chat_completions(
+ chat_req, request, current_subject, cancel_on_disconnect = True
+ )
+ if not isinstance(upstream, StreamingResponse):
+ return upstream
+ body = {
+ "tools": chat_req.tools,
+ "tool_choice": chat_req.tool_choice,
+ "response_format": chat_req.response_format,
+ }
+ cancel_event = threading.Event()
+ elif not llama_backend.is_loaded:
# The direct pass-through is GGUF-only. Non-GGUF /v1/responses streaming
# isn't a Codex-compatible path today, and wrapping the transformers
# backend's streaming generator here would re-introduce the
@@ -39343,60 +39685,61 @@ async def _responses_stream(
)
raise HTTPException(status_code = _status, detail = _detail)
- # Direct pass-through bypasses the openai_chat_completions tool-image note and image gate.
- if not llama_backend.is_vision:
- _omit_tool_images(chat_req.messages)
- if _messages_have_image(chat_req.messages):
- raise HTTPException(
- status_code = 400,
- detail = "Image provided but current GGUF model does not support vision.",
- )
+ if upstream is None:
+ # Direct pass-through bypasses the openai_chat_completions tool-image note and image gate.
+ if not llama_backend.is_vision:
+ _omit_tool_images(chat_req.messages)
+ if _messages_have_image(chat_req.messages):
+ raise HTTPException(
+ status_code = 400,
+ detail = "Image provided but current GGUF model does not support vision.",
+ )
- # Same bypass, same reason as the image gate: without this the non-streaming half of this
- # very route folds a Studio tool thread and answers while the streaming half still ships
- # role="tool" to a template that cannot render it, so one thread's replies depend on which
- # half the client called.
- if _folds_studio_tool_history(chat_req, llama_backend):
- chat_req.messages = _folded_studio_tool_messages(chat_req.messages)
+ # Same bypass, same reason as the image gate: without this the non-streaming half of this
+ # very route folds a Studio tool thread and answers while the streaming half still ships
+ # role="tool" to a template that cannot render it, so one thread's replies depend on which
+ # half the client called.
+ if _folds_studio_tool_history(chat_req, llama_backend):
+ chat_req.messages = _folded_studio_tool_messages(chat_req.messages)
- # Streaming /v1/responses builds the passthrough body directly (bypassing
- # openai_chat_completions), so apply recommended sampling here too.
- _fill_recommended_sampling_openai(
- chat_req,
- getattr(llama_backend, "model_identifier", None),
- thinking = _sampling_thinking_mode(llama_backend, chat_req),
- )
- body = await _build_openai_passthrough_body_async(
- chat_req, backend_ctx = llama_backend.context_length, llama_backend = llama_backend
- )
- body["stream_options"] = {"include_usage": True}
- target_url = f"{llama_backend.base_url}/v1/chat/completions"
- # The stream's own disconnect event, shared with the cancel/active-generation registries:
- # this path decodes on llama-server, so a non-forced /unload must see it and refuse instead
- # of tearing the server down mid-response. Entered inside the body generator below, so a
- # response whose body never starts leaves nothing behind.
- cancel_event = threading.Event()
- _tracker = _TrackedCancel.for_payload(cancel_event, payload, resp_id)
- try:
- reservation, admission_config = await _openai_llama_admission_reserve_async(
- request = request,
- llama_backend = llama_backend,
- # chat_req, not payload: a ResponsesRequest carries `input` and
- # `max_output_tokens`, so the estimator found no `messages` and fell back to
- # one equal cache share no matter how large the request really was.
- payload = chat_req,
- )
- except LlamaAdmissionQueueFull as exc:
- _llama_admission_log(
- "queue-full",
- snapshot = exc.snapshot,
- request = request,
- mode = "responses_stream",
- completion_id = resp_id,
- level = "warning",
+ # Streaming /v1/responses builds the passthrough body directly (bypassing
+ # openai_chat_completions), so apply recommended sampling here too.
+ _fill_recommended_sampling_openai(
+ chat_req,
+ getattr(llama_backend, "model_identifier", None),
+ thinking = _sampling_thinking_mode(llama_backend, chat_req),
)
- api_monitor.fail(monitor_id, str(exc))
- raise _openai_admission_http_exception(exc, status_code = 429)
+ body = await _build_openai_passthrough_body_async(
+ chat_req, backend_ctx = llama_backend.context_length, llama_backend = llama_backend
+ )
+ body["stream_options"] = {"include_usage": True}
+ target_url = f"{llama_backend.base_url}/v1/chat/completions"
+ # The stream's own disconnect event, shared with the cancel/active-generation registries:
+ # this path decodes on llama-server, so a non-forced /unload must see it and refuse instead
+ # of tearing the server down mid-response. Entered inside the body generator below, so a
+ # response whose body never starts leaves nothing behind.
+ cancel_event = threading.Event()
+ _tracker = _TrackedCancel.for_payload(cancel_event, payload, resp_id)
+ try:
+ reservation, admission_config = await _openai_llama_admission_reserve_async(
+ request = request,
+ llama_backend = llama_backend,
+ # chat_req, not payload: a ResponsesRequest carries `input` and
+ # `max_output_tokens`, so the estimator found no `messages` and fell back to
+ # one equal cache share no matter how large the request really was.
+ payload = chat_req,
+ )
+ except LlamaAdmissionQueueFull as exc:
+ _llama_admission_log(
+ "queue-full",
+ snapshot = exc.snapshot,
+ request = request,
+ mode = "responses_stream",
+ completion_id = resp_id,
+ level = "warning",
+ )
+ api_monitor.fail(monitor_id, str(exc))
+ raise _openai_admission_http_exception(exc, status_code = 429)
def _responses_admission_failed_sse(exc: Exception, *, code: str) -> str:
# Emitted in-band after the 200 headers, so flag failed: the middleware must not
@@ -39439,7 +39782,9 @@ async def _responses_stream(
# helper, not the raw identifier: after an auto-switch to a cached HF GGUF
# the identifier is the snapshot path while the repo id lives in
# _openai_advertised_id, so the raw form would stream a snapshot basename.
- _clean_model = _llama_public_model_id(llama_backend, payload.model) or payload.model
+ _clean_model = payload.model
+ if upstream is None:
+ _clean_model = _llama_public_model_id(llama_backend, payload.model) or payload.model
full_text = ""
full_reasoning = ""
input_tokens = 0
@@ -39448,8 +39793,13 @@ async def _responses_stream(
# From the chat chunks; applied once before finish, so chunk order does not matter.
stream_finish_reason: Optional[str] = None
extractor = _ResponsesReasoningExtractor(
- parse_think_markers = _responses_should_parse_think_markers(chat_req, llama_backend),
- single_block = _response_format_constrains_decoding(chat_req),
+ # A Connection's native thinking arrives as a leading <think> block even when thinking is on
+ # by default; only that block is read, so a literal tag later in the answer stays text.
+ parse_think_markers = (upstream is not None and chat_req.enable_thinking is not False)
+ or (
+ upstream is None and _responses_should_parse_think_markers(chat_req, llama_backend)
+ ),
+ single_block = upstream is not None or _response_format_constrains_decoding(chat_req),
)
reasoning_state: dict[str, Any] = {"output_index": None, "item_id": None, "opened": False}
message_state: dict[str, Any] = {
@@ -39570,7 +39920,12 @@ async def _responses_stream(
output_tokens = u.get("completion_tokens", output_tokens)
cached = cached_prompt_tokens(u)
cached_input = cached_input if cached is None else cached
- _monitor_usage(monitor_id, u, llama_backend.context_length, timings = timings)
+ _monitor_usage(
+ monitor_id,
+ u,
+ None if upstream is not None else llama_backend.context_length,
+ timings = timings,
+ )
def _ensure_reasoning_open() -> list[str]:
if reasoning_state["opened"]:
@@ -39849,11 +40204,7 @@ async def _responses_stream(
# `async with`, explicit aclose of lines_iter BEFORE resp / client so
# the innermost httpcore byte stream is finalised in this task (not via
# the asyncgen GC in a sibling task).
- client = httpx.AsyncClient(
- timeout = _llama_streaming_generation_timeout(),
- trust_env = False,
- verify = _local_ssl_context(),
- )
+ client = None
resp = None
lines_iter = None
items_iter = None
@@ -39861,98 +40212,108 @@ async def _responses_stream(
# Tracked per-run event: a client disconnect and a forced reload both land here.
disconnect_event = cancel_event
try:
- req = client.build_request(
- "POST",
- target_url,
- json = body,
- headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),
- )
- first_token_deadline = start_first_token_wait(_first_token_timeout_s())
- try:
- # Same event the loop below polls: prefill can run for the whole first-token window,
- # and only the send watcher can end it early.
- resp = await _send_stream_with_preheader_cancel(
- client, req, disconnect_event, request = request
+ if upstream is not None:
+ items_iter = _sse_data_lines(
+ _relay_recording_tool_extras(upstream.body_iterator, current_subject)
)
- if resp is None:
- api_monitor.finish(monitor_id, "cancelled")
- return
- except httpx.RequestError as e:
- logger.error("responses stream: upstream unreachable: %s", e)
- api_monitor.fail(monitor_id, _friendly_error(e))
- # 200 headers already flushed; this is an in-band failure, so the
- # middleware must not claim a preview-owned model for it.
- mark_response_failed(getattr(request, "scope", None))
- yield _sse(
- "response.failed",
- {
- "type": "response.failed",
- "response": {
- "id": resp_id,
- "object": "response",
- "created_at": created_at,
- "status": "failed",
- "model": _clean_model,
- "output": [],
- "error": {"code": "server_error", "message": _friendly_error(e)},
- },
- },
+ else:
+ client = httpx.AsyncClient(
+ timeout = _llama_streaming_generation_timeout(),
+ trust_env = False,
+ verify = _local_ssl_context(),
)
- return
-
- if resp.status_code != 200:
- err_bytes = await resp.aread()
- err_text = err_bytes.decode("utf-8", errors = "replace")
- logger.error(
- "responses stream upstream error: status=%s body=%s",
- resp.status_code,
- err_text[:500],
+ req = client.build_request(
+ "POST",
+ target_url,
+ json = body,
+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),
)
- api_monitor.fail(monitor_id, err_text[:500])
- # 200 headers already flushed; an upstream non-200 is an in-band
- # failure, so the middleware must not claim a preview-owned model.
- mark_response_failed(getattr(request, "scope", None))
- yield _sse(
- "response.failed",
- {
- "type": "response.failed",
- "response": {
- "id": resp_id,
- "object": "response",
- "created_at": created_at,
- "status": "failed",
- "model": _clean_model,
- "output": [],
- "error": {
- "code": _responses_error_code(RuntimeError(err_text)),
- "message": _friendly_upstream_error(err_text[:500]),
+ first_token_deadline = start_first_token_wait(_first_token_timeout_s())
+ try:
+ # Same event the loop below polls: prefill can run for the whole first-token window,
+ # and only the send watcher can end it early.
+ resp = await _send_stream_with_preheader_cancel(
+ client, req, disconnect_event, request = request
+ )
+ if resp is None:
+ api_monitor.finish(monitor_id, "cancelled")
+ return
+ except httpx.RequestError as e:
+ logger.error("responses stream: upstream unreachable: %s", e)
+ api_monitor.fail(monitor_id, _friendly_error(e))
+ # 200 headers already flushed; this is an in-band failure, so the
+ # middleware must not claim a preview-owned model for it.
+ mark_response_failed(getattr(request, "scope", None))
+ yield _sse(
+ "response.failed",
+ {
+ "type": "response.failed",
+ "response": {
+ "id": resp_id,
+ "object": "response",
+ "created_at": created_at,
+ "status": "failed",
+ "model": _clean_model,
+ "output": [],
+ "error": {"code": "server_error", "message": _friendly_error(e)},
},
},
- },
- )
- return
+ )
+ return
- lines_iter = resp.aiter_lines()
- disconnect_watcher = asyncio.create_task(
- _await_disconnect_then_close(request, resp, disconnect_event)
- )
- items_iter = _aiter_llama_stream_items(
- lines_iter,
- cancel_event = disconnect_event,
- request = request,
- first_token_deadline = first_token_deadline,
- response = resp,
- keepalive_interval_s = _openai_passthrough_stream_keepalive_interval(),
- )
+ if resp.status_code != 200:
+ err_bytes = await resp.aread()
+ err_text = err_bytes.decode("utf-8", errors = "replace")
+ logger.error(
+ "responses stream upstream error: status=%s body=%s",
+ resp.status_code,
+ err_text[:500],
+ )
+ api_monitor.fail(monitor_id, err_text[:500])
+ # 200 headers already flushed; an upstream non-200 is an in-band
+ # failure, so the middleware must not claim a preview-owned model.
+ mark_response_failed(getattr(request, "scope", None))
+ yield _sse(
+ "response.failed",
+ {
+ "type": "response.failed",
+ "response": {
+ "id": resp_id,
+ "object": "response",
+ "created_at": created_at,
+ "status": "failed",
+ "model": _clean_model,
+ "output": [],
+ "error": {
+ "code": _responses_error_code(RuntimeError(err_text)),
+ "message": _friendly_upstream_error(err_text[:500]),
+ },
+ },
+ },
+ )
+ return
+
+ lines_iter = resp.aiter_lines()
+ disconnect_watcher = asyncio.create_task(
+ _await_disconnect_then_close(request, resp, disconnect_event)
+ )
+ items_iter = _aiter_llama_stream_items(
+ lines_iter,
+ cancel_event = disconnect_event,
+ request = request,
+ first_token_deadline = first_token_deadline,
+ response = resp,
+ keepalive_interval_s = _openai_passthrough_stream_keepalive_interval(),
+ )
async for raw_line in items_iter:
if raw_line is _LLAMA_STREAM_KEEPALIVE:
yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
continue
if not raw_line:
continue
- if not raw_line.startswith("data: "):
+ if not raw_line.startswith("data:"):
continue
- data_str = raw_line[6:]
+ data_str = raw_line[5:]
if data_str.strip() == "[DONE]":
break
try:
@@ -39963,7 +40324,9 @@ async def _responses_stream(
_drop_parallel_tool_call_deltas(chunk_data)
choices = chunk_data.get("choices", [])
- if not choices:
+ # A Connection can report a mid-stream error beside a non-empty choices array
+ # (OpenRouter does), so it is checked before choices there.
+ if not choices or (upstream is not None and chunk_data.get("error")):
error_message = _monitor_openai_error_message(chunk_data)
if error_message:
# Upstream HTTP-200 SSE error payload (data: {"error": ...}) is a failed
@@ -39971,11 +40334,15 @@ async def _responses_stream(
# response.failed (which flags the scope) so a later response.completed
# can't fire and the middleware can't claim a preview-owned slot.
api_monitor.fail(monitor_id, error_message)
- yield _sse(
- "response.failed",
- _failed_response_payload(RuntimeError(error_message)),
- )
+ failed = _failed_response_payload(RuntimeError(error_message))
+ if upstream is not None:
+ failed["response"]["error"]["message"] = error_message
+ error = chunk_data.get("error")
+ if isinstance(error, dict) and str(error.get("code")) == "429":
+ failed["response"]["error"]["code"] = "rate_limit_exceeded"
+ yield _sse("response.failed", failed)
return
+ if not choices:
_apply_usage(chunk_data.get("usage"), chunk_data.get("timings"))
continue
if choices[0].get("finish_reason"):
@@ -40348,6 +40715,9 @@ async def _responses_stream(
api_monitor.finish(monitor_id)
yield _sse(terminal_event, terminal_response)
+ if upstream is not None:
+ return _sse_streaming_response(event_generator(), monitor_id = monitor_id)
+
async def admitted_event_generator():
# Register for the body's whole lifetime, admission wait included: the run holds a decode
# slot from here on, so /load and /unload must count it. __exit__ runs from the finally below.
@@ -40473,6 +40843,7 @@ async def openai_responses(
(output array, input_tokens/output_tokens, named SSE events for streaming).
"""
_admit_tool_access(payload)
+ connection = _connection_model_ref(payload.model)
for history_param in ("previous_response_id", "conversation"):
if getattr(payload, history_param, None) is not None:
raise HTTPException(
@@ -40526,27 +40897,28 @@ async def openai_responses(
param = "tool_choice",
),
)
- if payload.stream:
+ if payload.stream or connection is not None:
# streaming preflights here; non-streaming delegates its complete preflight to chat.
_responses_has_image = _messages_have_image(m for m in messages if m.role != "tool")
_responses_image_b64s = _local_image_payloads_from_messages(messages)
- await _maybe_auto_switch_model(
- _switch_model_for_payload(payload),
- request,
- current_subject,
- require_vision = _responses_has_image,
- claim_resident = False,
- gguf_only = True,
- image_preflight = (
- {
- "b64": _responses_image_b64s[0],
- "b64s": _responses_image_b64s,
- "multiple": False,
- }
- if _responses_image_b64s
- else None
- ),
- )
+ if connection is None:
+ await _maybe_auto_switch_model(
+ _switch_model_for_payload(payload),
+ request,
+ current_subject,
+ require_vision = _responses_has_image,
+ claim_resident = False,
+ gguf_only = True,
+ image_preflight = (
+ {
+ "b64": _responses_image_b64s[0],
+ "b64s": _responses_image_b64s,
+ "multiple": False,
+ }
+ if _responses_image_b64s
+ else None
+ ),
+ )
monitor_id = None
if not getattr(request.state, "skip_api_monitor", False):
monitor_id = api_monitor.start(
@@ -40555,11 +40927,13 @@ async def openai_responses(
method = request.method,
model = payload.model,
prompt = _monitor_prompt_from_messages(messages),
- context_length = _monitor_context_length(),
+ context_length = None if connection else _monitor_context_length(),
subject = current_subject,
)
try:
- return await _responses_stream(payload, messages, request, monitor_id)
+ response = await _responses_stream(
+ payload, messages, request, monitor_id, current_subject, connection
+ )
except HTTPException as exc:
detail = exc.detail
if not isinstance(detail, str):
@@ -40569,9 +40943,39 @@ async def openai_responses(
except Exception as exc:
api_monitor.fail(monitor_id, _friendly_error(exc))
raise
+ if payload.stream or not isinstance(response, StreamingResponse):
+ return response
+ # A Connection streams whatever the client asked for; answer from the terminal event.
+ return await _responses_from_stream(response, payload)
return await _responses_non_streaming(payload, messages, request, current_subject)
+async def _responses_from_stream(response: StreamingResponse, payload: ResponsesRequest):
+ terminal = None
+ async for line in _sse_data_lines(response.body_iterator):
+ if line.startswith("data:"):
+ event = json.loads(line[5:])
+ if event.get("type") in (
+ "response.completed",
+ "response.incomplete",
+ "response.failed",
+ ):
+ terminal = event["response"]
+ if terminal is None or terminal["status"] == "failed":
+ error = (terminal or {}).get("error") or {}
+ raise HTTPException(
+ status_code = 429 if error.get("code") == "rate_limit_exceeded" else 502,
+ detail = error.get("message") or "The connection failed.",
+ )
+ terminal.update(
+ temperature = payload.temperature,
+ top_p = payload.top_p,
+ max_output_tokens = payload.max_output_tokens,
+ instructions = payload.instructions,
+ )
+ return JSONResponse(terminal)
+
+
# =====================================================================
# Anthropic-Compatible Messages API (/messages → /v1/messages)
@@ -41894,6 +42298,367 @@ async def chat_count_tokens(
)
+def _anthropic_connection_error_event(error: Any) -> str:
+ message = error.get("message") if isinstance(error, dict) else None
+ try:
+ code = int(error.get("code"))
+ except (AttributeError, TypeError, ValueError):
+ code = 500
+ # Keep the provider's class (an expired saved key is a 401, not a bad request).
+ status = code if code in ANTHROPIC_TYPE_BY_STATUS or 500 <= code < 600 else 400
+ return build_anthropic_sse_event(
+ "error", anthropic_error_body(message or "The connection failed.", status = status)
+ )
+
+
+class _StopSequenceFilter:
+ """Cut visible text at the first stop sequence, holding back a tail that could start one: the
+ provider proxy forwards no stop sequences, so the relay enforces them like llama-server does."""
+
+ def __init__(self, stops: Any) -> None:
+ self.stops = [stop for stop in stops or [] if isinstance(stop, str) and stop]
+ self.held = ""
+ self.hit: Optional[str] = None
+
+ def feed(self, text: str) -> str:
+ if not self.stops or self.hit is not None:
+ return "" if self.hit is not None else text
+ self.held += text
+ found = [(self.held.find(stop), stop) for stop in self.stops if stop in self.held]
+ if found:
+ index, self.hit = min(found)
+ out, self.held = self.held[:index], ""
+ return out
+ keep = max(len(stop) for stop in self.stops) - 1
+ cut = max(len(self.held) - keep, 0)
+ out, self.held = self.held[:cut], self.held[cut:]
+ return out
+
+ def flush(self) -> str:
+ out, self.held = self.held, ""
+ return out
+
+
+def _split_connection_text(chunk: dict, think: Any, stop: _StopSequenceFilter) -> dict:
+ """Native thinking arrives as <think> markup in content (the Studio chat's form): hand it to the
+ emitter as reasoning_content, and apply the stop sequences to what remains visible."""
+ choices = chunk.get("choices") or []
+ if not any(
+ isinstance(choice.get("delta"), dict)
+ and (choice["delta"].get("content") or choice["delta"].get("reasoning_content"))
+ for choice in choices
+ ):
+ return chunk
+ chunk = {**chunk, "choices": []}
+ for choice in choices:
+ delta = dict(choice.get("delta") or {})
+ text = delta.get("content") if isinstance(delta.get("content"), str) else ""
+ if think is not None:
+ reasoning, text = think.feed(text, delta.get("reasoning_content"))
+ delta["reasoning_content"] = reasoning or None
+ delta["content"] = stop.feed(text) or None
+ chunk["choices"].append(
+ {**choice, "delta": {k: v for k, v in delta.items() if v is not None}}
+ )
+ return chunk
+
+
+# The emitter closes a tool_use block as soon as another call's fragment arrives, so interleaved
+# parallel calls would close with partial arguments: hold calls whole until the reply finishes.
+def _hold_tool_calls(chunk: dict, held: dict) -> dict:
+ choices = chunk.get("choices") or []
+ if not any((choice.get("delta") or {}).get("tool_calls") for choice in choices):
+ return chunk
+ chunk = {**chunk, "choices": []}
+ for choice in choices:
+ delta = dict(choice.get("delta") or {})
+ for part in delta.pop("tool_calls", None) or []:
+ if not isinstance(part, dict):
+ continue
+ call = held.setdefault(
+ part.get("index", len(held)),
+ {"id": None, "name": "", "arguments": "", "extra_content": None},
+ )
+ call["id"] = call["id"] or part.get("id")
+ function = part.get("function") or {}
+ call["name"] += function.get("name") or ""
+ call["arguments"] += function.get("arguments") or ""
+ call["extra_content"] = part.get("extra_content") or call["extra_content"]
+ chunk["choices"].append({**choice, "delta": delta})
+ return chunk
+
+
+def _released_tool_calls(held: dict):
+ for index in sorted(held):
+ call = held[index]
+ part = {
+ "index": index,
+ "id": call["id"],
+ "type": "function",
+ "function": {"name": call["name"], "arguments": call["arguments"]},
+ }
+ if call["extra_content"]:
+ part["extra_content"] = call["extra_content"]
+ yield {"choices": [{"index": 0, "delta": {"tool_calls": [part]}}]}
+ held.clear()
+
+
+def _pending_connection_text(think: Optional[Any], stop: _StopSequenceFilter, held: dict):
+ """What is still held back once the reply ends: unterminated markup, a possible stop prefix,
+ then the buffered tool calls."""
+ reasoning, text = think.finish() if think is not None else ("", "")
+ text = stop.feed(text) + stop.flush() if stop.hit is None else ""
+ delta = {
+ key: value for key, value in (("reasoning_content", reasoning), ("content", text)) if value
+ }
+ if delta:
+ yield {"choices": [{"index": 0, "delta": delta}]}
+ yield from _released_tool_calls(held)
+
+
+async def _anthropic_events_from_openai_sse(
+ lines,
+ message_id: str,
+ model: str,
+ subject: Optional[str] = None,
+ *,
+ single_tool_use: bool = False,
+ stop_sequences: Any = None,
+ parse_thinking: bool = True,
+):
+ emitter = AnthropicPassthroughEmitter()
+ # Always, like the Studio chat: a provider can think by default with no thinking controls sent.
+ # Native thinking leads the reply, so only a leading block is read: a literal tag later in the
+ # answer (or inside JSON) stays text.
+ think = _ResponsesReasoningExtractor(parse_think_markers = parse_thinking, single_block = True)
+ stop = _StopSequenceFilter(stop_sequences)
+ recorder = _ConnectionExtrasRecorder(subject)
+ dropped: set[int] = set()
+ tool_blocks = 0
+
+ def relay(event: str, extras: list) -> Optional[str]:
+ nonlocal tool_blocks
+ if stop.hit is not None and event.startswith("event: message_delta"):
+ data = json.loads(event.partition("\ndata: ")[2])
+ data["delta"].update(stop_reason = "stop_sequence", stop_sequence = stop.hit)
+ return build_anthropic_sse_event(data["type"], data)
+ if not (extras or single_tool_use) or not event.startswith("event: content_block"):
+ return event
+ data = json.loads(event.partition("\ndata: ")[2])
+ index = data.get("index")
+ if (
+ data["type"] == "content_block_start"
+ and data["content_block"].get("type") == "tool_use"
+ ):
+ extra = extras.pop(0) if extras else None
+ # disable_parallel_tool_use: at most one tool_use, the way the local route caps it.
+ if single_tool_use and tool_blocks:
+ dropped.add(index)
+ return None
+ tool_blocks += 1
+ # The emitter renames tool ids (toolu_*): file the call's extras under the id it publishes.
+ recorder.call(data["content_block"].get("id"), extra)
+ if index in dropped:
+ return None
+ shift = sum(1 for gone in dropped if gone < index)
+ if not shift:
+ return event
+ data["index"] = index - shift
+ return build_anthropic_sse_event(data["type"], data)
+
+ def feed(chunk: dict):
+ extras = [
+ call.get("extra_content")
+ for choice in chunk.get("choices") or []
+ for call in (choice.get("delta") or {}).get("tool_calls") or []
+ if isinstance(call, dict) and call.get("id")
+ ]
+ for event in emitter.feed_chunk(chunk):
+ event = relay(event, extras)
+ if event is not None:
+ yield event
+
+ held: dict[int, dict] = {}
+ for event in emitter.start(message_id, model):
+ yield event
+ items = _sse_data_lines(lines)
+ try:
+ async for line in items:
+ data = line[len("data:") :].strip() if line.startswith("data:") else ""
+ if data == "[DONE]":
+ break
+ if not data:
+ continue
+ parsed = json.loads(data)
+ if parsed.get("error"):
+ yield _anthropic_connection_error_event(parsed["error"])
+ return
+ recorder.chunk(parsed, calls = False)
+ parsed = _split_connection_text(_hold_tool_calls(parsed, held), think, stop)
+ if stop.hit is not None:
+ for event in feed(parsed):
+ yield event
+ break
+ finishes = [c for c in parsed.get("choices") or [] if c.get("finish_reason")]
+ if finishes:
+ # This chunk's own text goes before the held-back tail, then the finish.
+ text_part = {
+ **{k: v for k, v in parsed.items() if k != "usage"},
+ "choices": [{**c, "finish_reason": None} for c in parsed["choices"]],
+ }
+ for event in feed(text_part):
+ yield event
+ for chunk in _pending_connection_text(think, stop, held):
+ for event in feed(chunk):
+ yield event
+ if stop.hit is not None:
+ break
+ parsed = {
+ **parsed,
+ "choices": [
+ {
+ "index": c.get("index", 0),
+ "delta": {},
+ "finish_reason": c["finish_reason"],
+ }
+ for c in finishes
+ ],
+ }
+ for event in feed(parsed):
+ yield event
+ finally:
+ await items.aclose()
+ for chunk in _pending_connection_text(think, stop, held):
+ for event in feed(chunk):
+ yield event
+ if stop.hit is not None:
+ for event in feed({"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}):
+ yield event
+ for event in emitter.finish():
+ event = relay(event, [])
+ if event is not None:
+ yield event
+
+
+def _anthropic_message_from_events(events: list[str]) -> dict:
+ message: dict = {}
+ blocks: list[dict] = []
+ partial_json: dict[int, str] = {}
+ for event in events:
+ data = json.loads(event.partition("\ndata: ")[2])
+ if data["type"] == "error":
+ status_code = next(
+ (
+ code
+ for code, err_type in ANTHROPIC_TYPE_BY_STATUS.items()
+ if err_type == data["error"]["type"] and err_type != "api_error"
+ ),
+ 502,
+ )
+ raise HTTPException(status_code = status_code, detail = data["error"]["message"])
+ if data["type"] == "message_start":
+ message = data["message"]
+ elif data["type"] == "content_block_start":
+ blocks.append(dict(data["content_block"]))
+ elif data["type"] == "content_block_delta":
+ block, delta = blocks[data["index"]], data["delta"]
+ if delta["type"] == "text_delta":
+ block["text"] += delta["text"]
+ elif delta["type"] == "thinking_delta":
+ block["thinking"] += delta["thinking"]
+ elif delta["type"] == "input_json_delta":
+ partial_json[data["index"]] = (
+ partial_json.get(data["index"], "") + delta["partial_json"]
+ )
+ elif data["type"] == "message_delta":
+ message.update(data["delta"])
+ message["usage"] = data["usage"]
+ for index, text in partial_json.items():
+ try:
+ blocks[index]["input"] = json.loads(text)
+ except json.JSONDecodeError:
+ blocks[index]["input"] = {}
+ message["content"] = blocks
+ return message
+
+
+async def _anthropic_messages_via_connection(
+ payload: AnthropicMessagesRequest,
+ request: Request,
+ current_subject: str,
+ provider_id: str,
+ model: str,
+):
+ if payload.max_tokens is None:
+ raise HTTPException(
+ status_code = 400,
+ detail = anthropic_error_body(
+ "max_tokens: field required",
+ status = 400,
+ err_type = "invalid_request_error",
+ ),
+ )
+ _validate_anthropic_client_tools(payload.tools)
+ if _anthropic_requested_studio_tools(payload.tools) or payload.enable_tools is True:
+ raise HTTPException(
+ status_code = 400,
+ detail = "Unsloth server tools are not available with a Connection model.",
+ )
+ # Keep every turn's thinking: _build_external_messages drops it for providers that must not get it,
+ # and DeepSeek answers 400 when tools ride on a history without it.
+ messages = anthropic_messages_to_openai(
+ [m.model_dump() for m in payload.messages], payload.system, preserve_thinking = True
+ )
+ _restore_connection_tool_extras(current_subject, messages)
+ fields = {
+ "model": model,
+ "provider_id": provider_id,
+ "external_model": model,
+ "messages": messages,
+ "tools": anthropic_tools_to_openai(payload.tools or []) or None,
+ "tool_choice": anthropic_tool_choice_to_openai(payload.tool_choice),
+ "max_tokens": payload.max_tokens,
+ "stop": payload.stop_sequences or None,
+ "temperature": payload.temperature,
+ "top_p": payload.top_p,
+ "top_k": payload.top_k,
+ "min_p": payload.min_p,
+ "repetition_penalty": payload.repetition_penalty,
+ "presence_penalty": payload.presence_penalty,
+ "seed": payload.seed,
+ "session_id": payload.session_id,
+ "cancel_id": payload.cancel_id,
+ "response_format": _anthropic_response_format(payload, require_llama_compatibility = False),
+ "stream": True,
+ "stream_options": {"include_usage": True},
+ }
+ reasoning = _anthropic_reasoning_args(payload)
+ fields["enable_thinking"] = reasoning["enable_thinking"]
+ fields["reasoning_effort"] = reasoning["reasoning_effort"]
+ fields["preserve_thinking"] = reasoning.get("preserve_thinking")
+ chat_payload = ChatCompletionRequest(
+ **{name: value for name, value in fields.items() if value is not None}
+ )
+ response = await produce_openai_chat_completions(
+ chat_payload, request, current_subject, cancel_on_disconnect = True
+ )
+ if not isinstance(response, StreamingResponse):
+ return response
+ events = _anthropic_events_from_openai_sse(
+ response.body_iterator,
+ f"msg_{uuid.uuid4().hex[:24]}",
+ payload.model,
+ current_subject,
+ single_tool_use = isinstance(payload.tool_choice, dict)
+ and bool(payload.tool_choice.get("disable_parallel_tool_use")),
+ stop_sequences = payload.stop_sequences,
+ parse_thinking = chat_payload.enable_thinking is not False,
+ )
+ if payload.stream:
+ return _sse_streaming_response(events)
+ return JSONResponse(_anthropic_message_from_events([event async for event in events]))
+
+
@router.post("/messages/count_tokens")
async def anthropic_count_tokens(
payload: AnthropicMessagesRequest,
@@ -41907,6 +42672,11 @@ async def anthropic_count_tokens(
tokenizer, and returns ``{"input_tokens": int}`` only. Unlike /messages,
max_tokens is NOT required here.
"""
+ if _connection_model_ref(payload.model) is not None:
+ raise HTTPException(
+ status_code = 400,
+ detail = "Token counting is not available for a Connection model.",
+ )
# Reject malformed tools before the switch, like /messages, so an invalid
# count request can't evict the loaded model.
_validate_anthropic_client_tools(payload.tools)
@@ -42147,6 +42917,11 @@ async def anthropic_messages(
JSON).
"""
_admit_tool_access(payload)
+ connection = _connection_model_ref(payload.model)
+ if connection is not None:
+ return await _anthropic_messages_via_connection(
+ payload, request, current_subject, *connection
+ )
await _route_to_extra_slot(_switch_model_for_payload(payload))
llama_backend = get_llama_cpp_backend()
diff --git a/studio/backend/tests/test_anthropic_code_execution.py b/studio/backend/tests/test_anthropic_code_execution.py
index 22bbb191256..caaa723936e 100644
--- a/studio/backend/tests/test_anthropic_code_execution.py
+++ b/studio/backend/tests/test_anthropic_code_execution.py
@@ -394,3 +394,52 @@ def test_code_execution_error_renders_error_code(monkeypatch):
end = events[1]
assert end["type"] == "tool_end"
assert end["result"] == "Error: container_expired"
+
+
+def _captured_body(monkeypatch, **kwargs) -> dict:
+ captured: dict = {}
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ captured["body"] = json.loads(request.content.decode("utf-8"))
+ return httpx.Response(
+ 200,
+ content = _anthropic_sse([{"type": "message_stop"}]),
+ headers = {"content-type": "text/event-stream"},
+ )
+
+ _mock_http_client(monkeypatch, handler)
+
+ async def run():
+ client = _make_client()
+ async for _ in client.stream_chat_completion(
+ messages = [{"role": "user", "content": "extract"}],
+ model = "claude-sonnet-4-6",
+ max_tokens = 1024,
+ **kwargs,
+ ):
+ pass
+ await client.close()
+
+ _drive(run())
+ return captured["body"]
+
+
+def test_a_json_schema_response_format_reaches_output_config(monkeypatch):
+ # platform.claude.com/docs/en/build-with-claude/structured-outputs: output_config.format, GA.
+ schema = {
+ "type": "object",
+ "properties": {"a": {"type": "string"}},
+ "additionalProperties": False,
+ }
+ body = _captured_body(
+ monkeypatch,
+ response_format = {"type": "json_schema", "json_schema": {"name": "r", "schema": schema}},
+ reasoning_effort = "low",
+ )
+ assert body["output_config"]["format"] == {"type": "json_schema", "schema": schema}
+ assert body["output_config"]["effort"] == "low"
+
+
+def test_no_response_format_leaves_output_config_alone(monkeypatch):
+ body = _captured_body(monkeypatch)
+ assert "format" not in (body.get("output_config") or {})
diff --git a/studio/backend/tests/test_connection_model_routes.py b/studio/backend/tests/test_connection_model_routes.py
new file mode 100644
index 00000000000..1c0c14fd721
--- /dev/null
+++ b/studio/backend/tests/test_connection_model_routes.py
@@ -0,0 +1,1114 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import asyncio
+import json
+
+import pytest
+
+USER_KEY = "sk-unsloth-deadbeefdeadbeef"
+SCRIPT_KEY = "sk-unsloth-0123456789abcdef"
+CONNECTION_MODEL = "external::c1::models%2Fgemini-2.5-flash"
+TOOL = {
+ "name": "Read",
+ "description": "Read a file",
+ "input_schema": {"type": "object", "properties": {"file_path": {"type": "string"}}},
+}
+
+
+class FakeExternalClient:
+ calls: list = []
+ tool_extra: dict = {}
+
+ def __init__(self, **kwargs):
+ FakeExternalClient.calls.append({"ctor": kwargs})
+
+ def stream_chat_completion(self, **kwargs):
+ FakeExternalClient.calls[-1]["stream"] = kwargs
+
+ async def gen():
+ for chunk in (
+ {"choices": [{"index": 0, "delta": {"content": "Reading it."}}]},
+ {
+ "choices": [
+ {
+ "index": 0,
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_1",
+ "function": {"name": "Read", "arguments": '{"file_'},
+ **FakeExternalClient.tool_extra,
+ }
+ ]
+ },
+ }
+ ]
+ },
+ {
+ "choices": [
+ {
+ "index": 0,
+ "delta": {
+ "tool_calls": [
+ {"index": 0, "function": {"arguments": 'path": "a.txt"}'}}
+ ]
+ },
+ "finish_reason": "tool_calls",
+ }
+ ]
+ },
+ {"choices": [], "usage": {"prompt_tokens": 11, "completion_tokens": 7}},
+ ):
+ yield f"data: {json.dumps(chunk)}"
+ yield "data: [DONE]"
+
+ return gen()
+
+ async def served_context_window(self, model):
+ return None
+
+ async def close(self):
+ return None
+
+
+class NoLocalBackend:
+ def __getattr__(self, name):
+ raise AssertionError(f"a Connection request touched the local GGUF ({name})")
+
+
+def v1(monkeypatch):
+ from fastapi import FastAPI
+ from fastapi.testclient import TestClient
+
+ import auth.authentication as authentication
+ import routes.inference as inf
+ from auth.authentication import get_current_subject
+ from utils.api_errors import install_api_error_handlers
+
+ FakeExternalClient.calls = []
+ FakeExternalClient.tool_extra = {}
+ inf._CONNECTION_TOOL_EXTRAS.clear()
+ allowed = []
+
+ def resolve(
+ provider_id,
+ encrypted,
+ *,
+ allow_saved_key = True,
+ ):
+ allowed.append(allow_saved_key)
+ return "saved-key" if allow_saved_key else ""
+
+ monkeypatch.setattr(
+ inf.providers_db,
+ "get_provider",
+ lambda pid: {
+ "id": pid,
+ "provider_type": "custom",
+ "base_url": "http://10.0.0.5:8080/v1",
+ "api_type": "chat_completions",
+ "display_name": "Box",
+ "is_enabled": True,
+ "reasoning_config": None,
+ },
+ )
+ monkeypatch.setattr(inf, "resolve_provider_api_key_or_400", resolve)
+ monkeypatch.setattr(inf, "ExternalProviderClient", FakeExternalClient)
+ monkeypatch.setattr(inf, "get_llama_cpp_backend", lambda: NoLocalBackend())
+ monkeypatch.setattr(inf, "current_date_prompt_line", lambda **_kwargs: "")
+ monkeypatch.setattr(inf.auth_storage, "is_internal_api_key", lambda raw: False)
+ monkeypatch.setattr(
+ inf.auth_storage,
+ "validate_api_key_account",
+ lambda raw, touch = True: {
+ USER_KEY: ({"role": "owner", "key_name": "Coding agents (unsloth start)"}, "s"),
+ SCRIPT_KEY: ({"role": "owner", "key_name": "my script"}, "s"),
+ }.get(raw),
+ )
+ monkeypatch.setattr(authentication, "request_admitted_without_credential", lambda r: False)
+
+ app = FastAPI()
+ app.include_router(inf.router, prefix = "/v1")
+ install_api_error_handlers(app)
+ app.dependency_overrides[get_current_subject] = lambda: "owner"
+ client = TestClient(app)
+ client.headers["Authorization"] = f"Bearer {USER_KEY}"
+ client.allowed = allowed
+ return client
+
+
+def _messages_body(**overrides):
+ body = {
+ "model": CONNECTION_MODEL,
+ "max_tokens": 64,
+ "messages": [{"role": "user", "content": "read a.txt"}],
+ "tools": [TOOL],
+ }
+ body.update(overrides)
+ return body
+
+
+def _sse(text):
+ return [
+ json.loads(line[len("data: ") :]) for line in text.splitlines() if line.startswith("data: ")
+ ]
+
+
+def test_messages_streams_a_connection_model_with_tool_use(v1):
+ resp = v1.post("/v1/messages", json = _messages_body(stream = True))
+
+ assert resp.status_code == 200, resp.text
+ events = _sse(resp.text)
+ assert [e["type"] for e in events][0] == "message_start"
+ assert events[-1]["type"] == "message_stop"
+ starts = [e["content_block"] for e in events if e["type"] == "content_block_start"]
+ assert [b["type"] for b in starts] == ["text", "tool_use"]
+ assert starts[1]["name"] == "Read"
+ args = "".join(
+ e["delta"]["partial_json"]
+ for e in events
+ if e["type"] == "content_block_delta" and e["delta"]["type"] == "input_json_delta"
+ )
+ assert json.loads(args) == {"file_path": "a.txt"}
+ delta = next(e for e in events if e["type"] == "message_delta")
+ assert delta["delta"]["stop_reason"] == "tool_use"
+ assert delta["usage"]["output_tokens"] == 7
+
+ [call] = FakeExternalClient.calls
+ assert call["ctor"]["base_url"] == "http://10.0.0.5:8080/v1"
+ assert call["ctor"]["api_key"] == "saved-key"
+ assert call["stream"]["model"] == "models/gemini-2.5-flash"
+ assert call["stream"]["tools"][0]["function"]["name"] == "Read"
+ assert v1.allowed == [True]
+
+
+def test_messages_non_streaming_assembles_the_connection_reply(v1):
+ resp = v1.post("/v1/messages", json = _messages_body())
+
+ assert resp.status_code == 200, resp.text
+ body = resp.json()
+ assert body["model"] == CONNECTION_MODEL
+ assert body["stop_reason"] == "tool_use"
+ assert body["content"][0] == {"type": "text", "text": "Reading it."}
+ assert body["content"][1]["name"] == "Read"
+ assert body["content"][1]["input"] == {"file_path": "a.txt"}
+
+
+def test_messages_replays_tool_results_to_the_connection(v1):
+ history = [
+ {"role": "user", "content": "read a.txt"},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "tool_use", "id": "toolu_1", "name": "Read", "input": {"file_path": "a"}}
+ ],
+ },
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "hello"}],
+ },
+ ]
+ resp = v1.post("/v1/messages", json = _messages_body(messages = history, stream = True))
+
+ assert resp.status_code == 200, resp.text
+ sent = FakeExternalClient.calls[0]["stream"]["messages"]
+ assert sent[-2]["tool_calls"][0]["id"] == "toolu_1"
+ assert sent[-1] == {"role": "tool", "content": "hello", "tool_call_id": "toolu_1"}
+
+
+def test_messages_surfaces_a_connection_failure_as_an_anthropic_error(v1, monkeypatch):
+ class Failing(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ async def gen():
+ yield 'data: {"error": {"message": "quota exceeded", "code": "429"}}'
+
+ return gen()
+
+ import routes.inference as inf
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", Failing)
+ events = _sse(v1.post("/v1/messages", json = _messages_body(stream = True)).text)
+ assert events[-1]["type"] == "error"
+ assert events[-1]["error"] == {"type": "rate_limit_error", "message": "quota exceeded"}
+
+ resp = v1.post("/v1/messages", json = _messages_body())
+ assert resp.status_code == 429
+ assert resp.json()["error"]["message"] == "quota exceeded"
+
+
+def test_chat_completions_forwards_a_connection_model(v1):
+ resp = v1.post(
+ "/v1/chat/completions",
+ json = {
+ "model": CONNECTION_MODEL,
+ "stream": True,
+ "messages": [{"role": "user", "content": "hi"}],
+ },
+ )
+
+ assert resp.status_code == 200, resp.text
+ assert FakeExternalClient.calls[0]["stream"]["model"] == "models/gemini-2.5-flash"
+ assert v1.allowed == [True]
+
+
+def test_responses_streams_a_connection_model_for_codex(v1):
+ tool = {
+ "type": "function",
+ "name": "Read",
+ "description": "Read a file",
+ "parameters": TOOL["input_schema"],
+ }
+ resp = v1.post(
+ "/v1/responses",
+ json = {"model": CONNECTION_MODEL, "stream": True, "input": "read a.txt", "tools": [tool]},
+ )
+
+ assert resp.status_code == 200, resp.text
+ events = _sse(resp.text)
+ assert events[0]["type"] == "response.created"
+ assert events[-1]["type"] == "response.completed"
+ assert "".join(e["delta"] for e in events if e["type"] == "response.output_text.delta") == (
+ "Reading it."
+ )
+ [call] = [
+ e["item"]
+ for e in events
+ if e["type"] == "response.output_item.done" and e["item"]["type"] == "function_call"
+ ]
+ assert call["name"] == "Read"
+ assert json.loads(call["arguments"]) == {"file_path": "a.txt"}
+ assert events[-1]["response"]["usage"]["output_tokens"] == 7
+ [upstream] = FakeExternalClient.calls
+ assert upstream["stream"]["model"] == "models/gemini-2.5-flash"
+ assert upstream["stream"]["tools"][0]["function"]["name"] == "Read"
+ assert v1.allowed == [True]
+
+
+def test_responses_answers_a_connection_model_without_streaming(v1):
+ resp = v1.post("/v1/responses", json = {"model": CONNECTION_MODEL, "input": "read a.txt"})
+
+ assert resp.status_code == 200, resp.text
+ body = resp.json()
+ assert body["object"] == "response"
+ assert body["status"] == "completed"
+ assert body["output"][0]["content"][0]["text"] == "Reading it."
+ assert body["output"][1]["type"] == "function_call"
+ assert json.loads(body["output"][1]["arguments"]) == {"file_path": "a.txt"}
+ assert body["usage"]["input_tokens"] == 11
+
+
+def test_responses_reports_the_connection_failure(v1, monkeypatch):
+ class Failing(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ async def gen():
+ yield 'data: {"error": {"message": "API key expired.", "code": "401"}}'
+ yield "data: [DONE]"
+
+ return gen()
+
+ import routes.inference as inf
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", Failing)
+ events = _sse(
+ v1.post(
+ "/v1/responses", json = {"model": CONNECTION_MODEL, "input": "ping", "stream": True}
+ ).text
+ )
+ assert events[-1]["type"] == "response.failed"
+ assert events[-1]["response"]["error"]["message"] == "API key expired."
+
+ resp = v1.post("/v1/responses", json = {"model": CONNECTION_MODEL, "input": "ping"})
+ assert resp.status_code == 502
+ assert "API key expired." in resp.text
+
+
+def test_responses_tells_codex_a_connection_is_rate_limited(v1, monkeypatch):
+ class Limited(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ async def gen():
+ yield 'data: {"error": {"message": "Quota exceeded.", "code": "429"}}'
+
+ return gen()
+
+ import routes.inference as inf
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", Limited)
+ body = {"model": CONNECTION_MODEL, "input": "ping", "stream": True}
+ events = _sse(v1.post("/v1/responses", json = body).text)
+ assert events[-1]["response"]["error"] == {
+ "code": "rate_limit_exceeded",
+ "message": "Quota exceeded.",
+ }
+
+ resp = v1.post("/v1/responses", json = {"model": CONNECTION_MODEL, "input": "ping"})
+ assert resp.status_code == 429
+
+
+def test_count_tokens_cannot_serve_a_connection_model(v1):
+ resp = v1.post("/v1/messages/count_tokens", json = _messages_body())
+
+ assert resp.status_code == 400
+ assert "Connection model" in resp.text
+ assert FakeExternalClient.calls == []
+
+
+def test_a_keyless_caller_cannot_reach_a_connection(v1, monkeypatch):
+ import auth.authentication as authentication
+
+ monkeypatch.setattr(authentication, "request_admitted_without_credential", lambda r: True)
+ resp = v1.post("/v1/messages", json = _messages_body(stream = True))
+
+ assert resp.status_code == 403
+ assert FakeExternalClient.calls == []
+
+
+def test_a_recipe_key_still_cannot_spend_the_saved_credential(v1, monkeypatch):
+ import routes.inference as inf
+
+ monkeypatch.setattr(inf.auth_storage, "is_internal_api_key", lambda raw: True)
+ monkeypatch.setattr(inf.auth_storage, "internal_api_key_name", lambda raw: "data-recipe")
+ v1.post("/v1/messages", json = _messages_body(stream = True))
+
+ assert v1.allowed == [False]
+ assert FakeExternalClient.calls[0]["ctor"]["api_key"] == ""
+
+
+def test_an_unknown_key_cannot_spend_the_saved_credential(v1):
+ v1.headers["Authorization"] = "Bearer sk-unsloth-notarealkey0000"
+ v1.post("/v1/messages", json = _messages_body(stream = True))
+
+ assert v1.allowed == [False]
+
+
+def test_another_user_key_reaches_the_connection_without_its_saved_key(v1):
+ v1.headers["Authorization"] = f"Bearer {SCRIPT_KEY}"
+ v1.post("/v1/messages", json = _messages_body(stream = True))
+
+ assert v1.allowed == [False]
+ assert FakeExternalClient.calls[0]["ctor"]["api_key"] == ""
+
+
[email protected]("path", ["/v1/responses", "/v1/messages", "/v1/chat/completions"])
+def test_a_connection_stream_ends_at_done_even_if_the_provider_keeps_it_open(v1, monkeypatch, path):
+ import routes.inference as inf
+
+ class KeepsOpen(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ async def gen():
+ yield 'data: {"choices": [{"index": 0, "delta": {"content": "hi"}}]}'
+ yield 'data: {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}'
+ yield "data: [DONE]"
+ await asyncio.sleep(3600)
+
+ return gen()
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", KeepsOpen)
+ if path == "/v1/chat/completions":
+ # stream omitted: the collected answer must not wait for the provider to hang up.
+ resp = v1.post(
+ path, json = {"model": CONNECTION_MODEL, "messages": [{"role": "user", "content": "hi"}]}
+ )
+ assert resp.json()["choices"][0]["message"]["content"] == "hi"
+ return
+ body = (
+ {"model": CONNECTION_MODEL, "stream": True, "input": "hi"}
+ if path == "/v1/responses"
+ else _messages_body(stream = True)
+ )
+ events = _sse(v1.post(path, json = body).text)
+
+ assert events[-1]["type"] in ("response.completed", "message_stop")
+
+
+def test_a_connection_stream_without_a_space_after_data_still_reaches_the_agent(v1, monkeypatch):
+ class Compact(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ async def gen():
+ yield 'data:{"choices": [{"index": 0, "delta": {"content": "pong"}}]}'
+ yield 'data:{"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}'
+ yield "data:[DONE]"
+
+ return gen()
+
+ import routes.inference as inf
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", Compact)
+ message = v1.post("/v1/messages", json = _messages_body()).json()
+ assert message["content"] == [{"type": "text", "text": "pong"}]
+
+ response = v1.post("/v1/responses", json = {"model": CONNECTION_MODEL, "input": "ping"}).json()
+ assert response["output"][0]["content"][0]["text"] == "pong"
+
+
+SIGNED = {"extra_content": {"google": {"thought_signature": "sig-abc"}}}
+
+
+def gemini(v1, monkeypatch):
+ import routes.inference as inf
+
+ monkeypatch.setattr(
+ inf.providers_db,
+ "get_provider",
+ lambda pid: {
+ "id": pid,
+ "provider_type": "gemini",
+ "base_url": "https://generativelanguage.googleapis.com/v1beta",
+ "api_type": "chat_completions",
+ "display_name": "Gemini",
+ "is_enabled": True,
+ "reasoning_config": None,
+ },
+ )
+ FakeExternalClient.tool_extra = SIGNED
+ return v1
+
+
+def _replayed_call(messages, tool_id):
+ return next(
+ call
+ for message in messages
+ for call in (message.get("tool_calls") or [])
+ if call.get("id") == tool_id
+ )
+
+
+def test_messages_replays_a_tool_calls_thought_signature(gemini):
+ # Gemini 3 rejects a replayed functionCall without its thoughtSignature; Claude Code drops it.
+ v1 = gemini
+ first = v1.post("/v1/messages", json = _messages_body()).json()
+ tool_use = first["content"][1]
+ assert tool_use["type"] == "tool_use"
+
+ history = [
+ {"role": "user", "content": "read a.txt"},
+ {"role": "assistant", "content": [tool_use]},
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": tool_use["id"], "content": "hi"}],
+ },
+ ]
+ resp = v1.post("/v1/messages", json = _messages_body(messages = history))
+
+ assert resp.status_code == 200, resp.text
+ call = _replayed_call(FakeExternalClient.calls[-1]["stream"]["messages"], tool_use["id"])
+ assert call["extra_content"] == SIGNED["extra_content"]
+
+
+def test_responses_replays_a_tool_calls_thought_signature(gemini):
+ v1 = gemini
+ first = v1.post("/v1/responses", json = {"model": CONNECTION_MODEL, "input": "read a.txt"})
+ assert first.status_code == 200, first.text
+ call_item = next(i for i in first.json()["output"] if i["type"] == "function_call")
+
+ resp = v1.post(
+ "/v1/responses",
+ json = {
+ "model": CONNECTION_MODEL,
+ "input": [
+ {"role": "user", "content": "read a.txt"},
+ call_item,
+ {"type": "function_call_output", "call_id": call_item["call_id"], "output": "hi"},
+ ],
+ },
+ )
+
+ assert resp.status_code == 200, resp.text
+ call = _replayed_call(FakeExternalClient.calls[-1]["stream"]["messages"], call_item["call_id"])
+ assert call["extra_content"] == SIGNED["extra_content"]
+
+
+def test_a_signature_is_only_replayed_for_its_own_account(v1):
+ import routes.inference as inf
+
+ inf._remember_connection_extra("someone-else", "call", "call_1", SIGNED["extra_content"])
+ messages = [{"role": "assistant", "tool_calls": [{"id": "call_1", "type": "function"}]}]
+
+ inf._restore_connection_tool_extras("owner", messages)
+
+ assert "extra_content" not in messages[0]["tool_calls"][0]
+
+
+def test_chat_completions_without_stream_returns_one_completion_object(v1):
+ FakeExternalClient.tool_extra = SIGNED
+ resp = v1.post(
+ "/v1/chat/completions",
+ json = {"model": CONNECTION_MODEL, "messages": [{"role": "user", "content": "read a.txt"}]},
+ )
+
+ assert resp.status_code == 200, resp.text
+ assert resp.headers["content-type"].startswith("application/json")
+ body = resp.json()
+ assert body["object"] == "chat.completion"
+ message = body["choices"][0]["message"]
+ assert message["content"] == "Reading it."
+ assert message["tool_calls"][0]["function"] == {
+ "name": "Read",
+ "arguments": '{"file_path": "a.txt"}',
+ }
+ assert body["choices"][0]["finish_reason"] == "tool_calls"
+ assert body["usage"]["completion_tokens"] == 7
+ assert FakeExternalClient.calls[-1]["stream"]["stream"] is True
+
+
+def test_messages_forwards_an_output_format_to_the_connection(v1):
+ schema = {"type": "object", "properties": {"a": {"type": "string"}}}
+ resp = v1.post(
+ "/v1/messages",
+ json = _messages_body(output_config = {"format": {"type": "json_schema", "schema": schema}}),
+ )
+
+ assert resp.status_code == 200, resp.text
+ assert FakeExternalClient.calls[-1]["stream"]["response_format"] == {
+ "type": "json_schema",
+ "json_schema": {"name": "response", "schema": schema},
+ }
+
+
+def _two_tool_calls(**kwargs):
+ async def gen():
+ for index, path in enumerate(("a.txt", "b.txt")):
+ call = {
+ "index": index,
+ "id": f"call_{index}",
+ "function": {"name": "Read", "arguments": json.dumps({"file_path": path})},
+ }
+ yield f"data: {json.dumps({'choices': [{'index': 0, 'delta': {'tool_calls': [call]}}]})}"
+ yield 'data: {"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]}'
+ yield "data: [DONE]"
+
+ return gen()
+
+
[email protected]("stream", [True, False])
+def test_messages_honour_disable_parallel_tool_use(v1, monkeypatch, stream):
+ import routes.inference as inf
+
+ class TwoCalls(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ return _two_tool_calls(**kwargs)
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", TwoCalls)
+ body = _messages_body(
+ stream = stream, tool_choice = {"type": "auto", "disable_parallel_tool_use": True}
+ )
+ resp = v1.post("/v1/messages", json = body)
+
+ assert resp.status_code == 200, resp.text
+ if stream:
+ events = _sse(resp.text)
+ starts = [e for e in events if e["type"] == "content_block_start"]
+ assert [e["content_block"]["type"] for e in starts] == ["tool_use"]
+ indices = {e["index"] for e in events if "index" in e}
+ assert indices == {0}
+ else:
+ blocks = resp.json()["content"]
+ assert [b["type"] for b in blocks] == ["tool_use"]
+ assert blocks[0]["input"] == {"file_path": "a.txt"}
+
+
+def test_messages_keep_parallel_tool_calls_by_default(v1, monkeypatch):
+ import routes.inference as inf
+
+ class TwoCalls(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ return _two_tool_calls(**kwargs)
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", TwoCalls)
+ blocks = v1.post("/v1/messages", json = _messages_body()).json()["content"]
+
+ assert [b["input"] for b in blocks] == [{"file_path": "a.txt"}, {"file_path": "b.txt"}]
+
+
+def test_messages_replay_an_anthropic_connections_signed_thinking(v1, monkeypatch):
+ # Anthropic refuses a tool continuation whose assistant turn lost its signed thinking block.
+ import routes.inference as inf
+
+ native = [
+ {"type": "thinking", "thinking": "plan", "signature": "sig-xyz"},
+ {"type": "tool_use", "id": "toolu_abc", "name": "Read", "input": {"file_path": "a.txt"}},
+ ]
+
+ class Signed(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ FakeExternalClient.calls[-1]["stream"] = kwargs
+
+ async def gen():
+ call = {
+ "index": 0,
+ "id": "toolu_abc",
+ "function": {"name": "Read", "arguments": '{"file_path": "a.txt"}'},
+ }
+ yield f"data: {json.dumps({'choices': [{'index': 0, 'delta': {'tool_calls': [call]}}]})}"
+ extra = {"extra_content": {"anthropic": {"content": native}}}
+ yield f"data: {json.dumps({'choices': [{'index': 0, 'delta': extra}]})}"
+ yield 'data: {"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]}'
+ yield "data: [DONE]"
+
+ return gen()
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", Signed)
+ monkeypatch.setattr(
+ inf.providers_db,
+ "get_provider",
+ lambda pid: {
+ "id": pid,
+ "provider_type": "anthropic",
+ "base_url": "https://api.anthropic.com/v1",
+ "api_type": "chat_completions",
+ "display_name": "Claude",
+ "is_enabled": True,
+ "reasoning_config": None,
+ },
+ )
+ first = v1.post("/v1/messages", json = _messages_body()).json()
+ tool_use = next(b for b in first["content"] if b["type"] == "tool_use")
+ assert tool_use["id"] == "toolu_abc"
+
+ history = [
+ {"role": "user", "content": "read a.txt"},
+ {"role": "assistant", "content": [tool_use]},
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": "toolu_abc", "content": "hi"}],
+ },
+ ]
+ v1.post("/v1/messages", json = _messages_body(messages = history))
+
+ assistant = next(
+ m for m in FakeExternalClient.calls[-1]["stream"]["messages"] if m["role"] == "assistant"
+ )
+ assert assistant["extra_content"] == {"anthropic": {"content": native}}
+
+
+def test_messages_forward_the_agents_sampling_controls(v1):
+ v1.post(
+ "/v1/messages",
+ json = _messages_body(min_p = 0.05, repetition_penalty = 1.1, presence_penalty = 0.5),
+ )
+
+ sent = FakeExternalClient.calls[-1]["stream"]
+ assert (sent["min_p"], sent["repetition_penalty"], sent["presence_penalty"]) == (
+ 0.05,
+ 1.1,
+ 0.5,
+ )
+
+
+ "code, err_type", [("401", "authentication_error"), ("404", "not_found_error")]
+)
+def test_messages_keep_the_connections_error_class(v1, monkeypatch, code, err_type):
+ import routes.inference as inf
+
+ class Failing(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ async def gen():
+ yield f'data: {{"error": {{"message": "nope", "code": "{code}"}}}}'
+ yield "data: [DONE]"
+
+ return gen()
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", Failing)
+ events = _sse(v1.post("/v1/messages", json = _messages_body(stream = True)).text)
+ assert events[-1]["error"]["type"] == err_type
+
+ resp = v1.post("/v1/messages", json = _messages_body())
+ assert resp.status_code == int(code)
+ assert resp.json()["error"]["type"] == err_type
+
+
[email protected]("stream", [True, False])
+def test_messages_complete_interleaved_parallel_tool_calls(v1, monkeypatch, stream):
+ import routes.inference as inf
+
+ class Interleaved(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ async def gen():
+ fragments = [
+ {
+ "index": 0,
+ "id": "call_a",
+ "function": {"name": "Read", "arguments": '{"file_'},
+ },
+ {
+ "index": 1,
+ "id": "call_b",
+ "function": {"name": "Read", "arguments": '{"file_'},
+ },
+ {"index": 0, "function": {"arguments": 'path": "a.txt"}'}},
+ {"index": 1, "function": {"arguments": 'path": "b.txt"}'}},
+ ]
+ for part in fragments:
+ yield f"data: {json.dumps({'choices': [{'index': 0, 'delta': {'tool_calls': [part]}}]})}"
+ yield 'data: {"choices": [{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]}'
+ yield "data: [DONE]"
+
+ return gen()
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", Interleaved)
+ resp = v1.post("/v1/messages", json = _messages_body(stream = stream))
+
+ assert resp.status_code == 200, resp.text
+ if stream:
+ events = _sse(resp.text)
+ args: dict[int, str] = {}
+ for event in events:
+ if (
+ event["type"] == "content_block_delta"
+ and event["delta"]["type"] == "input_json_delta"
+ ):
+ # A delta after its block stopped would reach the client as a finished call with {}.
+ assert event["index"] not in {
+ e["index"]
+ for e in events[: events.index(event)]
+ if e["type"] == "content_block_stop"
+ }
+ args[event["index"]] = args.get(event["index"], "") + event["delta"]["partial_json"]
+ inputs = [json.loads(text) for _, text in sorted(args.items())]
+ else:
+ inputs = [b["input"] for b in resp.json()["content"] if b["type"] == "tool_use"]
+ assert inputs == [{"file_path": "a.txt"}, {"file_path": "b.txt"}]
+
+
+def test_messages_keep_earlier_thinking_for_a_deepseek_connection(v1, monkeypatch):
+ # DeepSeek answers 400 when tools ride on a history whose reasoning_content was dropped.
+ import routes.inference as inf
+
+ monkeypatch.setattr(
+ inf.providers_db,
+ "get_provider",
+ lambda pid: {
+ "id": pid,
+ "provider_type": "deepseek",
+ "base_url": "https://api.deepseek.com/v1",
+ "api_type": "chat_completions",
+ "display_name": "DeepSeek",
+ "is_enabled": True,
+ "reasoning_config": None,
+ },
+ )
+ history = [
+ {"role": "user", "content": "first"},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "earlier thought", "signature": ""},
+ {"type": "text", "text": "done"},
+ ],
+ },
+ {"role": "user", "content": "second"},
+ ]
+ v1.post("/v1/messages", json = _messages_body(messages = history))
+
+ assistant = next(
+ m for m in FakeExternalClient.calls[-1]["stream"]["messages"] if m["role"] == "assistant"
+ )
+ assert assistant["reasoning_content"] == "earlier thought"
+
+
+def _content_stream(*pieces):
+ class Content(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ FakeExternalClient.calls[-1]["stream"] = kwargs
+
+ async def gen():
+ for piece in pieces:
+ yield f"data: {json.dumps({'choices': [{'index': 0, 'delta': {'content': piece}}]})}"
+ yield 'data: {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}'
+ yield "data: [DONE]"
+
+ return gen()
+
+ return Content
+
+
[email protected]("stream", [True, False])
+def test_messages_turn_native_thinking_markup_into_a_thinking_block(v1, monkeypatch, stream):
+ # The native Anthropic translator streams thinking as <think> markup, the Studio chat's form.
+ import routes.inference as inf
+
+ monkeypatch.setattr(
+ inf, "ExternalProviderClient", _content_stream("<think>plan", "ning</think>ans", "wer")
+ )
+ body = _messages_body(stream = stream, thinking = {"type": "enabled", "budget_tokens": 1024})
+ resp = v1.post("/v1/messages", json = body)
+
+ assert resp.status_code == 200, resp.text
+ if stream:
+ events = _sse(resp.text)
+ kinds = [e["content_block"]["type"] for e in events if e["type"] == "content_block_start"]
+ text = "".join(
+ e["delta"].get("text", "") for e in events if e["type"] == "content_block_delta"
+ )
+ thinking = "".join(
+ e["delta"].get("thinking", "") for e in events if e["type"] == "content_block_delta"
+ )
+ assert kinds == ["thinking", "text"]
+ assert (thinking, text) == ("planning", "answer")
+ else:
+ blocks = resp.json()["content"]
+ assert [b["type"] for b in blocks] == ["thinking", "text"]
+ assert blocks[0]["thinking"] == "planning"
+ assert blocks[1]["text"] == "answer"
+
+
[email protected]("stream", [True, False])
+def test_messages_stop_at_a_requested_stop_sequence(v1, monkeypatch, stream):
+ import routes.inference as inf
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", _content_stream("before E", "ND after"))
+ body = _messages_body(stream = stream, stop_sequences = ["END"])
+ resp = v1.post("/v1/messages", json = body)
+
+ assert resp.status_code == 200, resp.text
+ if stream:
+ events = _sse(resp.text)
+ text = "".join(
+ e["delta"].get("text", "") for e in events if e["type"] == "content_block_delta"
+ )
+ delta = next(e for e in events if e["type"] == "message_delta")["delta"]
+ else:
+ message = resp.json()
+ text = "".join(b.get("text", "") for b in message["content"])
+ delta = message
+ assert text == "before "
+ assert (delta["stop_reason"], delta["stop_sequence"]) == ("stop_sequence", "END")
+
+
+def test_messages_without_a_stop_hit_keep_all_text(v1, monkeypatch):
+ import routes.inference as inf
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", _content_stream("before E", "N after"))
+ message = v1.post("/v1/messages", json = _messages_body(stop_sequences = ["END"])).json()
+
+ assert "".join(b.get("text", "") for b in message["content"]) == "before EN after"
+ assert message["stop_reason"] == "end_turn"
+
+
[email protected]("stream", [True, False])
+def test_messages_keep_text_order_when_the_finish_chunk_carries_text(v1, monkeypatch, stream):
+ # vLLM can put the last text and finish_reason in one chunk; the held-back tail must follow it.
+ import routes.inference as inf
+
+ class TerminalText(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ async def gen():
+ yield 'data: {"choices": [{"index": 0, "delta": {"content": "hello world"}, "finish_reason": "stop"}]}'
+ yield "data: [DONE]"
+
+ return gen()
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", TerminalText)
+ resp = v1.post("/v1/messages", json = _messages_body(stream = stream, stop_sequences = ["END"]))
+
+ if stream:
+ events = _sse(resp.text)
+ text = "".join(
+ e["delta"].get("text", "") for e in events if e["type"] == "content_block_delta"
+ )
+ else:
+ text = "".join(b.get("text", "") for b in resp.json()["content"])
+ assert text == "hello world"
+
+
+def test_messages_keep_default_thinking_out_of_the_stop_filter(v1, monkeypatch):
+ # No thinking controls sent, yet the provider thinks by default: a stop word in the thought
+ # must neither cut the answer nor leak the markup.
+ import routes.inference as inf
+
+ monkeypatch.setattr(
+ inf, "ExternalProviderClient", _content_stream("<think>private plan", "</think>answer")
+ )
+ message = v1.post("/v1/messages", json = _messages_body(stop_sequences = ["plan"])).json()
+
+ blocks = message["content"]
+ assert [b["type"] for b in blocks] == ["thinking", "text"]
+ assert blocks[1]["text"] == "answer"
+ assert message["stop_reason"] == "end_turn"
+
+
[email protected]("stream", [True, False])
+def test_chat_completions_move_native_thinking_to_reasoning_content(v1, monkeypatch, stream):
+ import routes.inference as inf
+
+ monkeypatch.setattr(
+ inf, "ExternalProviderClient", _content_stream("<think>pl", "an</think>ans", "wer")
+ )
+ body = {"model": CONNECTION_MODEL, "messages": [{"role": "user", "content": "hi"}]}
+ resp = v1.post("/v1/chat/completions", json = {**body, "stream": stream})
+
+ assert resp.status_code == 200, resp.text
+ if stream:
+ chunks = _sse(resp.text.replace("data: [DONE]", ""))
+ deltas = [c["choices"][0]["delta"] for c in chunks if c.get("choices")]
+ reasoning = "".join(d.get("reasoning_content") or "" for d in deltas)
+ content = "".join(d.get("content") or "" for d in deltas)
+ else:
+ message = resp.json()["choices"][0]["message"]
+ reasoning, content = message.get("reasoning_content"), message["content"]
+ assert (reasoning, content) == ("plan", "answer")
+
+
+def test_responses_move_default_thinking_out_of_the_answer(v1, monkeypatch):
+ # No reasoning controls sent, yet the provider thinks by default.
+ import routes.inference as inf
+
+ monkeypatch.setattr(
+ inf, "ExternalProviderClient", _content_stream("<think>plan</think>", "answer")
+ )
+ body = v1.post("/v1/responses", json = {"model": CONNECTION_MODEL, "input": "hi"}).json()
+
+ message = next(item for item in body["output"] if item["type"] == "message")
+ assert message["content"][0]["text"] == "answer"
+ assert any(item["type"] == "reasoning" for item in body["output"])
+
+
+def _chat_text(resp, stream):
+ if stream:
+ chunks = _sse(resp.text.replace("data: [DONE]", ""))
+ return "".join(
+ (c["choices"][0]["delta"].get("content") or "") for c in chunks if c.get("choices")
+ )
+ return resp.json()["choices"][0]["message"]["content"]
+
+
[email protected]("stream", [True, False])
+def test_chat_completions_stop_at_a_requested_stop_sequence(v1, monkeypatch, stream):
+ import routes.inference as inf
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", _content_stream("before E", "ND after"))
+ body = {
+ "model": CONNECTION_MODEL,
+ "messages": [{"role": "user", "content": "hi"}],
+ "stop": ["END"],
+ "stream": stream,
+ }
+ resp = v1.post("/v1/chat/completions", json = body)
+
+ assert resp.status_code == 200, resp.text
+ assert _chat_text(resp, stream) == "before "
+ if not stream:
+ assert resp.json()["choices"][0]["finish_reason"] == "stop"
+
+
[email protected]("stream", [True, False])
+def test_chat_completions_keep_a_literal_think_tag_inside_the_answer(v1, monkeypatch, stream):
+ # Only a leading block is thinking; JSON that mentions the tag must come back intact.
+ import routes.inference as inf
+
+ answer = '{"template": "<think>example</think>"}'
+ monkeypatch.setattr(inf, "ExternalProviderClient", _content_stream(answer[:12], answer[12:]))
+ body = {
+ "model": CONNECTION_MODEL,
+ "messages": [{"role": "user", "content": "hi"}],
+ "response_format": {"type": "json_object"},
+ "stream": stream,
+ }
+ resp = v1.post("/v1/chat/completions", json = body)
+
+ assert resp.status_code == 200, resp.text
+ assert _chat_text(resp, stream) == answer
+
+
[email protected]("stream", [True, False])
+def test_responses_fail_on_a_provider_error_beside_choices(v1, monkeypatch, stream):
+ # OpenRouter's mid-stream errors carry both `error` and a non-empty choices array.
+ import routes.inference as inf
+
+ class MidStreamError(FakeExternalClient):
+ def stream_chat_completion(self, **kwargs):
+ async def gen():
+ yield 'data: {"choices": [{"index": 0, "delta": {"content": "par"}}]}'
+ yield (
+ 'data: {"error": {"message": "Rate limited.", "code": "429"}, '
+ '"choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "error"}]}'
+ )
+ yield "data: [DONE]"
+
+ return gen()
+
+ monkeypatch.setattr(inf, "ExternalProviderClient", MidStreamError)
+ resp = v1.post(
+ "/v1/responses", json = {"model": CONNECTION_MODEL, "input": "hi", "stream": stream}
+ )
+
+ if stream:
+ events = _sse(resp.text)
+ assert events[-1]["type"] == "response.failed"
+ assert events[-1]["response"]["error"]["code"] == "rate_limit_exceeded"
+ else:
+ assert resp.status_code == 429
+ assert "Rate limited." in resp.text
+
+
+def test_responses_keep_a_literal_think_tag_inside_the_answer(v1, monkeypatch):
+ import routes.inference as inf
+
+ answer = 'Here is the template: {"template": "<think>example</think>"}'
+ monkeypatch.setattr(inf, "ExternalProviderClient", _content_stream(answer[:20], answer[20:]))
+ body = v1.post("/v1/responses", json = {"model": CONNECTION_MODEL, "input": "hi"}).json()
+
+ message = next(item for item in body["output"] if item["type"] == "message")
+ assert message["content"][0]["text"] == answer
+
+
[email protected]("path", ["/v1/chat/completions", "/v1/responses", "/v1/messages"])
+def test_a_leading_tag_stays_text_when_thinking_is_off(v1, monkeypatch, path):
+ import routes.inference as inf
+
+ answer = "<think>This is literal XML</think>remaining text"
+ monkeypatch.setattr(inf, "ExternalProviderClient", _content_stream(answer))
+ if path == "/v1/chat/completions":
+ body = {
+ "model": CONNECTION_MODEL,
+ "messages": [{"role": "user", "content": "hi"}],
+ "enable_thinking": False,
+ }
+ text = v1.post(path, json = body).json()["choices"][0]["message"]["content"]
+ elif path == "/v1/responses":
+ body = {"model": CONNECTION_MODEL, "input": "hi", "reasoning": {"effort": "none"}}
+ output = v1.post(path, json = body).json()["output"]
+ text = next(i for i in output if i["type"] == "message")["content"][0]["text"]
+ else:
+ body = _messages_body(thinking = {"type": "disabled"})
+ text = "".join(b.get("text", "") for b in v1.post(path, json = body).json()["content"])
+ assert text == answer
+
+
+def test_messages_carry_preserve_thinking_to_a_vllm_connection(v1, monkeypatch):
+ import routes.inference as inf
+
+ monkeypatch.setattr(
+ inf.providers_db,
+ "get_provider",
+ lambda pid: {
+ "id": pid,
+ "provider_type": "vllm",
+ "base_url": "http://10.0.0.5:8000/v1",
+ "api_type": "chat_completions",
+ "display_name": "vLLM",
+ "is_enabled": True,
+ "reasoning_config": None,
+ },
+ )
+ history = [
+ {"role": "user", "content": "first"},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "thinking", "thinking": "earlier thought", "signature": ""},
+ {"type": "text", "text": "done"},
+ ],
+ },
+ {"role": "user", "content": "second"},
+ ]
+ v1.post("/v1/messages", json = _messages_body(messages = history, preserve_thinking = True))
+
+ assistant = next(
+ m for m in FakeExternalClient.calls[-1]["stream"]["messages"] if m["role"] == "assistant"
+ )
+ assert assistant["reasoning_content"] == "earlier thought"
diff --git a/studio/backend/tests/test_custom_provider_api_type.py b/studio/backend/tests/test_custom_provider_api_type.py
index 75b20fd11bd..f1d8471b628 100644
--- a/studio/backend/tests/test_custom_provider_api_type.py
+++ b/studio/backend/tests/test_custom_provider_api_type.py
@@ -327,3 +327,27 @@ def test_responses_follow_up_preserves_reasoning_metadata():
messages, supports_vision = False, provider_type = "custom", api_type = "responses"
)
assert built[0]["extra_content"] == reasoning
+
+
+def test_native_openai_follow_up_keeps_only_its_reasoning_items():
+ # OpenAI wants a tool call's reasoning items back with its outputs (function-calling guide).
+ from routes.inference import _build_external_messages
+
+ reasoning = [{"type": "reasoning", "id": "rs_1", "encrypted_content": "enc_blob"}]
+ messages = [
+ ChatMessage(
+ role = "assistant",
+ content = None,
+ tool_calls = [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "lookup", "arguments": "{}"},
+ }
+ ],
+ extra_content = {"openai_responses_reasoning": reasoning, "google": {"x": 1}},
+ ),
+ ChatMessage(role = "tool", tool_call_id = "call_1", content = "result"),
+ ]
+ built = _build_external_messages(messages, supports_vision = False, provider_type = "openai")
+ assert built[0]["extra_content"] == {"openai_responses_reasoning": reasoning}
diff --git a/studio/backend/tests/test_external_confirm_gate_and_saved_keys.py b/studio/backend/tests/test_external_confirm_gate_and_saved_keys.py
index d6b5774b894..8669a32b193 100644
--- a/studio/backend/tests/test_external_confirm_gate_and_saved_keys.py
+++ b/studio/backend/tests/test_external_confirm_gate_and_saved_keys.py
@@ -398,6 +398,39 @@ def test_a_third_party_key_never_unlocks_a_saved_connection(monkeypatch):
assert seen == [False]
+ "record, allowed",
+ [
+ ({"role": "owner", "key_name": "Coding agents (unsloth start)"}, True),
+ ({"role": "owner", "key_name": "my script"}, False),
+ ({"role": "member", "key_name": "Coding agents (unsloth start)"}, False),
+ ],
+)
+def test_only_the_owners_unsloth_start_key_unlocks_a_saved_connection(monkeypatch, record, allowed):
+ from auth.authentication import API_KEY_PREFIX
+ from routes import inference as inf
+
+ seen: list[bool] = []
+ inf = _install(monkeypatch, "openai", record_keys = seen)
+ monkeypatch.setattr(inf.auth_storage, "is_internal_api_key", lambda raw: False)
+ monkeypatch.setattr(
+ inf.auth_storage, "validate_api_key_account", lambda raw, touch = True: (record, "s")
+ )
+ _run(inf, _payload(), request = _request(f"Bearer {API_KEY_PREFIX}deadbeefdeadbeef"))
+ assert seen == [allowed]
+
+
+def test_a_keyless_caller_never_unlocks_a_saved_connection(monkeypatch):
+ from auth import authentication
+ from routes import inference as inf
+
+ seen: list[bool] = []
+ inf = _install(monkeypatch, "openai", record_keys = seen)
+ monkeypatch.setattr(authentication, "admitted_without_session", lambda request: True)
+ _run(inf, _payload(), request = _request())
+ assert seen == [False]
+
+
def test_an_interactive_session_still_uses_its_saved_connection(monkeypatch):
"""Unsloth's own chat sends a session JWT and no API key at all."""
from routes import inference as inf
diff --git a/studio/backend/tests/test_external_preserve_thinking.py b/studio/backend/tests/test_external_preserve_thinking.py
index 4f983feaae7..1dc7a72830e 100644
--- a/studio/backend/tests/test_external_preserve_thinking.py
+++ b/studio/backend/tests/test_external_preserve_thinking.py
@@ -153,3 +153,16 @@ def test_tool_round_thinking_reaches_the_next_request(monkeypatch, provider, pre
assert replayed.get("reasoning_content") == expected
if provider == "vllm":
assert replayed["reasoning"] == expected
+
+
+def test_deepseek_history_keeps_reasoning_for_tool_continuations():
+ # DeepSeek thinking mode answers 400 to a tool continuation without the turn's reasoning_content.
+ calls = [
+ {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}
+ ]
+ message = ChatMessage(
+ role = "assistant", content = "", reasoning_content = "prior thought", tool_calls = calls
+ )
+ result = _build_external_messages([message], False, provider_type = "deepseek")
+ assert result[0]["reasoning_content"] == "prior thought"
+ assert result[0]["tool_calls"][0]["function"]["name"] == "lookup"
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 9e84e6cac69..38c6e83bc15 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -617,7 +617,7 @@ def test_responses_endpoint_wires_auto_switch_before_streaming_dispatch():
src = inspect.getsource(inference_route.openai_responses)
assert "_maybe_auto_switch_model" in src
hook_at = src.index("_maybe_auto_switch_model")
- assert src.index("if payload.stream:") < hook_at
+ assert src.index("if payload.stream or connection is not None:") < hook_at
assert hook_at < src.index("_responses_stream")
diff --git a/studio/backend/tests/test_unsloth_start_api_key.py b/studio/backend/tests/test_unsloth_start_api_key.py
new file mode 100644
index 00000000000..0eb23feeb9c
--- /dev/null
+++ b/studio/backend/tests/test_unsloth_start_api_key.py
@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import importlib.util
+import secrets
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from auth import storage
+from auth.authentication import create_access_token
+
+START = storage.UNSLOTH_START_API_KEY_NAME
+
+
[email protected](autouse = True)
+def isolated_auth_db(tmp_path, monkeypatch):
+ monkeypatch.setattr(storage, "DB_PATH", tmp_path / "auth.db")
+ monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", tmp_path / ".bootstrap_password")
+ monkeypatch.setattr(storage, "_bootstrap_password", None)
+ monkeypatch.setattr(storage, "_api_key_pbkdf2_salt_cache", None)
+ monkeypatch.setattr(storage, "_credential_encryption_key_cache", None)
+ storage.create_initial_user(
+ username = storage.DEFAULT_ADMIN_USERNAME,
+ password = "human-password-123",
+ jwt_secret = secrets.token_urlsafe(64),
+ )
+ yield
+
+
+def client():
+ route_path = Path(__file__).resolve().parents[1] / "routes" / "auth.py"
+ spec = importlib.util.spec_from_file_location("_unsloth_start_key_auth_route", route_path)
+ auth_route = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(auth_route)
+ app = FastAPI()
+ app.include_router(auth_route.router, prefix = "/api/auth")
+ return TestClient(app)
+
+
+def _mint(client, bearer, name):
+ return client.post(
+ "/api/auth/api-keys", headers = {"Authorization": f"Bearer {bearer}"}, json = {"name": name}
+ )
+
+
+def test_only_a_desktop_session_can_mint_the_unsloth_start_key(client):
+ owner = storage.DEFAULT_ADMIN_USERNAME
+ user_key, _row = storage.create_api_key(owner, name = "my script")
+
+ assert _mint(client, user_key, START).status_code == 403
+ assert _mint(client, create_access_token(owner), START).status_code == 403
+ assert _mint(client, user_key, "another script").status_code == 200
+ minted = _mint(client, create_access_token(owner, desktop = True), START)
+ assert minted.status_code == 200, minted.text
+ assert minted.json()["api_key"]["name"] == START
+
+
+def test_only_the_unsloth_start_key_may_use_a_saved_connection():
+ from routes import inference as inf
+
+ owner = storage.DEFAULT_ADMIN_USERNAME
+ start_key, _ = storage.create_api_key(owner, name = START)
+ user_key, _ = storage.create_api_key(owner, name = "my script")
+ internal_key, _ = storage.create_api_key(owner, name = START, internal = True)
+
+ def allowed(raw):
+ return inf._request_may_use_saved_provider_key(
+ SimpleNamespace(headers = {"authorization": f"Bearer {raw}"})
+ )
+
+ assert allowed(start_key) is True
+ assert allowed(user_key) is False
+ assert allowed(internal_key) is False
+ assert allowed("sk-unsloth-0000000000000000") is False
+
+
+def test_the_cli_mints_the_reserved_name():
+ start_py = Path(__file__).resolve().parents[3] / "unsloth_cli" / "commands" / "start.py"
+ assert f'{{"name": "{START}"}}' in start_py.read_text(encoding = "utf-8")
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py
index c703a74498c..24982e77ea0 100644
--- a/unsloth_cli/commands/start.py
+++ b/unsloth_cli/commands/start.py
@@ -25,7 +25,7 @@ import urllib.error
import urllib.request
from pathlib import Path
from typing import Literal, NamedTuple, NoReturn, Optional
-from urllib.parse import urlencode, urlparse
+from urllib.parse import quote, urlencode, urlparse
import click
import typer
@@ -1878,6 +1878,7 @@ def _agent_api_key(
explicit: Optional[str],
*,
auto_started: bool = False,
+ minted_only: bool = False,
) -> str:
cache = _key_cache_path()
if explicit:
@@ -1887,7 +1888,7 @@ def _agent_api_key(
# The server was auto-started for this run, so an exported UNSLOTH_API_KEY meant for some other server must not fail the launch: the loopback mint path below is guaranteed to work. An explicit key the fresh server accepts is still honored above.
# Replay a key the user saved for THIS EXACT server first (scoped per base, so it only goes back there, including a remote or SSH-tunnelled Unsloth whose secret the local handshake cannot match). Skip ones the server rejects.
- for key in _cached_keys(cache, base, "saved"):
+ for key in [] if minted_only else _cached_keys(cache, base, "saved"):
if _key_accepted(base, key):
_remember_key(cache, base, key, "saved")
return key
@@ -4613,6 +4614,52 @@ def _launch(
return code if code >= 0 else 128 - code
+_CONNECTION_MODEL_PREFIX = "external::"
+
+
+def _connection_entry(base: str, key: str, model: Optional[str]) -> Optional[dict]:
+ # Studio names a Connection model external::<connection id>::<url-encoded model>; a bare name counts when one enabled Connection lists it and no loaded model has it.
+ if not model:
+ return None
+ explicit = model.startswith(_CONNECTION_MODEL_PREFIX)
+ if not explicit and any(
+ _model_id_matches(entry.get("id"), name, allow_casefold = is_loopback_url(base))
+ for entry in _loaded_models(base, key)
+ if entry.get("loaded") is not False
+ for name in (model, _split_repo_variant(model)[0])
+ ):
+ return None
+ try:
+ providers = _http_json("GET", f"{base}/api/providers/", key)
+ except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc:
+ # A 4xx is an older server or a key without Connections; anything else must not fall
+ # through to a load that could evict the resident model.
+ if explicit or not (isinstance(exc, urllib.error.HTTPError) and exc.code < 500):
+ _fail_request(exc, "Couldn't list Connections")
+ return None
+ enabled = [p for p in providers or [] if isinstance(p, dict) and p.get("is_enabled")]
+ if explicit:
+ provider_id = model[len(_CONNECTION_MODEL_PREFIX) :].partition("::")[0]
+ matches = [p for p in enabled if p.get("id") == provider_id]
+ if not matches:
+ _fail(f"No enabled Connection {provider_id!r} on {base}. Check Settings → Connections.")
+ else:
+ matches = [p for p in enabled if model in (p.get("models") or [])]
+ if not matches:
+ return None
+ usable = [p for p in matches if p.get("auth_kind") != "chatgpt_oauth"]
+ if not usable:
+ _fail(
+ f"{model} is on a ChatGPT subscription Connection, which only the Unsloth chat can use."
+ )
+ if explicit:
+ return {"id": model, "connection": usable[0].get("display_name") or usable[0]["id"]}
+ ids = [f"{_CONNECTION_MODEL_PREFIX}{p['id']}::{quote(model, safe = '')}" for p in usable]
+ if len(ids) > 1:
+ _fail(f"{model} is offered by more than one Connection. Pass one of: {', '.join(ids)}")
+ return {"id": ids[0], "connection": usable[0].get("display_name") or usable[0]["id"]}
+
+
def _connect(
api_key: Optional[str],
model: Optional[str],
@@ -4622,19 +4669,59 @@ def _connect(
launch: bool = True,
server_options: ServerOptions = ServerOptions(),
preload_check = None,
+ connections: bool = False,
) -> tuple:
+ requested = model
+ explicit_connection = bool(model) and model.startswith(_CONNECTION_MODEL_PREFIX)
# `--model org/name:QUANT` is shorthand for `--model org/name --gguf-variant QUANT`. Split it before we match or serve so the attach path resolves against the already-loaded `org/name` (listed without the suffix) instead of reloading a `:`-suffixed repo id, which Unsloth rejects and which would evict a model another session is using.
- if model:
+ if model and not explicit_connection:
repo, variant = _split_repo_variant(model)
if variant:
model = repo
if not load.gguf_variant:
load = load._replace(gguf_variant = variant)
base, server = _require_studio(
- model, load, serve = serve, launch = launch, server_options = server_options
+ model,
+ load,
+ serve = serve and not explicit_connection,
+ launch = launch,
+ server_options = server_options,
)
try:
key = _agent_api_key(base, api_key, auto_started = server is not None)
+ connection = None if server is not None else _connection_entry(base, key, requested)
+ if connection is not None:
+ if not connections:
+ _fail(
+ f"{requested} is a Connection model, and this agent can only use models "
+ "loaded in Unsloth. Connection models work with `unsloth start claude`, "
+ "`unsloth start codex` and `unsloth start opencode`."
+ )
+ typer.echo(f"Using {requested} from the Connection {connection['connection']}.")
+ # A Connection reports no context window; an explicit one lets agents size compaction.
+ if load.max_seq_length > 0:
+ connection["context_length"] = load.max_seq_length
+ else:
+ typer.echo(
+ "Pass --max-seq-length with the model's context window so the agent can "
+ "compact before the provider runs out of context.",
+ err = True,
+ )
+ if (
+ key not in _cached_keys(_key_cache_path(), base, "minted")
+ and not api_key
+ and is_loopback_url(base)
+ and verify_studio_identity(base)
+ ):
+ key = _agent_api_key(base, None, minted_only = True)
+ if key not in _cached_keys(_key_cache_path(), base, "minted"):
+ typer.echo(
+ "Warning: Studio only sends a Connection's saved API key for the key "
+ "`unsloth start` issues, so drop --api-key / UNSLOTH_API_KEY unless this "
+ "Connection needs no key.",
+ err = True,
+ )
+ return base, key, connection
# A server we just started has exactly the requested model loaded, so resolve to whatever it is serving instead of re-matching the raw --model string. Only an attach can still trigger an evicting load, so _resolve_model gets the pre-load check and runs it only when that load is imminent.
entry = _resolve_model(
base,
@@ -6626,16 +6713,22 @@ def claude(
launch = launch,
preload_check = functools.partial(_attach_gguf_check, _CLAUDE_GGUF_AGENT),
server_options = server_options,
+ connections = True,
)
# Before the launch owns the server, so a rejection tears it down, not atexit.
try:
- _require_gguf_for_agent(_CLAUDE_GGUF_AGENT, base, key, entry["id"])
+ if not entry.get("connection"):
+ _require_gguf_for_agent(_CLAUDE_GGUF_AGENT, base, key, entry["id"])
except BaseException:
_shutdown_auto_served()
raise
model_id = entry["id"]
if as_subagent:
- subagent_id = _subagent_model_id(base, key, entry, model, gguf_variant)
+ subagent_id = (
+ model_id
+ if entry.get("connection")
+ else _subagent_model_id(base, key, entry, model, gguf_variant)
+ )
subagent_model = {**entry, "id": subagent_id}
window = subagent_model.get("context_length") or subagent_model.get("max_context_length")
server_env = {
@@ -6772,17 +6865,25 @@ def codex(
launch = launch,
preload_check = functools.partial(_attach_gguf_check, _CODEX_GGUF_AGENT),
server_options = server_options,
+ connections = True,
)
# This preflight runs after _connect may have auto-started a server but before _run takes over its lifecycle, so tear the server down here if it rejects the model (a transformers-backend model) rather than leaving it on the atexit backstop.
try:
- _require_gguf_for_agent(_CODEX_GGUF_AGENT, base, key, entry["id"])
+ if not entry.get("connection"):
+ _require_gguf_for_agent(_CODEX_GGUF_AGENT, base, key, entry["id"])
except BaseException:
_shutdown_auto_served()
raise
+ if app and entry.get("connection"):
+ _fail("The Codex app can't use a Connection model yet. Drop --app to run it in Codex.")
if app:
return _codex_app_session(base, api_key, entry)
if as_subagent:
- subagent_id = _subagent_model_id(base, key, entry, model, gguf_variant)
+ subagent_id = (
+ entry["id"]
+ if entry.get("connection")
+ else _subagent_model_id(base, key, entry, model, gguf_variant)
+ )
subagent_model = {**entry, "id": subagent_id}
with _session_config("codex-subagent", launch, persist = persist) as home:
bridge_config = write_codex_subagent_bridge(
@@ -7005,7 +7106,12 @@ def opencode(
serve = serve,
launch = launch,
server_options = server_options,
+ connections = True,
)
+ if app and entry.get("connection"):
+ _fail(
+ "The OpenCode app can't use a Connection model yet. Drop --app to run it in OpenCode."
+ )
if app:
target = _opencode_app_target(max_tokens, server_options.request_body())
return _add_app_provider(target, base, api_key, entry)
@@ -7015,7 +7121,11 @@ def opencode(
err = True,
)
if as_subagent:
- subagent_id = _subagent_model_id(base, key, entry, model, gguf_variant)
+ subagent_id = (
+ entry["id"]
+ if entry.get("connection")
+ else _subagent_model_id(base, key, entry, model, gguf_variant)
+ )
subagent_model = {**entry, "id": subagent_id}
# Stay append-safe for a bare no-launch recipe: a later `run <prompt>` would make `opencode --auto run ...` parse as the TUI, so keep yolo in the inline fallback.
route_native_auto = (
diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py
index cc43fd0944d..3b94077c913 100644
--- a/unsloth_cli/tests/test_start.py
+++ b/unsloth_cli/tests/test_start.py
@@ -1497,9 +1497,13 @@ def test_missing_pi_subagent_extension_fails_before_install_or_connect(monkeypat
assert "Missing Pi subagent extension" in result.output
+class _Calls(list):
+ pass
+
+
@pytest.fixture()
def fake_studio(tmp_path, monkeypatch):
- calls = []
+ calls = _Calls()
state = {"models": [MODEL]}
def http_json(
@@ -1521,6 +1525,10 @@ def fake_studio(tmp_path, monkeypatch):
return {"key": "sk-unsloth-feedfacefeedface"}
if url.endswith("/api/settings/embedding-model"):
return {"embedding_model": "unsloth/bge-small-en-v1.5"}
+ if url.endswith("/api/providers/"):
+ if state.get("providers_status"):
+ raise urllib.error.HTTPError(url, state["providers_status"], "error", None, None)
+ return state.get("providers", [])
if url.endswith("/api/inference/load"):
already_loaded = state["models"][0]["id"] == payload["model_path"]
state["models"] = [{"id": payload["model_path"], "context_length": 4096}]
@@ -1543,6 +1551,7 @@ def fake_studio(tmp_path, monkeypatch):
# No `claude` on PATH, so _claude_flags never probes the real binary.
monkeypatch.setattr(start.shutil, "which", lambda _: None)
monkeypatch.delenv("UNSLOTH_API_KEY", raising = False)
+ calls.state = state
return calls
@@ -6432,6 +6441,228 @@ def test_connect_opencode_as_subagent_preserves_cloud_parent(fake_studio, tmp_pa
assert "Unsloth is available as @unsloth and in /models." in result.output
+GEMINI_CONNECTION = {
+ "id": "c1",
+ "display_name": "Gemini",
+ "is_enabled": True,
+ "models": ["gemini-2.5-flash"],
+}
+GEMINI_ID = "external::c1::gemini-2.5-flash"
+
+
+def _loads(calls) -> list:
+ return [call for call in calls if call[1].endswith(("/api/inference/load", "/unload"))]
+
+
+def test_claude_uses_a_connection_model_without_loading_anything(fake_studio):
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ result = CliRunner().invoke(
+ start.start_app, ["claude", "--no-launch", "--model", "gemini-2.5-flash"]
+ )
+ assert result.exit_code == 0, result.output
+ assert _loads(fake_studio) == []
+ _assert_env_set(result.output, "ANTHROPIC_MODEL", GEMINI_ID)
+ _assert_env_set(result.output, "ANTHROPIC_BASE_URL", BASE)
+ assert "CLAUDE_CODE_MAX_CONTEXT_TOKENS" not in result.output
+ assert "Using gemini-2.5-flash from the Connection Gemini." in result.output
+ assert "saved API key" not in result.output
+
+
+def test_a_user_supplied_key_is_warned_it_carries_no_connection_key(fake_studio):
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ result = CliRunner().invoke(
+ start.start_app,
+ ["claude", "--no-launch", "--model", "gemini-2.5-flash", "--api-key", "sk-unsloth-mine"],
+ )
+ assert result.exit_code == 0, result.output
+ assert "only sends a Connection's saved API key for the key `unsloth start` issues" in (
+ result.output
+ )
+
+
+def test_a_remembered_user_key_gives_way_to_the_minted_key_for_a_connection(fake_studio):
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ first = CliRunner().invoke(
+ start.start_app, ["claude", "--no-launch", "--api-key", "sk-unsloth-mine"]
+ )
+ assert first.exit_code == 0, first.output
+ result = CliRunner().invoke(
+ start.start_app, ["claude", "--no-launch", "--model", "gemini-2.5-flash"]
+ )
+ assert result.exit_code == 0, result.output
+ assert "saved API key" not in result.output
+ assert "sk-unsloth-mine" not in result.output
+
+
+def test_claude_subagent_runs_on_a_connection_model(fake_studio, tmp_path):
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ result = CliRunner().invoke(
+ start.start_app, ["claude", "--as-subagent", "--no-launch", "--model", GEMINI_ID]
+ )
+ assert result.exit_code == 0, result.output
+ assert _loads(fake_studio) == []
+ plugin = tmp_path / "agents" / "claude-subagent" / "unsloth-local-agent"
+ env = json.loads((plugin / ".mcp.json").read_text())["mcpServers"]["unsloth"]["env"]
+ assert env["UNSLOTH_CLAUDE_SUBAGENT_MODEL"] == GEMINI_ID
+ assert "UNSLOTH_CLAUDE_SUBAGENT_CONTEXT_WINDOW" not in env
+
+
+def test_opencode_uses_a_connection_model(fake_studio, tmp_path):
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ result = CliRunner().invoke(
+ start.start_app, ["opencode", "--no-launch", "--model", "gemini-2.5-flash"]
+ )
+ assert result.exit_code == 0, result.output
+ assert _loads(fake_studio) == []
+ assert (
+ _opencode_inline_config(result.output)["model"] == f"{start._OPENCODE_PROVIDER}/{GEMINI_ID}"
+ )
+
+
+def test_opencode_sizes_a_connection_model_from_max_seq_length(fake_studio, tmp_path):
+ # Without a limit OpenCode treats the context as 0 and never compacts.
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ result = CliRunner().invoke(
+ start.start_app,
+ ["opencode", "--no-launch", "--model", "gemini-2.5-flash", "--max-seq-length", "1048576"],
+ )
+ assert result.exit_code == 0, result.output
+ assert _loads(fake_studio) == []
+ config = json.loads((tmp_path / "agents" / "opencode" / "opencode.json").read_text())
+ model = config["provider"][start._OPENCODE_PROVIDER]["models"][GEMINI_ID]
+ assert model["limit"]["context"] == 1048576
+ assert "Pass --max-seq-length" not in result.output
+
+
+def test_a_connection_model_without_a_window_says_how_to_give_one(fake_studio):
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ result = CliRunner().invoke(
+ start.start_app, ["opencode", "--no-launch", "--model", "gemini-2.5-flash"]
+ )
+ assert result.exit_code == 0, result.output
+ assert "Pass --max-seq-length with the model's context window" in result.output
+
+
[email protected]("agent", ["pi", "hermes", "openclaw"])
+def test_agents_without_connection_support_fail_before_loading(fake_studio, agent):
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ result = CliRunner().invoke(
+ start.start_app, [agent, "--no-launch", "--model", "gemini-2.5-flash"]
+ )
+ assert result.exit_code == 1
+ assert "gemini-2.5-flash is a Connection model" in result.output
+ assert _loads(fake_studio) == []
+
+
+def test_codex_uses_a_connection_model(fake_studio, tmp_path):
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ result = CliRunner().invoke(
+ start.start_app, ["codex", "--no-launch", "--model", "gemini-2.5-flash"]
+ )
+ assert result.exit_code == 0, result.output
+ assert _loads(fake_studio) == []
+ profile = (tmp_path / "agents" / "codex" / "unsloth_api.config.toml").read_text()
+ assert _parse_toml(profile)["model"] == GEMINI_ID
+
+
+def test_codex_subagent_runs_on_a_connection_model(fake_studio, tmp_path, monkeypatch):
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ source_home = tmp_path / "user-codex"
+ source_home.mkdir()
+ monkeypatch.setenv("CODEX_HOME", str(source_home))
+ result = CliRunner().invoke(
+ start.start_app, ["codex", "--as-subagent", "--no-launch", "--model", GEMINI_ID]
+ )
+ assert result.exit_code == 0, result.output
+ assert _loads(fake_studio) == []
+ child = tmp_path / "agents" / "codex-subagent" / "child" / "unsloth_api.config.toml"
+ assert _parse_toml(child.read_text())["model"] == GEMINI_ID
+
+
+def test_a_model_two_connections_offer_must_be_named_explicitly(fake_studio):
+ other = {**GEMINI_CONNECTION, "id": "c2", "display_name": "Proxy"}
+ fake_studio.state["providers"] = [GEMINI_CONNECTION, other]
+ result = CliRunner().invoke(
+ start.start_app, ["claude", "--no-launch", "--model", "gemini-2.5-flash"]
+ )
+ assert result.exit_code == 1
+ assert "external::c1::gemini-2.5-flash, external::c2::gemini-2.5-flash" in result.output
+ assert _loads(fake_studio) == []
+
+
+def test_a_loaded_model_wins_over_a_connection_with_the_same_name(fake_studio):
+ fake_studio.state["providers"] = [{**GEMINI_CONNECTION, "models": [MODEL["id"]]}]
+ result = CliRunner().invoke(start.start_app, ["claude", "--no-launch", "--model", MODEL["id"]])
+ assert result.exit_code == 0, result.output
+ _assert_env_set(result.output, "ANTHROPIC_MODEL", MODEL["id"])
+
+
+def test_a_disabled_connection_or_unknown_name_keeps_the_old_load(fake_studio):
+ fake_studio.state["providers"] = [{**GEMINI_CONNECTION, "is_enabled": False}]
+ result = CliRunner().invoke(
+ start.start_app, ["claude", "--no-launch", "--model", "gemini-2.5-flash"]
+ )
+ assert [call[2]["model_path"] for call in _loads(fake_studio)] == ["gemini-2.5-flash"]
+ assert "Connection" not in result.output
+
+
[email protected]("model", ["gpt-5.4", "external::c1::gpt-5.4"])
+def test_a_chatgpt_subscription_connection_is_refused_before_loading(fake_studio, model):
+ fake_studio.state["providers"] = [
+ {**GEMINI_CONNECTION, "models": ["gpt-5.4"], "auth_kind": "chatgpt_oauth"}
+ ]
+ result = CliRunner().invoke(start.start_app, ["codex", "--no-launch", "--model", model])
+ assert result.exit_code == 1
+ assert "ChatGPT subscription Connection" in result.output
+ assert _loads(fake_studio) == []
+
+
[email protected]("agent", ["codex", "opencode"])
+def test_app_mode_refuses_a_connection_model(fake_studio, agent):
+ fake_studio.state["providers"] = [GEMINI_CONNECTION]
+ result = CliRunner().invoke(start.start_app, [agent, "--app", "--model", "gemini-2.5-flash"])
+ assert result.exit_code == 1
+ assert "app can't use a Connection model yet" in result.output
+ assert not [call for call in fake_studio if str((call[2] or {}).get("name")).endswith(" app")]
+
+
+def test_an_explicit_connection_model_never_starts_a_server(fake_studio, monkeypatch):
+ monkeypatch.setattr(start, "find_studio_server", lambda: None)
+ started = []
+ monkeypatch.setattr(start, "_start_studio_server", lambda *args, **kwargs: started.append(args))
+ result = CliRunner().invoke(start.start_app, ["claude", "--model", GEMINI_ID])
+ assert result.exit_code == 1
+ assert "No running Unsloth server found" in result.output
+ assert started == []
+
+
[email protected]("status", [500, 503])
+def test_a_failed_connection_listing_does_not_load_the_name(fake_studio, status):
+ fake_studio.state["providers_status"] = status
+ result = CliRunner().invoke(
+ start.start_app, ["opencode", "--no-launch", "--model", "gemini-2.5-flash"]
+ )
+ assert result.exit_code == 1
+ assert "Couldn't list Connections" in result.output
+ assert _loads(fake_studio) == []
+
+
+def test_a_server_without_connections_still_loads_the_name(fake_studio):
+ fake_studio.state["providers_status"] = 404
+ result = CliRunner().invoke(
+ start.start_app, ["opencode", "--no-launch", "--model", "unsloth/other-GGUF"]
+ )
+ assert result.exit_code == 0, result.output
+ assert len(_loads(fake_studio)) == 1
+
+
+def test_an_explicit_connection_model_names_a_missing_connection(fake_studio):
+ result = CliRunner().invoke(start.start_app, ["claude", "--no-launch", "--model", GEMINI_ID])
+ assert result.exit_code == 1
+ assert "No enabled Connection 'c1'" in result.output
+ assert _loads(fake_studio) == []
+
+
def test_claude_subagent_allowed_tools_precede_forwarded_delimiter(fake_studio):
result = CliRunner().invoke(
start.start_app,