Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Client-side function-calling pass-through for /v1/responses; no running server or GPU is needed."""
import os
import sys
import asyncio
from types import SimpleNamespace
def _shared_setup_1(monitor):
monitor_id = monitor.start(
endpoint = "/v1/responses",
method = "POST",
model = "m",
prompt = "hi",
)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
return messages, monitor_id, payload
def _shared_setup_3(api_monitor, tool):
payload = ResponsesRequest(input = "hi", stream = True, tools = [tool])
messages = [ChatMessage(role = "user", content = "hi")]
monitor_id = api_monitor.start(
endpoint = "/v1/responses", method = "POST", model = "org/M-GGUF", prompt = "hi"
)
return messages, monitor_id, payload
def _shared_setup_4():
messages = [ChatMessage(role = "user", content = "hi")]
request = SimpleNamespace(
state = SimpleNamespace(),
url = SimpleNamespace(path = "/v1/responses"),
method = "POST",
)
return messages, request
def _shared_setup_5(payload):
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = False)
assert _extract_response_format(chat_req) is None
def _shared_setup_6():
from core.inference.api_monitor import api_monitor
xml = TestResponsesStreamHealing._XML
tool = TestResponsesStreamHealing._TOOL
return api_monitor, tool, xml
_backend = os.path.join(os.path.dirname(__file__), "..")
sys.path.insert(0, _backend)
import json
import httpx
import pytest
from fastapi import HTTPException
from fastapi.responses import JSONResponse
from pydantic import ValidationError
from core.inference.api_monitor import ApiMonitor
from core.inference.llama_admission import (
ADMISSION_QUEUE_TIMEOUT_ENV,
LlamaAdmissionConfig,
get_llama_admission_queue,
)
from models.inference import (
ChatMessage,
ResponsesCustomToolCallInputItem,
ResponsesCustomToolCallOutputInputItem,
ResponsesFunctionCallInputItem,
ResponsesFunctionCallOutputInputItem,
ResponsesFunctionTool,
ResponsesInputMessage,
ResponsesOutputFunctionCall,
ResponsesOutputCustomToolCall,
ResponsesOutputMessage,
ResponsesOutputReasoning,
ResponsesOutputTextContent,
ResponsesOutputTextPart,
ResponsesRequest,
ResponsesResponse,
ResponsesUnknownContentPart,
ResponsesUnknownInputItem,
ResponsesUsage,
)
from routes.inference import (
_ResponsesReasoningExtractor,
_SameTaskStreamingResponse,
_build_chat_request,
_build_openai_passthrough_body,
_chat_tool_calls_to_responses_output,
_extract_response_format,
_extract_responses_reasoning,
_normalise_responses_input,
_responses_custom_tool_input,
_responses_tool_output_content,
_responses_non_streaming,
_responses_stream,
_translate_responses_tool_choice_to_chat,
_translate_responses_tools_to_chat,
)
def _codex_apply_patch_tool():
return {
"type": "custom",
"name": "apply_patch",
"format": {
"type": "grammar",
"syntax": "lark",
"definition": 'start: "*** Begin Patch" LF hunk+ "*** End Patch" LF?',
},
}
def _codex_namespace_tools():
forecast_params = {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
}
return [
{"type": "function", "name": "exec_command", "parameters": {"type": "object"}},
{
"type": "namespace",
"name": "multi_agent_v1",
"description": "Tools for spawning and managing sub-agents.",
"tools": [
{"type": "function", "name": "spawn_agent", "parameters": {"type": "object"}},
{"type": "function", "name": "wait_agent", "parameters": {"type": "object"}},
],
},
{
"type": "namespace",
"name": "mcp__weather",
"description": "Tools in the mcp__weather namespace.",
"tools": [
{
"type": "function",
"name": "get_forecast",
"description": "Get the weather forecast for a city.",
"strict": False,
"parameters": forecast_params,
}
],
},
{"type": "web_search"},
]
def _namespaced_forecast_call_chunk():
return {
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"id": "call_wx",
"type": "function",
"function": {
"name": "mcp__weather__get_forecast",
"arguments": '{"city":"Paris"}',
},
}
]
}
}
]
}
# =====================================================================
# Request model — tools / tool_choice / parallel_tool_calls
# =====================================================================
class TestResponsesRequestTools:
def test_flat_function_tool_accepted(self):
req = ResponsesRequest(
input = "hi",
tools = [
{
"type": "function",
"name": "get_weather",
"description": "Get the weather for a city.",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
"strict": True,
}
],
)
assert req.tools is not None
assert req.tools[0]["name"] == "get_weather"
assert req.tools[0]["type"] == "function"
assert req.tools[0]["strict"] is True
def test_tool_choice_string_values(self):
for choice in ("auto", "required", "none"):
req = ResponsesRequest(input = "hi", tool_choice = choice)
assert req.tool_choice == choice
def test_tool_choice_forcing_object(self):
req = ResponsesRequest(
input = "hi",
tool_choice = {"type": "function", "name": "get_weather"},
)
assert req.tool_choice == {"type": "function", "name": "get_weather"}
def test_parallel_tool_calls(self):
req = ResponsesRequest(input = "hi", parallel_tool_calls = True)
assert req.parallel_tool_calls is True
def test_builtin_tool_type_passes_validation(self):
"""Built-in tools like web_search must not fail validation; translation filters them out."""
req = ResponsesRequest(
input = "hi",
tools = [{"type": "web_search_preview"}],
)
assert req.tools == [{"type": "web_search_preview"}]
def test_function_tool_model_direct(self):
tool = ResponsesFunctionTool(
type = "function",
name = "send_email",
parameters = {"type": "object", "properties": {}},
)
assert tool.name == "send_email"
assert tool.description is None
def test_function_tool_rejects_other_type(self):
with pytest.raises(ValidationError):
ResponsesFunctionTool(type = "web_search", name = "x")
class TestResponsesMultiTurnInput:
def test_function_call_input_item(self):
req = ResponsesRequest(
input = [
{"role": "user", "content": "Weather in Paris?"},
{
"type": "function_call",
"id": "fc_abc",
"call_id": "call_abc",
"name": "get_weather",
"arguments": '{"city": "Paris"}',
},
{
"type": "function_call_output",
"call_id": "call_abc",
"output": '{"temp": 12}',
},
],
)
assert len(req.input) == 3
assert isinstance(req.input[1], ResponsesFunctionCallInputItem)
assert req.input[1].call_id == "call_abc"
assert isinstance(req.input[2], ResponsesFunctionCallOutputInputItem)
assert req.input[2].call_id == "call_abc"
assert req.input[2].output == '{"temp": 12}'
def test_function_call_output_missing_call_id_rejected(self):
with pytest.raises(ValidationError):
ResponsesFunctionCallOutputInputItem(type = "function_call_output", output = "x")
def test_function_call_output_accepts_content_array(self):
item = ResponsesFunctionCallOutputInputItem(
type = "function_call_output",
call_id = "call_1",
output = [{"type": "output_text", "text": "done"}],
)
assert isinstance(item.output, list)
def test_custom_tool_call_items(self):
req = ResponsesRequest(
input = [
{
"type": "custom_tool_call",
"call_id": "call_patch",
"name": "apply_patch",
"input": "*** Begin Patch\n*** End Patch",
},
{
"type": "custom_tool_call_output",
"call_id": "call_patch",
"output": "Done!",
},
]
)
assert isinstance(req.input[0], ResponsesCustomToolCallInputItem)
assert isinstance(req.input[1], ResponsesCustomToolCallOutputInputItem)
class TestToolsTranslation:
def test_flat_to_nested(self):
tools = [
{
"type": "function",
"name": "get_weather",
"description": "Returns weather.",
"parameters": {"type": "object"},
"strict": True,
}
]
out = _translate_responses_tools_to_chat(tools)
assert out == [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Returns weather.",
"parameters": {"type": "object"},
"strict": True,
},
}
]
def test_builtin_tools_dropped(self):
out = _translate_responses_tools_to_chat(
[
{"type": "web_search_preview"},
{"type": "file_search"},
{
"type": "function",
"name": "search",
"parameters": {"type": "object"},
},
]
)
assert len(out) == 1
assert out[0]["function"]["name"] == "search"
def test_codex_namespace_tools_reach_the_model(self):
out = _translate_responses_tools_to_chat(_codex_namespace_tools())
assert [tool["function"]["name"] for tool in out] == [
"exec_command",
"multi_agent_v1__spawn_agent",
"multi_agent_v1__wait_agent",
"mcp__weather__get_forecast",
]
assert out[3]["function"] == {
"name": "mcp__weather__get_forecast",
"description": "Get the weather forecast for a city.",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
"strict": False,
}
def test_codex_apply_patch_custom_tool_becomes_a_local_function(self):
grammar = (
'start: "*** Begin Patch" LF add_hunk "*** End Patch" LF?\n'
'add_hunk: "*** Add File: " filename LF add_line+\n'
'add_line: "+" /(.*)/ LF'
)
out = _translate_responses_tools_to_chat(
[
{
"type": "custom",
"name": "apply_patch",
"description": "freeform",
"format": {"type": "grammar", "syntax": "lark", "definition": grammar},
}
]
)
assert out == [
{
"type": "function",
"function": {
"name": "apply_patch",
"description": (
"Edit files by passing one complete patch in the input field. "
"Every added content line must start with +. Example:\n"
"*** Begin Patch\n"
"*** Add File: path/to/file.txt\n"
"+first line\n"
"*** End Patch\n\n"
"The input must match this Lark grammar:\n" + grammar
),
"parameters": {
"type": "object",
"properties": {
"input": {
"type": "string",
"description": "A complete patch matching the tool grammar.",
}
},
"required": ["input"],
"additionalProperties": False,
},
"strict": False,
},
}
]
def test_unrelated_custom_tools_stay_unsupported(self):
assert (
_translate_responses_tools_to_chat(
[{"type": "custom", "name": "code_exec", "description": "raw input"}]
)
is None
)
def test_apply_patch_without_the_codex_grammar_stays_unsupported(self):
assert (
_translate_responses_tools_to_chat([{"type": "custom", "name": "apply_patch"}]) is None
)
def test_empty_returns_none(self):
assert _translate_responses_tools_to_chat(None) is None
assert _translate_responses_tools_to_chat([]) is None
def test_only_builtin_tools_returns_none(self):
assert _translate_responses_tools_to_chat([{"type": "web_search_preview"}]) is None
def test_description_optional(self):
out = _translate_responses_tools_to_chat(
[
{
"type": "function",
"name": "noop",
"parameters": {"type": "object"},
}
]
)
assert "description" not in out[0]["function"]
class TestToolChoiceTranslation:
def test_string_passthrough(self):
for v in ("auto", "required", "none"):
assert _translate_responses_tool_choice_to_chat(v) == v
def test_none_passthrough(self):
assert _translate_responses_tool_choice_to_chat(None) is None
def test_forcing_object_converted(self):
assert _translate_responses_tool_choice_to_chat(
{"type": "function", "name": "get_weather"}
) == {"type": "function", "function": {"name": "get_weather"}}
def test_forced_apply_patch_custom_tool_converted(self):
assert _translate_responses_tool_choice_to_chat(
{"type": "custom", "name": "apply_patch"}, {"apply_patch"}
) == {"type": "function", "function": {"name": "apply_patch"}}
def test_forced_apply_patch_without_valid_catalog_passes_through(self):
choice = {"type": "custom", "name": "apply_patch"}
assert _translate_responses_tool_choice_to_chat(choice, set()) is choice
def test_unrelated_custom_tool_choice_passes_through(self):
choice = {"type": "custom", "name": "code_exec"}
assert _translate_responses_tool_choice_to_chat(choice) is choice
def test_already_chat_nested_shape_passes_through(self):
"""A client sending the Chat Completions nested shape must not be double-wrapped."""
already_nested = {"type": "function", "function": {"name": "get_weather"}}
assert _translate_responses_tool_choice_to_chat(already_nested) == already_nested
def test_unknown_shape_passes_through(self):
obj = {"type": "allowed_tools", "tools": [{"type": "function", "name": "x"}]}
assert _translate_responses_tool_choice_to_chat(obj) == obj
class TestBuildChatRequest:
def test_seed_reaches_the_llama_passthrough_policy(self):
payload = ResponsesRequest(input = "hi", seed = 3407)
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = True)
body = _build_openai_passthrough_body(chat_req, backend_ctx = 4096)
assert chat_req.seed == 3407
assert body["seed"] == 3407
assert body["cache_prompt"] is False
def test_parallel_tool_calls_false_is_preserved_for_passthrough_caps(self):
payload = ResponsesRequest(
input = "hi",
tools = [
{
"type": "function",
"name": "lookup",
"parameters": {"type": "object"},
}
],
parallel_tool_calls = False,
)
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = True)
assert chat_req.parallel_tool_calls is False
def test_text_format_json_schema_becomes_response_format(self):
schema = {"type": "object", "properties": {"name": {"type": "string"}}}
payload = ResponsesRequest(
input = "hi",
text = {
"format": {
"type": "json_schema",
"name": "Person",
"schema": schema,
"strict": True,
}
},
)
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = False)
assert _extract_response_format(chat_req) == {
"type": "json_schema",
"json_schema": {"name": "Person", "schema": schema, "strict": True},
}
def test_text_format_does_not_lock_out_callable_tools(self):
payload = ResponsesRequest(
input = "Weather in Paris?",
tools = [
{
"type": "function",
"name": "get_weather",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
}
],
text = {
"format": {
"type": "json_schema",
"name": "final_output",
"schema": {"type": "object", "properties": {"temp_c": {"type": "number"}}},
"strict": True,
}
},
)
messages = [ChatMessage(role = "user", content = "Weather in Paris?")]
chat_req = _build_chat_request(payload, messages, stream = True)
body = _build_openai_passthrough_body(chat_req, backend_ctx = 4096)
assert [tool["function"]["name"] for tool in body["tools"]] == ["get_weather"]
assert "response_format" not in body
def test_text_format_json_object_becomes_response_format(self):
payload = ResponsesRequest(input = "hi", text = {"format": {"type": "json_object"}})
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = False)
assert _extract_response_format(chat_req) == {"type": "json_object"}
def test_text_format_text_carries_no_response_format(self):
# Codex and Agents SDK send this on ordinary calls; it must not force the schema path.
payload = ResponsesRequest(input = "hi", text = {"format": {"type": "text"}})
_shared_setup_5(payload)
def test_text_verbosity_only_carries_no_response_format(self):
payload = ResponsesRequest(input = "hi", text = {"verbosity": "low"})
_shared_setup_5(payload)
def test_text_format_json_schema_without_schema_is_ignored(self):
payload = ResponsesRequest(input = "hi", text = {"format": {"type": "json_schema"}})
_shared_setup_5(payload)
def test_chat_template_kwargs_enable_thinking_true_is_lifted(self):
payload = ResponsesRequest(
input = "hi",
chat_template_kwargs = {"enable_thinking": True},
)
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = False)
assert chat_req.enable_thinking is True
def test_chat_template_kwargs_enable_thinking_false_is_lifted(self):
payload = ResponsesRequest(
input = "hi",
chat_template_kwargs = {"enable_thinking": False},
)
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = False)
assert chat_req.enable_thinking is False
def test_chat_template_kwargs_enable_thinking_requires_json_boolean(self):
payload = ResponsesRequest(
input = "hi",
chat_template_kwargs = {"enable_thinking": "false"},
)
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = False)
assert chat_req.enable_thinking is None
def test_reasoning_effort_high_enables_local_thinking(self):
payload = ResponsesRequest(input = "hi", reasoning = {"effort": "high"})
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = False)
assert chat_req.reasoning_effort == "high"
assert chat_req.enable_thinking is True
def test_reasoning_effort_none_disables_local_thinking(self):
payload = ResponsesRequest(input = "hi", reasoning = {"effort": "none"})
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = False)
assert chat_req.reasoning_effort == "none"
assert chat_req.enable_thinking is False
def test_explicit_enable_thinking_false_disables_reasoning_effort(self):
payload = ResponsesRequest(
input = "hi",
reasoning = {"effort": "high"},
chat_template_kwargs = {"enable_thinking": False},
)
messages = [ChatMessage(role = "user", content = "hi")]
chat_req = _build_chat_request(payload, messages, stream = False)
assert chat_req.reasoning_effort == "none"
assert chat_req.enable_thinking is False
class TestNormaliseResponsesInputWithTools:
def test_function_call_output_maps_to_tool_role(self):
payload = ResponsesRequest(
input = [
{"role": "user", "content": "Weather?"},
{
"type": "function_call",
"call_id": "call_1",
"name": "get_weather",
"arguments": "{}",
},
{
"type": "function_call_output",
"call_id": "call_1",
"output": '{"temp": 20}',
},
],
)
msgs = _normalise_responses_input(payload)
assert len(msgs) == 3
assert msgs[0].role == "user"
assert msgs[1].role == "assistant"
assert msgs[1].tool_calls is not None
assert msgs[1].tool_calls[0]["id"] == "call_1"
assert msgs[1].tool_calls[0]["function"]["name"] == "get_weather"
assert msgs[2].role == "tool"
assert msgs[2].tool_call_id == "call_1"
assert msgs[2].content == '{"temp": 20}'
def test_replayed_namespaced_call_keeps_the_name_the_model_called(self):
payload = ResponsesRequest(
input = [
{"role": "user", "content": "Weather in Paris?"},
{
"type": "function_call",
"call_id": "call_wx",
"namespace": "mcp__weather",
"name": "get_forecast",
"arguments": '{"city":"Paris"}',
},
{"type": "function_call_output", "call_id": "call_wx", "output": "17C"},
],
tools = _codex_namespace_tools(),
)
msgs = _normalise_responses_input(payload)
assert msgs[1].tool_calls[0]["function"]["name"] == "mcp__weather__get_forecast"
assert msgs[2].tool_call_id == "call_wx"
def test_custom_tool_call_round_trip_maps_to_local_function(self):
patch = "*** Begin Patch\n*** Add File: café.txt\n+héllo\n*** End Patch"
payload = ResponsesRequest(
input = [
{
"type": "custom_tool_call",
"call_id": "call_patch",
"name": "apply_patch",
"input": patch,
},
{
"type": "custom_tool_call_output",
"call_id": "call_patch",
"output": "Done!",
},
],
tools = [_codex_apply_patch_tool()],
)
messages = _normalise_responses_input(payload)
assert len(messages) == 2
assert messages[0].role == "assistant"
call = messages[0].tool_calls[0]
assert call["id"] == "call_patch"
assert call["function"]["name"] == "apply_patch"
assert json.loads(call["function"]["arguments"]) == {"input": patch}
assert messages[1].role == "tool"
assert messages[1].tool_call_id == "call_patch"
assert messages[1].content == "Done!"
def test_unrelated_custom_tool_replay_stays_ignored(self):
payload = ResponsesRequest(
input = [
{
"type": "custom_tool_call",
"call_id": "call_code",
"name": "code_exec",
"input": "print('hidden')",
},
{
"type": "custom_tool_call_output",
"call_id": "call_code",
"output": "hidden output",
},
],
tools = [
{
"type": "custom",
"name": "code_exec",
"format": {
"type": "grammar",
"syntax": "lark",
"definition": "start: /.+/",
},
}
],
)
assert _normalise_responses_input(payload) == []
def test_instructions_plus_developer_message_are_merged(self):
"""Strict templates reject a second system message; instructions and developer merge."""
payload = ResponsesRequest(
instructions = "Base instructions.",
input = [
{"role": "developer", "content": "Developer override."},
{"role": "user", "content": "Hi"},
],
)
msgs = _normalise_responses_input(payload)
system_roles = [m for m in msgs if m.role == "system"]
assert len(system_roles) == 1
assert "Base instructions." in system_roles[0].content
assert "Developer override." in system_roles[0].content
# System must be the first message for strict templates.
assert msgs[0].role == "system"
assert msgs[1].role == "user"
def test_developer_message_after_user_is_still_hoisted(self):
"""Late developer messages are hoisted into one leading system message; strict templates
reject two."""
payload = ResponsesRequest(
input = [
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi!"},
{"role": "developer", "content": "Updated rules."},
{"role": "user", "content": "Continue"},
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].role == "system"
assert "Updated rules." in msgs[0].content
for m in msgs[1:]:
assert m.role != "system", "no trailing system message permitted"
def test_no_system_output_when_no_system_input(self):
payload = ResponsesRequest(input = "Hi")
msgs = _normalise_responses_input(payload)
assert all(m.role != "system" for m in msgs)
def test_multiple_system_messages_in_input_are_merged(self):
payload = ResponsesRequest(
input = [
{"role": "system", "content": "A"},
{"role": "system", "content": "B"},
{"role": "user", "content": "Hi"},
],
)
msgs = _normalise_responses_input(payload)
assert sum(1 for m in msgs if m.role == "system") == 1
assert "A" in msgs[0].content and "B" in msgs[0].content
def test_content_array_text_output_flattens_to_tool_text(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [{"type": "input_text", "text": "ok"}],
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].role == "tool"
assert msgs[0].content == "ok"
def test_content_array_image_output_becomes_multimodal_tool_content(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{"type": "input_text", "text": "see image"},
{
"type": "input_image",
"image_url": "data:image/png;base64,AAA",
"detail": "high",
},
],
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].role == "tool"
assert msgs[0].tool_call_id == "call_1"
assert msgs[0].model_dump(exclude_none = True)["content"] == [
{"type": "text", "text": "see image"},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,AAA",
"detail": "high",
},
},
]
chat_req = _build_chat_request(payload, msgs, stream = False)
assert chat_req.model_dump(exclude_none = True)["messages"][0]["content"] == [
{"type": "text", "text": "see image"},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,AAA",
"detail": "high",
},
},
]
def test_content_array_image_output_allows_original_detail(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{
"type": "input_image",
"image_url": "https://example.com/screenshot.png",
"detail": "original",
},
],
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].model_dump(exclude_none = True)["content"] == [
{
"type": "image_url",
"image_url": {
"url": "https://example.com/screenshot.png",
"detail": "original",
},
},
]
def test_content_array_file_id_image_output_rejected_clearly(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{"type": "input_text", "text": "see image"},
{"type": "input_image", "file_id": "file_abc"},
],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "file_id" in str(exc.value.detail)
def test_content_array_file_output_rejected_clearly(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{"type": "input_text", "text": "see file"},
{
"type": "input_file",
"file_data": "data:application/pdf;base64,AAA",
"filename": "report.pdf",
},
],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "input_file" in str(exc.value.detail)
def test_content_array_malformed_image_output_rejected_clearly(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [{"type": "input_image", "detail": "high"}],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "image_url" in str(exc.value.detail)
def test_input_file_message_part_rejected_clearly(self):
payload = ResponsesRequest(
input = [
{
"role": "user",
"content": [
{"type": "input_text", "text": "summarise this"},
{
"type": "input_file",
"file_data": "data:application/pdf;base64,AAA",
"filename": "report.pdf",
},
],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "input_file" in str(exc.value.detail)
def test_file_id_image_message_part_rejected_clearly(self):
payload = ResponsesRequest(
input = [
{
"role": "user",
"content": [
{"type": "input_text", "text": "what is in this?"},
{"type": "input_image", "file_id": "file_abc"},
],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "file_id" in str(exc.value.detail)
def test_unmodelled_message_part_is_named_not_dropped(self):
payload = ResponsesRequest(
input = [
{
"role": "user",
"content": [
{"type": "input_text", "text": "hello"},
{"type": "input_something_new", "value": 1},
],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "input_something_new" in str(exc.value.detail)
def test_image_message_part_without_any_source_rejected_clearly(self):
payload = ResponsesRequest(
input = [
{
"role": "user",
"content": [
{"type": "input_text", "text": "what is in this?"},
{"type": "input_image"},
],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "require an image_url string" in str(exc.value.detail)
def test_image_message_part_with_unknown_detail_rejected_clearly(self):
payload = ResponsesRequest(
input = [
{
"role": "user",
"content": [
{"type": "input_text", "text": "what is in this?"},
{
"type": "input_image",
"image_url": "https://example.com/a.png",
"detail": "medium",
},
],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "auto, low, high, or original" in str(exc.value.detail)
def test_image_message_part_with_url_and_file_id_is_served_from_the_url(self):
payload = ResponsesRequest(
input = [
{
"role": "user",
"content": [
{"type": "input_text", "text": "what is in this?"},
{
"type": "input_image",
"image_url": "https://example.com/a.png",
"file_id": "file_abc",
},
],
}
],
)
msgs = _normalise_responses_input(payload)
assert len(msgs) == 1
assert msgs[0].content[1].image_url.url == "https://example.com/a.png"
@pytest.mark.parametrize("role", ["system", "developer", "assistant"])
def test_attachment_refused_on_the_roles_that_exit_early(self, role):
text_part = (
{"type": "output_text", "text": "hi"}
if role == "assistant"
else {"type": "input_text", "text": "hi"}
)
payload = ResponsesRequest(
input = [
{
"role": role,
"content": [text_part, {"type": "input_file", "filename": "r.pdf"}],
},
{"role": "user", "content": "and now?"},
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "input_file" in str(exc.value.detail)
def test_assistant_replay_keeps_its_lenient_text_flatten(self):
payload = ResponsesRequest(
input = [
{
"role": "assistant",
"content": [
{"type": "output_text", "text": "earlier answer"},
{"type": "refusal", "refusal": "I cannot help with that"},
],
},
{"role": "user", "content": "why not?"},
],
)
msgs = _normalise_responses_input(payload)
assert [(m.role, m.content) for m in msgs] == [
("assistant", "earlier answer"),
("user", "why not?"),
]
def test_string_content_never_hides_an_attachment(self):
payload = ResponsesRequest(
input = [
{"role": "system", "content": "be brief"},
{"role": "assistant", "content": "earlier"},
{"role": "user", "content": "hello"},
],
)
msgs = _normalise_responses_input(payload)
assert [(m.role, m.content) for m in msgs] == [
("system", "be brief"),
("assistant", "earlier"),
("user", "hello"),
]
def test_refusal_body_is_the_openai_unsupported_parameter_shape(self):
payload = ResponsesRequest(
input = [{"role": "user", "content": [{"type": "input_file", "filename": "r.pdf"}]}],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
error = exc.value.detail["error"]
assert error["code"] == "unsupported_parameter"
assert error["param"] == "input"
assert error["type"] == "invalid_request_error"
_IMG = "https://example.com/a.png"
_RESPONSES_PART_MATRIX = [
({"type": "input_text", "text": "hi"}, False, ""),
({"type": "output_text", "text": "hi"}, False, ""),
({"type": "input_image", "image_url": _IMG}, False, ""),
({"type": "input_image", "image_url": _IMG, "detail": "auto"}, False, ""),
({"type": "input_image", "image_url": _IMG, "detail": "low"}, False, ""),
({"type": "input_image", "image_url": _IMG, "detail": "high"}, False, ""),
({"type": "input_image", "image_url": _IMG, "detail": "original"}, False, ""),
({"type": "input_image", "image_url": _IMG, "detail": None}, False, ""),
({"type": "input_image", "image_url": _IMG, "file_id": "f1"}, False, ""),
({"type": "input_image", "image_url": "data:image/png;base64,AAAA"}, False, ""),
({"type": "input_image", "file_id": "f1"}, True, "file_id"),
({"type": "input_image"}, True, "require an image_url string"),
({"type": "input_image", "image_url": None}, True, "require an image_url string"),
({"type": "input_image", "image_url": ""}, True, "require an image_url string"),
({"type": "input_image", "image_url": 17}, True, "require an image_url string"),
({"type": "input_image", "image_url": {"url": _IMG}}, True, "require an image_url string"),
({"type": "input_image", "image_url": _IMG, "detail": "medium"}, True, "auto, low, high"),
({"type": "input_image", "image_url": _IMG, "detail": 4}, True, "auto, low, high"),
({"type": "input_file", "file_id": "f1"}, True, "input_file"),
({"type": "input_file", "file_data": "data:application/pdf;base64,AA"}, True, "input_file"),
({"type": "input_file", "file_url": "https://example.com/d.pdf"}, True, "input_file"),
({"type": "input_file", "filename": "r.pdf"}, True, "input_file"),
({"type": "input_file"}, True, "input_file"),
({"type": "input_audio", "input_audio": {"data": "AA", "format": "wav"}}, True, "input_audio"),
({"type": "input_something_new", "value": 1}, True, "input_something_new"),
]
def _matrix_id(case):
part, refused, _ = case
keys = "+".join(k for k in part if k != "type")
return f"{part['type']}({keys or 'bare'})-{'400' if refused else '200'}"
class TestResponsesMessagePartMatrix:
"""One expected answer per (part shape, role, position). No server, no GPU."""
@pytest.mark.parametrize("case", _RESPONSES_PART_MATRIX, ids = _matrix_id)
@pytest.mark.parametrize("alone", [True, False], ids = ["alone", "with_text"])
def test_user_turn(self, case, alone):
part, refused, needle = case
content = [part] if alone else [{"type": "input_text", "text": "hi"}, part]
payload = ResponsesRequest(input = [{"role": "user", "content": content}])
if refused:
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert needle in str(exc.value.detail)
else:
assert _normalise_responses_input(payload)
@pytest.mark.parametrize("case", _RESPONSES_PART_MATRIX, ids = _matrix_id)
@pytest.mark.parametrize("role", ["system", "developer", "assistant"])
def test_attachments_refused_on_every_role(self, case, role):
part, refused, needle = case
payload = ResponsesRequest(
input = [
{"role": role, "content": [{"type": "input_text", "text": "hi"}, part]},
{"role": "user", "content": "go on"},
],
)
survives = part["type"] in ("input_text", "output_text") or (
role == "assistant" and part["type"] in ("refusal", "summary_text")
)
if survives:
assert _normalise_responses_input(payload)
return
if part["type"] == "input_file":
expected = needle
elif part["type"] == "input_image":
expected = needle if refused else "only supported on user messages"
else:
expected = part["type"]
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert expected in str(exc.value.detail)
@pytest.mark.parametrize("role", ["system", "developer", "assistant"])
def test_a_servable_image_is_refused_on_a_role_that_flattens(self, role):
text_part = (
{"type": "output_text", "text": "hi"}
if role == "assistant"
else {"type": "input_text", "text": "hi"}
)
payload = ResponsesRequest(
input = [
{
"role": role,
"content": [
text_part,
{"type": "input_image", "image_url": "https://example.com/a.png"},
],
},
{"role": "user", "content": "go on"},
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "only supported on user messages" in str(exc.value.detail)
assert role in str(exc.value.detail)
@pytest.mark.parametrize("role", ["system", "developer", "assistant"])
@pytest.mark.parametrize(
"part",
[
{"type": "input_audio", "input_audio": {"data": "AA", "format": "wav"}},
{"type": "computer_screenshot", "image_url": "https://example.com/a.png"},
{"type": "input_brand_new", "value": 1},
],
ids = ["input_audio", "computer_screenshot", "future"],
)
def test_a_non_text_part_is_refused_on_a_role_that_flattens(self, role, part):
text_part = (
{"type": "output_text", "text": "hi"}
if role == "assistant"
else {"type": "input_text", "text": "hi"}
)
payload = ResponsesRequest(
input = [
{"role": role, "content": [text_part, part]},
{"role": "user", "content": "go on"},
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert part["type"] in str(exc.value.detail)
assert role in str(exc.value.detail)
@pytest.mark.parametrize("role", ["system", "developer", "user", "assistant"])
@pytest.mark.parametrize(
"part",
[
{"type": "input_text"},
{"type": "input_text", "text": None},
{"type": "input_text", "text": 123},
{"type": "output_text"},
],
ids = ["no_text", "null_text", "int_text", "output_no_text"],
)
def test_a_text_part_without_text_is_named_for_what_is_wrong(self, role, part):
payload = ResponsesRequest(
input = [
{"role": role, "content": [part]},
{"role": "user", "content": "go on"},
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "require a text field" in str(exc.value.detail)
assert part["type"] in str(exc.value.detail)
def test_a_malformed_text_part_does_not_hide_behind_a_good_one(self):
payload = ResponsesRequest(
input = [
{
"role": "system",
"content": [{"type": "input_text", "text": "keep"}, {"type": "input_text"}],
},
{"role": "user", "content": "go on"},
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert "require a text field" in str(exc.value.detail)
@pytest.mark.parametrize("part_type", ["refusal", "summary_text"])
@pytest.mark.parametrize("role", ["system", "developer"])
def test_output_metadata_is_caller_content_on_a_non_assistant_turn(self, role, part_type):
payload = ResponsesRequest(
input = [
{
"role": role,
"content": [
{"type": "input_text", "text": "keep"},
{"type": part_type, "text": "x", "refusal": "x"},
],
},
{"role": "user", "content": "go on"},
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert part_type in str(exc.value.detail)
@pytest.mark.parametrize("part_type", ["refusal", "summary_text"])
def test_assistant_output_metadata_survives_the_flatten(self, part_type):
payload = ResponsesRequest(
input = [
{
"role": "assistant",
"content": [
{"type": "output_text", "text": "earlier answer"},
{"type": part_type, "text": "x", "refusal": "x"},
],
},
{"role": "user", "content": "why?"},
],
)
msgs = _normalise_responses_input(payload)
assert [(m.role, m.content) for m in msgs] == [
("assistant", "earlier answer"),
("user", "why?"),
]
def test_a_servable_image_still_passes_on_a_user_turn(self):
payload = ResponsesRequest(
input = [
{
"role": "user",
"content": [
{"type": "input_text", "text": "hi"},
{"type": "input_image", "image_url": "https://example.com/a.png"},
],
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].content[1].image_url.url == "https://example.com/a.png"
def test_instructions_still_merge_when_a_turn_is_servable(self):
payload = ResponsesRequest(
instructions = "be brief",
input = [
{"role": "developer", "content": [{"type": "input_text", "text": "and kind"}]},
{"role": "user", "content": [{"type": "input_text", "text": "hello"}]},
],
)
msgs = _normalise_responses_input(payload)
assert [(m.role, m.content) for m in msgs] == [
("system", "be brief\n\nand kind"),
("user", "hello"),
]
def test_top_level_item_types_are_untouched_by_the_refusal(self):
payload = ResponsesRequest(
input = [
{"type": "reasoning", "summary": [], "id": "rs_1"},
{
"type": "function_call",
"call_id": "call_1",
"name": "get_weather",
"arguments": "{}",
},
{"type": "function_call_output", "call_id": "call_1", "output": "sunny"},
{"role": "user", "content": "thanks"},
],
)
msgs = _normalise_responses_input(payload)
assert [m.role for m in msgs] == ["assistant", "tool", "user"]
def test_empty_function_call_output_gets_no_output_sentinel(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": "",
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].role == "tool"
assert msgs[0].tool_call_id == "call_1"
assert msgs[0].content == "(no output)"
ChatMessage(**msgs[0].model_dump(exclude_none = True))
def test_whitespace_function_call_output_gets_no_output_sentinel(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": " \n\t",
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].content == "(no output)"
def test_empty_content_array_output_gets_no_output_sentinel(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [],
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].content == "(no output)"
def test_image_content_array_tool_output_is_serialised(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/png",
"data": "iVBORw0KGgo=",
},
}
],
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].role == "tool"
assert json.loads(msgs[0].content)[0]["type"] == "image"
def test_image_payload_outside_output_gets_no_output_sentinel(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": "",
"content": [
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/png",
"data": "iVBORw0KGgo=",
},
}
],
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].role == "tool"
assert msgs[0].tool_call_id == "call_1"
assert msgs[0].content == "(no output)"
def test_tool_output_serializer_preserves_non_empty_text(self):
assert _responses_tool_output_content("done") == "done"
assert _responses_tool_output_content(" done ") == " done "
class TestChatToolCallsToResponsesOutput:
def test_basic_mapping(self):
items = _chat_tool_calls_to_responses_output(
[
{
"id": "call_abc",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city":"Paris"}',
},
}
]
)
assert len(items) == 1
assert items[0]["type"] == "function_call"
assert items[0]["call_id"] == "call_abc"
assert items[0]["name"] == "get_weather"
assert items[0]["arguments"] == '{"city":"Paris"}'
assert items[0]["status"] == "completed"
assert items[0]["id"].startswith("fc_")
def test_multiple_tool_calls_preserved(self):
items = _chat_tool_calls_to_responses_output(
[
{
"id": "call_1",
"type": "function",
"function": {"name": "a", "arguments": "{}"},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "b", "arguments": "{}"},
},
]
)
assert [it["call_id"] for it in items] == ["call_1", "call_2"]
def test_non_function_tool_call_dropped(self):
items = _chat_tool_calls_to_responses_output([{"id": "x", "type": "retrieval"}])
assert items == []
def test_missing_arguments_coerced_to_empty_string(self):
items = _chat_tool_calls_to_responses_output(
[{"id": "call_1", "type": "function", "function": {"name": "x"}}]
)
assert items[0]["arguments"] == ""
def test_apply_patch_function_is_restored_to_a_custom_tool_call(self):
patch = "*** Begin Patch\n*** Add File: café.txt\n+héllo\n*** End Patch"
items = _chat_tool_calls_to_responses_output(
[
{
"id": "call_patch",
"type": "function",
"function": {
"name": "apply_patch",
"arguments": json.dumps({"input": patch}),
},
}
],
{"apply_patch"},
)
assert items[0]["type"] == "custom_tool_call"
assert items[0]["call_id"] == "call_patch"
assert items[0]["name"] == "apply_patch"
assert items[0]["input"] == patch
def test_regular_function_named_apply_patch_stays_a_function_call(self):
items = _chat_tool_calls_to_responses_output(
[
{
"id": "call_patch",
"type": "function",
"function": {"name": "apply_patch", "arguments": '{"path":"x"}'},
}
]
)
assert items[0]["type"] == "function_call"
@pytest.mark.parametrize(
("arguments", "expected"),
[
('{"input":"patch"', '{"input":"patch"'),
('{"other":"value"}', '{"other":"value"}'),
(None, ""),
],
)
def test_malformed_custom_arguments_are_not_hidden(self, arguments, expected):
assert _responses_custom_tool_input(arguments) == expected
class TestResponsesNonStreamingAdapter:
class _Request:
pass
@staticmethod
def _run_with_message(
monkeypatch,
message,
payload = None,
llama_backend = None,
finish_reason = None,
):
import routes.inference as inf_mod
async def fake_chat_completions(
chat_req,
request,
current_subject = None,
):
return JSONResponse(
content = {
"model": "test-model",
"choices": [{"message": message, "finish_reason": finish_reason}],
"usage": {"prompt_tokens": 2, "completion_tokens": 3},
}
)
monkeypatch.setattr(inf_mod, "openai_chat_completions", fake_chat_completions)
if llama_backend is not None:
monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: llama_backend)
payload = payload or ResponsesRequest(input = "hi")
messages = [ChatMessage(role = "user", content = "hi")]
async def run():
response = await _responses_non_streaming(
payload, messages, TestResponsesNonStreamingAdapter._Request()
)
return json.loads(response.body.decode())
return asyncio.run(run())
def test_a_truncated_turn_is_reported_as_incomplete(self, monkeypatch):
body = self._run_with_message(
monkeypatch,
{
"content": "half an ans",
"reasoning_content": "partial plan",
"tool_calls": [
{
"id": "call_partial",
"type": "function",
"function": {"name": "lookup", "arguments": '{"q":"x"'},
}
],
},
finish_reason = "length",
)
assert body["status"] == "incomplete"
assert body["incomplete_details"] == {"reason": "max_output_tokens"}
assert [item["status"] for item in body["output"]] == [
"incomplete",
"incomplete",
"incomplete",
]
def test_a_natural_stop_is_still_completed(self, monkeypatch):
body = self._run_with_message(
monkeypatch,
{"content": "a whole answer"},
finish_reason = "stop",
)
assert body["status"] == "completed"
assert body["incomplete_details"] is None
def test_a_content_filtered_turn_is_reported_as_incomplete(self, monkeypatch):
body = self._run_with_message(
monkeypatch,
{"content": "filtered partial output"},
finish_reason = "content_filter",
)
assert body["status"] == "incomplete"
assert body["incomplete_details"] == {"reason": "content_filter"}
assert [item["status"] for item in body["output"]] == ["incomplete"]
def test_think_block_becomes_reasoning_item_before_message(self, monkeypatch):
payload = ResponsesRequest(input = "hi", reasoning = {"effort": "high"})
body = self._run_with_message(
monkeypatch,
{"content": "<think>plan</think>33"},
payload = payload,
)
assert [item["type"] for item in body["output"]] == ["reasoning", "message"]
assert body["output"][0]["content"] == [{"type": "reasoning_text", "text": "plan"}]
assert body["output"][0]["summary"] == []
assert body["output"][1]["content"][0]["text"] == "33"
assert "<think>" not in body["output"][1]["content"][0]["text"]
assert "</think>" not in body["output"][1]["content"][0]["text"]
def test_namespaced_call_returns_to_its_namespace(self, monkeypatch):
payload = ResponsesRequest(input = "Weather in Paris?", tools = _codex_namespace_tools())
body = self._run_with_message(
monkeypatch,
{
"content": None,
"tool_calls": [
_namespaced_forecast_call_chunk()["choices"][0]["delta"]["tool_calls"][0],
{
"id": "call_ls",
"type": "function",
"function": {"name": "exec_command", "arguments": "{}"},
},
],
},
payload = payload,
)
wx, ls = body["output"]
assert (wx["type"], wx["namespace"], wx["name"]) == (
"function_call",
"mcp__weather",
"get_forecast",
)
assert wx["call_id"] == "call_wx"
assert ls["name"] == "exec_command"
assert "namespace" not in ls
def test_apply_patch_function_becomes_a_custom_tool_call(self, monkeypatch):
patch = "*** Begin Patch\n*** Add File: nested/a.txt\n+ok\n*** End Patch"
payload = ResponsesRequest(
input = "edit the file",
tools = [_codex_apply_patch_tool()],
)
body = self._run_with_message(
monkeypatch,
{
"content": None,
"tool_calls": [
{
"id": "call_patch",
"type": "function",
"function": {
"name": "apply_patch",
"arguments": json.dumps({"input": patch}),
},
}
],
},
payload = payload,
)
assert len(body["output"]) == 1
assert body["output"][0]["type"] == "custom_tool_call"
assert body["output"][0]["input"] == patch
def test_unclosed_think_block_extracts_as_reasoning(self):
reasoning, visible = _extract_responses_reasoning(
"<think>partial plan",
parse_think_markers = True,
)
assert reasoning == "partial plan"
assert visible == ""
def test_monitor_records_translated_visible_text(self, monkeypatch):
import routes.inference as inf_mod
import routes.inference as inf_mod
async def fake_chat_completions(
chat_req,
request,
current_subject = None,
):
assert request.state.skip_api_monitor is True
return JSONResponse(
content = {
"model": "test-model",
"choices": [{"message": {"content": "<think>plan</think>answer"}}],
"usage": {"prompt_tokens": 2, "completion_tokens": 3},
}
)
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
monkeypatch.setattr(inf_mod, "openai_chat_completions", fake_chat_completions)
payload = ResponsesRequest(input = "hi", reasoning = {"effort": "high"})
messages, request = _shared_setup_4()
async def run():
response = await _responses_non_streaming(payload, messages, request)
return json.loads(response.body.decode())
body = asyncio.run(run())
assert body["output"][0]["content"] == [{"type": "reasoning_text", "text": "plan"}]
assert body["output"][1]["content"][0]["text"] == "answer"
[entry] = monitor.snapshot()
assert entry["status"] == "completed"
assert entry["reply"] == "answer"
assert entry["prompt_tokens"] == 2
assert entry["completion_tokens"] == 3
assert request.state.skip_api_monitor is False
def test_monitor_records_client_disconnect_as_cancelled(self, monkeypatch):
import routes.inference as inf_mod
async def fake_chat_completions(
chat_req,
request,
current_subject = None,
):
return JSONResponse(
content = {
"model": "test-model",
"choices": [{"message": {"content": "par"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 2, "completion_tokens": 1},
}
)
async def is_disconnected():
return True
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
monkeypatch.setattr(inf_mod, "openai_chat_completions", fake_chat_completions)
messages, request = _shared_setup_4()
request.is_disconnected = is_disconnected
asyncio.run(_responses_non_streaming(ResponsesRequest(input = "hi"), messages, request))
[entry] = monitor.snapshot()
assert entry["status"] == "cancelled"
assert monitor.active_count() == 0
@staticmethod
def _run_in_process_completion(
monkeypatch,
monitor,
timings,
*,
observations = None,
):
"""Drive the wrapper over an in-process chat completion: its own monitor row is
suppressed and a ``ChatCompletion`` has no ``timings`` field to carry them out."""
import routes.inference as inf_mod
from models.inference import (
ChatCompletion,
CompletionChoice,
CompletionMessage,
CompletionUsage,
)
usage = {"prompt_tokens": 11, "completion_tokens": 50, "total_tokens": 61}
async def fake_chat_completions(
chat_req,
request,
current_subject = None,
):
assert request.state.skip_api_monitor is True
if observations is not None:
observations["perf_callback"] = inf_mod._monitor_perf_callback(None, 4096)
inf_mod._monitor_usage(None, usage, 4096, timings = timings)
if observations is not None:
observations["live_entries"] = monitor.snapshot()
return inf_mod._model_json_response(
ChatCompletion(
model = "test-model",
choices = [
CompletionChoice(
index = 0,
message = CompletionMessage(content = "answer"),
finish_reason = "stop",
)
],
usage = CompletionUsage(**usage),
)
)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
monkeypatch.setattr(inf_mod, "openai_chat_completions", fake_chat_completions)
request = SimpleNamespace(
state = SimpleNamespace(),
url = SimpleNamespace(path = "/v1/responses"),
method = "POST",
)
async def run():
response = await _responses_non_streaming(
ResponsesRequest(input = "hi"),
[ChatMessage(role = "user", content = "hi")],
request,
)
return json.loads(response.body.decode())
return asyncio.run(run())
def test_in_process_engine_timings_reach_the_monitor(self, monkeypatch):
"""A local non-streamed /v1/responses never reached the Throughput tile: the
wrapper read decode_ms off the body, which a ChatCompletion cannot carry."""
from models.inference import ChatCompletion
# Asserted, not assumed: pydantic drops undeclared fields, so a body lookup finds nothing.
assert "timings" not in json.loads(
ChatCompletion(choices = [], timings = {"predicted_ms": 1000.0}).model_dump_json()
)
monitor = ApiMonitor(max_entries = 3)
body = self._run_in_process_completion(
monkeypatch,
monitor,
{"prompt_ms": 9000.0, "predicted_ms": 1000.0, "predicted_per_second": 50.0},
)
assert body["output"][0]["content"][0]["text"] == "answer"
[entry] = monitor.snapshot()
assert entry["completion_tokens"] == 50
assert entry["decode_ms"] == 1000
assert entry["completion_tokens"] / (entry["decode_ms"] / 1000) == 50.0
def test_in_process_engine_timings_update_outer_monitor_live(self, monkeypatch):
monitor = ApiMonitor(max_entries = 3)
observations = {}
self._run_in_process_completion(
monkeypatch,
monitor,
{
"prompt_per_second": 90.0,
"predicted_ms": 1000.0,
"predicted_per_second": 50.0,
},
observations = observations,
)
assert observations["perf_callback"] is not None
[entry] = observations["live_entries"]
assert entry["status"] == "running"
assert entry["prompt_tok_per_sec"] == 90.0
assert entry["tok_per_sec"] == 50.0
assert entry["decode_ms"] == 1000
assert entry["ttft_ms"] is None
def test_disabled_monitor_does_not_install_responses_perf_callback(self, monkeypatch):
monitor = ApiMonitor(max_entries = 3, enabled = False)
observations = {}
self._run_in_process_completion(
monkeypatch,
monitor,
{"prompt_per_second": 90.0, "predicted_per_second": 50.0},
observations = observations,
)
assert observations["perf_callback"] is None
assert observations["live_entries"] == []
assert monitor.snapshot() == []
def test_a_relayed_decode_span_does_not_outlive_its_request(self, monkeypatch):
"""The relay is scoped to one inner call, so a timing-less request inherits nothing."""
monitor = ApiMonitor(max_entries = 3)
self._run_in_process_completion(
monkeypatch,
monitor,
{"predicted_ms": 1000.0},
)
self._run_in_process_completion(monkeypatch, monitor, None)
newest, previous = monitor.snapshot()
assert previous["decode_ms"] == 1000
assert newest["decode_ms"] is None
def test_monitor_records_tool_only_reply(self, monkeypatch):
import routes.inference as inf_mod
async def fake_chat_completions(
chat_req,
request,
current_subject = None,
):
assert request.state.skip_api_monitor is True
return JSONResponse(
content = {
"model": "test-model",
"choices": [
{
"message": {
"content": "",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "lookup",
"arguments": '{"query":"weather"}',
},
}
],
}
}
],
"usage": {"prompt_tokens": 2, "completion_tokens": 3},
}
)
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
monkeypatch.setattr(inf_mod, "openai_chat_completions", fake_chat_completions)
payload = ResponsesRequest(
input = "hi",
tools = [{"type": "function", "name": "lookup"}],
)
messages, request = _shared_setup_4()
async def run():
response = await _responses_non_streaming(payload, messages, request)
return json.loads(response.body.decode())
body = asyncio.run(run())
assert body["output"][0]["type"] == "function_call"
[entry] = monitor.snapshot()
assert entry["status"] == "completed"
assert entry["reply"] == 'Tool call: lookup({"query":"weather"})'
assert request.state.skip_api_monitor is False
def test_cancelled_chat_completion_finalizes_monitor(self, monkeypatch):
import routes.inference as inf_mod
async def fake_chat_completions(
chat_req,
request,
current_subject = None,
):
assert request.state.skip_api_monitor is True
raise asyncio.CancelledError()
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
monkeypatch.setattr(inf_mod, "openai_chat_completions", fake_chat_completions)
payload = ResponsesRequest(input = "hi")
messages, request = _shared_setup_4()
async def run():
with pytest.raises(asyncio.CancelledError):
await _responses_non_streaming(payload, messages, request)
asyncio.run(run())
[entry] = monitor.snapshot()
assert entry["status"] == "cancelled"
assert monitor.active_count() == 0
assert request.state.skip_api_monitor is False
def test_literal_think_tags_remain_visible_without_reasoning_request(self, monkeypatch):
body = self._run_with_message(monkeypatch, {"content": "show <think>x</think> tags"})
assert [item["type"] for item in body["output"]] == ["message"]
assert body["output"][0]["content"][0]["text"] == "show <think>x</think> tags"
def test_non_reasoning_gguf_keeps_literal_think_tags_visible(self, monkeypatch):
payload = ResponsesRequest(input = "hi", reasoning = {"effort": "high"})
body = self._run_with_message(
monkeypatch,
{"content": "show <think>x</think> tags"},
payload = payload,
llama_backend = SimpleNamespace(
is_loaded = True,
reasoning_always_on = False,
supports_reasoning = False,
),
)
assert [item["type"] for item in body["output"]] == ["message"]
assert body["output"][0]["content"][0]["text"] == "show <think>x</think> tags"
def test_reasoning_capable_gguf_parses_think_tags_by_default(self, monkeypatch):
body = self._run_with_message(
monkeypatch,
{"content": "<think>plan</think>answer"},
llama_backend = SimpleNamespace(
is_loaded = True,
reasoning_always_on = False,
supports_reasoning = True,
),
)
assert [item["type"] for item in body["output"]] == ["reasoning", "message"]
assert body["output"][0]["content"] == [{"type": "reasoning_text", "text": "plan"}]
assert body["output"][1]["content"][0]["text"] == "answer"
def test_reasoning_capable_gguf_keeps_think_tags_visible_when_disabled(self, monkeypatch):
payload = ResponsesRequest(input = "hi", reasoning = {"effort": "none"})
body = self._run_with_message(
monkeypatch,
{"content": "Use <think>hi</think> in your prompt."},
payload = payload,
llama_backend = SimpleNamespace(
is_loaded = True,
reasoning_always_on = False,
supports_reasoning = True,
),
)
assert [item["type"] for item in body["output"]] == ["message"]
assert body["output"][0]["content"][0]["text"] == "Use <think>hi</think> in your prompt."
def test_effort_dial_gguf_still_parses_think_tags_when_disabled(self, monkeypatch):
payload = ResponsesRequest(input = "hi", reasoning = {"effort": "none"})
body = self._run_with_message(
monkeypatch,
{"content": "<think>plan</think>answer"},
payload = payload,
llama_backend = SimpleNamespace(
is_loaded = True,
reasoning_always_on = False,
supports_reasoning = True,
_request_reasoning_kwargs = (
lambda enable_thinking, reasoning_effort = None, preserve_thinking = None: (
{"reasoning_effort": "low"}
)
),
),
)
assert [item["type"] for item in body["output"]] == ["reasoning", "message"]
assert body["output"][0]["content"] == [{"type": "reasoning_text", "text": "plan"}]
assert body["output"][1]["content"][0]["text"] == "answer"
def test_inkling_numeric_zero_effort_keeps_think_tags_visible(self, monkeypatch):
# Real resolver: for Inkling, 'none' becomes numeric 0, which a string check misreads.
from core.inference.llama_cpp import LlamaCppBackend
backend = LlamaCppBackend.__new__(LlamaCppBackend)
backend._process = object()
backend._healthy = True
backend._supports_reasoning = True
backend._reasoning_always_on = False
backend._reasoning_style = "reasoning_effort"
backend._reasoning_effort_levels = ["none", "low", "medium", "high", "max"]
backend._supports_preserve_thinking = False
backend._preserve_thinking_default = False
backend._reasoning_default = True
backend._architecture = "inkling"
assert backend._request_reasoning_kwargs(False, "none", None) == {"reasoning_effort": 0.0}
payload = ResponsesRequest(input = "hi", reasoning = {"effort": "none"})
body = self._run_with_message(
monkeypatch,
{"content": "Use <think>hi</think> in your prompt."},
payload = payload,
llama_backend = backend,
)
assert [item["type"] for item in body["output"]] == ["message"]
assert body["output"][0]["content"][0]["text"] == "Use <think>hi</think> in your prompt."
def test_launch_default_thinking_off_keeps_think_tags_visible(self, monkeypatch):
body = self._run_with_message(
monkeypatch,
{"content": "Use <think>hi</think> in your prompt."},
llama_backend = SimpleNamespace(
is_loaded = True,
reasoning_always_on = False,
supports_reasoning = True,
reasoning_default = False,
),
)
assert [item["type"] for item in body["output"]] == ["message"]
assert body["output"][0]["content"][0]["text"] == "Use <think>hi</think> in your prompt."
def test_launch_default_thinking_on_still_parses_think_tags(self, monkeypatch):
body = self._run_with_message(
monkeypatch,
{"content": "<think>plan</think>answer"},
llama_backend = SimpleNamespace(
is_loaded = True,
reasoning_always_on = False,
supports_reasoning = True,
reasoning_default = True,
),
)
assert [item["type"] for item in body["output"]] == ["reasoning", "message"]
assert body["output"][0]["content"] == [{"type": "reasoning_text", "text": "plan"}]
assert body["output"][1]["content"][0]["text"] == "answer"
def test_structured_reasoning_content_extracts_text_parts(self, monkeypatch):
body = self._run_with_message(
monkeypatch,
{
"content": "33",
"reasoning_content": [
{"type": "reasoning_text", "text": "plan"},
{"type": "reasoning_text", "text": " next"},
],
},
)
assert [item["type"] for item in body["output"]] == ["reasoning", "message"]
assert body["output"][0]["content"] == [{"type": "reasoning_text", "text": "plan next"}]
assert body["output"][1]["content"][0]["text"] == "33"
def test_plain_content_remains_message_only(self, monkeypatch):
body = self._run_with_message(monkeypatch, {"content": "33"})
assert [item["type"] for item in body["output"]] == ["message"]
assert body["output"][0]["content"][0]["text"] == "33"
def test_reasoning_only_stays_out_of_visible_message_text(self, monkeypatch):
payload = ResponsesRequest(input = "hi", reasoning = {"effort": "high"})
body = self._run_with_message(
monkeypatch,
{"content": "<think>plan</think>"},
payload = payload,
)
assert [item["type"] for item in body["output"]] == ["reasoning"]
assert body["output"][0]["content"][0]["text"] == "plan"
class TestResponsesStreamAdapter:
class _Request:
async def is_disconnected(self):
return False
@staticmethod
async def _collect(response):
chunks = []
async for chunk in response.body_iterator:
chunks.append(chunk.decode() if isinstance(chunk, bytes) else chunk)
return chunks
def _stream_lines(
self,
payload,
messages,
monitor_id = None,
):
"""Drive _responses_stream to completion and return the SSE lines it wrote."""
async def run():
response = await _responses_stream(
payload, messages, self._Request(), monitor_id = monitor_id
)
return await self._collect(response)
return asyncio.run(run())
def _stream_deltas(self, payload, messages):
"""The SSE lines plus the reasoning and output-text deltas carried in them."""
lines = self._stream_lines(payload, messages)
return (
lines,
self._payloads(lines, "response.reasoning_text.delta"),
self._payloads(lines, "response.output_text.delta"),
)
@staticmethod
def _payloads(lines, event_name):
prefix = f"event: {event_name}\n"
return [
json.loads(line.split("data: ", 1)[1].strip())
for line in lines
if line.startswith(prefix)
]
@staticmethod
def _install_stream_mock(
monkeypatch,
chunks,
*,
supports_reasoning = True,
reasoning_always_on = False,
):
import routes.inference as inf_mod
def handler(request: httpx.Request) -> httpx.Response:
content = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks)
content += "data: [DONE]\n\n"
return httpx.Response(
200,
content = content.encode(),
headers = {"content-type": "text/event-stream"},
)
transport = httpx.MockTransport(handler)
real_async_client = httpx.AsyncClient
def _client(*args, **kwargs):
return real_async_client(
transport = transport,
timeout = kwargs.get("timeout", 600),
)
monkeypatch.setattr(inf_mod.httpx, "AsyncClient", _client)
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_vision = False,
context_length = 4096,
base_url = "http://llama.test",
supports_reasoning = supports_reasoning,
reasoning_always_on = reasoning_always_on,
_request_reasoning_kwargs = (
lambda enable_thinking = None, reasoning_effort = None, preserve_thinking = None: None
),
),
)
def test_stream_response_avoids_legacy_receive_watcher(self, monkeypatch):
self._install_stream_mock(
monkeypatch,
[{"choices": [{"delta": {"content": "33"}}]}],
)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
async def run():
response = await _responses_stream(payload, messages, self._Request())
assert isinstance(response, _SameTaskStreamingResponse)
sent = []
async def receive():
raise AssertionError("Responses streams poll disconnects in the generator")
async def send(message):
sent.append(message)
await response({"type": "http", "asgi": {"spec_version": "2.3"}}, receive, send)
return sent
sent = asyncio.run(run())
assert sent[0]["type"] == "http.response.start"
body = b"".join(message.get("body", b"") for message in sent).decode()
assert "response.output_text.delta" in body
assert '"delta":"33"' in body.replace(" ", "")
def test_split_think_markers_stream_as_reasoning_and_visible_text(self, monkeypatch):
chunks = [
{"choices": [{"delta": {"content": "<thi"}}]},
{"choices": [{"delta": {"content": "nk>pla"}}]},
{"choices": [{"delta": {"content": "n</th"}}]},
{"choices": [{"delta": {"content": "ink>33"}}]},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
self._install_stream_mock(monkeypatch, chunks)
payload = ResponsesRequest(input = "hi", stream = True, reasoning = {"effort": "high"})
messages = [ChatMessage(role = "user", content = "hi")]
lines, reasoning_deltas, text_deltas = self._stream_deltas(payload, messages)
assert "".join(event["delta"] for event in reasoning_deltas) == "plan"
assert "".join(event["delta"] for event in text_deltas) == "33"
completed = self._payloads(lines, "response.completed")[0]
assert [item["type"] for item in completed["response"]["output"]] == [
"reasoning",
"message",
]
assert completed["response"]["output"][0]["content"][0]["text"] == "plan"
assert completed["response"]["output"][1]["content"][0]["text"] == "33"
def test_usage_only_chunk_updates_monitor(self, monkeypatch):
import routes.inference as inf_mod
chunks = [
{"choices": [{"delta": {"content": "33"}}]},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
self._install_stream_mock(monkeypatch, chunks)
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
messages, monitor_id, payload = _shared_setup_1(monitor)
self._stream_lines(payload, messages, monitor_id)
[entry] = monitor.snapshot()
assert entry["status"] == "completed"
assert entry["reply"] == "33"
assert entry["prompt_tokens"] == 2
assert entry["completion_tokens"] == 3
assert entry["total_tokens"] == 5
assert entry["context_length"] == 4096
def test_final_chunk_timings_reach_the_monitor(self, monkeypatch):
"""Responses recorded usage but never timings, so a local request never reached
the Throughput tile. The final chunk can carry timings with no usage of its own."""
import routes.inference as inf_mod
chunks = [
{"choices": [{"delta": {"content": "33"}}]},
{"choices": [], "usage": {"prompt_tokens": 11, "completion_tokens": 50}},
{"choices": [], "timings": {"prompt_ms": 9000.0, "predicted_ms": 1000.0}},
]
self._install_stream_mock(monkeypatch, chunks)
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
messages, monitor_id, payload = _shared_setup_1(monitor)
self._stream_lines(payload, messages, monitor_id)
[entry] = monitor.snapshot()
assert entry["decode_ms"] == 1000
assert entry["completion_tokens"] == 50
assert entry["completion_tokens"] / (entry["decode_ms"] / 1000) == 50.0
def test_function_call_chunk_updates_monitor_reply(self, monkeypatch):
import routes.inference as inf_mod
chunks = [
{
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"id": "call_1",
"function": {
"name": "lookup",
"arguments": '{"query":"weather"}',
},
}
]
}
}
]
}
]
self._install_stream_mock(monkeypatch, chunks)
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
messages, monitor_id, payload = _shared_setup_1(monitor)
lines = self._stream_lines(payload, messages, monitor_id)
assert self._payloads(lines, "response.output_item.done")[-1]["item"]["name"] == "lookup"
[entry] = monitor.snapshot()
assert entry["status"] == "completed"
assert entry["reply"] == 'Tool call: lookup({"query":"weather"})'
def test_preheader_cancel_finalizes_monitor(self, monkeypatch):
import routes.inference as inf_mod
self._install_stream_mock(monkeypatch, [])
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
monitor_id = monitor.start(
endpoint = "/v1/responses",
method = "POST",
model = "m",
prompt = "hi",
)
async def fake_send(*_args, **_kwargs):
return None
monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fake_send)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
self._stream_lines(payload, messages, monitor_id)
[entry] = monitor.snapshot()
assert entry["status"] == "cancelled"
assert monitor.active_count() == 0
def test_stream_task_cancel_finalizes_monitor(self, monkeypatch):
async def _run():
import routes.inference as inf_mod
async def fake_send(*_args, **_kwargs):
return httpx.Response(200, content = b"")
async def fake_items(*_args, **_kwargs):
yield 'data: {"choices":[{"delta":{"content":"hello"}}]}'
await asyncio.sleep(3600)
self._install_stream_mock(monkeypatch, [])
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fake_send)
monkeypatch.setattr(inf_mod, "_aiter_llama_stream_items", fake_items)
monitor_id = monitor.start(
endpoint = "/v1/responses",
method = "POST",
model = "m",
prompt = "hi",
)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
response = await _responses_stream(
payload,
messages,
self._Request(),
monitor_id = monitor_id,
)
iterator = response.body_iterator
first = ""
for _ in range(8):
first = await anext(iterator)
if "hello" in first:
break
else:
pytest.fail("stream did not emit text delta")
pending = asyncio.create_task(anext(iterator))
await asyncio.sleep(0)
pending.cancel()
with pytest.raises(asyncio.CancelledError):
await pending
[entry] = monitor.snapshot()
assert entry["status"] == "cancelled"
assert entry["reply"] == "hello"
assert monitor.active_count() == 0
asyncio.run(_run())
def test_final_visible_text_updates_monitor(self, monkeypatch):
import routes.inference as inf_mod
class FakeExtractor:
def __init__(self, **_kwargs):
pass
def feed(
self,
_content,
_reasoning_content = None,
):
return "", ""
def finish(self):
return "", "tail"
self._install_stream_mock(monkeypatch, [{"choices": [{"delta": {"content": "<tai"}}]}])
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
monkeypatch.setattr(inf_mod, "_ResponsesReasoningExtractor", FakeExtractor)
messages, monitor_id, payload = _shared_setup_1(monitor)
lines = self._stream_lines(payload, messages, monitor_id)
assert self._payloads(lines, "response.output_text.delta")[-1]["delta"] == "tail"
[entry] = monitor.snapshot()
assert entry["status"] == "completed"
assert entry["reply"] == "tail"
def test_reasoning_only_stream_does_not_update_visible_monitor_reply(self, monkeypatch):
import routes.inference as inf_mod
class FakeExtractor:
def __init__(self, **_kwargs):
pass
def feed(
self,
_content,
_reasoning_content = None,
):
return "", ""
def finish(self):
return "plan", ""
self._install_stream_mock(monkeypatch, [{"choices": [{"delta": {"content": "<think>"}}]}])
monitor = ApiMonitor(max_entries = 3)
monkeypatch.setattr(inf_mod, "api_monitor", monitor)
monkeypatch.setattr(inf_mod, "_ResponsesReasoningExtractor", FakeExtractor)
messages, monitor_id, payload = _shared_setup_1(monitor)
lines = self._stream_lines(payload, messages, monitor_id)
assert self._payloads(lines, "response.output_text.delta") == []
assert self._payloads(lines, "response.reasoning_text.delta")[-1]["delta"] == "plan"
[entry] = monitor.snapshot()
assert entry["status"] == "completed"
assert entry["reply"] == ""
def test_reasoning_capable_gguf_stream_parses_think_tags_by_default(self, monkeypatch):
chunks = [
{"choices": [{"delta": {"content": "<thi"}}]},
{"choices": [{"delta": {"content": "nk>plan</think>answer"}}]},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
self._install_stream_mock(monkeypatch, chunks)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
lines, reasoning_deltas, text_deltas = self._stream_deltas(payload, messages)
assert "".join(event["delta"] for event in reasoning_deltas) == "plan"
assert "".join(event["delta"] for event in text_deltas) == "answer"
completed = self._payloads(lines, "response.completed")[0]
assert [item["type"] for item in completed["response"]["output"]] == [
"reasoning",
"message",
]
assert completed["response"]["output"][0]["content"][0]["text"] == "plan"
assert completed["response"]["output"][1]["content"][0]["text"] == "answer"
def test_non_reasoning_gguf_stream_keeps_literal_think_tags_visible(self, monkeypatch):
chunks = [
{"choices": [{"delta": {"content": "show <thi"}}]},
{"choices": [{"delta": {"content": "nk>x</think> tags"}}]},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
self._install_stream_mock(monkeypatch, chunks, supports_reasoning = False)
payload = ResponsesRequest(input = "hi", stream = True, reasoning = {"effort": "high"})
messages = [ChatMessage(role = "user", content = "hi")]
lines, reasoning_deltas, text_deltas = self._stream_deltas(payload, messages)
assert reasoning_deltas == []
assert "".join(event["delta"] for event in text_deltas) == "show <think>x</think> tags"
completed = self._payloads(lines, "response.completed")[0]
assert [item["type"] for item in completed["response"]["output"]] == ["message"]
assert completed["response"]["output"][0]["content"][0]["text"] == (
"show <think>x</think> tags"
)
def test_reasoning_only_stream_stays_out_of_visible_message_text(self, monkeypatch):
chunks = [
{"choices": [{"delta": {"content": "<think>plan</think>"}}]},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
self._install_stream_mock(monkeypatch, chunks)
payload = ResponsesRequest(input = "hi", stream = True, reasoning = {"effort": "high"})
messages = [ChatMessage(role = "user", content = "hi")]
lines, reasoning_deltas, text_deltas = self._stream_deltas(payload, messages)
assert "".join(event["delta"] for event in reasoning_deltas) == "plan"
assert text_deltas == []
completed = self._payloads(lines, "response.completed")[0]
assert [item["type"] for item in completed["response"]["output"]] == ["reasoning"]
assert completed["response"]["output"][0]["content"][0]["text"] == "plan"
def test_unclosed_think_stream_stays_out_of_visible_message_text(self, monkeypatch):
chunks = [
{"choices": [{"delta": {"content": "<thi"}}]},
{"choices": [{"delta": {"content": "nk>plan"}}]},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
self._install_stream_mock(monkeypatch, chunks)
payload = ResponsesRequest(input = "hi", stream = True, reasoning = {"effort": "high"})
messages = [ChatMessage(role = "user", content = "hi")]
lines, reasoning_deltas, text_deltas = self._stream_deltas(payload, messages)
assert "".join(event["delta"] for event in reasoning_deltas) == "plan"
assert text_deltas == []
completed = self._payloads(lines, "response.completed")[0]
assert [item["type"] for item in completed["response"]["output"]] == ["reasoning"]
assert completed["response"]["output"][0]["content"][0]["text"] == "plan"
def test_structured_reasoning_content_streams_as_reasoning(self, monkeypatch):
chunks = [
{"choices": [{"delta": {"reasoning_content": "plan"}}]},
{"choices": [{"delta": {"content": "33"}}]},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
self._install_stream_mock(monkeypatch, chunks)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
lines, reasoning_deltas, text_deltas = self._stream_deltas(payload, messages)
assert "".join(event["delta"] for event in reasoning_deltas) == "plan"
assert "".join(event["delta"] for event in text_deltas) == "33"
completed = self._payloads(lines, "response.completed")[0]
assert completed["response"]["output"][0]["type"] == "reasoning"
assert completed["response"]["output"][1]["type"] == "message"
def test_structured_reasoning_content_parts_stream_as_reasoning(self, monkeypatch):
chunks = [
{
"choices": [
{
"delta": {
"reasoning_content": {
"content": [
{"type": "reasoning_text", "text": "plan"},
{"type": "reasoning_text", "text": " next"},
]
}
}
}
]
},
{"choices": [{"delta": {"content": "33"}}]},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
self._install_stream_mock(monkeypatch, chunks)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
lines, reasoning_deltas, text_deltas = self._stream_deltas(payload, messages)
assert "".join(event["delta"] for event in reasoning_deltas) == "plan next"
assert "".join(event["delta"] for event in text_deltas) == "33"
assert "reasoning_text" not in "".join(event["delta"] for event in reasoning_deltas)
completed = self._payloads(lines, "response.completed")[0]
assert completed["response"]["output"][0]["content"][0]["text"] == "plan next"
assert completed["response"]["output"][1]["content"][0]["text"] == "33"
def test_tool_first_stream_closes_items_in_output_index_order(self, monkeypatch):
chunks = [
{
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"id": "call_0",
"type": "function",
"function": {"name": "lookup", "arguments": "{}"},
}
]
}
}
]
},
{"choices": [{"delta": {"content": "done"}}]},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
self._install_stream_mock(monkeypatch, chunks)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
lines = self._stream_lines(payload, messages)
done_events = self._payloads(lines, "response.output_item.done")
assert [event["output_index"] for event in done_events] == [0, 1]
assert [event["item"]["type"] for event in done_events] == ["function_call", "message"]
completed = self._payloads(lines, "response.completed")[0]
assert [item["type"] for item in completed["response"]["output"]] == [
"function_call",
"message",
]
def test_streamed_namespaced_call_returns_to_its_namespace(self, monkeypatch):
self._install_stream_mock(monkeypatch, [_namespaced_forecast_call_chunk()])
payload = ResponsesRequest(
input = "Weather in Paris?", stream = True, tools = _codex_namespace_tools()
)
messages = [ChatMessage(role = "user", content = "Weather in Paris?")]
lines = self._stream_lines(payload, messages)
added = self._payloads(lines, "response.output_item.added")[0]["item"]
args_done = self._payloads(lines, "response.function_call_arguments.done")[0]
done = self._payloads(lines, "response.output_item.done")[0]["item"]
completed = self._payloads(lines, "response.completed")[0]["response"]["output"]
assert (args_done["namespace"], args_done["name"]) == ("mcp__weather", "get_forecast")
for item in (added, done, completed[0]):
assert (item["type"], item["namespace"], item["name"]) == (
"function_call",
"mcp__weather",
"get_forecast",
)
assert done["call_id"] == "call_wx"
assert done["arguments"] == '{"city":"Paris"}'
def test_apply_patch_stream_is_restored_to_one_custom_tool_call(self, monkeypatch):
patch = "*** Begin Patch\n*** Add File: nested/café.txt\n+héllo\n*** End Patch"
arguments = json.dumps({"input": patch}, ensure_ascii = False)
split = len(arguments) // 2
chunks = [
{
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"id": "call_patch",
"type": "function",
"function": {
"name": "apply_patch",
"arguments": arguments[:split],
},
}
]
}
}
]
},
{
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"function": {"arguments": arguments[split:]},
}
]
}
}
]
},
]
self._install_stream_mock(monkeypatch, chunks)
payload = ResponsesRequest(
input = "edit",
stream = True,
tools = [_codex_apply_patch_tool()],
)
async def run():
response = await _responses_stream(
payload, [ChatMessage(role = "user", content = "edit")], self._Request()
)
return await self._collect(response)
lines = asyncio.run(run())
assert self._payloads(lines, "response.function_call_arguments.delta") == []
assert self._payloads(lines, "response.function_call_arguments.done") == []
done = self._payloads(lines, "response.output_item.done")
assert len(done) == 1
assert done[0]["item"] == {
"type": "custom_tool_call",
"status": "completed",
"call_id": "call_patch",
"name": "apply_patch",
"input": patch,
}
completed = self._payloads(lines, "response.completed")[0]
assert completed["response"]["output"] == [done[0]["item"]]
def test_regular_apply_patch_function_stream_stays_a_function_call(self, monkeypatch):
chunks = [
{
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"id": "call_patch",
"type": "function",
"function": {
"name": "apply_patch",
"arguments": '{"path":"x"}',
},
}
]
}
}
]
}
]
self._install_stream_mock(monkeypatch, chunks)
payload = ResponsesRequest(
input = "edit",
stream = True,
tools = [{"type": "function", "name": "apply_patch"}],
)
async def run():
response = await _responses_stream(
payload, [ChatMessage(role = "user", content = "edit")], self._Request()
)
return await self._collect(response)
lines = asyncio.run(run())
done = self._payloads(lines, "response.output_item.done")
assert done[0]["item"]["type"] == "function_call"
assert self._payloads(lines, "response.function_call_arguments.done")
def test_studio_ownership_marker_reaches_the_chat_request(self):
"""The Studio marker must reach the chat request, or a Studio thread is refused or forwarded raw."""
from routes.inference import _build_chat_request
payload = ResponsesRequest.model_validate(
{"input": "hi", "stream": True, "model": "org/M-GGUF", "studio_tool_history": True}
)
chat_req = _build_chat_request(
payload, [ChatMessage(role = "user", content = "hi")], stream = True
)
assert chat_req.studio_tool_history is True
plain = _build_chat_request(
ResponsesRequest.model_validate({"input": "hi", "model": "org/M-GGUF"}),
[ChatMessage(role = "user", content = "hi")],
stream = False,
)
assert not plain.studio_tool_history
def test_studio_tool_history_is_folded_on_the_direct_stream(self, monkeypatch):
"""This half of /v1/responses builds the passthrough body itself, so it has to fold the
way openai_chat_completions does. Otherwise the same thread on the same model answers
non-streaming and ships role="tool" to a toolless template when streamed."""
import routes.inference as inf_mod
captured = {}
def handler(request: httpx.Request) -> httpx.Response:
captured["body"] = json.loads(request.content.decode())
content = 'data: {"choices": [{"delta": {"content": "ok"}}]}\n\ndata: [DONE]\n\n'
return httpx.Response(
200,
content = content.encode(),
headers = {"content-type": "text/event-stream"},
)
transport = httpx.MockTransport(handler)
real_async_client = httpx.AsyncClient
monkeypatch.setattr(
inf_mod.httpx,
"AsyncClient",
lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)),
)
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_vision = False,
context_length = 4096,
base_url = "http://llama.test",
supports_tools = False,
supports_tool_passthrough = False,
_request_reasoning_kwargs = (
lambda enable_thinking = None, reasoning_effort = None, preserve_thinking = None: None
),
),
)
payload = ResponsesRequest(input = "and now?", stream = True, model = "org/M-GGUF")
messages = [
ChatMessage(role = "user", content = "what did we say about seeds?"),
ChatMessage(
role = "assistant",
content = None,
tool_calls = [
{
"id": "call_1",
"type": "function",
"function": {
"name": "search_conversation",
"arguments": '{"query": "seeds"}',
},
}
],
),
ChatMessage(
role = "tool",
tool_call_id = "call_1",
name = "search_conversation",
content = "we said 3407",
),
ChatMessage(role = "user", content = "and now?"),
]
async def run():
response = await _responses_stream(payload, messages, self._Request())
return await self._collect(response)
asyncio.run(run())
roles = [m.get("role") for m in captured["body"]["messages"]]
assert "tool" not in roles, roles
assert not any(a == "user" and b == "user" for a, b in zip(roles, roles[1:])), roles
assert "we said 3407" in json.dumps(captured["body"]["messages"])
def test_requests_usage_and_caps_parallel_tool_calls(self, monkeypatch):
import routes.inference as inf_mod
captured = {}
def handler(request: httpx.Request) -> httpx.Response:
captured["body"] = json.loads(request.content.decode())
chunks = [
{
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"id": "call_0",
"type": "function",
"function": {"name": "first", "arguments": "{}"},
},
{
"index": 1,
"id": "call_1",
"type": "function",
"function": {"name": "second", "arguments": "{}"},
},
]
}
}
]
},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
content = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks)
content += "data: [DONE]\n\n"
return httpx.Response(
200,
content = content.encode(),
headers = {"content-type": "text/event-stream"},
)
transport = httpx.MockTransport(handler)
real_async_client = httpx.AsyncClient
def _client(*args, **kwargs):
return real_async_client(
transport = transport,
timeout = kwargs.get("timeout", 600),
)
monkeypatch.setattr(inf_mod.httpx, "AsyncClient", _client)
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_vision = False,
context_length = 4096,
base_url = "http://llama.test",
_request_reasoning_kwargs = (
lambda enable_thinking = None, reasoning_effort = None, preserve_thinking = None: None
),
),
)
payload = ResponsesRequest(
input = "hi",
stream = True,
parallel_tool_calls = False,
tools = [
{
"type": "function",
"name": "first",
"parameters": {"type": "object"},
},
{
"type": "function",
"name": "second",
"parameters": {"type": "object"},
},
],
)
messages = [ChatMessage(role = "user", content = "hi")]
lines = self._stream_lines(payload, messages)
assert captured["body"]["stream_options"] == {"include_usage": True}
joined = "".join(lines)
assert "call_0" in joined
assert "call_1" not in joined
completed = self._payloads(lines, "response.completed")[0]
assert completed["response"]["usage"] == {
"input_tokens": 2,
"output_tokens": 3,
"total_tokens": 5,
}
class TestResponsesOutputFunctionCall:
def test_reasoning_output_item_serialises_full_reasoning_content(self):
item = ResponsesOutputReasoning(content = [{"type": "reasoning_text", "text": "plan"}])
d = item.model_dump()
assert d["type"] == "reasoning"
assert d["id"].startswith("rs_")
assert d["status"] == "completed"
assert d["summary"] == []
assert d["content"] == [{"type": "reasoning_text", "text": "plan"}]
def test_direct_construction(self):
fc = ResponsesOutputFunctionCall(
call_id = "call_1",
name = "get_weather",
arguments = '{"city":"Paris"}',
)
d = fc.model_dump()
assert d["type"] == "function_call"
assert d["call_id"] == "call_1"
assert d["status"] == "completed"
assert d["id"].startswith("fc_")
def test_custom_tool_call_construction(self):
call = ResponsesOutputCustomToolCall(
call_id = "call_patch",
name = "apply_patch",
input = "*** Begin Patch\n*** End Patch",
).model_dump()
assert call["type"] == "custom_tool_call"
assert call["id"].startswith("ctc_")
assert call["status"] == "completed"
def test_response_with_tool_call_output(self):
resp = ResponsesResponse(
model = "test",
output = [
ResponsesOutputFunctionCall(
call_id = "call_1",
name = "get_weather",
arguments = "{}",
)
],
usage = ResponsesUsage(input_tokens = 1, output_tokens = 1, total_tokens = 2),
)
d = json.loads(resp.model_dump_json())
assert d["output"][0]["type"] == "function_call"
assert d["output"][0]["call_id"] == "call_1"
def test_response_with_mixed_output(self):
resp = ResponsesResponse(
model = "test",
output = [
ResponsesOutputMessage(
content = [ResponsesOutputTextContent(text = "Calling...")],
),
ResponsesOutputFunctionCall(
call_id = "call_1",
name = "get_weather",
arguments = '{"city":"Paris"}',
),
],
)
d = resp.model_dump()
assert d["output"][0]["type"] == "message"
assert d["output"][1]["type"] == "function_call"
class TestCodexStyleRequestShapes:
"""Regression tests for the request shapes OpenAI Codex CLI sends."""
def test_assistant_replay_output_text_accepted(self):
"""Codex replays assistant turns with `output_text` content, which must validate."""
req = ResponsesRequest(
input = [
{"role": "user", "content": "Hi"},
{
"type": "message",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": "Hello!",
"annotations": [],
"logprobs": [],
}
],
},
{"role": "user", "content": "Continue"},
],
)
assert len(req.input) == 3
parts = req.input[1].content
assert isinstance(parts, list)
assert isinstance(parts[0], ResponsesOutputTextPart)
assert parts[0].text == "Hello!"
def test_reasoning_item_accepted_as_unknown(self):
"""Replayed `reasoning` items must not fail validation; Codex keeps them across turns."""
req = ResponsesRequest(
input = [
{"role": "user", "content": "Hi"},
{
"type": "reasoning",
"id": "rs_1",
"summary": [],
"encrypted_content": "opaque",
},
{"role": "assistant", "content": "Hello!"},
],
)
assert len(req.input) == 3
assert isinstance(req.input[1], ResponsesUnknownInputItem)
def test_emitted_reasoning_item_replays_as_reasoning_content(self):
payload = ResponsesRequest(
input = [
{"role": "user", "content": "Hi"},
{
"type": "reasoning",
"id": "rs_1",
"summary": [{"type": "summary_text", "text": "summary"}],
"content": [{"type": "reasoning_text", "text": "plan"}],
},
{"role": "assistant", "content": "33"},
{"role": "user", "content": "Continue"},
],
)
msgs = _normalise_responses_input(payload)
assert [m.role for m in msgs] == ["user", "assistant", "user"]
assert msgs[1].content == "33"
assert msgs[1].reasoning_content == "plan"
def test_codex_parallel_calls_replay_as_one_turn_with_reasoning(self):
payload = ResponsesRequest(
store = False,
input = [
{"type": "message", "role": "user", "content": "list files then read README"},
{
"type": "reasoning",
"id": "rs_1",
"summary": [],
"content": [{"type": "reasoning_text", "text": "PLAN: ls and cat"}],
"encrypted_content": None,
},
{
"type": "function_call",
"call_id": "c1",
"name": "shell",
"arguments": '{"cmd":"ls"}',
},
{
"type": "function_call",
"call_id": "c2",
"name": "shell",
"arguments": '{"cmd":"cat"}',
},
{"type": "function_call_output", "call_id": "c1", "output": "README"},
{"type": "function_call_output", "call_id": "c2", "output": "hello"},
],
tools = [{"type": "function", "name": "shell", "parameters": {"type": "object"}}],
)
msgs = _normalise_responses_input(payload)
assert [m.role for m in msgs] == ["user", "assistant", "tool", "tool"]
assert msgs[1].content is None
assert msgs[1].reasoning_content == "PLAN: ls and cat"
assert [c["id"] for c in msgs[1].tool_calls] == ["c1", "c2"]
body = _build_openai_passthrough_body(
_build_chat_request(payload, msgs, stream = True), backend_ctx = 4096
)
assert [m.get("reasoning_content") for m in body["messages"]] == [
None,
"PLAN: ls and cat",
None,
None,
]
def test_each_turn_keeps_its_own_text_reasoning_and_calls(self):
payload = ResponsesRequest(
input = [
{"role": "user", "content": "fix it"},
{
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "read first"}],
},
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "Reading."}],
},
{"type": "function_call", "call_id": "c1", "name": "shell", "arguments": "{}"},
{"type": "function_call_output", "call_id": "c1", "output": "code"},
{"type": "reasoning", "summary": [], "encrypted_content": "opaque"},
{"type": "custom_tool_call", "call_id": "c2", "name": "apply_patch", "input": "p"},
{"type": "custom_tool_call_output", "call_id": "c2", "output": "Done!"},
{
"type": "reasoning",
"summary": [],
"content": [{"type": "reasoning_text", "text": "ok"}],
},
{"role": "user", "content": "thanks"},
],
tools = [_codex_apply_patch_tool()],
)
msgs = _normalise_responses_input(payload)
assert [m.role for m in msgs] == ["user", "assistant", "tool", "assistant", "tool", "user"]
assert (msgs[1].content, msgs[1].reasoning_content) == ("Reading.", "read first")
assert [c["id"] for c in msgs[1].tool_calls] == ["c1"]
assert (msgs[3].content, msgs[3].reasoning_content) == (None, None)
assert [c["id"] for c in msgs[3].tool_calls] == ["c2"]
def test_unknown_content_part_type_accepted(self):
"""Unknown content-part types (e.g. future input_audio) validate as
ResponsesUnknownContentPart so the request doesn't 422."""
req = ResponsesRequest(
input = [
{
"role": "user",
"content": [
{"type": "input_text", "text": "See:"},
{"type": "input_audio", "audio": {"data": "..."}},
],
}
],
)
parts = req.input[0].content
assert isinstance(parts[1], ResponsesUnknownContentPart)
assert parts[1].type == "input_audio"
def test_codex_full_shape_roundtrip(self):
"""End-to-end: developer + user + assistant(output_text) +
function_call + function_call_output + reasoning in one request."""
payload = ResponsesRequest(
instructions = "Base instructions.",
input = [
{
"type": "message",
"role": "developer",
"content": [{"type": "input_text", "text": "Dev override."}],
},
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "Weather?"}],
},
{
"type": "reasoning",
"id": "rs_1",
"summary": [],
},
{
"type": "function_call",
"call_id": "call_1",
"name": "get_weather",
"arguments": "{}",
},
{
"type": "function_call_output",
"call_id": "call_1",
"output": '{"temp":20}',
},
{
"type": "message",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": "It's 20°C.",
"annotations": [],
"logprobs": [],
}
],
},
{"role": "user", "content": "And tomorrow?"},
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].role == "system"
assert sum(1 for m in msgs if m.role == "system") == 1
assert "Base instructions." in msgs[0].content
assert "Dev override." in msgs[0].content
roles = [m.role for m in msgs[1:]]
assert roles == ["user", "assistant", "tool", "assistant", "user"]
assert msgs[2].tool_calls is not None
assert msgs[3].role == "tool"
assert msgs[3].tool_call_id == "call_1"
assert msgs[4].content == "It's 20°C."
def test_single_output_text_part_flattens_to_string(self):
"""A single output_text part flattens to a string so legacy chat templates avoid multimodal
paths."""
payload = ResponsesRequest(
input = [
{
"role": "assistant",
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
},
{"role": "user", "content": "next"},
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].role == "assistant"
assert msgs[0].content == "ok"
class TestTranslatedMessagesValidate:
"""Messages from _normalise_responses_input satisfy ChatMessage's
role-shape validator so the downstream /v1/chat/completions
pass-through doesn't reject them."""
def test_round_trip_multi_turn(self):
payload = ResponsesRequest(
input = [
{"role": "user", "content": "Weather in Paris?"},
{
"type": "function_call",
"call_id": "call_1",
"name": "get_weather",
"arguments": '{"city": "Paris"}',
},
{
"type": "function_call_output",
"call_id": "call_1",
"output": '{"temp": 20}',
},
{"role": "user", "content": "Thanks!"},
],
)
msgs = _normalise_responses_input(payload)
for m in msgs:
ChatMessage(**m.model_dump(exclude_none = True))
def test_empty_tool_output_round_trips_through_chat_message_validator(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_empty",
"output": "",
},
],
)
msgs = _normalise_responses_input(payload)
for m in msgs:
ChatMessage(**m.model_dump(exclude_none = True))
# enable_thinking templates prefill an unclosed <think>, so generation starts in reasoning.
class TestReasoningPrefilledExtractor:
@pytest.mark.parametrize(
"text, parse_think_markers, reasoning_prefilled, expected, expected_visible",
[
pytest.param(
"plan</think>answer",
True,
True,
"plan",
"answer",
id = "prefilled_single_feed_splits_lone_close",
),
pytest.param(
"still thinking with no close",
True,
True,
"still thinking with no close",
"",
id = "prefilled_never_closed_is_all_reasoning",
),
pytest.param("", True, True, "", "", id = "prefilled_empty_generation"),
pytest.param(
"plan</think>\n\nanswer",
True,
True,
"plan",
"\n\nanswer",
id = "prefilled_whitespace_after_close_is_visible",
),
pytest.param(
"</think>hi", True, True, "", "hi", id = "prefilled_close_at_start_empty_reasoning"
),
pytest.param(
"reasoning</think>ans",
True,
False,
"",
"reasoningans",
id = "not_prefilled_lone_close_preserves_current_behavior",
),
pytest.param(
"<think>r</think>v",
True,
False,
"r",
"v",
id = "not_prefilled_full_pair_still_splits",
),
pytest.param(
"just an answer",
False,
False,
"",
"just an answer",
id = "prefilled_ignored_when_markers_not_parsed",
),
],
)
def test_reasoning_prefilled_extractor_cases(
self, text, parse_think_markers, reasoning_prefilled, expected, expected_visible
):
reasoning, visible = _extract_responses_reasoning(
text, parse_think_markers = parse_think_markers, reasoning_prefilled = reasoning_prefilled
)
assert reasoning == expected
assert visible == expected_visible
def test_prefilled_close_split_across_feeds(self):
ex = _ResponsesReasoningExtractor(parse_think_markers = True, reasoning_prefilled = True)
r1, v1 = ex.feed("plan</th")
r2, v2 = ex.feed("ink>ans")
fr, fv = ex.finish()
assert (r1 + r2 + fr) == "plan"
assert (v1 + v2 + fv) == "ans"
def test_prefilled_close_split_one_char_per_feed(self):
ex = _ResponsesReasoningExtractor(parse_think_markers = True, reasoning_prefilled = True)
reasoning, visible = "", ""
for ch in "plan</think>x":
r, v = ex.feed(ch)
reasoning += r
visible += v
fr, fv = ex.finish()
assert (reasoning + fr) == "plan"
assert (visible + fv) == "x"
def test_prefilled_stray_open_tag_is_suppressed(self):
reasoning, visible = _extract_responses_reasoning(
"a<think>b</think>c",
parse_think_markers = True,
reasoning_prefilled = True,
)
assert reasoning == "ab"
assert visible == "c"
assert "<think>" not in reasoning
class TestResponsesStreamHealing:
"""Healed text-form tool calls go through structured items and must keep their order."""
_XML = '<tool_call>{"name":"lookup","arguments":{"q":"x"}}</tool_call>'
_TOOL = {"type": "function", "name": "lookup", "parameters": {"type": "object"}}
@staticmethod
def _ordered_events(lines):
events = []
for line in lines:
if not line.startswith("event: "):
continue
name, _, rest = line.partition("\n")
payload = json.loads(rest.split("data: ", 1)[1].strip())
events.append((name[len("event: ") :], payload))
return events
def _run_stream(self, monkeypatch, content, **payload_kwargs):
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch, [{"choices": [{"delta": {"content": content}}]}]
)
payload = ResponsesRequest(input = "hi", stream = True, tools = [self._TOOL], **payload_kwargs)
messages = [ChatMessage(role = "user", content = "hi")]
async def run():
response = await _responses_stream(
payload, messages, TestResponsesStreamAdapter._Request()
)
return await TestResponsesStreamAdapter._collect(response)
return self._ordered_events(asyncio.run(run()))
def test_text_around_healed_call_keeps_order(self, monkeypatch):
events = self._run_stream(monkeypatch, f"before {self._XML} after.")
pos_before = pos_item = pos_after = None
for i, (name, payload) in enumerate(events):
if name == "response.output_text.delta":
if "before" in payload["delta"] and pos_before is None:
pos_before = i
if "after" in payload["delta"]:
pos_after = i
if (
name == "response.output_item.added"
and payload["item"]["type"] == "function_call"
and pos_item is None
):
pos_item = i
assert payload["item"]["name"] == "lookup"
assert pos_before is not None and pos_item is not None and pos_after is not None
assert pos_before < pos_item < pos_after
def test_call_before_trailing_text_claims_lower_output_index(self, monkeypatch):
events = self._run_stream(monkeypatch, f"{self._XML} done.")
item_added = [
(name, payload) for name, payload in events if name == "response.output_item.added"
]
assert [payload["item"]["type"] for _, payload in item_added] == [
"function_call",
"message",
]
call_idx = item_added[0][1]["output_index"]
msg_idx = item_added[1][1]["output_index"]
assert call_idx < msg_idx
text = "".join(
payload["delta"] for name, payload in events if name == "response.output_text.delta"
)
assert "done." in text
assert "<tool_call>" not in text
def test_tool_choice_none_streams_raw_text(self, monkeypatch):
events = self._run_stream(monkeypatch, self._XML, tool_choice = "none")
assert not any(
payload["item"]["type"] == "function_call"
for name, payload in events
if name == "response.output_item.added"
)
text = "".join(
payload["delta"] for name, payload in events if name == "response.output_text.delta"
)
assert text == self._XML
def test_healed_call_splits_message_items(self, monkeypatch):
events = self._run_stream(monkeypatch, f"before {self._XML} after.")
added = [
(payload["output_index"], payload["item"]["type"], payload["item"].get("id"))
for name, payload in events
if name == "response.output_item.added"
]
assert [item_type for _, item_type, _ in added] == [
"message",
"function_call",
"message",
]
assert [idx for idx, _, _ in added] == sorted(idx for idx, _, _ in added)
assert added[0][2] != added[2][2]
deltas = [
(payload["item_id"], payload["delta"])
for name, payload in events
if name == "response.output_text.delta"
]
assert [d for i, d in deltas if i == added[0][2]] == ["before "]
assert [d for i, d in deltas if i == added[2][2]] == [" after."]
completed = [payload for name, payload in events if name == "response.completed"]
output = completed[0]["response"]["output"]
assert [item["type"] for item in output] == ["message", "function_call", "message"]
assert output[0]["content"][0]["text"] == "before "
assert output[2]["content"][0]["text"] == " after."
def test_parallel_cap_drops_native_after_healed(self, monkeypatch):
# parallel_tool_calls=false: the healed call takes the one slot, so a later native call
# (index 0 survives _drop_parallel_tool_call_deltas) must not open a second item.
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch,
[
{"choices": [{"delta": {"content": self._XML}}]},
{
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"id": "call_up",
"function": {"name": "lookup", "arguments": "{}"},
}
]
}
}
]
},
],
)
payload = ResponsesRequest(
input = "hi",
stream = True,
tools = [self._TOOL],
parallel_tool_calls = False,
)
messages = [ChatMessage(role = "user", content = "hi")]
async def run():
response = await _responses_stream(
payload, messages, TestResponsesStreamAdapter._Request()
)
return await TestResponsesStreamAdapter._collect(response)
events = self._ordered_events(asyncio.run(run()))
calls = [
payload
for name, payload in events
if name == "response.output_item.added" and payload["item"]["type"] == "function_call"
]
assert len(calls) == 1
assert calls[0]["item"]["name"] == "lookup"
def test_healed_responses_tool_call_stamps_first_token(monkeypatch):
# Healed output bypasses append_reply, so the call went untimed until its item closed.
api_monitor, tool, xml = _shared_setup_6()
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch, [{"choices": [{"delta": {"content": xml}}]}]
)
messages, monitor_id, payload = _shared_setup_3(api_monitor, tool)
stamped: list[str] = []
real_mark = api_monitor.mark_first_token
monkeypatch.setattr(
api_monitor,
"mark_first_token",
lambda mid: (stamped.append(mid), real_mark(mid))[1],
)
async def run():
response = await _responses_stream(
payload, messages, TestResponsesStreamAdapter._Request(), monitor_id
)
return await TestResponsesStreamAdapter._collect(response)
asyncio.run(run())
assert stamped, "a healed tool call is output the client already received"
def test_finalized_healed_tool_call_stamps_first_token(monkeypatch):
from core.inference.api_monitor import api_monitor
unclosed = '<tool_call>{"name":"lookup","arguments":{"q":"x"}}'
tool = TestResponsesStreamHealing._TOOL
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch, [{"choices": [{"delta": {"content": unclosed}}]}]
)
messages, monitor_id, payload = _shared_setup_3(api_monitor, tool)
stamped: list[str] = []
real_mark = api_monitor.mark_first_token
monkeypatch.setattr(
api_monitor,
"mark_first_token",
lambda mid: (stamped.append(mid), real_mark(mid))[1],
)
async def run():
response = await _responses_stream(
payload, messages, TestResponsesStreamAdapter._Request(), monitor_id
)
return await TestResponsesStreamAdapter._collect(response)
lines = asyncio.run(run())
assert any("response.function_call_arguments.delta" in line for line in lines)
assert stamped, "a finalized healed call is output the client already received"
def test_healed_responses_tool_call_reports_a_tool_call_stop(monkeypatch):
# Upstream still says 'stop' while a function_call was emitted.
api_monitor, tool, xml = _shared_setup_6()
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch,
[{"choices": [{"delta": {"content": xml}, "finish_reason": "stop"}]}],
)
messages, monitor_id, payload = _shared_setup_3(api_monitor, tool)
async def run():
response = await _responses_stream(
payload, messages, TestResponsesStreamAdapter._Request(), monitor_id
)
return await TestResponsesStreamAdapter._collect(response)
asyncio.run(run())
row = next(r for r in api_monitor.snapshot() if r["id"] == monitor_id)
assert row["stop_reason"] == "tool_calls"
def test_unhealed_responses_stream_keeps_the_upstream_stop(monkeypatch):
from core.inference.api_monitor import api_monitor
tool = TestResponsesStreamHealing._TOOL
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch,
[{"choices": [{"delta": {"content": "plain text"}, "finish_reason": "stop"}]}],
)
messages, monitor_id, payload = _shared_setup_3(api_monitor, tool)
async def run():
response = await _responses_stream(
payload, messages, TestResponsesStreamAdapter._Request(), monitor_id
)
return await TestResponsesStreamAdapter._collect(response)
asyncio.run(run())
row = next(r for r in api_monitor.snapshot() if r["id"] == monitor_id)
assert row["stop_reason"] == "stop"
@pytest.mark.parametrize(
("finish_reason", "incomplete_reason"),
[("length", "max_output_tokens"), ("content_filter", "content_filter")],
)
def test_a_truncated_responses_stream_ends_on_response_incomplete(
monkeypatch, finish_reason, incomplete_reason
):
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch,
[{"choices": [{"delta": {"content": "half an ans"}, "finish_reason": finish_reason}]}],
)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
async def run():
response = await _responses_stream(payload, messages, TestResponsesStreamAdapter._Request())
return await TestResponsesStreamAdapter._collect(response)
lines = asyncio.run(run())
assert TestResponsesStreamAdapter._payloads(lines, "response.completed") == []
incomplete = TestResponsesStreamAdapter._payloads(lines, "response.incomplete")[0]
assert incomplete["response"]["status"] == "incomplete"
assert incomplete["response"]["incomplete_details"] == {"reason": incomplete_reason}
assert [item["status"] for item in incomplete["response"]["output"]] == ["incomplete"]
item_done = TestResponsesStreamAdapter._payloads(lines, "response.output_item.done")[0]
assert item_done["item"]["status"] == "incomplete"
@pytest.mark.parametrize(
("finish_reason", "incomplete_reason"),
[("length", "max_output_tokens"), ("content_filter", "content_filter")],
)
def test_a_healed_truncated_tool_call_remains_incomplete(
monkeypatch, finish_reason, incomplete_reason
):
api_monitor, tool, xml = _shared_setup_6()
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch,
[{"choices": [{"delta": {"content": xml}, "finish_reason": finish_reason}]}],
)
messages, monitor_id, payload = _shared_setup_3(api_monitor, tool)
async def run():
response = await _responses_stream(
payload, messages, TestResponsesStreamAdapter._Request(), monitor_id
)
return await TestResponsesStreamAdapter._collect(response)
lines = asyncio.run(run())
assert TestResponsesStreamAdapter._payloads(lines, "response.completed") == []
incomplete = TestResponsesStreamAdapter._payloads(lines, "response.incomplete")[0]
assert incomplete["response"]["status"] == "incomplete"
assert incomplete["response"]["incomplete_details"] == {"reason": incomplete_reason}
assert [item["status"] for item in incomplete["response"]["output"]] == ["incomplete"]
row = next(r for r in api_monitor.snapshot() if r["id"] == monitor_id)
assert row["stop_reason"] == finish_reason
item_done = TestResponsesStreamAdapter._payloads(lines, "response.output_item.done")[0]
assert item_done["item"]["status"] == "incomplete"
def test_a_call_cut_at_length_stays_text_in_a_responses_stream(monkeypatch):
api_monitor, tool, _xml = _shared_setup_6()
name = tool["name"]
cut = f"<tool_call><function={name}><parameter=q>line1\nli"
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch,
[{"choices": [{"delta": {"content": cut}, "finish_reason": "length"}]}],
)
messages, monitor_id, payload = _shared_setup_3(api_monitor, tool)
async def run():
response = await _responses_stream(
payload, messages, TestResponsesStreamAdapter._Request(), monitor_id
)
return await TestResponsesStreamAdapter._collect(response)
lines = asyncio.run(run())
incomplete = TestResponsesStreamAdapter._payloads(lines, "response.incomplete")[0]
assert [item["type"] for item in incomplete["response"]["output"]] == ["message"]
assert incomplete["response"]["output"][0]["content"][0]["text"] == cut
def test_a_complete_responses_stream_still_ends_on_response_completed(monkeypatch):
TestResponsesStreamAdapter._install_stream_mock(
monkeypatch,
[{"choices": [{"delta": {"content": "a whole answer"}, "finish_reason": "stop"}]}],
)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
async def run():
response = await _responses_stream(payload, messages, TestResponsesStreamAdapter._Request())
return await TestResponsesStreamAdapter._collect(response)
lines = asyncio.run(run())
assert TestResponsesStreamAdapter._payloads(lines, "response.incomplete") == []
completed = TestResponsesStreamAdapter._payloads(lines, "response.completed")[0]
assert completed["response"]["status"] == "completed"
assert completed["response"]["incomplete_details"] is None
assert [item["status"] for item in completed["response"]["output"]] == ["completed"]
_OVERFLOW_BODY = json.dumps(
{
"error": {
"code": 400,
"message": "request (16608 tokens) exceeds the available context size (2048 tokens), try increasing it",
"type": "exceed_context_size_error",
"n_prompt_tokens": 16608,
"n_ctx": 2048,
}
}
)
def _failed_stream_error(
monkeypatch,
handler = None,
chunks = (),
):
import routes.inference as inf_mod
real_async_client = httpx.AsyncClient
TestResponsesStreamAdapter._install_stream_mock(monkeypatch, list(chunks))
if handler is not None:
monkeypatch.setattr(
inf_mod.httpx,
"AsyncClient",
lambda *args, **kwargs: real_async_client(transport = httpx.MockTransport(handler)),
)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
async def run():
response = await _responses_stream(payload, messages, TestResponsesStreamAdapter._Request())
return await TestResponsesStreamAdapter._collect(response)
[failed] = TestResponsesStreamAdapter._payloads(asyncio.run(run()), "response.failed")
return failed["response"]["error"]
@pytest.mark.parametrize(
("status", "body", "code"),
[
(400, _OVERFLOW_BODY, "context_length_exceeded"),
(500, '{"error":{"code":500,"message":"Context size has been exceeded."}}', "server_error"),
],
)
def test_upstream_rejection_fails_the_stream_with_a_string_code(monkeypatch, status, body, code):
error = _failed_stream_error(
monkeypatch, handler = lambda request: httpx.Response(status, content = body.encode())
)
assert error["code"] == code
@pytest.mark.parametrize(
("message", "code"),
[
(
"request (3868 tokens) exceeds the available context size (2048 tokens), try increasing it",
"context_length_exceeded",
),
("Context size has been exceeded.", "server_error"),
],
)
def test_in_band_upstream_error_fails_the_stream_with_a_string_code(monkeypatch, message, code):
error = _failed_stream_error(monkeypatch, chunks = [{"error": {"code": 500, "message": message}}])
assert error["code"] == code
def test_unreachable_upstream_fails_the_stream_with_server_error(monkeypatch):
def refuse(request):
raise httpx.ConnectError("connection refused", request = request)
assert _failed_stream_error(monkeypatch, handler = refuse)["code"] == "server_error"
def test_admission_timeout_fails_the_stream_with_server_is_overloaded(monkeypatch):
import routes.inference as inf_mod
async def fail_send(*_args, **_kwargs):
raise AssertionError("a request that never got a slot must not reach llama-server")
base_url = "http://llama.responses.admission-timeout.test"
monkeypatch.setenv(ADMISSION_QUEUE_TIMEOUT_ENV, "0.01")
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_vision = False,
base_url = base_url,
context_length = 4096,
effective_parallel_slots = 1,
_request_reasoning_kwargs = lambda *_args, **_kwargs: None,
),
)
monkeypatch.setattr(inf_mod, "_send_stream_with_preheader_cancel", fail_send)
payload = ResponsesRequest(input = "hi", stream = True)
messages = [ChatMessage(role = "user", content = "hi")]
async def run():
queue = get_llama_admission_queue(base_url)
blocker = queue.reserve(capacity = 1, config = LlamaAdmissionConfig()).lease_nowait()
try:
response = await _responses_stream(
payload, messages, TestResponsesStreamAdapter._Request()
)
return await TestResponsesStreamAdapter._collect(response)
finally:
blocker.release()
[failed] = TestResponsesStreamAdapter._payloads(asyncio.run(run()), "response.failed")
assert failed["response"]["error"]["code"] == "server_is_overloaded"
assert failed["sequence_number"] == 0
def _sequence_numbers(lines):
return [
json.loads(line.split("data: ", 1)[1].strip())["sequence_number"]
for line in lines
if line.startswith("event: ")
]
@pytest.mark.parametrize(
"chunks",
[
[{"choices": [{"delta": {"content": "a whole answer"}, "finish_reason": "stop"}]}],
[{"error": {"code": 500, "message": "Context size has been exceeded."}}],
],
)
def test_every_responses_stream_event_carries_its_sequence_number(monkeypatch, chunks):
TestResponsesStreamAdapter._install_stream_mock(monkeypatch, chunks)
lines = TestResponsesStreamAdapter()._stream_lines(
ResponsesRequest(input = "hi", stream = True), [ChatMessage(role = "user", content = "hi")]
)
numbers = _sequence_numbers(lines)
assert len(numbers) > 1
assert numbers == list(range(len(numbers)))
def test_upstream_rejection_event_carries_its_sequence_number(monkeypatch):
import routes.inference as inf_mod
real_async_client = httpx.AsyncClient
TestResponsesStreamAdapter._install_stream_mock(monkeypatch, [])
monkeypatch.setattr(
inf_mod.httpx,
"AsyncClient",
lambda *args, **kwargs: real_async_client(
transport = httpx.MockTransport(
lambda request: httpx.Response(400, content = _OVERFLOW_BODY.encode())
)
),
)
lines = TestResponsesStreamAdapter()._stream_lines(
ResponsesRequest(input = "hi", stream = True), [ChatMessage(role = "user", content = "hi")]
)
assert _sequence_numbers(lines) == [0, 1]
class TestResponsesInProcessStream:
_TC = {"index": 0, "id": "c1", "function": {"name": "f", "arguments": "{}"}}
_CHUNKS = [
{"choices": [{"delta": {"reasoning_content": "plan"}}]},
{"choices": [{"delta": {"content": "33"}}]},
{"choices": [{"delta": {"tool_calls": [_TC]}}]},
{"choices": [{"delta": {}, "finish_reason": "length"}]},
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
]
@staticmethod
def _run(
monkeypatch,
chunks,
stop_at = None,
taken = -1,
entry = {},
headers = (),
raise_on_stop = False,
):
# taken >= 0 cancels after that many events; None disconnects before the body.
import routes.inference as inf_mod
from starlette.requests import Request
seen = {"closed": [], "unstarted": 0, "finish": []}
async def fake_chat(chat_req, request, subject):
seen.update(req = chat_req, skip = request.state.skip_api_monitor)
seen["ui"] = inf_mod._ui_stream_events_enabled(request)
event = seen["event"] = request.state.generation_cancel_event
async def body():
try:
for i, chunk in enumerate(chunks):
if i == stop_at:
event.set() # a Stop through the run's tracked event
if raise_on_stop: # as the managed vLLM / SGLang tool path does
raise asyncio.CancelledError()
yield f"data: {json.dumps(chunk)}\n\n"
yield "data: [DONE]\n\n"
except BaseException as exc:
seen["closed"].append(type(exc).__name__)
raise
async def unstarted():
seen["unstarted"] += 1
return _SameTaskStreamingResponse(body(), unstarted_cleanup = unstarted)
monkeypatch.setattr(inf_mod, "openai_chat_completions", fake_chat)
monkeypatch.setattr(
inf_mod, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
orchestrator = SimpleNamespace(
active_model_name = None if entry is None else "m", models = {"m": entry}
)
monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: orchestrator)
monkeypatch.setattr(inf_mod, "_peek_inference_backend", lambda: orchestrator)
monkeypatch.setattr(inf_mod.api_monitor, "finish", lambda _id, *a: seen["finish"].append(a))
request = seen["request"] = Request({"type": "http", "headers": list(headers)})
payload = ResponsesRequest(input = "hi", stream = True, thread_id = "t1", generation_run_id = "r1")
async def run():
messages = [ChatMessage(role = "user", content = "hi")]
response = await _responses_stream(payload, messages, request, monitor_id = "m")
if taken is None:
return await response._unstarted_cleanup()
if taken < 0:
return [line async for line in response.body_iterator]
for _ in range(taken):
await response.body_iterator.__anext__()
with pytest.raises(asyncio.CancelledError):
await response.body_iterator.athrow(asyncio.CancelledError())
seen["lines"] = asyncio.run(run())
return seen
def test_a_source_raising_on_stop_still_ends_the_stream_cleanly(self, monkeypatch):
seen = self._run(monkeypatch, self._CHUNKS, stop_at = 2, raise_on_stop = True)
events = "".join(seen["lines"])
assert "response.created" in events and "response.completed" not in events
assert ("cancelled",) in seen["finish"]
def test_without_task_cancelling_a_stop_raised_by_the_source_still_propagates(
self, monkeypatch
):
# Python < 3.11 has no Task.cancelling, so the old behaviour stays.
import routes.inference as inf_mod
real = inf_mod.asyncio.current_task
monkeypatch.setattr(
inf_mod.asyncio, "current_task", lambda: SimpleNamespace(wrapped = real())
)
with pytest.raises(asyncio.CancelledError):
self._run(monkeypatch, self._CHUNKS, stop_at = 2, raise_on_stop = True)
def test_delegated_chat_never_offers_a_control_channel_it_cannot_relay(self, monkeypatch):
# An approval_id in a dropped tool_start frame would leave the tool loop waiting.
seen = self._run(monkeypatch, self._CHUNKS, headers = [(b"x-unsloth-events", b"1")])
assert seen["ui"] is False
# Restored after the call: the outer request still carries its own opt-in.
import routes.inference as inf_mod
assert inf_mod._ui_stream_events_enabled(seen["request"]) is True
def test_chat_chunks_stream_the_same_events_as_llama_server(self, monkeypatch):
import re
def norm(lines):
return [
re.sub(r'_[0-9a-f]{12}|"created_at": \d+|"model": "[^"]*"', "", l) for l in lines
]
TestResponsesStreamAdapter._install_stream_mock(monkeypatch, self._CHUNKS)
gguf = TestResponsesStreamAdapter()._stream_lines(
ResponsesRequest(input = "hi", stream = True), [ChatMessage(role = "user", content = "hi")]
)
seen = self._run(monkeypatch, self._CHUNKS)
assert norm(seen["lines"]) == norm(gguf)
assert gguf[-1].startswith("event: response.incomplete")
# Reachable by the response's keys; only the outer monitor counts the run.
req = seen["req"]
resp_id = json.loads(seen["lines"][0].split("data: ", 1)[1])["response"]["id"]
keys = (req.cancel_id, req.thread_id, req.generation_run_id, seen["skip"])
assert keys == (resp_id, "t1", "r1", True)
assert req.stream_options == {"include_usage": True}
assert not hasattr(seen["request"].state, "skip_api_monitor")
@pytest.mark.parametrize("stop_at", [None, 2])
def test_inner_failure_and_stop_end_without_success(self, monkeypatch, stop_at):
chunks = self._CHUNKS if stop_at else self._CHUNKS[1:2] + [{"error": {"message": "boom"}}]
seen = self._run(monkeypatch, chunks, stop_at)
names = [line.split("\n", 1)[0] for line in seen["lines"]]
assert not {"event: response.completed", "event: response.incomplete"} & set(names)
assert (names[-1] == "event: response.failed") is (stop_at is None)
assert (seen["finish"][-1:] == [("cancelled",)]) is (stop_at is not None)
@pytest.mark.parametrize(
"taken, closed, unstarted", [(None, [], 1), (1, [], 1), (2, ["CancelledError"], 0)]
)
def test_outer_cancel_closes_the_inner_stream(self, monkeypatch, taken, closed, unstarted):
seen = self._run(monkeypatch, self._CHUNKS, taken = taken)
assert (seen["closed"], seen["unstarted"]) == (closed, unstarted)
assert seen["finish"][-1] == ("cancelled",)
assert seen["event"].is_set() is (taken is not None)
@pytest.mark.parametrize("entry", [None, {"is_audio": True, "audio_type": "tts"}])
def test_only_a_text_model_in_the_orchestrator_streams_through_chat(self, monkeypatch, entry):
# An NPU model's provider proxy keeps its own cancel event, so the stream cannot see a
# Stop; chat answers a TTS model with a finished clip, not a text stream.
with pytest.raises(HTTPException) as err:
self._run(monkeypatch, self._CHUNKS, entry = entry)
assert err.value.status_code == 400