Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
from __future__ import annotations
import asyncio
import json
import sys
from http.server import BaseHTTPRequestHandler
from pathlib import Path
import pytest
from starlette.requests import Request
import routes.inference as ri
from core.inference import external_provider as ep_mod
from core.inference.context_window import estimate_messages_tokens_conservative
from models.inference import ChatCompletionRequest
sys.path.insert(0, str(Path(__file__).resolve().parent))
from test_external_provider_sampling_over_the_wire import _Handler, _run, _Server # noqa: E402
_TURN = "word " * 400
def _long_chat(turns: int = 12, turn: str = _TURN) -> list[dict]:
messages: list[dict] = [{"role": "system", "content": "Answer in French."}]
for index in range(turns):
messages.append({"role": "user", "content": f"question {index} {turn}"})
messages.append({"role": "assistant", "content": f"answer {index} {turn}"})
messages.append({"role": "user", "content": "latest question"})
return messages
class _AnthropicCompactingHandler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *args, **kwargs) -> None:
return
events = [
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "compaction", "content": None},
},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "compaction_delta", "content": "Earlier: twelve questions."},
},
{"type": "content_block_stop", "index": 0},
{"type": "content_block_start", "index": 1, "content_block": {"type": "text", "text": ""}},
{
"type": "content_block_delta",
"index": 1,
"delta": {"type": "text_delta", "text": "Bonjour"},
},
{"type": "content_block_stop", "index": 1},
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {}},
{"type": "message_stop"},
]
def do_POST(self) -> None: # noqa: N802
length = int(self.headers.get("Content-Length") or 0)
self.server.recorded.append(json.loads(self.rfile.read(length) or b"{}")) # type: ignore[attr-defined]
sse = "".join(
f"event: {e['type']}\ndata: {json.dumps(e)}\n\n" for e in self.events
).encode()
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Content-Length", str(len(sse)))
self.end_headers()
self.wfile.write(sse)
class _AnthropicQuotingHandler(_AnthropicCompactingHandler):
events = [
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": 'The event is called "compaction_block'},
},
{"type": "content_block_stop", "index": 0},
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {}},
{"type": "message_stop"},
]
class _OpenAICompactionRejectingHandler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *args, **kwargs) -> None:
return
def do_POST(self) -> None: # noqa: N802
length = int(self.headers.get("Content-Length") or 0)
body = json.loads(self.rfile.read(length) or b"{}")
self.server.recorded.append(body) # type: ignore[attr-defined]
if "context_management" in body:
payload = json.dumps(
{
"error": {
"message": "context_management is unavailable for this deployment",
"type": "invalid_request_error",
}
}
).encode()
self.send_response(400)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
return
sse = (
b"event: response.completed\n"
b'data: {"type":"response.completed",'
b'"response":{"output":[],"usage":{"input_tokens":0,"output_tokens":0}}}\n\n'
)
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Content-Length", str(len(sse)))
self.end_headers()
self.wfile.write(sse)
def _proxy(
provider_type: str,
messages: list[dict],
handler = _Handler,
**fields,
):
async def receive() -> dict:
return {"type": "http.disconnect"}
chunks: list[dict] = []
with _Server(handler = handler) as server:
payload = ChatCompletionRequest(
provider_type = provider_type,
provider_base_url = server.base_url,
messages = messages,
model = fields.pop("model", "a-model"),
stream = True,
max_tokens = fields.pop("max_tokens", 256),
**fields,
)
request = Request(
{
"type": "http",
"method": "POST",
"path": "/v1/chat/completions",
"headers": [(b"content-type", b"application/json")],
"query_string": b"",
},
receive,
)
async def go() -> None:
response = await ri._proxy_to_external_provider(payload, request)
async for part in response.body_iterator:
text = part.decode() if isinstance(part, bytes) else part
for line in text.splitlines():
body = line[len("data:") :].strip() if line.startswith("data:") else ""
if body and body != "[DONE]":
chunks.append(json.loads(body))
_run(go)
assert server.bodies, "the route never reached the provider"
return chunks, server.bodies[-1]
def _truncations(chunks: list[dict]) -> list[dict]:
return [c["context_truncated"] for c in chunks if "context_truncated" in c]
def test_auto_compact_drops_the_oldest_turns_before_a_provider_without_compaction():
chunks, sent = _proxy(
"openrouter",
_long_chat(),
context_overflow = "truncate_oldest",
compaction_threshold = 6_000,
)
roles = [m["role"] for m in sent["messages"]]
assert len(sent["messages"]) < len(_long_chat())
assert roles[0] == "system" and "Answer in French." in json.dumps(sent["messages"][0])
assert sent["messages"][-1]["content"] == "latest question"
[truncation] = _truncations(chunks)
assert truncation["fits"] is True
assert truncation["dropped_messages"] == len(_long_chat()) - len(sent["messages"])
def test_auto_compact_off_forwards_the_whole_chat():
chunks, sent = _proxy("openrouter", _long_chat())
assert len(sent["messages"]) == len(_long_chat())
assert _truncations(chunks) == []
def test_a_chat_within_the_threshold_is_forwarded_untouched():
short = _long_chat(turns = 1)
chunks, sent = _proxy(
"openrouter", short, context_overflow = "truncate_oldest", compaction_threshold = 6_000
)
assert len(sent["messages"]) == len(short)
assert _truncations(chunks) == []
def test_a_provider_compaction_is_reported_as_a_summarized_compaction():
chat = _long_chat(turns = 2)
chunks, sent = _proxy(
"anthropic",
chat,
handler = _AnthropicCompactingHandler,
model = "claude-sonnet-4-6",
context_overflow = "truncate_oldest",
compaction_threshold = 150_000,
)
assert sent["context_management"]["edits"][0]["trigger"]["value"] == 150_000
assert len(sent["messages"]) == len(chat) - 1
[truncation] = _truncations(chunks)
assert truncation == {
"dropped_messages": 4,
"boundary_messages": 4,
"fits": True,
"summarized": True,
}
def test_a_rejected_provider_compaction_falls_back_to_local_fitting(monkeypatch):
monkeypatch.setattr(ep_mod, "_is_openai_family_cloud", lambda _base_url: True)
monkeypatch.setattr(ri, "compacts_server_side", lambda *_args: True)
chat = _long_chat()
chunks, sent = _proxy(
"openai",
chat,
handler = _OpenAICompactionRejectingHandler,
model = "gpt-5.5",
context_overflow = "truncate_oldest",
compaction_threshold = 6_000,
)
serialized = json.dumps(sent["input"])
assert "context_management" not in sent
assert "question 0" not in serialized
assert "latest question" in serialized
[truncation] = _truncations(chunks)
assert truncation["fits"] is True
assert truncation["dropped_messages"] > 0
def test_model_text_naming_a_compaction_block_is_not_a_compaction():
chunks, _ = _proxy(
"anthropic",
_long_chat(turns = 2),
handler = _AnthropicQuotingHandler,
model = "claude-sonnet-4-6",
context_overflow = "truncate_oldest",
compaction_threshold = 150_000,
)
assert _truncations(chunks) == []
def test_the_trimmed_prompt_leaves_the_whole_reply_room_in_the_window():
chat = _long_chat(turns = 9, turn = "word " * 370)
assert 16_384 - 8_192 < estimate_messages_tokens_conservative(chat) < 9_216
chunks, sent = _proxy(
"openrouter",
chat,
max_tokens = 8_192,
context_overflow = "truncate_oldest",
compaction_threshold = 12_288,
context_window = 16_384,
)
assert sent["max_tokens"] == 8_192
assert estimate_messages_tokens_conservative(sent["messages"]) + 8_192 < 16_384
assert sent["messages"][-1]["content"] == "latest question"
[truncation] = _truncations(chunks)
assert truncation["dropped_messages"] == len(chat) - len(sent["messages"])
def test_a_reply_cap_near_the_window_is_lowered_rather_than_emptying_the_chat():
chat = _long_chat(turns = 9, turn = "word " * 370)
_, sent = _proxy(
"openrouter",
chat,
max_tokens = 16_000,
context_overflow = "truncate_oldest",
compaction_threshold = 12_288,
context_window = 16_384,
)
assert sum(m["content"].startswith("question") for m in sent["messages"]) >= 3
assert sent["max_tokens"] <= 16_384 // 2
assert estimate_messages_tokens_conservative(sent["messages"]) + sent["max_tokens"] < 16_384
def test_a_window_without_auto_compact_changes_nothing():
chat = _long_chat(turns = 9, turn = "word " * 370)
chunks, sent = _proxy("openrouter", chat, max_tokens = 16_000, context_window = 16_384)
assert sent["max_tokens"] == 16_000
assert len(sent["messages"]) == len(chat)
assert _truncations(chunks) == []
def test_a_short_chat_keeps_a_reply_cap_past_half_the_window():
short = _long_chat(turns = 1)
chunks, sent = _proxy(
"openrouter",
short,
max_tokens = 12_000,
context_overflow = "truncate_oldest",
compaction_threshold = 12_288,
context_window = 16_384,
)
assert sent["max_tokens"] == 12_000
assert len(sent["messages"]) == len(short)
assert _truncations(chunks) == []
def test_a_tool_enabled_chat_starts_from_the_saved_boundary(monkeypatch):
from core.inference import llama_cpp
monkeypatch.setattr(
llama_cpp,
"_sticky_compaction_state",
lambda thread_id, *a, **k: (20 if thread_id else 0, False),
)
monkeypatch.setattr(llama_cpp, "_keeps_compaction_boundary", lambda thread_id: False)
async def connected(self) -> bool:
return False
# The tool loop polls for a disconnect and would stop before reaching the provider.
monkeypatch.setattr(Request, "is_disconnected", connected)
_, sent = _proxy(
"openrouter",
_long_chat(),
thread_id = "saved-thread",
enable_tools = True,
enabled_tools = ["python"],
permission_mode = "off",
context_overflow = "truncate_oldest",
compaction_threshold = 6_000,
)
assert sent["tools"]
assert len(_long_chat()) - len(sent["messages"]) == 20
def test_external_fit_reserves_image_embeddings():
messages = [
{"role": "system", "content": "Keep this."},
{
"role": "user",
"content": [
{"type": "text", "text": "old image"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,cGljdHVyZQ=="},
},
],
},
{"role": "assistant", "content": "old answer"},
{"role": "user", "content": "latest question"},
]
payload = ChatCompletionRequest(
messages = [{"role": "user", "content": "latest question"}],
provider_type = "openrouter",
model = "a-model",
context_overflow = "truncate_oldest",
compaction_threshold = 3_000,
context_window = 4_000,
max_tokens = 512,
)
fitted, truncation, _ = ri._fit_external_context(messages, payload)
assert truncation and truncation["dropped_messages"] == 2
assert fitted[-1]["content"] == "latest question"
def test_external_fit_reserves_tool_schemas():
messages = _long_chat(turns = 2, turn = "word " * 80)
payload = ChatCompletionRequest(
messages = [{"role": "user", "content": "latest question"}],
provider_type = "openrouter",
model = "a-model",
context_overflow = "truncate_oldest",
compaction_threshold = 3_000,
context_window = 4_000,
max_tokens = 512,
)
tools = [
{
"type": "function",
"function": {
"name": "large_tool",
"description": "schema " * 1_200,
"parameters": {"type": "object", "properties": {}},
},
}
]
fitted, truncation, _ = ri._fit_external_context(messages, payload, tools = tools)
assert truncation and truncation["dropped_messages"] > 0
assert fitted[-1]["content"] == "latest question"
class _ServedWindowHandler(_Handler):
models: list[dict] = []
props: dict | None = None
ps: list[dict] | None = None
key: str | None = None
gets: list[str] = []
def do_GET(self) -> None: # noqa: N802
type(self).gets.append(self.path)
if self.key is not None and self.headers.get("Authorization") != f"Bearer {self.key}":
status, body = 401, {"error": "unauthorized"}
elif self.path == "/v1/models":
status, body = 200, {"object": "list", "data": self.models}
elif self.path.startswith("/props") and self.props is not None:
status, body = 200, self.props
elif self.path == "/api/ps" and self.ps is not None:
status, body = 200, {"models": self.ps}
else:
status, body = 404, {"error": "not found"}
raw = json.dumps(body).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
def _served(**attrs):
ep_mod._served_windows.clear()
ep_mod._served_templates.clear()
ep_mod._ollama_last_windows.clear()
return type("Served", (_ServedWindowHandler,), {"gets": [], **attrs})
def _llama_server():
# llama-server -c 16384 -np 2 reports n_ctx per slot and n_ctx_train as the trained maximum.
return _served(
models = [
{
"id": "qwen3-next",
"aliases": ["qwen3-next"],
"owned_by": "llamacpp",
"meta": {"n_ctx": 8_192, "n_ctx_train": 262_144},
}
]
)
def _fits(sent: dict, window: int) -> bool:
return estimate_messages_tokens_conservative(sent["messages"]) + sent["max_tokens"] <= window
def test_a_llama_server_connection_compacts_to_the_window_the_server_reports():
chunks, sent = _proxy(
"llama_cpp",
_long_chat(),
handler = _llama_server(),
model = "qwen3-next",
context_overflow = "truncate_oldest",
)
assert len(sent["messages"]) < len(_long_chat())
assert sent["messages"][-1]["content"] == "latest question"
assert _fits(sent, 8_192)
[truncation] = _truncations(chunks)
assert truncation["fits"] is True
def test_a_custom_connection_to_vllm_compacts_to_max_model_len():
vllm = _served(models = [{"id": "a-model", "owned_by": "vllm", "max_model_len": 8_192}])
chunks, sent = _proxy("custom", _long_chat(), handler = vllm, context_overflow = "truncate_oldest")
assert len(sent["messages"]) < len(_long_chat())
assert _fits(sent, 8_192)
assert len(_truncations(chunks)) == 1
def test_a_lora_adapter_uses_its_base_model_window():
vllm = _served(
models = [
{"id": "base", "owned_by": "vllm", "max_model_len": 8_192},
{"id": "my-lora", "owned_by": "vllm", "parent": "base"},
]
)
_, sent = _proxy(
"vllm", _long_chat(), handler = vllm, model = "my-lora", context_overflow = "truncate_oldest"
)
assert len(sent["messages"]) < len(_long_chat())
assert _fits(sent, 8_192)
def test_an_older_llama_server_is_read_from_props_for_the_model():
old = _served(
models = [{"id": "a-model", "owned_by": "llamacpp", "meta": {"n_ctx_train": 262_144}}],
props = {"default_generation_settings": {"n_ctx": 8_192}},
)
_, sent = _proxy("llama_cpp", _long_chat(), handler = old, context_overflow = "truncate_oldest")
assert len(sent["messages"]) < len(_long_chat())
assert _fits(sent, 8_192)
assert old.gets == ["/v1/models", "/props?model=a-model"]
def test_a_llama_server_template_default_is_sent_along_with_the_date(monkeypatch):
monkeypatch.setattr(
ri, "current_date_prompt_line", lambda **_kwargs: "The current date is 2026-08-15."
)
qwen = (
"{% if messages[0]['role'] == 'system' %}{% set sys = messages[0]['content'] %}"
"{% set rest = messages[1:] %}{% else %}"
"{% set sys = 'You are Qwen, created by Alibaba Cloud.' %}{% set rest = messages %}{% endif %}"
"<|im_start|>system\n{{ sys }}<|im_end|>\n"
"{% for m in rest %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}"
)
server = _served(
models = [{"id": "a-model", "owned_by": "llamacpp"}], props = {"chat_template": qwen}
)
_, sent = _proxy("llama_cpp", [{"role": "user", "content": "who are you?"}], handler = server)
assert sent["messages"] == [
{
"role": "system",
"content": "The current date is 2026-08-15.\n\nYou are Qwen, created by Alibaba Cloud.",
},
{"role": "user", "content": "who are you?"},
]
assert server.gets == ["/props?model=a-model"]
def test_a_window_the_request_states_is_kept_over_the_served_one():
server = _llama_server()
_, sent = _proxy(
"llama_cpp",
_long_chat(),
handler = server,
model = "qwen3-next",
context_overflow = "truncate_oldest",
context_window = 4_096,
)
assert _fits(sent, 4_096)
assert server.gets == []
def test_a_server_that_reports_no_window_gets_the_whole_chat():
silent = _served(models = [{"id": "a-model", "owned_by": "someone"}])
chunks, sent = _proxy(
"custom", _long_chat(), handler = silent, context_overflow = "truncate_oldest"
)
assert len(sent["messages"]) == len(_long_chat())
assert _truncations(chunks) == []
def test_auto_compact_off_never_reads_the_served_window():
server = _llama_server()
_, sent = _proxy("llama_cpp", _long_chat(), handler = server, model = "qwen3-next")
assert len(sent["messages"]) == len(_long_chat())
assert server.gets == []
def test_a_hosted_provider_is_not_probed_for_a_window():
server = _llama_server()
_, sent = _proxy(
"openrouter",
_long_chat(),
handler = server,
model = "qwen3-next",
context_overflow = "truncate_oldest",
)
assert len(sent["messages"]) == len(_long_chat())
assert server.gets == []
def _ollama(context_length: int = 4_096):
loaded = {"name": "qwen3:1.7b", "model": "qwen3:1.7b", "context_length": context_length}
return _served(ps = [loaded])
def test_an_ollama_connection_compacts_to_the_window_ollama_loaded():
chunks, sent = _proxy(
"ollama",
_long_chat(),
handler = _ollama(),
model = "qwen3:1.7b",
context_overflow = "truncate_oldest",
compaction_threshold = 30_720,
context_window = 40_960,
)
assert len(sent["messages"]) < len(_long_chat())
assert sent["messages"][-1]["content"] == "latest question"
assert _fits(sent, 4_096)
[truncation] = _truncations(chunks)
assert truncation["fits"] is True
def test_an_ollama_window_above_the_catalogued_one_does_not_raise_it():
_, sent = _proxy(
"ollama",
_long_chat(),
handler = _ollama(262_144),
model = "qwen3:1.7b",
context_overflow = "truncate_oldest",
compaction_threshold = 6_144,
context_window = 8_192,
)
assert len(sent["messages"]) < len(_long_chat())
assert _fits(sent, 8_192)
def test_an_ollama_model_not_loaded_yet_keeps_the_catalogued_window():
idle = _served(ps = [])
chunks, sent = _proxy(
"ollama",
_long_chat(),
handler = idle,
model = "qwen3:1.7b",
context_overflow = "truncate_oldest",
compaction_threshold = 30_720,
context_window = 40_960,
)
assert len(sent["messages"]) == len(_long_chat())
assert _truncations(chunks) == []
assert idle.gets == ["/api/ps"]
def test_an_ollama_model_is_read_again_once_it_has_loaded():
server = _served(ps = [])
windows: list = []
with _Server(handler = server) as upstream:
async def go() -> None:
client = ep_mod.ExternalProviderClient(
provider_type = "ollama", base_url = upstream.base_url, api_key = None
)
windows.append(await client.served_context_window("qwen3:1.7b"))
server.ps = [{"name": "qwen3:1.7b", "context_length": 4_096}]
windows.append(await client.served_context_window("qwen3:1.7b"))
windows.append(await client.served_context_window("qwen3:1.7b"))
_run(go)
assert windows == [None, 4_096, 4_096]
assert server.gets == ["/api/ps", "/api/ps"]
def test_an_idle_ollama_model_keeps_the_window_it_last_loaded(monkeypatch):
monkeypatch.setattr(ep_mod, "_SERVED_WINDOW_TTL_S", 0.0)
server = _ollama()
windows: list = []
with _Server(handler = server) as upstream:
async def go() -> None:
client = ep_mod.ExternalProviderClient(
provider_type = "ollama", base_url = upstream.base_url, api_key = None
)
windows.append(await client.served_context_window("qwen3:1.7b"))
server.ps = []
# another connection's refresh purges expired entries from the TTL cache.
ep_mod._served_windows.clear()
windows.append(await client.served_context_window("qwen3:1.7b"))
_run(go)
assert windows == [4_096, 4_096]
assert server.gets == ["/api/ps", "/api/ps"]
def test_a_failed_ollama_read_keeps_the_window_it_last_loaded(monkeypatch):
monkeypatch.setattr(ep_mod, "_SERVED_WINDOW_TTL_S", 0.0)
server = _ollama()
windows: list = []
with _Server(handler = server) as upstream:
async def go() -> None:
client = ep_mod.ExternalProviderClient(
provider_type = "ollama", base_url = upstream.base_url, api_key = None
)
windows.append(await client.served_context_window("qwen3:1.7b"))
server.ps = None
windows.append(await client.served_context_window("qwen3:1.7b"))
_run(go)
assert windows == [4_096, 4_096]
def test_a_custom_miss_on_the_same_server_does_not_hide_ollamas_window():
server = _ollama()
windows: list = []
with _Server(handler = server) as upstream:
async def go() -> None:
for provider_type in ("custom", "ollama"):
client = ep_mod.ExternalProviderClient(
provider_type = provider_type, base_url = upstream.base_url, api_key = None
)
windows.append(await client.served_context_window("qwen3:1.7b"))
_run(go)
assert windows == [None, 4_096]
@pytest.mark.parametrize(
("base", "probe"),
[
("http://h:11434/v1", "http://h:11434/api/ps"),
("http://h:11434", "http://h:11434/api/ps"),
("https://gw.example/ollama/v1?tenant=a", "https://gw.example/ollama/api/ps?tenant=a"),
],
)
def test_the_ollama_probe_keeps_the_base_path_and_query(base, probe):
assert ep_mod._native_server_path(base, "/api/ps") == probe
def test_a_bare_ollama_model_name_matches_its_latest_tag():
server = _served(
ps = [{"name": "qwen3:latest", "model": "qwen3:latest", "context_length": 4_096}]
)
_, sent = _proxy(
"ollama",
_long_chat(),
handler = server,
model = "qwen3",
context_overflow = "truncate_oldest",
compaction_threshold = 30_720,
context_window = 40_960,
)
assert _fits(sent, 4_096)
def test_auto_compact_off_never_asks_ollama_for_its_window():
server = _ollama()
_, sent = _proxy("ollama", _long_chat(), handler = server, model = "qwen3:1.7b")
assert len(sent["messages"]) == len(_long_chat())
assert server.gets == []
def test_a_tool_enabled_llama_server_chat_compacts_to_the_reported_window(monkeypatch):
async def connected(self) -> bool:
return False
# the tool loop polls for disconnects and would stop before reaching the provider.
monkeypatch.setattr(Request, "is_disconnected", connected)
_, sent = _proxy(
"llama_cpp",
_long_chat(),
handler = _llama_server(),
model = "qwen3-next",
enable_tools = True,
enabled_tools = ["python"],
permission_mode = "off",
context_overflow = "truncate_oldest",
)
assert sent["tools"]
assert len(sent["messages"]) < len(_long_chat())
assert sent["messages"][-1]["content"] == "latest question"
def test_an_unauthorized_read_does_not_hide_the_window_from_a_valid_key():
server = _served(
models = [{"id": "a-model", "owned_by": "vllm", "max_model_len": 8_192}], key = "good"
)
windows: list = []
with _Server(handler = server) as upstream:
async def go() -> None:
for key in ("bad", "good", "bad", "good"):
client = ep_mod.ExternalProviderClient(
provider_type = "vllm", base_url = upstream.base_url, api_key = key
)
windows.append(await client.served_context_window("a-model"))
_run(go)
assert windows == [None, 8_192, None, 8_192]
# each credential is read once, then served from the cache.
assert len(server.gets) == 2
def test_a_refused_connection_is_not_remembered():
# a down or restarting server must be queried again on the next turn.
ep_mod._served_windows.clear()
windows: list = []
async def go() -> None:
client = ep_mod.ExternalProviderClient(
provider_type = "llama_cpp", base_url = "http://127.0.0.1:9/v1", api_key = None
)
windows.append(await client.served_context_window("a-model"))
_run(go)
assert windows == [None]
assert ep_mod._served_windows == {}
def test_a_stalled_server_costs_one_deadline_and_is_remembered(monkeypatch):
ep_mod._served_windows.clear()
monkeypatch.setattr(ep_mod, "_SERVED_WINDOW_TIMEOUT_S", 0.2)
reads: list[str] = []
async def stalled(self, model):
reads.append(model)
await asyncio.sleep(5)
monkeypatch.setattr(ep_mod.ExternalProviderClient, "_read_served_window", stalled)
windows: list = []
async def go() -> None:
for _ in range(2):
client = ep_mod.ExternalProviderClient(
provider_type = "custom", base_url = "http://127.0.0.1:9/v1", api_key = None
)
windows.append(await client.served_context_window("a-model"))
_run(go)
assert windows == [None, None]
assert reads == ["a-model"]
def test_a_checkpoint_turn_the_fit_creates_still_carries_the_date(monkeypatch):
monkeypatch.setattr(
ri, "current_date_prompt_line", lambda **_kwargs: "The current date is 2026-08-15."
)
def fit(
messages,
payload,
tools = None,
**_kwargs,
):
return [{"role": "system", "content": "Earlier: a summary."}, *messages], None, None
monkeypatch.setattr(ri, "_fit_external_context", fit)
_, sent = _proxy(
"vllm",
[{"role": "user", "content": "hi"}],
context_overflow = "truncate_oldest",
compaction_threshold = 6_000,
)
assert sent["messages"][0] == {
"role": "system",
"content": "The current date is 2026-08-15.\n\nEarlier: a summary.",
}
def test_a_failed_template_read_logs_no_url_credentials(monkeypatch):
import httpx
lines = []
monkeypatch.setattr(
ep_mod, "logger", type("L", (), {"info": lambda self, *a: lines.append(a[0] % a[1:])})()
)
monkeypatch.setattr(
ep_mod,
"_client",
lambda: httpx.AsyncClient(
transport = httpx.MockTransport(lambda request: httpx.Response(404, request = request))
),
)
ep_mod._served_templates.clear()
provider = ep_mod.ExternalProviderClient("llama_cpp", "http://user:secret@host/v1", "")
got = []
async def go():
got.append(await provider.served_chat_template("a-model"))
_run(go)
assert got == [None]
assert lines and not any("secret" in line for line in lines)
@pytest.mark.parametrize(
"enable_thinking,reasoning_effort,preserve_thinking",
[
(None, None, None),
(True, None, None),
(False, None, True),
(None, "none", None),
(True, "max", False),
(None, "medium", None),
],
)
def test_the_probed_template_controls_are_the_ones_sent(
enable_thinking, reasoning_effort, preserve_thinking
):
with _Server() as server:
provider = ep_mod.ExternalProviderClient("llama_cpp", server.base_url, "")
async def go():
async for _ in provider.stream_chat_completion(
messages = [{"role": "user", "content": "hi"}],
model = "a-model",
enable_thinking = enable_thinking,
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
):
pass
_run(go)
body = server.bodies[-1]
sent = dict(body.get("chat_template_kwargs") or {})
if "reasoning_effort" in body:
sent["reasoning_effort"] = body["reasoning_effort"]
assert ep_mod.llama_cpp_template_controls(
enable_thinking, reasoning_effort, preserve_thinking
) == tuple(sorted(sent.items()))