Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Min P, repetition penalty and top K forward only when set; local schema defaults are non-None."""
from __future__ import annotations
import ast
import asyncio
import json
import pathlib
import httpx
import pytest
from core.inference import external_provider as ep_mod
from core.inference.external_provider import ExternalProviderClient
from models.inference import ChatCompletionRequest
_ROUTE_SOURCE = pathlib.Path(__file__).resolve().parents[1] / "routes" / "inference.py"
def _capture_body(provider_type: str, **kwargs) -> dict:
captured: dict = {}
def handler(request: httpx.Request) -> httpx.Response:
captured["body"] = json.loads(request.content.decode())
sse = 'data: {"choices":[{"index":0,"delta":{"content":"ok"}}]}\n\ndata: [DONE]\n\n'
return httpx.Response(200, content = sse, headers = {"content-type": "text/event-stream"})
mock_client = httpx.AsyncClient(transport = httpx.MockTransport(handler))
client = ExternalProviderClient(
provider_type = provider_type,
base_url = "http://127.0.0.1:8000/v1",
api_key = "",
)
async def run() -> None:
try:
async for _ in client.stream_chat_completion(
messages = [{"role": "user", "content": "hi"}],
model = "a-model",
**kwargs,
):
pass
finally:
await mock_client.aclose()
event_loop = asyncio.new_event_loop()
previous_client = ep_mod._http_client
ep_mod._http_client = mock_client
try:
event_loop.run_until_complete(run())
finally:
ep_mod._http_client = previous_client
event_loop.close()
return captured["body"]
@pytest.mark.parametrize("provider_type", ["vllm", "openrouter"])
def test_the_sliders_reach_the_server(provider_type):
body = _capture_body(
provider_type,
top_k = 40,
min_p = 0.07,
repetition_penalty = 1.15,
)
assert body["top_k"] == 40
assert body["min_p"] == 0.07
assert body["repetition_penalty"] == 1.15
@pytest.mark.parametrize("provider_type", ["vllm", "openrouter", "custom", "llama_cpp"])
def test_a_request_that_set_none_of_them_sends_none(provider_type):
body = _capture_body(provider_type)
for field in ("top_k", "min_p", "repetition_penalty", "repeat_penalty"):
assert field not in body, field
assert body["temperature"] == 0.7
assert body["top_p"] == 0.95
assert body["presence_penalty"] == 0.0
def test_llama_server_gets_repeat_penalty_not_repetition_penalty():
body = _capture_body("llama_cpp", min_p = 0.07, repetition_penalty = 1.15, top_k = 40)
assert body["repeat_penalty"] == 1.15
assert "repetition_penalty" not in body
assert body["min_p"] == 0.07
assert body["top_k"] == 40
def test_a_zero_value_is_forwarded_rather_than_read_as_unset():
# 0 means "disabled", so a truthiness gate would restore the server default
body = _capture_body("vllm", top_k = 0, min_p = 0.0)
assert body["top_k"] == 0
assert body["min_p"] == 0.0
def test_the_schema_defaults_are_not_none_so_the_route_cannot_test_for_none():
bare = ChatCompletionRequest(messages = [{"role": "user", "content": "hi"}])
assert bare.top_k == 20
assert bare.min_p == 0.01
assert bare.repetition_penalty == 1.0
assert "min_p" not in bare.model_fields_set
assert "repetition_penalty" not in bare.model_fields_set
asked = ChatCompletionRequest(
messages = [{"role": "user", "content": "hi"}],
min_p = 0.0,
repetition_penalty = 1.0,
)
# both equal a default, so only model_fields_set tells them from an omission
assert {"min_p", "repetition_penalty"} <= asked.model_fields_set
def test_the_proxy_hands_the_client_explicit_values_not_schema_defaults():
tree = ast.parse(_ROUTE_SOURCE.read_text(encoding = "utf-8"))
proxy = next(
node
for node in ast.walk(tree)
if isinstance(node, ast.AsyncFunctionDef) and node.name == "_proxy_to_external_provider"
)
kwargs = next(
node
for node in ast.walk(proxy)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "dict"
and any(keyword.arg == "top_k" for keyword in node.keywords)
)
passed = {
keyword.arg: ast.unparse(keyword.value)
for keyword in kwargs.keywords
if keyword.arg in ("top_k", "min_p", "repetition_penalty")
}
assert passed == {
"top_k": "_top_k_explicit",
"min_p": "_min_p_explicit",
"repetition_penalty": "_repetition_penalty_explicit",
}
@pytest.mark.parametrize("provider_type", ["ollama", "custom"])
def test_the_registry_strips_the_three_for_providers_that_cannot_take_them(provider_type):
# Ollama ignores them; a gateway behind a Custom base URL rejects the whole turn
body = _capture_body(
provider_type,
temperature = 0.31,
top_k = 42,
min_p = 0.07,
repetition_penalty = 1.23,
presence_penalty = 0.11,
)
assert "top_k" not in body
assert "min_p" not in body
assert "repetition_penalty" not in body
assert body["temperature"] == 0.31
assert body["presence_penalty"] == 0.11
@pytest.mark.parametrize("provider", ["llama_cpp", "vllm"])
@pytest.mark.parametrize("value", [True, False, None])
def test_llama_preserve_thinking_template_option(provider, value):
body = _capture_body(provider, preserve_thinking = value)
if value is None:
assert "chat_template_kwargs" not in body
else:
assert body["chat_template_kwargs"] == {"preserve_thinking": value}
def test_llama_preserve_thinking_keeps_enable_thinking():
body = _capture_body("llama_cpp", preserve_thinking = True, enable_thinking = False)
assert body["chat_template_kwargs"] == {"preserve_thinking": True, "enable_thinking": False}
@pytest.mark.parametrize("provider", ["custom", "ollama", "openrouter"])
def test_preserve_thinking_does_not_leak_to_other_providers(provider):
body = _capture_body(provider, preserve_thinking = True)
assert "preserve_thinking" not in body.get("chat_template_kwargs", {})