Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import asyncio
import json
import threading
import time
from datetime import date, datetime, timezone
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from starlette.requests import Request
from core.inference import chat_generation_runs, llama_keepwarm
from core.inference.chat_generation_runs import (
_EVENT_FLUSH_SECONDS,
ChatGenerationSupervisor,
)
from models.inference import ChatCompletionRequest
from routes import chat_generation_runs as run_routes
from routes import inference
from state import active_generations
from storage import chat_generation_runs_db as runs_db
from storage import studio_db
from utils.current_date_prompt_settings import _request_local_date
@pytest.fixture
def durable_run(request):
engine = getattr(request, "param", "gguf")
model = "local.gguf" if engine == "gguf" else "local.safetensors"
studio_db.upsert_chat_thread(
{"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": model, "createdAt": 1}
)
studio_db.upsert_chat_message(
{
"id": "user-1",
"threadId": "thread-1",
"role": "user",
"content": [{"type": "text", "text": "Hello"}],
"createdAt": 2,
}
)
run, _created = runs_db.create_run(
run_id = "run-1",
owner_subject = "alice",
thread_id = "thread-1",
user_message_id = "user-1",
assistant_message_id = "assistant-1",
request_payload = {
"model": model,
"messages": [{"role": "user", "content": "Hello"}],
"stream": True,
"cancel_id": "run-1",
"thread_id": "thread-1",
"generation_run_id": "run-1",
},
)
active_generations.reset_for_tests()
yield run
active_generations.reset_for_tests()
def _create_payload(content = "Hello"):
return run_routes.CreateChatGenerationRun(
runId = "run-1",
threadId = "thread-1",
userMessageId = "user-1",
assistantMessageId = "assistant-1",
requestPayload = {"model": "local", "messages": [{"role": "user", "content": content}]},
)
def _route_request(supervisor):
return SimpleNamespace(
app = SimpleNamespace(state = SimpleNamespace(chat_generation_supervisor = supervisor)),
headers = {},
)
@pytest.mark.asyncio
@pytest.mark.parametrize("scope", ["off", "full"])
@pytest.mark.parametrize(
"caller,expected",
[("session", False), ("keyless", True), ("api-key", True), ("workflow", False)],
)
async def test_durable_producer_preserves_monitor_origin(monkeypatch, scope, caller, expected):
from auth import policy, storage
from utils import keyless_api_access as keyless
studio_db.upsert_chat_thread(
{"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}
)
studio_db.upsert_chat_message(
{"id": "user-1", "threadId": "thread-1", "role": "user", "content": [], "createdAt": 2}
)
app = SimpleNamespace(state = SimpleNamespace(bind_host = "127.0.0.1"))
headers = (
[]
if caller == "keyless"
else [
(
b"authorization",
b"Bearer session-jwt" if caller == "session" else b"Bearer sk-unsloth-test",
)
]
)
request = Request(
{
"type": "http",
"method": "POST",
"path": "/api/inference/chat-runs",
"headers": headers,
"app": app,
"client": ("127.0.0.1", 5000),
"server": ("127.0.0.1", 8000),
}
)
keyless.mark_keyless_admission(request, caller == "keyless")
monkeypatch.setattr(policy, "installation_has_managed_accounts", lambda: False)
monkeypatch.setattr(storage, "is_internal_api_key", lambda _token: caller == "workflow")
run = await run_routes.create_chat_generation_run(_create_payload(), request, "alice")
assert run["requestPayload"][runs_db.API_MONITOR_ORIGIN_FIELD] is expected
request.state.api_monitor_via_api_key = not expected
retry = await run_routes.create_chat_generation_run(_create_payload(), request, "alice")
assert retry["created"] is False
assert retry["requestPayload"][runs_db.API_MONITOR_ORIGIN_FIELD] is expected
monkeypatch.setattr(keyless, "get_keyless_api_access_scope", lambda: scope)
observed = []
async def body():
yield "data: [DONE]\n\n"
async def fake(payload, background, _subject, *, cancel_on_disconnect):
observed.append(
(
inference._request_used_api_key(background),
runs_db.API_MONITOR_ORIGIN_FIELD in payload.model_extra,
"authorization" in background.headers,
)
)
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
await ChatGenerationSupervisor(app)._produce("run-1")
assert observed == [(expected, False, False)]
def test_monitor_origin_is_not_part_of_retry_identity():
args = dict(thread_id = "t", user_message_id = "u", assistant_message_id = "a")
payload = {"model": "local"}
_, original = runs_db.canonical_request(**args, request_payload = payload)
for origin in (False, True):
encoded, identity = runs_db.canonical_request(
**args, request_payload = {**payload, runs_db.API_MONITOR_ORIGIN_FIELD: origin}
)
assert identity == original
assert json.loads(encoded)[runs_db.API_MONITOR_ORIGIN_FIELD] is origin
def test_client_cannot_supply_monitor_origin():
from fastapi import HTTPException
payload = _create_payload()
payload.requestPayload[runs_db.API_MONITOR_ORIGIN_FIELD] = False
with pytest.raises(HTTPException) as exc:
run_routes._sanitize_request(payload)
assert exc.value.status_code == 400
@pytest.mark.asyncio
async def test_public_chat_wrapper_keeps_cancel_on_disconnect(monkeypatch):
observed = []
async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
observed.append(cancel_on_disconnect)
return "response"
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
payload = ChatCompletionRequest(
model = "local",
messages = [{"role": "user", "content": "Hello"}],
)
assert await inference.openai_chat_completions(payload, object(), "alice") == "response"
assert observed == [True]
@pytest.mark.parametrize(
"is_mlx,durable_run,generation_run_id,completion_tokens,expected",
[
(True, True, "run-1", 8, "length"),
(True, True, "run-1", 7, "stop"),
(True, False, "run-1", 8, "stop"),
(False, True, "run-1", 8, "stop"),
],
)
def test_only_durable_mlx_normalizes_stop_at_token_cap(
is_mlx, durable_run, generation_run_id, completion_tokens, expected
):
payload = SimpleNamespace(
generation_run_id = generation_run_id,
max_tokens = 8,
max_completion_tokens = None,
)
stats = {"usage": {"completion_tokens": completion_tokens}}
assert (
inference._safetensors_finish_reason(stats, payload, is_mlx = is_mlx, durable_run = durable_run)
== expected
)
@pytest.mark.asyncio
async def test_create_route_schedules_producer_on_request_loop(monkeypatch):
started = []
supervisor = SimpleNamespace(
start = lambda run_id, **identity: started.append((run_id, identity))
)
request = _route_request(supervisor)
def create_run(**_kwargs):
assert started == []
return (
{
"id": "run-1",
"status": "queued",
"threadId": "thread-1",
"requestPayload": {"model": "local"},
},
True,
)
monkeypatch.setattr(run_routes.db, "create_run", create_run)
response = await run_routes.create_chat_generation_run(_create_payload(), request, "alice")
assert response["created"] is True
assert started == [("run-1", {"thread_id": "thread-1", "model": "local", "images": {}})]
@pytest.mark.asyncio
async def test_terminal_idempotent_create_does_not_reserve_generation(monkeypatch):
terminal = {
"id": "run-1",
"status": "completed",
"threadId": "thread-1",
"requestPayload": {"model": "local"},
}
supervisor = SimpleNamespace(
start = lambda *_args, **_kwargs: pytest.fail("terminal run must not start"),
)
request = _route_request(supervisor)
monkeypatch.setattr(run_routes.db, "create_run", lambda **_kwargs: (terminal, False))
response = await run_routes.create_chat_generation_run(_create_payload("Hi"), request, "alice")
assert response["created"] is False
@pytest.mark.asyncio
async def test_background_producer_persists_chunks_and_completes(durable_run, monkeypatch):
observed = []
leaked = []
chunks = [
{"choices": [{"delta": {"content": "Hello"}, "finish_reason": None}]},
{"choices": [{"delta": {}, "finish_reason": "stop"}]},
]
async def body():
for chunk in chunks:
yield f"data: {json.dumps(chunk)}\n\n"
yield "data: [DONE]\n\n"
async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
observed.append(cancel_on_disconnect)
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
loop = asyncio.get_running_loop()
previous_handler = loop.get_exception_handler()
loop.set_exception_handler(lambda _loop, context: leaked.append(context))
try:
await supervisor._produce("run-1")
await asyncio.sleep(0)
finally:
loop.set_exception_handler(previous_handler)
run = runs_db.get_run("run-1", "alice")
assert leaked == []
assert observed == [False]
assert (run["status"], run["finishReason"]) == ("completed", "stop")
assert [
event["payload"] for event in runs_db.list_events("run-1") if event["type"] == "chunk"
] == chunks
@pytest.mark.asyncio
@pytest.mark.parametrize(
"finish, cut, expected",
[
("stop", True, {"reason": "quote_cut"}),
("stop", False, None),
("length", True, {"reason": "length"}),
],
ids = ["cut", "clean-stop", "length-wins"],
)
async def test_producer_stamps_a_reported_quote_cut(
durable_run, monkeypatch, finish, cut, expected
):
"""The producer must persist the reason before the client can settle with it."""
chunks = [
{"choices": [{"delta": {"reasoning_content": "The tokens are `"}, "finish_reason": None}]},
*([{"choices": [], "quote_cut": True}] if cut else []),
{"choices": [{"delta": {}, "finish_reason": finish}]},
]
async def body():
for chunk in chunks:
yield f"data: {json.dumps(chunk)}\n\n"
yield "data: [DONE]\n\n"
async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
await supervisor._produce("run-1")
run = runs_db.get_run("run-1", "alice")
assert (run["status"], run["finishReason"]) == ("completed", finish)
message = studio_db.get_chat_message("thread-1", "assistant-1")
assert message["metadata"].get("incomplete") == expected
@pytest.mark.asyncio
async def test_producer_dates_the_prompt_in_the_browser_timezone(monkeypatch):
studio_db.upsert_chat_thread(
{"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}
)
studio_db.upsert_chat_message(
{"id": "user-1", "threadId": "thread-1", "role": "user", "content": [], "createdAt": 2}
)
browser = Request(
{
"type": "http",
"method": "POST",
"path": "/api/inference/chat-runs",
"query_string": b"",
"headers": [
(b"x-unsloth-timezone", b"Pacific/Kiritimati"),
(b"x-unsloth-timezone-offset-minutes", b"-840"),
],
"app": SimpleNamespace(state = SimpleNamespace()),
}
)
await run_routes.create_chat_generation_run(_create_payload(), browser, "alice")
instant = datetime(2026, 9, 24, 10, 5, tzinfo = timezone.utc)
observed = []
async def body():
yield "data: [DONE]\n\n"
async def fake(payload, request, _subject, *, cancel_on_disconnect):
observed.append((sorted(payload.model_extra), _request_local_date(request, instant)))
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
await supervisor._produce("run-1")
assert observed == [(["generation_run_id"], date(2026, 9, 25))]
@pytest.mark.asyncio
async def test_a_follow_up_reaches_the_model_with_an_image_the_run_never_stored(monkeypatch):
studio_db.upsert_chat_thread(
{"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}
)
studio_db.upsert_chat_message(
{"id": "user-1", "threadId": "thread-1", "role": "user", "content": [], "createdAt": 2}
)
url = "data:image/png;base64,iVBORw0KGgo="
image_turn = [
{"type": "text", "text": "what?"},
{"type": "image_url", "image_url": {"url": url}},
]
payload = _create_payload("and the colour?")
payload.requestPayload["messages"] = [
{"role": "user", "content": image_turn},
{"role": "assistant", "content": "A square."},
*payload.requestPayload["messages"],
]
observed = []
async def body():
yield "data: [DONE]\n\n"
async def fake(payload, _request, _subject, *, cancel_on_disconnect):
observed.append(payload.messages[0].content[1].image_url.url)
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
await run_routes.create_chat_generation_run(payload, _route_request(supervisor), "alice")
await supervisor._tasks["run-1"]
assert observed == [url]
assert url not in json.dumps(runs_db.get_run("run-1")["requestPayload"])
assert supervisor._detached_images == {}
@pytest.mark.asyncio
async def test_a_create_retried_across_a_dst_change_returns_the_committed_run():
studio_db.upsert_chat_thread(
{"id": "thread-1", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}
)
studio_db.upsert_chat_message(
{"id": "user-1", "threadId": "thread-1", "role": "user", "content": [], "createdAt": 2}
)
def browser(offset):
request = _route_request(None)
request.headers = {
"x-unsloth-timezone": "America/Los_Angeles",
"x-unsloth-timezone-offset-minutes": offset,
}
return request
first = await run_routes.create_chat_generation_run(_create_payload(), browser("420"), "alice")
retry = await run_routes.create_chat_generation_run(_create_payload(), browser("480"), "alice")
assert (first["created"], retry["created"]) == (True, False)
assert retry["requestPayload"]["timezone_headers"]["x-unsloth-timezone-offset-minutes"] == "420"
@pytest.mark.asyncio
async def test_a_prefill_reporting_only_progress_renews_the_lease(durable_run, monkeypatch):
"""A 250K prefill outruns the 1200s lease before its first token and only progress writes renew
it, so dropping content-less chunks would reap a healthy prefill."""
released = asyncio.Event()
sampled: dict = {}
def _progress(processed):
return {
"choices": [{"delta": {"role": "assistant", "content": None}, "finish_reason": None}],
"prompt_progress": {
"total": 250000,
"cache": 0,
"processed": processed,
"time_ms": processed,
},
}
async def body():
for processed in (1024, 8192, 65536):
yield f"data: {json.dumps(_progress(processed))}\n\n"
# Polled, not slept: a sleep sized against the flush timer flakes under load.
_deadline = time.monotonic() + 10.0
while time.monotonic() < _deadline:
sampled["events"] = [
e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"
]
if len(sampled["events"]) >= 3:
break
await asyncio.sleep(0.01)
sampled["progress"] = runs_db.get_progress("run-1")
released.set()
yield f"data: {json.dumps({'choices': [{'delta': {'content': 'Hi'}, 'finish_reason': 'stop'}]})}\n\n"
yield "data: [DONE]\n\n"
async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
await supervisor._produce("run-1")
assert released.is_set()
assert len(sampled["events"]) == 3, sampled["events"]
assert all("prompt_progress" in e for e in sampled["events"])
assert sampled["progress"][1] == 3, sampled["progress"]
assert runs_db.get_run("run-1", "alice")["status"] == "completed"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"comment, reaped",
[
("_OPENAI_TOOL_HEARTBEAT_SSE", []),
("_OPENAI_PREFILL_PROGRESS_SSE", []),
("_OPENAI_PASSTHROUGH_SSE_KEEPALIVE", ["run-1"]),
],
)
async def test_progress_comments_hold_the_lease_but_a_stalled_stream_does_not(
durable_run, monkeypatch, comment, reaped
):
comment = getattr(inference, comment)
now = {"ms": runs_db.now_ms()}
monkeypatch.setattr(runs_db, "now_ms", lambda: now["ms"])
sampled = {}
async def body():
now["ms"] += 21 * 60_000
yield comment
yield comment
sampled["reaped"] = runs_db.reconcile_runs(stale_after_ms = 1_200_000)
yield "data: [DONE]\n\n"
async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
monkeypatch.setattr(chat_generation_runs, "_renew_interval_seconds", lambda: 0.0)
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
await supervisor._produce("run-1")
assert sampled["reaped"] == reaped
@pytest.mark.asyncio
@pytest.mark.parametrize("tools", [False, True], ids = ["chat", "tool-loop"])
@pytest.mark.parametrize("reports_progress, reaped", [(True, []), (False, ["run-1"])])
async def test_native_gguf_prefill_progress_holds_the_lease(
durable_run, monkeypatch, tools, reports_progress, reaped
):
"""Native GGUF generators are silent until the first token; advancing progress must still renew the lease."""
from core.inference.llama_cpp import _report_live_llama_timings
entered, release = threading.Event(), threading.Event()
def _prefill(perf_callback):
entered.set()
processed = 0
while not release.wait(0.005):
if reports_progress:
processed += 512
_report_live_llama_timings(
perf_callback,
{"choices": [], "prompt_progress": {"total": 10**6, "processed": processed}},
)
def generate(*, perf_callback = None, **_kwargs):
_prefill(perf_callback)
yield "Hi"
yield {"type": "metadata", "finish_reason": "stop"}
def generate_with_tools(*, perf_callback = None, **_kwargs):
_prefill(perf_callback)
yield {"type": "content", "text": "Hi"}
yield {"type": "metadata", "finish_reason": "stop"}
llama = SimpleNamespace(
is_loaded = True,
model_identifier = "local.gguf",
base_url = "http://llama.test",
effective_parallel_slots = 1,
supports_tools = tools,
supports_tool_passthrough = tools,
is_vision = False,
_is_audio = False,
context_length = None,
generate_chat_completion = generate,
generate_chat_completion_with_tools = generate_with_tools,
release_idle_chat_slot = lambda *_a, **_k: False,
count_chat_tokens = lambda *_a, **_k: 16,
_request_reasoning_kwargs = lambda *_a, **_k: None,
)
monkeypatch.setattr(inference, "get_llama_cpp_backend", lambda: llama)
monkeypatch.setattr(inference, "_automatic_model_load_may_run", lambda: False)
async def no_switch(*_a, **_k):
return None
monkeypatch.setattr(inference, "_maybe_auto_switch_model", no_switch)
monkeypatch.setattr(inference, "_LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S", 0.02)
if not tools:
monkeypatch.setattr(inference, "_effective_enable_tools", lambda _payload: False)
now = {"ms": runs_db.now_ms()}
monkeypatch.setattr(runs_db, "now_ms", lambda: now["ms"])
real_produce = inference.produce_openai_chat_completions
sampled = {"comments": set()}
async def wrapped(payload, request, subject, *, cancel_on_disconnect):
if tools:
payload.enable_tools = True
payload.enabled_tools = ["web_search"]
response = await real_produce(
payload, request, subject, cancel_on_disconnect = cancel_on_disconnect
)
inner = response.body_iterator
monkeypatch.setattr(chat_generation_runs, "_renew_interval_seconds", lambda: 0.0)
async def body():
comments = 0
chunks = 0
async for raw in inner:
text = raw.decode() if isinstance(raw, bytes) else str(raw)
chunks += text.count("data: {")
if text.startswith(":") and entered.is_set():
comments += 1
sampled["comments"].add(text.strip())
if comments == 2:
# Commits run beside the stream now, so let the chunks read so far land before the clock
# jumps; a slow runner would otherwise stamp them as progress made after it.
await _await_chunk_payloads("run-1", chunks, 10.0)
now["ms"] += 21 * 60_000
elif comments == 5 and "reaped" not in sampled:
sampled["reaped"] = runs_db.reconcile_runs(stale_after_ms = 1_200_000)
release.set()
yield raw
response.body_iterator = body()
return response
monkeypatch.setattr(inference, "produce_openai_chat_completions", wrapped)
try:
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
await asyncio.wait_for(supervisor._produce("run-1"), 30)
finally:
release.set()
assert sampled["reaped"] == reaped
expected = ": prefill-progress" if reports_progress else ": keep-alive"
assert expected in sampled["comments"], sampled["comments"]
if not reports_progress:
assert ": prefill-progress" not in sampled["comments"]
def test_prefill_signal_counts_only_moving_progress():
from core.inference.llama_cpp import _perf_callback_wants_timings
forwarded = []
bare = inference._PrefillProgressSignal(None)
assert _perf_callback_wants_timings(bare) is False
assert bare.needs_phase is False
assert bare.advanced() is False
bare({"prompt_progress": {"processed": 10}})
bare({"prompt_progress": {"processed": 10}})
assert bare.advanced() is True
assert bare.advanced() is False
bare({"prompt_progress": {"processed": 2}})
assert bare.advanced() is True
bare({"timings": {"predicted_n": 1}})
assert bare.advanced() is False
def monitor(sample):
forwarded.append(sample)
monitor.needs_phase = False
wrapped = inference._PrefillProgressSignal(monitor)
assert _perf_callback_wants_timings(wrapped) is True
assert wrapped.needs_phase is False
wrapped({"prompt_progress": {"processed": 1}})
assert forwarded == [{"prompt_progress": {"processed": 1}}]
async def _subscriber_sequences(after = 0):
response = await run_routes.chat_generation_events(
"run-1",
SimpleNamespace(is_disconnected = AsyncMock(return_value = True)),
after = after,
last_event_id = None,
current_subject = "alice",
)
raw = ""
async for part in response.body_iterator:
raw += part.decode() if isinstance(part, bytes) else part
return [int(line[4:]) for line in raw.splitlines() if line.startswith("id: ")]
def _route_engine(monkeypatch, model, first, release):
gguf = model.endswith(".gguf")
def generate(*, stats_holder = None, **_kwargs):
first.set()
yield "A"
assert release.wait(5)
yield "AB"
if stats_holder is None:
yield {"type": "metadata", "finish_reason": "stop"}
else:
stats_holder["stats"] = {"usage": {"completion_tokens": 2}}
llama = SimpleNamespace(
is_loaded = gguf,
model_identifier = model,
base_url = "http://llama.test",
effective_parallel_slots = 1,
supports_tools = False,
is_vision = False,
_is_audio = False,
context_length = None,
generate_chat_completion = generate,
)
mlx = SimpleNamespace(
active_model_name = model,
models = {model: {"is_mlx": True, "chat_template_info": {"template": "chatml"}}},
generate_chat_response = generate,
reset_generation_state = lambda *_a, **_k: None,
)
monkeypatch.setattr(inference, "get_llama_cpp_backend", lambda: llama)
monkeypatch.setattr(inference, "get_inference_backend", lambda: mlx)
monkeypatch.setattr(inference, "_automatic_model_load_may_run", lambda: False)
monkeypatch.setattr(
inference, "_detect_safetensors_features", lambda *_a, **_k: {"supports_tools": False}
)
monkeypatch.setattr(inference, "_effective_enable_tools", lambda _payload: False)
async def no_switch(*_args, **_kwargs):
return None
monkeypatch.setattr(inference, "_maybe_auto_switch_model", no_switch)
@pytest.mark.asyncio
@pytest.mark.parametrize("durable_run", ["gguf", "mlx"], indirect = True)
async def test_subscribers_detach_then_replay_the_same_engine_run(durable_run, monkeypatch):
first_chunk, release = threading.Event(), threading.Event()
_route_engine(monkeypatch, durable_run["requestPayload"]["model"], first_chunk, release)
task = asyncio.create_task(
ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))._produce("run-1")
)
if not await asyncio.to_thread(first_chunk.wait, 5):
await task
pytest.fail(str(runs_db.get_run("run-1", "alice")))
while len(runs_db.list_events("run-1")) < 4:
await asyncio.sleep(0.01)
first, second = await asyncio.gather(_subscriber_sequences(), _subscriber_sequences())
assert first == second == list(range(1, max(first) + 1))
assert runs_db.get_run("run-1", "alice")["status"] == "running"
release.set()
await task
tail = await _subscriber_sequences(after = max(first))
run = runs_db.get_run("run-1", "alice")
assert first + tail == list(range(1, run["lastEventSeq"] + 1))
deltas = [
event["payload"].get("choices", [{}])[0].get("delta", {}).get("content")
for event in runs_db.list_events("run-1")
if event["type"] == "chunk"
]
assert [text for text in deltas if text] == ["A", "B"]
async def _await_chunk_payloads(run_id: str, count: int, deadline_s: float) -> list:
"""Returns chunk payloads once count are durable or the deadline passes; the caller asserts."""
started = time.monotonic()
while True:
stored = [e["payload"] for e in runs_db.list_events(run_id) if e["type"] == "chunk"]
if len(stored) >= count or time.monotonic() - started >= deadline_s:
return stored
await asyncio.sleep(0.005)
@pytest.mark.asyncio
async def test_event_batch_flushes_while_upstream_is_idle(durable_run, monkeypatch):
release = asyncio.Event()
chunks = [
{"choices": [{"delta": {"role": "assistant"}}]},
{"choices": [{"delta": {"content": "Hello"}}]},
]
async def body():
for chunk in chunks:
yield f"data: {json.dumps(chunk)}\n\n"
await release.wait()
yield "data: [DONE]\n\n"
async def fake(*_args, **_kwargs):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
task = asyncio.create_task(supervisor._produce("run-1"))
stored = await _await_chunk_payloads("run-1", len(chunks), 0.5)
assert stored == chunks
release.set()
await task
def test_event_flush_interval_is_one_display_frame():
assert _EVENT_FLUSH_SECONDS <= 1 / 60
@pytest.mark.asyncio
async def test_a_lone_chunk_is_appended_without_waiting_for_another(durable_run, monkeypatch):
release = asyncio.Event()
chunk = {"choices": [{"delta": {"content": "Hello"}}]}
async def body():
yield f"data: {json.dumps(chunk)}\n\n"
await release.wait()
yield "data: [DONE]\n\n"
async def fake(*_args, **_kwargs):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
task = asyncio.create_task(supervisor._produce("run-1"))
stored = await _await_chunk_payloads("run-1", 1, 0.5)
assert stored == [chunk]
release.set()
await task
def _numbered_chunks(count):
return [{"choices": [{"delta": {"content": f"t{i} "}}]} for i in range(count)]
def _held_append(monkeypatch):
"""Patch append_events so every commit blocks until `gate` is set. Returns (entered, gate)."""
entered = threading.Event()
gate = threading.Event()
original = runs_db.append_events
def held(run_id, worker_token, events):
entered.set()
assert gate.wait(10)
return original(run_id, worker_token, events)
monkeypatch.setattr(runs_db, "append_events", held)
return entered, gate
@pytest.mark.asyncio
async def test_a_slow_commit_does_not_stop_the_stream_being_read(durable_run, monkeypatch):
chunks = _numbered_chunks(50)
drained = asyncio.Event()
release = asyncio.Event()
async def body():
yield f"data: {json.dumps(chunks[0])}\n\n"
# The rest is only offered once the first commit is under way and held.
assert await asyncio.to_thread(entered.wait, 2)
for chunk in chunks[1:]:
yield f"data: {json.dumps(chunk)}\n\n"
drained.set()
await release.wait()
yield "data: [DONE]\n\n"
async def fake(*_args, **_kwargs):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
entered, gate = _held_append(monkeypatch)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
task = asyncio.create_task(supervisor._produce("run-1"))
try:
await asyncio.wait_for(drained.wait(), timeout = 2)
assert not gate.is_set()
finally:
gate.set()
release.set()
await task
stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
assert stored == chunks
assert runs_db.get_run("run-1", "alice")["status"] == "completed"
@pytest.mark.asyncio
async def test_slow_commits_batch_a_burst_instead_of_one_chunk_each(durable_run, monkeypatch):
chunks = _numbered_chunks(400)
calls = []
original = runs_db.append_events
def slow(run_id, worker_token, events):
calls.append(len(events))
time.sleep(0.04)
return original(run_id, worker_token, events)
async def body():
for chunk in chunks:
yield f"data: {json.dumps(chunk)}\n\n"
yield "data: [DONE]\n\n"
async def fake(*_args, **_kwargs):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
monkeypatch.setattr(runs_db, "append_events", slow)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
await supervisor._produce("run-1")
stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
assert stored == chunks
assert len(calls) <= 5, calls
assert runs_db.get_run("run-1", "alice")["status"] == "completed"
@pytest.mark.asyncio
async def test_a_failed_commit_keeps_its_events_and_fails_the_run(durable_run, monkeypatch):
chunks = _numbered_chunks(3)
def broken(*_args, **_kwargs):
raise RuntimeError("database is locked")
async def body():
for chunk in chunks:
yield f"data: {json.dumps(chunk)}\n\n"
yield "data: [DONE]\n\n"
async def fake(*_args, **_kwargs):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
monkeypatch.setattr(runs_db, "append_events", broken)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
await supervisor._produce("run-1")
run = runs_db.get_run("run-1", "alice")
assert (run["status"], run["finishReason"], run["error"]) == (
"failed",
"error",
"database is locked",
)
stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
assert stored and stored == chunks[: len(stored)]
@pytest.mark.asyncio
@pytest.mark.parametrize("cancels", [1, 2], ids = ["cancel", "cancel-again-while-settling"])
async def test_cancel_during_a_commit_waits_for_it_before_settling(
durable_run, monkeypatch, cancels
):
chunks = _numbered_chunks(20)
leaked = []
drained = asyncio.Event()
async def body():
for chunk in chunks:
yield f"data: {json.dumps(chunk)}\n\n"
drained.set()
await asyncio.Event().wait()
async def fake(*_args, **_kwargs):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
entered, gate = _held_append(monkeypatch)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
loop = asyncio.get_running_loop()
previous_handler = loop.get_exception_handler()
loop.set_exception_handler(lambda _loop, context: leaked.append(context))
cancel_event = threading.Event()
task = asyncio.create_task(supervisor._produce("run-1", cancel_event))
try:
assert await asyncio.to_thread(entered.wait, 2)
await asyncio.wait_for(drained.wait(), timeout = 2)
cancel_event.set()
task.cancel()
# A second cancel (shutdown's forced one) lands while settlement waits on the commit; the run must still settle.
for _ in range(cancels - 1):
loop.call_later(0.05, task.cancel)
# finish_run must not overtake the commit still holding its batch.
loop.call_later(0.1, gate.set)
with pytest.raises(asyncio.CancelledError):
await task
await asyncio.sleep(0)
finally:
gate.set()
loop.set_exception_handler(previous_handler)
assert leaked == []
assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"
stored = [e["payload"] for e in runs_db.list_events("run-1") if e["type"] == "chunk"]
assert stored == chunks
@pytest.mark.asyncio
async def test_model_lifecycle_cancel_reaches_same_registered_event(durable_run, monkeypatch):
registered = asyncio.Event()
async def body(cancel_event):
with active_generations.ActiveGeneration(
cancel_event,
thread_id = "thread-1",
run_id = "run-1",
):
registered.set()
while not cancel_event.is_set():
await asyncio.sleep(0.01)
if False:
yield ""
async def fake(_payload, request, *_args, **_kwargs):
return SimpleNamespace(
status_code = 200,
body_iterator = body(request.state.generation_cancel_event),
)
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
task = asyncio.create_task(supervisor._produce("run-1"))
await asyncio.wait_for(registered.wait(), timeout = 2)
assert active_generations.cancel_all() == 1
await asyncio.wait_for(task, timeout = 2)
assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"
metadata = studio_db.get_chat_message("thread-1", "assistant-1")["metadata"]
assert metadata["incomplete"] == {"reason": "cancelled"}
assert active_generations.count() == 0
@pytest.mark.asyncio
async def test_cancel_before_registration_signals_load_event(durable_run, monkeypatch):
entered = asyncio.Event()
cancel_ids = []
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
async def fake(_payload, request, *_args, **_kwargs):
event = request.state.generation_cancel_event
entered.set()
while not event.is_set():
await asyncio.sleep(0.01)
async def body():
if False:
yield ""
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
monkeypatch.setattr(
inference,
"_cancel_by_cancel_id_or_stash",
lambda run_id: cancel_ids.append(run_id) or 0,
)
supervisor.start("run-1")
await asyncio.wait_for(entered.wait(), timeout = 2)
supervisor.cancel("run-1")
await asyncio.wait_for(supervisor._tasks["run-1"], timeout = 2)
assert cancel_ids == ["run-1"]
assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"
@pytest.mark.asyncio
async def test_start_reserves_slot_and_lifecycle_before_worker_runs(durable_run, monkeypatch):
monkeypatch.setattr(llama_keepwarm, "_pending", 0)
monkeypatch.setattr(llama_keepwarm, "_inflight", 0)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
supervisor.start("run-1", thread_id = "thread-1", model = "local")
assert (llama_keepwarm._pending, active_generations.count()) == (1, 1)
assert active_generations.snapshot()[0]["thread_id"] == "thread-1"
supervisor.cancel("run-1")
await asyncio.wait_for(supervisor._tasks["run-1"], timeout = 2)
assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"
assert (llama_keepwarm._pending, llama_keepwarm._inflight) == (0, 0)
@pytest.mark.asyncio
async def test_cancelled_producer_error_is_cancelled(durable_run, monkeypatch):
entered = asyncio.Event()
async def fake(_payload, request, *_args, **_kwargs):
entered.set()
while not request.state.generation_cancel_event.is_set():
await asyncio.sleep(0.01)
raise RuntimeError("Generation cancelled")
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
supervisor.start("run-1")
await asyncio.wait_for(entered.wait(), timeout = 2)
assert active_generations.cancel_all() == 1
await asyncio.wait_for(supervisor._tasks["run-1"], timeout = 2)
assert runs_db.get_run("run-1", "alice")["status"] == "cancelled"
@pytest.mark.asyncio
async def test_uncancelled_partial_eof_is_interrupted(durable_run, monkeypatch):
async def body():
yield 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'
async def fake(*_args, **_kwargs):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
await ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))._produce("run-1")
run = runs_db.get_run("run-1", "alice")
assert (run["status"], run["finishReason"]) == ("failed", "interrupted")
@pytest.mark.asyncio
async def test_a_caught_up_reconnect_to_a_settled_run_does_not_block(durable_run, monkeypatch):
"""Nothing left to replay must return at once, not after the 15s event wait; otherwise a
finished answer reads as still generating and holds an event-wait worker for the timeout."""
async def body():
yield 'data: {"choices":[{"delta":{"content":"done"},"finish_reason":"stop"}]}\n\n'
yield "data: [DONE]\n\n"
async def fake(*_args, **_kwargs):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
await ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))._produce("run-1")
settled = runs_db.get_run("run-1", "alice")
assert settled["status"] == "completed"
caught_up = await asyncio.wait_for(
_subscriber_sequences(after = int(settled["lastEventSeq"])),
timeout = 5,
)
assert caught_up == []
assert await _subscriber_sequences() == list(range(1, int(settled["lastEventSeq"]) + 1))
@pytest.mark.asyncio
async def test_streamed_error_outranks_cleanup_cancellation(durable_run, monkeypatch):
"""A backend failure must not be recorded as if the user pressed Stop.
``gguf_stream_chunks`` emits the error in band, follows it with ``[DONE]`` and then
sets this same ``cancel_event`` from its ``finally`` because the stream did not
complete. Nobody asked to cancel, so the run has to settle as ``failed`` carrying
the diagnostic the user needs to act on.
"""
async def body(cancel_event):
try:
yield 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'
yield 'data: {"error": {"message": "Out of memory"}}\n\ndata: [DONE]\n\n'
finally:
cancel_event.set()
async def fake(_payload, request, *_args, **_kwargs):
return SimpleNamespace(
status_code = 200,
body_iterator = body(request.state.generation_cancel_event),
)
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
await ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))._produce("run-1")
run = runs_db.get_run("run-1", "alice")
assert run["status"] == "failed"
assert run["error"] == "Out of memory"
assert run["finishReason"] != "cancelled"
metadata = studio_db.get_chat_message("thread-1", "assistant-1")["metadata"]
assert metadata["incomplete"] != {"reason": "cancelled"}
@pytest.mark.asyncio
async def test_graceful_supervisor_shutdown_is_interrupted(durable_run, monkeypatch):
entered = asyncio.Event()
async def body(request):
entered.set()
while not request.state.generation_cancel_event.is_set():
await asyncio.sleep(0.01)
if False:
yield ""
async def fake(_payload, request, *_args, **_kwargs):
return SimpleNamespace(status_code = 200, body_iterator = body(request))
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
supervisor.start("run-1")
await asyncio.wait_for(entered.wait(), timeout = 2)
await supervisor.stop()
run = runs_db.get_run("run-1", "alice")
assert (run["status"], run["finishReason"]) == ("failed", "interrupted")
assert run["error"] == "Unsloth shut down during generation"
def test_thread_delete_captures_durable_run_before_cascade(durable_run):
research_ids, chat_ids = studio_db.delete_chat_threads_with_active_runs(["thread-1"])
assert research_ids == []
assert chat_ids == ["run-1"]
assert runs_db.get_run("run-1", "alice") is None
def test_project_delete_captures_durable_run_before_cascade(durable_run):
studio_db.upsert_chat_project(
{"id": "project-1", "name": "Project", "createdAt": 1, "updatedAt": 1}
)
studio_db.update_chat_thread("thread-1", {"projectId": "project-1"})
deleted = studio_db.delete_chat_project("project-1")
assert deleted["activeChatGenerationRunIds"] == ["run-1"]
def test_clear_captures_durable_run_before_cascade(durable_run):
removed, research_ids, chat_ids = studio_db.clear_chat_history(
include_chat_generation_runs = True
)
assert (removed, research_ids, chat_ids) == (["thread-1"], [], ["run-1"])
def test_startup_reconcile_marks_stored_assistant_interrupted(durable_run):
worker_token = runs_db.get_worker_token("run-1")
assert runs_db.mark_running("run-1", worker_token)
assert runs_db.reconcile_orphaned_runs() == 1
message = studio_db.get_chat_message("thread-1", "assistant-1")
assert message["metadata"]["generationStatus"] == "failed"
assert message["metadata"]["incomplete"] == {"reason": "interrupted"}
@pytest.mark.asyncio
async def test_shutdown_returns_even_when_a_producer_will_not_unwind(durable_run, monkeypatch):
"""A generator whose teardown blocks must not block uvicorn's shutdown; the gather after
task.cancel() must be bounded too, since an engine draining a subprocess in aclose never
finishes cancelling and stop() would wait forever."""
import core.inference.chat_generation_runs as chat_generation_runs
monkeypatch.setattr(chat_generation_runs, "_SHUTDOWN_GRACE_SECONDS", 0.2)
monkeypatch.setattr(chat_generation_runs, "_SHUTDOWN_CANCEL_SECONDS", 0.5)
wedged = asyncio.Event()
release = asyncio.Event()
async def body():
yield 'data: {"choices":[{"delta":{"content":"a"}}]}\n\n'
wedged.set()
try:
await release.wait()
except (asyncio.CancelledError, GeneratorExit):
await release.wait()
raise
yield "data: [DONE]\n\n"
async def fake(_payload, _request, _subject, *, cancel_on_disconnect):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
supervisor.start("run-1", thread_id = "thread-1", model = "local.gguf")
await asyncio.wait_for(wedged.wait(), 10)
try:
await asyncio.wait_for(supervisor.stop(), timeout = 10)
finally:
release.set()
await asyncio.sleep(0)
# Pins the line in _ensure_reservation that sets cancel.durable; other tests set it by hand.
def test_the_producers_cancel_event_carries_the_durable_marker_and_its_run_id():
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
try:
assert supervisor._ensure_reservation("run-durable", thread_id = "thread-1") is True
cancel_event = supervisor._cancel_events["run-durable"]
assert getattr(cancel_event, "durable", False) is True, (
"state.tool_approvals.wait_tool_decision reads this to park an approval instead of "
"auto-denying it, so without it a tool turn that outlives its tab still loses the call"
)
assert getattr(cancel_event, "durable_run_id", None) == "run-durable", (
"the gate resolves attendance by run id (state/run_subscribers.py); without it an "
"attended approval cannot be told from an abandoned one and expires at the park ceiling"
)
finally:
supervisor.cancel("run-durable")
registration = supervisor._active_registrations.pop("run-durable", None)
if registration is not None:
registration.__exit__(None, None, None)
supervisor._cancel_events.pop("run-durable", None)
activity = supervisor._activities.pop("run-durable", None)
if activity is not None:
activity.finish()
@pytest.mark.asyncio
async def test_permitted_prefill_renews_lease_but_decode_silence_does_not(durable_run, monkeypatch):
from utils.first_token_timeout import start_first_token_wait, finish_first_token_wait
monkeypatch.setattr(chat_generation_runs, "_renew_interval_seconds", lambda: 0.01)
supervisor = ChatGenerationSupervisor(SimpleNamespace(state = SimpleNamespace()))
touch = AsyncMock(wraps = supervisor._try_touch_progress)
monkeypatch.setattr(supervisor, "_try_touch_progress", touch)
observations = []
async def body():
# Sync reader runs in a copied context; the supervisor polls the shared wait object.
await asyncio.to_thread(start_first_token_wait, float("inf"))
before = touch.await_count
await asyncio.sleep(0.08)
observations.append(touch.await_count > before)
await asyncio.to_thread(finish_first_token_wait)
yield 'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
before = touch.await_count
await asyncio.sleep(0.08)
observations.append(touch.await_count == before)
yield "data: [DONE]\n\n"
async def fake(*_args, **_kwargs):
return SimpleNamespace(status_code = 200, body_iterator = body())
monkeypatch.setattr(inference, "produce_openai_chat_completions", fake)
await supervisor._produce("run-1")
assert observations == [True, True]