Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""httpcore snapshots the read timeout at body start, so the stall timeout must be re-read per call."""
from __future__ import annotations
import inspect
import sys
import threading
import types as _types
from pathlib import Path
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
sys.modules.setdefault("structlog", _types.ModuleType("structlog"))
import httpcore # noqa: E402
import httpx # noqa: E402
from core.inference import llama_cpp as llama_cpp_mod # noqa: E402
from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
_PREFILL_TIMEOUT = 1200.0
_STALL_TIMEOUT = 120.0
class _Obj:
pass
def _install(response, clock, silent_stream):
"""Wire fake client/pool so _install_cancel_aware_read finds the stream; return the wrapped stream.read."""
inner = _Obj()
inner._network_stream = silent_stream
connection = _Obj()
connection._connection = inner
pool = _Obj()
pool._connections = [connection]
transport = _Obj()
transport._pool = pool
client = _Obj()
client._transport = transport
cancel_event = threading.Event()
sig = inspect.signature(LlamaCppBackend._install_cancel_aware_read)
if "response" in sig.parameters:
LlamaCppBackend._install_cancel_aware_read(client, cancel_event, response)
else:
# Pre-fix signature: the stall assertion then fails, proving the bug.
LlamaCppBackend._install_cancel_aware_read(client, cancel_event)
return silent_stream.read
def test_stall_timeout_honored_after_first_token(monkeypatch):
clock = {"t": 0.0}
monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])
def silent_read(max_bytes, timeout = None):
clock["t"] += timeout if timeout is not None else 0.0
raise httpcore.ReadTimeout("slice timed out on silence")
stream = _Obj()
stream.read = silent_read
request = _Obj()
request.extensions = {"timeout": {"read": _STALL_TIMEOUT}}
response = _Obj()
response.request = request
wrapped_read = _install(response, clock, stream)
# httpcore still passes the stale prefill timeout it snapshotted at body start.
with pytest.raises(httpcore.ReadTimeout):
wrapped_read(65536, timeout = _PREFILL_TIMEOUT)
assert clock["t"] <= _STALL_TIMEOUT * 1.5, (
f"stall timeout not honored: waited {clock['t']}s "
f"(expected ~{_STALL_TIMEOUT}s, not {_PREFILL_TIMEOUT}s)"
)
assert clock["t"] >= _STALL_TIMEOUT * 0.5
def test_prompt_progress_keeps_the_prefill_read_timeout():
progress = (
'data: {"choices":[{"delta":{"role":"assistant","content":null}}],'
'"prompt_progress":{"processed":512,"cache":0,"time_ms":64}}\n\n'
)
output = 'data: {"choices":[{"delta":{"content":"x"}}]}\n\n'
fragments = (progress[:32], progress[32:], output[:24], output[24:])
class Response:
request = _types.SimpleNamespace(extensions = {"timeout": {"read": _PREFILL_TIMEOUT}})
@staticmethod
def iter_text():
yield from fragments
@staticmethod
def close():
return None
iterator = LlamaCppBackend._iter_text_cancellable(
Response(),
first_token_deadline = llama_cpp_mod.time.monotonic() + _PREFILL_TIMEOUT,
post_first_chunk_read_timeout_s = _STALL_TIMEOUT,
)
assert next(iterator) == fragments[0]
assert Response.request.extensions["timeout"]["read"] > _STALL_TIMEOUT
assert next(iterator) == fragments[1]
assert Response.request.extensions["timeout"]["read"] > _STALL_TIMEOUT
assert next(iterator) == fragments[2]
assert Response.request.extensions["timeout"]["read"] > _STALL_TIMEOUT
assert next(iterator) == fragments[3]
assert Response.request.extensions["timeout"]["read"] == _STALL_TIMEOUT
iterator.close()
_OUTPUT_EVENT = 'data: {"choices":[{"delta":{"content":"x"}}]}\n\n'
def _progress_event(processed: int) -> str:
return (
'data: {"choices":[{"delta":{"role":"assistant","content":null}}],'
f'"prompt_progress":{{"processed":{processed},"cache":0,"time_ms":1}}}}\n\n'
)
def _run(events, clock):
"""Drain _iter_text_cancellable over ``events``, 200s of fake time apart."""
class Response:
request = _types.SimpleNamespace(extensions = {"timeout": {"read": _PREFILL_TIMEOUT}})
@staticmethod
def iter_text():
for event in events:
clock["t"] += 200.0
yield event
@staticmethod
def close():
return None
return list(
LlamaCppBackend._iter_text_cancellable(
Response(),
first_token_deadline = clock["t"] + _PREFILL_TIMEOUT,
post_first_chunk_read_timeout_s = _STALL_TIMEOUT,
)
)
def test_advancing_prefill_progress_extends_the_first_token_deadline(monkeypatch):
"""A prompt that keeps prefilling past the window is not a hang (#9037)."""
clock = {"t": 0.0}
monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])
events = [_progress_event(i * 4096) for i in range(1, 21)] + [_OUTPUT_EVENT]
assert _run(events, clock)[-1] == _OUTPUT_EVENT
assert clock["t"] > _PREFILL_TIMEOUT
def test_repeated_prefill_progress_still_times_out(monkeypatch):
"""A server replaying one progress value is stuck, not prefilling."""
clock = {"t": 0.0}
monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])
with pytest.raises(httpx.ReadTimeout):
_run([_progress_event(4096)] * 20 + [_OUTPUT_EVENT], clock)
def test_prefill_without_progress_events_still_times_out(monkeypatch):
"""Content-free events alone are not proof of life: the window still applies."""
clock = {"t": 0.0}
monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])
keepalive = 'data: {"choices":[{"delta":{"role":"assistant","content":null}}]}\n\n'
with pytest.raises(httpx.ReadTimeout):
_run([keepalive] * 20 + [_OUTPUT_EVENT], clock)
def test_prefill_timeout_used_when_no_live_override(monkeypatch):
"""Without a lowered live timeout, the wrapper honors the passed prefill timeout, so the normal first-token wait is unchanged."""
clock = {"t": 0.0}
monkeypatch.setattr(llama_cpp_mod.time, "monotonic", lambda: clock["t"])
def silent_read(max_bytes, timeout = None):
clock["t"] += timeout if timeout is not None else 0.0
raise httpcore.ReadTimeout("slice timed out on silence")
stream = _Obj()
stream.read = silent_read
request = _Obj()
request.extensions = {}
response = _Obj()
response.request = request
wrapped_read = _install(response, clock, stream)
with pytest.raises(httpcore.ReadTimeout):
wrapped_read(65536, timeout = _PREFILL_TIMEOUT)
assert clock["t"] >= _PREFILL_TIMEOUT * 0.9