Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Load and unload must see which streaming chats they would interrupt, via a dict under a lock."""
import multiprocessing as _mp
import os
import sys
import threading
import uuid
import pytest
_backend = os.path.join(os.path.dirname(__file__), "..")
sys.path.insert(0, _backend)
from core.inference.worker import _SLOTS, StopLedger, _Stops
from state import active_generations
@pytest.fixture(autouse = True)
def _clean_registry():
active_generations.reset_for_tests()
yield
active_generations.reset_for_tests()
def test_registry_starts_empty():
assert active_generations.count() == 0
assert active_generations.snapshot() == []
assert active_generations.active_thread_ids() == []
def test_entry_lives_only_for_the_block():
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1", model = "m"):
assert active_generations.count() == 1
assert active_generations.active_thread_ids() == ["t1"]
assert active_generations.count() == 0
assert active_generations.active_thread_ids() == []
def test_entry_is_removed_even_when_the_block_raises():
ev = threading.Event()
with pytest.raises(RuntimeError):
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
raise RuntimeError("stream blew up")
assert active_generations.count() == 0
def test_overlapping_runs_on_one_thread_both_register():
a, b = threading.Event(), threading.Event()
with active_generations.ActiveGeneration(a, thread_id = "t1"):
with active_generations.ActiveGeneration(b, thread_id = "t1"):
assert active_generations.count() == 2
assert active_generations.active_thread_ids() == ["t1"]
assert active_generations.count() == 1
assert active_generations.count() == 0
def test_snapshot_is_json_safe_and_ordered_by_start():
a, b = threading.Event(), threading.Event()
with active_generations.ActiveGeneration(a, thread_id = "first", model = "m1"):
with active_generations.ActiveGeneration(b, thread_id = "second", model = "m2"):
snap = active_generations.snapshot()
assert [e["thread_id"] for e in snap] == ["first", "second"]
assert all("event" not in e for e in snap)
assert {"handle", "thread_id", "run_id", "model", "kind", "account_id", "started_at"} == set(
snap[0]
)
def test_thread_ids_are_deduped_and_skip_unnamed_runs():
a, b, c = threading.Event(), threading.Event(), threading.Event()
with active_generations.ActiveGeneration(a, thread_id = "t1"):
with active_generations.ActiveGeneration(b, thread_id = "t1"):
with active_generations.ActiveGeneration(c, thread_id = None):
assert active_generations.active_thread_ids() == ["t1"]
assert active_generations.count() == 3
def test_cancel_all_sets_every_event():
a, b = threading.Event(), threading.Event()
with active_generations.ActiveGeneration(a, thread_id = "t1"):
with active_generations.ActiveGeneration(b, thread_id = "t2"):
assert active_generations.cancel_all() == 2
assert a.is_set() and b.is_set()
def test_cancel_all_on_an_empty_registry_is_a_no_op():
assert active_generations.cancel_all() == 0
def test_cancel_thread_leaves_siblings_alone():
a, b = threading.Event(), threading.Event()
with active_generations.ActiveGeneration(a, thread_id = "t1"):
with active_generations.ActiveGeneration(b, thread_id = "t2"):
assert active_generations.cancel_thread("t1") == 1
assert a.is_set()
assert not b.is_set()
def test_cancel_thread_with_no_match_is_a_no_op():
a = threading.Event()
with active_generations.ActiveGeneration(a, thread_id = "t1"):
assert active_generations.cancel_thread("nope") == 0
assert active_generations.cancel_thread("") == 0
assert not a.is_set()
def test_cancel_run_targets_only_matching_durable_generation():
durable, sibling = threading.Event(), threading.Event()
with active_generations.ActiveGeneration(durable, thread_id = "t1", run_id = "run-1"):
with active_generations.ActiveGeneration(sibling, thread_id = "t1", run_id = "run-2"):
assert active_generations.cancel_run("run-1") == 1
assert durable.is_set()
assert not sibling.is_set()
def test_same_durable_run_and_event_borrows_existing_registration():
event = threading.Event()
with active_generations.ActiveGeneration(
event, run_id = "run-1", thread_id = "stale", model = "stale"
):
with active_generations.ActiveGeneration(
event, run_id = "run-1", thread_id = "thread-1", model = "local"
):
snapshot = active_generations.snapshot()[0]
assert (active_generations.count(), snapshot["thread_id"], snapshot["model"]) == (
1,
"thread-1",
"local",
)
assert active_generations.cancel_all() == 1
assert event.is_set()
assert active_generations.count() == 0
def test_cancel_does_not_unregister_entries():
a = threading.Event()
with active_generations.ActiveGeneration(a, thread_id = "t1"):
active_generations.cancel_all()
assert active_generations.count() == 1
def test_registry_survives_concurrent_register_unregister():
errors: list[BaseException] = []
barrier = threading.Barrier(8)
def worker(i: int) -> None:
try:
barrier.wait(timeout = 10)
for _ in range(50):
with active_generations.ActiveGeneration(threading.Event(), thread_id = f"t{i}"):
active_generations.snapshot()
except BaseException as exc: # noqa: BLE001 - surfaced via assert below
errors.append(exc)
threads = [threading.Thread(target = worker, args = (i,)) for i in range(8)]
for t in threads:
t.start()
for t in threads:
t.join(timeout = 30)
assert errors == []
assert active_generations.count() == 0
# The gate lives in routes.inference, which pulls the whole inference stack.
def _route_gate():
pytest.importorskip("fastapi", reason = "inference stack not installed")
routes_inference = pytest.importorskip(
"routes.inference", reason = "inference stack not installed"
)
return routes_inference._raise_or_cancel_active_generations
@pytest.fixture
def gate():
return _route_gate()
def test_gate_allows_a_swap_when_nothing_is_generating(gate):
assert gate(force = False, action = "Loading a model") == 0
def test_gate_refuses_with_409_and_names_the_chats(gate):
from fastapi import HTTPException
a, b = threading.Event(), threading.Event()
with active_generations.ActiveGeneration(a, thread_id = "t1"):
with active_generations.ActiveGeneration(b, thread_id = "t2"):
with pytest.raises(HTTPException) as exc:
gate(force = False, action = "Loading a model")
assert exc.value.status_code == 409
detail = exc.value.detail
assert detail["error"] == "active_generations"
assert detail["running"] == 2
assert detail["thread_ids"] == ["t1", "t2"]
assert not a.is_set() and not b.is_set()
def test_gate_message_is_singular_for_one_chat(gate):
from fastapi import HTTPException
with active_generations.ActiveGeneration(threading.Event(), thread_id = "t1"):
with pytest.raises(HTTPException) as exc:
gate(force = False, action = "Unloading the model")
message = exc.value.detail["message"]
assert "1 chat that is still generating" in message
assert "Unloading the model" in message
def test_gate_force_cancels_and_returns_the_count(gate):
a, b = threading.Event(), threading.Event()
with active_generations.ActiveGeneration(a, thread_id = "t1"):
with active_generations.ActiveGeneration(b, thread_id = "t2"):
assert gate(force = True, action = "Loading a model") == 2
assert a.is_set() and b.is_set()
def test_gate_force_with_nothing_running_is_a_no_op(gate):
assert gate(force = True, action = "Loading a model") == 0
def test_tracked_cancel_registers_the_thread_for_its_block():
_route_gate()
from routes.inference import _TrackedCancel
ev = threading.Event()
tracker = _TrackedCancel(ev, "cancel-1", thread_id = "t1", model = "m")
tracker.__enter__()
try:
assert active_generations.active_thread_ids() == ["t1"]
assert active_generations.snapshot()[0]["model"] == "m"
finally:
tracker.__exit__(None, None, None)
assert active_generations.count() == 0
def test_tracked_cancel_shares_its_event_with_the_registry():
_route_gate()
from routes.inference import _TrackedCancel
ev = threading.Event()
tracker = _TrackedCancel(ev, "cancel-1", thread_id = "t1")
tracker.__enter__()
try:
active_generations.cancel_all()
assert ev.is_set()
finally:
tracker.__exit__(None, None, None)
def _stub_load_route(
monkeypatch,
*,
active_model_name,
backend = None,
):
from types import SimpleNamespace
import routes.inference as inf_mod
from core.inference.orchestrator import InferenceOrchestrator
if backend is None:
backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
backend._stop_ledger = None
backend._pending_teardowns = None
backend.active_model_name = active_model_name
backend.models = {}
monkeypatch.setattr(inf_mod, "_raise_if_sidecar_swap_in_progress", lambda: None)
monkeypatch.setattr(inf_mod, "validate_extra_args", lambda args: [])
monkeypatch.setattr(
inf_mod,
"resolve_effective_chat_template_override",
lambda model_identifier = None, user_override = None: None,
)
monkeypatch.setattr(inf_mod, "load_inference_config", lambda name: {})
monkeypatch.setattr(
inf_mod,
"_detect_safetensors_features",
lambda backend, template, tools = None: {
"supports_reasoning": False,
"reasoning_style": "enable_thinking",
"reasoning_effort_levels": [],
"reasoning_always_on": False,
"supports_preserve_thinking": False,
"supports_tools": False,
},
)
monkeypatch.setattr(inf_mod, "_resolve_loaded_trust_remote_code", lambda *a, **k: False)
monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend)
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_loaded = False, hf_variant = None, model_identifier = None),
)
return inf_mod
def test_idempotent_load_neither_refuses_nor_cancels_running_chats(monkeypatch):
_route_gate()
import asyncio
from models.inference import LoadRequest
inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/A")
for force in (False, True):
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
response = asyncio.run(
inf_mod.load_model(
LoadRequest(model_path = "org/A", force_cancel_active = force),
object(),
"tester",
)
)
assert response.status == "already_loaded"
assert not ev.is_set()
def test_a_width_change_reaches_the_resident_model_without_replacing_it(monkeypatch):
_route_gate()
import asyncio
from models.inference import LoadRequest
inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/A")
applied = []
inf_mod.get_inference_backend().set_parallel_slots = applied.append
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
response = asyncio.run(
inf_mod.load_model(LoadRequest(model_path = "org/A", n_parallel = 2), object(), "tester")
)
assert response.status == "already_loaded"
assert applied == [2]
assert not ev.is_set(), "a width change ended a reply that was decoding"
def test_a_real_reload_still_refuses_while_chats_stream(monkeypatch):
_route_gate()
import asyncio
from fastapi import HTTPException
from models.inference import LoadRequest
inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER")
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
with pytest.raises(HTTPException) as exc:
asyncio.run(inf_mod.load_model(LoadRequest(model_path = "org/A"), object(), "tester"))
assert exc.value.status_code == 409
assert exc.value.detail["thread_ids"] == ["t1"]
assert not ev.is_set()
def test_a_forced_load_that_fails_preflight_leaves_the_chats_alone(monkeypatch):
_route_gate()
import asyncio
import contextlib
from fastapi import HTTPException
from models.inference import LoadRequest
inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER")
monkeypatch.setattr(inf_mod, "_hf_offline_if_unreachable", contextlib.nullcontext)
monkeypatch.setattr(inf_mod.ModelConfig, "from_identifier", staticmethod(lambda **kwargs: None))
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
with pytest.raises(HTTPException) as exc:
asyncio.run(
inf_mod.load_model(
LoadRequest(model_path = "org/A", force_cancel_active = True),
object(),
"tester",
)
)
assert not ev.is_set()
assert active_generations.count() == 1
assert exc.value.status_code == 400
def _stub_standard_load_route(monkeypatch):
"""Drive _load_model_impl down the Unsloth path as far as the pre-teardown drain."""
import contextlib
from types import SimpleNamespace
import routes.inference as inf_mod
real_sidecar_check = inf_mod._raise_if_sidecar_swap_in_progress
_stub_load_route(monkeypatch, active_model_name = "org/OTHER")
monkeypatch.setattr(inf_mod, "_raise_if_sidecar_swap_in_progress", real_sidecar_check)
monkeypatch.setattr(inf_mod, "_hf_offline_if_unreachable", contextlib.nullcontext)
monkeypatch.setattr(inf_mod, "_mlx_distributed_launch_detected", lambda: False)
monkeypatch.setattr(
inf_mod.ModelConfig,
"from_identifier",
staticmethod(
lambda **kwargs: SimpleNamespace(
is_gguf = False,
identifier = "org/A",
display_name = "A",
is_vision = False,
gguf_hf_repo = None,
gguf_variant = None,
)
),
)
monkeypatch.setattr(inf_mod, "_effective_load_in_4bit", lambda config, requested: False)
monkeypatch.setattr(inf_mod, "_resolve_inherited_extra_args", lambda *a, **k: None)
monkeypatch.setattr(inf_mod, "_guard_chat_load_against_training", lambda *a, **k: None)
return inf_mod
def test_a_sidecar_swap_reserved_during_the_drain_never_strands_cancelled_chats(monkeypatch):
# The sidecar recheck must precede the cancel, or chats die for nothing.
_route_gate()
import asyncio
import time
from types import SimpleNamespace
from fastapi import HTTPException
from core.inference import llama_keepwarm as kw
from models.inference import LoadRequest
import utils.transformers_version as tv
inf_mod = _stub_standard_load_route(monkeypatch)
reserved = {"v": False}
monkeypatch.setattr(tv, "sidecar_swap_in_progress", lambda: reserved["v"])
monkeypatch.setattr(kw, "_inflight", 2)
def _installer():
time.sleep(0.10)
kw._inflight = 1
reserved["v"] = True
time.sleep(0.35)
kw._inflight = 0
thread = threading.Thread(target = _installer, daemon = True)
ev = threading.Event()
try:
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
thread.start()
with pytest.raises(HTTPException) as exc:
asyncio.run(
inf_mod.load_model(
LoadRequest(model_path = "org/A", force_cancel_active = True),
SimpleNamespace(
app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1))
),
"tester",
)
)
assert not ev.is_set()
assert active_generations.count() == 1
assert exc.value.status_code == 409
assert "transformers installation" in str(exc.value.detail)
finally:
thread.join(timeout = 5)
kw._inflight = 0
def _stub_unload_backends(monkeypatch, *, llama, backend):
"""Point the /unload route at in-memory backends."""
import routes.inference as inf_mod
from core.inference import llama_keepwarm as kw
monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: llama)
monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend)
monkeypatch.setattr(inf_mod, "is_registered_native_path_label", lambda *a: False)
monkeypatch.setattr(kw, "note_model_unloaded", lambda: None)
return inf_mod, kw
def test_unload_rechecks_active_generations_under_the_lifecycle_gate(monkeypatch):
_route_gate()
import asyncio
from types import SimpleNamespace
from fastapi import HTTPException
from models.inference import UnloadRequest
torn_down: list[str] = []
inf_mod, kw = _stub_unload_backends(
monkeypatch,
llama = SimpleNamespace(
is_active = True,
is_loaded = True,
model_identifier = "org/A-GGUF",
unload_model = lambda: torn_down.append("gguf"),
),
backend = SimpleNamespace(
get_loading_model = lambda: None,
unload_model = lambda path: torn_down.append("unsloth"),
),
)
ev = threading.Event()
started = active_generations.ActiveGeneration(ev, thread_id = "t1")
async def drive():
kw._lifecycle_lock.acquire()
task = asyncio.create_task(
inf_mod.unload_model(UnloadRequest(model_path = "org/A-GGUF"), "tester")
)
entered = False
try:
await asyncio.sleep(0.1)
started.__enter__()
entered = True
finally:
kw._lifecycle_lock.release()
try:
return await asyncio.wait_for(task, timeout = 5)
finally:
if entered:
started.__exit__(None, None, None)
with pytest.raises(HTTPException) as exc:
asyncio.run(drive())
assert exc.value.status_code == 409
assert exc.value.detail["error"] == "active_generations"
assert torn_down == []
assert not ev.is_set()
def _run_unload(
inf_mod,
monkeypatch,
*,
loaded_gguf,
requested,
force,
torn_down,
unload_model = None,
):
"""Drive POST /unload; the stub unload_model observes the world at teardown, not after it."""
import asyncio
from types import SimpleNamespace
from models.inference import UnloadRequest
_stub_unload_backends(
monkeypatch,
llama = SimpleNamespace(
is_active = True,
is_loaded = True,
model_identifier = loaded_gguf,
unload_model = unload_model or (lambda: torn_down.append("gguf")),
),
backend = SimpleNamespace(
get_loading_model = lambda: None,
active_model_name = None,
models = {},
unload_model = lambda path: torn_down.append("unsloth"),
),
)
return asyncio.run(
inf_mod.unload_model(
UnloadRequest(model_path = requested, force_cancel_active = force), "tester"
)
)
_PINNED_SNAPSHOT = "/tmp/hf/hub/models--Org--Quant/snapshots/" + "d" * 40
def test_unload_finds_a_gguf_loaded_from_a_pinned_snapshot(monkeypatch):
"""A GGUF pinned to a snapshot reports its repo id, so unload by that id must reach it."""
_route_gate()
import routes.inference as inf_mod
torn_down: list[str] = []
_run_unload(
inf_mod,
monkeypatch,
loaded_gguf = _PINNED_SNAPSHOT,
requested = "Org/Quant",
force = False,
torn_down = torn_down,
)
assert torn_down == ["gguf"]
other: list[str] = []
_run_unload(
inf_mod,
monkeypatch,
loaded_gguf = _PINNED_SNAPSHOT,
requested = "Org/Other",
force = False,
torn_down = other,
)
assert other == ["unsloth"]
def test_stop_loading_cancels_an_in_flight_pinned_load(monkeypatch):
"""Cancel sends the id the picker shows. A pinned row loads under a snapshot path, so a raw
compare let Stop loading report success while the multi minute load kept running."""
_route_gate()
import asyncio
from types import SimpleNamespace
from models.inference import UnloadRequest
def _cancel(loading, requested):
cancelled: list[str] = []
inf_mod, _kw = _stub_unload_backends(
monkeypatch,
llama = SimpleNamespace(is_active = False, is_loaded = False, model_identifier = None),
backend = SimpleNamespace(
get_loading_model = lambda: loading,
cancel_load = lambda path: (cancelled.append(path), True)[1],
active_model_name = None,
models = {},
unload_model = lambda path: None,
),
)
asyncio.run(
inf_mod.unload_model(
UnloadRequest(model_path = requested, force_cancel_active = False), "tester"
)
)
return cancelled
assert _cancel(_PINNED_SNAPSHOT, "Org/Quant") == [_PINNED_SNAPSHOT]
assert _cancel("Org/Quant", "Org/Quant") == ["Org/Quant"]
assert _cancel(_PINNED_SNAPSHOT, "Org/Other") == []
def test_unload_evicts_a_pinned_standard_model_under_its_registered_name(monkeypatch):
"""The standard backend refuses a name it never loaded, so a pinned load has to be evicted
under the path it was registered with rather than the id the picker shows."""
_route_gate()
import asyncio
from types import SimpleNamespace
from models.inference import UnloadRequest
unloaded: list[str] = []
inf_mod, _kw = _stub_unload_backends(
monkeypatch,
llama = SimpleNamespace(is_active = False, is_loaded = False, model_identifier = None),
backend = SimpleNamespace(
get_loading_model = lambda: None,
active_model_name = _PINNED_SNAPSHOT,
models = {_PINNED_SNAPSHOT: {}},
unload_model = lambda path: unloaded.append(path),
),
)
asyncio.run(
inf_mod.unload_model(
UnloadRequest(model_path = "Org/Quant", force_cancel_active = False), "tester"
)
)
assert unloaded == [_PINNED_SNAPSHOT]
def test_forced_unload_of_a_stale_model_path_leaves_the_chats_alone(monkeypatch):
_route_gate()
import routes.inference as inf_mod
torn_down: list[str] = []
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
response = _run_unload(
inf_mod,
monkeypatch,
loaded_gguf = "org/B-GGUF", # what the other tab actually loaded
requested = "org/A-GGUF", # this tab's stale idea of it
force = True,
torn_down = torn_down,
)
assert not ev.is_set()
assert active_generations.count() == 1
assert "gguf" not in torn_down
assert response.status == "unloaded"
def test_forced_unload_of_the_loaded_model_still_stops_its_chats(monkeypatch):
_route_gate()
import routes.inference as inf_mod
torn_down: list[str] = []
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
response = _run_unload(
inf_mod,
monkeypatch,
loaded_gguf = "org/A-GGUF",
requested = "org/A-GGUF",
force = True,
torn_down = torn_down,
)
assert ev.is_set()
assert torn_down == ["gguf"]
assert response.status == "unloaded"
def test_forced_unload_lets_the_cancelled_chats_unwind_before_teardown(monkeypatch):
_route_gate()
import core.inference.llama_keepwarm as keepwarm
import routes.inference as inf_mod
inflight = {"n": 1}
seen = {}
def _count(current_request_counted = True, *, include_pending = True):
if inflight["n"] > 0:
inflight["n"] -= 1
return inflight["n"]
monkeypatch.setattr(keepwarm, "other_inference_request_count", _count)
monkeypatch.setattr(inf_mod, "_switch_waiter_count", lambda: 0)
torn_down: list[str] = []
ev = threading.Event()
def _record_teardown():
seen["inflight_at_teardown"] = inflight["n"]
torn_down.append("gguf")
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
response = _run_unload(
inf_mod,
monkeypatch,
loaded_gguf = "org/A-GGUF",
requested = "org/A-GGUF",
force = True,
torn_down = torn_down,
unload_model = _record_teardown,
)
assert ev.is_set()
assert torn_down == ["gguf"]
assert seen["inflight_at_teardown"] == 0
assert response.status == "unloaded"
def test_unload_drains_on_the_middleware_count_not_just_the_registry(monkeypatch):
# The drain reads the middleware count: counted-but-unregistered requests exist.
_route_gate()
import core.inference.llama_keepwarm as keepwarm
import routes.inference as inf_mod
polls = {"n": 0}
def _count(current_request_counted = True, *, include_pending = True):
polls["n"] += 1
return 0
monkeypatch.setattr(keepwarm, "other_inference_request_count", _count)
torn_down: list[str] = []
response = _run_unload(
inf_mod,
monkeypatch,
loaded_gguf = "org/A-GGUF",
requested = "org/A-GGUF",
force = True,
torn_down = torn_down,
)
assert torn_down == ["gguf"]
assert polls["n"] == 1
assert response.status == "unloaded"
def test_unforced_unload_of_a_stale_model_path_is_still_a_no_op(monkeypatch):
_route_gate()
import routes.inference as inf_mod
torn_down: list[str] = []
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
response = _run_unload(
inf_mod,
monkeypatch,
loaded_gguf = "org/B-GGUF",
requested = "org/A-GGUF",
force = False,
torn_down = torn_down,
)
assert not ev.is_set()
assert active_generations.count() == 1
assert torn_down == ["unsloth"]
assert response.status == "unloaded"
def test_unforced_unload_of_the_loaded_model_still_refuses_while_chats_stream(monkeypatch):
_route_gate()
import routes.inference as inf_mod
from fastapi import HTTPException
torn_down: list[str] = []
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
with pytest.raises(HTTPException) as exc:
_run_unload(
inf_mod,
monkeypatch,
loaded_gguf = "org/A-GGUF",
requested = "org/A-GGUF",
force = False,
torn_down = torn_down,
)
assert exc.value.status_code == 409
assert exc.value.detail["thread_ids"] == ["t1"]
assert torn_down == []
assert not ev.is_set()
def test_unforced_unload_still_refuses_while_a_gguf_load_is_in_flight(monkeypatch):
_route_gate()
import asyncio
from types import SimpleNamespace
from fastapi import HTTPException
from models.inference import UnloadRequest
torn_down: list[str] = []
inf_mod, _kw = _stub_unload_backends(
monkeypatch,
llama = SimpleNamespace(
is_active = True,
is_loaded = False, # spawned, health check not passed: mid-load
model_identifier = "org/B-GGUF",
unload_model = lambda: torn_down.append("gguf"),
),
backend = SimpleNamespace(
get_loading_model = lambda: None,
active_model_name = None,
models = {},
unload_model = lambda path: torn_down.append("unsloth"),
),
)
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
with pytest.raises(HTTPException) as exc:
asyncio.run(
inf_mod.unload_model(
UnloadRequest(model_path = "org/A-GGUF", force_cancel_active = False),
"tester",
)
)
assert exc.value.status_code == 409
assert torn_down == []
assert not ev.is_set()
def test_cancelling_an_in_flight_standard_load_is_not_refused_by_the_chat_gate(monkeypatch):
_route_gate()
import asyncio
from types import SimpleNamespace
from models.inference import UnloadRequest
cancelled: list[str] = []
torn_down: list[str] = []
inf_mod, _kw = _stub_unload_backends(
monkeypatch,
llama = SimpleNamespace(
is_active = False,
is_loaded = False,
model_identifier = None,
unload_model = lambda: torn_down.append("gguf"),
),
backend = SimpleNamespace(
get_loading_model = lambda: "org/B",
cancel_load = lambda path: bool(cancelled.append(path)) or True,
active_model_name = None,
models = {},
unload_model = lambda path: torn_down.append("unsloth"),
),
)
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
response = asyncio.run(
inf_mod.unload_model(
UnloadRequest(model_path = "org/B", force_cancel_active = False), "tester"
)
)
assert not ev.is_set()
assert active_generations.count() == 1
assert response.status == "unloaded"
assert cancelled == ["org/B"]
assert torn_down == []
def test_cancelling_an_in_flight_gguf_load_is_not_refused_by_the_chat_gate(monkeypatch):
_route_gate()
import asyncio
from types import SimpleNamespace
from models.inference import UnloadRequest
torn_down: list[str] = []
inf_mod, _kw = _stub_unload_backends(
monkeypatch,
llama = SimpleNamespace(
is_active = True,
is_loaded = False, # spawned, health check not passed: mid-load
model_identifier = "org/B-GGUF",
unload_model = lambda: torn_down.append("gguf"),
),
backend = SimpleNamespace(
get_loading_model = lambda: None,
active_model_name = None,
models = {},
unload_model = lambda path: torn_down.append("unsloth"),
),
)
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
response = asyncio.run(
inf_mod.unload_model(
UnloadRequest(model_path = "org/B-GGUF", force_cancel_active = False), "tester"
)
)
assert not ev.is_set()
assert active_generations.count() == 1
assert response.status == "unloaded"
assert torn_down == ["gguf"]
def _install_responses_stream_mock(monkeypatch, chunks):
"""Points /v1/responses GGUF pass-through at an in-process llama-server."""
import json
from types import SimpleNamespace
import httpx
import routes.inference as inf_mod
def handler(request):
content = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks)
content += "data: [DONE]\n\n"
return httpx.Response(
200,
content = content.encode(),
headers = {"content-type": "text/event-stream"},
)
transport = httpx.MockTransport(handler)
real_async_client = httpx.AsyncClient
monkeypatch.setattr(
inf_mod.httpx,
"AsyncClient",
lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)),
)
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_vision = False,
context_length = 4096,
base_url = "http://llama.test",
supports_reasoning = True,
reasoning_always_on = False,
_request_reasoning_kwargs = (
lambda enable_thinking = None, reasoning_effort = None, preserve_thinking = None: None
),
),
)
return inf_mod
class _NeverDisconnectedRequest:
async def is_disconnected(self):
return False
def test_direct_responses_stream_is_visible_to_the_swap_gate(monkeypatch):
_route_gate()
import asyncio
from models.inference import ChatMessage, ResponsesRequest
inf_mod = _install_responses_stream_mock(
monkeypatch, [{"choices": [{"delta": {"content": "33"}}]}]
)
payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF")
messages = [ChatMessage(role = "user", content = "hi")]
seen = {}
async def run():
response = await inf_mod._responses_stream(payload, messages, _NeverDisconnectedRequest())
iterator = response.body_iterator
await iterator.__anext__()
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
async for _ in iterator:
pass
asyncio.run(run())
assert seen["count"] == 1
assert seen["snapshot"][0]["model"] == "org/M-GGUF"
assert active_generations.count() == 0
def test_forced_reload_stops_a_direct_responses_stream(monkeypatch):
_route_gate()
import asyncio
from models.inference import ChatMessage, ResponsesRequest
inf_mod = _install_responses_stream_mock(
monkeypatch,
[
{"choices": [{"delta": {"content": "3"}}]},
{"choices": [{"delta": {"content": "3"}}]},
],
)
payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF")
messages = [ChatMessage(role = "user", content = "hi")]
async def run():
response = await inf_mod._responses_stream(payload, messages, _NeverDisconnectedRequest())
iterator = response.body_iterator
chunks = [await iterator.__anext__()]
assert active_generations.cancel_all() == 1
async for chunk in iterator:
chunks.append(chunk)
return "".join(c.decode() if isinstance(c, bytes) else c for c in chunks)
body = asyncio.run(run())
assert "response.completed" not in body
assert active_generations.count() == 0
def test_forced_reload_stops_a_responses_stream_still_queued_for_a_slot(monkeypatch):
# The run registers before holding a decode slot, so cancel_all() must reach it while queued.
_route_gate()
import asyncio
from core.inference import llama_admission
from models.inference import ChatMessage, ResponsesRequest
for name in (
llama_admission.ADMISSION_CONTROL_ENV,
llama_admission.ADMISSION_QUEUE_TIMEOUT_ENV,
llama_admission.ADMISSION_KEEPALIVE_INTERVAL_ENV,
llama_admission.ADMISSION_MAX_QUEUE_ENV,
):
monkeypatch.delenv(name, raising = False)
inf_mod = _install_responses_stream_mock(
monkeypatch, [{"choices": [{"delta": {"content": "33"}}]}]
)
payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF")
messages = [ChatMessage(role = "user", content = "hi")]
llama_admission.reset_llama_admission_queues()
try:
async def run():
queue = llama_admission.get_llama_admission_queue("http://llama.test")
holder = queue.reserve(capacity = 1, config = llama_admission.LlamaAdmissionConfig())
assert holder.lease_nowait() is not None
response = await inf_mod._responses_stream(
payload, messages, _NeverDisconnectedRequest()
)
chunks = []
async def drain():
async for chunk in response.body_iterator:
chunks.append(chunk)
task = asyncio.create_task(drain())
for _ in range(500):
if active_generations.count() == 1:
break
await asyncio.sleep(0.01)
assert active_generations.count() == 1, "the queued run never registered"
assert active_generations.cancel_all() == 1
# Unbounded queue by default: without the tracked event this never returns while the slot is held.
await asyncio.wait_for(task, timeout = 5)
return chunks
chunks = asyncio.run(run())
finally:
llama_admission.reset_llama_admission_queues()
body = "".join(c.decode() if isinstance(c, bytes) else c for c in chunks)
assert "response.created" not in body
assert active_generations.count() == 0
def _install_completions_stream_mock(monkeypatch, events):
"""Point the /v1/completions proxy at an in-process llama-server."""
import json
from types import SimpleNamespace
import httpx
import routes.inference as inf_mod
def handler(request):
# One network chunk per SSE event: the relay polls its cancel flag between upstream chunks.
async def _chunks():
for event in events:
yield f"data: {json.dumps(event)}\n\n".encode()
yield b"data: [DONE]\n\n"
return httpx.Response(
200,
content = _chunks(),
headers = {"content-type": "text/event-stream"},
)
transport = httpx.MockTransport(handler)
real_async_client = httpx.AsyncClient
monkeypatch.setattr(
inf_mod.httpx,
"AsyncClient",
lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)),
)
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
context_length = 4096,
base_url = "http://llama.test",
model_identifier = "org/M-GGUF",
),
)
monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False)
async def _no_auto_switch(request, current_subject, **_kwargs):
return await request.json()
monkeypatch.setattr(inf_mod, "_auto_switch_from_request_body", _no_auto_switch)
return inf_mod
class _CompletionsRequest(_NeverDisconnectedRequest):
"""Minimal stand-in for the Starlette Request /v1/completions reads."""
def __init__(self, body):
from types import SimpleNamespace
self._body = body
self.method = "POST"
self.url = SimpleNamespace(path = "/v1/completions")
async def json(self):
return self._body
def test_completions_proxy_stream_is_visible_to_the_swap_gate(monkeypatch):
_route_gate()
import asyncio
inf_mod = _install_completions_stream_mock(monkeypatch, [{"choices": [{"text": "33"}]}])
request = _CompletionsRequest(
{"prompt": "hi", "stream": True, "model": "org/M-GGUF", "max_tokens": 8}
)
seen = {}
async def run():
response = await inf_mod.openai_completions(request, "tester")
iterator = response.body_iterator
await iterator.__anext__()
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
async for _ in iterator:
pass
asyncio.run(run())
assert seen["count"] == 1
assert seen["snapshot"][0]["model"] == "org/M-GGUF"
assert active_generations.count() == 0
def test_forced_reload_stops_a_completions_proxy_stream(monkeypatch):
_route_gate()
import asyncio
inf_mod = _install_completions_stream_mock(
monkeypatch,
[{"choices": [{"text": "3"}]}, {"choices": [{"text": "3"}]}],
)
request = _CompletionsRequest(
{"prompt": "hi", "stream": True, "model": "org/M-GGUF", "max_tokens": 8}
)
async def run():
response = await inf_mod.openai_completions(request, "tester")
iterator = response.body_iterator
chunks = [await iterator.__anext__()]
assert active_generations.cancel_all() == 1
async for chunk in iterator:
chunks.append(chunk)
return b"".join(c if isinstance(c, bytes) else c.encode() for c in chunks)
body = asyncio.run(run())
assert body.count(b'"text"') == 1
assert active_generations.count() == 0
def test_completions_proxy_non_stream_is_visible_to_the_swap_gate(monkeypatch):
# Non-streaming is the common shape and holds llama-server unregistered, so /unload counts zero.
_route_gate()
import asyncio
from types import SimpleNamespace
import httpx
import routes.inference as inf_mod
seen = {}
def handler(request):
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
seen["cancelled"] = active_generations.cancel_all()
return httpx.Response(200, json = {"id": "cmpl-x", "choices": [{"text": "33"}]})
transport = httpx.MockTransport(handler)
monkeypatch.setattr(
inf_mod,
"_cancelable_nonstreaming_client",
lambda: httpx.AsyncClient(transport = transport),
)
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
context_length = 4096,
base_url = "http://llama.test",
model_identifier = "org/M-GGUF",
),
)
monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False)
async def _no_auto_switch(request, current_subject, **_kwargs):
return await request.json()
monkeypatch.setattr(inf_mod, "_auto_switch_from_request_body", _no_auto_switch)
request = _CompletionsRequest({"prompt": "hi", "model": "org/M-GGUF", "max_tokens": 8})
with pytest.raises(inf_mod.HTTPException) as exc:
asyncio.run(inf_mod.openai_completions(request, "tester"))
assert exc.value.status_code == 499
assert seen["count"] == 1
assert seen["snapshot"][0]["model"] == "org/M-GGUF"
assert seen["cancelled"] == 1
assert active_generations.count() == 0
class _EmbeddingsRequest(_NeverDisconnectedRequest):
"""Minimal stand-in for the Starlette Request /v1/embeddings reads."""
def __init__(self, body):
from types import SimpleNamespace
self._body = body
self.method = "POST"
self.url = SimpleNamespace(path = "/v1/embeddings")
self.state = SimpleNamespace(skip_api_monitor = True)
async def json(self):
return self._body
def test_embeddings_proxy_is_visible_to_the_swap_gate(monkeypatch):
_route_gate()
import asyncio
from types import SimpleNamespace
import httpx
import routes.inference as inf_mod
seen = {}
def handler(request):
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
seen["cancelled"] = active_generations.cancel_all()
return httpx.Response(200, json = {"data": [{"embedding": [0.1, 0.2]}]})
transport = httpx.MockTransport(handler)
monkeypatch.setattr(
inf_mod,
"_cancelable_nonstreaming_client",
lambda: httpx.AsyncClient(transport = transport),
)
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
context_length = 4096,
base_url = "http://llama.test",
model_identifier = "org/M-GGUF",
),
)
monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False)
async def _no_auto_switch(request, current_subject, **_kwargs):
return await request.json()
monkeypatch.setattr(inf_mod, "_auto_switch_from_request_body", _no_auto_switch)
request = _EmbeddingsRequest({"input": "hi", "model": "org/M-GGUF"})
with pytest.raises(inf_mod.HTTPException) as exc:
asyncio.run(inf_mod.openai_embeddings(request, "tester"))
assert exc.value.status_code == 499
assert seen["count"] == 1
assert seen["snapshot"][0]["model"] == "org/M-GGUF"
assert seen["cancelled"] == 1
assert active_generations.count() == 0
def test_active_generations_redacts_native_model_paths(monkeypatch):
_route_gate()
import asyncio
import threading
from types import SimpleNamespace
import routes.inference as inf_mod
from utils.native_path_leases import _remember_native_path_for_redaction
secret_path = "/home/somebody/models/private-model.gguf"
_remember_native_path_for_redaction(secret_path, "private-model.gguf")
request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 4)))
monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: SimpleNamespace())
with active_generations.ActiveGeneration(threading.Event(), thread_id = "t1", model = secret_path):
body = asyncio.run(inf_mod.get_active_generations(request, "tester"))
assert body["count"] == 1
assert secret_path not in str(body)
assert body["active"][0]["model"] == "<native_path>"
def test_legacy_generate_stream_is_visible_to_the_swap_gate(monkeypatch):
_route_gate()
import asyncio
from types import SimpleNamespace
import routes.inference as inf_mod
from models.inference import GenerateRequest
seen = {}
def _fake_generate_chat_response(**kwargs):
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
seen["cancelled"] = active_generations.cancel_all()
yield "hello"
yield "world"
backend = SimpleNamespace(
active_model_name = "org/M",
models = {"org/M": {}},
generate_chat_response = lambda **kw: _fake_generate_chat_response(**kw),
reset_generation_state = lambda *a: None,
resize_image = lambda img: img,
)
monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend)
async def _drain():
response = await inf_mod.generate_stream(
GenerateRequest(messages = [{"role": "user", "content": "hi"}]),
_NeverDisconnectedRequest(),
current_subject = "tester",
)
async for _ in response.body_iterator:
pass
asyncio.run(_drain())
assert seen["count"] == 1
assert seen["snapshot"][0]["model"] == "org/M"
assert seen["cancelled"] == 1
assert active_generations.count() == 0
def _anthropic_stream_args(chunks):
"""(request, cancel_event, run_gen) for the local Anthropic stream helpers."""
cancel_event = threading.Event()
def run_gen():
def _gen():
for chunk in chunks:
if cancel_event.is_set():
return
yield chunk
return _gen()
return _NeverDisconnectedRequest(), cancel_event, run_gen
def test_local_anthropic_plain_stream_is_visible_to_the_swap_gate(monkeypatch):
_route_gate()
import asyncio
import routes.inference as inf_mod
request, cancel_event, run_gen = _anthropic_stream_args(["3", "33"])
seen = {}
async def run():
response = await inf_mod._anthropic_plain_stream(
request, cancel_event, run_gen, "msg_1", "org/M-GGUF"
)
iterator = response.body_iterator
await iterator.__anext__()
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
async for _ in iterator:
pass
asyncio.run(run())
assert seen["count"] == 1
assert seen["snapshot"][0]["model"] == "org/M-GGUF"
assert active_generations.count() == 0
def test_forced_reload_stops_a_local_anthropic_plain_stream(monkeypatch):
_route_gate()
import asyncio
import routes.inference as inf_mod
request, cancel_event, run_gen = _anthropic_stream_args(["3", "33", "333"])
async def run():
response = await inf_mod._anthropic_plain_stream(
request, cancel_event, run_gen, "msg_1", "org/M-GGUF"
)
iterator = response.body_iterator
chunks = [await iterator.__anext__()]
assert active_generations.cancel_all() == 1
async for chunk in iterator:
chunks.append(chunk)
return "".join(c.decode() if isinstance(c, bytes) else c for c in chunks)
body = asyncio.run(run())
assert cancel_event.is_set()
assert "message_stop" not in body
assert active_generations.count() == 0
def test_local_anthropic_tool_stream_is_visible_to_the_swap_gate(monkeypatch):
_route_gate()
import asyncio
import routes.inference as inf_mod
request, cancel_event, run_gen = _anthropic_stream_args(
[{"type": "content", "text": "3"}, {"type": "content", "text": "33"}]
)
seen = {}
async def run():
response = await inf_mod._anthropic_tool_stream(
request, cancel_event, run_gen, "msg_1", "org/M-GGUF"
)
iterator = response.body_iterator
await iterator.__anext__()
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
async for _ in iterator:
pass
asyncio.run(run())
assert seen["count"] == 1
assert seen["snapshot"][0]["model"] == "org/M-GGUF"
assert active_generations.count() == 0
def test_load_and_unload_requests_default_to_not_cancelling():
pytest.importorskip("pydantic", reason = "pydantic not installed")
from models.inference import LoadRequest, UnloadRequest
assert LoadRequest(model_path = "m").force_cancel_active is False
assert UnloadRequest(model_path = "m").force_cancel_active is False
assert LoadRequest(model_path = "m", force_cancel_active = True).force_cancel_active is True
def _parallel_constants(path: str) -> dict:
"""Read _PARALLEL_* constants from source via ast; importing run.py would start the server."""
import ast
with open(path, encoding = "utf-8") as f:
tree = ast.parse(f.read())
found = {}
for node in tree.body:
if not isinstance(node, ast.Assign):
continue
for target in node.targets:
name = getattr(target, "id", "")
if name.startswith("_PARALLEL_") and isinstance(node.value, ast.Constant):
found[name] = node.value.value
return found
def test_studio_defaults_to_more_than_one_decode_slot():
# With one slot the admission queue serialises every chat.
consts = _parallel_constants(os.path.join(_backend, "run.py"))
assert consts["_PARALLEL_DEFAULT_PLAIN"] > 1
assert consts["_PARALLEL_MIN"] <= consts["_PARALLEL_DEFAULT_PLAIN"] <= consts["_PARALLEL_MAX"]
def test_cli_and_backend_parallel_defaults_agree():
backend = _parallel_constants(os.path.join(_backend, "run.py"))
cli_path = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(_backend))),
"unsloth_cli",
"commands",
"studio.py",
)
cli = _parallel_constants(cli_path)
assert cli["_PARALLEL_DEFAULT_PLAIN"] == backend["_PARALLEL_DEFAULT_PLAIN"]
def _run_server_parallel_default(path: str, consts: dict):
"""Resolve run_server()'s llama_parallel_slots default from run.py's source."""
import ast
with open(path, encoding = "utf-8") as f:
tree = ast.parse(f.read())
for node in tree.body:
if not isinstance(node, ast.FunctionDef) or node.name != "run_server":
continue
args = node.args.args
defaults = node.args.defaults
# defaults align with the tail of the positional arg list.
for arg, default in zip(args[len(args) - len(defaults) :], defaults):
if arg.arg != "llama_parallel_slots":
continue
if isinstance(default, ast.Constant):
return default.value
if isinstance(default, ast.Name):
return consts.get(default.id)
return None
return None
def test_run_server_default_matches_the_cli_parallel_default():
# colab.py omits llama_parallel_slots, so the signature default is what Colab runs with.
run_path = os.path.join(_backend, "run.py")
consts = _parallel_constants(run_path)
default = _run_server_parallel_default(run_path, consts)
assert default is not None, "run_server() must keep a llama_parallel_slots default"
assert default == consts["_PARALLEL_DEFAULT_PLAIN"]
assert default > 1
def test_colab_launcher_inherits_the_parallel_default():
# Guard the inheritance itself: an explicit 1 here would resurrect the bug.
import ast
colab_path = os.path.join(_backend, "colab.py")
with open(colab_path, encoding = "utf-8") as f:
tree = ast.parse(f.read())
consts = _parallel_constants(os.path.join(_backend, "run.py"))
calls = [
node
for node in ast.walk(tree)
if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "run_server"
]
assert calls, "colab.py must still launch the backend through run_server()"
for call in calls:
for kw in call.keywords:
if kw.arg != "llama_parallel_slots":
continue
value = kw.value.value if isinstance(kw.value, ast.Constant) else None
assert (
value is None or value > 1
), "colab.py pins llama_parallel_slots to 1; Colab chats would serialise"
assert consts["_PARALLEL_DEFAULT_PLAIN"] > 1
def test_a_forced_load_that_loses_to_a_sidecar_install_leaves_the_chats_alone(monkeypatch):
# The sidecar recheck must run before the destructive cancel.
_route_gate()
import asyncio
import contextlib
from types import SimpleNamespace
from fastapi import HTTPException
from models.inference import LoadRequest
inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER")
monkeypatch.setattr(inf_mod, "_hf_offline_if_unreachable", contextlib.nullcontext)
monkeypatch.setattr(
inf_mod.ModelConfig,
"from_identifier",
staticmethod(
lambda **kwargs: SimpleNamespace(
is_gguf = False,
identifier = "org/A",
display_name = "A",
is_vision = False,
is_lora = False,
path = None,
)
),
)
monkeypatch.setattr(inf_mod, "_mlx_distributed_launch_detected", lambda: False)
monkeypatch.setattr(inf_mod, "_guard_chat_load_against_training", lambda *a, **k: None)
monkeypatch.setattr(inf_mod, "_resolve_inherited_extra_args", lambda *a, **k: None)
seen = {"calls": 0}
def _sidecar_reserved_during_preflight():
seen["calls"] += 1
if seen["calls"] > 2:
raise HTTPException(
status_code = 409,
detail = "A transformers installation is in progress. Retry when it completes.",
)
monkeypatch.setattr(
inf_mod, "_raise_if_sidecar_swap_in_progress", _sidecar_reserved_during_preflight
)
fastapi_request = SimpleNamespace(
app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1))
)
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
with pytest.raises(HTTPException) as exc:
asyncio.run(
inf_mod.load_model(
LoadRequest(
model_path = "org/A",
load_in_4bit = False,
force_cancel_active = True,
),
fastapi_request,
"tester",
)
)
assert not ev.is_set()
assert active_generations.count() == 1
assert exc.value.status_code == 409
def test_anthropic_passthrough_registers_nothing_until_its_body_starts():
# A never-started async generator runs no body code (PEP 342), so an eager enter leaks.
_route_gate()
import asyncio
import inspect
from types import SimpleNamespace
from starlette.requests import ClientDisconnect
import routes.inference as inf_mod
llama_backend = SimpleNamespace(
base_url = "http://127.0.0.1:8080",
context_length = 4096,
count_chat_tokens = lambda messages, _unused, tools, **_kwargs: 7,
)
async def _build():
return await inf_mod._anthropic_passthrough_stream(
SimpleNamespace(),
threading.Event(),
llama_backend,
[{"role": "user", "content": "hi"}],
[],
0.7,
0.9,
40,
128,
"msg_1",
"org/A",
session_id = "s1",
cancel_id = "c1",
)
asyncio.run(_build())
assert active_generations.count() == 0
assert not inf_mod._CANCEL_REGISTRY
async def _drive():
response = await _build()
async def _receive():
return {"type": "http.disconnect"}
async def _send(message):
raise OSError("client disconnected")
with pytest.raises(ClientDisconnect):
await response({"type": "http"}, _receive, _send)
asyncio.run(_drive())
assert active_generations.count() == 0
assert not inf_mod._CANCEL_REGISTRY
src = inspect.getsource(inf_mod._anthropic_passthrough_stream)
assert src.index("async def _stream()") < src.index("_tracker.__enter__()")
assert src.index("_tracker.__enter__()") < src.index("_tracker.__exit__(None, None, None)")
def test_audio_generation_is_visible_to_the_swap_gate(monkeypatch):
_route_gate()
import asyncio
from types import SimpleNamespace
import routes.inference as inf_mod
from models.inference import ChatCompletionRequest
seen = {}
class _TtsBackend:
active_model_name = "org/TTS"
models = {"org/TTS": {"is_audio": True, "audio_type": "snac"}}
def generate_audio_response(self, **kwargs):
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
return (b"RIFFfake", 24000)
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_loaded = False, _is_audio = False),
)
monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: _TtsBackend())
async def _no_auto_switch(*a, **k):
return None
monkeypatch.setattr(inf_mod, "_maybe_auto_switch_model", _no_auto_switch)
payload = ChatCompletionRequest(
model = "org/TTS",
messages = [{"role": "user", "content": "hi"}],
thread_id = "thread-tts",
)
asyncio.run(inf_mod.generate_audio(payload, request = None, current_subject = "tester"))
assert seen["count"] == 1
assert seen["snapshot"][0]["thread_id"] == "thread-tts"
assert active_generations.count() == 0
class _ChatRequest(_NeverDisconnectedRequest):
"""Minimal stand-in for the Starlette Request /v1/chat/completions reads."""
def __init__(self):
from types import SimpleNamespace
self.method = "POST"
self.url = SimpleNamespace(path = "/v1/chat/completions")
self.state = SimpleNamespace(skip_api_monitor = True)
self.scope: dict = {}
def _standard_chat_stubs(monkeypatch, backend):
"""supports_tools False keeps chat off the safetensors server-tool loop, the default branch."""
from types import SimpleNamespace
import routes.inference as inf_mod
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = False,
supports_tools = False,
is_vision = False,
context_length = None,
),
)
monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend)
monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False)
monkeypatch.setattr(
inf_mod, "_detect_safetensors_features", lambda *a, **k: {"supports_tools": False}
)
async def _no_auto_switch(*a, **k):
return None
monkeypatch.setattr(inf_mod, "_maybe_auto_switch_model", _no_auto_switch)
return inf_mod
def test_standard_non_stream_chat_is_visible_to_the_swap_gate(monkeypatch):
_route_gate()
import asyncio
import routes.inference as inf_mod
from models.inference import ChatCompletionRequest
seen = {}
class _StandardBackend:
active_model_name = "org/M"
models = {"org/M": {"chat_template_info": {"template": "chatml"}}}
def generate_chat_response(
self,
*,
cancel_event = None,
stats_holder = None,
**kwargs,
):
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
seen["cancelled"] = active_generations.cancel_all()
seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set()
yield "33"
def reset_generation_state(self, caller_cancel_event = None):
pass
_standard_chat_stubs(monkeypatch, _StandardBackend())
payload = ChatCompletionRequest(
model = "org/M",
messages = [{"role": "user", "content": "hi"}],
thread_id = "thread-chat",
)
response = asyncio.run(
inf_mod.openai_chat_completions(payload, _ChatRequest(), current_subject = "tester")
)
assert response.status_code == 200
assert seen["count"] == 1
assert seen["snapshot"][0]["thread_id"] == "thread-chat"
assert seen["cancelled"] == 1
assert seen["reached_the_decode"]
assert active_generations.count() == 0
def test_standard_non_stream_chat_unregisters_when_it_fails(monkeypatch):
_route_gate()
import asyncio
from fastapi import HTTPException
import routes.inference as inf_mod
from models.inference import ChatCompletionRequest
class _BrokenBackend:
active_model_name = "org/M"
models = {"org/M": {"chat_template_info": {"template": "chatml"}}}
def generate_chat_response(self, **kwargs):
raise RuntimeError("decode exploded")
yield # pragma: no cover - generator marker
def reset_generation_state(self, caller_cancel_event = None):
pass
_standard_chat_stubs(monkeypatch, _BrokenBackend())
payload = ChatCompletionRequest(model = "org/M", messages = [{"role": "user", "content": "hi"}])
with pytest.raises(HTTPException):
asyncio.run(
inf_mod.openai_chat_completions(payload, _ChatRequest(), current_subject = "tester")
)
assert active_generations.count() == 0
def test_audio_input_non_stream_chat_is_visible_to_the_swap_gate(monkeypatch):
_route_gate()
import asyncio
import routes.inference as inf_mod
from models.inference import ChatCompletionRequest
seen = {}
class _AudioInputBackend:
active_model_name = "org/AUDIO-IN"
models = {"org/AUDIO-IN": {"has_audio_input": True}}
def generate_audio_input_response(
self,
*,
cancel_event = None,
**kwargs,
):
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
seen["cancelled"] = active_generations.cancel_all()
seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set()
yield "33"
def reset_generation_state(self, caller_cancel_event = None):
pass
_standard_chat_stubs(monkeypatch, _AudioInputBackend())
monkeypatch.setattr(inf_mod, "_decode_audio_base64", lambda _b64: object())
payload = ChatCompletionRequest(
model = "org/AUDIO-IN",
messages = [{"role": "user", "content": "transcribe this"}],
audio_base64 = "ZmFrZQ==",
thread_id = "thread-audio-in",
)
response = asyncio.run(
inf_mod.openai_chat_completions(payload, _ChatRequest(), current_subject = "tester")
)
assert response.status_code == 200
assert seen["count"] == 1
assert seen["snapshot"][0]["thread_id"] == "thread-audio-in"
assert seen["cancelled"] == 1
assert seen["reached_the_decode"]
assert active_generations.count() == 0
def _anthropic_route_stubs(monkeypatch, **overrides):
"""Minimal GGUF backend + request stub for the /v1/messages route."""
from types import SimpleNamespace
import routes.inference as inf_mod
from state.tool_policy import reset_tool_policy
reset_tool_policy()
backend = SimpleNamespace(
is_loaded = True,
is_vision = False,
supports_tools = True,
supports_tool_passthrough = True,
model_identifier = "org/M-GGUF",
base_url = "http://llama.test",
context_length = 4096,
count_chat_tokens = lambda *a, **k: 2,
)
backend.__dict__.update(overrides)
monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend)
monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False)
return inf_mod
class _MessagesRequest(_NeverDisconnectedRequest):
"""Minimal stand-in for the Starlette Request /v1/messages reads."""
def __init__(self):
from types import SimpleNamespace
self.method = "POST"
self.url = SimpleNamespace(path = "/v1/messages")
self.state = SimpleNamespace(skip_api_monitor = True)
@pytest.mark.parametrize("with_server_tools", [False, True])
def test_local_anthropic_non_stream_is_visible_to_the_swap_gate(monkeypatch, with_server_tools):
_route_gate()
import asyncio
from models.inference import AnthropicMessagesRequest
seen = {}
def _sample():
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
seen["cancelled"] = active_generations.cancel_all()
def _gen_plain(*, cancel_event = None, **kwargs):
_sample()
seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set()
yield "ok"
def _gen_tools(*, cancel_event = None, **kwargs):
_sample()
seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set()
yield {"type": "content", "text": "ok"}
inf_mod = _anthropic_route_stubs(
monkeypatch,
generate_chat_completion = _gen_plain,
generate_chat_completion_with_tools = _gen_tools,
)
fields = {"max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}
if with_server_tools:
fields["enable_tools"] = True
fields["tools"] = [{"type": "web_search_20250305", "name": "web_search"}]
payload = AnthropicMessagesRequest(**fields)
response = asyncio.run(
inf_mod.anthropic_messages(payload, request = _MessagesRequest(), current_subject = "tester")
)
assert response.status_code == 200
assert seen["count"] == 1
assert seen["snapshot"][0]["model"] == "org/M-GGUF"
assert seen["cancelled"] == 1
assert seen["reached_the_decode"]
assert active_generations.count() == 0
def test_anthropic_passthrough_non_stream_is_visible_to_the_swap_gate(monkeypatch):
_route_gate()
import asyncio
import httpx
from models.inference import AnthropicMessagesRequest
seen = {}
def handler(request):
seen["count"] = active_generations.count()
seen["snapshot"] = active_generations.snapshot()
seen["cancelled"] = active_generations.cancel_all()
return httpx.Response(
200,
json = {
"choices": [
{"message": {"role": "assistant", "content": "33"}, "finish_reason": "stop"}
]
},
)
inf_mod = _anthropic_route_stubs(monkeypatch)
transport = httpx.MockTransport(handler)
real_async_client = httpx.AsyncClient
monkeypatch.setattr(
inf_mod,
"_cancelable_nonstreaming_client",
lambda: real_async_client(transport = transport),
)
payload = AnthropicMessagesRequest(
max_tokens = 16,
messages = [{"role": "user", "content": "hi"}],
enable_tools = False,
tools = [{"name": "lookup", "input_schema": {"type": "object", "properties": {}}}],
)
with pytest.raises(inf_mod.HTTPException) as exc:
asyncio.run(
inf_mod.anthropic_messages(
payload, request = _MessagesRequest(), current_subject = "tester"
)
)
assert exc.value.status_code == 499
assert seen["count"] == 1
assert seen["snapshot"][0]["model"] == "org/M-GGUF"
assert seen["cancelled"] == 1
assert active_generations.count() == 0
def test_anthropic_passthrough_non_stream_stops_when_the_swap_cancels_it(monkeypatch):
_route_gate()
import asyncio
import httpx
from models.inference import AnthropicMessagesRequest
seen = {}
def handler(request):
seen["cancelled"] = active_generations.cancel_all()
raise httpx.ConnectError("client closed")
inf_mod = _anthropic_route_stubs(monkeypatch)
transport = httpx.MockTransport(handler)
real_async_client = httpx.AsyncClient
monkeypatch.setattr(
inf_mod,
"_cancelable_nonstreaming_client",
lambda: real_async_client(transport = transport),
)
payload = AnthropicMessagesRequest(
max_tokens = 16,
messages = [{"role": "user", "content": "hi"}],
enable_tools = False,
tools = [{"name": "lookup", "input_schema": {"type": "object", "properties": {}}}],
)
with pytest.raises(inf_mod.HTTPException) as exc:
asyncio.run(
inf_mod.anthropic_messages(
payload, request = _MessagesRequest(), current_subject = "tester"
)
)
assert exc.value.status_code == 499
assert seen["cancelled"] == 1
assert active_generations.count() == 0
def test_audio_generation_unregisters_when_it_fails(monkeypatch):
_route_gate()
import asyncio
from types import SimpleNamespace
from fastapi import HTTPException
import routes.inference as inf_mod
from models.inference import ChatCompletionRequest
class _BrokenTtsBackend:
active_model_name = "org/TTS"
models = {"org/TTS": {"is_audio": True, "audio_type": "snac"}}
def generate_audio_response(self, **kwargs):
raise RuntimeError("codec exploded")
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_loaded = False, _is_audio = False),
)
monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: _BrokenTtsBackend())
async def _no_auto_switch(*a, **k):
return None
monkeypatch.setattr(inf_mod, "_maybe_auto_switch_model", _no_auto_switch)
payload = ChatCompletionRequest(
model = "org/TTS",
messages = [{"role": "user", "content": "hi"}],
)
with pytest.raises(HTTPException):
asyncio.run(inf_mod.generate_audio(payload, request = None, current_subject = "tester"))
assert active_generations.count() == 0
def _stub_install_route(monkeypatch, *, in_flight_events):
"""A request counts as in flight until its stream unwinds on cancel; the installer reads that count."""
from types import SimpleNamespace
import core.inference.llama_keepwarm as keepwarm
import routes.inference as inf_mod
import utils.transformers_latest as latest_mod
import utils.transformers_version as version_mod
calls = {"installed": [], "released": 0}
monkeypatch.setattr(version_mod, "try_begin_sidecar_swap", lambda: True)
def _end_sidecar_swap():
calls["released"] += 1
monkeypatch.setattr(version_mod, "end_sidecar_swap", _end_sidecar_swap)
import core.export as export_mod
import core.training as training_mod
monkeypatch.setattr(
training_mod,
"get_training_backend",
lambda: SimpleNamespace(is_training_active = lambda: False),
)
monkeypatch.setattr(
export_mod,
"get_export_backend",
lambda: SimpleNamespace(is_export_active = lambda: False, current_checkpoint = None),
)
monkeypatch.setattr(
inf_mod,
"get_inference_backend",
lambda: SimpleNamespace(active_model_name = None, load_generation = 0),
)
def _fake_in_flight(current_request_counted = True, *, include_pending = True):
return sum(1 for ev in in_flight_events if not ev.is_set())
monkeypatch.setattr(keepwarm, "other_inference_request_count", _fake_in_flight)
def _install(version, before_swap, *args, **kwargs):
calls["installed"].append(version)
return {"success": True, "version": version, "message": "installed"}
monkeypatch.setattr(latest_mod, "install_latest_transformers", _install)
return inf_mod, calls
def test_confirmed_install_stops_the_chats_it_was_given_permission_to_stop(monkeypatch):
_route_gate()
import asyncio
from models.inference import InstallLatestTransformersRequest
ev = threading.Event()
inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev])
with active_generations.ActiveGeneration(ev, thread_id = "t1", model = "org/M-GGUF"):
response = asyncio.run(
inf_mod.install_latest_transformers_route(
InstallLatestTransformersRequest(version = "5.0.0", force_cancel_active = True),
"tester",
)
)
assert ev.is_set()
assert response.success is True
assert calls["installed"] == ["5.0.0"]
def test_unconfirmed_install_still_refuses_while_chats_stream(monkeypatch):
_route_gate()
import asyncio
from fastapi import HTTPException
from models.inference import InstallLatestTransformersRequest
ev = threading.Event()
inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev])
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
with pytest.raises(HTTPException) as exc:
asyncio.run(
inf_mod.install_latest_transformers_route(
InstallLatestTransformersRequest(version = "5.0.0"),
"tester",
)
)
assert not ev.is_set()
assert active_generations.count() == 1
assert exc.value.status_code == 409
assert calls["installed"] == []
def test_a_confirmed_install_that_cannot_drain_refuses_instead_of_swapping(monkeypatch):
_route_gate()
import asyncio
from fastapi import HTTPException
from models.inference import InstallLatestTransformersRequest
ev = threading.Event()
stuck = threading.Event()
stuck.set() # already "cancelled", yet still counted: it never unwinds
inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev, stuck])
monkeypatch.setattr(inf_mod, "_POST_CANCEL_DRAIN_TIMEOUT_S", 0.05)
def _never_unwinds(current_request_counted = True, *, include_pending = True):
return 1
import core.inference.llama_keepwarm as keepwarm
monkeypatch.setattr(keepwarm, "other_inference_request_count", _never_unwinds)
async def _install():
# Deadline here too: a regression that drops the drain's bound must fail, not hang the suite.
return await asyncio.wait_for(
inf_mod.install_latest_transformers_route(
InstallLatestTransformersRequest(version = "5.0.0", force_cancel_active = True),
"tester",
),
timeout = 5,
)
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
with pytest.raises(HTTPException) as exc:
asyncio.run(_install())
assert exc.value.status_code == 409
assert calls["installed"] == []
def test_confirmed_install_does_not_spend_its_cancel_on_an_install_that_will_refuse(monkeypatch):
_route_gate()
import asyncio
from fastapi import HTTPException
from models.inference import InstallLatestTransformersRequest
ev = threading.Event()
inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev])
import core.inference.llama_keepwarm as keepwarm
def _never_drains(current_request_counted = True, *, include_pending = True):
return 2
monkeypatch.setattr(keepwarm, "other_inference_request_count", _never_drains)
monkeypatch.setattr(inf_mod, "_POST_CANCEL_DRAIN_TIMEOUT_S", 0.05)
async def _install():
return await asyncio.wait_for(
inf_mod.install_latest_transformers_route(
InstallLatestTransformersRequest(version = "5.0.0", force_cancel_active = True),
"tester",
),
timeout = 5,
)
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
with pytest.raises(HTTPException) as exc:
asyncio.run(_install())
assert not ev.is_set()
assert active_generations.count() == 1
assert exc.value.status_code == 409
assert calls["installed"] == []
def _drain_with_counts(monkeypatch, counts, **kwargs):
"""Scripted counts repeat their last value, so a trailing non-zero is a request that never unwinds."""
_route_gate()
import asyncio
import core.inference.llama_keepwarm as keepwarm
import routes.inference as inf_mod
remaining = list(counts)
polls = {"n": 0}
def _count(current_request_counted = True, *, include_pending = True):
polls["n"] += 1
return remaining.pop(0) if len(remaining) > 1 else remaining[0]
monkeypatch.setattr(keepwarm, "other_inference_request_count", _count)
monkeypatch.setattr(inf_mod, "_switch_waiter_count", lambda: 0)
async def _run():
# Hard test-side deadline: a drain that regresses to waiting forever must fail red, not hang.
await asyncio.wait_for(
inf_mod._wait_for_model_switch_idle(current_request_counted = False, **kwargs),
timeout = 5,
)
asyncio.run(_run())
return polls["n"]
def test_forced_swap_does_not_wait_out_the_generations_it_is_about_to_cancel(monkeypatch):
# cancel_pending discounts registered generations; without it the drain waits forever.
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
polls = _drain_with_counts(monkeypatch, [1], cancel_pending = True)
assert polls == 1
def test_the_same_drain_without_the_discount_would_keep_waiting(monkeypatch):
ev = threading.Event()
with active_generations.ActiveGeneration(ev, thread_id = "t1"):
polls = _drain_with_counts(monkeypatch, [1], timeout_s = 0.05)
assert polls > 1
def test_post_cancel_drain_gives_up_on_a_request_that_never_unwinds(monkeypatch):
polls = _drain_with_counts(monkeypatch, [1], timeout_s = 0.05)
assert polls > 1
def test_drain_returns_as_soon_as_the_cancelled_requests_unwind(monkeypatch):
polls = _drain_with_counts(monkeypatch, [2, 1, 0], timeout_s = 30)
assert polls == 3
def _ids(n):
return [str(uuid.uuid4()) for _ in range(n)]
def _stopped(ledger):
"""What the worker would read: the only way the record is read."""
return ledger.snapshot()[1]
def test_a_request_reads_as_stopped_once_it_is():
ledger = StopLedger(_mp.get_context("spawn"))
mine, theirs = _ids(2)
assert _stopped(ledger) == set()
assert ledger.stop(mine)
stops = _Stops(ledger, None, None, [])
assert stops.unread() and stops._refresh() is None and mine in stops and not stops.unread()
assert _stopped(ledger) == {mine}, "a stop names one request and no other"
assert theirs not in _stopped(ledger)
def test_the_oldest_stop_is_the_one_that_ages_out():
ledger = StopLedger(_mp.get_context("spawn"))
recorded = _ids(_SLOTS + 1)
for request_id in recorded:
assert ledger.stop(request_id)
assert _stopped(ledger) == set(recorded[1:]), "the oldest made way for the newest"
def test_stopping_the_same_request_twice_spends_one_slot():
ledger = StopLedger(_mp.get_context("spawn"))
theirs, mine = _ids(2)
assert ledger.stop(theirs)
for _ in range(_SLOTS):
assert ledger.stop(mine)
assert _stopped(ledger) == {mine, theirs}, "the one stopped first is still stopped"
def test_a_snapshot_answers_for_every_reply_at_once():
ledger = StopLedger(_mp.get_context("spawn"))
mine, absent = _ids(2)
theirs = "short"
ledger.stop(mine)
ledger.stop(theirs)
written, stopped = ledger.snapshot()
assert written == 2
assert stopped == {mine, theirs}, "the ids as recorded, with nothing padded onto them"
assert absent not in stopped
class _Ledger:
def __init__(self, read_by_worker = False):
self.stopped: set = set()
self._read_by_worker = read_by_worker
def stop(self, request_id):
self.stopped.add(request_id)
return True
def read_by_worker(self) -> bool:
return self._read_by_worker
def _orchestrator_for_ownership():
"""A real InferenceOrchestrator with just enough stubbed to drive the lock."""
_route_gate()
orch_mod = pytest.importorskip(
"core.inference.orchestrator", reason = "inference stack not installed"
)
orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator)
orch._stop_ledger = _Ledger()
orch._pending_teardowns = None
orch._gen_lock = threading.Lock()
orch._active_cancel_events = []
orch._executing_cancel_events = []
orch._active_cancel_lock = threading.Lock()
orch._send_order_lock = threading.Lock()
orch._mailbox_lock = threading.Lock()
orch._request_cancel_events = {}
orch._cancel_event = threading.Event()
orch._ensure_subprocess_alive = lambda: False
return orch
def test_a_queued_chat_cannot_reset_the_chat_that_is_generating():
# One shared worker cancel event: Stop on a queued chat must not kill the running one.
orch = _orchestrator_for_ownership()
a_event = threading.Event()
b_event = threading.Event()
orch._claim_worker(a_event)
orch._mark_worker_started(a_event)
orch.reset_generation_state(b_event)
assert not orch._cancel_event.is_set()
orch.reset_generation_state(a_event)
assert orch._cancel_event.is_set()
def test_a_global_reset_still_cancels_whatever_is_running():
orch = _orchestrator_for_ownership()
_running = threading.Event()
orch._claim_worker(_running)
orch._mark_worker_started(_running)
orch.reset_generation_state()
assert orch._cancel_event.is_set()
def test_a_reset_with_no_generation_running_is_not_dropped():
orch = _orchestrator_for_ownership()
orch.reset_generation_state(threading.Event())
assert orch._cancel_event.is_set()
def test_unload_waits_for_a_request_that_is_admitted_but_not_yet_registered(monkeypatch):
_route_gate()
import core.inference.llama_keepwarm as keepwarm
import routes.inference as inf_mod
remaining = [1, 1, 0]
seen = {}
def _count(current_request_counted = True, *, include_pending = True):
return remaining.pop(0) if len(remaining) > 1 else remaining[0]
monkeypatch.setattr(keepwarm, "other_inference_request_count", _count)
monkeypatch.setattr(inf_mod, "_switch_waiter_count", lambda: 0)
torn_down: list[str] = []
def _record_teardown():
seen["counted_at_teardown"] = remaining[0]
torn_down.append("gguf")
response = _run_unload(
inf_mod,
monkeypatch,
loaded_gguf = "org/A-GGUF",
requested = "org/A-GGUF",
force = True,
torn_down = torn_down,
unload_model = _record_teardown,
)
assert active_generations.count() == 0
assert torn_down == ["gguf"]
assert seen["counted_at_teardown"] == 0
assert response.status == "unloaded"
def test_a_dispatched_chat_cannot_reset_its_concurrently_dispatched_sibling():
orch = _orchestrator_for_ownership()
a_event = threading.Event()
b_event = threading.Event()
c_event = threading.Event()
orch._claim_worker(a_event)
orch._mark_worker_started(a_event)
orch._claim_worker(b_event)
orch._mark_worker_started(b_event)
orch.reset_generation_state(c_event)
assert not orch._cancel_event.is_set()
orch.reset_generation_state(b_event)
assert orch._cancel_event.is_set()
def test_releasing_one_generation_leaves_the_other_claimed():
orch = _orchestrator_for_ownership()
a_event = threading.Event()
b_event = threading.Event()
orch._claim_worker(a_event)
orch._mark_worker_started(a_event)
orch._claim_worker(b_event)
orch._mark_worker_started(b_event)
orch._release_worker(a_event)
orch.reset_generation_state(a_event)
assert not orch._cancel_event.is_set()
orch._release_worker(b_event)
orch.reset_generation_state(a_event)
assert orch._cancel_event.is_set()
def test_a_dispatched_request_queued_behind_another_is_not_an_owner():
orch = _orchestrator_for_ownership()
a_event = threading.Event()
b_event = threading.Event()
orch._claim_worker(a_event)
orch._mark_worker_started(a_event)
orch._claim_worker(b_event)
orch.reset_generation_state(b_event)
assert not orch._cancel_event.is_set(), "a queued request must not reset A"
orch._mark_worker_started(b_event)
orch.reset_generation_state(b_event)
assert orch._cancel_event.is_set()
def test_a_queued_request_cannot_reset_during_the_other_ones_prefill():
orch = _orchestrator_for_ownership()
a_event = threading.Event()
b_event = threading.Event()
orch._claim_worker(a_event)
orch._claim_worker(b_event)
orch.reset_generation_state(b_event)
assert not orch._cancel_event.is_set(), "B must not reset A during prefill"
orch.reset_generation_state(a_event)
assert orch._cancel_event.is_set()
def test_the_oldest_claim_is_the_one_the_worker_is_prefilling():
orch = _orchestrator_for_ownership()
a_event = threading.Event()
b_event = threading.Event()
orch._claim_worker(a_event)
orch._claim_worker(b_event)
orch._release_worker(a_event)
orch.reset_generation_state(b_event)
assert orch._cancel_event.is_set(), "B is now the oldest claim"
def test_claim_order_matches_send_order_under_concurrent_dispatch():
# The claim must be atomic with the enqueue, or claim order diverges from the subprocess queue.
_route_gate()
orch_mod = pytest.importorskip(
"core.inference.orchestrator", reason = "inference stack not installed"
)
orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator)
orch._stop_ledger = None
orch._pending_teardowns = None
orch._active_cancel_events = []
orch._executing_cancel_events = []
orch._active_cancel_lock = threading.Lock()
orch._send_order_lock = threading.Lock()
sent: list = []
barrier = threading.Barrier(4)
def worker(ev):
barrier.wait(timeout = 10)
with orch._send_order_lock:
orch._claim_worker(ev)
sent.append(ev)
events = [threading.Event() for _ in range(4)]
threads = [threading.Thread(target = worker, args = (e,)) for e in events]
for t in threads:
t.start()
for t in threads:
t.join(timeout = 30)
assert orch._active_cancel_events == sent, "claim order must equal send order"
def test_responses_stream_reports_reasoning_ttft_and_stop_reason(monkeypatch):
# This adapter parses SSE itself, so it must stamp TTFT for reasoning-first turns.
import asyncio
from core.inference.api_monitor import api_monitor
from models.inference import ChatMessage, ResponsesRequest
inf_mod = _install_responses_stream_mock(
monkeypatch,
[
{"choices": [{"delta": {"reasoning_content": "thinking"}}]},
{"choices": [{"delta": {"content": "hi"}, "finish_reason": "length"}]},
],
)
payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF")
messages = [ChatMessage(role = "user", content = "hi")]
monitor_id = api_monitor.start(
endpoint = "/v1/responses", method = "POST", model = "org/M-GGUF", prompt = "hi"
)
async def run():
response = await inf_mod._responses_stream(
payload, messages, _NeverDisconnectedRequest(), monitor_id
)
async for _ in response.body_iterator:
pass
asyncio.run(run())
rows = [r for r in api_monitor.snapshot() if r["id"] == monitor_id]
assert rows, "the stream should have opened a monitor row"
assert rows[0]["ttft_ms"] is not None
assert rows[0]["stop_reason"] == "length"
def test_responses_stream_stamps_tool_call_deltas(monkeypatch):
import asyncio
from core.inference.api_monitor import api_monitor
from models.inference import ChatMessage, ResponsesRequest
inf_mod = _install_responses_stream_mock(
monkeypatch,
[
{
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"id": "call_1",
"function": {"name": "f", "arguments": "{}"},
}
]
}
}
]
},
{"choices": [{"delta": {}, "finish_reason": "tool_calls"}]},
],
)
payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF")
messages = [ChatMessage(role = "user", content = "hi")]
monitor_id = api_monitor.start(
endpoint = "/v1/responses", method = "POST", model = "org/M-GGUF", prompt = "hi"
)
stamped: list[str] = []
real_mark = api_monitor.mark_first_token
monkeypatch.setattr(
api_monitor,
"mark_first_token",
lambda mid: (stamped.append(mid), real_mark(mid))[1],
)
async def run():
response = await inf_mod._responses_stream(
payload, messages, _NeverDisconnectedRequest(), monitor_id
)
async for _ in response.body_iterator:
pass
asyncio.run(run())
assert stamped, "the tool-call delta should stamp the first token"
[row] = [r for r in api_monitor.snapshot() if r["id"] == monitor_id]
assert row["ttft_ms"] is not None
assert row["stop_reason"] == "tool_calls"
def _run_external_tool_chat(during):
import asyncio
import json
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from types import SimpleNamespace
import httpx
from starlette.requests import Request
import routes.inference as inf_mod
from core.inference import external_provider as ep_mod
from models.inference import ChatCompletionRequest
release = threading.Event()
class _SlowProvider(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *args, **kwargs):
return
def do_POST(self):
self.rfile.read(int(self.headers.get("Content-Length") or 0))
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Connection", "close")
self.end_headers()
for word in ("one ", "two "):
chunk = {"choices": [{"index": 0, "delta": {"content": word}}]}
self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())
self.wfile.flush()
release.wait(10)
done = {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
self.wfile.write(f"data: {json.dumps(done)}\n\ndata: [DONE]\n\n".encode())
self.wfile.flush()
server = ThreadingHTTPServer(("127.0.0.1", 0), _SlowProvider)
threading.Thread(target = server.serve_forever, daemon = True).start()
async def receive():
await asyncio.Event().wait()
payload = ChatCompletionRequest(
provider_type = "custom",
provider_base_url = f"http://127.0.0.1:{server.server_address[1]}/v1",
model = "remote-model",
messages = [{"role": "user", "content": "hi"}],
stream = True,
thread_id = "external-chat",
cancel_id = "external-run",
enable_tools = True,
enabled_tools = ["python"],
)
request = Request(
{
"type": "http",
"http_version": "1.1",
"method": "POST",
"path": "/api/inference/chat/completions",
"raw_path": b"/api/inference/chat/completions",
"root_path": "",
"scheme": "http",
"query_string": b"",
"headers": [(b"content-type", b"application/json"), (b"x-unsloth-events", b"1")],
"client": ("127.0.0.1", 12345),
"server": ("127.0.0.1", 8000),
"app": SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1)),
},
receive,
)
async def run():
previous = ep_mod._http_client
ep_mod._http_client = httpx.AsyncClient()
try:
response = await inf_mod._proxy_to_external_provider(payload, request)
iterator = response.body_iterator
chunks = []
while active_generations.count() == 0:
chunks.append(await asyncio.wait_for(iterator.__anext__(), 10))
seen = await during(inf_mod, request)
release.set()
async for chunk in iterator:
chunks.append(chunk)
return seen, "".join(c.decode() if isinstance(c, bytes) else c for c in chunks)
finally:
release.set()
await ep_mod._http_client.aclose()
ep_mod._http_client = previous
server.shutdown()
server.server_close()
return asyncio.run(run())
def test_an_external_tool_chat_is_not_stopped_by_a_local_model_swap():
_route_gate()
async def during(inf_mod, request):
[entry] = active_generations.snapshot()
listed = await inf_mod.get_active_generations(request, "tester")
inf_mod._raise_or_cancel_active_generations(force = False, action = "Unloading the model")
cancelled = inf_mod._raise_or_cancel_active_generations(
force = True, action = "Unloading the model"
)
return entry, listed, cancelled
(entry, listed, cancelled), body = _run_external_tool_chat(during)
assert entry["thread_id"] == "external-chat"
assert listed["count"] == 0 and listed["thread_ids"] == []
assert cancelled == 0
assert "two" in body and "[DONE]" in body
assert active_generations.count() == 0
def test_external_registration_racing_the_swap_gate_stays_excluded(monkeypatch):
gate = _route_gate()
event = threading.Event()
tracker = active_generations.ActiveGeneration(
event,
thread_id = "external-chat",
external = True,
)
original_count = active_generations.count
entered = False
def count_after_registration(*args, **kwargs):
nonlocal entered
if not entered:
tracker.__enter__()
entered = True
return original_count(*args, **kwargs)
monkeypatch.setattr(active_generations, "count", count_after_registration)
try:
assert gate(force = False, action = "Unloading the model") == 0
assert not event.is_set()
finally:
if entered:
tracker.__exit__(None, None, None)
def test_an_external_tool_chat_still_stops_on_its_own_stop():
_route_gate()
async def during(inf_mod, request):
return inf_mod._cancel_by_cancel_id_or_stash("external-run")
stopped, _ = _run_external_tool_chat(during)
assert stopped == 1
assert active_generations.count() == 0
def test_deleting_an_external_tool_chat_still_stops_it():
_route_gate()
async def during(inf_mod, request):
return active_generations.cancel_thread("external-chat")
stopped, _ = _run_external_tool_chat(during)
assert stopped == 1