Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Replies joining and leaving a batch the MLX worker keeps open."""
import functools
import importlib
import inspect
import multiprocessing as _mp
import queue as _queue
import sys
import threading
import types
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
def _stub_if_missing(name, attrs):
if name in sys.modules:
return
try:
importlib.import_module(name)
return
except Exception: # noqa: BLE001 - stub unusable imports
pass
mod = types.ModuleType(name)
mod.__spec__ = None
for attr in attrs:
setattr(mod, attr, MagicMock())
sys.modules[name] = mod
_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
from core.inference import worker # noqa: E402
from core.inference.worker import RowRefused # noqa: E402
from core.inference.worker import StopLedger # noqa: E402
class _RespQueue:
def __init__(self):
self.sent = []
def put(self, item, *a, **k):
self.sent.append(item)
class _Session:
"""A batch that hands back whatever the test scripted for each reply."""
def __init__(self, *, width):
self.opened_at = width
self.rows, self.ending, self.refusals = [], [], []
self.settled, self.script, self.stats_at_retire = {}, {}, {}
self.will_not_take, self.closed, self.admitted = set(), False, []
@property
def rows_in_flight(self):
return len(self.rows)
@property
def handles(self):
return list(self.rows)
def take_stats(self, handle):
return self.settled.pop(handle, None)
def admit(self, request, handle):
if handle[0] in self.will_not_take or handle in self.will_not_take:
self.refusals.append(handle)
raise RowRefused("it does not prepare like the replies in the batch")
self.admitted.append((handle, request))
self.rows.append(handle)
return ""
def _retire(self, handle):
self.rows.remove(handle)
self.settled[handle] = self.stats_at_retire.get(handle, {})
def step(self, waiting):
for handle in list(self.rows):
events = self.script.get(handle) or []
if not events:
continue
event = events.pop(0)
if event is None:
self._retire(handle)
yield handle, event
def withdraw(self, handles):
for handle in [h for h in handles if h in self.rows]:
self._retire(handle)
yield handle, None
def close(self):
self.closed = True
class _Backend:
def __init__(
self,
reason = None,
script = None,
):
self.reason = reason
self.script = script or {}
self.sessions = []
self.on_open = None
self.on_reason = None
self.apart = []
def generate_chat_response(self, **kwargs):
self.apart.append(kwargs)
yield "apart"
def resident_unavailable_reason(self, request):
if self.on_reason is not None:
self.on_reason()
return self.reason
def open_resident_batch(
self,
*,
width,
adapter_state = None,
):
session = _Session(width = width)
session.script.update(self.script)
if self.on_open is not None:
self.on_open(session)
self.sessions.append(session)
return session
def _cmd(request_id = "r1", **extra):
return {"type": "generate", "request_id": request_id, "messages": [], **extra}
def _batch(backend = None):
resp = _RespQueue()
return worker._ResidentBatch(backend or _Backend(), resp), resp
def test_replies_a_command_asked_for_together_are_addressed_and_counted_by_row():
batch, resp = _batch()
batch.admit(_cmd(rows = [{"seed": 1}, {"seed": 2}]), None)
for row, stats in ((0, {"n": 1}), (1, {"n": 2})):
batch.session.script[("r1", row)] = ["ab"[row], None]
batch.session.stats_at_retire[("r1", row)] = stats
for _ in range(2):
batch.step()
rows = [(m["type"], m.get("row")) for m in resp.sent]
assert ("token", 0) in rows and ("token", 1) in rows
assert rows.count(("row_done", 0)) == 1 and rows.count(("row_done", 1)) == 1
done = [m for m in resp.sent if m["type"] == "row_done"]
assert sorted(m["stats"]["n"] for m in done) == [1, 2]
assert [m["type"] for m in resp.sent][-1] == "gen_done"
assert resp.sent[-1]["stats"] is None
def test_a_reply_whose_own_decode_fails_is_answered_alone_and_the_batch_decodes_on():
batch, resp = _batch()
batch.admit(_cmd("r1", rows = [{"seed": 1}, {"seed": 2}], parallel_slots = 4), None)
batch.admit(_cmd("r2", parallel_slots = 4), None)
batch.admit(_cmd("r3", parallel_slots = 4), None)
session = batch.session
def step(_waiting):
session.rows.remove(("r1", 0))
session.rows.remove(("r3", None))
yield ("r1", 0), ValueError("schema")
yield ("r3", None), KeyError("bind")
yield ("r2", None), "b"
# The failed reply's sibling ending in the same step does not answer it as done.
yield ("r1", 1), None
session.step = step
batch.step()
sent = [(m["type"], m["request_id"], m.get("error")) for m in resp.sent]
assert sent == [
("gen_error", "r1", "schema"),
("gen_error", "r3", "'bind'"),
("token", "r2", None),
]
assert "ValueError" in resp.sent[0]["stack"] and "KeyError" in resp.sent[1]["stack"]
assert session.rows == [("r2", None)] and not session.closed
def test_a_failed_reply_whose_siblings_cannot_be_withdrawn_is_still_answered():
batch, resp = _batch()
batch.admit(_cmd("r1", rows = [{"seed": 1}, {"seed": 2}], parallel_slots = 4), None)
batch.admit(_cmd("r2", parallel_slots = 4), None)
session = batch.session
def step(_waiting):
session.rows.remove(("r1", 0))
yield ("r1", 0), ValueError("schema")
def withdraw(_handles):
raise RuntimeError("metal")
yield
session.step, session.withdraw = step, withdraw
batch.step()
errors = {m["request_id"]: m["error"] for m in resp.sent if m["type"] == "gen_error"}
assert errors == {"r1": "schema", "r2": "schema"} and batch.session is None
def test_raw_completion_rows_join_the_batch_and_decode_alone_through_the_completion_path():
batch, _ = _batch()
cmd = _cmd(
rows = [{"prompt_ids": [5], "seed": 1}, {"prompt_ids": [6], "seed": 2}],
penalty_window = 3,
parallel_slots = 2,
)
assert batch.unavailable_reason(cmd) is None and batch.admit(cmd, None)
assert [(r["prompt_ids"], r["penalty_window"]) for _, r in batch.session.admitted] == [
([5], 3),
([6], 3),
]
backend = SimpleNamespace(generate_completion = lambda ids, **kw: iter([f"{ids} {kw['seed']}"]))
apart = worker._rows_apart(backend, [{"prompt_ids": [5], "seed": 1}], threading.Event(), [None])
assert list(apart) == [(0, "[5] 1"), (0, None)]
IDLE = object()
def _run_loop(monkeypatch, backend, cmds):
"""Drive the real worker loop over a scripted queue."""
import queue as _stdqueue
from utils.hardware import hardware as _hw
from loggers.config import LogConfig
from utils import worker_stderr
import core.inference.mlx_inference as mlx_mod
monkeypatch.setenv("ENVIRONMENT_TYPE", "development")
inert = lambda *a, **k: None # noqa: E731
for name, value in (
("is_apple_silicon", lambda: True),
("apply_gpu_ids", inert),
("_recorded_local_base", lambda m: (None, False)),
("_hub_targets_are_local", lambda *a, **k: True),
("_activate_transformers_version", inert),
("_handle_load", inert),
):
monkeypatch.setattr(worker, name, value)
monkeypatch.setattr(_hw, "detect_hardware", inert)
monkeypatch.setattr(_hw, "DEVICE", _hw.DeviceType.MLX)
monkeypatch.setattr(LogConfig, "setup_logging", staticmethod(lambda *a, **k: None))
monkeypatch.setattr(worker_stderr, "mark_log_record_continuations", inert)
monkeypatch.setattr(mlx_mod, "MLXInferenceBackend", lambda *a, **k: backend)
class _Script:
def __init__(self, items):
self._items = list(items)
def get(self, timeout = None):
if self._items and (item := self._items.pop(0)) is not IDLE:
return item
raise _stdqueue.Empty
resp = _RespQueue()
never = SimpleNamespace(is_set = lambda: False, clear = lambda: None, set = lambda: None)
worker.run_inference_process(
cmd_queue = _Script([*cmds, {"type": "shutdown"}]),
resp_queue = resp,
cancel_event = never,
config = {"model_name": "unsloth/orpheus-3b-0.1-ft"},
)
return resp
def test_a_one_slot_load_runs_its_reply_without_ever_opening_a_batch(monkeypatch):
"""A batch one reply wide is one nothing can join, so it is never opened."""
backend = _Backend()
resp = _run_loop(monkeypatch, backend, [_cmd("r1", parallel_slots = 1), IDLE])
assert backend.sessions == [], "a batch was opened for a single reply"
assert [m["type"] for m in resp.sent if m.get("request_id") == "r1"] == ["token", "gen_done"]
_HOLD_CASES = [
pytest.param(
dict(
cmds = [("r1", 4), ("r2", 2), ("r3", 4), ("r4", 4)],
script = {"r1": ["a", None], "r2": ["b", None]},
widths = [4, 2, 4],
order = ["r1", "r2", "r3", "r4"],
),
id = "the head runs on a queue that never falls quiet",
),
pytest.param(
dict(
cmds = [("r1", 4), ("r2", 2), ("r3", 2), ("r4", 2)],
script = {
"r1": ["a"] * 4 + [None],
"r2": ["b", "b", "b", None],
"r3": ["c", "c", None],
"r4": ["d", None],
},
idle = 12,
widths = [4, 2, 2],
order = ["r1", "r2", "r3", "r4"],
),
id = "everything held behind one incompatible command still batches, within the width",
),
pytest.param(
dict(
cmds = [("r1", 2), ("r2", 2), ("r3", 2)],
script = {"r1": ["a"] * 4 + [None], "r2": ["b", "b", None], "r3": ["c", None]},
refuse = ("r2", 0),
idle = 12,
refusals = (0, "r2"),
order = ["r1", "r2", "r3"],
),
id = "a command the batch turned away is not offered to it again",
),
pytest.param(
dict(
cmds = [("r1", 4), ("r0", 2), ("r2", 2), ("r3", 2)],
script = {
"r1": ["a"] * 5 + [None],
"r0": ["x", "x", "x", None],
"r2": ["b", None],
"r3": ["c", None],
},
refuse = ("r2", 1),
idle = 16,
refusals = (1, "r2"),
order = ["r1", "r0", "r2", "r3"],
),
id = "a head the batch turns away keeps its place over what waited behind it",
),
]
@pytest.mark.parametrize("case", _HOLD_CASES)
def test_the_hold_releases_its_head_and_keeps_its_order(monkeypatch, case):
"""The hold's whole contract, over the traffic that breaks each part of it."""
backend = _Backend()
backend.script = {(name, None): events for name, events in case["script"].items()}
if "refuse" in case:
who, when = case["refuse"]
backend.on_open = lambda session: (
session.will_not_take.add(who) if len(backend.sessions) == when else None
)
resp = _run_loop(
monkeypatch,
backend,
[_cmd(name, parallel_slots = width) for name, width in case["cmds"]]
+ [IDLE] * case.get("idle", 0),
)
admitted = [handle[0] for session in backend.sessions for handle, _r in session.admitted]
assert admitted == case["order"], f"the hold lost or reordered a command: {admitted}"
if "widths" in case:
assert [session.opened_at for session in backend.sessions] == case["widths"]
if "refusals" in case:
index, name = case["refusals"]
assert backend.sessions[index].refusals == [(name, None)]
if case.get("idle"):
terminal = [
m["request_id"] for m in resp.sent if m.get("type") in ("gen_done", "gen_error")
]
assert sorted(terminal) == sorted(case["order"]), f"a held reply was lost: {resp.sent}"
class _DecliningBackend:
def __init__(self, reason):
self._reason, self.batched = reason, 0
self.last_generation_stats = {"usage": {"completion_tokens": 2}}
def batch_unavailable_reason(self, requests):
return None if len(requests) < 2 else self._reason
def generate_chat_response(
self,
seed = None,
**kwargs,
):
yield f"seed {seed}"
yield f"seed {seed} done"
def generate_chat_batch(self, requests, **kwargs):
self.batched += 1
raise AssertionError("a declined request set must not reach the batch")
def _run(
reason,
cancelled = False,
backend = None,
):
from core.inference import worker
sent, cancel = [], threading.Event()
backend = backend or _DecliningBackend(reason)
if cancelled:
cancel.set()
worker._handle_generate_rows(
backend,
{
"request_id": "r1",
"messages": [{"role": "user", "content": "hi"}],
"rows": [{"seed": 1}, {"seed": 2}],
"parallel_slots": 2,
},
SimpleNamespace(put = sent.append),
cancel,
)
return backend, sent
def test_a_declined_request_set_is_decoded_one_reply_at_a_time():
"""Reporting the command done without decoding would answer a caller's two."""
backend, sent = _run("a reply asks for stop sequences")
kinds = [event["type"] for event in sent if event["type"] != "batch_state"]
tokens = [event for event in sent if event["type"] == "token"]
assert backend.batched == 0
assert kinds == ["token", "token", "row_done", "token", "token", "row_done", "gen_done"]
assert [(event["row"], event["text"]) for event in tokens] == [
(0, "seed 1"),
(0, "seed 1 done"),
(1, "seed 2"),
(1, "seed 2 done"),
], "each row decodes with the request it was given, not with row zero's"
assert [event["row"] for event in sent if event["type"] == "row_done"] == [0, 1]
assert all(event["request_id"] == "r1" for event in sent)
assert all(
event["stats"] == backend.last_generation_stats
for event in sent
if event["type"] == "row_done"
)
def test_a_cancelled_fallback_still_reports_every_row_done():
"""A row nobody decoded is still a row somebody is waiting on."""
_backend, sent = _run("a reply asks for stop sequences", cancelled = True)
assert [event["type"] for event in sent if event["type"] != "batch_state"] == [
"row_done",
"row_done",
"gen_done",
]
assert [event["row"] for event in sent if event["type"] == "row_done"] == [0, 1]
assert all(event["stats"] is None for event in sent if event["type"] == "row_done")
def test_a_batch_that_will_not_open_still_answers_the_reply(monkeypatch):
"""Building the batch is what fails, and the reply predates any batch existing."""
def refuse(**_kwargs):
raise RuntimeError("BatchGenerator.insert missing")
backend = _Backend()
backend.open_resident_batch = refuse
resp = _run_loop(monkeypatch, backend, [_cmd(parallel_slots = 2), IDLE])
assert [m["type"] for m in resp.sent if m["type"] != "status"] == ["token", "gen_done"]
assert len(backend.apart) == 1, "it was decoded one reply at a time"
def test_a_row_override_the_backend_cannot_take_is_dropped_not_raised():
"""The route varies the seed per row, and the fallback decodes on backends without one."""
InferenceBackend = pytest.importorskip("core.inference.inference").InferenceBackend
class _NoSeedBackend(_DecliningBackend):
"""Transformers-shaped: wraps() carries the real signature, which declares no seed."""
@functools.wraps(InferenceBackend.generate_chat_response)
def generate_chat_response(self, *args, **kwargs):
inspect.signature(InferenceBackend.generate_chat_response).bind(self, *args, **kwargs)
yield "reply"
_backend, sent = _run(None, backend = _NoSeedBackend("a reply asks for stop sequences"))
kinds = [event["type"] for event in sent if event["type"] != "batch_state"]
assert kinds == ["token", "row_done", "token", "row_done", "gen_done"], f"{sent}"
def test_a_count_is_refused_while_the_batch_decodes(monkeypatch):
"""Dispatched generations hold no orchestrator lock, so the loop keeps its promise."""
backend = _Backend(script = {("r1", None): ["a", None]})
backend.count_chat_tokens, backend.active_model_name = (lambda *a, **k: 7), "m"
count = {"type": "count_tokens", "request_id": "c1", "messages": []}
cmds = [_cmd("r1", parallel_slots = 2), count, IDLE, IDLE, IDLE, count, IDLE]
sent = _run_loop(monkeypatch, backend, cmds).sent
counts = [m for m in sent if m["type"] == "count_tokens_response"]
assert [m.get("error") for m in counts] == [
"A generation is in progress.",
None,
], f"the count was served mid-decode, or never ran once the batch drained: {counts}"
assert counts[1]["input_tokens"] == 7