Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""The dictation engine lives in a spawn child, since an accelerator context is freed only at exit."""
import queue
import signal
import threading
import time
from types import SimpleNamespace
import numpy as np
import pytest
import core.inference.stt_transformers_worker as worker_module
from core.inference.stt_sidecar import (
SttLoadCancelledError,
SttModelNotDownloadedError,
SttTranscriptionCancelledError,
)
from core.inference.stt_transformers_worker import SttWorkerError, WhisperWorker
# Windows has no SIGKILL name and reports -9 by number.
_SIGKILL_TEXT = "SIGKILL" if hasattr(signal, "SIGKILL") else "SIG9"
class _FakeTensor:
def __init__(self, dtype = None) -> None:
self.dtype = dtype
self.moved_to = []
def to(self, value):
self.moved_to.append(value)
return self
class _FakeProcessor:
def __init__(self) -> None:
self.seen_audio = None
self.seen_rate = None
self.features = _FakeTensor()
self.attention_mask = _FakeTensor()
def __call__(
self,
audio,
sampling_rate = None,
return_tensors = None,
return_attention_mask = False,
):
self.seen_audio = audio
self.seen_rate = sampling_rate
result = SimpleNamespace(input_features = self.features)
if return_attention_mask:
result.attention_mask = self.attention_mask
return result
def batch_decode(self, _generated, **_kwargs):
return ["hello"]
class _FakeModel:
def __init__(self, dtype = "float16") -> None:
self.dtype = dtype
self.device = "cuda"
self.generation_config = SimpleNamespace(is_multilingual = True)
self.generate_kwargs = None
self.moved_to = None
self.evaluated = False
def to(self, device):
self.moved_to = device
return self
def eval(self):
self.evaluated = True
return self
def generate(self, _features, **kwargs):
self.generate_kwargs = kwargs
return [[1]]
class _FakeProcess:
"""Stands in for mp.Process; alive until something ends it."""
def __init__(
self,
pid = 4242,
alive = True,
) -> None:
self.pid = pid
self._alive = alive
self.exitcode = None
self.terminated = False
self.killed = False
def is_alive(self):
return self._alive
def join(self, _timeout = None):
return None
def terminate(self):
self.terminated = True
self._alive = False
self.exitcode = -15
def kill(self):
self.killed = True
self._alive = False
self.exitcode = -9
def _wired_worker(process = None):
"""A handle wired to in-process queues, so no child is ever spawned."""
handle = WhisperWorker()
handle._process = process if process is not None else _FakeProcess()
handle._cmd_queue = queue.Queue()
handle._resp_queue = queue.Queue()
handle._cancel_event = threading.Event()
return handle
def _install_fake_transformers(
monkeypatch,
model = None,
processor = None,
):
fake_model = model if model is not None else _FakeModel()
fake_processor = processor if processor is not None else _FakeProcessor()
calls = []
class FakeWhisperForConditionalGeneration:
@classmethod
def from_pretrained(cls, path, **kwargs):
calls.append(("model", path, kwargs))
return fake_model
class FakeWhisperProcessor:
@classmethod
def from_pretrained(cls, path, **kwargs):
calls.append(("processor", path, kwargs))
return fake_processor
class _NoGrad:
def __enter__(self):
return None
def __exit__(self, *_args):
return False
monkeypatch.setitem(
__import__("sys").modules,
"transformers",
SimpleNamespace(
WhisperForConditionalGeneration = FakeWhisperForConditionalGeneration,
WhisperProcessor = FakeWhisperProcessor,
StoppingCriteriaList = list,
),
)
monkeypatch.setitem(
__import__("sys").modules,
"torch",
SimpleNamespace(
float16 = "float16",
float32 = "float32",
device = lambda value: value,
no_grad = _NoGrad,
),
)
return calls, fake_model, fake_processor
def test_child_loads_from_the_model_hub_cache_without_an_implicit_download(monkeypatch):
calls, model, _processor = _install_fake_transformers(monkeypatch)
worker_module.load_whisper("/cached/model", "cuda", "float16")
assert {(kind, path) for kind, path, _ in calls} == {
("processor", "/cached/model"),
("model", "/cached/model"),
}
assert all(kwargs.get("local_files_only") is True for _, _, kwargs in calls)
# The weight load forces safetensors so a pickle checkpoint cannot execute.
model_kwargs = next(kwargs for kind, _, kwargs in calls if kind == "model")
assert model_kwargs.get("use_safetensors") is True
assert model_kwargs.get("torch_dtype") == "float16"
assert model.moved_to == "cuda"
assert model.evaluated is True
def test_child_load_stops_at_the_first_checkpoint_after_a_cancel(monkeypatch):
_calls, model, _processor = _install_fake_transformers(monkeypatch)
cancel_event = threading.Event()
cancel_event.set()
with pytest.raises(SttLoadCancelledError):
worker_module.load_whisper("/cached/model", "cuda", "float16", cancel_event)
assert model.moved_to is None
def test_child_falls_back_to_float32_for_an_unknown_dtype_name(monkeypatch):
calls, _model, _processor = _install_fake_transformers(monkeypatch)
worker_module.load_whisper("/cached/model", "cpu", "bfloat9")
model_kwargs = next(kwargs for kind, _, kwargs in calls if kind == "model")
assert model_kwargs.get("torch_dtype") == "float32"
def test_child_feeds_decoded_pcm_and_matches_the_model_dtype(monkeypatch):
_calls, model, processor = _install_fake_transformers(monkeypatch)
pcm = np.arange(4, dtype = np.float32).tobytes()
result = worker_module.transcribe_window(
model, processor, pcm, {"task": "transcribe", "num_beams": 5}
)
assert result == ("hello", 4)
assert processor.seen_rate == 16000
assert np.array_equal(processor.seen_audio, np.arange(4, dtype = np.float32))
# to(device) then to(dtype): features must match the weights they meet.
assert processor.features.moved_to == ["cuda", "float16"]
assert model.generate_kwargs == {"task": "transcribe", "num_beams": 5}
def test_child_only_installs_stopping_criteria_for_a_cancellable_request(monkeypatch):
_calls, model, processor = _install_fake_transformers(monkeypatch)
cancel_event = threading.Event()
pcm = np.zeros(4, dtype = np.float32).tobytes()
worker_module.transcribe_window(model, processor, pcm, {}, cancel_event)
criteria = model.generate_kwargs["stopping_criteria"]
assert criteria[0]() is False
cancel_event.set()
assert criteria[0]() is True
worker_module.transcribe_window(model, processor, pcm, {})
assert "stopping_criteria" not in model.generate_kwargs
_TIMESTAMP_BEGIN = 50364
_EOS = 50257
class _TimestampedModel(_FakeModel):
def __init__(self, tokens) -> None:
super().__init__()
self.generation_config = SimpleNamespace(
is_multilingual = True, no_timestamps_token_id = _TIMESTAMP_BEGIN - 1, eos_token_id = _EOS
)
self.tokens = tokens
def generate(self, _features, **kwargs):
self.generate_kwargs = kwargs
return [self.tokens]
class _RecordingProcessor(_FakeProcessor):
def batch_decode(self, generated, **_kwargs):
self.decoded = [list(row) for row in generated]
return ["decoded"]
def _at(seconds):
return _TIMESTAMP_BEGIN + round(seconds * 50)
@pytest.mark.parametrize(
"tail, kept, consumed",
[
pytest.param([_at(28.4), _at(28.4), 33], [_at(28.4)], 454400, id = "cut-mid-sentence"),
pytest.param([33, _at(29.0)], [33, _at(29.0)], 480000, id = "last-segment-closed"),
],
)
def test_child_resumes_a_long_clip_after_its_last_complete_segment(
monkeypatch, tail, kept, consumed
):
head = [_at(0), 11, _at(5.0), _at(5.0), 22]
model = _TimestampedModel([50258, 50259, 50359] + head + tail + [_EOS])
processor = _RecordingProcessor()
_install_fake_transformers(monkeypatch, model = model, processor = processor)
pcm = np.zeros(480000, dtype = np.float32).tobytes()
result = worker_module.transcribe_window(
model, processor, pcm, {"num_beams": 5, "return_timestamps": True}
)
assert result == ("decoded", consumed)
assert processor.decoded == [head + kept]
assert model.generate_kwargs["force_unique_generate_call"] is True
assert model.generate_kwargs["attention_mask"] is processor.attention_mask
assert processor.attention_mask.moved_to == ["cuda"]
class _DeviceRow:
"""An accelerator row: each iterated element is its own device-to-host read."""
def __init__(self, tokens) -> None:
self.tokens = list(tokens)
self.element_reads = 0
self.host_copies = 0
def __iter__(self):
for token in self.tokens:
self.element_reads += 1
yield token
def tolist(self):
self.host_copies += 1
return list(self.tokens)
def test_child_copies_timestamped_tokens_to_the_host_once(monkeypatch):
head = [_at(0), 11, _at(5.0), _at(5.0), 22]
tail = [_at(28.4), _at(28.4), 33]
row = _DeviceRow([50258, 50259, 50359] + head + tail + [_EOS])
model = _TimestampedModel(row)
processor = _RecordingProcessor()
_install_fake_transformers(monkeypatch, model = model, processor = processor)
pcm = np.zeros(480000, dtype = np.float32).tobytes()
result = worker_module.transcribe_window(
model, processor, pcm, {"num_beams": 5, "return_timestamps": True}
)
assert result == ("decoded", 454400)
assert processor.decoded == [head + [_at(28.4)]]
assert (row.host_copies, row.element_reads) == (1, 0)
def _run_child(
monkeypatch,
commands,
*,
load = None,
transcribe = None,
):
"""The bootstrap handshake is consumed here, so each test sees only the answers to its own commands."""
cmd_queue: queue.Queue = queue.Queue()
resp_queue: queue.Queue = queue.Queue()
cancel_event = threading.Event()
if load is not None:
monkeypatch.setattr(worker_module, "load_whisper", load)
if transcribe is not None:
monkeypatch.setattr(worker_module, "transcribe_window", transcribe)
for command in commands:
cmd_queue.put(command)
ready_event = threading.Event()
thread = threading.Thread(
target = worker_module.run_stt_worker,
kwargs = {
"cmd_queue": cmd_queue,
"resp_queue": resp_queue,
"cancel_event": cancel_event,
"ready_event": ready_event,
"config": {},
},
daemon = True,
)
thread.start()
thread.join(timeout = 10)
assert thread.is_alive() is False
assert ready_event.is_set() is True
responses = []
while not resp_queue.empty():
responses.append(resp_queue.get_nowait())
return responses, cancel_event
def test_child_reports_the_loaded_model_then_transcribes_then_exits(monkeypatch):
model = _FakeModel()
model.generation_config = SimpleNamespace(is_multilingual = False)
responses, _cancel = _run_child(
monkeypatch,
[
{
"type": "load",
"snapshot_path": "/cached/model",
"device": "cuda",
"dtype": "float16",
},
{"type": "transcribe", "audio": b"", "generate_kwargs": {}, "cancellable": False},
{"type": "shutdown"},
],
load = lambda *_args, **_kwargs: (model, _FakeProcessor()),
transcribe = lambda *_args, **_kwargs: ("hello", 16000),
)
assert responses == [
{
"type": "loaded",
"device": "cuda",
"is_multilingual": False,
"supports_timestamps": False,
},
{"type": "text", "text": "hello", "consumed": 16000},
{"type": "shutdown_ack"},
]
def test_child_exits_after_a_failed_load_so_a_half_taken_context_goes_with_it(monkeypatch):
def boom(*_args, **_kwargs):
raise RuntimeError("out of memory")
responses, _cancel = _run_child(
monkeypatch,
[
{
"type": "load",
"snapshot_path": "/cached/model",
"device": "cuda",
"dtype": "float16",
},
{"type": "transcribe", "audio": b"", "generate_kwargs": {}, "cancellable": False},
],
load = boom,
)
assert responses == [{"type": "error", "kind": "RuntimeError", "error": "out of memory"}]
def test_child_survives_a_failed_transcription_and_keeps_the_model(monkeypatch):
def boom(*_args, **_kwargs):
raise ValueError("bad audio")
responses, _cancel = _run_child(
monkeypatch,
[
{"type": "load", "snapshot_path": "/cached/model", "device": "cpu", "dtype": "float32"},
{"type": "transcribe", "audio": b"", "generate_kwargs": {}, "cancellable": False},
{"type": "shutdown"},
],
load = lambda *_args, **_kwargs: (_FakeModel(), _FakeProcessor()),
transcribe = boom,
)
assert [response["type"] for response in responses] == ["loaded", "error", "shutdown_ack"]
assert responses[1]["error"] == "bad audio"
def test_child_reports_a_cancelled_generation_rather_than_partial_text(monkeypatch):
def stop_early(
_model,
_processor,
_pcm,
_kwargs,
cancel_event = None,
):
cancel_event.set()
return "half a sen", 0
responses, _cancel = _run_child(
monkeypatch,
[
{"type": "load", "snapshot_path": "/cached/model", "device": "cpu", "dtype": "float32"},
{"type": "transcribe", "audio": b"", "generate_kwargs": {}, "cancellable": True},
{"type": "shutdown"},
],
load = lambda *_args, **_kwargs: (_FakeModel(), _FakeProcessor()),
transcribe = stop_early,
)
assert responses[1]["kind"] == "SttTranscriptionCancelledError"
def test_child_answers_an_unknown_command_instead_of_dropping_it(monkeypatch):
responses, _cancel = _run_child(
monkeypatch,
[{"type": "explode"}, {"type": "shutdown"}],
)
assert responses[0]["type"] == "error"
assert "explode" in responses[0]["error"]
def test_a_local_cache_miss_crosses_as_a_not_downloaded_error():
class LocalEntryNotFoundError(RuntimeError):
pass
response = worker_module._error_response(LocalEntryNotFoundError("not cached"))
assert response["kind"] == "SttModelNotDownloadedError"
with pytest.raises(SttModelNotDownloadedError):
worker_module._raise_worker_error(response)
def test_cancellation_keeps_its_class_across_the_process_boundary():
response = worker_module._error_response(
SttTranscriptionCancelledError("Transcription cancelled.")
)
with pytest.raises(SttTranscriptionCancelledError, match = "cancelled"):
worker_module._raise_worker_error(response)
def test_an_unknown_failure_arrives_as_a_worker_error_carrying_its_message():
# The exception is never sent, since an unpicklable torch error would cost a full timeout.
response = worker_module._error_response(TypeError("weird"))
assert response == {"type": "error", "kind": "TypeError", "error": "weird"}
with pytest.raises(SttWorkerError, match = "weird"):
worker_module._raise_worker_error(response)
def test_handle_sends_one_window_and_returns_its_text():
handle = _wired_worker()
handle._resp_queue.put({"type": "text", "text": "hello", "consumed": 1})
result = handle.transcribe_window(b"\x00\x00\x00\x00", {"num_beams": 1})
assert result == ("hello", 1)
command = handle._cmd_queue.get_nowait()
assert command["type"] == "transcribe"
assert command["generate_kwargs"] == {"num_beams": 1}
assert command["cancellable"] is False
def test_handle_reports_a_dead_child_instead_of_waiting_out_its_timeout():
process = _FakeProcess(alive = False)
process.exitcode = -9
handle = _wired_worker(process)
with pytest.raises(SttWorkerError, match = _SIGKILL_TEXT):
handle.transcribe_window(b"", {})
def test_handle_kills_a_child_that_stops_answering():
process = _FakeProcess()
handle = _wired_worker(process)
with pytest.raises(SttWorkerError, match = "stopped responding"):
handle._await("text", 0.0, None, "transcribe")
assert process.killed or process.terminated
def test_handle_mirrors_a_request_cancel_into_the_child():
handle = _wired_worker()
cancel_event = threading.Event()
cancel_event.set()
def answer_once():
time.sleep(0.2)
handle._resp_queue.put(
{
"type": "error",
"kind": "SttTranscriptionCancelledError",
"error": "Transcription cancelled.",
}
)
thread = threading.Thread(target = answer_once, daemon = True)
thread.start()
with pytest.raises(SttTranscriptionCancelledError):
handle._await("text", 30.0, cancel_event, "transcribe")
thread.join(timeout = 5)
assert handle._cancel_event.is_set()
def test_a_cancelled_load_that_never_answers_is_killed_rather_than_waited_on(monkeypatch):
monkeypatch.setattr(worker_module, "_CANCEL_GRACE_SECONDS", 0.0)
process = _FakeProcess()
handle = _wired_worker(process)
cancel_event = threading.Event()
cancel_event.set()
with pytest.raises(SttLoadCancelledError):
handle._await("loaded", 30.0, cancel_event, "load")
assert handle.is_alive() is False
def test_the_cancel_grace_is_not_followed_by_a_second_shutdown_wait(monkeypatch):
# The grace is the graceful shutdown: a child stuck in from_pretrained reads neither.
monkeypatch.setattr(worker_module, "_CANCEL_GRACE_SECONDS", 0.0)
monkeypatch.setattr("utils.process_lifetime.forget_pid", lambda _pid: None)
class _Recording(_FakeProcess):
def __init__(self) -> None:
super().__init__()
self.joins = []
def join(self, timeout = None):
self.joins.append(timeout)
process = _Recording()
handle = _wired_worker(process)
cancel_event = threading.Event()
cancel_event.set()
with pytest.raises(SttLoadCancelledError):
handle._await("loaded", 30.0, cancel_event, "load")
assert worker_module._SHUTDOWN_TIMEOUT_SECONDS not in process.joins
assert process.terminated is True
assert handle.is_alive() is False
@pytest.mark.parametrize(
("phase", "expected"),
[("load", SttLoadCancelledError), ("transcribe", SttTranscriptionCancelledError)],
)
def test_a_cancel_that_lands_near_the_command_timeout_keeps_its_cancellation(
monkeypatch, phase, expected
):
# A cancel late in the timeout is still owed a 409/499, not a 500.
monkeypatch.setattr(worker_module, "_CANCEL_GRACE_SECONDS", 30.0)
monkeypatch.setattr("utils.process_lifetime.forget_pid", lambda _pid: None)
class _Recording(_FakeProcess):
def __init__(self) -> None:
super().__init__()
self.joins = []
def join(self, timeout = None):
self.joins.append(timeout)
process = _Recording()
handle = _wired_worker(process)
cancel_event = threading.Event()
cancel_event.set()
with pytest.raises(expected):
handle._await("text" if phase == "transcribe" else "loaded", 0.0, cancel_event, phase)
assert worker_module._SHUTDOWN_TIMEOUT_SECONDS not in process.joins
assert handle.is_alive() is False
def test_closing_a_handle_normally_still_asks_the_child_to_exit_first(monkeypatch):
monkeypatch.setattr("utils.process_lifetime.forget_pid", lambda _pid: None)
class _Recording(_FakeProcess):
def __init__(self) -> None:
super().__init__()
self.joins = []
def join(self, timeout = None):
self.joins.append(timeout)
self._alive = False
process = _Recording()
handle = _wired_worker(process)
cmd_queue = handle._cmd_queue
handle.close()
assert cmd_queue.get_nowait() == {"type": "shutdown"}
assert process.joins[0] == worker_module._SHUTDOWN_TIMEOUT_SECONDS
assert process.terminated is False
def test_closing_the_handle_ends_the_child_and_drops_its_pid(monkeypatch):
forgotten = []
monkeypatch.setattr("utils.process_lifetime.forget_pid", lambda pid: forgotten.append(pid))
process = _FakeProcess()
handle = _wired_worker(process)
handle.close()
assert forgotten == [4242]
assert handle.is_alive() is False
assert handle._cmd_queue is None
def test_a_child_that_survives_terminate_and_kill_keeps_its_pid_and_handle(monkeypatch):
# A child wedged in a driver call outlives SIGKILL and still holds its accelerator
# memory; forgetting its pid leaves terminate_all and the sweep nothing to find it by.
forgotten = []
monkeypatch.setattr("utils.process_lifetime.forget_pid", lambda pid: forgotten.append(pid))
class _Unkillable(_FakeProcess):
def terminate(self):
self.terminated = True
def kill(self):
self.killed = True
process = _Unkillable()
handle = _wired_worker(process)
closed = handle.close()
assert forgotten == []
assert closed is False
assert handle._process is process
assert handle.is_alive() is True
assert handle._cmd_queue is not None
def test_a_child_that_outlived_a_cancelled_command_marks_its_handle_unusable(monkeypatch):
# A child that outlived both signals answers nothing later, so the handle marks itself spent.
monkeypatch.setattr(worker_module, "_CANCEL_GRACE_SECONDS", 0.0)
monkeypatch.setattr("utils.process_lifetime.forget_pid", lambda _pid: None)
class _Unkillable(_FakeProcess):
def terminate(self):
self.terminated = True
def kill(self):
self.killed = True
handle = _wired_worker(_Unkillable())
assert handle.survived_kill is False
cancel_event = threading.Event()
cancel_event.set()
with pytest.raises(SttTranscriptionCancelledError):
handle._await("text", 30.0, cancel_event, "transcribe")
assert handle.is_alive() is True
assert handle.survived_kill is True
def test_a_handle_whose_child_did_exit_is_still_usable(monkeypatch):
# Set only for a child that outlived both signals, so a normal close keeps the handle.
monkeypatch.setattr("utils.process_lifetime.forget_pid", lambda _pid: None)
handle = _wired_worker()
assert handle.close() is True
assert handle.survived_kill is False
def test_closing_a_handle_that_ignores_shutdown_escalates_to_a_kill(monkeypatch):
monkeypatch.setattr("utils.process_lifetime.forget_pid", lambda _pid: None)
class _Stubborn(_FakeProcess):
def terminate(self):
self.terminated = True
process = _Stubborn()
handle = _wired_worker(process)
handle.close()
assert process.terminated is True
assert process.killed is True
class _RefusingProcess:
"""A child that cannot be created: a sandbox, or a frozen POSIX build."""
def __init__(self, error) -> None:
self.pid = None
self.exitcode = None
self._error = error
def is_alive(self):
return False
def start(self):
raise self._error
def join(self, _timeout = None):
return None
class _RefusingContext:
def __init__(self, error = None) -> None:
self.error = error or PermissionError("spawn is not permitted here")
def Queue(self):
return queue.Queue()
def Event(self):
return threading.Event()
def Process(self, **_kwargs):
return _RefusingProcess(self.error)
def test_dictation_still_loads_and_transcribes_when_no_child_can_be_started(monkeypatch):
# Moves work out of the backend only; a host forbidding spawn must keep working as before.
from core.inference.stt_sidecar import WhisperSttSidecar
monkeypatch.setattr(worker_module, "_CTX", _RefusingContext())
_calls, _model, _processor = _install_fake_transformers(monkeypatch)
engine = WhisperSttSidecar(keep_alive_seconds = 0)._build_model(
"/cached/model", "cpu", "float32", threading.Event()
)
assert isinstance(engine, worker_module.InProcessWhisperEngine)
assert engine.device == "cpu"
assert engine.is_alive() is True
assert engine.transcribe_window(np.zeros(4, dtype = np.float32).tobytes(), {}) == ("hello", 4)
def test_a_spawn_failure_on_an_accelerator_leaves_the_cpu_retry_to_the_sidecar(monkeypatch):
# An in-process load takes the context this module exists to avoid, so the fallback
# is CPU only; the accelerator attempt must reach the sidecar's own CPU retry first.
from core.inference.stt_sidecar import WhisperSttSidecar
monkeypatch.setattr(worker_module, "_CTX", _RefusingContext())
monkeypatch.setattr(
worker_module,
"load_whisper",
lambda *_args, **_kwargs: pytest.fail("no in-process load on an accelerator"),
)
with pytest.raises(worker_module.SttWorkerSpawnError, match = "not permitted"):
WhisperSttSidecar(keep_alive_seconds = 0)._build_model(
"/cached/model", "cuda", "float16", threading.Event()
)
class _StillbornProcess:
"""A frozen POSIX build re-runs its own binary, so the child can exit before it reads any command."""
def __init__(self, exitcode = 1) -> None:
self.pid = 4243
self.exitcode = None
self._exitcode = exitcode
def start(self):
self.exitcode = self._exitcode
def is_alive(self):
return False
def join(self, _timeout = None):
return None
def terminate(self):
pass
def kill(self):
pass
class _StillbornContext:
def __init__(self, exitcode = 1) -> None:
self.exitcode = exitcode
def Queue(self):
return queue.Queue()
def Event(self):
return threading.Event()
def Process(self, **_kwargs):
return _StillbornProcess(self.exitcode)
def test_a_child_that_never_bootstraps_reads_as_a_host_that_cannot_spawn(monkeypatch):
# start() only proves exec; a child dying before answering holds no device, so fall back.
from core.inference.stt_sidecar import WhisperSttSidecar
monkeypatch.setattr(worker_module, "_CTX", _StillbornContext())
_calls, _model, _processor = _install_fake_transformers(monkeypatch)
engine = WhisperSttSidecar(keep_alive_seconds = 0)._build_model(
"/cached/model", "cpu", "float32", threading.Event()
)
assert isinstance(engine, worker_module.InProcessWhisperEngine)
assert engine.device == "cpu"
def test_a_child_killed_by_a_signal_keeps_its_crash_instead_of_falling_back(monkeypatch):
# A signal-killed child bootstrapped, so spawn works; a backend load just repeats it.
monkeypatch.setattr(worker_module, "_CTX", _StillbornContext(exitcode = -9))
monkeypatch.setattr(
worker_module,
"load_whisper",
lambda *_args, **_kwargs: pytest.fail("no in-process load after a real crash"),
)
handle = WhisperWorker()
with pytest.raises(SttWorkerError, match = _SIGKILL_TEXT) as caught:
handle.start("/cached/model", "cpu", "float32")
assert isinstance(caught.value, worker_module.SttWorkerSpawnError) is False
class _NativeCrashProcess:
"""A native-fault child exits with a positive code, since Windows has no signals to report."""
def __init__(self, kwargs, faulted: threading.Event) -> None:
self.pid = 4244
self.exitcode = None
self._kwargs = kwargs
self._faulted = faulted
def start(self):
thread = threading.Thread(
target = worker_module.run_stt_worker,
kwargs = self._kwargs,
daemon = True,
)
thread.start()
def is_alive(self):
if self._faulted.is_set():
self.exitcode = 3221225477 # 0xC0000005, STATUS_ACCESS_VIOLATION
return False
return True
def join(self, _timeout = None):
return None
def terminate(self):
pass
def kill(self):
pass
class _NativeCrashContext:
"""Spawns a child that comes up and then faults in the model load."""
def __init__(self, faulted: threading.Event) -> None:
self._faulted = faulted
self._queues: list = []
def Queue(self):
made: queue.Queue = queue.Queue()
self._queues.append(made)
return made
def Event(self):
return threading.Event()
def Process(self, **kwargs):
# Pass start()'s kwargs through, since rebuilding them would drop ready_event.
process = _NativeCrashProcess(dict(kwargs.get("kwargs") or {}), self._faulted)
self._process = process
return process
def _fault_in_the_native_load(monkeypatch, faulted: threading.Event, forever: threading.Event):
def _fault(*_args, **_kwargs):
faulted.set()
forever.wait(30)
raise AssertionError("the crashed child was resumed")
monkeypatch.setattr(worker_module, "load_whisper", _fault)
def test_a_child_that_crashed_in_the_load_is_not_read_as_a_host_that_cannot_spawn(monkeypatch):
# On Windows a native crash exits with a positive code; the child bootstrapped, so spawn works.
faulted = threading.Event()
forever = threading.Event()
monkeypatch.setattr(worker_module, "_CTX", _NativeCrashContext(faulted))
_fault_in_the_native_load(monkeypatch, faulted, forever)
handle = WhisperWorker()
try:
with pytest.raises(SttWorkerError) as caught:
handle.start("/cached/model", "cpu", "float32")
finally:
forever.set()
assert faulted.is_set()
assert isinstance(caught.value, worker_module.SttWorkerSpawnError) is False
def test_a_crash_in_the_child_load_is_never_repeated_inside_the_backend(monkeypatch):
# The fallback exists for a host that cannot bring a child up. A load that crashes
# the child crashes the backend too, and the backend is what the user talks to.
from core.inference.stt_sidecar import WhisperSttSidecar
faulted = threading.Event()
forever = threading.Event()
monkeypatch.setattr(worker_module, "_CTX", _NativeCrashContext(faulted))
_fault_in_the_native_load(monkeypatch, faulted, forever)
monkeypatch.setattr(
worker_module.InProcessWhisperEngine,
"start",
lambda *_args, **_kwargs: pytest.fail("no in-process load after a crash in the child"),
)
try:
with pytest.raises(SttWorkerError):
WhisperSttSidecar(keep_alive_seconds = 0)._build_model(
"/cached/model", "cpu", "float32", threading.Event()
)
finally:
forever.set()
def test_the_child_says_it_is_ready_before_it_touches_a_command(monkeypatch):
# The handshake separates no-spawn hosts from failed children, so it precedes every load.
cmd_queue: queue.Queue = queue.Queue()
resp_queue: queue.Queue = queue.Queue()
def boom(*_args, **_kwargs):
raise RuntimeError("kaboom")
monkeypatch.setattr(worker_module, "load_whisper", boom)
cmd_queue.put(
{"type": "load", "snapshot_path": "/cached/model", "device": "cpu", "dtype": "float32"}
)
ready_event = threading.Event()
worker_module.run_stt_worker(
cmd_queue = cmd_queue,
resp_queue = resp_queue,
cancel_event = threading.Event(),
ready_event = ready_event,
config = {},
)
assert ready_event.is_set() is True
assert resp_queue.get_nowait()["kind"] == "RuntimeError"
def test_the_in_process_fallback_reports_the_checkpoint_language_support(monkeypatch):
_calls, model, _processor = _install_fake_transformers(monkeypatch)
model.generation_config = SimpleNamespace(is_multilingual = False)
engine = worker_module.InProcessWhisperEngine()
engine.start("/cached/model", "cpu", "float32")
# The sidecar reads this to drop the kwargs an English-only model rejects.
assert engine.generation_config.is_multilingual is False
assert engine.generation_config.supports_timestamps is False
engine.close()
assert engine.is_alive() is False
class _LosesTheReadyMessage(queue.Queue):
"""Queue.put returns before its feeder thread sends, so a fault can lose the ready message."""
def put(self, item, *args, **kwargs):
if isinstance(item, dict) and item.get("type") == "ready":
return
return super().put(item, *args, **kwargs)
class _LossyNativeCrashContext(_NativeCrashContext):
def Queue(self):
made = _LosesTheReadyMessage()
self._queues.append(made)
return made
def test_a_crashed_child_whose_ready_word_was_lost_is_still_not_read_as_a_bad_host(monkeypatch):
# The child came up but its ready never arrived; retrying in the backend would crash it too.
faulted = threading.Event()
forever = threading.Event()
monkeypatch.setattr(worker_module, "_CTX", _LossyNativeCrashContext(faulted))
_fault_in_the_native_load(monkeypatch, faulted, forever)
handle = WhisperWorker()
try:
with pytest.raises(SttWorkerError) as caught:
handle.start("/cached/model", "cpu", "float32")
finally:
forever.set()
assert faulted.is_set()
assert isinstance(caught.value, worker_module.SttWorkerSpawnError) is False