Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop declaring a live backend dead, and keep a crashed worker's stderr (#11070)

Daniel Hanc2d2fca7b7f7
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 9a2c6a4f187..a795698e592 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -14,6 +14,7 @@ import signal
from loggers import get_logger
import multiprocessing as mp
import queue
+import re
import threading
import time
import uuid
@@ -45,6 +46,134 @@ def __getattr__(name: str):
logger = get_logger(__name__)
+# Delimited, not a whitelist: a whitelist stopped at the apostrophe in `/home/o'connor/`.
+_PATH_COMPONENT = r"[^\s\\/](?:(?:(?![A-Za-z]:[\\/])[^\\/\n\",;])*[^\s\\/])?"
+# Second alternative: the root-level case (`/model.gguf`, `\\server\share`) has no trailing separator.
+_ABSOLUTE_PATH_RE = re.compile(
+ r"(?<![\w:/])(?:\\\\[^\\/\s]+[\\/]|[A-Za-z]:[\\/]|/)"
+ r"(?:(?:" + _PATH_COMPONENT + r"[\\/])+[^\s\\/]*|[^\s\\/\",;]+[\\/]?)"
+)
+
+
+def _shorten_path(match: "re.Match[str]") -> str:
+ text = match.group(0)
+ tail = text.replace("\\", "/").rstrip("/").rsplit("/", 1)[-1]
+ return f".../{tail}" if tail else "..."
+
+
+# A line written by a logger; `decode_bicodec` logs 500 characters of generated text this way.
+_LOG_RECORD_RE = re.compile(
+ r"""^\s*(?:
+ \d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2} # a leading ISO timestamp
+ | \[?(?:DEBUG|INFO|WARNING|WARN|ERROR|CRITICAL|FATAL)\b
+ | [\w]+(?:\.[\w]+)+\s*: # a dotted logger name before the colon
+ )""",
+ re.VERBOSE,
+)
+
+# Only the writer can tell a logged traceback from a dying process's; the bytes are identical. See `utils.worker_stderr`.
+_LOG_CONTINUATION_PREFIX = " | "
+_LOG_RECORD_START_MARK = "\x1f"
+
+
+def _looks_like_a_log_record(line: str) -> bool:
+ """The mark is the reliable half; the patterns are the fallback for a producer that bypassed that handler."""
+ if line.startswith(_LOG_CONTINUATION_PREFIX) or line.startswith(_LOG_RECORD_START_MARK):
+ return True
+ return bool(_LOG_RECORD_RE.match(line))
+
+
+# Errs towards dropping: a missed diagnostic costs a detail, a kept content line leaves the host.
+_DIAGNOSTIC_START_RE = re.compile(
+ r"""^(?:
+ Traceback\ \(most\ recent\ call\ last\):
+ | terminate\ called
+ | what\(\):
+ | Fatal\ Python\ error:
+ | (?:Current\ )?[Tt]hread\ 0x
+ | Stack\ \(most\ recent\ call\ first\):
+ | Segmentation\ fault | Bus\ error | Illegal\ instruction
+ | Floating\ point\ exception | Aborted | Killed | Trace/breakpoint\ trap
+ | \*\*\* # *** stack smashing detected ***
+ | double\ free | free\(\) | malloc\(\) | munmap_chunk | corrupted\ (?:size|double-linked)
+ | std::(?:bad_alloc|terminate) | libc\+\+abi
+ | GGML_ASSERT | CUDA\ error | HIP\ error | cudaError
+ | [A-Za-z_][\w.]*(?:Error|Exception|Exit|Interrupt|Abort|Fault|Signal)\s*:
+ )""",
+ re.VERBOSE,
+)
+
+
+_TRACEBACK_HEADER = "Traceback (most recent call last):"
+
+
+def _starts_a_new_diagnostic(line: str) -> bool:
+ return bool(_DIAGNOSTIC_START_RE.match(line))
+
+
+def _diagnostic_lines_only(lines: "list[str]") -> "list[str]":
+ """Line by line is not a filter, because a log record is not always one line."""
+ kept: "list[str]" = []
+ inside_a_diagnostic = False
+ for line in lines:
+ if line.startswith(_LOG_CONTINUATION_PREFIX):
+ # Another thread logging mid-abort does not end the abort.
+ continue
+ if _looks_like_a_log_record(line):
+ inside_a_diagnostic = False
+ continue
+ if _starts_a_new_diagnostic(line):
+ inside_a_diagnostic = True
+ kept.append(line)
+ continue
+ if inside_a_diagnostic and (not line.strip() or line[:1] in (" ", "\t")):
+ kept.append(line)
+ continue
+ inside_a_diagnostic = False
+ return kept
+
+
+def _crash_lines(lines: "list[str]") -> "list[str]":
+ """The last traceback, unless a diagnostic follows it: `logger.exception` for a RECOVERED failure leaves one too."""
+ starts = [
+ index for index, line in enumerate(lines) if line.lstrip().startswith(_TRACEBACK_HEADER)
+ ]
+ if not starts:
+ return lines
+ header = starts[-1]
+ end = len(lines)
+ for index in range(header + 1, len(lines)):
+ line = lines[index]
+ if line.strip() and line[:1] not in (" ", "\t"):
+ end = index + 1
+ break
+ after = lines[end:]
+ return after if after else lines[header:]
+
+
+def _redact_worker_output(text: str) -> str:
+ """Three redactors because none subsumes another: native path leases, token shapes, filesystem layout."""
+ if not text:
+ return ""
+ redacted = text
+ try:
+ from utils.native_path_leases import redact_native_paths
+ redacted = redact_native_paths(redacted)
+ except Exception: # noqa: BLE001 -- a redactor that cannot run must not lose the others
+ pass
+ try:
+ from hub.utils.download_registry import scrub_secrets
+ redacted = scrub_secrets(redacted)
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ # `redact_log_text` is idempotent, so it runs after `scrub_secrets`.
+ from utils.log_redaction import redact_log_text
+ redacted = redact_log_text(redacted)
+ except Exception: # noqa: BLE001
+ pass
+ return _ABSOLUTE_PATH_RE.sub(_shorten_path, redacted)
+
class _LoadCancelled(Exception):
"""Internal control flow for a caller-cancelled model load."""
@@ -235,6 +364,8 @@ class InferenceOrchestrator:
def __init__(self):
self._proc: Optional[mp.Process] = None
+ # Retired when the next worker is spawned; read long after _proc has been cleared.
+ self._stderr_capture: Any = None
self._cmd_queue: Any = None
self._resp_queue: Any = None
self._subprocess_shutdown_lock = threading.Lock()
@@ -439,6 +570,17 @@ class InferenceOrchestrator:
else get_hf_cache_paths().child_env({})
)
+ # Retired here, not at shutdown: a crash message outlives _shutdown_subprocess clearing _proc.
+ self._retire_stderr_capture()
+ self._worker_stopped_deliberately = False
+ try:
+ from utils.worker_stderr import WorkerStderrCapture
+ self._stderr_capture = WorkerStderrCapture(prefix = "unsloth-inference-worker-")
+ except Exception as exc:
+ # No sink is the old behaviour; never a failed spawn.
+ logger.debug("Could not open a worker stderr mirror: %s", exc)
+ self._stderr_capture = None
+
with (
child_environment_for_spawn(cache_env),
native_path_secret_removed_for_child_start(),
@@ -454,16 +596,22 @@ class InferenceOrchestrator:
# rely on from here on: snapshotting it after start() would capture the None
# and lose the only reference to a live child, which is the orphan this
# change exists to prevent.
+ _child_kwargs: dict = {
+ "cmd_queue": self._cmd_queue,
+ "resp_queue": self._resp_queue,
+ "cancel_event": self._cancel_event,
+ "drain_event": self._drain_event,
+ "config": config,
+ }
+ if self._stderr_capture is not None:
+ from utils.native_path_leases import STDERR_MIRROR_KWARG
+
+ # Consumed by run_without_native_path_secret; it never reaches the entrypoint.
+ _child_kwargs[STDERR_MIRROR_KWARG] = self._stderr_capture.path
_spawned_proc = _CTX.Process(
target = run_without_native_path_secret,
args = ("core.inference.worker", "run_inference_process", cache_env),
- kwargs = {
- "cmd_queue": self._cmd_queue,
- "resp_queue": self._resp_queue,
- "cancel_event": self._cancel_event,
- "drain_event": self._drain_event,
- "config": config,
- },
+ kwargs = _child_kwargs,
daemon = True,
)
self._proc = _spawned_proc
@@ -606,6 +754,9 @@ class InferenceOrchestrator:
handle and refuse the destructive sidecar swap."""
self._stop_dispatcher() # before killing subprocess
if self._proc is None or not self._proc.is_alive():
+ # Already gone: a nonzero status is an unwaited crash and keeps its replay.
+ exitcode = getattr(self._proc, "exitcode", 0) if self._proc is not None else 0
+ self._worker_stopped_deliberately = exitcode == 0
self._proc = None
return True
@@ -656,6 +807,8 @@ class InferenceOrchestrator:
)
return False
+ # Without this flag every model switch replayed a healthy worker's stderr at ERROR.
+ self._worker_stopped_deliberately = True
self._proc = None
self._cmd_queue = None
self._resp_queue = None
@@ -755,12 +908,90 @@ class InferenceOrchestrator:
def _cleanup(self):
self._shutdown_subprocess(timeout = 5.0)
+ self._retire_stderr_capture()
+
+ def _retire_stderr_capture(self) -> None:
+ """A native fault BETWEEN requests has no waiter, so nothing reaches `_subprocess_crash_message`."""
+ capture = getattr(self, "_stderr_capture", None)
+ if capture is None:
+ return
+ proc = getattr(self, "_proc", None)
+ try:
+ worker_is_gone = proc is None or not proc.is_alive()
+ except Exception: # noqa: BLE001 -- a handle in teardown; treat it as gone
+ worker_is_gone = True
+ # `_shutdown_subprocess_locked` has already cleared `_proc`, so the flag is all that still knows.
+ if worker_is_gone and not getattr(self, "_worker_stopped_deliberately", False):
+ self._log_worker_stderr_once(
+ getattr(proc, "pid", None),
+ getattr(proc, "exitcode", None),
+ )
+ try:
+ capture.close()
+ except Exception:
+ pass
+ self._stderr_capture = None
+
+ def _public_worker_stderr_tail(self) -> str:
+ """Spans the worker's WHOLE lifetime and goes out verbatim through `GenStreamError(public = True)`."""
+ raw = self._worker_stderr_tail()
+ if not raw:
+ return ""
+ # Filtered FIRST, then searched: searching raw selected logger-written traceback headers.
+ kept = _crash_lines(_diagnostic_lines_only(raw.splitlines()))
+ block = "\n".join(kept).strip()
+ if not block:
+ return ""
+ return _redact_worker_output(block)
+
+ def _log_worker_stderr_once(
+ self,
+ pid,
+ exitcode,
+ *,
+ worker_exited: bool = True,
+ ) -> None:
+ """The RAW tail, at most once per worker; *worker_exited* is False for a still-live replacement."""
+ capture = getattr(self, "_stderr_capture", None)
+ if capture is None:
+ return
+ logged = getattr(self, "_stderr_tail_logged", None)
+ if isinstance(logged, tuple) and logged[0] is capture:
+ # Already replayed for a real exit, or this is a second non-terminal call.
+ if logged[1] or not worker_exited:
+ return
+ # Marked before the read, so a failure here cannot become a log line per call.
+ self._stderr_tail_logged = (capture, bool(worker_exited))
+ raw = self._worker_stderr_tail()
+ if not raw:
+ return
+ raw = raw.replace(_LOG_RECORD_START_MARK, "")
+ logger.error(
+ "Inference worker stderr (pid=%s, exitcode=%s):\n%s",
+ pid,
+ exitcode,
+ raw,
+ )
+
+ def _worker_stderr_tail(self) -> str:
+ capture = getattr(self, "_stderr_capture", None)
+ if capture is None:
+ return ""
+ try:
+ return capture.tail()
+ except Exception:
+ return ""
def _ensure_subprocess_alive(self) -> bool:
return self._proc is not None and self._proc.is_alive()
- def _subprocess_crash_message(self, context: str) -> str:
- """Return a user-facing crash message with the worker exit status."""
+ def _subprocess_crash_message(
+ self,
+ context: str,
+ *,
+ with_worker_output: bool = False,
+ ) -> str:
+ """``with_worker_output`` defaults to FALSE: the opposite default discloses a traceback to a merely QUEUED request."""
context_label = {
"wait": "loading the model",
"generation": "generating a response",
@@ -769,14 +1000,25 @@ class InferenceOrchestrator:
}.get(context, context)
message = f"The inference worker stopped unexpectedly while {context_label}."
- if self._proc is None:
+ proc = self._proc
+ if proc is None:
+ # A concurrent teardown can clear `_proc`; the bytes outlive the handle.
+ self._log_worker_stderr_once(None, None)
return f"{message} Details: process missing."
- exitcode = self._proc.exitcode
- pid = self._proc.pid
+ exitcode = proc.exitcode
+ pid = proc.pid
if exitcode is None:
+ # NOT terminal: the capture belongs to the live replacement, whose crash must stay replayable.
+ self._log_worker_stderr_once(pid, None, worker_exited = False)
return f"{message} Details: pid={pid}."
+ # What the worker said before it went (#7843), narrowed to what a client may see.
+ tail = self._public_worker_stderr_tail() if with_worker_output else ""
+ details = f"\n\nWorker error output:\n{tail}" if tail else ""
+ # The operator's unredacted copy: the worker's forwarding daemon thread can die first.
+ self._log_worker_stderr_once(pid, exitcode)
+
if exitcode < 0:
signum = -exitcode
try:
@@ -790,9 +1032,12 @@ class InferenceOrchestrator:
" This usually means the system killed it under memory pressure. "
"Try a smaller model, lower context length, or close other GPU-heavy apps."
)
- return f"{message}{suffix} Details: pid={pid}, signal={sig_name}, exitcode={exitcode}."
+ return (
+ f"{message}{suffix} Details: pid={pid}, signal={sig_name}, "
+ f"exitcode={exitcode}.{details}"
+ )
- return f"{message} Details: pid={pid}, exitcode={exitcode}."
+ return f"{message} Details: pid={pid}, exitcode={exitcode}.{details}"
def _send_cmd(self, cmd: dict) -> None:
if self._cmd_queue is None:
@@ -840,7 +1085,11 @@ class InferenceOrchestrator:
if resp is None:
if not self._ensure_subprocess_alive():
- raise RuntimeError(self._subprocess_crash_message("wait"))
+ raise RuntimeError(
+ self._subprocess_crash_message(
+ "wait", with_worker_output = self._owns_worker(cancel_event)
+ )
+ )
continue
rtype = resp.get("type", "")
@@ -1058,18 +1307,18 @@ class InferenceOrchestrator:
initial_resp_queue = self._resp_queue
while True:
if self._proc is not initial_proc or self._resp_queue is not initial_resp_queue:
- yield GenStreamError(
- f"Error: {self._subprocess_crash_message(crash_context)}",
- public = True,
- )
+ # No tail here whatever the lists say: the capture is the REPLACEMENT's.
+ detail = self._subprocess_crash_message(crash_context)
+ yield GenStreamError(f"Error: {detail}", public = True)
return
resp = read_one(read_timeout)
if resp is None:
if not self._ensure_subprocess_alive():
- yield GenStreamError(
- f"Error: {self._subprocess_crash_message(crash_context)}",
- public = True,
+ # Only the request the worker was RUNNING gets its last words; the rest were queued behind it.
+ detail = self._subprocess_crash_message(
+ crash_context, with_worker_output = self._owns_worker(cancel_event)
)
+ yield GenStreamError(f"Error: {detail}", public = True)
return
continue
@@ -1457,7 +1706,12 @@ class InferenceOrchestrator:
resp = self._read_resp(timeout = min(remaining, 1.0))
if resp is None:
if not self._ensure_subprocess_alive():
- raise RuntimeError(self._subprocess_crash_message("sharing chat turn"))
+ raise RuntimeError(
+ self._subprocess_crash_message(
+ "sharing chat turn",
+ with_worker_output = self._owns_worker(None),
+ )
+ )
continue
rtype = resp.get("type", "")
@@ -2013,7 +2267,12 @@ class InferenceOrchestrator:
candidate = read_one(timeout = min(1.0, deadline - time.monotonic()))
if candidate is None:
if not self._ensure_subprocess_alive():
- raise RuntimeError(self._subprocess_crash_message("count"))
+ # A count takes no part in the claim bookkeeping.
+ raise RuntimeError(
+ self._subprocess_crash_message(
+ "count", with_worker_output = self._owns_worker(None)
+ )
+ )
continue
# _direct_reader already drops a reply whose mailbox is gone; this is the backstop.
if (
@@ -2580,7 +2839,10 @@ class InferenceOrchestrator:
if resp is None:
if not self._ensure_subprocess_alive():
raise RuntimeError(
- self._subprocess_crash_message("audio generation")
+ self._subprocess_crash_message(
+ "audio generation",
+ with_worker_output = self._owns_worker(cancel_event),
+ )
)
continue
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 52e5f9fa5d1..877afed0ae8 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -1120,6 +1120,13 @@ def run_inference_process(
service_name = "unsloth-studio-inference-worker",
env = os.getenv("ENVIRONMENT_TYPE", "production"),
)
+ # Must follow setup_logging. Structlog records go to fd 1, but a third-party library
+ # logging through stdlib `logging` reaches fd 2 via `logging.lastResort`, and that
+ # traceback is byte-identical to a dying process's: unmarked, the parent hands a
+ # RECOVERED failure to the NEXT caller on a shared worker as their crash.
+ from utils.worker_stderr import mark_log_record_continuations
+
+ mark_log_record_continuations()
apply_gpu_ids(config.get("resolved_gpu_ids"), backend = config.get("device_backend"))
diff --git a/studio/backend/tests/_worker_stderr_boom.py b/studio/backend/tests/_worker_stderr_boom.py
new file mode 100644
index 00000000000..3ea2df2ac92
--- /dev/null
+++ b/studio/backend/tests/_worker_stderr_boom.py
@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import sys
+
+
+def exit_one_after_writing_to_stderr(**_kwargs) -> None:
+ print("Received command: generate", file = sys.stderr)
+ print("Starting text generation", file = sys.stderr)
+ sys.stderr.flush()
+ raise RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB")
+
+
+def record_received_kwargs(**kwargs) -> None:
+ print(f"entrypoint kwargs: {sorted(kwargs)}", file = sys.stderr)
+ sys.stderr.flush()
+ raise SystemExit(1)
+
+
+def hold_a_second_handle_on_stderr_then_exit(**_kwargs) -> None:
+ import os
+
+ _keep_alive = os.dup(2) # noqa: F841
+ print("held a second handle on stderr", file = sys.stderr)
+ sys.stderr.flush()
+ raise SystemExit(1)
+
+
+def write_non_ascii_far_past_the_read_window(**_kwargs) -> None:
+ from utils.worker_stderr import TAIL_READ_BYTES
+
+ padding = "é" * 40
+ for index in range((TAIL_READ_BYTES // len(padding.encode("utf-8"))) + 200):
+ sys.stderr.write(f"{padding} warmup {index}\n")
+ sys.stderr.write("RuntimeError: modellädt nicht — CUDA out of memory\n")
+ sys.stderr.flush()
+ raise SystemExit(1)
+
+
+def write_far_more_than_the_cap(**_kwargs) -> None:
+ from utils.worker_stderr import MIRROR_FILE_CAP_BYTES
+
+ line = "x" * 1023 + "\n"
+ for _ in range((MIRROR_FILE_CAP_BYTES * 3) // len(line)):
+ sys.stderr.write(line)
+ sys.stderr.flush()
+ raise RuntimeError("the last line before the worker died")
diff --git a/studio/backend/tests/test_inference_worker_stderr_tail.py b/studio/backend/tests/test_inference_worker_stderr_tail.py
new file mode 100644
index 00000000000..282c0d36e65
--- /dev/null
+++ b/studio/backend/tests/test_inference_worker_stderr_tail.py
@@ -0,0 +1,1243 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""unsloth#7843: a worker that exits without answering must not lose its own error output."""
+
+import subprocess
+from pathlib import Path
+from types import SimpleNamespace
+import importlib.util
+import multiprocessing as mp
+import os
+import signal
+import sys
+
+import pytest
+
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from utils.worker_stderr import ( # noqa: E402
+ MIRROR_FILE_CAP_BYTES,
+ STDERR_MIRROR_KWARG,
+ WorkerStderrCapture,
+ decode_worker_stderr,
+ stderr_tail_from_bytes,
+)
+
+
+def _load_orchestrator_module():
+ spec = importlib.util.spec_from_file_location(
+ "inference_orchestrator_stderr_tail_under_test",
+ Path(_BACKEND_DIR) / "core/inference/orchestrator.py",
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _orchestrator_with(
+ exitcode,
+ capture = None,
+ pid = 6145,
+):
+ module = _load_orchestrator_module()
+ orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)
+ orchestrator._proc = SimpleNamespace(pid = pid, exitcode = exitcode)
+ orchestrator._stderr_capture = capture
+ return orchestrator
+
+
+def test_a_windows_worker_arrives_as_crlf_cp1252_and_decodes_cleanly():
+ raw = "Traceback (most recent call last):\r\n File “worker.py”, line 1\r\n".encode(
+ "cp1252",
+ )
+ assert b"\r\n" in raw
+ text = decode_worker_stderr(raw)
+ assert "\r" not in text
+ assert text.splitlines()[0] == "Traceback (most recent call last):"
+ assert "“worker.py”" in text
+
+
+def test_a_utf8_worker_is_never_mis_read_as_cp1252():
+ raw = "RuntimeError: — out of memory é\n".encode("utf-8")
+ assert decode_worker_stderr(raw) == "RuntimeError: — out of memory é\n"
+
+
+def test_cp1252_bytes_that_are_not_valid_utf8_still_decode():
+ raw = b"warning: caf\xe9 layer skipped\r\n"
+ assert decode_worker_stderr(raw) == "warning: café layer skipped\n"
+
+
+def test_a_cp1252_character_at_the_very_end_is_not_trimmed_away():
+ assert decode_worker_stderr(b"RuntimeError: caf\xe9") == "RuntimeError: café"
+ assert decode_worker_stderr("é at the start".encode("utf-8")[1:]).endswith(" at the start")
+ truncated = "RuntimeError: é".encode("utf-8")[:-1]
+ assert decode_worker_stderr(truncated, ends_at_eof = False) == "RuntimeError: "
+ assert decode_worker_stderr(truncated) == "RuntimeError: \u00c3"
+ assert decode_worker_stderr("done é".encode("utf-8")) == "done é"
+
+
+def test_undecodable_bytes_are_replaced_rather_than_dropped():
+ # 0x81 and 0x90 are undefined in cp1252 and invalid as UTF-8.
+ raw = b"RuntimeError: \x81\x90 bad bytes\n"
+ text = decode_worker_stderr(raw)
+ assert "RuntimeError:" in text
+ assert "bad bytes" in text
+
+
+def test_a_progress_bar_redrawn_with_carriage_returns_does_not_hide_the_traceback():
+ raw = (
+ b"Loading checkpoint: 10%\rLoading checkpoint: 55%\rLoading checkpoint: 100%\r\n"
+ b"RuntimeError: the real failure\n"
+ )
+ tail = stderr_tail_from_bytes(raw, max_lines = 2)
+ assert tail.splitlines()[-1] == "RuntimeError: the real failure"
+
+
+def test_the_tail_keeps_the_last_lines_and_drops_blank_ones():
+ raw = ("\n".join(f"line {index}" for index in range(100)) + "\n\n\n").encode("utf-8")
+ tail = stderr_tail_from_bytes(raw, max_lines = 3)
+ assert tail == "line 97\nline 98\nline 99"
+
+
+def test_the_character_cap_trims_from_the_front_and_never_shows_a_half_line():
+ raw = ("first line that is quite long\n" + "b" * 200 + "\nlast line\n").encode("utf-8")
+ tail = stderr_tail_from_bytes(raw, max_lines = 10, max_chars = 60)
+ assert tail.endswith("last line")
+ assert len(tail) <= 60
+ assert not tail.startswith("b")
+
+
+def test_an_empty_sink_produces_no_tail():
+ assert stderr_tail_from_bytes(b"") == ""
+ assert stderr_tail_from_bytes(b"\n \n\r\n") == ""
+
+
+def test_a_capture_whose_file_is_gone_reports_no_tail_instead_of_raising():
+ capture = WorkerStderrCapture(prefix = "unsloth-test-")
+ capture.close()
+ assert capture.tail() == ""
+ # Idempotent: a Windows host takes a second close when the child still holds it.
+ capture.close()
+
+
+def test_a_positive_exit_carries_the_worker_stderr_tail(tmp_path):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ Path(capture.path).write_bytes(
+ b"Starting text generation\r\n"
+ b"Traceback (most recent call last):\r\n"
+ b"RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB\r\n",
+ )
+
+ message = _orchestrator_with(1, capture)._subprocess_crash_message(
+ "generation", with_worker_output = True
+ )
+
+ assert message.startswith(
+ "The inference worker stopped unexpectedly while generating a response.",
+ )
+ assert "Details: pid=6145, exitcode=1." in message
+ assert "Worker error output:" in message
+ assert "RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB" in message
+ assert "\r" not in message
+
+
+def test_a_signalled_exit_keeps_its_hint_and_gains_the_tail(tmp_path):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ Path(capture.path).write_bytes(b"terminate called after throwing an instance of 'c10::Error'\n")
+
+ message = _orchestrator_with(-9, capture)._subprocess_crash_message(
+ "wait", with_worker_output = True
+ )
+
+ assert "exitcode=-9" in message
+ assert "c10::Error" in message
+
+ # POSIX only: a Windows process never reports a negative code and signal.Signals(9)
+ # raises there, so that branch names the signal SIG9 and offers no hint.
+ if hasattr(signal, "SIGKILL"):
+ assert "memory pressure" in message
+ assert "signal=SIGKILL" in message
+ else:
+ assert "signal=SIG9" in message
+
+
+def test_nothing_captured_leaves_the_message_exactly_as_it_was(tmp_path):
+ empty = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ for capture in (None, empty):
+ message = _orchestrator_with(1, capture)._subprocess_crash_message(
+ "generation", with_worker_output = True
+ )
+ assert message == (
+ "The inference worker stopped unexpectedly while generating a response. "
+ "Details: pid=6145, exitcode=1."
+ )
+
+
+def test_a_worker_still_running_is_not_given_a_tail(tmp_path):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ Path(capture.path).write_bytes(b"still going\n")
+ message = _orchestrator_with(None, capture)._subprocess_crash_message(
+ "generation", with_worker_output = True
+ )
+ assert message.endswith("Details: pid=6145.")
+
+
+def test_the_orchestrator_hands_the_sink_to_the_child_it_spawns():
+ # Read from the source: exercising it for real needs a model load.
+ source = (
+ (Path(_BACKEND_DIR) / "core/inference/orchestrator.py")
+ .read_text(
+ encoding = "utf-8",
+ )
+ .replace("\r\n", "\n")
+ )
+ start = source.index("def _spawn_subprocess(")
+ body = source[start : source.index("\n def ", start + 10)]
+ assert "WorkerStderrCapture" in body, "the spawn path no longer opens a stderr sink"
+ assert "STDERR_MIRROR_KWARG" in body, "the sink is no longer passed to the child"
+ assert "_retire_stderr_capture()" in body, "a stale sink now survives into the next worker"
+
+
+def test_the_reserved_kwarg_name_is_the_same_on_both_sides():
+ # utils/native_path_leases.py spells the name out to keep its import graph stdlib only.
+ from utils.native_path_leases import STDERR_MIRROR_KWARG as entrypoint_name
+ assert entrypoint_name == STDERR_MIRROR_KWARG
+
+
+def _spawn(
+ function_name,
+ mirror_path,
+ timeout = 180,
+):
+ from utils.native_path_leases import run_without_native_path_secret
+
+ # Bare name, with this directory on sys.path: a dotted name finds the repository's own
+ # top-level "tests" package instead. Spawn hands the child the parent's sys.path.
+ tests_dir = str(Path(__file__).resolve().parent)
+ if tests_dir not in sys.path:
+ sys.path.insert(0, tests_dir)
+
+ context = mp.get_context("spawn")
+ kwargs = {} if mirror_path is None else {STDERR_MIRROR_KWARG: mirror_path}
+ process = context.Process(
+ target = run_without_native_path_secret,
+ args = ("_worker_stderr_boom", function_name, {}),
+ kwargs = kwargs,
+ )
+ process.start()
+ process.join(timeout)
+ assert not process.is_alive(), "the test worker did not exit"
+ return process
+
+
+def test_a_worker_that_exits_one_after_writing_to_stderr_keeps_its_traceback(tmp_path):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ process = _spawn("exit_one_after_writing_to_stderr", capture.path)
+ assert process.exitcode == 1
+
+ orchestrator = _orchestrator_with(process.exitcode, capture, pid = process.pid)
+ message = orchestrator._subprocess_crash_message("generation", with_worker_output = True)
+
+ assert "RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB" in message
+ assert "Traceback (most recent call last)" in message
+ assert "Starting text generation" in capture.tail(max_lines = 100)
+
+
+def test_the_reserved_kwarg_is_consumed_and_never_reaches_the_entrypoint(tmp_path):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ process = _spawn("record_received_kwargs", capture.path)
+ assert process.exitcode == 1
+ tail = capture.tail(max_lines = 100)
+ assert "entrypoint kwargs: []" in tail, tail
+
+
+def test_the_sink_is_bounded_even_when_the_worker_floods_stderr(tmp_path):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ process = _spawn("write_far_more_than_the_cap", capture.path)
+ assert process.exitcode == 1
+
+ size = os.path.getsize(capture.path)
+ assert size <= 2 * MIRROR_FILE_CAP_BYTES + 65536, size
+ assert "the last line before the worker died" in capture.tail(max_lines = 200)
+
+
+_STALLED_READER_CHILD = r"""
+import os, sys
+sys.path.insert(0, %(backend)r)
+from utils.worker_stderr import MIRROR_FILE_CAP_BYTES, install_worker_stderr_mirror
+
+assert install_worker_stderr_mirror(%(path)r) is True
+line = b"x" * 1023 + b"\n"
+for _ in range((MIRROR_FILE_CAP_BYTES * 32) // len(line)):
+ os.write(2, line)
+os.write(2, b"the last line before the worker died\n")
+"""
+
+
+def test_the_sink_is_bounded_even_when_nobody_drains_the_operators_stderr(tmp_path):
+ """The mirrored copy is a courtesy; the cap is not. 68 MB landed here when a full pipe
+ parked the pump inside os.write and the cap counted bytes relayed rather than bytes written."""
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ script = _STALLED_READER_CHILD % {"backend": _BACKEND_DIR, "path": capture.path}
+ read_end, write_end = os.pipe()
+ try:
+ process = subprocess.Popen([sys.executable, "-c", script], stderr = write_end)
+ os.close(write_end)
+ write_end = None
+ # Nothing ever reads `read_end`, which is what a stalled log consumer looks like.
+ assert process.wait(timeout = 300) == 0
+ finally:
+ if write_end is not None:
+ os.close(write_end)
+ os.close(read_end)
+
+ size = os.path.getsize(capture.path)
+ assert size <= 2 * MIRROR_FILE_CAP_BYTES + 65536, size
+ assert "the last line before the worker died" in capture.tail(max_lines = 200)
+ capture.close()
+
+
+def test_a_caller_that_passes_no_mirror_is_unchanged(tmp_path):
+ process = _spawn("exit_one_after_writing_to_stderr", None)
+ assert process.exitcode == 1
+ assert _orchestrator_with(process.exitcode, None, pid = process.pid)._subprocess_crash_message(
+ "generation",
+ ) == (
+ "The inference worker stopped unexpectedly while generating a response. "
+ f"Details: pid={process.pid}, exitcode=1."
+ )
+
+
+def test_a_server_that_exits_leaves_no_sink_behind(tmp_path):
+ script = (
+ "import sys; sys.path.insert(0, %r)\n"
+ "from utils.worker_stderr import WorkerStderrCapture\n"
+ "print(WorkerStderrCapture(directory = %r, prefix = 'unsloth-test-').path)\n"
+ ) % (_BACKEND_DIR, str(tmp_path))
+ import subprocess
+
+ path = subprocess.run(
+ [sys.executable, "-c", script],
+ capture_output = True,
+ text = True,
+ check = True,
+ ).stdout.strip()
+ assert path.startswith(str(tmp_path))
+ assert not os.path.exists(path), "the sink outlived the process that opened it"
+
+
+def test_a_severed_multibyte_character_does_not_mojibake_the_whole_tail():
+ from utils.worker_stderr import TAIL_READ_BYTES
+
+ line = "RuntimeError: modellädt nicht — CUDA out of memory\n".encode("utf-8")
+ body = line * 30
+ window = b"\xa9" + b"." * (TAIL_READ_BYTES - 1 - len(body)) + body
+ assert len(window) == TAIL_READ_BYTES
+ with pytest.raises(UnicodeDecodeError):
+ window.decode("utf-8")
+
+ tail = stderr_tail_from_bytes(window)
+ assert "modellädt nicht — CUDA out of memory" in tail, tail
+ assert "ä" not in tail and "—" not in tail, tail
+
+
+def test_a_severed_character_at_the_end_of_the_window_is_dropped_too():
+ raw = "RuntimeError: café blew up — ".encode("utf-8") + "é".encode("utf-8")[:1]
+ text = decode_worker_stderr(raw)
+ assert "café blew up —" in text, text
+ assert "é" not in text, text
+
+
+def test_a_real_worker_whose_output_outgrows_the_read_window_still_reads_cleanly(tmp_path):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ process = _spawn("write_non_ascii_far_past_the_read_window", capture.path, timeout = 60)
+ assert process.exitcode == 1
+ tail = capture.tail(max_lines = 5)
+ assert "modellädt nicht — CUDA out of memory" in tail, tail
+ assert "ä" not in tail, tail
+ capture.close()
+
+
+def test_a_missing_sink_is_never_re_created_by_the_child(tmp_path):
+ from utils.worker_stderr import install_worker_stderr_mirror
+
+ missing = tmp_path / "already-retired.stderr"
+ assert install_worker_stderr_mirror(str(missing)) is False
+ assert not missing.exists(), "the child created a sink the parent had retired"
+
+
[email protected](not hasattr(os, "O_NOFOLLOW"), reason = "O_NOFOLLOW is POSIX only")
+def test_a_symlink_at_the_sink_path_is_not_followed(tmp_path):
+ from utils.worker_stderr import install_worker_stderr_mirror
+
+ victim = tmp_path / "victim.txt"
+ victim.write_text("ORIGINAL\n", encoding = "utf-8")
+ link = tmp_path / "sink.stderr"
+ link.symlink_to(victim)
+
+ assert install_worker_stderr_mirror(str(link)) is False
+ assert victim.read_text(encoding = "utf-8") == "ORIGINAL\n"
+
+
+def test_a_worker_holding_a_second_handle_on_stderr_still_exits_promptly(tmp_path):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ process = _spawn("hold_a_second_handle_on_stderr_then_exit", capture.path, timeout = 60)
+ assert process.exitcode == 1
+ assert "held a second handle on stderr" in capture.tail(), capture.tail()
+ capture.close()
+
+ # The race cannot be asserted from outside: a run that does not hit it proves nothing.
+ source = (Path(_BACKEND_DIR) / "utils/worker_stderr.py").read_text(encoding = "utf-8")
+ teardown = source.split("def _stop_mirror(", 1)[1].split("\ndef ", 1)[0]
+ close_at = teardown.index("os.close(inherited_fd)")
+ # Both threads write to the inherited descriptor, so both must be done with it.
+ for guard in ("pump.is_alive()", "relay.is_alive()"):
+ assert guard in teardown[:close_at], (
+ "the teardown closes the inherited stderr without first checking that the "
+ f"thread behind {guard} has finished with it"
+ )
+
+
[email protected](not hasattr(os, "fork"), reason = "fork is POSIX only")
+def test_a_forked_child_does_not_retire_its_parents_live_sink(tmp_path):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ Path(capture.path).write_bytes(b"the parent is still writing here\n")
+
+ pid = os.fork()
+ if pid == 0: # pragma: no cover - runs in the child
+ import atexit as _atexit
+ try:
+ _atexit._run_exitfuncs()
+ finally:
+ os._exit(0)
+ os.waitpid(pid, 0)
+
+ assert os.path.exists(capture.path), "a forked child retired its parent's live sink"
+ assert "the parent is still writing here" in capture.tail()
+ capture.close()
+
+
+def test_the_sink_directory_is_never_swept_by_pattern():
+ source = (Path(_BACKEND_DIR) / "utils/worker_stderr.py").read_text(encoding = "utf-8")
+ for forbidden in (
+ "glob(",
+ "iglob(",
+ "iterdir(",
+ "listdir(",
+ "scandir(",
+ "walk(",
+ "st_mtime",
+ "rmtree(",
+ ):
+ assert forbidden not in source, f"cleanup grew a directory sweep ({forbidden})"
+
+
+# Spaces and non-ASCII are what break code that builds paths by string, and a Windows
+# profile routinely has both.
+_INSTALL_TEMP_SHAPES = {
+ "linux": "tmp",
+ "windows-profile": "C_/Users/Jos\u00e9 Mu\u00f1oz/AppData/Local/Temp",
+ "wsl-windows-drive": "mnt/c/Users/Jos\u00e9 Mu\u00f1oz/AppData/Local/Temp",
+ "macos-per-user": "var/folders/9k/9k1_2p_s0qz3h/T",
+}
+
+
[email protected]("shape", sorted(_INSTALL_TEMP_SHAPES))
+def test_the_sink_works_on_every_install_path_shape(tmp_path, shape):
+ directory = tmp_path.joinpath(*_INSTALL_TEMP_SHAPES[shape].split("/"))
+ try:
+ directory.mkdir(parents = True)
+ except (OSError, UnicodeError) as exc:
+ pytest.skip(f"this filesystem cannot hold a {shape} path: {exc}")
+
+ capture = WorkerStderrCapture(directory = str(directory), prefix = "unsloth-test-")
+ assert Path(capture.path).parent == directory
+ process = _spawn("exit_one_after_writing_to_stderr", capture.path)
+ assert process.exitcode == 1
+
+ message = _orchestrator_with(
+ process.exitcode,
+ capture,
+ pid = process.pid,
+ )._subprocess_crash_message("generation", with_worker_output = True)
+ assert "RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB" in message
+
+ capture.close()
+
+
[email protected](os.name == "nt", reason = "the parent's stderr is redirected per platform")
+def test_the_mirror_writes_through_to_the_inherited_stderr(tmp_path, capfd):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ _spawn("exit_one_after_writing_to_stderr", capture.path)
+ captured = capfd.readouterr()
+ assert "Starting text generation" in captured.err
+ assert "CUDA out of memory" in captured.err
+
+
+class _FixedCapture:
+ def __init__(self, text):
+ self._text = text
+
+ def tail(self):
+ return self._text
+
+
+def _orchestrator_with_capture(text):
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ instance = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ instance._stderr_capture = _FixedCapture(text)
+ return instance
+
+
+CROSS_ACCOUNT = (
+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text: "
+ "another account's private prompt\n"
+ "2026-09-16 10:00:02 worker: request 41 finished\n"
+)
+TRACEBACK = (
+ "Traceback (most recent call last):\n"
+ ' File "/home/alice/.unsloth/studio/worker.py", line 42, in run\n'
+ ' raise RuntimeError("boom")\n'
+ "RuntimeError: boom\n"
+)
+
+
+def test_the_public_tail_drops_everything_before_the_traceback():
+ public = _orchestrator_with_capture(CROSS_ACCOUNT + TRACEBACK)._public_worker_stderr_tail()
+ assert "another account" not in public, public
+ assert "request 41 finished" not in public, public
+ assert public.startswith("Traceback (most recent call last):"), public
+ assert "RuntimeError: boom" in public
+
+
+def test_the_public_tail_keeps_only_the_last_traceback():
+ earlier = TRACEBACK.replace("boom", "an earlier unrelated failure")
+ public = _orchestrator_with_capture(
+ earlier + CROSS_ACCOUNT + TRACEBACK
+ )._public_worker_stderr_tail()
+ assert "an earlier unrelated failure" not in public, public
+ assert "RuntimeError: boom" in public
+
+
+def test_a_capture_of_nothing_but_logging_is_dropped_entirely():
+ assert _orchestrator_with_capture(CROSS_ACCOUNT)._public_worker_stderr_tail() == ""
+ assert _orchestrator_with_capture("")._public_worker_stderr_tail() == ""
+
+
+def test_a_native_abort_with_no_traceback_is_still_reported():
+ abort = "terminate called after throwing an instance of 'c10::Error'\n"
+ public = _orchestrator_with_capture(CROSS_ACCOUNT + abort)._public_worker_stderr_tail()
+ assert "c10::Error" in public, public
+ assert "another account" not in public, public
+
+
+def test_the_log_record_shapes_are_the_ones_content_arrives_in():
+ from core.inference.orchestrator import _looks_like_a_log_record
+ for record in (
+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text: hello",
+ "2026-09-16T10:00:01 worker: started",
+ "INFO: loaded the model",
+ "[WARNING] falling back to CPU",
+ "audio_codecs.decode_bicodec: generated text: hello",
+ ):
+ assert _looks_like_a_log_record(record) is True, record
+
+ for kept in (
+ "terminate called after throwing an instance of 'c10::Error'",
+ "Traceback (most recent call last):",
+ "RuntimeError: boom",
+ ' File "/x/worker.py", line 42, in run',
+ "Segmentation fault (core dumped)",
+ ):
+ assert _looks_like_a_log_record(kept) is False, kept
+
+
+def test_the_public_tail_redacts_paths_and_credentials():
+ from core.inference.orchestrator import _redact_worker_output
+
+ text = (
+ "Traceback (most recent call last):\n"
+ ' File "/home/alice/.unsloth/studio/worker.py", line 42, in run\n'
+ ' login(token="hf_abcdefghijklmnopqrstuvwxyz012345")\n'
+ "RuntimeError: refused, Authorization: Bearer sk-secret-value\n"
+ )
+ public = _redact_worker_output(text)
+ assert "/home/alice" not in public, public
+ assert "worker.py" in public, public
+ assert "hf_abcdefghijklmnopqrstuvwxyz012345" not in public, public
+ assert "sk-secret-value" not in public, public
+ assert "RuntimeError: refused" in public, public
+
+
+def test_the_crash_message_uses_the_public_tail():
+ import inspect
+
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ source = inspect.getsource(InferenceOrchestrator._subprocess_crash_message)
+ assert "_public_worker_stderr_tail()" in source
+ assert "= self._worker_stderr_tail()" not in source
+
+
+_FATAL_CHILD = r"""
+import os, signal, sys
+sys.path.insert(0, %(backend)r)
+from utils.worker_stderr import install_worker_stderr_mirror
+
+assert install_worker_stderr_mirror(%(path)r) is True
+os.write(2, b"terminate called after throwing an instance of 'c10::Error'\n")
+# No flush, no atexit, no interpreter shutdown: the process is gone between one
+# instruction and the next, which is what SIGSEGV and SIGABRT do.
+# SIGKILL on POSIX; Windows has no such signal, and os.kill there calls
+# TerminateProcess, which is the same thing for this purpose: no handler, no cleanup.
+os.kill(os.getpid(), getattr(signal, "SIGKILL", signal.SIGTERM))
+"""
+
+
+def test_a_worker_killed_outright_still_leaves_its_last_words(tmp_path):
+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")
+ backend = str(Path(__file__).resolve().parent.parent)
+ script = _FATAL_CHILD % {"backend": backend, "path": capture.path}
+
+ completed = subprocess.run(
+ [sys.executable, "-c", script],
+ capture_output = True,
+ timeout = 120,
+ )
+ if os.name == "nt":
+ assert completed.returncode != 0, completed.stderr[-400:]
+ else:
+ assert completed.returncode == -signal.SIGKILL, completed.stderr[-400:]
+ assert "c10::Error" in capture.tail(), capture.tail()
+
+
+def test_the_mirror_puts_the_sink_on_fd_two_rather_than_a_pipe():
+ from utils import worker_stderr
+
+ source = Path(worker_stderr.__file__).read_text(encoding = "utf-8")
+ install = source.split("def install_worker_stderr_mirror(", 1)[1]
+ assert "os.pipe()" not in install, "fd 2 is a pipe again"
+ assert "_open_sink_for_append" in install
+ assert "os.dup2(writer_fd, 2)" in install
+ # Compaction rewrites the file from the front while fd 2 still points at it.
+ append = source.split("def _open_sink_for_append(", 1)[1].split("\ndef ", 1)[0]
+ assert "os.O_APPEND" in append
+
+
+MULTILINE_CROSS_ACCOUNT = (
+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text: the first line\n"
+ "another account's second line\n"
+ "and a third line of the same prompt\n"
+)
+
+
+def test_a_logged_message_cannot_leak_through_its_own_continuation_lines():
+ abort = (
+ "terminate called after throwing an instance of 'c10::Error'\n"
+ " what(): CUDA error: device-side assert triggered\n"
+ )
+ public = _orchestrator_with_capture(
+ MULTILINE_CROSS_ACCOUNT + abort
+ )._public_worker_stderr_tail()
+ assert "c10::Error" in public, public
+ assert "device-side assert" in public, public
+ assert "another account" not in public, public
+ assert "third line" not in public, public
+
+
+def test_compaction_never_deletes_what_a_racing_writer_appended(tmp_path):
+ from utils.worker_stderr import _compact_sink
+
+ path = tmp_path / "sink"
+ path.write_bytes(b"x" * 4096)
+ appender = os.open(str(path), os.O_WRONLY | os.O_APPEND)
+ handle = open(path, "r+b", buffering = 0)
+
+ class _WriterRacesTheRead:
+ def __init__(self, inner):
+ self._inner = inner
+ self._raced = False
+ self._raced_the_write = False
+
+ def __getattr__(self, name):
+ return getattr(self._inner, name)
+
+ def read(self, *args):
+ data = self._inner.read(*args)
+ if not self._raced:
+ self._raced = True
+ os.write(appender, b"terminate called after throwing an instance of 'c10::Error'\n")
+ return data
+
+ def write(self, payload):
+ written = self._inner.write(payload)
+ if not self._raced_the_write:
+ self._raced_the_write = True
+ os.write(appender, b"what(): CUDA error: device-side assert triggered\n")
+ return written
+
+ try:
+ _compact_sink(_WriterRacesTheRead(handle), 1024)
+ finally:
+ handle.close()
+ os.close(appender)
+
+ kept = path.read_bytes()
+ # Both windows: the append during the tail READ and the one during the REWRITE.
+ assert b"terminate called" in kept, kept[-300:]
+ assert b"device-side assert" in kept, kept[-300:]
+ assert len(kept) <= 1024 + 256, len(kept)
+
+
+def test_the_operator_still_gets_the_last_words_in_the_server_log(monkeypatch):
+ from core.inference import orchestrator as orchestrator_module
+
+ written = []
+ monkeypatch.setattr(
+ orchestrator_module.logger,
+ "error",
+ lambda message, *args, **kwargs: written.append(message % args if args else message),
+ )
+
+ abort = (
+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text: private\n"
+ "terminate called after throwing an instance of 'c10::Error'\n"
+ " what(): CUDA error at /home/alice/.unsloth/studio/worker.py\n"
+ )
+ instance = _orchestrator_with_capture(abort)
+ instance._proc = SimpleNamespace(exitcode = -6, pid = 4242, is_alive = lambda: False)
+ message = instance._subprocess_crash_message("generation", with_worker_output = True)
+
+ logged = "\n".join(written)
+ assert "terminate called" in logged, logged
+ assert "/home/alice" in logged, logged
+ assert "generated text: private" in logged, logged
+ assert "/home/alice" not in message, message
+ assert "generated text" not in message, message
+
+ written.clear()
+ instance._subprocess_crash_message("wait")
+ assert written == [], written
+
+
+def test_a_teardown_that_clears_the_handle_first_still_logs_the_tail(monkeypatch):
+ from core.inference import orchestrator as orchestrator_module
+
+ written = []
+ monkeypatch.setattr(
+ orchestrator_module.logger,
+ "error",
+ lambda message, *args, **kwargs: written.append(message % args if args else message),
+ )
+ instance = _orchestrator_with_capture(
+ "terminate called after throwing an instance of 'c10::Error'\n"
+ )
+ instance._proc = None
+ message = instance._subprocess_crash_message("generation", with_worker_output = True)
+ assert "process missing" in message
+ assert any("terminate called" in line for line in written), written
+
+ written.clear()
+ other = _orchestrator_with_capture("Fatal Python error: Segmentation fault\n")
+ other._proc = SimpleNamespace(exitcode = None, pid = 99, is_alive = lambda: True)
+ other._subprocess_crash_message("wait")
+ assert any("Segmentation fault" in line for line in written), written
+
+
+def test_a_logged_traceback_is_not_mistaken_for_the_crash():
+ logged = (
+ "2026-09-16 10:00:03 worker: request 41 failed, retrying\n"
+ "Traceback (most recent call last):\n"
+ ' File "/home/alice/.unsloth/studio/worker.py", line 9, in handle\n'
+ ' raise ValueError("another account\'s prompt was rejected")\n'
+ "ValueError: another account's prompt was rejected\n"
+ )
+ abort = "Fatal Python error: Aborted\n\nCurrent thread 0x00007f1a (most recent call first):\n"
+ public = _orchestrator_with_capture(logged + abort)._public_worker_stderr_tail()
+ assert "another account" not in public, public
+ assert "ValueError" not in public, public
+ assert "Fatal Python error: Aborted" in public, public
+
+
+def test_a_runtimes_own_traceback_after_a_log_line_is_still_reported():
+ text = "2026-09-16 10:00:02 worker: request 41 finished\n\n" + TRACEBACK
+ public = _orchestrator_with_capture(text)._public_worker_stderr_tail()
+ assert "RuntimeError: boom" in public, public
+ assert "request 41 finished" not in public, public
+
+
+def test_a_path_component_with_punctuation_in_it_is_still_redacted():
+ from core.inference.orchestrator import _redact_worker_output
+
+ for path in (
+ "/home/o'connor/private/model.py",
+ "/home/a(1)/private/model.py",
+ "/srv/models/llama:8b/weights.gguf",
+ "C:\\Users\\O'Brien\\models\\weights.gguf",
+ "\\\\share\\team\\o'brien\\model.gguf",
+ ):
+ public = _redact_worker_output(f' File "{path}", line 1, in run\n')
+ assert "private" not in public, public
+ assert "o'connor" not in public.lower(), public
+ assert "o'brien" not in public.lower(), public
+ assert "llama:8b" not in public, public
+ assert public.strip().startswith('File ".../'), public
+
+ spaced = _redact_worker_output("C:\\Program Files\\unsloth\\weights.gguf failed\n")
+ assert spaced.strip() == ".../weights.gguf failed", spaced
+ two = _redact_worker_output(' File "/a/b.py", line 1, then /etc/passwd here\n')
+ assert "/etc/passwd" not in two, two
+ assert "line 1, then" in two, two
+ assert _redact_worker_output("a ratio of 3/4 at https://host/path/x\n").strip() == (
+ "a ratio of 3/4 at https://host/path/x"
+ )
+
+
+MARK = " | "
+START_MARK = "\x1f"
+
+
+def test_the_worker_marks_every_continuation_line_of_a_record():
+ import logging
+ from utils.worker_stderr import LOG_RECORD_CONTINUATION_PREFIX, mark_log_record_continuations
+
+ assert LOG_RECORD_CONTINUATION_PREFIX == MARK
+
+ logger_object = logging.getLogger("unsloth-test-marking")
+ logger_object.handlers = []
+ handler = logging.StreamHandler()
+ handler.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
+ logger_object.addHandler(handler)
+
+ # cover_later_handlers off so this case does not edit the interpreter's logging for
+ # the rest of the session; the process-wide half is exercised on its own below.
+ assert mark_log_record_continuations(logger_object, cover_later_handlers = False) == 1
+ assert mark_log_record_continuations(logger_object, cover_later_handlers = False) == 0
+
+ record = logger_object.makeRecord(
+ "unsloth-test-marking",
+ logging.ERROR,
+ __file__,
+ 1,
+ "generated text: line one\nRuntimeError: not really\nTraceback (most recent call last):",
+ (),
+ None,
+ )
+ formatted = handler.formatter.format(record)
+ lines = formatted.split("\n")
+ assert lines[0].startswith(START_MARK + "ERROR unsloth-test-marking:"), lines
+ assert all(line.startswith(MARK) for line in lines[1:]), lines
+
+ single = logger_object.makeRecord(
+ "unsloth-test-marking",
+ logging.ERROR,
+ __file__,
+ 1,
+ "one line",
+ (),
+ None,
+ )
+ one_line = handler.formatter.format(single)
+ assert "\n" not in one_line
+ # A single-line record through a default formatter is otherwise the same bytes a dying
+ # runtime writes.
+ assert one_line.startswith(START_MARK), repr(one_line)
+
+
+def test_a_default_formatter_record_is_not_mistaken_for_a_crash():
+ logged = START_MARK + "RuntimeError: another account's prompt was rejected\n"
+ assert _orchestrator_with_capture(logged)._public_worker_stderr_tail() == ""
+
+ public = _orchestrator_with_capture(
+ logged
+ + "Fatal Python error: Aborted\n"
+ + ' File "/home/alice/.unsloth/studio/worker.py", line 9, in handle\n'
+ )._public_worker_stderr_tail()
+ assert "another account" not in public, public
+ assert "Fatal Python error: Aborted" in public, public
+
+
+def test_a_logged_traceback_is_not_returned_when_the_worker_dies_silently():
+ logged = (
+ "2026-09-16 10:00:03 worker: Generation error: another account's prompt was rejected\n"
+ + MARK
+ + "Traceback (most recent call last):\n"
+ + MARK
+ + ' File "/home/alice/.unsloth/studio/worker.py", line 9, in handle\n'
+ + MARK
+ + "ValueError: another account's prompt was rejected\n"
+ )
+ public = _orchestrator_with_capture(logged)._public_worker_stderr_tail()
+ assert public == "", public
+
+
+def test_a_marked_continuation_that_reads_like_a_diagnostic_is_still_content():
+ content = (
+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text:\n"
+ + MARK
+ + "RuntimeError: another account's private prompt\n"
+ + MARK
+ + "Fatal Python error: also theirs\n"
+ + MARK
+ + "Killed\n"
+ )
+ public = _orchestrator_with_capture(content)._public_worker_stderr_tail()
+ assert public == "", public
+ assert "another account" not in public
+
+
+def test_a_crash_after_a_marked_record_is_still_the_crash():
+ text = (
+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text:\n"
+ + MARK
+ + "another account's private prompt\n"
+ + TRACEBACK
+ )
+ public = _orchestrator_with_capture(text)._public_worker_stderr_tail()
+ assert "another account" not in public, public
+ assert "RuntimeError: boom" in public, public
+
+
+def test_the_worker_installs_the_marking_where_its_logging_is_configured():
+ import inspect
+ from core.inference import worker as worker_module
+
+ source = inspect.getsource(worker_module)
+ assert "mark_log_record_continuations()" in source
+ setup = source.index("LogConfig.setup_logging(")
+ assert 0 < setup < source.index("mark_log_record_continuations()")
+
+
+def test_a_marked_record_under_a_diagnostic_is_not_adopted_by_it():
+ text = (
+ "terminate called after throwing an instance of 'c10::Error'\n"
+ + MARK
+ + "another account's private prompt\n"
+ " what(): CUDA error: device-side assert triggered\n"
+ )
+ public = _orchestrator_with_capture(text)._public_worker_stderr_tail()
+ assert "another account" not in public, public
+ assert "terminate called" in public, public
+ assert "CUDA error: device-side assert triggered" in public, public
+
+
+def test_a_live_replacements_own_crash_is_still_written_to_the_log(monkeypatch):
+ import importlib.util as _ilu
+
+ spec = _ilu.spec_from_file_location(
+ "inference_orchestrator_stderr_log_once_under_test",
+ Path(_BACKEND_DIR) / "core/inference/orchestrator.py",
+ )
+ module = _ilu.module_from_spec(spec)
+ spec.loader.exec_module(module)
+
+ written: "list[tuple]" = []
+ monkeypatch.setattr(
+ module.logger,
+ "error",
+ lambda *args, **kwargs: written.append(args),
+ raising = False,
+ )
+
+ capture = _FixedCapture("Fatal Python error: Aborted\n")
+ orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)
+ orchestrator._stderr_capture = capture
+
+ orchestrator._proc = SimpleNamespace(pid = 4242, exitcode = None)
+ orchestrator._subprocess_crash_message("generation")
+ assert len(written) == 1, written
+ orchestrator._subprocess_crash_message("generation")
+ assert len(written) == 1, written
+
+ orchestrator._proc = SimpleNamespace(pid = 4242, exitcode = -6)
+ orchestrator._subprocess_crash_message("generation", with_worker_output = True)
+ assert len(written) == 2, "the replacement's own fatal output was skipped as already logged"
+ orchestrator._subprocess_crash_message("generation", with_worker_output = True)
+ assert len(written) == 2, written
+
+
+def test_a_worker_we_stopped_on_purpose_is_not_replayed_as_a_crash(monkeypatch):
+ import importlib.util as _ilu
+
+ spec = _ilu.spec_from_file_location(
+ "inference_orchestrator_deliberate_stop_under_test",
+ Path(_BACKEND_DIR) / "core/inference/orchestrator.py",
+ )
+ module = _ilu.module_from_spec(spec)
+ spec.loader.exec_module(module)
+
+ written: "list[tuple]" = []
+ monkeypatch.setattr(
+ module.logger,
+ "error",
+ lambda *args, **kwargs: written.append(args),
+ raising = False,
+ )
+
+ orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)
+ orchestrator._stderr_capture = _FixedCapture("loading shards: 100%\n")
+ orchestrator._proc = None
+ orchestrator._worker_stopped_deliberately = True
+ orchestrator._retire_stderr_capture()
+ assert written == [], written
+
+ orchestrator._stderr_capture = _FixedCapture("Fatal Python error: Aborted\n")
+ orchestrator._worker_stopped_deliberately = False
+ orchestrator._retire_stderr_capture()
+ assert written, "an unattended crash was skipped because a previous stop was deliberate"
+
+
+def test_a_worker_that_died_between_requests_is_replayed_before_its_sink_closes(monkeypatch):
+ import importlib.util as _ilu
+
+ spec = _ilu.spec_from_file_location(
+ "inference_orchestrator_idle_retire_under_test",
+ Path(_BACKEND_DIR) / "core/inference/orchestrator.py",
+ )
+ module = _ilu.module_from_spec(spec)
+ spec.loader.exec_module(module)
+
+ written: "list[tuple]" = []
+ monkeypatch.setattr(
+ module.logger,
+ "error",
+ lambda *args, **kwargs: written.append(args),
+ raising = False,
+ )
+
+ closed: "list[bool]" = []
+
+ class _ClosableCapture(_FixedCapture):
+ def close(self):
+ closed.append(True)
+
+ orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)
+ orchestrator._stderr_capture = _ClosableCapture("Fatal Python error: Segmentation fault\n")
+ orchestrator._proc = SimpleNamespace(
+ pid = 7331,
+ exitcode = -11,
+ is_alive = lambda: False,
+ )
+
+ orchestrator._retire_stderr_capture()
+ assert closed == [True]
+ assert written, "the capture of a worker that died unattended was closed unread"
+ assert "Fatal Python error: Segmentation fault" in str(written[0])
+ assert orchestrator._stderr_capture is None
+
+ written.clear()
+ orchestrator._stderr_capture = _ClosableCapture("loading shards: 40%\n")
+ orchestrator._proc = SimpleNamespace(pid = 7332, exitcode = None, is_alive = lambda: True)
+ orchestrator._retire_stderr_capture()
+ assert written == [], written
+
+
+def test_a_request_queued_behind_the_crash_is_not_given_its_last_words(monkeypatch):
+ capture = _FixedCapture(
+ "Traceback (most recent call last):\n"
+ ' File "/home/alice/.unsloth/studio/worker.py", line 42, in run\n'
+ "RuntimeError: another account's prompt overflowed the context\n"
+ )
+ orchestrator = _orchestrator_with(-9, capture)
+ logged: "list[str]" = []
+ monkeypatch.setattr(
+ type(orchestrator),
+ "_log_worker_stderr_once",
+ lambda self, pid, exitcode, **_kwargs: logged.append(str(pid)),
+ raising = False,
+ )
+
+ executing = orchestrator._subprocess_crash_message("generation", with_worker_output = True)
+ assert "Worker error output:" in executing
+ assert "RuntimeError: another account" in executing
+
+ orchestrator._stderr_tail_logged = None
+ queued = orchestrator._subprocess_crash_message("generation", with_worker_output = False)
+ assert "Worker error output:" not in queued, queued
+ assert "another account" not in queued, queued
+ assert "generating a response" in queued
+ # `signal.Signals(9)` raises on Windows, so the message says SIG9 there.
+ assert "signal=SIGKILL" in queued or "signal=SIG9" in queued, queued
+ assert "exitcode=-9" in queued, queued
+ assert logged, "the server log lost the tail for the queued request's path"
+
+
+def test_the_stream_asks_who_owned_the_worker_before_handing_over_the_tail():
+ import inspect
+ from core.inference import orchestrator as orchestrator_module
+
+ body = inspect.getsource(orchestrator_module.InferenceOrchestrator._consume_token_stream)
+ # The other crash exit, the swapped-worker branch, passes no tail at all.
+ assert body.count("with_worker_output = self._owns_worker(cancel_event)") == 1, body
+
+
+def test_a_queued_compare_request_does_not_own_the_worker():
+ module = _load_orchestrator_module()
+ orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)
+ import threading
+
+ orchestrator._active_cancel_lock = threading.Lock()
+ first, second = threading.Event(), threading.Event()
+ orchestrator._active_cancel_events = []
+ orchestrator._executing_cancel_events = []
+ assert orchestrator._owns_worker(first) is True
+
+ orchestrator._active_cancel_events = [first, second]
+ orchestrator._executing_cancel_events = [first]
+ assert orchestrator._owns_worker(first) is True
+ assert orchestrator._owns_worker(second) is False
+
+
+def test_the_public_tail_redacts_the_credentials_a_crash_actually_carries():
+ from core.inference.orchestrator import _redact_worker_output
+
+ text = (
+ "Traceback (most recent call last):\n"
+ ' File "/home/alice/.unsloth/studio/worker.py", line 42, in run\n'
+ " client = OpenAI() # OPENAI_API_KEY=sk-proj-abcdefghijklmnopqrstuvwxyz0123456789\n"
+ " aws = 'AKIAIOSFODNN7EXAMPLE'\n"
+ " gh = 'ghp_abcdefghijklmnopqrstuvwxyz0123456789'\n"
+ " password=hunter2seventeen\n"
+ "RuntimeError: refused\n"
+ )
+ public = _redact_worker_output(text)
+ for secret in (
+ "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789",
+ "AKIAIOSFODNN7EXAMPLE",
+ "ghp_abcdefghijklmnopqrstuvwxyz0123456789",
+ "hunter2seventeen",
+ ):
+ assert secret not in public, (secret, public)
+ assert "RuntimeError: refused" in public, public
+ assert "worker.py" in public, public
+
+
+def test_the_worker_output_is_opt_in_at_every_call_site():
+ import inspect
+ from core.inference import orchestrator as orchestrator_module
+
+ source = inspect.getsource(orchestrator_module)
+ assert "with_worker_output: bool = False" in source
+ calls = source.count("self._subprocess_crash_message(")
+ owned = source.count("with_worker_output = self._owns_worker(")
+ # Every call site but one asks ownership; the exception is the swapped-worker branch.
+ assert calls == owned + 1, (calls, owned)
+
+
+def test_a_stream_whose_worker_was_swapped_gets_no_tail_at_all():
+ import inspect
+ from core.inference import orchestrator as orchestrator_module
+
+ body = inspect.getsource(orchestrator_module.InferenceOrchestrator._consume_token_stream)
+ swap = body.index("initial_proc or self._resp_queue is not initial_resp_queue")
+ following = body[swap : body.index("resp = read_one(read_timeout)", swap)]
+ assert "_subprocess_crash_message(crash_context)" in following, following
+ assert "with_worker_output" not in following, following
+
+
+def _logging_restored():
+ import logging
+
+ from utils import worker_stderr
+
+ # getattr so a build without the hook lets the CASE report the gap, not this fixture.
+ saved = (
+ logging.Handler.setFormatter,
+ logging.Logger.addHandler,
+ getattr(logging.lastResort, "formatter", None),
+ getattr(worker_stderr, "_UNHOOKED_SET_FORMATTER", None),
+ getattr(worker_stderr, "_UNHOOKED_ADD_HANDLER", None),
+ )
+ try:
+ yield
+ finally:
+ logging.Handler.setFormatter = saved[0]
+ logging.Logger.addHandler = saved[1]
+ logging.lastResort.formatter = saved[2]
+ if hasattr(worker_stderr, "_UNHOOKED_SET_FORMATTER"):
+ worker_stderr._UNHOOKED_SET_FORMATTER = saved[3]
+ worker_stderr._UNHOOKED_ADD_HANDLER = saved[4]
+
+
+def test_the_marking_covers_the_last_resort_handler(_logging_restored):
+ import logging
+
+ from utils.worker_stderr import mark_log_record_continuations
+
+ logging.lastResort.setFormatter(logging.Formatter("%(message)s"))
+ mark_log_record_continuations(logging.getLogger("unsloth-test-lastresort"))
+
+ record = logging.LogRecord(
+ "unsloth-test-lastresort", logging.ERROR, __file__, 1, "one\ntwo\nthree", (), None
+ )
+ lines = logging.lastResort.format(record).split("\n")
+ assert lines[0] == START_MARK + "one", lines
+ assert all(line.startswith(MARK) for line in lines[1:]), lines
+
+
+def test_a_handler_installed_after_startup_is_marked_too(_logging_restored):
+ import logging
+
+ from utils.worker_stderr import mark_log_record_continuations
+
+ mark_log_record_continuations(logging.getLogger("unsloth-test-late"))
+
+ late = logging.getLogger("unsloth-test-late-library")
+ late.handlers = []
+ handler = logging.StreamHandler()
+ handler.setFormatter(logging.Formatter("%(message)s"))
+ late.addHandler(handler)
+
+ record = logging.LogRecord(
+ "unsloth-test-late-library", logging.ERROR, __file__, 1, "one\ntwo", (), None
+ )
+ assert handler.format(record).split("\n")[1].startswith(MARK), handler.format(record)
+
+ handler.setFormatter(logging.Formatter("%(levelname)s %(message)s"))
+ formatted = handler.format(record)
+ assert formatted.split("\n")[0] == START_MARK + "ERROR one", formatted
+ assert formatted.split("\n")[1].startswith(MARK), formatted
+
+ # Clearing the formatter is the one case the hook cannot wrap; it must not crash.
+ handler.setFormatter(None)
+ assert handler.formatter is None
+
+
+def test_a_root_level_path_is_redacted_too():
+ from core.inference.orchestrator import _redact_worker_output
+ for path, tail in (
+ ("/model.gguf", "model.gguf"),
+ ("C:\\model.gguf", "model.gguf"),
+ ("\\\\fileserver\\share", "share"),
+ ("/opt", "opt"),
+ ):
+ public = _redact_worker_output(f"could not open {path}\n")
+ assert public.strip() == f"could not open .../{tail}", public
+ assert not public.count("C:"), public
+
+ for text in (
+ 'the "/" separator is not a path',
+ "a ratio of 3/4 at https://host/path/x",
+ "use / to split and/or join",
+ ):
+ assert _redact_worker_output(text + "\n").strip() == text, text
+
+
+def test_two_paths_on_one_line_are_two_matches():
+ from core.inference.orchestrator import _redact_worker_output
+
+ windows = _redact_worker_output("copy C:\\Users\\ann\\old.gguf to C:\\tmp\\new.gguf\n")
+ assert windows.strip() == "copy .../old.gguf to .../new.gguf", windows
+ posix = _redact_worker_output("cannot copy /home/ann/old.gguf to /tmp/new.gguf\n")
+ assert posix.strip() == "cannot copy .../old.gguf to .../new.gguf", posix
+ unc = _redact_worker_output("copy \\\\share\\team\\old.gguf to \\\\other\\team\\new.gguf\n")
+ assert unc.strip() == "copy .../old.gguf to .../new.gguf", unc
+
+ spaced = _redact_worker_output("C:\\Program Files\\unsloth\\weights.gguf failed\n")
+ assert spaced.strip() == ".../weights.gguf failed", spaced
diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py
index be511315723..33ffd7c6f3a 100644
--- a/studio/backend/tests/test_orchestrator_unload_cancel.py
+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py
@@ -231,7 +231,7 @@ def test_consume_token_stream_bails_when_subprocess_swapped(monkeypatch):
o = _bare_orchestrator()
monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True)
monkeypatch.setattr(
- o, "_subprocess_crash_message", lambda ctx: "inference subprocess restarted"
+ o, "_subprocess_crash_message", lambda ctx, **kwargs: "inference subprocess restarted"
)
def read_one(timeout):
diff --git a/studio/backend/utils/native_path_leases.py b/studio/backend/utils/native_path_leases.py
index 15ed307ca7a..0b8f6e645c0 100644
--- a/studio/backend/utils/native_path_leases.py
+++ b/studio/backend/utils/native_path_leases.py
@@ -28,6 +28,8 @@ from pathlib import Path
from typing import Any, Callable, Collection, Iterable, Iterator, Mapping
LEASE_SECRET_ENV = "UNSLOTH_STUDIO_NATIVE_PATH_LEASE_SECRET"
+# Spelled out, not imported from utils/worker_stderr.py, to keep this module's import graph stdlib only.
+STDERR_MIRROR_KWARG = "unsloth_stderr_mirror_path"
_MAX_NATIVE_PATH_REDACTIONS = 100
_MAX_NATIVE_PATH_LABELS = 10_000
_MIN_LEASE_SECRET_BYTES = 32
@@ -119,6 +121,15 @@ def run_without_native_path_secret(
) -> Any:
"""Run a multiprocessing child target without the native path lease secret."""
+ # First, before anything else can raise, so a child dying before its entrypoint is explainable (#7843).
+ _stderr_mirror_path = kwargs.pop(STDERR_MIRROR_KWARG, None)
+ if _stderr_mirror_path:
+ try:
+ from utils.worker_stderr import install_worker_stderr_mirror
+ install_worker_stderr_mirror(_stderr_mirror_path)
+ except Exception:
+ pass
+
# Runs in the spawned child to bind it to the parent's death, since multiprocessing children get no
# preexec_fn. Shared entrypoint for the inference/export/training/data-recipe workers. Two try
# blocks, because allow_child_processes is the newer name: on an older process_lifetime.py a
diff --git a/studio/backend/utils/worker_stderr.py b/studio/backend/utils/worker_stderr.py
new file mode 100644
index 00000000000..f4a02295664
--- /dev/null
+++ b/studio/backend/utils/worker_stderr.py
@@ -0,0 +1,559 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""#7843: a spawn child's traceback goes down the inherited stderr, not the response queue. Best effort throughout."""
+
+from __future__ import annotations
+
+import atexit
+import collections
+import logging
+import os
+import sys
+import tempfile
+import threading
+
+__all__ = [
+ "LOG_RECORD_CONTINUATION_PREFIX",
+ "LOG_RECORD_START_MARK",
+ "STDERR_MIRROR_KWARG",
+ "WorkerStderrCapture",
+ "decode_worker_stderr",
+ "install_worker_stderr_mirror",
+ "mark_log_record_continuations",
+ "stderr_tail_from_bytes",
+]
+
+# Marks a record's continuation lines: a recovered request's `exc_info` traceback is byte-identical to a dying process's.
+LOG_RECORD_CONTINUATION_PREFIX = " | "
+
+# And its first line. UNIT SEPARATOR so it stays invisible in the operator's mirrored copy.
+LOG_RECORD_START_MARK = "\x1f"
+
+# A kwarg, not an environment variable: a process-wide value would cross workers spawning at once.
+STDERR_MIRROR_KWARG = "unsloth_stderr_mirror_path"
+
+DEFAULT_TAIL_LINES = 20
+DEFAULT_TAIL_CHARS = 4000
+
+MIRROR_FILE_CAP_BYTES = 256 * 1024
+
+TAIL_READ_BYTES = 64 * 1024
+
+_PUMP_JOIN_TIMEOUT_S = 2.0
+
+
+def _utf8_edge_bounds(data: bytes, *, trim_end: bool = True) -> "tuple[int, int]":
+ start = 0
+ while start < len(data) and start < 3 and 0x80 <= data[start] < 0xC0:
+ start += 1
+ end = len(data)
+ if not trim_end:
+ return start, end
+ for back in range(1, min(4, end - start) + 1):
+ byte = data[end - back]
+ if byte < 0x80:
+ break
+ if byte >= 0xC0:
+ width = 2 if byte < 0xE0 else 3 if byte < 0xF0 else 4
+ if back < width:
+ end -= back
+ break
+ return start, end
+
+
+def _without_partial_utf8_edges(data: bytes, *, trim_end: bool = True) -> bytes:
+ """``trim_end`` is off at EOF, where a trailing 0xC0+ byte is a complete cp1252 character, not a severed one."""
+ start, end = _utf8_edge_bounds(data, trim_end = trim_end)
+ return data[start:end]
+
+
+def decode_worker_stderr(data: bytes, *, ends_at_eof: bool = True) -> str:
+ """UTF-8 before cp1252: cp1252 has a meaning for almost every byte, so the stricter encoding goes first."""
+ text: str | None = None
+ try:
+ text = data.decode("utf-8")
+ except UnicodeDecodeError:
+ text = None
+ if text is None:
+ start, end = _utf8_edge_bounds(data)
+ try:
+ text = data[start:end].decode("utf-8")
+ except UnicodeDecodeError:
+ text = None
+ else:
+ # cp1252 costs at worst one spurious character; discarding loses crash detail.
+ tail_bytes = data[end:] if ends_at_eof else b""
+ if tail_bytes:
+ try:
+ text += tail_bytes.decode("cp1252")
+ except (UnicodeDecodeError, LookupError):
+ pass
+ if text is None:
+ try:
+ text = data.decode("cp1252")
+ except (UnicodeDecodeError, LookupError):
+ text = data.decode("utf-8", errors = "replace")
+ return text.replace("\r\n", "\n").replace("\r", "\n")
+
+
+def stderr_tail_from_bytes(
+ data: bytes,
+ max_lines: int = DEFAULT_TAIL_LINES,
+ max_chars: int = DEFAULT_TAIL_CHARS,
+ *,
+ ends_at_eof: bool = True,
+) -> str:
+ text = decode_worker_stderr(data, ends_at_eof = ends_at_eof)
+ lines = [line.rstrip() for line in text.split("\n")]
+ lines = [line for line in lines if line.strip()]
+ if not lines:
+ return ""
+ if max_lines > 0:
+ lines = lines[-max_lines:]
+ joined = "\n".join(lines)
+ if max_chars > 0 and len(joined) > max_chars:
+ joined = joined[-max_chars:]
+ first_break = joined.find("\n")
+ if first_break != -1:
+ joined = joined[first_break + 1 :]
+ return joined
+
+
+# Exact paths, never a pattern: Studios share one temporary directory.
+_OPEN_SINKS: "set[str]" = set()
+_ATEXIT_REGISTERED = False
+# A fork inherits the set and the atexit registration, so the handler must check these paths are its own.
+_SINKS_OWNER_PID: "int | None" = None
+
+
+def _unlink_quietly(path: str) -> None:
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+
+def _remove_open_sinks() -> None:
+ if os.getpid() != _SINKS_OWNER_PID:
+ # An inherited handler in a forked child: these paths are the parent's.
+ return
+ for path in list(_OPEN_SINKS):
+ _unlink_quietly(path)
+ _OPEN_SINKS.clear()
+
+
+class WorkerStderrCapture:
+ def __init__(
+ self,
+ directory: "str | None" = None,
+ prefix: str = "unsloth-worker-",
+ ) -> None:
+ global _ATEXIT_REGISTERED, _SINKS_OWNER_PID
+ handle, self._path = tempfile.mkstemp(
+ prefix = prefix,
+ suffix = ".stderr",
+ dir = directory,
+ )
+ os.close(handle)
+ _OPEN_SINKS.add(self._path)
+ _SINKS_OWNER_PID = os.getpid()
+ if not _ATEXIT_REGISTERED:
+ atexit.register(_remove_open_sinks)
+ _ATEXIT_REGISTERED = True
+
+ @property
+ def path(self) -> str:
+ return self._path
+
+ def tail(
+ self,
+ max_lines: int = DEFAULT_TAIL_LINES,
+ max_chars: int = DEFAULT_TAIL_CHARS,
+ ) -> str:
+ try:
+ # O_NOFOLLOW: a tail is never worth following a symlink in a shared tmpdir for.
+ fd = os.open(self._path, os.O_RDONLY | _O_NOFOLLOW | _O_BINARY)
+ with os.fdopen(fd, "rb") as handle:
+ handle.seek(0, os.SEEK_END)
+ size = handle.tell()
+ handle.seek(max(0, size - TAIL_READ_BYTES))
+ data = handle.read()
+ except OSError:
+ return ""
+ return stderr_tail_from_bytes(data, max_lines = max_lines, max_chars = max_chars)
+
+ def close(self) -> None:
+ """Tolerates a child still holding the sink open, which is the norm on Windows."""
+ _OPEN_SINKS.discard(self._path)
+ _unlink_quietly(self._path)
+
+
+# O_NOFOLLOW is POSIX only and O_BINARY is Windows only; both are absent-means-zero here.
+_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
+_O_BINARY = getattr(os, "O_BINARY", 0)
+
+
+def _open_existing_sink(path: str):
+ """``open(path, "wb")`` is WRONG here: world-readable under the default umask, and follows a planted symlink."""
+ flags = os.O_RDWR | _O_NOFOLLOW | _O_BINARY
+ try:
+ handle = os.open(path, flags)
+ except OSError:
+ return None
+ try:
+ return os.fdopen(handle, "r+b", buffering = 0)
+ except OSError:
+ try:
+ os.close(handle)
+ except OSError:
+ pass
+ return None
+
+
+_TAIL_POLL_S = 0.05
+
+# 64 x 64 KiB: a stalled operator stderr costs 4 MiB of buffer before the mirror starts dropping.
+_MIRROR_RELAY_CHUNKS = 64
+
+
+def _open_sink_for_append(path: str):
+ flags = os.O_WRONLY | os.O_APPEND | _O_NOFOLLOW | _O_BINARY
+ try:
+ return os.open(path, flags)
+ except OSError:
+ return None
+
+
+def _open_sink_for_reading(path: str):
+ flags = os.O_RDONLY | _O_NOFOLLOW | _O_BINARY
+ try:
+ handle = os.open(path, flags)
+ except OSError:
+ return None
+ try:
+ return os.fdopen(handle, "rb", buffering = 0)
+ except OSError:
+ try:
+ os.close(handle)
+ except OSError:
+ pass
+ return None
+
+
+_COMPACT_CATCH_UP_ROUNDS = 8
+
+
+def _compact_sink(
+ sink,
+ cap_bytes: int,
+ reader = None,
+ emit = None,
+) -> int:
+ """Rewrite *sink* to roughly its last *cap_bytes* bytes.
+
+ fd 2 is ``O_APPEND`` on it, so a plain read-rewrite-truncate loses concurrent appends.
+ """
+ if reader is not None and emit is not None:
+ while True:
+ try:
+ pending = reader.read(65536)
+ except (OSError, ValueError):
+ break
+ if not pending:
+ break
+ emit(pending)
+ size = sink.seek(0, os.SEEK_END)
+ keep = min(size, cap_bytes)
+ sink.seek(size - keep)
+ data = sink.read(keep)
+ # Everything past `total` is an append that arrived after this function started looking.
+ total = size
+ for _ in range(_COMPACT_CATCH_UP_ROUNDS):
+ appended = sink.read()
+ if not appended:
+ break
+ if emit is not None:
+ emit(appended)
+ data += appended
+ total += len(appended)
+ sink.seek(0)
+ sink.write(data)
+ end = len(data)
+ # A native thread can write into the region about to be truncated during the rewrite.
+ for _ in range(_COMPACT_CATCH_UP_ROUNDS):
+ sink.seek(total)
+ late = sink.read()
+ if not late:
+ break
+ if emit is not None:
+ emit(late)
+ total += len(late)
+ sink.seek(end)
+ sink.write(late)
+ end += len(late)
+ sink.truncate(end)
+ return end
+
+
+class _MirrorRelay:
+ """Owns the only blocking write, so the thread that bounds the sink never parks in one.
+
+ The operator's stderr is a pipe in the packaged app, and a reader that stops draining it
+ would otherwise stall the pump inside ``os.write`` and leave the sink growing without limit.
+ The mirrored copy is best effort, so a full buffer drops its oldest chunk; the sink keeps
+ the tail, which is the crash record the parent reports.
+ """
+
+ def __init__(self, inherited_fd: int, max_chunks: int) -> None:
+ self._fd = inherited_fd
+ self._chunks = collections.deque(maxlen = max_chunks)
+ self._wake = threading.Event()
+ self._closed = False
+ self._thread = threading.Thread(
+ target = self._run,
+ name = "unsloth-worker-stderr-relay",
+ daemon = True,
+ )
+ self._thread.start()
+
+ def emit(self, data: bytes) -> None:
+ if not data:
+ return
+ self._chunks.append(data)
+ self._wake.set()
+
+ def close(self) -> None:
+ self._closed = True
+ self._wake.set()
+
+ def join(self, timeout: "float | None" = None) -> None:
+ self._thread.join(timeout = timeout)
+
+ def is_alive(self) -> bool:
+ return self._thread.is_alive()
+
+ def _run(self) -> None:
+ while True:
+ try:
+ data = self._chunks.popleft()
+ except IndexError:
+ if self._closed:
+ return
+ self._wake.wait(_TAIL_POLL_S)
+ self._wake.clear()
+ continue
+ try:
+ os.write(self._fd, data)
+ except OSError:
+ pass
+
+
+def _sink_size(sink) -> int:
+ try:
+ return os.fstat(sink.fileno()).st_size
+ except (OSError, ValueError):
+ return 0
+
+
+def _tail_sink_to_stderr(
+ reader, relay: "_MirrorRelay", sink, cap_bytes: int, stop: "threading.Event"
+) -> None:
+ """fd 2 must be the SINK, not a pipe: a fatal signal kills anything buffered in a pipe with its writer."""
+ while True:
+ try:
+ chunk = reader.read(65536)
+ except (OSError, ValueError):
+ break
+ if not chunk:
+ if stop.is_set():
+ break
+ stop.wait(_TAIL_POLL_S)
+ continue
+ relay.emit(chunk)
+ # Bytes ON DISK, not bytes relayed: the relay can fall arbitrarily far behind.
+ if cap_bytes > 0 and _sink_size(sink) > 2 * cap_bytes:
+ try:
+ _compact_sink(sink, cap_bytes, reader = reader, emit = relay.emit)
+ # The file was rewritten from the front; the old offset is now meaningless.
+ reader.seek(0, os.SEEK_END)
+ except (OSError, ValueError):
+ pass
+ try:
+ sink.close()
+ except OSError:
+ pass
+ try:
+ reader.close()
+ except (OSError, ValueError):
+ pass
+
+
+def _stop_mirror(
+ inherited_fd: int,
+ pump: threading.Thread,
+ stop = None,
+ relay = None,
+) -> None:
+ """The pump must be a daemon or ``BaseProcess._bootstrap`` waits on it for ever.
+
+ ``multiprocessing`` prints the traceback before atexit, so draining here still catches it.
+ """
+ try:
+ sys.stderr.flush()
+ except Exception:
+ pass
+ try:
+ os.dup2(inherited_fd, 2)
+ except OSError:
+ pass
+ if stop is not None:
+ # fd 2 no longer points at the sink, so the next empty read is the end, not a pause.
+ stop.set()
+ pump.join(timeout = _PUMP_JOIN_TIMEOUT_S)
+ if relay is not None:
+ relay.close()
+ relay.join(timeout = _PUMP_JOIN_TIMEOUT_S)
+ if pump.is_alive() or (relay is not None and relay.is_alive()):
+ # A thread may be inside os.write(inherited_fd, ...); closing frees the number for any thread's next open().
+ return
+ try:
+ os.close(inherited_fd)
+ except OSError:
+ pass
+
+
+def install_worker_stderr_mirror(
+ path: "str | None", cap_bytes: int = MIRROR_FILE_CAP_BYTES
+) -> bool:
+ """At the descriptor level: a worker dying inside a C extension writes fd 2 directly and bypasses a Python wrapper."""
+ if not path:
+ return False
+ sink = _open_existing_sink(path)
+ if sink is None:
+ return False
+ try:
+ inherited = os.dup(2)
+ except OSError:
+ sink.close()
+ return False
+ # O_APPEND: compaction rewrites from the front and two handles share the file.
+ writer_fd = _open_sink_for_append(path)
+ if writer_fd is None:
+ os.close(inherited)
+ sink.close()
+ return False
+ reader = _open_sink_for_reading(path)
+ if reader is None:
+ os.close(writer_fd)
+ os.close(inherited)
+ sink.close()
+ return False
+ try:
+ sys.stderr.flush()
+ except Exception:
+ pass
+ try:
+ reader.seek(0, os.SEEK_END)
+ except (OSError, ValueError):
+ pass
+ try:
+ os.dup2(writer_fd, 2)
+ except OSError:
+ os.close(writer_fd)
+ reader.close()
+ os.close(inherited)
+ sink.close()
+ return False
+ os.close(writer_fd)
+ stop = threading.Event()
+ relay = _MirrorRelay(inherited, _MIRROR_RELAY_CHUNKS)
+ pump = threading.Thread(
+ target = _tail_sink_to_stderr,
+ args = (reader, relay, sink, cap_bytes, stop),
+ name = "unsloth-worker-stderr-mirror",
+ daemon = True,
+ )
+ pump.start()
+ atexit.register(_stop_mirror, inherited, pump, stop, relay)
+ return True
+
+
+class _EveryLineCarriesThePrefix(logging.Formatter):
+ def __init__(self, inner: "logging.Formatter") -> None:
+ super().__init__()
+ self._inner = inner
+
+ def format(self, record: "logging.LogRecord") -> str:
+ text = self._inner.format(record)
+ first, newline, rest = text.partition("\n")
+ # The first line too: a default-formatted single-line record has no shape to spot.
+ marked_first = (
+ first if first.startswith(LOG_RECORD_START_MARK) else LOG_RECORD_START_MARK + first
+ )
+ if not newline:
+ return marked_first
+ return (
+ marked_first
+ + "\n"
+ + "\n".join(LOG_RECORD_CONTINUATION_PREFIX + line for line in rest.split("\n"))
+ )
+
+ def __getattr__(self, name: str):
+ return getattr(self._inner, name)
+
+
+_UNHOOKED_SET_FORMATTER = None
+_UNHOOKED_ADD_HANDLER = None
+
+
+def _mark_handler(handler) -> bool:
+ formatter = getattr(handler, "formatter", None)
+ if isinstance(formatter, _EveryLineCarriesThePrefix):
+ return False
+ setter = _UNHOOKED_SET_FORMATTER or type(handler).setFormatter
+ setter(handler, _EveryLineCarriesThePrefix(formatter or logging.Formatter()))
+ return True
+
+
+def _install_continuation_hook() -> bool:
+ """Patched on the CLASS: the ML stack is imported after logging is configured and adds its own handlers."""
+ global _UNHOOKED_SET_FORMATTER, _UNHOOKED_ADD_HANDLER
+ if _UNHOOKED_SET_FORMATTER is not None:
+ return False
+ unhooked_set = logging.Handler.setFormatter
+ unhooked_add = logging.Logger.addHandler
+
+ def setFormatter(self, fmt): # noqa: N802 -- matches logging's own spelling
+ if fmt is not None and not isinstance(fmt, _EveryLineCarriesThePrefix):
+ fmt = _EveryLineCarriesThePrefix(fmt)
+ unhooked_set(self, fmt)
+
+ def addHandler(self, hdlr): # noqa: N802 -- matches logging's own spelling
+ unhooked_add(self, hdlr)
+ try:
+ _mark_handler(hdlr)
+ except Exception: # noqa: BLE001 -- marking must never break someone's logging
+ pass
+
+ _UNHOOKED_SET_FORMATTER = unhooked_set
+ _UNHOOKED_ADD_HANDLER = unhooked_add
+ logging.Handler.setFormatter = setFormatter
+ logging.Logger.addHandler = addHandler
+ return True
+
+
+def mark_log_record_continuations(logger_object = None, *, cover_later_handlers = True) -> int:
+ """Covers ``logging.lastResort`` and later handlers too: a miss hands a RECOVERED
+ request's traceback to the next caller as their crash."""
+ root = logger_object if logger_object is not None else logging.getLogger()
+ wrapped = 0
+ for handler in list(getattr(root, "handlers", ())):
+ if _mark_handler(handler):
+ wrapped += 1
+ if cover_later_handlers:
+ last_resort = getattr(logging, "lastResort", None)
+ if last_resort is not None and _mark_handler(last_resort):
+ wrapped += 1
+ _install_continuation_hook()
+ return wrapped
diff --git a/studio/frontend/src/features/auth/api.ts b/studio/frontend/src/features/auth/api.ts
index c654d7941af..a0899389302 100644
--- a/studio/frontend/src/features/auth/api.ts
+++ b/studio/frontend/src/features/auth/api.ts
@@ -2,7 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { accountTransitionPending } from "@/lib/account-transition";
-import { apiUrl, isTauri } from "@/lib/api-base";
+import { apiUrl, getApiPort, isTauri } from "@/lib/api-base";
import {
clearAuthTokens,
getAuthToken,
@@ -29,10 +29,30 @@ let refreshInflight: Promise<boolean> | null = null;
let refreshInflightToken: string | null = null;
let logoutGeneration = 0;
-const TAURI_FETCH_RETRY_DELAYS_MS = [250, 750, 1500] as const;
+// #10520: sums to 10.5s, just past the launcher's 10s HEALTH_PROBE_TIMEOUT. Guarded by
+// `the_frontend_retry_ladder_outlives_one_probe_budget` in src-tauri/src/commands.rs.
+const TAURI_FETCH_RETRY_DELAYS_MS = [250, 750, 1500, 3000, 5000] as const;
+// A network error is not an answer: retrying a committed POST creates a second API key,
+// project or job, so non-idempotent methods keep the shorter ladder.
+const TAURI_FETCH_RETRY_DELAYS_UNSAFE_MS = [250, 750, 1500] as const;
+const IDEMPOTENT_METHODS = new Set(["GET", "HEAD", "OPTIONS", "PUT", "DELETE"]);
+
+function retryDelaysFor(
+ input: RequestInfo | URL,
+ init?: RequestInit,
+): readonly number[] {
+ const method = (
+ init?.method ??
+ (typeof Request !== "undefined" && input instanceof Request
+ ? input.method
+ : "GET")
+ ).toUpperCase();
+ return IDEMPOTENT_METHODS.has(method)
+ ? TAURI_FETCH_RETRY_DELAYS_MS
+ : TAURI_FETCH_RETRY_DELAYS_UNSAFE_MS;
+}
const BROWSER_TIMEZONE_HEADER = "X-Unsloth-Timezone";
-const BROWSER_TIMEZONE_OFFSET_HEADER =
- "X-Unsloth-Timezone-Offset-Minutes";
+const BROWSER_TIMEZONE_OFFSET_HEADER = "X-Unsloth-Timezone-Offset-Minutes";
function addBrowserTimezoneHeaders(headers: Headers): void {
try {
@@ -61,6 +81,7 @@ async function fetchWithTauriNetworkRetry(
retryNetworkErrors = true,
beforeRetry?: () => void,
): Promise<Response> {
+ const delays = retryDelaysFor(input, init);
for (let attempt = 0; ; attempt++) {
try {
return await fetch(input, init);
@@ -69,11 +90,11 @@ async function fetchWithTauriNetworkRetry(
!isTauri ||
!retryNetworkErrors ||
!(error instanceof TypeError) ||
- attempt >= TAURI_FETCH_RETRY_DELAYS_MS.length
+ attempt >= delays.length
) {
throw error;
}
- await wait(TAURI_FETCH_RETRY_DELAYS_MS[attempt]);
+ await wait(delays[attempt]);
beforeRetry?.();
}
}
@@ -105,9 +126,10 @@ async function redirectToAuth(passwordChangeRequired = false): Promise<void> {
login_mode?: "single" | "multi";
};
// Public status describes the owner. A managed session carries its own requirement.
- const requiresChange = data.login_mode === "multi"
- ? passwordChangeRequired || mustChangePassword()
- : data.requires_password_change;
+ const requiresChange =
+ data.login_mode === "multi"
+ ? passwordChangeRequired || mustChangePassword()
+ : data.requires_password_change;
if (requiresChange !== mustChangePassword()) {
setMustChangePassword(requiresChange);
}
@@ -124,9 +146,54 @@ async function redirectToAuth(passwordChangeRequired = false): Promise<void> {
window.location.href = target;
}
-function asTransportFailure(err: unknown): unknown {
+/** Copy shown when the backend really is unreachable and the launcher agrees. */
+export const BACKEND_NOT_RUNNING_MESSAGE =
+ "Unsloth isn't running -- please relaunch it.";
+/** Copy shown when the webview could not reach the backend but the launcher says it is up. */
+export const BACKEND_NOT_ANSWERING_MESSAGE =
+ "Unsloth is running but did not answer in time. It may still be starting up. Please try again in a moment.";
+
+/** `check_backend_present` and NOT `check_health`: the latter reports a probe that ran out of budget exactly as a refused connection. */
+// The port is carried WITH the promise: `setApiBase` can move it inside the 10s budget.
+let nativeHealthInflight: { port: number; probe: Promise<boolean> } | null = null;
+
+async function nativeBackendIsAlive(): Promise<boolean> {
+ if (!isTauri) {
+ return false;
+ }
+ const port = getApiPort();
+ if (port === null) {
+ return false;
+ }
+ // Single flight: one 10s probe rather than one per panel. Not cached beyond the call.
+ if (nativeHealthInflight !== null && nativeHealthInflight.port === port) {
+ return nativeHealthInflight.probe;
+ }
+ const probe = (async () => {
+ try {
+ const { invoke } = await import("@tauri-apps/api/core");
+ return (
+ (await invoke<boolean>("check_backend_present", { port })) === true
+ );
+ } catch {
+ return false;
+ }
+ })();
+ const inflight = { port, probe };
+ nativeHealthInflight = inflight;
+ try {
+ return await probe;
+ } finally {
+ // Identity, not the port: a probe for a newer port owns the slot now.
+ if (nativeHealthInflight === inflight) {
+ nativeHealthInflight = null;
+ }
+ }
+}
+
+async function asTransportFailure(err: unknown): Promise<unknown> {
// fetch TypeError = offline | backend down | CORS/DNS. Tagged so callers tell "never reached"
- // from "rejected"; Tauri is always backend-down, the web build distinguishes offline.
+ // from "rejected"; under Tauri the launcher is asked before claiming the backend is gone.
if (!(err instanceof TypeError)) return err;
if (
!isTauri &&
@@ -140,10 +207,17 @@ function asTransportFailure(err: unknown): unknown {
{ unslothTransportFailure: true },
);
}
- return Object.assign(
- new Error("Unsloth isn't running -- please relaunch it."),
- { unslothTransportFailure: true },
- );
+ // A failed fetch in the webview is not proof the backend died, and "please relaunch it"
+ // throws away a running backend and whatever it has in flight.
+ if (await nativeBackendIsAlive()) {
+ return Object.assign(new Error(BACKEND_NOT_ANSWERING_MESSAGE), {
+ unslothTransportFailure: true,
+ unslothBackendStillRunning: true,
+ });
+ }
+ return Object.assign(new Error(BACKEND_NOT_RUNNING_MESSAGE), {
+ unslothTransportFailure: true,
+ });
}
async function retryWithCurrentToken(
@@ -166,7 +240,7 @@ async function retryWithCurrentToken(
beforeRetry,
);
} catch (err) {
- throw asTransportFailure(err);
+ throw await asTransportFailure(err);
}
}
@@ -257,7 +331,7 @@ export async function authFetch(
options?.beforeRetry,
);
} catch (err) {
- throw asTransportFailure(err);
+ throw await asTransportFailure(err);
}
if (await isPasswordChangeRequiredResponse(response)) {
diff --git a/studio/frontend/src/features/chat/utils/chat-history-storage.ts b/studio/frontend/src/features/chat/utils/chat-history-storage.ts
index 077f1c68d60..ac71909b00b 100644
--- a/studio/frontend/src/features/chat/utils/chat-history-storage.ts
+++ b/studio/frontend/src/features/chat/utils/chat-history-storage.ts
@@ -250,12 +250,20 @@ function sortMessages(messages: MessageRecord[]): MessageRecord[] {
}
export function isExpectedBackgroundChatStorageError(error: unknown): boolean {
+ // The transport marker rather than the copy, which is how a merely busy backend came to
+ // be reported as an unexpected error when the copy changed.
+ if (
+ error instanceof Error &&
+ (error as { unslothTransportFailure?: boolean }).unslothTransportFailure ===
+ true
+ ) {
+ return true;
+ }
return (
error instanceof Error &&
(error.message === "Invalid or expired token" ||
error.message === "Not authenticated" ||
- error.message === "Request failed (401)" ||
- error.message === "Unsloth isn't running -- please relaunch it.")
+ error.message === "Request failed (401)")
);
}
diff --git a/studio/frontend/src/lib/api-base.ts b/studio/frontend/src/lib/api-base.ts
index 33efafdf543..0483b8ee9cd 100644
--- a/studio/frontend/src/lib/api-base.ts
+++ b/studio/frontend/src/lib/api-base.ts
@@ -20,6 +20,8 @@ if (isTauri) {
const initialApiBase = apiBase
+const LOOPBACK_BASE_PORT = /^https?:\/\/127\.0\.0\.1:(\d+)$/
+
export function resetApiBase() {
apiBase = initialApiBase
}
@@ -32,6 +34,19 @@ export function getApiBase(): string {
return apiBase
}
+/**
+ * The port the backend is currently expected on, or null when none is known yet. The
+ * placeholder base above is port 0, which reads as "no port yet".
+ */
+export function getApiPort(): number | null {
+ const match = LOOPBACK_BASE_PORT.exec(apiBase)
+ if (!match) {
+ return null
+ }
+ const port = Number(match[1])
+ return Number.isInteger(port) && port > 0 && port <= 65535 ? port : null
+}
+
export function apiUrl(path: string): string {
if (path.startsWith('http')) return path
return `${apiBase}${path}`
diff --git a/studio/frontend/tests/tauri-backend-transport-failure.test.ts b/studio/frontend/tests/tauri-backend-transport-failure.test.ts
new file mode 100644
index 00000000000..be5cca65b99
--- /dev/null
+++ b/studio/frontend/tests/tauri-backend-transport-failure.test.ts
@@ -0,0 +1,443 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// #10520: a firewall filter delayed the loopback handshake and Desktop said "Unsloth isn't running" about a healthy backend.
+
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import { createServer, type Server } from "node:http";
+import test from "node:test";
+
+import { loadWithStubs } from "./helpers/module-stubs.ts";
+
+type AuthApi = {
+ authFetch: (
+ input: string,
+ init?: RequestInit,
+ options?: { retryNetworkErrors?: boolean },
+ ) => Promise<Response>;
+ BACKEND_NOT_RUNNING_MESSAGE: string;
+ BACKEND_NOT_ANSWERING_MESSAGE: string;
+};
+
+function loadAuthApi(options: {
+ port: number | null;
+ getPort?: () => number | null;
+ checkHealth?: (port: number) => boolean | Promise<boolean>;
+ onInvoke?: (command: string, args: Record<string, unknown>) => void;
+}): AuthApi {
+ const currentPort = options.getPort ?? (() => options.port);
+ return loadWithStubs<AuthApi>(
+ new URL("../src/features/auth/api.ts", import.meta.url),
+ {
+ "@/lib/api-base": {
+ apiUrl: (path: string) => {
+ const port = currentPort();
+ return port === null
+ ? `http://127.0.0.1:0${path}`
+ : `http://127.0.0.1:${port}${path}`;
+ },
+ getApiPort: () => currentPort(),
+ isTauri: true,
+ },
+ "@/lib/account-transition": { accountTransitionPending: () => false },
+ "./session": {
+ clearAuthTokens: () => {},
+ getAuthToken: () => "access-token",
+ getRefreshToken: () => null,
+ mustChangePassword: () => false,
+ setMustChangePassword: () => {},
+ storeAuthTokens: () => {},
+ },
+ "@tauri-apps/api/core": {
+ invoke: async (command: string, args: Record<string, unknown>) => {
+ options.onInvoke?.(command, args);
+ if (command !== "check_backend_present") {
+ throw new Error(`unexpected command ${command}`);
+ }
+ return options.checkHealth?.(args.port as number) ?? false;
+ },
+ },
+ },
+ );
+}
+
+async function reserveLoopbackPort(): Promise<number> {
+ const probe = createServer();
+ const port = await new Promise<number>((resolve, reject) => {
+ probe.once("error", reject);
+ probe.listen(0, "127.0.0.1", () => {
+ const address = probe.address();
+ if (typeof address === "string" || address === null) {
+ reject(new Error("no loopback address"));
+ return;
+ }
+ resolve(address.port);
+ });
+ });
+ await new Promise<void>((resolve) => probe.close(() => resolve()));
+ return port;
+}
+
+function listenAfter(server: Server, port: number, delayMs: number): void {
+ setTimeout(() => {
+ server.listen(port, "127.0.0.1");
+ }, delayMs).unref();
+}
+
+// The host below starts accepting at 3s: inside the new ladder, outside the old 250 + 750 + 1500ms one.
+const SLOW_LOOPBACK_ACCEPT_DELAY_MS = 3_000;
+const OLD_LADDER_ATTEMPTS = 4;
+
+test("a backend that accepts late is reached instead of declared not running", async () => {
+ const port = await reserveLoopbackPort();
+ const server = createServer((_request, response) => {
+ response.writeHead(200, { "Content-Type": "application/json" });
+ response.end(JSON.stringify({ ok: true }));
+ });
+ const originalFetch = globalThis.fetch;
+ let attempts = 0;
+ globalThis.fetch = async (input, init) => {
+ attempts += 1;
+ return await originalFetch(input, init);
+ };
+
+ try {
+ listenAfter(server, port, SLOW_LOOPBACK_ACCEPT_DELAY_MS);
+ const authApi = loadAuthApi({ port });
+ const response = await authApi.authFetch("/api/models");
+
+ assert.equal(response.status, 200);
+ assert.ok(
+ attempts > OLD_LADDER_ATTEMPTS,
+ `the backend was only reached on attempt ${attempts}; a ladder of ` +
+ `${OLD_LADDER_ATTEMPTS} attempts would have declared it not running`,
+ );
+ } finally {
+ globalThis.fetch = originalFetch;
+ await new Promise<void>((resolve) => server.close(() => resolve()));
+ }
+});
+
+test("a backend the launcher still sees is not reported as not running", async () => {
+ const port = 61799;
+ const probed: Array<Record<string, unknown>> = [];
+ const originalFetch = globalThis.fetch;
+ globalThis.fetch = async () => {
+ throw new TypeError("fetch failed");
+ };
+
+ try {
+ const authApi = loadAuthApi({
+ port,
+ checkHealth: () => true,
+ onInvoke: (_command, args) => probed.push(args),
+ });
+ const error = await authApi
+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })
+ .then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_ANSWERING_MESSAGE);
+ assert.notEqual(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);
+ assert.ok(!/relaunch/i.test(error.message));
+ assert.equal(
+ (error as { unslothTransportFailure?: boolean }).unslothTransportFailure,
+ true,
+ );
+ assert.equal(
+ (error as { unslothBackendStillRunning?: boolean })
+ .unslothBackendStillRunning,
+ true,
+ );
+ assert.deepEqual(probed, [{ port }]);
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("a backend the launcher cannot see either still says to relaunch", async () => {
+ const port = 61800;
+ const originalFetch = globalThis.fetch;
+ globalThis.fetch = async () => {
+ throw new TypeError("fetch failed");
+ };
+
+ try {
+ const authApi = loadAuthApi({ port, checkHealth: () => false });
+ const error = await authApi
+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })
+ .then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);
+ assert.equal(
+ (error as { unslothBackendStillRunning?: boolean })
+ .unslothBackendStillRunning,
+ undefined,
+ );
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("a presence probe that throws leaves the original verdict in place", async () => {
+ const originalFetch = globalThis.fetch;
+ globalThis.fetch = async () => {
+ throw new TypeError("fetch failed");
+ };
+
+ try {
+ const authApi = loadAuthApi({
+ port: 61801,
+ checkHealth: () => {
+ throw new Error("command not registered on this build");
+ },
+ });
+ const error = await authApi
+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })
+ .then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("no validated port yet means nothing is asked and nothing is claimed", async () => {
+ const invoked: string[] = [];
+ const originalFetch = globalThis.fetch;
+ globalThis.fetch = async () => {
+ throw new TypeError("fetch failed");
+ };
+
+ try {
+ // The placeholder base before server-port arrives is port 0, which never connects.
+ const authApi = loadAuthApi({
+ port: null,
+ checkHealth: () => true,
+ onInvoke: (command) => invoked.push(command),
+ });
+ const error = await authApi
+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })
+ .then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);
+ assert.deepEqual(invoked, []);
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("panels that all lose the backend at once share one native probe", async () => {
+ // The whole hub loses its connection in the same tick and each probe waits out the launcher's budget.
+ const originalFetch = globalThis.fetch;
+ globalThis.fetch = async () => {
+ throw new TypeError("fetch failed");
+ };
+
+ let probes = 0;
+ let releaseProbe: (() => void) | null = null;
+ const probeStarted = new Promise<void>((resolve) => {
+ releaseProbe = resolve;
+ });
+
+ try {
+ const authApi = loadAuthApi({
+ port: 61802,
+ checkHealth: async () => {
+ probes += 1;
+ releaseProbe?.();
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ return true;
+ },
+ });
+ const call = () =>
+ authApi
+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })
+ .then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+
+ const first = call();
+ await probeStarted;
+ const errors = await Promise.all([first, call(), call()]);
+
+ assert.equal(
+ probes,
+ 1,
+ "each lost panel opened its own native health probe",
+ );
+ for (const error of errors) {
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_ANSWERING_MESSAGE);
+ }
+
+ const later = await call();
+ assert.equal(probes, 2);
+ assert.ok(later instanceof Error);
+ assert.equal(later.message, authApi.BACKEND_NOT_ANSWERING_MESSAGE);
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("updating an existing install does not go through the changed path", async () => {
+ // The update deliberately stops the backend, so pin that this flow uses commands and events, not HTTP.
+ const update = await readFile(
+ new URL("../src/hooks/use-tauri-update.ts", import.meta.url),
+ "utf8",
+ );
+ assert.ok(update.includes('invoke("start_backend_update")'));
+ assert.ok(
+ !update.includes("authFetch"),
+ "the update flow now goes through authFetch",
+ );
+ assert.ok(
+ !/(?<![.\w])fetch\(/.test(update),
+ "the update flow now issues its own fetch, which the transport path wraps",
+ );
+
+ // A webview newer than its shell asks for nothing new; the rejected invoke is covered above.
+ const backend = await readFile(
+ new URL("../src/hooks/use-tauri-backend.ts", import.meta.url),
+ "utf8",
+ );
+ assert.ok(backend.includes('invoke<boolean>("check_health", { port })'));
+ const main = await readFile(
+ new URL("../../src-tauri/src/main.rs", import.meta.url),
+ "utf8",
+ );
+ assert.ok(main.includes("commands::check_health,"));
+ // check_health collapses a stalled probe onto false, the verdict this file exists to stop showing.
+ assert.ok(main.includes("commands::check_backend_present,"));
+ const authApiSrc = await readFile(
+ new URL("../src/features/auth/api.ts", import.meta.url),
+ "utf8",
+ );
+ assert.ok(
+ authApiSrc.includes('invoke<boolean>("check_backend_present", { port })'),
+ "the auth transport probe must ask for presence, not health",
+ );
+});
+
+test("the background chat storage filter accepts every transport verdict", async () => {
+ // Read as source: the module pulls in the Dexie database only a browser build can load.
+ const source = await readFile(
+ new URL(
+ "../src/features/chat/utils/chat-history-storage.ts",
+ import.meta.url,
+ ),
+ "utf8",
+ );
+ const filterStart = source.indexOf(
+ "export function isExpectedBackgroundChatStorageError",
+ );
+ assert.ok(filterStart !== -1, "the background storage error filter moved");
+ const filter = source.slice(filterStart, source.indexOf("\n}", filterStart));
+
+ assert.ok(
+ filter.includes("unslothTransportFailure"),
+ "the filter does not recognise the transport marker, so a sync against a backend " +
+ "that is merely busy is reported as an unexpected error",
+ );
+ assert.ok(
+ !filter.includes("please relaunch it"),
+ "the filter still matches one exact wording of the transport failure",
+ );
+});
+
+test("a POST is not retried on the long ladder", async () => {
+ // A network error is not an answer: the backend may have committed the request, so a retry can duplicate it.
+ const port = 61797;
+ const originalFetch = globalThis.fetch;
+ let attempts = 0;
+ globalThis.fetch = async () => {
+ attempts += 1;
+ throw new TypeError("fetch failed");
+ };
+
+ try {
+ const authApi = loadAuthApi({ port, checkHealth: () => false });
+ await authApi
+ .authFetch("/api/auth/api-keys", { method: "POST", body: "{}" })
+ .catch(() => undefined);
+ assert.ok(
+ attempts <= OLD_LADDER_ATTEMPTS,
+ `a POST was sent ${attempts} times; the unsafe ladder allows ${OLD_LADDER_ATTEMPTS}`,
+ );
+ assert.ok(
+ attempts > 1,
+ "the unsafe ladder still retries, as it always did",
+ );
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("a probe pending against the old port is not the answer about the new one", async () => {
+ // setApiBase can move the port inside a probe's 10s budget, so a shared pending promise reports the PREVIOUS backend.
+ const originalFetch = globalThis.fetch;
+ globalThis.fetch = async () => {
+ throw new TypeError("fetch failed");
+ };
+
+ const asked: number[] = [];
+ let port = 61810;
+ let releaseFirst: (() => void) | null = null;
+ const firstProbeStarted = new Promise<void>((resolve) => {
+ releaseFirst = resolve;
+ });
+
+ try {
+ const authApi = loadAuthApi({
+ port,
+ getPort: () => port,
+ checkHealth: async (probedPort: number) => {
+ asked.push(probedPort);
+ if (asked.length === 1) {
+ releaseFirst?.();
+ await new Promise((resolve) => setTimeout(resolve, 60));
+ return false;
+ }
+ return true;
+ },
+ });
+ const call = () =>
+ authApi
+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })
+ .then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+
+ const first = call();
+ await firstProbeStarted;
+ port = 61811;
+ const second = await call();
+ const firstError = await first;
+
+ assert.deepEqual(asked, [61810, 61811], "the new port was never probed");
+ assert.ok(second instanceof Error);
+ assert.equal(second.message, authApi.BACKEND_NOT_ANSWERING_MESSAGE);
+ assert.ok(firstError instanceof Error);
+ assert.equal(firstError.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
diff --git a/studio/src-tauri/src/commands.rs b/studio/src-tauri/src/commands.rs
index 8a6443253f8..438cdf0750d 100644
--- a/studio/src-tauri/src/commands.rs
+++ b/studio/src-tauri/src/commands.rs
@@ -443,6 +443,8 @@ struct BackendLiveness {
/// holding the port. Silence from a closed port is death; silence from an accepted
/// connection is a stall.
probe_timed_out: bool,
+ /// An HTTP RESPONSE, whatever its status. Weaker than `alive`, which also requires the payload to name this service.
+ answered: bool,
}
/// Check if an Unsloth backend is running on the given port.
@@ -459,6 +461,48 @@ pub async fn check_health(port: u16) -> Result<bool, String> {
}
}
+/// Whether a process still holds the port. `check_health_inner` only returns `Ok` with
+/// `probe_timed_out: false`, so a stall arrives through the `Err` arm; a REFUSED connection
+/// is not a timeout.
+#[tauri::command]
+pub async fn check_backend_present(
+ state: tauri::State<'_, BackendState>,
+ port: u16,
+) -> Result<bool, String> {
+ // Silence does not say which side went quiet, so ownership of the port settles it.
+ let state = state.inner();
+ backend_presence(port, || we_manage_a_backend_on(state, port)).await
+}
+
+/// *we_manage_it* is called AFTER the probe: an ownership answer taken before it died would report an exited backend as running.
+async fn backend_presence(
+ port: u16,
+ we_manage_it: impl Fn() -> bool,
+) -> Result<bool, String> {
+ match check_health_inner(port, HEALTH_PROBE_TIMEOUT).await {
+ Ok(liveness) => Ok(backend_is_present(&liveness, we_manage_it())),
+ Err(e) => {
+ info!("Backend presence check on port {} failed: {}", port, e);
+ Ok(backend_is_present(
+ &liveness_from_probe_error(&e),
+ we_manage_it(),
+ ))
+ }
+ }
+}
+
+/// Whether this app's own backend handle names *port*. The handle outlives the process it names, so the owner is checked too.
+fn we_manage_a_backend_on(state: &BackendState, port: u16) -> bool {
+ // Handle and process state in one pass under the same lock, so they cannot disagree.
+ process::owned_backend_on_port_is_running(state, port)
+}
+
+/// The rule `check_backend_present` applies, as a value so the test for it can actually fail.
+fn backend_is_present(liveness: &BackendLiveness, we_manage_it: bool) -> bool {
+ // A healthy answer is presence whoever owns the port; a timeout or unhealthy reply counts only for a backend we manage.
+ liveness.alive || ((liveness.probe_timed_out || liveness.answered) && we_manage_it)
+}
+
/// Probe the backend for process liveness.
///
/// `/api/liveness` rather than `/api/health`: health awaits hardware detection through
@@ -487,13 +531,31 @@ async fn check_health_inner(
continue;
}
if !resp.status().is_success() {
- return Ok(BackendLiveness::default());
+ // Not healthy, but not silence either: something answered on that port.
+ return Ok(BackendLiveness {
+ answered: true,
+ ..BackendLiveness::default()
+ });
}
- json = Some(resp.json::<serde_json::Value>().await?);
+ json = match resp.json::<serde_json::Value>().await {
+ Ok(value) => Some(value),
+ Err(error) => {
+ // Propagating the error made presence indistinguishable from a refusal.
+ info!("Backend on port {} answered unparseable JSON: {}", port, error);
+ return Ok(BackendLiveness {
+ answered: true,
+ ..BackendLiveness::default()
+ });
+ }
+ };
break;
}
let Some(json) = json else {
- return Ok(BackendLiveness::default());
+ // Every path answered 404, which is still an answer.
+ return Ok(BackendLiveness {
+ answered: true,
+ ..BackendLiveness::default()
+ });
};
// Liveness answers "alive" and health answers "healthy". Accept either, so the fallback
@@ -546,6 +608,7 @@ async fn check_health_inner(
warming_up: alive && (warming || (detecting && !deferred)),
inference_active: alive && inference_active,
probe_timed_out: false,
+ answered: true,
})
}
@@ -655,6 +718,8 @@ fn adopted_backend_liveness(
// port a different Unsloth backend has taken over is excluded by `different_owner`,
// since that one answered rather than fell silent.
probe_timed_out: adopted_failure_is_a_stall(verified, served.alive, different_owner),
+ // The pre-probe got a reply out of this port, whatever the re-check then did.
+ answered: served.answered || served.alive,
}
}
@@ -1379,8 +1444,18 @@ mod tests {
super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
.await
.unwrap(),
- super::BackendLiveness::default()
+ super::BackendLiveness {
+ // It answered, and that is all `answered` says: not OUR backend.
+ answered: true,
+ ..super::BackendLiveness::default()
+ }
);
+ assert!(!super::backend_is_present(
+ &super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
+ .await
+ .unwrap(),
+ false
+ ));
}
#[test]
@@ -1389,6 +1464,83 @@ mod tests {
assert!(super::HEALTH_PROBE_TIMEOUT < super::HEALTH_WATCHDOG_INTERVAL);
}
+ #[test]
+ fn the_frontend_retry_ladder_outlives_one_probe_budget() {
+ // #10520: the ladder lives in TypeScript and the budget here, so only this guard keeps them in step.
+ let src = include_str!("../../frontend/src/features/auth/api.ts").replace("\r\n", "\n");
+ let marker = "const TAURI_FETCH_RETRY_DELAYS_MS = [";
+ let start = src
+ .find(marker)
+ .expect("the Tauri fetch retry ladder moved; update this guard")
+ + marker.len();
+ let ladder = &src[start..];
+ let ladder = &ladder[..ladder.find(']').expect("unterminated retry ladder")];
+ let total_ms: u64 = ladder
+ .split(',')
+ .map(str::trim)
+ .filter(|delay| !delay.is_empty())
+ .map(|delay| {
+ delay
+ .parse::<u64>()
+ .expect("a retry delay stopped being a plain number of milliseconds")
+ })
+ .sum();
+ assert!(
+ total_ms >= super::HEALTH_PROBE_TIMEOUT.as_millis() as u64,
+ "the webview gives up after {total_ms}ms while one native liveness probe is \
+ allowed {}ms, so a backend the launcher still considers alive is reported to \
+ the user as not running",
+ super::HEALTH_PROBE_TIMEOUT.as_millis()
+ );
+ assert!(
+ src.contains("invoke<boolean>(\"check_backend_present\""),
+ "the transport-failure path no longer asks the native side before it tells the \
+ user to relaunch"
+ );
+ // And not the health command, which reports a spent budget as a refused connection.
+ assert!(
+ !src.contains("invoke<boolean>(\"check_health\""),
+ "the transport-failure path is back on check_health, which collapses a stalled \
+ probe onto \"not running\""
+ );
+ }
+
+ #[test]
+ fn a_stalled_probe_is_not_reported_as_an_absent_backend() {
+ // check_health answers `alive`, so a timeout and a closed port are the same answer.
+ let stalled = super::BackendLiveness {
+ alive: false,
+ probe_timed_out: true,
+ ..Default::default()
+ };
+ let closed = super::BackendLiveness::default();
+ let answered = super::BackendLiveness {
+ alive: true,
+ ..Default::default()
+ };
+
+ // Through the rule the command applies: an inlined copy asserts its own arithmetic.
+ assert!(!stalled.alive, "a stall is not an answer");
+ assert!(
+ super::backend_is_present(&stalled, true),
+ "a stalled probe must read as a backend that is still present"
+ );
+ assert!(
+ !super::backend_is_present(&stalled, false),
+ "a stall on a port this app does not manage proves nothing about our backend"
+ );
+ assert!(
+ !super::backend_is_present(&closed, true),
+ "a refused connection must still read as absent"
+ );
+ assert!(super::backend_is_present(&answered, false));
+ // And the health command must keep collapsing the stall for its own caller.
+ assert!(
+ !stalled.alive,
+ "check_health still reports a stall as not usable"
+ );
+ }
+
#[test]
fn the_startup_grace_survives_the_mac_cold_start_timeline() {
// Replays the macOS report this grace period exists for. The warm thread held the
@@ -1534,6 +1686,89 @@ mod tests {
port
}
+ /// A port that answers every request with *status* and *body*.
+ async fn answering_test_backend(status: &'static str, body: &'static str) -> u16 {
+ let listener = TcpListener::bind(("127.0.0.1", 0))
+ .await
+ .expect("probe test needs a loopback port");
+ let port = listener.local_addr().unwrap().port();
+ tokio::spawn(async move {
+ while let Ok((mut stream, _)) = listener.accept().await {
+ let mut buffer = [0; 2048];
+ let Ok(_) = stream.read(&mut buffer).await else {
+ return;
+ };
+ let response = format!(
+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ );
+ let _ = stream.write_all(response.as_bytes()).await;
+ }
+ });
+ port
+ }
+
+ #[tokio::test]
+ async fn ownership_is_read_after_the_probe_not_before_it() {
+ // The closure answers "we manage it" only once the request reached the server, so true means it ran AFTER the probe.
+ let probe_arrived = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let listener = TcpListener::bind(("127.0.0.1", 0))
+ .await
+ .expect("probe test needs a loopback port");
+ let port = listener.local_addr().unwrap().port();
+ let seen = std::sync::Arc::clone(&probe_arrived);
+ tokio::spawn(async move {
+ while let Ok((mut stream, _)) = listener.accept().await {
+ let mut buffer = [0; 2048];
+ let Ok(_) = stream.read(&mut buffer).await else {
+ return;
+ };
+ seen.store(true, std::sync::atomic::Ordering::SeqCst);
+ let response = "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";
+ let _ = stream.write_all(response.as_bytes()).await;
+ }
+ });
+
+ let asked = std::sync::Arc::clone(&probe_arrived);
+ assert_eq!(
+ super::backend_presence(port, move || asked.load(std::sync::atomic::Ordering::SeqCst))
+ .await,
+ Ok(true),
+ "ownership was read before the probe, so a backend that changed under it would \
+ still have been reported as running"
+ );
+ }
+
+ #[tokio::test]
+ async fn a_managed_backend_that_answers_unhealthily_is_still_present() {
+ // A backend still building its app answers a non-2xx, collapsed onto the refused-connection default.
+ let port = answering_test_backend("503 Service Unavailable", "").await;
+ assert_eq!(
+ super::backend_presence(port, || true).await,
+ Ok(true),
+ "a managed backend answering 503 was reported absent"
+ );
+ assert_eq!(super::backend_presence(port, || false).await, Ok(false));
+ }
+
+ #[tokio::test]
+ async fn a_reply_this_build_cannot_parse_is_still_an_answer() {
+ // Propagating the parse error made presence indistinguishable from a refusal.
+ let port = answering_test_backend("200 OK", "not json at all").await;
+ assert_eq!(
+ super::backend_presence(port, || true).await,
+ Ok(true),
+ "a managed backend answering an unparseable body was reported absent"
+ );
+ assert_eq!(super::backend_presence(port, || false).await, Ok(false));
+
+ let liveness = super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
+ .await
+ .expect("an answered probe is not a transport error");
+ assert!(!liveness.alive, "an unparseable reply is not a healthy backend");
+ assert!(liveness.answered, "the reply arrived, so the port is not silent");
+ }
+
#[tokio::test]
async fn a_port_that_accepts_and_never_answers_reads_as_a_stall() {
// The premise of the busy budget: a saturated backend still holds its port open, so
@@ -1554,6 +1789,45 @@ mod tests {
);
}
+ #[tokio::test]
+ async fn check_backend_present_reports_a_stalled_port_as_present() {
+ // The bug lives in the `Err` arm, which a struct-level test cannot reach; costs HEALTH_PROBE_TIMEOUT in wall clock.
+ let port = stalling_test_backend().await;
+ assert_eq!(
+ super::backend_presence(port, || true).await,
+ Ok(true),
+ "a backend holding the port and not answering was reported absent, which is the \
+ relaunch prompt this command was added to prevent"
+ );
+ }
+
+ #[tokio::test]
+ async fn a_stalled_port_this_app_does_not_manage_is_not_our_backend() {
+ // With no backend of our own on that port there is nothing to wait for.
+ let port = stalling_test_backend().await;
+ assert_eq!(
+ super::backend_presence(port, || false).await,
+ Ok(false),
+ "a stalled port with no managed backend behind it was reported as present"
+ );
+ }
+
+ #[tokio::test]
+ async fn check_backend_present_reports_a_closed_port_as_absent() {
+ // Classifying the error must not turn every failed probe into "present".
+ let port = {
+ let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
+ let port = listener.local_addr().unwrap().port();
+ drop(listener);
+ port
+ };
+ assert_eq!(
+ super::backend_presence(port, || true).await,
+ Ok(false),
+ "a closed port must still read as an absent backend"
+ );
+ }
+
#[tokio::test]
async fn a_port_with_nothing_on_it_reads_as_death_not_a_stall() {
// The other half: a backend that really exited leaves a closed port, and that must
@@ -1693,6 +1967,7 @@ mod tests {
warming_up: false,
inference_active: true,
probe_timed_out: false,
+ answered: true,
};
let confirmed = super::adopted_backend_liveness(false, &served, false);
assert!(
@@ -1714,6 +1989,7 @@ mod tests {
warming_up: false,
inference_active: false,
probe_timed_out: false,
+ answered: true,
},
false,
);
@@ -1838,6 +2114,7 @@ mod tests {
warming_up: warming,
inference_active: busy,
probe_timed_out: false,
+ answered: true,
},
Probe::TimedOut => {
elapsed += probe_budget;
diff --git a/studio/src-tauri/src/main.rs b/studio/src-tauri/src/main.rs
index cc5a8ff09b4..9559ea280bb 100644
--- a/studio/src-tauri/src/main.rs
+++ b/studio/src-tauri/src/main.rs
@@ -2125,6 +2125,7 @@ fn main() {
commands::start_managed_server,
commands::stop_server,
commands::check_health,
+ commands::check_backend_present,
commands::get_server_logs,
commands::open_logs_dir,
commands::open_models_dir,
diff --git a/studio/src-tauri/src/process.rs b/studio/src-tauri/src/process.rs
index 5879e422fa3..a951cd66d5b 100644
--- a/studio/src-tauri/src/process.rs
+++ b/studio/src-tauri/src/process.rs
@@ -1173,6 +1173,34 @@ pub(crate) fn owned_backend_snapshot(
Ok(snapshot)
}
+/// Whether the handle that names *port* still refers to a process that EXISTS: a handle
+/// outlives its process, and another process can bind the freed port.
+/// Anything this cannot read leaves the handle trusted, so a running backend is never declared dead.
+pub(crate) fn owned_backend_on_port_is_running(state: &BackendState, port: u16) -> bool {
+ let mut proc = match state.lock() {
+ Ok(guard) => guard,
+ Err(poisoned) => poisoned.into_inner(),
+ };
+ let handle = match proc.owned.as_mut() {
+ Some(handle) => handle,
+ None => return false,
+ };
+ if handle.port() != Some(port) {
+ return false;
+ }
+ match handle {
+ OwnedBackendHandle::Spawned { child, .. } => !matches!(child.try_wait(), Ok(Some(_))),
+ OwnedBackendHandle::Adopted { pid, .. } => backend_pid_is_running(*pid),
+ }
+}
+
+/// Whether *pid* still exists: false only when the pid is PROVABLY gone. A zombie has exited
+/// but is unreaped, which `kill(pid, 0)` alone cannot see.
+pub(crate) fn backend_pid_is_running(pid: u32) -> bool {
+ crate::desktop_backend_owner::pid_is_not_dead(pid)
+ && !crate::process_identity::is_zombie(pid)
+}
+
pub(crate) fn record_owned_backend_port_if_current(
state: &BackendState,
generation: u64,
@@ -6243,3 +6271,107 @@ mod exit_status_after_stdout_closed_tests {
}
}
}
+
+#[cfg(test)]
+mod owned_backend_liveness_tests {
+ use super::*;
+
+ #[cfg(unix)]
+ fn spawn_owned(args: &[&str]) -> Box<dyn ChildWrapper + Send> {
+ let mut cmd = Command::new(args[0]);
+ cmd.args(&args[1..])
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped());
+ let mut wrap = CommandWrap::from(cmd);
+ wrap.wrap(ProcessGroup::leader());
+ wrap.spawn().expect("spawn test child")
+ }
+
+ #[cfg(windows)]
+ fn spawn_owned(args: &[&str]) -> Box<dyn ChildWrapper + Send> {
+ let mut cmd = Command::new(args[0]);
+ cmd.args(&args[1..])
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped());
+ CommandWrap::from(cmd).spawn().expect("spawn test child")
+ }
+
+ #[cfg(unix)]
+ const LIVE_CHILD: [&str; 3] = ["/bin/sh", "-c", "exec sleep 30"];
+ #[cfg(unix)]
+ const DEAD_CHILD: [&str; 3] = ["/bin/sh", "-c", "exit 0"];
+ #[cfg(windows)]
+ const LIVE_CHILD: [&str; 3] = ["cmd.exe", "/C", "ping -n 30 127.0.0.1"];
+ #[cfg(windows)]
+ const DEAD_CHILD: [&str; 3] = ["cmd.exe", "/C", "exit 0"];
+
+ fn state_owning(child: Box<dyn ChildWrapper + Send>, port: u16) -> BackendState {
+ let state = new_backend_state();
+ {
+ let mut proc = state.lock().unwrap();
+ let pid = 0;
+ proc.owned = Some(OwnedBackendHandle::spawned(child, None, pid, 1));
+ if let Some(handle) = proc.owned.as_mut() {
+ handle.set_reported_port(port);
+ }
+ proc.port = Some(port);
+ }
+ state
+ }
+
+ #[test]
+ fn a_child_that_is_still_running_is_ours() {
+ let state = state_owning(spawn_owned(&LIVE_CHILD), 8765);
+ assert!(owned_backend_on_port_is_running(&state, 8765));
+ let mut proc = state.lock().unwrap();
+ if let Some(handle) = proc.owned.as_mut() {
+ if let Some(child) = handle.spawned_child_mut() {
+ let _ = child.start_kill();
+ }
+ }
+ }
+
+ #[test]
+ fn a_child_that_has_exited_leaves_the_port_to_strangers() {
+ let mut child = spawn_owned(&DEAD_CHILD);
+ let _ = child.wait();
+ let state = state_owning(child, 8765);
+ assert!(
+ !owned_backend_on_port_is_running(&state, 8765),
+ "an exited child still counted as the managed backend, so a foreign service on \
+ its port would be reported to the user as Unsloth still running"
+ );
+ }
+
+ #[test]
+ fn a_handle_for_another_port_is_not_this_port() {
+ let state = state_owning(spawn_owned(&LIVE_CHILD), 8765);
+ assert!(!owned_backend_on_port_is_running(&state, 8766));
+ let mut proc = state.lock().unwrap();
+ if let Some(handle) = proc.owned.as_mut() {
+ if let Some(child) = handle.spawned_child_mut() {
+ let _ = child.start_kill();
+ }
+ }
+ }
+
+ #[test]
+ fn no_handle_at_all_is_not_a_managed_backend() {
+ let state = new_backend_state();
+ assert!(!owned_backend_on_port_is_running(&state, 8765));
+ }
+
+ // The adopted half, where there is no child handle to wait on.
+ #[test]
+ fn an_adopted_pid_that_is_gone_is_not_running() {
+ assert!(backend_pid_is_running(std::process::id()));
+ // A real process run to completion, so this pid is PROVABLY gone; an unreadable pid stays trusted by design.
+ let mut child = spawn_owned(&DEAD_CHILD);
+ let pid = child.id();
+ let _ = child.wait();
+ assert!(
+ !backend_pid_is_running(pid),
+ "an adopted backend that has exited still read as running"
+ );
+ }
+}
diff --git a/tests/studio/test_auth_form_input_count.py b/tests/studio/test_auth_form_input_count.py
index a1d3d4f8158..48a0e4de990 100644
--- a/tests/studio/test_auth_form_input_count.py
+++ b/tests/studio/test_auth_form_input_count.py
@@ -314,12 +314,17 @@ def test_auth_flow_routes_do_not_mount_global_settings():
def test_auth_redirect_targets_are_idempotent_and_concurrent(tmp_path: Path):
if shutil.which("node") is None:
pytest.skip("node not available")
- probe = subprocess.run(
- ["node", "--experimental-strip-types", "--version"],
- capture_output = True,
- text = True,
- timeout = 5,
- )
+ # A timeout is a SKIP: the first `node` of a job on a Windows runner pays for image
+ # scanning and a cold file cache, and 5s was not enough for a `--version` there.
+ try:
+ probe = subprocess.run(
+ ["node", "--experimental-strip-types", "--version"],
+ capture_output = True,
+ text = True,
+ timeout = 120,
+ )
+ except subprocess.TimeoutExpired:
+ pytest.skip("node did not answer --version in time on this runner")
if probe.returncode != 0:
pytest.skip("node --experimental-strip-types not available")
@@ -335,6 +340,11 @@ def test_auth_redirect_targets_are_idempotent_and_concurrent(tmp_path: Path):
let access = null, refresh = null, passwordChange = false;
export const apiUrl = (path) => path;
export const isTauri = false;
+ // Read by the Tauri transport-failure path before it asks the native health
+ // check. This stub is the web build (isTauri false), where that path is never
+ // taken, but the import is unconditional and an ES module import of a name the
+ // stub does not export is a SyntaxError at instantiation, not at call time.
+ export const getApiPort = () => null;
export const accountTransitionPending = () => false;
export const reset = (a = null, r = null) => { access = a; refresh = r; passwordChange = false; };
export const clearAuthTokens = () => { access = null; refresh = null; };
@@ -406,6 +416,7 @@ def test_auth_redirect_targets_are_idempotent_and_concurrent(tmp_path: Path):
cwd = tmp_path,
capture_output = True,
text = True,
- timeout = 30,
+ # The same cold-start allowance as the probe above, plus the work itself.
+ timeout = 180,
)
assert result.returncode == 0, f"stderr: {result.stderr}\nstdout: {result.stdout}"
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index d6e63bd515a..14796dda433 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -2649,16 +2649,22 @@ def test_auth_retries_tag_transport_failures_like_the_first_attempt():
src = (WORKDIR / "studio" / "frontend" / "src" / "features" / "auth" / "api.ts").read_text(
encoding = "utf-8"
)
- assert src.count("unslothTransportFailure: true") == 2, "one tag per message, in one helper"
- tagger = src.split("function asTransportFailure", 1)[1].split("\n}\n", 1)[0]
+ tagger = src.split("async function asTransportFailure", 1)[1].split("\n}\n", 1)[0]
assert "err instanceof TypeError" in tagger
assert "navigator.onLine === false" in tagger
+ # Counted against the helper's own raises, not pinned to a number that a new message would trip.
+ assert tagger.count("unslothTransportFailure: true") == tagger.count("new Error(")
+ assert tagger.count("unslothTransportFailure: true") >= 2
+ assert src.count("unslothTransportFailure: true") == tagger.count(
+ "unslothTransportFailure: true"
+ ), "every tag belongs to the one helper"
retry = src.split("async function retryWithCurrentToken", 1)[1]
retry = retry.split("\n}\n", 1)[0]
assert "fetchWithTauriNetworkRetry" in retry
- assert "throw asTransportFailure(err);" in retry
+ # Awaited since #10520: dropping the await throws a pending promise and the tag is never seen.
+ assert "throw await asTransportFailure(err);" in retry
first = src.split("export async function authFetch", 1)[1]
- assert "throw asTransportFailure(err);" in first
+ assert "throw await asTransportFailure(err);" in first
def test_adoption_takes_its_own_pin_before_moving_the_checkpoint():