Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop declaring a live backend dead, and keep a crashed worker's stderr (#11070)
Daniel Hanc2d2fca7b7f7
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 9a2c6a4f187..a795698e592 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -14,6 +14,7 @@ import signalfrom loggers import get_loggerimport multiprocessing as mpimport queue+import reimport threadingimport timeimport uuid@@ -45,6 +46,134 @@ def __getattr__(name: str):logger = get_logger(__name__)+# Delimited, not a whitelist: a whitelist stopped at the apostrophe in `/home/o'connor/`.+_PATH_COMPONENT = r"[^\s\\/](?:(?:(?![A-Za-z]:[\\/])[^\\/\n\",;])*[^\s\\/])?"+# Second alternative: the root-level case (`/model.gguf`, `\\server\share`) has no trailing separator.+_ABSOLUTE_PATH_RE = re.compile(+ r"(?<![\w:/])(?:\\\\[^\\/\s]+[\\/]|[A-Za-z]:[\\/]|/)"+ r"(?:(?:" + _PATH_COMPONENT + r"[\\/])+[^\s\\/]*|[^\s\\/\",;]+[\\/]?)"+)+++def _shorten_path(match: "re.Match[str]") -> str:+ text = match.group(0)+ tail = text.replace("\\", "/").rstrip("/").rsplit("/", 1)[-1]+ return f".../{tail}" if tail else "..."+++# A line written by a logger; `decode_bicodec` logs 500 characters of generated text this way.+_LOG_RECORD_RE = re.compile(+ r"""^\s*(?:+ \d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2} # a leading ISO timestamp+ | \[?(?:DEBUG|INFO|WARNING|WARN|ERROR|CRITICAL|FATAL)\b+ | [\w]+(?:\.[\w]+)+\s*: # a dotted logger name before the colon+ )""",+ re.VERBOSE,+)++# Only the writer can tell a logged traceback from a dying process's; the bytes are identical. See `utils.worker_stderr`.+_LOG_CONTINUATION_PREFIX = " | "+_LOG_RECORD_START_MARK = "\x1f"+++def _looks_like_a_log_record(line: str) -> bool:+ """The mark is the reliable half; the patterns are the fallback for a producer that bypassed that handler."""+ if line.startswith(_LOG_CONTINUATION_PREFIX) or line.startswith(_LOG_RECORD_START_MARK):+ return True+ return bool(_LOG_RECORD_RE.match(line))+++# Errs towards dropping: a missed diagnostic costs a detail, a kept content line leaves the host.+_DIAGNOSTIC_START_RE = re.compile(+ r"""^(?:+ Traceback\ \(most\ recent\ call\ last\):+ | terminate\ called+ | what\(\):+ | Fatal\ Python\ error:+ | (?:Current\ )?[Tt]hread\ 0x+ | Stack\ \(most\ recent\ call\ first\):+ | Segmentation\ fault | Bus\ error | Illegal\ instruction+ | Floating\ point\ exception | Aborted | Killed | Trace/breakpoint\ trap+ | \*\*\* # *** stack smashing detected ***+ | double\ free | free\(\) | malloc\(\) | munmap_chunk | corrupted\ (?:size|double-linked)+ | std::(?:bad_alloc|terminate) | libc\+\+abi+ | GGML_ASSERT | CUDA\ error | HIP\ error | cudaError+ | [A-Za-z_][\w.]*(?:Error|Exception|Exit|Interrupt|Abort|Fault|Signal)\s*:+ )""",+ re.VERBOSE,+)+++_TRACEBACK_HEADER = "Traceback (most recent call last):"+++def _starts_a_new_diagnostic(line: str) -> bool:+ return bool(_DIAGNOSTIC_START_RE.match(line))+++def _diagnostic_lines_only(lines: "list[str]") -> "list[str]":+ """Line by line is not a filter, because a log record is not always one line."""+ kept: "list[str]" = []+ inside_a_diagnostic = False+ for line in lines:+ if line.startswith(_LOG_CONTINUATION_PREFIX):+ # Another thread logging mid-abort does not end the abort.+ continue+ if _looks_like_a_log_record(line):+ inside_a_diagnostic = False+ continue+ if _starts_a_new_diagnostic(line):+ inside_a_diagnostic = True+ kept.append(line)+ continue+ if inside_a_diagnostic and (not line.strip() or line[:1] in (" ", "\t")):+ kept.append(line)+ continue+ inside_a_diagnostic = False+ return kept+++def _crash_lines(lines: "list[str]") -> "list[str]":+ """The last traceback, unless a diagnostic follows it: `logger.exception` for a RECOVERED failure leaves one too."""+ starts = [+ index for index, line in enumerate(lines) if line.lstrip().startswith(_TRACEBACK_HEADER)+ ]+ if not starts:+ return lines+ header = starts[-1]+ end = len(lines)+ for index in range(header + 1, len(lines)):+ line = lines[index]+ if line.strip() and line[:1] not in (" ", "\t"):+ end = index + 1+ break+ after = lines[end:]+ return after if after else lines[header:]+++def _redact_worker_output(text: str) -> str:+ """Three redactors because none subsumes another: native path leases, token shapes, filesystem layout."""+ if not text:+ return ""+ redacted = text+ try:+ from utils.native_path_leases import redact_native_paths+ redacted = redact_native_paths(redacted)+ except Exception: # noqa: BLE001 -- a redactor that cannot run must not lose the others+ pass+ try:+ from hub.utils.download_registry import scrub_secrets+ redacted = scrub_secrets(redacted)+ except Exception: # noqa: BLE001+ pass+ try:+ # `redact_log_text` is idempotent, so it runs after `scrub_secrets`.+ from utils.log_redaction import redact_log_text+ redacted = redact_log_text(redacted)+ except Exception: # noqa: BLE001+ pass+ return _ABSOLUTE_PATH_RE.sub(_shorten_path, redacted)+class _LoadCancelled(Exception):"""Internal control flow for a caller-cancelled model load."""@@ -235,6 +364,8 @@ class InferenceOrchestrator:def __init__(self):self._proc: Optional[mp.Process] = None+ # Retired when the next worker is spawned; read long after _proc has been cleared.+ self._stderr_capture: Any = Noneself._cmd_queue: Any = Noneself._resp_queue: Any = Noneself._subprocess_shutdown_lock = threading.Lock()@@ -439,6 +570,17 @@ class InferenceOrchestrator:else get_hf_cache_paths().child_env({}))+ # Retired here, not at shutdown: a crash message outlives _shutdown_subprocess clearing _proc.+ self._retire_stderr_capture()+ self._worker_stopped_deliberately = False+ try:+ from utils.worker_stderr import WorkerStderrCapture+ self._stderr_capture = WorkerStderrCapture(prefix = "unsloth-inference-worker-")+ except Exception as exc:+ # No sink is the old behaviour; never a failed spawn.+ logger.debug("Could not open a worker stderr mirror: %s", exc)+ self._stderr_capture = None+with (child_environment_for_spawn(cache_env),native_path_secret_removed_for_child_start(),@@ -454,16 +596,22 @@ class InferenceOrchestrator:# rely on from here on: snapshotting it after start() would capture the None# and lose the only reference to a live child, which is the orphan this# change exists to prevent.+ _child_kwargs: dict = {+ "cmd_queue": self._cmd_queue,+ "resp_queue": self._resp_queue,+ "cancel_event": self._cancel_event,+ "drain_event": self._drain_event,+ "config": config,+ }+ if self._stderr_capture is not None:+ from utils.native_path_leases import STDERR_MIRROR_KWARG++ # Consumed by run_without_native_path_secret; it never reaches the entrypoint.+ _child_kwargs[STDERR_MIRROR_KWARG] = self._stderr_capture.path_spawned_proc = _CTX.Process(target = run_without_native_path_secret,args = ("core.inference.worker", "run_inference_process", cache_env),- kwargs = {- "cmd_queue": self._cmd_queue,- "resp_queue": self._resp_queue,- "cancel_event": self._cancel_event,- "drain_event": self._drain_event,- "config": config,- },+ kwargs = _child_kwargs,daemon = True,)self._proc = _spawned_proc@@ -606,6 +754,9 @@ class InferenceOrchestrator:handle and refuse the destructive sidecar swap."""self._stop_dispatcher() # before killing subprocessif self._proc is None or not self._proc.is_alive():+ # Already gone: a nonzero status is an unwaited crash and keeps its replay.+ exitcode = getattr(self._proc, "exitcode", 0) if self._proc is not None else 0+ self._worker_stopped_deliberately = exitcode == 0self._proc = Nonereturn True@@ -656,6 +807,8 @@ class InferenceOrchestrator:)return False+ # Without this flag every model switch replayed a healthy worker's stderr at ERROR.+ self._worker_stopped_deliberately = Trueself._proc = Noneself._cmd_queue = Noneself._resp_queue = None@@ -755,12 +908,90 @@ class InferenceOrchestrator:def _cleanup(self):self._shutdown_subprocess(timeout = 5.0)+ self._retire_stderr_capture()++ def _retire_stderr_capture(self) -> None:+ """A native fault BETWEEN requests has no waiter, so nothing reaches `_subprocess_crash_message`."""+ capture = getattr(self, "_stderr_capture", None)+ if capture is None:+ return+ proc = getattr(self, "_proc", None)+ try:+ worker_is_gone = proc is None or not proc.is_alive()+ except Exception: # noqa: BLE001 -- a handle in teardown; treat it as gone+ worker_is_gone = True+ # `_shutdown_subprocess_locked` has already cleared `_proc`, so the flag is all that still knows.+ if worker_is_gone and not getattr(self, "_worker_stopped_deliberately", False):+ self._log_worker_stderr_once(+ getattr(proc, "pid", None),+ getattr(proc, "exitcode", None),+ )+ try:+ capture.close()+ except Exception:+ pass+ self._stderr_capture = None++ def _public_worker_stderr_tail(self) -> str:+ """Spans the worker's WHOLE lifetime and goes out verbatim through `GenStreamError(public = True)`."""+ raw = self._worker_stderr_tail()+ if not raw:+ return ""+ # Filtered FIRST, then searched: searching raw selected logger-written traceback headers.+ kept = _crash_lines(_diagnostic_lines_only(raw.splitlines()))+ block = "\n".join(kept).strip()+ if not block:+ return ""+ return _redact_worker_output(block)++ def _log_worker_stderr_once(+ self,+ pid,+ exitcode,+ *,+ worker_exited: bool = True,+ ) -> None:+ """The RAW tail, at most once per worker; *worker_exited* is False for a still-live replacement."""+ capture = getattr(self, "_stderr_capture", None)+ if capture is None:+ return+ logged = getattr(self, "_stderr_tail_logged", None)+ if isinstance(logged, tuple) and logged[0] is capture:+ # Already replayed for a real exit, or this is a second non-terminal call.+ if logged[1] or not worker_exited:+ return+ # Marked before the read, so a failure here cannot become a log line per call.+ self._stderr_tail_logged = (capture, bool(worker_exited))+ raw = self._worker_stderr_tail()+ if not raw:+ return+ raw = raw.replace(_LOG_RECORD_START_MARK, "")+ logger.error(+ "Inference worker stderr (pid=%s, exitcode=%s):\n%s",+ pid,+ exitcode,+ raw,+ )++ def _worker_stderr_tail(self) -> str:+ capture = getattr(self, "_stderr_capture", None)+ if capture is None:+ return ""+ try:+ return capture.tail()+ except Exception:+ return ""def _ensure_subprocess_alive(self) -> bool:return self._proc is not None and self._proc.is_alive()- def _subprocess_crash_message(self, context: str) -> str:- """Return a user-facing crash message with the worker exit status."""+ def _subprocess_crash_message(+ self,+ context: str,+ *,+ with_worker_output: bool = False,+ ) -> str:+ """``with_worker_output`` defaults to FALSE: the opposite default discloses a traceback to a merely QUEUED request."""context_label = {"wait": "loading the model","generation": "generating a response",@@ -769,14 +1000,25 @@ class InferenceOrchestrator:}.get(context, context)message = f"The inference worker stopped unexpectedly while {context_label}."- if self._proc is None:+ proc = self._proc+ if proc is None:+ # A concurrent teardown can clear `_proc`; the bytes outlive the handle.+ self._log_worker_stderr_once(None, None)return f"{message} Details: process missing."- exitcode = self._proc.exitcode- pid = self._proc.pid+ exitcode = proc.exitcode+ pid = proc.pidif exitcode is None:+ # NOT terminal: the capture belongs to the live replacement, whose crash must stay replayable.+ self._log_worker_stderr_once(pid, None, worker_exited = False)return f"{message} Details: pid={pid}."+ # What the worker said before it went (#7843), narrowed to what a client may see.+ tail = self._public_worker_stderr_tail() if with_worker_output else ""+ details = f"\n\nWorker error output:\n{tail}" if tail else ""+ # The operator's unredacted copy: the worker's forwarding daemon thread can die first.+ self._log_worker_stderr_once(pid, exitcode)+if exitcode < 0:signum = -exitcodetry:@@ -790,9 +1032,12 @@ class InferenceOrchestrator:" This usually means the system killed it under memory pressure. ""Try a smaller model, lower context length, or close other GPU-heavy apps.")- return f"{message}{suffix} Details: pid={pid}, signal={sig_name}, exitcode={exitcode}."+ return (+ f"{message}{suffix} Details: pid={pid}, signal={sig_name}, "+ f"exitcode={exitcode}.{details}"+ )- return f"{message} Details: pid={pid}, exitcode={exitcode}."+ return f"{message} Details: pid={pid}, exitcode={exitcode}.{details}"def _send_cmd(self, cmd: dict) -> None:if self._cmd_queue is None:@@ -840,7 +1085,11 @@ class InferenceOrchestrator:if resp is None:if not self._ensure_subprocess_alive():- raise RuntimeError(self._subprocess_crash_message("wait"))+ raise RuntimeError(+ self._subprocess_crash_message(+ "wait", with_worker_output = self._owns_worker(cancel_event)+ )+ )continuertype = resp.get("type", "")@@ -1058,18 +1307,18 @@ class InferenceOrchestrator:initial_resp_queue = self._resp_queuewhile True:if self._proc is not initial_proc or self._resp_queue is not initial_resp_queue:- yield GenStreamError(- f"Error: {self._subprocess_crash_message(crash_context)}",- public = True,- )+ # No tail here whatever the lists say: the capture is the REPLACEMENT's.+ detail = self._subprocess_crash_message(crash_context)+ yield GenStreamError(f"Error: {detail}", public = True)returnresp = read_one(read_timeout)if resp is None:if not self._ensure_subprocess_alive():- yield GenStreamError(- f"Error: {self._subprocess_crash_message(crash_context)}",- public = True,+ # Only the request the worker was RUNNING gets its last words; the rest were queued behind it.+ detail = self._subprocess_crash_message(+ crash_context, with_worker_output = self._owns_worker(cancel_event))+ yield GenStreamError(f"Error: {detail}", public = True)returncontinue@@ -1457,7 +1706,12 @@ class InferenceOrchestrator:resp = self._read_resp(timeout = min(remaining, 1.0))if resp is None:if not self._ensure_subprocess_alive():- raise RuntimeError(self._subprocess_crash_message("sharing chat turn"))+ raise RuntimeError(+ self._subprocess_crash_message(+ "sharing chat turn",+ with_worker_output = self._owns_worker(None),+ )+ )continuertype = resp.get("type", "")@@ -2013,7 +2267,12 @@ class InferenceOrchestrator:candidate = read_one(timeout = min(1.0, deadline - time.monotonic()))if candidate is None:if not self._ensure_subprocess_alive():- raise RuntimeError(self._subprocess_crash_message("count"))+ # A count takes no part in the claim bookkeeping.+ raise RuntimeError(+ self._subprocess_crash_message(+ "count", with_worker_output = self._owns_worker(None)+ )+ )continue# _direct_reader already drops a reply whose mailbox is gone; this is the backstop.if (@@ -2580,7 +2839,10 @@ class InferenceOrchestrator:if resp is None:if not self._ensure_subprocess_alive():raise RuntimeError(- self._subprocess_crash_message("audio generation")+ self._subprocess_crash_message(+ "audio generation",+ with_worker_output = self._owns_worker(cancel_event),+ ))continuediff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 52e5f9fa5d1..877afed0ae8 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -1120,6 +1120,13 @@ def run_inference_process(service_name = "unsloth-studio-inference-worker",env = os.getenv("ENVIRONMENT_TYPE", "production"),)+ # Must follow setup_logging. Structlog records go to fd 1, but a third-party library+ # logging through stdlib `logging` reaches fd 2 via `logging.lastResort`, and that+ # traceback is byte-identical to a dying process's: unmarked, the parent hands a+ # RECOVERED failure to the NEXT caller on a shared worker as their crash.+ from utils.worker_stderr import mark_log_record_continuations++ mark_log_record_continuations()apply_gpu_ids(config.get("resolved_gpu_ids"), backend = config.get("device_backend"))diff --git a/studio/backend/tests/_worker_stderr_boom.py b/studio/backend/tests/_worker_stderr_boom.pynew file mode 100644index 00000000000..3ea2df2ac92--- /dev/null+++ b/studio/backend/tests/_worker_stderr_boom.py@@ -0,0 +1,47 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import sys+++def exit_one_after_writing_to_stderr(**_kwargs) -> None:+ print("Received command: generate", file = sys.stderr)+ print("Starting text generation", file = sys.stderr)+ sys.stderr.flush()+ raise RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB")+++def record_received_kwargs(**kwargs) -> None:+ print(f"entrypoint kwargs: {sorted(kwargs)}", file = sys.stderr)+ sys.stderr.flush()+ raise SystemExit(1)+++def hold_a_second_handle_on_stderr_then_exit(**_kwargs) -> None:+ import os++ _keep_alive = os.dup(2) # noqa: F841+ print("held a second handle on stderr", file = sys.stderr)+ sys.stderr.flush()+ raise SystemExit(1)+++def write_non_ascii_far_past_the_read_window(**_kwargs) -> None:+ from utils.worker_stderr import TAIL_READ_BYTES++ padding = "é" * 40+ for index in range((TAIL_READ_BYTES // len(padding.encode("utf-8"))) + 200):+ sys.stderr.write(f"{padding} warmup {index}\n")+ sys.stderr.write("RuntimeError: modellädt nicht — CUDA out of memory\n")+ sys.stderr.flush()+ raise SystemExit(1)+++def write_far_more_than_the_cap(**_kwargs) -> None:+ from utils.worker_stderr import MIRROR_FILE_CAP_BYTES++ line = "x" * 1023 + "\n"+ for _ in range((MIRROR_FILE_CAP_BYTES * 3) // len(line)):+ sys.stderr.write(line)+ sys.stderr.flush()+ raise RuntimeError("the last line before the worker died")diff --git a/studio/backend/tests/test_inference_worker_stderr_tail.py b/studio/backend/tests/test_inference_worker_stderr_tail.pynew file mode 100644index 00000000000..282c0d36e65--- /dev/null+++ b/studio/backend/tests/test_inference_worker_stderr_tail.py@@ -0,0 +1,1243 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""unsloth#7843: a worker that exits without answering must not lose its own error output."""++import subprocess+from pathlib import Path+from types import SimpleNamespace+import importlib.util+import multiprocessing as mp+import os+import signal+import sys++import pytest+++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from utils.worker_stderr import ( # noqa: E402+ MIRROR_FILE_CAP_BYTES,+ STDERR_MIRROR_KWARG,+ WorkerStderrCapture,+ decode_worker_stderr,+ stderr_tail_from_bytes,+)+++def _load_orchestrator_module():+ spec = importlib.util.spec_from_file_location(+ "inference_orchestrator_stderr_tail_under_test",+ Path(_BACKEND_DIR) / "core/inference/orchestrator.py",+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def _orchestrator_with(+ exitcode,+ capture = None,+ pid = 6145,+):+ module = _load_orchestrator_module()+ orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)+ orchestrator._proc = SimpleNamespace(pid = pid, exitcode = exitcode)+ orchestrator._stderr_capture = capture+ return orchestrator+++def test_a_windows_worker_arrives_as_crlf_cp1252_and_decodes_cleanly():+ raw = "Traceback (most recent call last):\r\n File “worker.py”, line 1\r\n".encode(+ "cp1252",+ )+ assert b"\r\n" in raw+ text = decode_worker_stderr(raw)+ assert "\r" not in text+ assert text.splitlines()[0] == "Traceback (most recent call last):"+ assert "“worker.py”" in text+++def test_a_utf8_worker_is_never_mis_read_as_cp1252():+ raw = "RuntimeError: — out of memory é\n".encode("utf-8")+ assert decode_worker_stderr(raw) == "RuntimeError: — out of memory é\n"+++def test_cp1252_bytes_that_are_not_valid_utf8_still_decode():+ raw = b"warning: caf\xe9 layer skipped\r\n"+ assert decode_worker_stderr(raw) == "warning: café layer skipped\n"+++def test_a_cp1252_character_at_the_very_end_is_not_trimmed_away():+ assert decode_worker_stderr(b"RuntimeError: caf\xe9") == "RuntimeError: café"+ assert decode_worker_stderr("é at the start".encode("utf-8")[1:]).endswith(" at the start")+ truncated = "RuntimeError: é".encode("utf-8")[:-1]+ assert decode_worker_stderr(truncated, ends_at_eof = False) == "RuntimeError: "+ assert decode_worker_stderr(truncated) == "RuntimeError: \u00c3"+ assert decode_worker_stderr("done é".encode("utf-8")) == "done é"+++def test_undecodable_bytes_are_replaced_rather_than_dropped():+ # 0x81 and 0x90 are undefined in cp1252 and invalid as UTF-8.+ raw = b"RuntimeError: \x81\x90 bad bytes\n"+ text = decode_worker_stderr(raw)+ assert "RuntimeError:" in text+ assert "bad bytes" in text+++def test_a_progress_bar_redrawn_with_carriage_returns_does_not_hide_the_traceback():+ raw = (+ b"Loading checkpoint: 10%\rLoading checkpoint: 55%\rLoading checkpoint: 100%\r\n"+ b"RuntimeError: the real failure\n"+ )+ tail = stderr_tail_from_bytes(raw, max_lines = 2)+ assert tail.splitlines()[-1] == "RuntimeError: the real failure"+++def test_the_tail_keeps_the_last_lines_and_drops_blank_ones():+ raw = ("\n".join(f"line {index}" for index in range(100)) + "\n\n\n").encode("utf-8")+ tail = stderr_tail_from_bytes(raw, max_lines = 3)+ assert tail == "line 97\nline 98\nline 99"+++def test_the_character_cap_trims_from_the_front_and_never_shows_a_half_line():+ raw = ("first line that is quite long\n" + "b" * 200 + "\nlast line\n").encode("utf-8")+ tail = stderr_tail_from_bytes(raw, max_lines = 10, max_chars = 60)+ assert tail.endswith("last line")+ assert len(tail) <= 60+ assert not tail.startswith("b")+++def test_an_empty_sink_produces_no_tail():+ assert stderr_tail_from_bytes(b"") == ""+ assert stderr_tail_from_bytes(b"\n \n\r\n") == ""+++def test_a_capture_whose_file_is_gone_reports_no_tail_instead_of_raising():+ capture = WorkerStderrCapture(prefix = "unsloth-test-")+ capture.close()+ assert capture.tail() == ""+ # Idempotent: a Windows host takes a second close when the child still holds it.+ capture.close()+++def test_a_positive_exit_carries_the_worker_stderr_tail(tmp_path):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ Path(capture.path).write_bytes(+ b"Starting text generation\r\n"+ b"Traceback (most recent call last):\r\n"+ b"RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB\r\n",+ )++ message = _orchestrator_with(1, capture)._subprocess_crash_message(+ "generation", with_worker_output = True+ )++ assert message.startswith(+ "The inference worker stopped unexpectedly while generating a response.",+ )+ assert "Details: pid=6145, exitcode=1." in message+ assert "Worker error output:" in message+ assert "RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB" in message+ assert "\r" not in message+++def test_a_signalled_exit_keeps_its_hint_and_gains_the_tail(tmp_path):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ Path(capture.path).write_bytes(b"terminate called after throwing an instance of 'c10::Error'\n")++ message = _orchestrator_with(-9, capture)._subprocess_crash_message(+ "wait", with_worker_output = True+ )++ assert "exitcode=-9" in message+ assert "c10::Error" in message++ # POSIX only: a Windows process never reports a negative code and signal.Signals(9)+ # raises there, so that branch names the signal SIG9 and offers no hint.+ if hasattr(signal, "SIGKILL"):+ assert "memory pressure" in message+ assert "signal=SIGKILL" in message+ else:+ assert "signal=SIG9" in message+++def test_nothing_captured_leaves_the_message_exactly_as_it_was(tmp_path):+ empty = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ for capture in (None, empty):+ message = _orchestrator_with(1, capture)._subprocess_crash_message(+ "generation", with_worker_output = True+ )+ assert message == (+ "The inference worker stopped unexpectedly while generating a response. "+ "Details: pid=6145, exitcode=1."+ )+++def test_a_worker_still_running_is_not_given_a_tail(tmp_path):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ Path(capture.path).write_bytes(b"still going\n")+ message = _orchestrator_with(None, capture)._subprocess_crash_message(+ "generation", with_worker_output = True+ )+ assert message.endswith("Details: pid=6145.")+++def test_the_orchestrator_hands_the_sink_to_the_child_it_spawns():+ # Read from the source: exercising it for real needs a model load.+ source = (+ (Path(_BACKEND_DIR) / "core/inference/orchestrator.py")+ .read_text(+ encoding = "utf-8",+ )+ .replace("\r\n", "\n")+ )+ start = source.index("def _spawn_subprocess(")+ body = source[start : source.index("\n def ", start + 10)]+ assert "WorkerStderrCapture" in body, "the spawn path no longer opens a stderr sink"+ assert "STDERR_MIRROR_KWARG" in body, "the sink is no longer passed to the child"+ assert "_retire_stderr_capture()" in body, "a stale sink now survives into the next worker"+++def test_the_reserved_kwarg_name_is_the_same_on_both_sides():+ # utils/native_path_leases.py spells the name out to keep its import graph stdlib only.+ from utils.native_path_leases import STDERR_MIRROR_KWARG as entrypoint_name+ assert entrypoint_name == STDERR_MIRROR_KWARG+++def _spawn(+ function_name,+ mirror_path,+ timeout = 180,+):+ from utils.native_path_leases import run_without_native_path_secret++ # Bare name, with this directory on sys.path: a dotted name finds the repository's own+ # top-level "tests" package instead. Spawn hands the child the parent's sys.path.+ tests_dir = str(Path(__file__).resolve().parent)+ if tests_dir not in sys.path:+ sys.path.insert(0, tests_dir)++ context = mp.get_context("spawn")+ kwargs = {} if mirror_path is None else {STDERR_MIRROR_KWARG: mirror_path}+ process = context.Process(+ target = run_without_native_path_secret,+ args = ("_worker_stderr_boom", function_name, {}),+ kwargs = kwargs,+ )+ process.start()+ process.join(timeout)+ assert not process.is_alive(), "the test worker did not exit"+ return process+++def test_a_worker_that_exits_one_after_writing_to_stderr_keeps_its_traceback(tmp_path):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ process = _spawn("exit_one_after_writing_to_stderr", capture.path)+ assert process.exitcode == 1++ orchestrator = _orchestrator_with(process.exitcode, capture, pid = process.pid)+ message = orchestrator._subprocess_crash_message("generation", with_worker_output = True)++ assert "RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB" in message+ assert "Traceback (most recent call last)" in message+ assert "Starting text generation" in capture.tail(max_lines = 100)+++def test_the_reserved_kwarg_is_consumed_and_never_reaches_the_entrypoint(tmp_path):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ process = _spawn("record_received_kwargs", capture.path)+ assert process.exitcode == 1+ tail = capture.tail(max_lines = 100)+ assert "entrypoint kwargs: []" in tail, tail+++def test_the_sink_is_bounded_even_when_the_worker_floods_stderr(tmp_path):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ process = _spawn("write_far_more_than_the_cap", capture.path)+ assert process.exitcode == 1++ size = os.path.getsize(capture.path)+ assert size <= 2 * MIRROR_FILE_CAP_BYTES + 65536, size+ assert "the last line before the worker died" in capture.tail(max_lines = 200)+++_STALLED_READER_CHILD = r"""+import os, sys+sys.path.insert(0, %(backend)r)+from utils.worker_stderr import MIRROR_FILE_CAP_BYTES, install_worker_stderr_mirror++assert install_worker_stderr_mirror(%(path)r) is True+line = b"x" * 1023 + b"\n"+for _ in range((MIRROR_FILE_CAP_BYTES * 32) // len(line)):+ os.write(2, line)+os.write(2, b"the last line before the worker died\n")+"""+++def test_the_sink_is_bounded_even_when_nobody_drains_the_operators_stderr(tmp_path):+ """The mirrored copy is a courtesy; the cap is not. 68 MB landed here when a full pipe+ parked the pump inside os.write and the cap counted bytes relayed rather than bytes written."""+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ script = _STALLED_READER_CHILD % {"backend": _BACKEND_DIR, "path": capture.path}+ read_end, write_end = os.pipe()+ try:+ process = subprocess.Popen([sys.executable, "-c", script], stderr = write_end)+ os.close(write_end)+ write_end = None+ # Nothing ever reads `read_end`, which is what a stalled log consumer looks like.+ assert process.wait(timeout = 300) == 0+ finally:+ if write_end is not None:+ os.close(write_end)+ os.close(read_end)++ size = os.path.getsize(capture.path)+ assert size <= 2 * MIRROR_FILE_CAP_BYTES + 65536, size+ assert "the last line before the worker died" in capture.tail(max_lines = 200)+ capture.close()+++def test_a_caller_that_passes_no_mirror_is_unchanged(tmp_path):+ process = _spawn("exit_one_after_writing_to_stderr", None)+ assert process.exitcode == 1+ assert _orchestrator_with(process.exitcode, None, pid = process.pid)._subprocess_crash_message(+ "generation",+ ) == (+ "The inference worker stopped unexpectedly while generating a response. "+ f"Details: pid={process.pid}, exitcode=1."+ )+++def test_a_server_that_exits_leaves_no_sink_behind(tmp_path):+ script = (+ "import sys; sys.path.insert(0, %r)\n"+ "from utils.worker_stderr import WorkerStderrCapture\n"+ "print(WorkerStderrCapture(directory = %r, prefix = 'unsloth-test-').path)\n"+ ) % (_BACKEND_DIR, str(tmp_path))+ import subprocess++ path = subprocess.run(+ [sys.executable, "-c", script],+ capture_output = True,+ text = True,+ check = True,+ ).stdout.strip()+ assert path.startswith(str(tmp_path))+ assert not os.path.exists(path), "the sink outlived the process that opened it"+++def test_a_severed_multibyte_character_does_not_mojibake_the_whole_tail():+ from utils.worker_stderr import TAIL_READ_BYTES++ line = "RuntimeError: modellädt nicht — CUDA out of memory\n".encode("utf-8")+ body = line * 30+ window = b"\xa9" + b"." * (TAIL_READ_BYTES - 1 - len(body)) + body+ assert len(window) == TAIL_READ_BYTES+ with pytest.raises(UnicodeDecodeError):+ window.decode("utf-8")++ tail = stderr_tail_from_bytes(window)+ assert "modellädt nicht — CUDA out of memory" in tail, tail+ assert "ä" not in tail and "—" not in tail, tail+++def test_a_severed_character_at_the_end_of_the_window_is_dropped_too():+ raw = "RuntimeError: café blew up — ".encode("utf-8") + "é".encode("utf-8")[:1]+ text = decode_worker_stderr(raw)+ assert "café blew up —" in text, text+ assert "é" not in text, text+++def test_a_real_worker_whose_output_outgrows_the_read_window_still_reads_cleanly(tmp_path):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ process = _spawn("write_non_ascii_far_past_the_read_window", capture.path, timeout = 60)+ assert process.exitcode == 1+ tail = capture.tail(max_lines = 5)+ assert "modellädt nicht — CUDA out of memory" in tail, tail+ assert "ä" not in tail, tail+ capture.close()+++def test_a_missing_sink_is_never_re_created_by_the_child(tmp_path):+ from utils.worker_stderr import install_worker_stderr_mirror++ missing = tmp_path / "already-retired.stderr"+ assert install_worker_stderr_mirror(str(missing)) is False+ assert not missing.exists(), "the child created a sink the parent had retired"++[email protected](not hasattr(os, "O_NOFOLLOW"), reason = "O_NOFOLLOW is POSIX only")+def test_a_symlink_at_the_sink_path_is_not_followed(tmp_path):+ from utils.worker_stderr import install_worker_stderr_mirror++ victim = tmp_path / "victim.txt"+ victim.write_text("ORIGINAL\n", encoding = "utf-8")+ link = tmp_path / "sink.stderr"+ link.symlink_to(victim)++ assert install_worker_stderr_mirror(str(link)) is False+ assert victim.read_text(encoding = "utf-8") == "ORIGINAL\n"+++def test_a_worker_holding_a_second_handle_on_stderr_still_exits_promptly(tmp_path):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ process = _spawn("hold_a_second_handle_on_stderr_then_exit", capture.path, timeout = 60)+ assert process.exitcode == 1+ assert "held a second handle on stderr" in capture.tail(), capture.tail()+ capture.close()++ # The race cannot be asserted from outside: a run that does not hit it proves nothing.+ source = (Path(_BACKEND_DIR) / "utils/worker_stderr.py").read_text(encoding = "utf-8")+ teardown = source.split("def _stop_mirror(", 1)[1].split("\ndef ", 1)[0]+ close_at = teardown.index("os.close(inherited_fd)")+ # Both threads write to the inherited descriptor, so both must be done with it.+ for guard in ("pump.is_alive()", "relay.is_alive()"):+ assert guard in teardown[:close_at], (+ "the teardown closes the inherited stderr without first checking that the "+ f"thread behind {guard} has finished with it"+ )++[email protected](not hasattr(os, "fork"), reason = "fork is POSIX only")+def test_a_forked_child_does_not_retire_its_parents_live_sink(tmp_path):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ Path(capture.path).write_bytes(b"the parent is still writing here\n")++ pid = os.fork()+ if pid == 0: # pragma: no cover - runs in the child+ import atexit as _atexit+ try:+ _atexit._run_exitfuncs()+ finally:+ os._exit(0)+ os.waitpid(pid, 0)++ assert os.path.exists(capture.path), "a forked child retired its parent's live sink"+ assert "the parent is still writing here" in capture.tail()+ capture.close()+++def test_the_sink_directory_is_never_swept_by_pattern():+ source = (Path(_BACKEND_DIR) / "utils/worker_stderr.py").read_text(encoding = "utf-8")+ for forbidden in (+ "glob(",+ "iglob(",+ "iterdir(",+ "listdir(",+ "scandir(",+ "walk(",+ "st_mtime",+ "rmtree(",+ ):+ assert forbidden not in source, f"cleanup grew a directory sweep ({forbidden})"+++# Spaces and non-ASCII are what break code that builds paths by string, and a Windows+# profile routinely has both.+_INSTALL_TEMP_SHAPES = {+ "linux": "tmp",+ "windows-profile": "C_/Users/Jos\u00e9 Mu\u00f1oz/AppData/Local/Temp",+ "wsl-windows-drive": "mnt/c/Users/Jos\u00e9 Mu\u00f1oz/AppData/Local/Temp",+ "macos-per-user": "var/folders/9k/9k1_2p_s0qz3h/T",+}++[email protected]("shape", sorted(_INSTALL_TEMP_SHAPES))+def test_the_sink_works_on_every_install_path_shape(tmp_path, shape):+ directory = tmp_path.joinpath(*_INSTALL_TEMP_SHAPES[shape].split("/"))+ try:+ directory.mkdir(parents = True)+ except (OSError, UnicodeError) as exc:+ pytest.skip(f"this filesystem cannot hold a {shape} path: {exc}")++ capture = WorkerStderrCapture(directory = str(directory), prefix = "unsloth-test-")+ assert Path(capture.path).parent == directory+ process = _spawn("exit_one_after_writing_to_stderr", capture.path)+ assert process.exitcode == 1++ message = _orchestrator_with(+ process.exitcode,+ capture,+ pid = process.pid,+ )._subprocess_crash_message("generation", with_worker_output = True)+ assert "RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB" in message++ capture.close()++[email protected](os.name == "nt", reason = "the parent's stderr is redirected per platform")+def test_the_mirror_writes_through_to_the_inherited_stderr(tmp_path, capfd):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ _spawn("exit_one_after_writing_to_stderr", capture.path)+ captured = capfd.readouterr()+ assert "Starting text generation" in captured.err+ assert "CUDA out of memory" in captured.err+++class _FixedCapture:+ def __init__(self, text):+ self._text = text++ def tail(self):+ return self._text+++def _orchestrator_with_capture(text):+ from core.inference.orchestrator import InferenceOrchestrator++ instance = InferenceOrchestrator.__new__(InferenceOrchestrator)+ instance._stderr_capture = _FixedCapture(text)+ return instance+++CROSS_ACCOUNT = (+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text: "+ "another account's private prompt\n"+ "2026-09-16 10:00:02 worker: request 41 finished\n"+)+TRACEBACK = (+ "Traceback (most recent call last):\n"+ ' File "/home/alice/.unsloth/studio/worker.py", line 42, in run\n'+ ' raise RuntimeError("boom")\n'+ "RuntimeError: boom\n"+)+++def test_the_public_tail_drops_everything_before_the_traceback():+ public = _orchestrator_with_capture(CROSS_ACCOUNT + TRACEBACK)._public_worker_stderr_tail()+ assert "another account" not in public, public+ assert "request 41 finished" not in public, public+ assert public.startswith("Traceback (most recent call last):"), public+ assert "RuntimeError: boom" in public+++def test_the_public_tail_keeps_only_the_last_traceback():+ earlier = TRACEBACK.replace("boom", "an earlier unrelated failure")+ public = _orchestrator_with_capture(+ earlier + CROSS_ACCOUNT + TRACEBACK+ )._public_worker_stderr_tail()+ assert "an earlier unrelated failure" not in public, public+ assert "RuntimeError: boom" in public+++def test_a_capture_of_nothing_but_logging_is_dropped_entirely():+ assert _orchestrator_with_capture(CROSS_ACCOUNT)._public_worker_stderr_tail() == ""+ assert _orchestrator_with_capture("")._public_worker_stderr_tail() == ""+++def test_a_native_abort_with_no_traceback_is_still_reported():+ abort = "terminate called after throwing an instance of 'c10::Error'\n"+ public = _orchestrator_with_capture(CROSS_ACCOUNT + abort)._public_worker_stderr_tail()+ assert "c10::Error" in public, public+ assert "another account" not in public, public+++def test_the_log_record_shapes_are_the_ones_content_arrives_in():+ from core.inference.orchestrator import _looks_like_a_log_record+ for record in (+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text: hello",+ "2026-09-16T10:00:01 worker: started",+ "INFO: loaded the model",+ "[WARNING] falling back to CPU",+ "audio_codecs.decode_bicodec: generated text: hello",+ ):+ assert _looks_like_a_log_record(record) is True, record++ for kept in (+ "terminate called after throwing an instance of 'c10::Error'",+ "Traceback (most recent call last):",+ "RuntimeError: boom",+ ' File "/x/worker.py", line 42, in run',+ "Segmentation fault (core dumped)",+ ):+ assert _looks_like_a_log_record(kept) is False, kept+++def test_the_public_tail_redacts_paths_and_credentials():+ from core.inference.orchestrator import _redact_worker_output++ text = (+ "Traceback (most recent call last):\n"+ ' File "/home/alice/.unsloth/studio/worker.py", line 42, in run\n'+ ' login(token="hf_abcdefghijklmnopqrstuvwxyz012345")\n'+ "RuntimeError: refused, Authorization: Bearer sk-secret-value\n"+ )+ public = _redact_worker_output(text)+ assert "/home/alice" not in public, public+ assert "worker.py" in public, public+ assert "hf_abcdefghijklmnopqrstuvwxyz012345" not in public, public+ assert "sk-secret-value" not in public, public+ assert "RuntimeError: refused" in public, public+++def test_the_crash_message_uses_the_public_tail():+ import inspect++ from core.inference.orchestrator import InferenceOrchestrator++ source = inspect.getsource(InferenceOrchestrator._subprocess_crash_message)+ assert "_public_worker_stderr_tail()" in source+ assert "= self._worker_stderr_tail()" not in source+++_FATAL_CHILD = r"""+import os, signal, sys+sys.path.insert(0, %(backend)r)+from utils.worker_stderr import install_worker_stderr_mirror++assert install_worker_stderr_mirror(%(path)r) is True+os.write(2, b"terminate called after throwing an instance of 'c10::Error'\n")+# No flush, no atexit, no interpreter shutdown: the process is gone between one+# instruction and the next, which is what SIGSEGV and SIGABRT do.+# SIGKILL on POSIX; Windows has no such signal, and os.kill there calls+# TerminateProcess, which is the same thing for this purpose: no handler, no cleanup.+os.kill(os.getpid(), getattr(signal, "SIGKILL", signal.SIGTERM))+"""+++def test_a_worker_killed_outright_still_leaves_its_last_words(tmp_path):+ capture = WorkerStderrCapture(directory = str(tmp_path), prefix = "unsloth-test-")+ backend = str(Path(__file__).resolve().parent.parent)+ script = _FATAL_CHILD % {"backend": backend, "path": capture.path}++ completed = subprocess.run(+ [sys.executable, "-c", script],+ capture_output = True,+ timeout = 120,+ )+ if os.name == "nt":+ assert completed.returncode != 0, completed.stderr[-400:]+ else:+ assert completed.returncode == -signal.SIGKILL, completed.stderr[-400:]+ assert "c10::Error" in capture.tail(), capture.tail()+++def test_the_mirror_puts_the_sink_on_fd_two_rather_than_a_pipe():+ from utils import worker_stderr++ source = Path(worker_stderr.__file__).read_text(encoding = "utf-8")+ install = source.split("def install_worker_stderr_mirror(", 1)[1]+ assert "os.pipe()" not in install, "fd 2 is a pipe again"+ assert "_open_sink_for_append" in install+ assert "os.dup2(writer_fd, 2)" in install+ # Compaction rewrites the file from the front while fd 2 still points at it.+ append = source.split("def _open_sink_for_append(", 1)[1].split("\ndef ", 1)[0]+ assert "os.O_APPEND" in append+++MULTILINE_CROSS_ACCOUNT = (+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text: the first line\n"+ "another account's second line\n"+ "and a third line of the same prompt\n"+)+++def test_a_logged_message_cannot_leak_through_its_own_continuation_lines():+ abort = (+ "terminate called after throwing an instance of 'c10::Error'\n"+ " what(): CUDA error: device-side assert triggered\n"+ )+ public = _orchestrator_with_capture(+ MULTILINE_CROSS_ACCOUNT + abort+ )._public_worker_stderr_tail()+ assert "c10::Error" in public, public+ assert "device-side assert" in public, public+ assert "another account" not in public, public+ assert "third line" not in public, public+++def test_compaction_never_deletes_what_a_racing_writer_appended(tmp_path):+ from utils.worker_stderr import _compact_sink++ path = tmp_path / "sink"+ path.write_bytes(b"x" * 4096)+ appender = os.open(str(path), os.O_WRONLY | os.O_APPEND)+ handle = open(path, "r+b", buffering = 0)++ class _WriterRacesTheRead:+ def __init__(self, inner):+ self._inner = inner+ self._raced = False+ self._raced_the_write = False++ def __getattr__(self, name):+ return getattr(self._inner, name)++ def read(self, *args):+ data = self._inner.read(*args)+ if not self._raced:+ self._raced = True+ os.write(appender, b"terminate called after throwing an instance of 'c10::Error'\n")+ return data++ def write(self, payload):+ written = self._inner.write(payload)+ if not self._raced_the_write:+ self._raced_the_write = True+ os.write(appender, b"what(): CUDA error: device-side assert triggered\n")+ return written++ try:+ _compact_sink(_WriterRacesTheRead(handle), 1024)+ finally:+ handle.close()+ os.close(appender)++ kept = path.read_bytes()+ # Both windows: the append during the tail READ and the one during the REWRITE.+ assert b"terminate called" in kept, kept[-300:]+ assert b"device-side assert" in kept, kept[-300:]+ assert len(kept) <= 1024 + 256, len(kept)+++def test_the_operator_still_gets_the_last_words_in_the_server_log(monkeypatch):+ from core.inference import orchestrator as orchestrator_module++ written = []+ monkeypatch.setattr(+ orchestrator_module.logger,+ "error",+ lambda message, *args, **kwargs: written.append(message % args if args else message),+ )++ abort = (+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text: private\n"+ "terminate called after throwing an instance of 'c10::Error'\n"+ " what(): CUDA error at /home/alice/.unsloth/studio/worker.py\n"+ )+ instance = _orchestrator_with_capture(abort)+ instance._proc = SimpleNamespace(exitcode = -6, pid = 4242, is_alive = lambda: False)+ message = instance._subprocess_crash_message("generation", with_worker_output = True)++ logged = "\n".join(written)+ assert "terminate called" in logged, logged+ assert "/home/alice" in logged, logged+ assert "generated text: private" in logged, logged+ assert "/home/alice" not in message, message+ assert "generated text" not in message, message++ written.clear()+ instance._subprocess_crash_message("wait")+ assert written == [], written+++def test_a_teardown_that_clears_the_handle_first_still_logs_the_tail(monkeypatch):+ from core.inference import orchestrator as orchestrator_module++ written = []+ monkeypatch.setattr(+ orchestrator_module.logger,+ "error",+ lambda message, *args, **kwargs: written.append(message % args if args else message),+ )+ instance = _orchestrator_with_capture(+ "terminate called after throwing an instance of 'c10::Error'\n"+ )+ instance._proc = None+ message = instance._subprocess_crash_message("generation", with_worker_output = True)+ assert "process missing" in message+ assert any("terminate called" in line for line in written), written++ written.clear()+ other = _orchestrator_with_capture("Fatal Python error: Segmentation fault\n")+ other._proc = SimpleNamespace(exitcode = None, pid = 99, is_alive = lambda: True)+ other._subprocess_crash_message("wait")+ assert any("Segmentation fault" in line for line in written), written+++def test_a_logged_traceback_is_not_mistaken_for_the_crash():+ logged = (+ "2026-09-16 10:00:03 worker: request 41 failed, retrying\n"+ "Traceback (most recent call last):\n"+ ' File "/home/alice/.unsloth/studio/worker.py", line 9, in handle\n'+ ' raise ValueError("another account\'s prompt was rejected")\n'+ "ValueError: another account's prompt was rejected\n"+ )+ abort = "Fatal Python error: Aborted\n\nCurrent thread 0x00007f1a (most recent call first):\n"+ public = _orchestrator_with_capture(logged + abort)._public_worker_stderr_tail()+ assert "another account" not in public, public+ assert "ValueError" not in public, public+ assert "Fatal Python error: Aborted" in public, public+++def test_a_runtimes_own_traceback_after_a_log_line_is_still_reported():+ text = "2026-09-16 10:00:02 worker: request 41 finished\n\n" + TRACEBACK+ public = _orchestrator_with_capture(text)._public_worker_stderr_tail()+ assert "RuntimeError: boom" in public, public+ assert "request 41 finished" not in public, public+++def test_a_path_component_with_punctuation_in_it_is_still_redacted():+ from core.inference.orchestrator import _redact_worker_output++ for path in (+ "/home/o'connor/private/model.py",+ "/home/a(1)/private/model.py",+ "/srv/models/llama:8b/weights.gguf",+ "C:\\Users\\O'Brien\\models\\weights.gguf",+ "\\\\share\\team\\o'brien\\model.gguf",+ ):+ public = _redact_worker_output(f' File "{path}", line 1, in run\n')+ assert "private" not in public, public+ assert "o'connor" not in public.lower(), public+ assert "o'brien" not in public.lower(), public+ assert "llama:8b" not in public, public+ assert public.strip().startswith('File ".../'), public++ spaced = _redact_worker_output("C:\\Program Files\\unsloth\\weights.gguf failed\n")+ assert spaced.strip() == ".../weights.gguf failed", spaced+ two = _redact_worker_output(' File "/a/b.py", line 1, then /etc/passwd here\n')+ assert "/etc/passwd" not in two, two+ assert "line 1, then" in two, two+ assert _redact_worker_output("a ratio of 3/4 at https://host/path/x\n").strip() == (+ "a ratio of 3/4 at https://host/path/x"+ )+++MARK = " | "+START_MARK = "\x1f"+++def test_the_worker_marks_every_continuation_line_of_a_record():+ import logging+ from utils.worker_stderr import LOG_RECORD_CONTINUATION_PREFIX, mark_log_record_continuations++ assert LOG_RECORD_CONTINUATION_PREFIX == MARK++ logger_object = logging.getLogger("unsloth-test-marking")+ logger_object.handlers = []+ handler = logging.StreamHandler()+ handler.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))+ logger_object.addHandler(handler)++ # cover_later_handlers off so this case does not edit the interpreter's logging for+ # the rest of the session; the process-wide half is exercised on its own below.+ assert mark_log_record_continuations(logger_object, cover_later_handlers = False) == 1+ assert mark_log_record_continuations(logger_object, cover_later_handlers = False) == 0++ record = logger_object.makeRecord(+ "unsloth-test-marking",+ logging.ERROR,+ __file__,+ 1,+ "generated text: line one\nRuntimeError: not really\nTraceback (most recent call last):",+ (),+ None,+ )+ formatted = handler.formatter.format(record)+ lines = formatted.split("\n")+ assert lines[0].startswith(START_MARK + "ERROR unsloth-test-marking:"), lines+ assert all(line.startswith(MARK) for line in lines[1:]), lines++ single = logger_object.makeRecord(+ "unsloth-test-marking",+ logging.ERROR,+ __file__,+ 1,+ "one line",+ (),+ None,+ )+ one_line = handler.formatter.format(single)+ assert "\n" not in one_line+ # A single-line record through a default formatter is otherwise the same bytes a dying+ # runtime writes.+ assert one_line.startswith(START_MARK), repr(one_line)+++def test_a_default_formatter_record_is_not_mistaken_for_a_crash():+ logged = START_MARK + "RuntimeError: another account's prompt was rejected\n"+ assert _orchestrator_with_capture(logged)._public_worker_stderr_tail() == ""++ public = _orchestrator_with_capture(+ logged+ + "Fatal Python error: Aborted\n"+ + ' File "/home/alice/.unsloth/studio/worker.py", line 9, in handle\n'+ )._public_worker_stderr_tail()+ assert "another account" not in public, public+ assert "Fatal Python error: Aborted" in public, public+++def test_a_logged_traceback_is_not_returned_when_the_worker_dies_silently():+ logged = (+ "2026-09-16 10:00:03 worker: Generation error: another account's prompt was rejected\n"+ + MARK+ + "Traceback (most recent call last):\n"+ + MARK+ + ' File "/home/alice/.unsloth/studio/worker.py", line 9, in handle\n'+ + MARK+ + "ValueError: another account's prompt was rejected\n"+ )+ public = _orchestrator_with_capture(logged)._public_worker_stderr_tail()+ assert public == "", public+++def test_a_marked_continuation_that_reads_like_a_diagnostic_is_still_content():+ content = (+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text:\n"+ + MARK+ + "RuntimeError: another account's private prompt\n"+ + MARK+ + "Fatal Python error: also theirs\n"+ + MARK+ + "Killed\n"+ )+ public = _orchestrator_with_capture(content)._public_worker_stderr_tail()+ assert public == "", public+ assert "another account" not in public+++def test_a_crash_after_a_marked_record_is_still_the_crash():+ text = (+ "2026-09-16 10:00:01 audio_codecs.decode_bicodec: generated text:\n"+ + MARK+ + "another account's private prompt\n"+ + TRACEBACK+ )+ public = _orchestrator_with_capture(text)._public_worker_stderr_tail()+ assert "another account" not in public, public+ assert "RuntimeError: boom" in public, public+++def test_the_worker_installs_the_marking_where_its_logging_is_configured():+ import inspect+ from core.inference import worker as worker_module++ source = inspect.getsource(worker_module)+ assert "mark_log_record_continuations()" in source+ setup = source.index("LogConfig.setup_logging(")+ assert 0 < setup < source.index("mark_log_record_continuations()")+++def test_a_marked_record_under_a_diagnostic_is_not_adopted_by_it():+ text = (+ "terminate called after throwing an instance of 'c10::Error'\n"+ + MARK+ + "another account's private prompt\n"+ " what(): CUDA error: device-side assert triggered\n"+ )+ public = _orchestrator_with_capture(text)._public_worker_stderr_tail()+ assert "another account" not in public, public+ assert "terminate called" in public, public+ assert "CUDA error: device-side assert triggered" in public, public+++def test_a_live_replacements_own_crash_is_still_written_to_the_log(monkeypatch):+ import importlib.util as _ilu++ spec = _ilu.spec_from_file_location(+ "inference_orchestrator_stderr_log_once_under_test",+ Path(_BACKEND_DIR) / "core/inference/orchestrator.py",+ )+ module = _ilu.module_from_spec(spec)+ spec.loader.exec_module(module)++ written: "list[tuple]" = []+ monkeypatch.setattr(+ module.logger,+ "error",+ lambda *args, **kwargs: written.append(args),+ raising = False,+ )++ capture = _FixedCapture("Fatal Python error: Aborted\n")+ orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)+ orchestrator._stderr_capture = capture++ orchestrator._proc = SimpleNamespace(pid = 4242, exitcode = None)+ orchestrator._subprocess_crash_message("generation")+ assert len(written) == 1, written+ orchestrator._subprocess_crash_message("generation")+ assert len(written) == 1, written++ orchestrator._proc = SimpleNamespace(pid = 4242, exitcode = -6)+ orchestrator._subprocess_crash_message("generation", with_worker_output = True)+ assert len(written) == 2, "the replacement's own fatal output was skipped as already logged"+ orchestrator._subprocess_crash_message("generation", with_worker_output = True)+ assert len(written) == 2, written+++def test_a_worker_we_stopped_on_purpose_is_not_replayed_as_a_crash(monkeypatch):+ import importlib.util as _ilu++ spec = _ilu.spec_from_file_location(+ "inference_orchestrator_deliberate_stop_under_test",+ Path(_BACKEND_DIR) / "core/inference/orchestrator.py",+ )+ module = _ilu.module_from_spec(spec)+ spec.loader.exec_module(module)++ written: "list[tuple]" = []+ monkeypatch.setattr(+ module.logger,+ "error",+ lambda *args, **kwargs: written.append(args),+ raising = False,+ )++ orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)+ orchestrator._stderr_capture = _FixedCapture("loading shards: 100%\n")+ orchestrator._proc = None+ orchestrator._worker_stopped_deliberately = True+ orchestrator._retire_stderr_capture()+ assert written == [], written++ orchestrator._stderr_capture = _FixedCapture("Fatal Python error: Aborted\n")+ orchestrator._worker_stopped_deliberately = False+ orchestrator._retire_stderr_capture()+ assert written, "an unattended crash was skipped because a previous stop was deliberate"+++def test_a_worker_that_died_between_requests_is_replayed_before_its_sink_closes(monkeypatch):+ import importlib.util as _ilu++ spec = _ilu.spec_from_file_location(+ "inference_orchestrator_idle_retire_under_test",+ Path(_BACKEND_DIR) / "core/inference/orchestrator.py",+ )+ module = _ilu.module_from_spec(spec)+ spec.loader.exec_module(module)++ written: "list[tuple]" = []+ monkeypatch.setattr(+ module.logger,+ "error",+ lambda *args, **kwargs: written.append(args),+ raising = False,+ )++ closed: "list[bool]" = []++ class _ClosableCapture(_FixedCapture):+ def close(self):+ closed.append(True)++ orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)+ orchestrator._stderr_capture = _ClosableCapture("Fatal Python error: Segmentation fault\n")+ orchestrator._proc = SimpleNamespace(+ pid = 7331,+ exitcode = -11,+ is_alive = lambda: False,+ )++ orchestrator._retire_stderr_capture()+ assert closed == [True]+ assert written, "the capture of a worker that died unattended was closed unread"+ assert "Fatal Python error: Segmentation fault" in str(written[0])+ assert orchestrator._stderr_capture is None++ written.clear()+ orchestrator._stderr_capture = _ClosableCapture("loading shards: 40%\n")+ orchestrator._proc = SimpleNamespace(pid = 7332, exitcode = None, is_alive = lambda: True)+ orchestrator._retire_stderr_capture()+ assert written == [], written+++def test_a_request_queued_behind_the_crash_is_not_given_its_last_words(monkeypatch):+ capture = _FixedCapture(+ "Traceback (most recent call last):\n"+ ' File "/home/alice/.unsloth/studio/worker.py", line 42, in run\n'+ "RuntimeError: another account's prompt overflowed the context\n"+ )+ orchestrator = _orchestrator_with(-9, capture)+ logged: "list[str]" = []+ monkeypatch.setattr(+ type(orchestrator),+ "_log_worker_stderr_once",+ lambda self, pid, exitcode, **_kwargs: logged.append(str(pid)),+ raising = False,+ )++ executing = orchestrator._subprocess_crash_message("generation", with_worker_output = True)+ assert "Worker error output:" in executing+ assert "RuntimeError: another account" in executing++ orchestrator._stderr_tail_logged = None+ queued = orchestrator._subprocess_crash_message("generation", with_worker_output = False)+ assert "Worker error output:" not in queued, queued+ assert "another account" not in queued, queued+ assert "generating a response" in queued+ # `signal.Signals(9)` raises on Windows, so the message says SIG9 there.+ assert "signal=SIGKILL" in queued or "signal=SIG9" in queued, queued+ assert "exitcode=-9" in queued, queued+ assert logged, "the server log lost the tail for the queued request's path"+++def test_the_stream_asks_who_owned_the_worker_before_handing_over_the_tail():+ import inspect+ from core.inference import orchestrator as orchestrator_module++ body = inspect.getsource(orchestrator_module.InferenceOrchestrator._consume_token_stream)+ # The other crash exit, the swapped-worker branch, passes no tail at all.+ assert body.count("with_worker_output = self._owns_worker(cancel_event)") == 1, body+++def test_a_queued_compare_request_does_not_own_the_worker():+ module = _load_orchestrator_module()+ orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)+ import threading++ orchestrator._active_cancel_lock = threading.Lock()+ first, second = threading.Event(), threading.Event()+ orchestrator._active_cancel_events = []+ orchestrator._executing_cancel_events = []+ assert orchestrator._owns_worker(first) is True++ orchestrator._active_cancel_events = [first, second]+ orchestrator._executing_cancel_events = [first]+ assert orchestrator._owns_worker(first) is True+ assert orchestrator._owns_worker(second) is False+++def test_the_public_tail_redacts_the_credentials_a_crash_actually_carries():+ from core.inference.orchestrator import _redact_worker_output++ text = (+ "Traceback (most recent call last):\n"+ ' File "/home/alice/.unsloth/studio/worker.py", line 42, in run\n'+ " client = OpenAI() # OPENAI_API_KEY=sk-proj-abcdefghijklmnopqrstuvwxyz0123456789\n"+ " aws = 'AKIAIOSFODNN7EXAMPLE'\n"+ " gh = 'ghp_abcdefghijklmnopqrstuvwxyz0123456789'\n"+ " password=hunter2seventeen\n"+ "RuntimeError: refused\n"+ )+ public = _redact_worker_output(text)+ for secret in (+ "sk-proj-abcdefghijklmnopqrstuvwxyz0123456789",+ "AKIAIOSFODNN7EXAMPLE",+ "ghp_abcdefghijklmnopqrstuvwxyz0123456789",+ "hunter2seventeen",+ ):+ assert secret not in public, (secret, public)+ assert "RuntimeError: refused" in public, public+ assert "worker.py" in public, public+++def test_the_worker_output_is_opt_in_at_every_call_site():+ import inspect+ from core.inference import orchestrator as orchestrator_module++ source = inspect.getsource(orchestrator_module)+ assert "with_worker_output: bool = False" in source+ calls = source.count("self._subprocess_crash_message(")+ owned = source.count("with_worker_output = self._owns_worker(")+ # Every call site but one asks ownership; the exception is the swapped-worker branch.+ assert calls == owned + 1, (calls, owned)+++def test_a_stream_whose_worker_was_swapped_gets_no_tail_at_all():+ import inspect+ from core.inference import orchestrator as orchestrator_module++ body = inspect.getsource(orchestrator_module.InferenceOrchestrator._consume_token_stream)+ swap = body.index("initial_proc or self._resp_queue is not initial_resp_queue")+ following = body[swap : body.index("resp = read_one(read_timeout)", swap)]+ assert "_subprocess_crash_message(crash_context)" in following, following+ assert "with_worker_output" not in following, following+++def _logging_restored():+ import logging++ from utils import worker_stderr++ # getattr so a build without the hook lets the CASE report the gap, not this fixture.+ saved = (+ logging.Handler.setFormatter,+ logging.Logger.addHandler,+ getattr(logging.lastResort, "formatter", None),+ getattr(worker_stderr, "_UNHOOKED_SET_FORMATTER", None),+ getattr(worker_stderr, "_UNHOOKED_ADD_HANDLER", None),+ )+ try:+ yield+ finally:+ logging.Handler.setFormatter = saved[0]+ logging.Logger.addHandler = saved[1]+ logging.lastResort.formatter = saved[2]+ if hasattr(worker_stderr, "_UNHOOKED_SET_FORMATTER"):+ worker_stderr._UNHOOKED_SET_FORMATTER = saved[3]+ worker_stderr._UNHOOKED_ADD_HANDLER = saved[4]+++def test_the_marking_covers_the_last_resort_handler(_logging_restored):+ import logging++ from utils.worker_stderr import mark_log_record_continuations++ logging.lastResort.setFormatter(logging.Formatter("%(message)s"))+ mark_log_record_continuations(logging.getLogger("unsloth-test-lastresort"))++ record = logging.LogRecord(+ "unsloth-test-lastresort", logging.ERROR, __file__, 1, "one\ntwo\nthree", (), None+ )+ lines = logging.lastResort.format(record).split("\n")+ assert lines[0] == START_MARK + "one", lines+ assert all(line.startswith(MARK) for line in lines[1:]), lines+++def test_a_handler_installed_after_startup_is_marked_too(_logging_restored):+ import logging++ from utils.worker_stderr import mark_log_record_continuations++ mark_log_record_continuations(logging.getLogger("unsloth-test-late"))++ late = logging.getLogger("unsloth-test-late-library")+ late.handlers = []+ handler = logging.StreamHandler()+ handler.setFormatter(logging.Formatter("%(message)s"))+ late.addHandler(handler)++ record = logging.LogRecord(+ "unsloth-test-late-library", logging.ERROR, __file__, 1, "one\ntwo", (), None+ )+ assert handler.format(record).split("\n")[1].startswith(MARK), handler.format(record)++ handler.setFormatter(logging.Formatter("%(levelname)s %(message)s"))+ formatted = handler.format(record)+ assert formatted.split("\n")[0] == START_MARK + "ERROR one", formatted+ assert formatted.split("\n")[1].startswith(MARK), formatted++ # Clearing the formatter is the one case the hook cannot wrap; it must not crash.+ handler.setFormatter(None)+ assert handler.formatter is None+++def test_a_root_level_path_is_redacted_too():+ from core.inference.orchestrator import _redact_worker_output+ for path, tail in (+ ("/model.gguf", "model.gguf"),+ ("C:\\model.gguf", "model.gguf"),+ ("\\\\fileserver\\share", "share"),+ ("/opt", "opt"),+ ):+ public = _redact_worker_output(f"could not open {path}\n")+ assert public.strip() == f"could not open .../{tail}", public+ assert not public.count("C:"), public++ for text in (+ 'the "/" separator is not a path',+ "a ratio of 3/4 at https://host/path/x",+ "use / to split and/or join",+ ):+ assert _redact_worker_output(text + "\n").strip() == text, text+++def test_two_paths_on_one_line_are_two_matches():+ from core.inference.orchestrator import _redact_worker_output++ windows = _redact_worker_output("copy C:\\Users\\ann\\old.gguf to C:\\tmp\\new.gguf\n")+ assert windows.strip() == "copy .../old.gguf to .../new.gguf", windows+ posix = _redact_worker_output("cannot copy /home/ann/old.gguf to /tmp/new.gguf\n")+ assert posix.strip() == "cannot copy .../old.gguf to .../new.gguf", posix+ unc = _redact_worker_output("copy \\\\share\\team\\old.gguf to \\\\other\\team\\new.gguf\n")+ assert unc.strip() == "copy .../old.gguf to .../new.gguf", unc++ spaced = _redact_worker_output("C:\\Program Files\\unsloth\\weights.gguf failed\n")+ assert spaced.strip() == ".../weights.gguf failed", spaceddiff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.pyindex be511315723..33ffd7c6f3a 100644--- a/studio/backend/tests/test_orchestrator_unload_cancel.py+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py@@ -231,7 +231,7 @@ def test_consume_token_stream_bails_when_subprocess_swapped(monkeypatch):o = _bare_orchestrator()monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True)monkeypatch.setattr(- o, "_subprocess_crash_message", lambda ctx: "inference subprocess restarted"+ o, "_subprocess_crash_message", lambda ctx, **kwargs: "inference subprocess restarted")def read_one(timeout):diff --git a/studio/backend/utils/native_path_leases.py b/studio/backend/utils/native_path_leases.pyindex 15ed307ca7a..0b8f6e645c0 100644--- a/studio/backend/utils/native_path_leases.py+++ b/studio/backend/utils/native_path_leases.py@@ -28,6 +28,8 @@ from pathlib import Pathfrom typing import Any, Callable, Collection, Iterable, Iterator, MappingLEASE_SECRET_ENV = "UNSLOTH_STUDIO_NATIVE_PATH_LEASE_SECRET"+# Spelled out, not imported from utils/worker_stderr.py, to keep this module's import graph stdlib only.+STDERR_MIRROR_KWARG = "unsloth_stderr_mirror_path"_MAX_NATIVE_PATH_REDACTIONS = 100_MAX_NATIVE_PATH_LABELS = 10_000_MIN_LEASE_SECRET_BYTES = 32@@ -119,6 +121,15 @@ def run_without_native_path_secret() -> Any:"""Run a multiprocessing child target without the native path lease secret."""+ # First, before anything else can raise, so a child dying before its entrypoint is explainable (#7843).+ _stderr_mirror_path = kwargs.pop(STDERR_MIRROR_KWARG, None)+ if _stderr_mirror_path:+ try:+ from utils.worker_stderr import install_worker_stderr_mirror+ install_worker_stderr_mirror(_stderr_mirror_path)+ except Exception:+ pass+# Runs in the spawned child to bind it to the parent's death, since multiprocessing children get no# preexec_fn. Shared entrypoint for the inference/export/training/data-recipe workers. Two try# blocks, because allow_child_processes is the newer name: on an older process_lifetime.py adiff --git a/studio/backend/utils/worker_stderr.py b/studio/backend/utils/worker_stderr.pynew file mode 100644index 00000000000..f4a02295664--- /dev/null+++ b/studio/backend/utils/worker_stderr.py@@ -0,0 +1,559 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""#7843: a spawn child's traceback goes down the inherited stderr, not the response queue. Best effort throughout."""++from __future__ import annotations++import atexit+import collections+import logging+import os+import sys+import tempfile+import threading++__all__ = [+ "LOG_RECORD_CONTINUATION_PREFIX",+ "LOG_RECORD_START_MARK",+ "STDERR_MIRROR_KWARG",+ "WorkerStderrCapture",+ "decode_worker_stderr",+ "install_worker_stderr_mirror",+ "mark_log_record_continuations",+ "stderr_tail_from_bytes",+]++# Marks a record's continuation lines: a recovered request's `exc_info` traceback is byte-identical to a dying process's.+LOG_RECORD_CONTINUATION_PREFIX = " | "++# And its first line. UNIT SEPARATOR so it stays invisible in the operator's mirrored copy.+LOG_RECORD_START_MARK = "\x1f"++# A kwarg, not an environment variable: a process-wide value would cross workers spawning at once.+STDERR_MIRROR_KWARG = "unsloth_stderr_mirror_path"++DEFAULT_TAIL_LINES = 20+DEFAULT_TAIL_CHARS = 4000++MIRROR_FILE_CAP_BYTES = 256 * 1024++TAIL_READ_BYTES = 64 * 1024++_PUMP_JOIN_TIMEOUT_S = 2.0+++def _utf8_edge_bounds(data: bytes, *, trim_end: bool = True) -> "tuple[int, int]":+ start = 0+ while start < len(data) and start < 3 and 0x80 <= data[start] < 0xC0:+ start += 1+ end = len(data)+ if not trim_end:+ return start, end+ for back in range(1, min(4, end - start) + 1):+ byte = data[end - back]+ if byte < 0x80:+ break+ if byte >= 0xC0:+ width = 2 if byte < 0xE0 else 3 if byte < 0xF0 else 4+ if back < width:+ end -= back+ break+ return start, end+++def _without_partial_utf8_edges(data: bytes, *, trim_end: bool = True) -> bytes:+ """``trim_end`` is off at EOF, where a trailing 0xC0+ byte is a complete cp1252 character, not a severed one."""+ start, end = _utf8_edge_bounds(data, trim_end = trim_end)+ return data[start:end]+++def decode_worker_stderr(data: bytes, *, ends_at_eof: bool = True) -> str:+ """UTF-8 before cp1252: cp1252 has a meaning for almost every byte, so the stricter encoding goes first."""+ text: str | None = None+ try:+ text = data.decode("utf-8")+ except UnicodeDecodeError:+ text = None+ if text is None:+ start, end = _utf8_edge_bounds(data)+ try:+ text = data[start:end].decode("utf-8")+ except UnicodeDecodeError:+ text = None+ else:+ # cp1252 costs at worst one spurious character; discarding loses crash detail.+ tail_bytes = data[end:] if ends_at_eof else b""+ if tail_bytes:+ try:+ text += tail_bytes.decode("cp1252")+ except (UnicodeDecodeError, LookupError):+ pass+ if text is None:+ try:+ text = data.decode("cp1252")+ except (UnicodeDecodeError, LookupError):+ text = data.decode("utf-8", errors = "replace")+ return text.replace("\r\n", "\n").replace("\r", "\n")+++def stderr_tail_from_bytes(+ data: bytes,+ max_lines: int = DEFAULT_TAIL_LINES,+ max_chars: int = DEFAULT_TAIL_CHARS,+ *,+ ends_at_eof: bool = True,+) -> str:+ text = decode_worker_stderr(data, ends_at_eof = ends_at_eof)+ lines = [line.rstrip() for line in text.split("\n")]+ lines = [line for line in lines if line.strip()]+ if not lines:+ return ""+ if max_lines > 0:+ lines = lines[-max_lines:]+ joined = "\n".join(lines)+ if max_chars > 0 and len(joined) > max_chars:+ joined = joined[-max_chars:]+ first_break = joined.find("\n")+ if first_break != -1:+ joined = joined[first_break + 1 :]+ return joined+++# Exact paths, never a pattern: Studios share one temporary directory.+_OPEN_SINKS: "set[str]" = set()+_ATEXIT_REGISTERED = False+# A fork inherits the set and the atexit registration, so the handler must check these paths are its own.+_SINKS_OWNER_PID: "int | None" = None+++def _unlink_quietly(path: str) -> None:+ try:+ os.unlink(path)+ except OSError:+ pass+++def _remove_open_sinks() -> None:+ if os.getpid() != _SINKS_OWNER_PID:+ # An inherited handler in a forked child: these paths are the parent's.+ return+ for path in list(_OPEN_SINKS):+ _unlink_quietly(path)+ _OPEN_SINKS.clear()+++class WorkerStderrCapture:+ def __init__(+ self,+ directory: "str | None" = None,+ prefix: str = "unsloth-worker-",+ ) -> None:+ global _ATEXIT_REGISTERED, _SINKS_OWNER_PID+ handle, self._path = tempfile.mkstemp(+ prefix = prefix,+ suffix = ".stderr",+ dir = directory,+ )+ os.close(handle)+ _OPEN_SINKS.add(self._path)+ _SINKS_OWNER_PID = os.getpid()+ if not _ATEXIT_REGISTERED:+ atexit.register(_remove_open_sinks)+ _ATEXIT_REGISTERED = True++ @property+ def path(self) -> str:+ return self._path++ def tail(+ self,+ max_lines: int = DEFAULT_TAIL_LINES,+ max_chars: int = DEFAULT_TAIL_CHARS,+ ) -> str:+ try:+ # O_NOFOLLOW: a tail is never worth following a symlink in a shared tmpdir for.+ fd = os.open(self._path, os.O_RDONLY | _O_NOFOLLOW | _O_BINARY)+ with os.fdopen(fd, "rb") as handle:+ handle.seek(0, os.SEEK_END)+ size = handle.tell()+ handle.seek(max(0, size - TAIL_READ_BYTES))+ data = handle.read()+ except OSError:+ return ""+ return stderr_tail_from_bytes(data, max_lines = max_lines, max_chars = max_chars)++ def close(self) -> None:+ """Tolerates a child still holding the sink open, which is the norm on Windows."""+ _OPEN_SINKS.discard(self._path)+ _unlink_quietly(self._path)+++# O_NOFOLLOW is POSIX only and O_BINARY is Windows only; both are absent-means-zero here.+_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)+_O_BINARY = getattr(os, "O_BINARY", 0)+++def _open_existing_sink(path: str):+ """``open(path, "wb")`` is WRONG here: world-readable under the default umask, and follows a planted symlink."""+ flags = os.O_RDWR | _O_NOFOLLOW | _O_BINARY+ try:+ handle = os.open(path, flags)+ except OSError:+ return None+ try:+ return os.fdopen(handle, "r+b", buffering = 0)+ except OSError:+ try:+ os.close(handle)+ except OSError:+ pass+ return None+++_TAIL_POLL_S = 0.05++# 64 x 64 KiB: a stalled operator stderr costs 4 MiB of buffer before the mirror starts dropping.+_MIRROR_RELAY_CHUNKS = 64+++def _open_sink_for_append(path: str):+ flags = os.O_WRONLY | os.O_APPEND | _O_NOFOLLOW | _O_BINARY+ try:+ return os.open(path, flags)+ except OSError:+ return None+++def _open_sink_for_reading(path: str):+ flags = os.O_RDONLY | _O_NOFOLLOW | _O_BINARY+ try:+ handle = os.open(path, flags)+ except OSError:+ return None+ try:+ return os.fdopen(handle, "rb", buffering = 0)+ except OSError:+ try:+ os.close(handle)+ except OSError:+ pass+ return None+++_COMPACT_CATCH_UP_ROUNDS = 8+++def _compact_sink(+ sink,+ cap_bytes: int,+ reader = None,+ emit = None,+) -> int:+ """Rewrite *sink* to roughly its last *cap_bytes* bytes.++ fd 2 is ``O_APPEND`` on it, so a plain read-rewrite-truncate loses concurrent appends.+ """+ if reader is not None and emit is not None:+ while True:+ try:+ pending = reader.read(65536)+ except (OSError, ValueError):+ break+ if not pending:+ break+ emit(pending)+ size = sink.seek(0, os.SEEK_END)+ keep = min(size, cap_bytes)+ sink.seek(size - keep)+ data = sink.read(keep)+ # Everything past `total` is an append that arrived after this function started looking.+ total = size+ for _ in range(_COMPACT_CATCH_UP_ROUNDS):+ appended = sink.read()+ if not appended:+ break+ if emit is not None:+ emit(appended)+ data += appended+ total += len(appended)+ sink.seek(0)+ sink.write(data)+ end = len(data)+ # A native thread can write into the region about to be truncated during the rewrite.+ for _ in range(_COMPACT_CATCH_UP_ROUNDS):+ sink.seek(total)+ late = sink.read()+ if not late:+ break+ if emit is not None:+ emit(late)+ total += len(late)+ sink.seek(end)+ sink.write(late)+ end += len(late)+ sink.truncate(end)+ return end+++class _MirrorRelay:+ """Owns the only blocking write, so the thread that bounds the sink never parks in one.++ The operator's stderr is a pipe in the packaged app, and a reader that stops draining it+ would otherwise stall the pump inside ``os.write`` and leave the sink growing without limit.+ The mirrored copy is best effort, so a full buffer drops its oldest chunk; the sink keeps+ the tail, which is the crash record the parent reports.+ """++ def __init__(self, inherited_fd: int, max_chunks: int) -> None:+ self._fd = inherited_fd+ self._chunks = collections.deque(maxlen = max_chunks)+ self._wake = threading.Event()+ self._closed = False+ self._thread = threading.Thread(+ target = self._run,+ name = "unsloth-worker-stderr-relay",+ daemon = True,+ )+ self._thread.start()++ def emit(self, data: bytes) -> None:+ if not data:+ return+ self._chunks.append(data)+ self._wake.set()++ def close(self) -> None:+ self._closed = True+ self._wake.set()++ def join(self, timeout: "float | None" = None) -> None:+ self._thread.join(timeout = timeout)++ def is_alive(self) -> bool:+ return self._thread.is_alive()++ def _run(self) -> None:+ while True:+ try:+ data = self._chunks.popleft()+ except IndexError:+ if self._closed:+ return+ self._wake.wait(_TAIL_POLL_S)+ self._wake.clear()+ continue+ try:+ os.write(self._fd, data)+ except OSError:+ pass+++def _sink_size(sink) -> int:+ try:+ return os.fstat(sink.fileno()).st_size+ except (OSError, ValueError):+ return 0+++def _tail_sink_to_stderr(+ reader, relay: "_MirrorRelay", sink, cap_bytes: int, stop: "threading.Event"+) -> None:+ """fd 2 must be the SINK, not a pipe: a fatal signal kills anything buffered in a pipe with its writer."""+ while True:+ try:+ chunk = reader.read(65536)+ except (OSError, ValueError):+ break+ if not chunk:+ if stop.is_set():+ break+ stop.wait(_TAIL_POLL_S)+ continue+ relay.emit(chunk)+ # Bytes ON DISK, not bytes relayed: the relay can fall arbitrarily far behind.+ if cap_bytes > 0 and _sink_size(sink) > 2 * cap_bytes:+ try:+ _compact_sink(sink, cap_bytes, reader = reader, emit = relay.emit)+ # The file was rewritten from the front; the old offset is now meaningless.+ reader.seek(0, os.SEEK_END)+ except (OSError, ValueError):+ pass+ try:+ sink.close()+ except OSError:+ pass+ try:+ reader.close()+ except (OSError, ValueError):+ pass+++def _stop_mirror(+ inherited_fd: int,+ pump: threading.Thread,+ stop = None,+ relay = None,+) -> None:+ """The pump must be a daemon or ``BaseProcess._bootstrap`` waits on it for ever.++ ``multiprocessing`` prints the traceback before atexit, so draining here still catches it.+ """+ try:+ sys.stderr.flush()+ except Exception:+ pass+ try:+ os.dup2(inherited_fd, 2)+ except OSError:+ pass+ if stop is not None:+ # fd 2 no longer points at the sink, so the next empty read is the end, not a pause.+ stop.set()+ pump.join(timeout = _PUMP_JOIN_TIMEOUT_S)+ if relay is not None:+ relay.close()+ relay.join(timeout = _PUMP_JOIN_TIMEOUT_S)+ if pump.is_alive() or (relay is not None and relay.is_alive()):+ # A thread may be inside os.write(inherited_fd, ...); closing frees the number for any thread's next open().+ return+ try:+ os.close(inherited_fd)+ except OSError:+ pass+++def install_worker_stderr_mirror(+ path: "str | None", cap_bytes: int = MIRROR_FILE_CAP_BYTES+) -> bool:+ """At the descriptor level: a worker dying inside a C extension writes fd 2 directly and bypasses a Python wrapper."""+ if not path:+ return False+ sink = _open_existing_sink(path)+ if sink is None:+ return False+ try:+ inherited = os.dup(2)+ except OSError:+ sink.close()+ return False+ # O_APPEND: compaction rewrites from the front and two handles share the file.+ writer_fd = _open_sink_for_append(path)+ if writer_fd is None:+ os.close(inherited)+ sink.close()+ return False+ reader = _open_sink_for_reading(path)+ if reader is None:+ os.close(writer_fd)+ os.close(inherited)+ sink.close()+ return False+ try:+ sys.stderr.flush()+ except Exception:+ pass+ try:+ reader.seek(0, os.SEEK_END)+ except (OSError, ValueError):+ pass+ try:+ os.dup2(writer_fd, 2)+ except OSError:+ os.close(writer_fd)+ reader.close()+ os.close(inherited)+ sink.close()+ return False+ os.close(writer_fd)+ stop = threading.Event()+ relay = _MirrorRelay(inherited, _MIRROR_RELAY_CHUNKS)+ pump = threading.Thread(+ target = _tail_sink_to_stderr,+ args = (reader, relay, sink, cap_bytes, stop),+ name = "unsloth-worker-stderr-mirror",+ daemon = True,+ )+ pump.start()+ atexit.register(_stop_mirror, inherited, pump, stop, relay)+ return True+++class _EveryLineCarriesThePrefix(logging.Formatter):+ def __init__(self, inner: "logging.Formatter") -> None:+ super().__init__()+ self._inner = inner++ def format(self, record: "logging.LogRecord") -> str:+ text = self._inner.format(record)+ first, newline, rest = text.partition("\n")+ # The first line too: a default-formatted single-line record has no shape to spot.+ marked_first = (+ first if first.startswith(LOG_RECORD_START_MARK) else LOG_RECORD_START_MARK + first+ )+ if not newline:+ return marked_first+ return (+ marked_first+ + "\n"+ + "\n".join(LOG_RECORD_CONTINUATION_PREFIX + line for line in rest.split("\n"))+ )++ def __getattr__(self, name: str):+ return getattr(self._inner, name)+++_UNHOOKED_SET_FORMATTER = None+_UNHOOKED_ADD_HANDLER = None+++def _mark_handler(handler) -> bool:+ formatter = getattr(handler, "formatter", None)+ if isinstance(formatter, _EveryLineCarriesThePrefix):+ return False+ setter = _UNHOOKED_SET_FORMATTER or type(handler).setFormatter+ setter(handler, _EveryLineCarriesThePrefix(formatter or logging.Formatter()))+ return True+++def _install_continuation_hook() -> bool:+ """Patched on the CLASS: the ML stack is imported after logging is configured and adds its own handlers."""+ global _UNHOOKED_SET_FORMATTER, _UNHOOKED_ADD_HANDLER+ if _UNHOOKED_SET_FORMATTER is not None:+ return False+ unhooked_set = logging.Handler.setFormatter+ unhooked_add = logging.Logger.addHandler++ def setFormatter(self, fmt): # noqa: N802 -- matches logging's own spelling+ if fmt is not None and not isinstance(fmt, _EveryLineCarriesThePrefix):+ fmt = _EveryLineCarriesThePrefix(fmt)+ unhooked_set(self, fmt)++ def addHandler(self, hdlr): # noqa: N802 -- matches logging's own spelling+ unhooked_add(self, hdlr)+ try:+ _mark_handler(hdlr)+ except Exception: # noqa: BLE001 -- marking must never break someone's logging+ pass++ _UNHOOKED_SET_FORMATTER = unhooked_set+ _UNHOOKED_ADD_HANDLER = unhooked_add+ logging.Handler.setFormatter = setFormatter+ logging.Logger.addHandler = addHandler+ return True+++def mark_log_record_continuations(logger_object = None, *, cover_later_handlers = True) -> int:+ """Covers ``logging.lastResort`` and later handlers too: a miss hands a RECOVERED+ request's traceback to the next caller as their crash."""+ root = logger_object if logger_object is not None else logging.getLogger()+ wrapped = 0+ for handler in list(getattr(root, "handlers", ())):+ if _mark_handler(handler):+ wrapped += 1+ if cover_later_handlers:+ last_resort = getattr(logging, "lastResort", None)+ if last_resort is not None and _mark_handler(last_resort):+ wrapped += 1+ _install_continuation_hook()+ return wrappeddiff --git a/studio/frontend/src/features/auth/api.ts b/studio/frontend/src/features/auth/api.tsindex c654d7941af..a0899389302 100644--- a/studio/frontend/src/features/auth/api.ts+++ b/studio/frontend/src/features/auth/api.ts@@ -2,7 +2,7 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import { accountTransitionPending } from "@/lib/account-transition";-import { apiUrl, isTauri } from "@/lib/api-base";+import { apiUrl, getApiPort, isTauri } from "@/lib/api-base";import {clearAuthTokens,getAuthToken,@@ -29,10 +29,30 @@ let refreshInflight: Promise<boolean> | null = null;let refreshInflightToken: string | null = null;let logoutGeneration = 0;-const TAURI_FETCH_RETRY_DELAYS_MS = [250, 750, 1500] as const;+// #10520: sums to 10.5s, just past the launcher's 10s HEALTH_PROBE_TIMEOUT. Guarded by+// `the_frontend_retry_ladder_outlives_one_probe_budget` in src-tauri/src/commands.rs.+const TAURI_FETCH_RETRY_DELAYS_MS = [250, 750, 1500, 3000, 5000] as const;+// A network error is not an answer: retrying a committed POST creates a second API key,+// project or job, so non-idempotent methods keep the shorter ladder.+const TAURI_FETCH_RETRY_DELAYS_UNSAFE_MS = [250, 750, 1500] as const;+const IDEMPOTENT_METHODS = new Set(["GET", "HEAD", "OPTIONS", "PUT", "DELETE"]);++function retryDelaysFor(+ input: RequestInfo | URL,+ init?: RequestInit,+): readonly number[] {+ const method = (+ init?.method ??+ (typeof Request !== "undefined" && input instanceof Request+ ? input.method+ : "GET")+ ).toUpperCase();+ return IDEMPOTENT_METHODS.has(method)+ ? TAURI_FETCH_RETRY_DELAYS_MS+ : TAURI_FETCH_RETRY_DELAYS_UNSAFE_MS;+}const BROWSER_TIMEZONE_HEADER = "X-Unsloth-Timezone";-const BROWSER_TIMEZONE_OFFSET_HEADER =- "X-Unsloth-Timezone-Offset-Minutes";+const BROWSER_TIMEZONE_OFFSET_HEADER = "X-Unsloth-Timezone-Offset-Minutes";function addBrowserTimezoneHeaders(headers: Headers): void {try {@@ -61,6 +81,7 @@ async function fetchWithTauriNetworkRetry(retryNetworkErrors = true,beforeRetry?: () => void,): Promise<Response> {+ const delays = retryDelaysFor(input, init);for (let attempt = 0; ; attempt++) {try {return await fetch(input, init);@@ -69,11 +90,11 @@ async function fetchWithTauriNetworkRetry(!isTauri ||!retryNetworkErrors ||!(error instanceof TypeError) ||- attempt >= TAURI_FETCH_RETRY_DELAYS_MS.length+ attempt >= delays.length) {throw error;}- await wait(TAURI_FETCH_RETRY_DELAYS_MS[attempt]);+ await wait(delays[attempt]);beforeRetry?.();}}@@ -105,9 +126,10 @@ async function redirectToAuth(passwordChangeRequired = false): Promise<void> {login_mode?: "single" | "multi";};// Public status describes the owner. A managed session carries its own requirement.- const requiresChange = data.login_mode === "multi"- ? passwordChangeRequired || mustChangePassword()- : data.requires_password_change;+ const requiresChange =+ data.login_mode === "multi"+ ? passwordChangeRequired || mustChangePassword()+ : data.requires_password_change;if (requiresChange !== mustChangePassword()) {setMustChangePassword(requiresChange);}@@ -124,9 +146,54 @@ async function redirectToAuth(passwordChangeRequired = false): Promise<void> {window.location.href = target;}-function asTransportFailure(err: unknown): unknown {+/** Copy shown when the backend really is unreachable and the launcher agrees. */+export const BACKEND_NOT_RUNNING_MESSAGE =+ "Unsloth isn't running -- please relaunch it.";+/** Copy shown when the webview could not reach the backend but the launcher says it is up. */+export const BACKEND_NOT_ANSWERING_MESSAGE =+ "Unsloth is running but did not answer in time. It may still be starting up. Please try again in a moment.";++/** `check_backend_present` and NOT `check_health`: the latter reports a probe that ran out of budget exactly as a refused connection. */+// The port is carried WITH the promise: `setApiBase` can move it inside the 10s budget.+let nativeHealthInflight: { port: number; probe: Promise<boolean> } | null = null;++async function nativeBackendIsAlive(): Promise<boolean> {+ if (!isTauri) {+ return false;+ }+ const port = getApiPort();+ if (port === null) {+ return false;+ }+ // Single flight: one 10s probe rather than one per panel. Not cached beyond the call.+ if (nativeHealthInflight !== null && nativeHealthInflight.port === port) {+ return nativeHealthInflight.probe;+ }+ const probe = (async () => {+ try {+ const { invoke } = await import("@tauri-apps/api/core");+ return (+ (await invoke<boolean>("check_backend_present", { port })) === true+ );+ } catch {+ return false;+ }+ })();+ const inflight = { port, probe };+ nativeHealthInflight = inflight;+ try {+ return await probe;+ } finally {+ // Identity, not the port: a probe for a newer port owns the slot now.+ if (nativeHealthInflight === inflight) {+ nativeHealthInflight = null;+ }+ }+}++async function asTransportFailure(err: unknown): Promise<unknown> {// fetch TypeError = offline | backend down | CORS/DNS. Tagged so callers tell "never reached"- // from "rejected"; Tauri is always backend-down, the web build distinguishes offline.+ // from "rejected"; under Tauri the launcher is asked before claiming the backend is gone.if (!(err instanceof TypeError)) return err;if (!isTauri &&@@ -140,10 +207,17 @@ function asTransportFailure(err: unknown): unknown {{ unslothTransportFailure: true },);}- return Object.assign(- new Error("Unsloth isn't running -- please relaunch it."),- { unslothTransportFailure: true },- );+ // A failed fetch in the webview is not proof the backend died, and "please relaunch it"+ // throws away a running backend and whatever it has in flight.+ if (await nativeBackendIsAlive()) {+ return Object.assign(new Error(BACKEND_NOT_ANSWERING_MESSAGE), {+ unslothTransportFailure: true,+ unslothBackendStillRunning: true,+ });+ }+ return Object.assign(new Error(BACKEND_NOT_RUNNING_MESSAGE), {+ unslothTransportFailure: true,+ });}async function retryWithCurrentToken(@@ -166,7 +240,7 @@ async function retryWithCurrentToken(beforeRetry,);} catch (err) {- throw asTransportFailure(err);+ throw await asTransportFailure(err);}}@@ -257,7 +331,7 @@ export async function authFetch(options?.beforeRetry,);} catch (err) {- throw asTransportFailure(err);+ throw await asTransportFailure(err);}if (await isPasswordChangeRequiredResponse(response)) {diff --git a/studio/frontend/src/features/chat/utils/chat-history-storage.ts b/studio/frontend/src/features/chat/utils/chat-history-storage.tsindex 077f1c68d60..ac71909b00b 100644--- a/studio/frontend/src/features/chat/utils/chat-history-storage.ts+++ b/studio/frontend/src/features/chat/utils/chat-history-storage.ts@@ -250,12 +250,20 @@ function sortMessages(messages: MessageRecord[]): MessageRecord[] {}export function isExpectedBackgroundChatStorageError(error: unknown): boolean {+ // The transport marker rather than the copy, which is how a merely busy backend came to+ // be reported as an unexpected error when the copy changed.+ if (+ error instanceof Error &&+ (error as { unslothTransportFailure?: boolean }).unslothTransportFailure ===+ true+ ) {+ return true;+ }return (error instanceof Error &&(error.message === "Invalid or expired token" ||error.message === "Not authenticated" ||- error.message === "Request failed (401)" ||- error.message === "Unsloth isn't running -- please relaunch it.")+ error.message === "Request failed (401)"));}diff --git a/studio/frontend/src/lib/api-base.ts b/studio/frontend/src/lib/api-base.tsindex 33efafdf543..0483b8ee9cd 100644--- a/studio/frontend/src/lib/api-base.ts+++ b/studio/frontend/src/lib/api-base.ts@@ -20,6 +20,8 @@ if (isTauri) {const initialApiBase = apiBase+const LOOPBACK_BASE_PORT = /^https?:\/\/127\.0\.0\.1:(\d+)$/+export function resetApiBase() {apiBase = initialApiBase}@@ -32,6 +34,19 @@ export function getApiBase(): string {return apiBase}+/**+ * The port the backend is currently expected on, or null when none is known yet. The+ * placeholder base above is port 0, which reads as "no port yet".+ */+export function getApiPort(): number | null {+ const match = LOOPBACK_BASE_PORT.exec(apiBase)+ if (!match) {+ return null+ }+ const port = Number(match[1])+ return Number.isInteger(port) && port > 0 && port <= 65535 ? port : null+}+export function apiUrl(path: string): string {if (path.startsWith('http')) return pathreturn `${apiBase}${path}`diff --git a/studio/frontend/tests/tauri-backend-transport-failure.test.ts b/studio/frontend/tests/tauri-backend-transport-failure.test.tsnew file mode 100644index 00000000000..be5cca65b99--- /dev/null+++ b/studio/frontend/tests/tauri-backend-transport-failure.test.ts@@ -0,0 +1,443 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// #10520: a firewall filter delayed the loopback handshake and Desktop said "Unsloth isn't running" about a healthy backend.++import assert from "node:assert/strict";+import { readFile } from "node:fs/promises";+import { createServer, type Server } from "node:http";+import test from "node:test";++import { loadWithStubs } from "./helpers/module-stubs.ts";++type AuthApi = {+ authFetch: (+ input: string,+ init?: RequestInit,+ options?: { retryNetworkErrors?: boolean },+ ) => Promise<Response>;+ BACKEND_NOT_RUNNING_MESSAGE: string;+ BACKEND_NOT_ANSWERING_MESSAGE: string;+};++function loadAuthApi(options: {+ port: number | null;+ getPort?: () => number | null;+ checkHealth?: (port: number) => boolean | Promise<boolean>;+ onInvoke?: (command: string, args: Record<string, unknown>) => void;+}): AuthApi {+ const currentPort = options.getPort ?? (() => options.port);+ return loadWithStubs<AuthApi>(+ new URL("../src/features/auth/api.ts", import.meta.url),+ {+ "@/lib/api-base": {+ apiUrl: (path: string) => {+ const port = currentPort();+ return port === null+ ? `http://127.0.0.1:0${path}`+ : `http://127.0.0.1:${port}${path}`;+ },+ getApiPort: () => currentPort(),+ isTauri: true,+ },+ "@/lib/account-transition": { accountTransitionPending: () => false },+ "./session": {+ clearAuthTokens: () => {},+ getAuthToken: () => "access-token",+ getRefreshToken: () => null,+ mustChangePassword: () => false,+ setMustChangePassword: () => {},+ storeAuthTokens: () => {},+ },+ "@tauri-apps/api/core": {+ invoke: async (command: string, args: Record<string, unknown>) => {+ options.onInvoke?.(command, args);+ if (command !== "check_backend_present") {+ throw new Error(`unexpected command ${command}`);+ }+ return options.checkHealth?.(args.port as number) ?? false;+ },+ },+ },+ );+}++async function reserveLoopbackPort(): Promise<number> {+ const probe = createServer();+ const port = await new Promise<number>((resolve, reject) => {+ probe.once("error", reject);+ probe.listen(0, "127.0.0.1", () => {+ const address = probe.address();+ if (typeof address === "string" || address === null) {+ reject(new Error("no loopback address"));+ return;+ }+ resolve(address.port);+ });+ });+ await new Promise<void>((resolve) => probe.close(() => resolve()));+ return port;+}++function listenAfter(server: Server, port: number, delayMs: number): void {+ setTimeout(() => {+ server.listen(port, "127.0.0.1");+ }, delayMs).unref();+}++// The host below starts accepting at 3s: inside the new ladder, outside the old 250 + 750 + 1500ms one.+const SLOW_LOOPBACK_ACCEPT_DELAY_MS = 3_000;+const OLD_LADDER_ATTEMPTS = 4;++test("a backend that accepts late is reached instead of declared not running", async () => {+ const port = await reserveLoopbackPort();+ const server = createServer((_request, response) => {+ response.writeHead(200, { "Content-Type": "application/json" });+ response.end(JSON.stringify({ ok: true }));+ });+ const originalFetch = globalThis.fetch;+ let attempts = 0;+ globalThis.fetch = async (input, init) => {+ attempts += 1;+ return await originalFetch(input, init);+ };++ try {+ listenAfter(server, port, SLOW_LOOPBACK_ACCEPT_DELAY_MS);+ const authApi = loadAuthApi({ port });+ const response = await authApi.authFetch("/api/models");++ assert.equal(response.status, 200);+ assert.ok(+ attempts > OLD_LADDER_ATTEMPTS,+ `the backend was only reached on attempt ${attempts}; a ladder of ` ++ `${OLD_LADDER_ATTEMPTS} attempts would have declared it not running`,+ );+ } finally {+ globalThis.fetch = originalFetch;+ await new Promise<void>((resolve) => server.close(() => resolve()));+ }+});++test("a backend the launcher still sees is not reported as not running", async () => {+ const port = 61799;+ const probed: Array<Record<string, unknown>> = [];+ const originalFetch = globalThis.fetch;+ globalThis.fetch = async () => {+ throw new TypeError("fetch failed");+ };++ try {+ const authApi = loadAuthApi({+ port,+ checkHealth: () => true,+ onInvoke: (_command, args) => probed.push(args),+ });+ const error = await authApi+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })+ .then(+ () => null,+ (rejection: unknown) => rejection,+ );++ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_ANSWERING_MESSAGE);+ assert.notEqual(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);+ assert.ok(!/relaunch/i.test(error.message));+ assert.equal(+ (error as { unslothTransportFailure?: boolean }).unslothTransportFailure,+ true,+ );+ assert.equal(+ (error as { unslothBackendStillRunning?: boolean })+ .unslothBackendStillRunning,+ true,+ );+ assert.deepEqual(probed, [{ port }]);+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("a backend the launcher cannot see either still says to relaunch", async () => {+ const port = 61800;+ const originalFetch = globalThis.fetch;+ globalThis.fetch = async () => {+ throw new TypeError("fetch failed");+ };++ try {+ const authApi = loadAuthApi({ port, checkHealth: () => false });+ const error = await authApi+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })+ .then(+ () => null,+ (rejection: unknown) => rejection,+ );++ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);+ assert.equal(+ (error as { unslothBackendStillRunning?: boolean })+ .unslothBackendStillRunning,+ undefined,+ );+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("a presence probe that throws leaves the original verdict in place", async () => {+ const originalFetch = globalThis.fetch;+ globalThis.fetch = async () => {+ throw new TypeError("fetch failed");+ };++ try {+ const authApi = loadAuthApi({+ port: 61801,+ checkHealth: () => {+ throw new Error("command not registered on this build");+ },+ });+ const error = await authApi+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })+ .then(+ () => null,+ (rejection: unknown) => rejection,+ );++ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("no validated port yet means nothing is asked and nothing is claimed", async () => {+ const invoked: string[] = [];+ const originalFetch = globalThis.fetch;+ globalThis.fetch = async () => {+ throw new TypeError("fetch failed");+ };++ try {+ // The placeholder base before server-port arrives is port 0, which never connects.+ const authApi = loadAuthApi({+ port: null,+ checkHealth: () => true,+ onInvoke: (command) => invoked.push(command),+ });+ const error = await authApi+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })+ .then(+ () => null,+ (rejection: unknown) => rejection,+ );++ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);+ assert.deepEqual(invoked, []);+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("panels that all lose the backend at once share one native probe", async () => {+ // The whole hub loses its connection in the same tick and each probe waits out the launcher's budget.+ const originalFetch = globalThis.fetch;+ globalThis.fetch = async () => {+ throw new TypeError("fetch failed");+ };++ let probes = 0;+ let releaseProbe: (() => void) | null = null;+ const probeStarted = new Promise<void>((resolve) => {+ releaseProbe = resolve;+ });++ try {+ const authApi = loadAuthApi({+ port: 61802,+ checkHealth: async () => {+ probes += 1;+ releaseProbe?.();+ await new Promise((resolve) => setTimeout(resolve, 50));+ return true;+ },+ });+ const call = () =>+ authApi+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })+ .then(+ () => null,+ (rejection: unknown) => rejection,+ );++ const first = call();+ await probeStarted;+ const errors = await Promise.all([first, call(), call()]);++ assert.equal(+ probes,+ 1,+ "each lost panel opened its own native health probe",+ );+ for (const error of errors) {+ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_ANSWERING_MESSAGE);+ }++ const later = await call();+ assert.equal(probes, 2);+ assert.ok(later instanceof Error);+ assert.equal(later.message, authApi.BACKEND_NOT_ANSWERING_MESSAGE);+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("updating an existing install does not go through the changed path", async () => {+ // The update deliberately stops the backend, so pin that this flow uses commands and events, not HTTP.+ const update = await readFile(+ new URL("../src/hooks/use-tauri-update.ts", import.meta.url),+ "utf8",+ );+ assert.ok(update.includes('invoke("start_backend_update")'));+ assert.ok(+ !update.includes("authFetch"),+ "the update flow now goes through authFetch",+ );+ assert.ok(+ !/(?<![.\w])fetch\(/.test(update),+ "the update flow now issues its own fetch, which the transport path wraps",+ );++ // A webview newer than its shell asks for nothing new; the rejected invoke is covered above.+ const backend = await readFile(+ new URL("../src/hooks/use-tauri-backend.ts", import.meta.url),+ "utf8",+ );+ assert.ok(backend.includes('invoke<boolean>("check_health", { port })'));+ const main = await readFile(+ new URL("../../src-tauri/src/main.rs", import.meta.url),+ "utf8",+ );+ assert.ok(main.includes("commands::check_health,"));+ // check_health collapses a stalled probe onto false, the verdict this file exists to stop showing.+ assert.ok(main.includes("commands::check_backend_present,"));+ const authApiSrc = await readFile(+ new URL("../src/features/auth/api.ts", import.meta.url),+ "utf8",+ );+ assert.ok(+ authApiSrc.includes('invoke<boolean>("check_backend_present", { port })'),+ "the auth transport probe must ask for presence, not health",+ );+});++test("the background chat storage filter accepts every transport verdict", async () => {+ // Read as source: the module pulls in the Dexie database only a browser build can load.+ const source = await readFile(+ new URL(+ "../src/features/chat/utils/chat-history-storage.ts",+ import.meta.url,+ ),+ "utf8",+ );+ const filterStart = source.indexOf(+ "export function isExpectedBackgroundChatStorageError",+ );+ assert.ok(filterStart !== -1, "the background storage error filter moved");+ const filter = source.slice(filterStart, source.indexOf("\n}", filterStart));++ assert.ok(+ filter.includes("unslothTransportFailure"),+ "the filter does not recognise the transport marker, so a sync against a backend " ++ "that is merely busy is reported as an unexpected error",+ );+ assert.ok(+ !filter.includes("please relaunch it"),+ "the filter still matches one exact wording of the transport failure",+ );+});++test("a POST is not retried on the long ladder", async () => {+ // A network error is not an answer: the backend may have committed the request, so a retry can duplicate it.+ const port = 61797;+ const originalFetch = globalThis.fetch;+ let attempts = 0;+ globalThis.fetch = async () => {+ attempts += 1;+ throw new TypeError("fetch failed");+ };++ try {+ const authApi = loadAuthApi({ port, checkHealth: () => false });+ await authApi+ .authFetch("/api/auth/api-keys", { method: "POST", body: "{}" })+ .catch(() => undefined);+ assert.ok(+ attempts <= OLD_LADDER_ATTEMPTS,+ `a POST was sent ${attempts} times; the unsafe ladder allows ${OLD_LADDER_ATTEMPTS}`,+ );+ assert.ok(+ attempts > 1,+ "the unsafe ladder still retries, as it always did",+ );+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("a probe pending against the old port is not the answer about the new one", async () => {+ // setApiBase can move the port inside a probe's 10s budget, so a shared pending promise reports the PREVIOUS backend.+ const originalFetch = globalThis.fetch;+ globalThis.fetch = async () => {+ throw new TypeError("fetch failed");+ };++ const asked: number[] = [];+ let port = 61810;+ let releaseFirst: (() => void) | null = null;+ const firstProbeStarted = new Promise<void>((resolve) => {+ releaseFirst = resolve;+ });++ try {+ const authApi = loadAuthApi({+ port,+ getPort: () => port,+ checkHealth: async (probedPort: number) => {+ asked.push(probedPort);+ if (asked.length === 1) {+ releaseFirst?.();+ await new Promise((resolve) => setTimeout(resolve, 60));+ return false;+ }+ return true;+ },+ });+ const call = () =>+ authApi+ .authFetch("/api/models", undefined, { retryNetworkErrors: false })+ .then(+ () => null,+ (rejection: unknown) => rejection,+ );++ const first = call();+ await firstProbeStarted;+ port = 61811;+ const second = await call();+ const firstError = await first;++ assert.deepEqual(asked, [61810, 61811], "the new port was never probed");+ assert.ok(second instanceof Error);+ assert.equal(second.message, authApi.BACKEND_NOT_ANSWERING_MESSAGE);+ assert.ok(firstError instanceof Error);+ assert.equal(firstError.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);+ } finally {+ globalThis.fetch = originalFetch;+ }+});diff --git a/studio/src-tauri/src/commands.rs b/studio/src-tauri/src/commands.rsindex 8a6443253f8..438cdf0750d 100644--- a/studio/src-tauri/src/commands.rs+++ b/studio/src-tauri/src/commands.rs@@ -443,6 +443,8 @@ struct BackendLiveness {/// holding the port. Silence from a closed port is death; silence from an accepted/// connection is a stall.probe_timed_out: bool,+ /// An HTTP RESPONSE, whatever its status. Weaker than `alive`, which also requires the payload to name this service.+ answered: bool,}/// Check if an Unsloth backend is running on the given port.@@ -459,6 +461,48 @@ pub async fn check_health(port: u16) -> Result<bool, String> {}}+/// Whether a process still holds the port. `check_health_inner` only returns `Ok` with+/// `probe_timed_out: false`, so a stall arrives through the `Err` arm; a REFUSED connection+/// is not a timeout.+#[tauri::command]+pub async fn check_backend_present(+ state: tauri::State<'_, BackendState>,+ port: u16,+) -> Result<bool, String> {+ // Silence does not say which side went quiet, so ownership of the port settles it.+ let state = state.inner();+ backend_presence(port, || we_manage_a_backend_on(state, port)).await+}++/// *we_manage_it* is called AFTER the probe: an ownership answer taken before it died would report an exited backend as running.+async fn backend_presence(+ port: u16,+ we_manage_it: impl Fn() -> bool,+) -> Result<bool, String> {+ match check_health_inner(port, HEALTH_PROBE_TIMEOUT).await {+ Ok(liveness) => Ok(backend_is_present(&liveness, we_manage_it())),+ Err(e) => {+ info!("Backend presence check on port {} failed: {}", port, e);+ Ok(backend_is_present(+ &liveness_from_probe_error(&e),+ we_manage_it(),+ ))+ }+ }+}++/// Whether this app's own backend handle names *port*. The handle outlives the process it names, so the owner is checked too.+fn we_manage_a_backend_on(state: &BackendState, port: u16) -> bool {+ // Handle and process state in one pass under the same lock, so they cannot disagree.+ process::owned_backend_on_port_is_running(state, port)+}++/// The rule `check_backend_present` applies, as a value so the test for it can actually fail.+fn backend_is_present(liveness: &BackendLiveness, we_manage_it: bool) -> bool {+ // A healthy answer is presence whoever owns the port; a timeout or unhealthy reply counts only for a backend we manage.+ liveness.alive || ((liveness.probe_timed_out || liveness.answered) && we_manage_it)+}+/// Probe the backend for process liveness.////// `/api/liveness` rather than `/api/health`: health awaits hardware detection through@@ -487,13 +531,31 @@ async fn check_health_inner(continue;}if !resp.status().is_success() {- return Ok(BackendLiveness::default());+ // Not healthy, but not silence either: something answered on that port.+ return Ok(BackendLiveness {+ answered: true,+ ..BackendLiveness::default()+ });}- json = Some(resp.json::<serde_json::Value>().await?);+ json = match resp.json::<serde_json::Value>().await {+ Ok(value) => Some(value),+ Err(error) => {+ // Propagating the error made presence indistinguishable from a refusal.+ info!("Backend on port {} answered unparseable JSON: {}", port, error);+ return Ok(BackendLiveness {+ answered: true,+ ..BackendLiveness::default()+ });+ }+ };break;}let Some(json) = json else {- return Ok(BackendLiveness::default());+ // Every path answered 404, which is still an answer.+ return Ok(BackendLiveness {+ answered: true,+ ..BackendLiveness::default()+ });};// Liveness answers "alive" and health answers "healthy". Accept either, so the fallback@@ -546,6 +608,7 @@ async fn check_health_inner(warming_up: alive && (warming || (detecting && !deferred)),inference_active: alive && inference_active,probe_timed_out: false,+ answered: true,})}@@ -655,6 +718,8 @@ fn adopted_backend_liveness(// port a different Unsloth backend has taken over is excluded by `different_owner`,// since that one answered rather than fell silent.probe_timed_out: adopted_failure_is_a_stall(verified, served.alive, different_owner),+ // The pre-probe got a reply out of this port, whatever the re-check then did.+ answered: served.answered || served.alive,}}@@ -1379,8 +1444,18 @@ mod tests {super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT).await.unwrap(),- super::BackendLiveness::default()+ super::BackendLiveness {+ // It answered, and that is all `answered` says: not OUR backend.+ answered: true,+ ..super::BackendLiveness::default()+ });+ assert!(!super::backend_is_present(+ &super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)+ .await+ .unwrap(),+ false+ ));}#[test]@@ -1389,6 +1464,83 @@ mod tests {assert!(super::HEALTH_PROBE_TIMEOUT < super::HEALTH_WATCHDOG_INTERVAL);}+ #[test]+ fn the_frontend_retry_ladder_outlives_one_probe_budget() {+ // #10520: the ladder lives in TypeScript and the budget here, so only this guard keeps them in step.+ let src = include_str!("../../frontend/src/features/auth/api.ts").replace("\r\n", "\n");+ let marker = "const TAURI_FETCH_RETRY_DELAYS_MS = [";+ let start = src+ .find(marker)+ .expect("the Tauri fetch retry ladder moved; update this guard")+ + marker.len();+ let ladder = &src[start..];+ let ladder = &ladder[..ladder.find(']').expect("unterminated retry ladder")];+ let total_ms: u64 = ladder+ .split(',')+ .map(str::trim)+ .filter(|delay| !delay.is_empty())+ .map(|delay| {+ delay+ .parse::<u64>()+ .expect("a retry delay stopped being a plain number of milliseconds")+ })+ .sum();+ assert!(+ total_ms >= super::HEALTH_PROBE_TIMEOUT.as_millis() as u64,+ "the webview gives up after {total_ms}ms while one native liveness probe is \+ allowed {}ms, so a backend the launcher still considers alive is reported to \+ the user as not running",+ super::HEALTH_PROBE_TIMEOUT.as_millis()+ );+ assert!(+ src.contains("invoke<boolean>(\"check_backend_present\""),+ "the transport-failure path no longer asks the native side before it tells the \+ user to relaunch"+ );+ // And not the health command, which reports a spent budget as a refused connection.+ assert!(+ !src.contains("invoke<boolean>(\"check_health\""),+ "the transport-failure path is back on check_health, which collapses a stalled \+ probe onto \"not running\""+ );+ }++ #[test]+ fn a_stalled_probe_is_not_reported_as_an_absent_backend() {+ // check_health answers `alive`, so a timeout and a closed port are the same answer.+ let stalled = super::BackendLiveness {+ alive: false,+ probe_timed_out: true,+ ..Default::default()+ };+ let closed = super::BackendLiveness::default();+ let answered = super::BackendLiveness {+ alive: true,+ ..Default::default()+ };++ // Through the rule the command applies: an inlined copy asserts its own arithmetic.+ assert!(!stalled.alive, "a stall is not an answer");+ assert!(+ super::backend_is_present(&stalled, true),+ "a stalled probe must read as a backend that is still present"+ );+ assert!(+ !super::backend_is_present(&stalled, false),+ "a stall on a port this app does not manage proves nothing about our backend"+ );+ assert!(+ !super::backend_is_present(&closed, true),+ "a refused connection must still read as absent"+ );+ assert!(super::backend_is_present(&answered, false));+ // And the health command must keep collapsing the stall for its own caller.+ assert!(+ !stalled.alive,+ "check_health still reports a stall as not usable"+ );+ }+#[test]fn the_startup_grace_survives_the_mac_cold_start_timeline() {// Replays the macOS report this grace period exists for. The warm thread held the@@ -1534,6 +1686,89 @@ mod tests {port}+ /// A port that answers every request with *status* and *body*.+ async fn answering_test_backend(status: &'static str, body: &'static str) -> u16 {+ let listener = TcpListener::bind(("127.0.0.1", 0))+ .await+ .expect("probe test needs a loopback port");+ let port = listener.local_addr().unwrap().port();+ tokio::spawn(async move {+ while let Ok((mut stream, _)) = listener.accept().await {+ let mut buffer = [0; 2048];+ let Ok(_) = stream.read(&mut buffer).await else {+ return;+ };+ let response = format!(+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len()+ );+ let _ = stream.write_all(response.as_bytes()).await;+ }+ });+ port+ }++ #[tokio::test]+ async fn ownership_is_read_after_the_probe_not_before_it() {+ // The closure answers "we manage it" only once the request reached the server, so true means it ran AFTER the probe.+ let probe_arrived = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));+ let listener = TcpListener::bind(("127.0.0.1", 0))+ .await+ .expect("probe test needs a loopback port");+ let port = listener.local_addr().unwrap().port();+ let seen = std::sync::Arc::clone(&probe_arrived);+ tokio::spawn(async move {+ while let Ok((mut stream, _)) = listener.accept().await {+ let mut buffer = [0; 2048];+ let Ok(_) = stream.read(&mut buffer).await else {+ return;+ };+ seen.store(true, std::sync::atomic::Ordering::SeqCst);+ let response = "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";+ let _ = stream.write_all(response.as_bytes()).await;+ }+ });++ let asked = std::sync::Arc::clone(&probe_arrived);+ assert_eq!(+ super::backend_presence(port, move || asked.load(std::sync::atomic::Ordering::SeqCst))+ .await,+ Ok(true),+ "ownership was read before the probe, so a backend that changed under it would \+ still have been reported as running"+ );+ }++ #[tokio::test]+ async fn a_managed_backend_that_answers_unhealthily_is_still_present() {+ // A backend still building its app answers a non-2xx, collapsed onto the refused-connection default.+ let port = answering_test_backend("503 Service Unavailable", "").await;+ assert_eq!(+ super::backend_presence(port, || true).await,+ Ok(true),+ "a managed backend answering 503 was reported absent"+ );+ assert_eq!(super::backend_presence(port, || false).await, Ok(false));+ }++ #[tokio::test]+ async fn a_reply_this_build_cannot_parse_is_still_an_answer() {+ // Propagating the parse error made presence indistinguishable from a refusal.+ let port = answering_test_backend("200 OK", "not json at all").await;+ assert_eq!(+ super::backend_presence(port, || true).await,+ Ok(true),+ "a managed backend answering an unparseable body was reported absent"+ );+ assert_eq!(super::backend_presence(port, || false).await, Ok(false));++ let liveness = super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)+ .await+ .expect("an answered probe is not a transport error");+ assert!(!liveness.alive, "an unparseable reply is not a healthy backend");+ assert!(liveness.answered, "the reply arrived, so the port is not silent");+ }+#[tokio::test]async fn a_port_that_accepts_and_never_answers_reads_as_a_stall() {// The premise of the busy budget: a saturated backend still holds its port open, so@@ -1554,6 +1789,45 @@ mod tests {);}+ #[tokio::test]+ async fn check_backend_present_reports_a_stalled_port_as_present() {+ // The bug lives in the `Err` arm, which a struct-level test cannot reach; costs HEALTH_PROBE_TIMEOUT in wall clock.+ let port = stalling_test_backend().await;+ assert_eq!(+ super::backend_presence(port, || true).await,+ Ok(true),+ "a backend holding the port and not answering was reported absent, which is the \+ relaunch prompt this command was added to prevent"+ );+ }++ #[tokio::test]+ async fn a_stalled_port_this_app_does_not_manage_is_not_our_backend() {+ // With no backend of our own on that port there is nothing to wait for.+ let port = stalling_test_backend().await;+ assert_eq!(+ super::backend_presence(port, || false).await,+ Ok(false),+ "a stalled port with no managed backend behind it was reported as present"+ );+ }++ #[tokio::test]+ async fn check_backend_present_reports_a_closed_port_as_absent() {+ // Classifying the error must not turn every failed probe into "present".+ let port = {+ let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();+ let port = listener.local_addr().unwrap().port();+ drop(listener);+ port+ };+ assert_eq!(+ super::backend_presence(port, || true).await,+ Ok(false),+ "a closed port must still read as an absent backend"+ );+ }+#[tokio::test]async fn a_port_with_nothing_on_it_reads_as_death_not_a_stall() {// The other half: a backend that really exited leaves a closed port, and that must@@ -1693,6 +1967,7 @@ mod tests {warming_up: false,inference_active: true,probe_timed_out: false,+ answered: true,};let confirmed = super::adopted_backend_liveness(false, &served, false);assert!(@@ -1714,6 +1989,7 @@ mod tests {warming_up: false,inference_active: false,probe_timed_out: false,+ answered: true,},false,);@@ -1838,6 +2114,7 @@ mod tests {warming_up: warming,inference_active: busy,probe_timed_out: false,+ answered: true,},Probe::TimedOut => {elapsed += probe_budget;diff --git a/studio/src-tauri/src/main.rs b/studio/src-tauri/src/main.rsindex cc5a8ff09b4..9559ea280bb 100644--- a/studio/src-tauri/src/main.rs+++ b/studio/src-tauri/src/main.rs@@ -2125,6 +2125,7 @@ fn main() {commands::start_managed_server,commands::stop_server,commands::check_health,+ commands::check_backend_present,commands::get_server_logs,commands::open_logs_dir,commands::open_models_dir,diff --git a/studio/src-tauri/src/process.rs b/studio/src-tauri/src/process.rsindex 5879e422fa3..a951cd66d5b 100644--- a/studio/src-tauri/src/process.rs+++ b/studio/src-tauri/src/process.rs@@ -1173,6 +1173,34 @@ pub(crate) fn owned_backend_snapshot(Ok(snapshot)}+/// Whether the handle that names *port* still refers to a process that EXISTS: a handle+/// outlives its process, and another process can bind the freed port.+/// Anything this cannot read leaves the handle trusted, so a running backend is never declared dead.+pub(crate) fn owned_backend_on_port_is_running(state: &BackendState, port: u16) -> bool {+ let mut proc = match state.lock() {+ Ok(guard) => guard,+ Err(poisoned) => poisoned.into_inner(),+ };+ let handle = match proc.owned.as_mut() {+ Some(handle) => handle,+ None => return false,+ };+ if handle.port() != Some(port) {+ return false;+ }+ match handle {+ OwnedBackendHandle::Spawned { child, .. } => !matches!(child.try_wait(), Ok(Some(_))),+ OwnedBackendHandle::Adopted { pid, .. } => backend_pid_is_running(*pid),+ }+}++/// Whether *pid* still exists: false only when the pid is PROVABLY gone. A zombie has exited+/// but is unreaped, which `kill(pid, 0)` alone cannot see.+pub(crate) fn backend_pid_is_running(pid: u32) -> bool {+ crate::desktop_backend_owner::pid_is_not_dead(pid)+ && !crate::process_identity::is_zombie(pid)+}+pub(crate) fn record_owned_backend_port_if_current(state: &BackendState,generation: u64,@@ -6243,3 +6271,107 @@ mod exit_status_after_stdout_closed_tests {}}}++#[cfg(test)]+mod owned_backend_liveness_tests {+ use super::*;++ #[cfg(unix)]+ fn spawn_owned(args: &[&str]) -> Box<dyn ChildWrapper + Send> {+ let mut cmd = Command::new(args[0]);+ cmd.args(&args[1..])+ .stdout(Stdio::piped())+ .stderr(Stdio::piped());+ let mut wrap = CommandWrap::from(cmd);+ wrap.wrap(ProcessGroup::leader());+ wrap.spawn().expect("spawn test child")+ }++ #[cfg(windows)]+ fn spawn_owned(args: &[&str]) -> Box<dyn ChildWrapper + Send> {+ let mut cmd = Command::new(args[0]);+ cmd.args(&args[1..])+ .stdout(Stdio::piped())+ .stderr(Stdio::piped());+ CommandWrap::from(cmd).spawn().expect("spawn test child")+ }++ #[cfg(unix)]+ const LIVE_CHILD: [&str; 3] = ["/bin/sh", "-c", "exec sleep 30"];+ #[cfg(unix)]+ const DEAD_CHILD: [&str; 3] = ["/bin/sh", "-c", "exit 0"];+ #[cfg(windows)]+ const LIVE_CHILD: [&str; 3] = ["cmd.exe", "/C", "ping -n 30 127.0.0.1"];+ #[cfg(windows)]+ const DEAD_CHILD: [&str; 3] = ["cmd.exe", "/C", "exit 0"];++ fn state_owning(child: Box<dyn ChildWrapper + Send>, port: u16) -> BackendState {+ let state = new_backend_state();+ {+ let mut proc = state.lock().unwrap();+ let pid = 0;+ proc.owned = Some(OwnedBackendHandle::spawned(child, None, pid, 1));+ if let Some(handle) = proc.owned.as_mut() {+ handle.set_reported_port(port);+ }+ proc.port = Some(port);+ }+ state+ }++ #[test]+ fn a_child_that_is_still_running_is_ours() {+ let state = state_owning(spawn_owned(&LIVE_CHILD), 8765);+ assert!(owned_backend_on_port_is_running(&state, 8765));+ let mut proc = state.lock().unwrap();+ if let Some(handle) = proc.owned.as_mut() {+ if let Some(child) = handle.spawned_child_mut() {+ let _ = child.start_kill();+ }+ }+ }++ #[test]+ fn a_child_that_has_exited_leaves_the_port_to_strangers() {+ let mut child = spawn_owned(&DEAD_CHILD);+ let _ = child.wait();+ let state = state_owning(child, 8765);+ assert!(+ !owned_backend_on_port_is_running(&state, 8765),+ "an exited child still counted as the managed backend, so a foreign service on \+ its port would be reported to the user as Unsloth still running"+ );+ }++ #[test]+ fn a_handle_for_another_port_is_not_this_port() {+ let state = state_owning(spawn_owned(&LIVE_CHILD), 8765);+ assert!(!owned_backend_on_port_is_running(&state, 8766));+ let mut proc = state.lock().unwrap();+ if let Some(handle) = proc.owned.as_mut() {+ if let Some(child) = handle.spawned_child_mut() {+ let _ = child.start_kill();+ }+ }+ }++ #[test]+ fn no_handle_at_all_is_not_a_managed_backend() {+ let state = new_backend_state();+ assert!(!owned_backend_on_port_is_running(&state, 8765));+ }++ // The adopted half, where there is no child handle to wait on.+ #[test]+ fn an_adopted_pid_that_is_gone_is_not_running() {+ assert!(backend_pid_is_running(std::process::id()));+ // A real process run to completion, so this pid is PROVABLY gone; an unreadable pid stays trusted by design.+ let mut child = spawn_owned(&DEAD_CHILD);+ let pid = child.id();+ let _ = child.wait();+ assert!(+ !backend_pid_is_running(pid),+ "an adopted backend that has exited still read as running"+ );+ }+}diff --git a/tests/studio/test_auth_form_input_count.py b/tests/studio/test_auth_form_input_count.pyindex a1d3d4f8158..48a0e4de990 100644--- a/tests/studio/test_auth_form_input_count.py+++ b/tests/studio/test_auth_form_input_count.py@@ -314,12 +314,17 @@ def test_auth_flow_routes_do_not_mount_global_settings():def test_auth_redirect_targets_are_idempotent_and_concurrent(tmp_path: Path):if shutil.which("node") is None:pytest.skip("node not available")- probe = subprocess.run(- ["node", "--experimental-strip-types", "--version"],- capture_output = True,- text = True,- timeout = 5,- )+ # A timeout is a SKIP: the first `node` of a job on a Windows runner pays for image+ # scanning and a cold file cache, and 5s was not enough for a `--version` there.+ try:+ probe = subprocess.run(+ ["node", "--experimental-strip-types", "--version"],+ capture_output = True,+ text = True,+ timeout = 120,+ )+ except subprocess.TimeoutExpired:+ pytest.skip("node did not answer --version in time on this runner")if probe.returncode != 0:pytest.skip("node --experimental-strip-types not available")@@ -335,6 +340,11 @@ def test_auth_redirect_targets_are_idempotent_and_concurrent(tmp_path: Path):let access = null, refresh = null, passwordChange = false;export const apiUrl = (path) => path;export const isTauri = false;+ // Read by the Tauri transport-failure path before it asks the native health+ // check. This stub is the web build (isTauri false), where that path is never+ // taken, but the import is unconditional and an ES module import of a name the+ // stub does not export is a SyntaxError at instantiation, not at call time.+ export const getApiPort = () => null;export const accountTransitionPending = () => false;export const reset = (a = null, r = null) => { access = a; refresh = r; passwordChange = false; };export const clearAuthTokens = () => { access = null; refresh = null; };@@ -406,6 +416,7 @@ def test_auth_redirect_targets_are_idempotent_and_concurrent(tmp_path: Path):cwd = tmp_path,capture_output = True,text = True,- timeout = 30,+ # The same cold-start allowance as the probe above, plus the work itself.+ timeout = 180,)assert result.returncode == 0, f"stderr: {result.stderr}\nstdout: {result.stdout}"diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.pyindex d6e63bd515a..14796dda433 100644--- a/tests/studio/test_model_picker_contracts.py+++ b/tests/studio/test_model_picker_contracts.py@@ -2649,16 +2649,22 @@ def test_auth_retries_tag_transport_failures_like_the_first_attempt():src = (WORKDIR / "studio" / "frontend" / "src" / "features" / "auth" / "api.ts").read_text(encoding = "utf-8")- assert src.count("unslothTransportFailure: true") == 2, "one tag per message, in one helper"- tagger = src.split("function asTransportFailure", 1)[1].split("\n}\n", 1)[0]+ tagger = src.split("async function asTransportFailure", 1)[1].split("\n}\n", 1)[0]assert "err instanceof TypeError" in taggerassert "navigator.onLine === false" in tagger+ # Counted against the helper's own raises, not pinned to a number that a new message would trip.+ assert tagger.count("unslothTransportFailure: true") == tagger.count("new Error(")+ assert tagger.count("unslothTransportFailure: true") >= 2+ assert src.count("unslothTransportFailure: true") == tagger.count(+ "unslothTransportFailure: true"+ ), "every tag belongs to the one helper"retry = src.split("async function retryWithCurrentToken", 1)[1]retry = retry.split("\n}\n", 1)[0]assert "fetchWithTauriNetworkRetry" in retry- assert "throw asTransportFailure(err);" in retry+ # Awaited since #10520: dropping the await throws a pending promise and the tag is never seen.+ assert "throw await asTransportFailure(err);" in retryfirst = src.split("export async function authFetch", 1)[1]- assert "throw asTransportFailure(err);" in first+ assert "throw await asTransportFailure(err);" in firstdef test_adoption_takes_its_own_pin_before_moving_the_checkpoint():