Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: report a backend that is provably gone without sleeping out the retry ladder (#11213)

Daniel Handdb2324e2423
diff --git a/studio/backend/tests/test_inference_worker_stderr_tail.py b/studio/backend/tests/test_inference_worker_stderr_tail.py
index 282c0d36e65..2c2e6901290 100644
--- a/studio/backend/tests/test_inference_worker_stderr_tail.py
+++ b/studio/backend/tests/test_inference_worker_stderr_tail.py
@@ -1241,3 +1241,89 @@ def test_two_paths_on_one_line_are_two_matches():
spaced = _redact_worker_output("C:\\Program Files\\unsloth\\weights.gguf failed\n")
assert spaced.strip() == ".../weights.gguf failed", spaced
+
+
+def test_the_mirror_module_does_not_pull_logging_into_a_spawned_worker():
+ """Every inference worker imports this module before its entrypoint runs.
+
+ A fresh spawn child has no ``logging`` yet, and importing it there was 4.2ms of the
+ 5.3ms this module added to each worker spawn. Nothing on the spawn path needs it: the
+ record marking runs later, after the worker has configured logging.
+ """
+ import ast
+
+ source = (Path(_BACKEND_DIR) / "utils/worker_stderr.py").read_text(encoding = "utf-8")
+ tree = ast.parse(source)
+ top_level = [
+ alias.name for node in tree.body if isinstance(node, ast.Import) for alias in node.names
+ ]
+ top_level += [node.module for node in tree.body if isinstance(node, ast.ImportFrom)]
+ assert "logging" not in top_level, (
+ "utils/worker_stderr.py imports logging at module scope again, which every "
+ "spawned worker pays for before its entrypoint runs"
+ )
+
+ # And the same thing as a fact rather than as a reading of the source. Skipped rather
+ # than failed where the interpreter already has logging for its own reasons, since the
+ # claim is about what this module drags in, not about what a bare start-up loads.
+ probe = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ "import sys\n"
+ "preloaded = 'logging' in sys.modules\n"
+ f"sys.path.insert(0, {_BACKEND_DIR!r})\n"
+ "import utils.worker_stderr\n"
+ "print(preloaded, 'logging' in sys.modules)\n",
+ ],
+ capture_output = True,
+ text = True,
+ timeout = 120,
+ )
+ assert probe.returncode == 0, probe.stderr
+ preloaded, after = probe.stdout.split()
+ if preloaded == "True":
+ pytest.skip("this interpreter loads logging before any of our code runs")
+ assert after == "False", "importing utils.worker_stderr imported logging"
+
+
+def test_a_second_formatter_class_cannot_wrap_a_handler_that_is_already_marked():
+ """The marking class is built on first use, so there can be more than one of it.
+
+ Two threads that both find the cache empty each build a class, and ``isinstance`` is
+ false across the pair, so identity would wrap one handler twice and emit every
+ continuation line with a doubled prefix. Rebuilding the class here is the same
+ observation as that race, without depending on an interleaving.
+ """
+ import io
+ import logging as logging_module
+
+ from utils import worker_stderr
+ from utils.worker_stderr import LOG_RECORD_CONTINUATION_PREFIX, LOG_RECORD_START_MARK
+
+ stream = io.StringIO()
+ handler = logging_module.StreamHandler(stream)
+ handler.setFormatter(logging_module.Formatter("%(message)s"))
+
+ original = worker_stderr._PREFIX_FORMATTER_CLASS
+ try:
+ assert worker_stderr._mark_handler(handler) is True
+ first_class = worker_stderr._PREFIX_FORMATTER_CLASS
+ worker_stderr._PREFIX_FORMATTER_CLASS = None
+ assert (
+ worker_stderr._mark_handler(handler) is False
+ ), "a handler already marked by one formatter class was marked again by another"
+ second_class = worker_stderr._prefix_formatter_class()
+ assert first_class is not second_class, "the rebuild this test needs did not happen"
+
+ logger = logging_module.getLogger(f"{__name__}.two-classes")
+ logger.handlers = [handler]
+ logger.propagate = False
+ logger.setLevel(logging_module.DEBUG)
+ logger.error("boom\n File x")
+ emitted = stream.getvalue()
+ finally:
+ worker_stderr._PREFIX_FORMATTER_CLASS = original
+
+ expected = LOG_RECORD_START_MARK + "boom\n" + LOG_RECORD_CONTINUATION_PREFIX + " File x\n"
+ assert emitted == expected, emitted
diff --git a/studio/backend/utils/worker_stderr.py b/studio/backend/utils/worker_stderr.py
index f4a02295664..65caca2752e 100644
--- a/studio/backend/utils/worker_stderr.py
+++ b/studio/backend/utils/worker_stderr.py
@@ -7,12 +7,13 @@ from __future__ import annotations
import atexit
import collections
-import logging
import os
import sys
import tempfile
import threading
+# `logging` is deliberately NOT imported here; see _prefix_formatter_class().
+
__all__ = [
"LOG_RECORD_CONTINUATION_PREFIX",
"LOG_RECORD_START_MARK",
@@ -479,28 +480,55 @@ def install_worker_stderr_mirror(
return True
-class _EveryLineCarriesThePrefix(logging.Formatter):
- def __init__(self, inner: "logging.Formatter") -> None:
- super().__init__()
- self._inner = inner
+_PREFIX_FORMATTER_CLASS = None
+# On the formatter, not the class: built lazily, so two threads racing the first call make two
+# classes, `isinstance` is false across the pair, and the handler gets wrapped twice.
+_MARKS_CONTINUATIONS = "_unsloth_marks_continuations"
+
+
+def _formatter_marks_continuations(formatter) -> bool:
+ return getattr(formatter, _MARKS_CONTINUATIONS, False) is True
- def format(self, record: "logging.LogRecord") -> str:
- text = self._inner.format(record)
- first, newline, rest = text.partition("\n")
- # The first line too: a default-formatted single-line record has no shape to spot.
- marked_first = (
- first if first.startswith(LOG_RECORD_START_MARK) else LOG_RECORD_START_MARK + first
- )
- if not newline:
- return marked_first
- return (
- marked_first
- + "\n"
- + "\n".join(LOG_RECORD_CONTINUATION_PREFIX + line for line in rest.split("\n"))
- )
- def __getattr__(self, name: str):
- return getattr(self._inner, name)
+def _prefix_formatter_class():
+ """Built on first use, so importing this module does not import ``logging``.
+
+ Every spawned worker imports this module before its entrypoint runs and a fresh spawn
+ child has no ``logging`` yet, which was 4.2ms of the 5.3ms this file cost each spawn.
+ """
+ global _PREFIX_FORMATTER_CLASS
+ if _PREFIX_FORMATTER_CLASS is not None:
+ return _PREFIX_FORMATTER_CLASS
+ import logging
+
+ class _EveryLineCarriesThePrefix(logging.Formatter):
+ # Read through `_formatter_marks_continuations`, never `isinstance`: see above.
+ _unsloth_marks_continuations = True
+
+ def __init__(self, inner: "logging.Formatter") -> None:
+ super().__init__()
+ self._inner = inner
+
+ def format(self, record: "logging.LogRecord") -> str:
+ text = self._inner.format(record)
+ first, newline, rest = text.partition("\n")
+ # The first line too: a default-formatted single-line record has no shape to spot.
+ marked_first = (
+ first if first.startswith(LOG_RECORD_START_MARK) else LOG_RECORD_START_MARK + first
+ )
+ if not newline:
+ return marked_first
+ return (
+ marked_first
+ + "\n"
+ + "\n".join(LOG_RECORD_CONTINUATION_PREFIX + line for line in rest.split("\n"))
+ )
+
+ def __getattr__(self, name: str):
+ return getattr(self._inner, name)
+
+ _PREFIX_FORMATTER_CLASS = _EveryLineCarriesThePrefix
+ return _PREFIX_FORMATTER_CLASS
_UNHOOKED_SET_FORMATTER = None
@@ -508,11 +536,14 @@ _UNHOOKED_ADD_HANDLER = None
def _mark_handler(handler) -> bool:
+ import logging
+
+ prefixed = _prefix_formatter_class()
formatter = getattr(handler, "formatter", None)
- if isinstance(formatter, _EveryLineCarriesThePrefix):
+ if _formatter_marks_continuations(formatter):
return False
setter = _UNHOOKED_SET_FORMATTER or type(handler).setFormatter
- setter(handler, _EveryLineCarriesThePrefix(formatter or logging.Formatter()))
+ setter(handler, prefixed(formatter or logging.Formatter()))
return True
@@ -521,12 +552,15 @@ def _install_continuation_hook() -> bool:
global _UNHOOKED_SET_FORMATTER, _UNHOOKED_ADD_HANDLER
if _UNHOOKED_SET_FORMATTER is not None:
return False
+ import logging
+
+ prefixed = _prefix_formatter_class()
unhooked_set = logging.Handler.setFormatter
unhooked_add = logging.Logger.addHandler
def setFormatter(self, fmt): # noqa: N802 -- matches logging's own spelling
- if fmt is not None and not isinstance(fmt, _EveryLineCarriesThePrefix):
- fmt = _EveryLineCarriesThePrefix(fmt)
+ if fmt is not None and not _formatter_marks_continuations(fmt):
+ fmt = prefixed(fmt)
unhooked_set(self, fmt)
def addHandler(self, hdlr): # noqa: N802 -- matches logging's own spelling
@@ -546,6 +580,8 @@ def _install_continuation_hook() -> bool:
def mark_log_record_continuations(logger_object = None, *, cover_later_handlers = True) -> int:
"""Covers ``logging.lastResort`` and later handlers too: a miss hands a RECOVERED
request's traceback to the next caller as their crash."""
+ import logging
+
root = logger_object if logger_object is not None else logging.getLogger()
wrapped = 0
for handler in list(getattr(root, "handlers", ())):
diff --git a/studio/frontend/src/features/auth/api.ts b/studio/frontend/src/features/auth/api.ts
index a0899389302..fda7e6ab5ce 100644
--- a/studio/frontend/src/features/auth/api.ts
+++ b/studio/frontend/src/features/auth/api.ts
@@ -94,6 +94,10 @@ async function fetchWithTauriNetworkRetry(
) {
throw error;
}
+ // Tauri only, below the guard above. `fetch` cannot tell a refused port from a silent
+ // one and the native side can, so ask it once on the FIRST failure rather than sleeping
+ // out #10520's 10.5s ladder to be told what the refusal already proved.
+ if (attempt === 0 && (await nativeBackendIsGone())) throw error;
await wait(delays[attempt]);
beforeRetry?.();
}
@@ -191,6 +195,51 @@ async function nativeBackendIsAlive(): Promise<boolean> {
}
}
+/** `check_backend_is_gone` and NOT `check_backend_present`: presence reports a backend of ours that has not bound its port yet as absent. */
+let nativeGoneInflight: { port: number; probe: Promise<boolean> } | null = null;
+
+/**
+ * Whether the retry ladder has anything left to wait for.
+ *
+ * Only ever answers true on positive proof, so every failure mode below returns false and
+ * leaves the ladder exactly as long as it is today: the browser build, which has no native
+ * side to ask; a port the webview has not been given yet; and a shell too old to carry the
+ * command, whose rejected `invoke` is caught here.
+ */
+async function nativeBackendIsGone(): Promise<boolean> {
+ if (!isTauri) {
+ return false;
+ }
+ const port = getApiPort();
+ if (port === null) {
+ return false;
+ }
+ // Single flight, like the presence probe: a hub losing the backend fails every panel at once.
+ if (nativeGoneInflight !== null && nativeGoneInflight.port === port) {
+ return nativeGoneInflight.probe;
+ }
+ const probe = (async () => {
+ try {
+ const { invoke } = await import("@tauri-apps/api/core");
+ return (
+ (await invoke<boolean>("check_backend_is_gone", { port })) === true
+ );
+ } catch {
+ return false;
+ }
+ })();
+ const inflight = { port, probe };
+ nativeGoneInflight = inflight;
+ try {
+ return await probe;
+ } finally {
+ // Identity, not the port: a probe for a newer port owns the slot now.
+ if (nativeGoneInflight === inflight) {
+ nativeGoneInflight = null;
+ }
+ }
+}
+
async function asTransportFailure(err: unknown): Promise<unknown> {
// fetch TypeError = offline | backend down | CORS/DNS. Tagged so callers tell "never reached"
// from "rejected"; under Tauri the launcher is asked before claiming the backend is gone.
diff --git a/studio/frontend/tests/tauri-backend-transport-failure.test.ts b/studio/frontend/tests/tauri-backend-transport-failure.test.ts
index be5cca65b99..1f5b8b3fa37 100644
--- a/studio/frontend/tests/tauri-backend-transport-failure.test.ts
+++ b/studio/frontend/tests/tauri-backend-transport-failure.test.ts
@@ -24,6 +24,9 @@ function loadAuthApi(options: {
port: number | null;
getPort?: () => number | null;
checkHealth?: (port: number) => boolean | Promise<boolean>;
+ /** `check_backend_is_gone`. Default false: the ladder the tree had before the fast path. */
+ checkGone?: (port: number) => boolean | Promise<boolean>;
+ isTauri?: boolean;
onInvoke?: (command: string, args: Record<string, unknown>) => void;
}): AuthApi {
const currentPort = options.getPort ?? (() => options.port);
@@ -38,7 +41,7 @@ function loadAuthApi(options: {
: `http://127.0.0.1:${port}${path}`;
},
getApiPort: () => currentPort(),
- isTauri: true,
+ isTauri: options.isTauri ?? true,
},
"@/lib/account-transition": { accountTransitionPending: () => false },
"./session": {
@@ -52,6 +55,9 @@ function loadAuthApi(options: {
"@tauri-apps/api/core": {
invoke: async (command: string, args: Record<string, unknown>) => {
options.onInvoke?.(command, args);
+ if (command === "check_backend_is_gone") {
+ return options.checkGone?.(args.port as number) ?? false;
+ }
if (command !== "check_backend_present") {
throw new Error(`unexpected command ${command}`);
}
@@ -104,7 +110,10 @@ test("a backend that accepts late is reached instead of declared not running", a
try {
listenAfter(server, port, SLOW_LOOPBACK_ACCEPT_DELAY_MS);
- const authApi = loadAuthApi({ port });
+ // The connects here are genuinely refused until 3s. `check_backend_is_gone` answers
+ // false anyway, because a backend this app is bringing up owns the port before it binds
+ // it -- which is why the fast path asks for absence rather than for presence.
+ const authApi = loadAuthApi({ port, checkGone: () => false });
const response = await authApi.authFetch("/api/models");
assert.equal(response.status, 200);
@@ -326,6 +335,8 @@ test("updating an existing install does not go through the changed path", async
assert.ok(main.includes("commands::check_health,"));
// check_health collapses a stalled probe onto false, the verdict this file exists to stop showing.
assert.ok(main.includes("commands::check_backend_present,"));
+ // The fast path's command. Unregistered, every invoke rejects and the ladder is the old one.
+ assert.ok(main.includes("commands::check_backend_is_gone,"));
const authApiSrc = await readFile(
new URL("../src/features/auth/api.ts", import.meta.url),
"utf8",
@@ -441,3 +452,256 @@ test("a probe pending against the old port is not the answer about the new one",
globalThis.fetch = originalFetch;
}
});
+
+// The ladder the fast path short-circuits, as the source states it.
+const LONG_LADDER_MS = 250 + 750 + 1500 + 3000 + 5000;
+const LONG_LADDER_ATTEMPTS = 6;
+
+test("a backend that is provably gone is reported without sleeping out the ladder", async () => {
+ // A refused port with nothing of ours coming up on it: the native side has already
+ // answered the question the remaining 10.5s of sleeping would ask five more times.
+ const port = 61820;
+ const asked: string[] = [];
+ const originalFetch = globalThis.fetch;
+ let attempts = 0;
+ globalThis.fetch = async () => {
+ attempts += 1;
+ throw new TypeError("fetch failed");
+ };
+
+ try {
+ const authApi = loadAuthApi({
+ port,
+ checkGone: () => true,
+ checkHealth: () => false,
+ onInvoke: (command) => asked.push(command),
+ });
+ const started = performance.now();
+ const error = await authApi.authFetch("/api/models").then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+ const elapsed = performance.now() - started;
+
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);
+ assert.equal(
+ attempts,
+ 1,
+ `the request was sent ${attempts} times after the backend was proven gone`,
+ );
+ assert.ok(
+ elapsed < LONG_LADDER_MS / 4,
+ `reporting a dead backend took ${Math.round(elapsed)}ms of a ${LONG_LADDER_MS}ms ladder`,
+ );
+ assert.deepEqual(asked, ["check_backend_is_gone", "check_backend_present"]);
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("a backend that is merely slow still gets every rung and the busy message", async () => {
+ // The other direction. `check_backend_is_gone` is false for a port we are bringing up and
+ // for one whose handshake is filtered, and neither may lose a single retry.
+ const port = 61821;
+ const originalFetch = globalThis.fetch;
+ let attempts = 0;
+ let goneProbes = 0;
+ globalThis.fetch = async () => {
+ attempts += 1;
+ throw new TypeError("fetch failed");
+ };
+
+ try {
+ const authApi = loadAuthApi({
+ port,
+ checkGone: () => {
+ goneProbes += 1;
+ return false;
+ },
+ checkHealth: () => true,
+ });
+ const started = performance.now();
+ const error = await authApi.authFetch("/api/models").then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+ const elapsed = performance.now() - started;
+
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_ANSWERING_MESSAGE);
+ assert.equal(
+ attempts,
+ LONG_LADDER_ATTEMPTS,
+ `a backend that may still be starting was given ${attempts} attempts`,
+ );
+ assert.ok(
+ elapsed >= LONG_LADDER_MS,
+ `the ladder finished in ${Math.round(elapsed)}ms, short of ${LONG_LADDER_MS}ms`,
+ );
+ assert.equal(
+ goneProbes,
+ 1,
+ "the fast path probed on more than the first failure",
+ );
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("a shell too old for the command keeps the ladder it has today", async () => {
+ // The webview ships ahead of the shell it runs in. A rejected invoke is not an answer.
+ const port = 61822;
+ const originalFetch = globalThis.fetch;
+ let attempts = 0;
+ globalThis.fetch = async () => {
+ attempts += 1;
+ throw new TypeError("fetch failed");
+ };
+
+ try {
+ const authApi = loadAuthApi({
+ port,
+ checkGone: () => {
+ throw new Error("command not registered on this build");
+ },
+ checkHealth: () => false,
+ });
+ const error = await authApi.authFetch("/api/models").then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);
+ assert.equal(attempts, LONG_LADDER_ATTEMPTS);
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("the browser build asks nothing and behaves exactly as it did", async () => {
+ // `fetch` in a browser reports a refused connection and a timed-out one as the same
+ // opaque TypeError, so there is nothing to be fast about and no native side to ask.
+ const invoked: string[] = [];
+ const originalFetch = globalThis.fetch;
+ let attempts = 0;
+ globalThis.fetch = async () => {
+ attempts += 1;
+ throw new TypeError("fetch failed");
+ };
+
+ try {
+ const authApi = loadAuthApi({
+ port: 61823,
+ isTauri: false,
+ checkGone: () => true,
+ checkHealth: () => true,
+ onInvoke: (command) => invoked.push(command),
+ });
+ const error = await authApi.authFetch("/api/models").then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);
+ assert.equal(attempts, 1, "the browser build started retrying");
+ assert.deepEqual(
+ invoked,
+ [],
+ "the browser build reached for the native side",
+ );
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("panels that all lose the backend at once share one absence probe", async () => {
+ const originalFetch = globalThis.fetch;
+ globalThis.fetch = async () => {
+ throw new TypeError("fetch failed");
+ };
+
+ let probes = 0;
+ let releaseProbe: (() => void) | null = null;
+ const probeStarted = new Promise<void>((resolve) => {
+ releaseProbe = resolve;
+ });
+
+ try {
+ const authApi = loadAuthApi({
+ port: 61824,
+ checkGone: async () => {
+ probes += 1;
+ releaseProbe?.();
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ return true;
+ },
+ checkHealth: () => false,
+ });
+ const call = () =>
+ authApi.authFetch("/api/models").then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+
+ const first = call();
+ // Bounded: a build that never probes must fail on the count below, not hang here.
+ await Promise.race([
+ probeStarted,
+ new Promise<void>((resolve) => {
+ setTimeout(resolve, 5_000).unref();
+ }),
+ ]);
+ const errors = await Promise.all([first, call(), call()]);
+
+ assert.equal(probes, 1, "each lost panel opened its own absence probe");
+ for (const error of errors) {
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);
+ }
+ } finally {
+ globalThis.fetch = originalFetch;
+ }
+});
+
+test("a port nobody here owns that binds late is given up on, and that is the trade", async () => {
+ // The other side of the fast path, pinned so it cannot change by accident. A refusal
+ // proves nothing is listening NOW, not that nothing will bind later, so a backend the
+ // shell did not start -- one attached to from outside Desktop, restarting -- is reported
+ // gone rather than waited for. The owned case above keeps every rung; this one does not,
+ // and the ownership answer is the whole difference between them.
+ const port = await reserveLoopbackPort();
+ const server = createServer((_request, response) => {
+ response.writeHead(200, { "Content-Type": "application/json" });
+ response.end(JSON.stringify({ ok: true }));
+ });
+ const originalFetch = globalThis.fetch;
+ let attempts = 0;
+ globalThis.fetch = async (input, init) => {
+ attempts += 1;
+ return await originalFetch(input, init);
+ };
+
+ try {
+ listenAfter(server, port, SLOW_LOOPBACK_ACCEPT_DELAY_MS);
+ // What a truthful native side answers for this port: refused, and owned by nobody here.
+ const authApi = loadAuthApi({ port, checkGone: () => true, checkHealth: () => false });
+ const error = await authApi.authFetch("/api/models").then(
+ () => null,
+ (rejection: unknown) => rejection,
+ );
+
+ assert.ok(error instanceof Error);
+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);
+ assert.equal(
+ attempts,
+ 1,
+ `the ladder kept climbing for a port reported gone (${attempts} attempts)`,
+ );
+ } finally {
+ globalThis.fetch = originalFetch;
+ await new Promise<void>((resolve) => server.close(() => resolve()));
+ }
+});
diff --git a/studio/src-tauri/src/commands.rs b/studio/src-tauri/src/commands.rs
index beff9a53c5d..a7d853a4d24 100644
--- a/studio/src-tauri/src/commands.rs
+++ b/studio/src-tauri/src/commands.rs
@@ -3,6 +3,7 @@ use crate::install;
use crate::process::{self, BackendState, ShutdownFlag};
use crate::update;
use log::{error, info, warn};
+use std::net::{Ipv4Addr, SocketAddr};
use std::time::{Duration, Instant};
use tauri::{AppHandle, Emitter};
@@ -36,6 +37,22 @@ const HEALTH_WATCHDOG_MAX_FAILURES_BUSY: u32 = 12;
/// than retrying, and `backend/tests/test_health_answers_within_probe_budget.py` derives
/// `_HEALTH_DETECT_BUDGET_S` from that number.
const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(10);
+/// How long one loopback connect gets to be REFUSED before silence starts meaning a filtered
+/// handshake (#10520) rather than an empty port, in which case the full ladder is kept.
+///
+/// Per platform because the wait before a refusal is: 0.1ms worst on linux and 0.2ms on
+/// macOS, but 2030.2ms on Windows, which retransmits the SYN first (as libuv documents when
+/// it disables that per socket via SIO_TCP_INITIAL_RTO). The legal window is therefore
+/// (2030, 2500] on Windows, since `the_refusal_probe_cannot_eat_the_ladder_it_short_circuits`
+/// caps it at a quarter of HEALTH_PROBE_TIMEOUT, and this takes the end with the headroom.
+/// Measurements per platform are in the PR description.
+///
+/// Spent BEFORE the first rung, so it is also the whole cost the fast path can add, and only
+/// a port nobody here manages that neither answers nor refuses ever pays it.
+#[cfg(windows)]
+const REFUSAL_PROBE_TIMEOUT: Duration = Duration::from_millis(2_500);
+#[cfg(not(windows))]
+const REFUSAL_PROBE_TIMEOUT: Duration = Duration::from_millis(250);
/// Budget for the single last-chance probe spent before a stalled backend is declared dead.
///
/// Deliberately above HEALTH_WATCHDOG_INTERVAL, unlike the per-cycle budget: this one is not
@@ -507,12 +524,87 @@ async fn backend_presence(
}
}
+/// Whether the webview can stop retrying and tell the user the backend is gone.
+///
+/// Not `check_backend_present`: presence reports a backend of ours that has not bound its
+/// port yet exactly as it reports one that was never there, and that first case is the slow
+/// start the ladder exists to survive. Ownership is asked FIRST for the same reason.
+#[tauri::command]
+pub async fn check_backend_is_gone(
+ state: tauri::State<'_, BackendState>,
+ port: u16,
+) -> Result<bool, String> {
+ let state = state.inner();
+ Ok(backend_is_gone(port, REFUSAL_PROBE_TIMEOUT, || {
+ we_could_be_bringing_up(state, port)
+ })
+ .await)
+}
+
+async fn backend_is_gone(port: u16, budget: Duration, we_manage_it: impl Fn() -> bool) -> bool {
+ // Port 0 is the placeholder base the webview holds before a validated port arrives.
+ if port == 0 {
+ return false;
+ }
+ // A backend we started may simply not have bound its port yet: #10520, keep waiting.
+ if we_manage_it() {
+ return false;
+ }
+ if !matches!(connect_outcome(port, budget).await, ConnectOutcome::Refused) {
+ return false;
+ }
+ // Ownership was read BEFORE the connect, and a refusal is not instant everywhere: Windows
+ // retransmits the SYN first, so that read can be 2s stale by the time it decides. Asking
+ // again costs one mutex read and no network, and only a port nobody was bringing up at
+ // either end is reported gone.
+ !we_manage_it()
+}
+
+/// What one loopback connect established, as three answers rather than two.
+#[derive(Debug, PartialEq, Eq)]
+enum ConnectOutcome {
+ /// The kernel answered that nothing holds the port. Proof, not an expiring budget.
+ Refused,
+ /// Something accepted, so an empty port is not what the webview ran into.
+ Accepted,
+ /// No answer inside the budget, or an error not naming a closed port: #10520 lands here.
+ Unsettled,
+}
+
+async fn connect_outcome(port: u16, budget: Duration) -> ConnectOutcome {
+ let target = SocketAddr::from((Ipv4Addr::LOCALHOST, port));
+ let settled = tokio::time::timeout(budget, tokio::net::TcpStream::connect(target))
+ .await
+ .ok()
+ .map(|attempt| attempt.map(|_stream| ()));
+ classify_connect(settled)
+}
+
+/// `None` is a spent budget. Split from the connect so the rule is testable without a port
+/// that behaves the way each branch needs.
+fn classify_connect(settled: Option<std::io::Result<()>>) -> ConnectOutcome {
+ match settled {
+ Some(Ok(())) => ConnectOutcome::Accepted,
+ Some(Err(err)) if err.kind() == std::io::ErrorKind::ConnectionRefused => {
+ ConnectOutcome::Refused
+ }
+ Some(Err(_)) => ConnectOutcome::Unsettled,
+ None => ConnectOutcome::Unsettled,
+ }
+}
+
/// Whether this app's own backend handle names *port*. The handle outlives the process it names, so the owner is checked too.
fn we_manage_a_backend_on(state: &BackendState, port: u16) -> bool {
// Handle and process state in one pass under the same lock, so they cannot disagree.
process::owned_backend_on_port_is_running(state, port)
}
+/// Deliberately wider than presence: a live backend of ours that has not reported a port yet
+/// may be about to bind THIS one, and a refusal about it is not proof of anything.
+fn we_could_be_bringing_up(state: &BackendState, port: u16) -> bool {
+ process::owned_backend_could_bind_port(state, port)
+}
+
/// The rule `check_backend_present` applies, as a value so the test for it can actually fail.
fn backend_is_present(liveness: &BackendLiveness, we_manage_it: bool) -> bool {
// A healthy answer is presence whoever owns the port; a timeout or unhealthy reply counts only for a backend we manage.
@@ -1850,6 +1942,183 @@ mod tests {
);
}
+ /// Below the ephemeral range (32768 on Linux, 49152 elsewhere) and in its own window, so
+ /// no sibling test can be handed this port back the way a dropped port-0 binding can be.
+ async fn a_closed_port_below_the_ephemeral_range() -> u16 {
+ for candidate in 20_064..20_128u16 {
+ if let Ok(listener) = TcpListener::bind(("127.0.0.1", candidate)).await {
+ drop(listener);
+ return candidate;
+ }
+ }
+ panic!("every port in the probe window was already bound")
+ }
+
+ /// Carried into the assertion message because the two failures need opposite fixes: an
+ /// answer that is not `ConnectionRefused` indicts the classifier, a late one the budget.
+ async fn describe_connect(port: u16, budget: Duration) -> String {
+ let target =
+ std::net::SocketAddr::from((std::net::Ipv4Addr::LOCALHOST, port));
+ let started = std::time::Instant::now();
+ let settled = tokio::time::timeout(budget, tokio::net::TcpStream::connect(target)).await;
+ let elapsed = started.elapsed();
+ match settled {
+ Err(_) => format!("no answer inside {budget:?} (waited {elapsed:?})"),
+ Ok(Ok(_)) => format!("accepted after {elapsed:?}"),
+ Ok(Err(error)) => format!(
+ "kind={:?} raw_os_error={:?} after {elapsed:?} ({error})",
+ error.kind(),
+ error.raw_os_error(),
+ ),
+ }
+ }
+
+ /// A budget under the platform's own wait before a refusal classifies every dead port as
+ /// Unsettled, and the fast path silently stops existing. The floors are well under
+ /// `REFUSAL_PROBE_TIMEOUT`, not equal to it, so only a real drop fails this.
+ #[test]
+ fn the_refusal_budget_clears_the_wait_this_platform_actually_takes() {
+ // Not the budget itself: the ladder invariant below also caps it, and pinning both
+ // ends would leave exactly one legal value.
+ let floor = if cfg!(windows) {
+ Duration::from_millis(2_100)
+ } else {
+ Duration::from_millis(50)
+ };
+ assert!(
+ super::REFUSAL_PROBE_TIMEOUT >= floor,
+ "REFUSAL_PROBE_TIMEOUT is {:?}, under the {floor:?} this platform needs to see a \
+ refusal at all. Below the real wait every dead port reads as Unsettled and the \
+ fast path never fires.",
+ super::REFUSAL_PROBE_TIMEOUT,
+ );
+ }
+
+ #[tokio::test]
+ async fn a_closed_port_nobody_here_owns_is_gone() {
+ let port = a_closed_port_below_the_ephemeral_range().await;
+ let observed = describe_connect(port, super::REFUSAL_PROBE_TIMEOUT).await;
+ assert!(
+ super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || false).await,
+ "a refused loopback connect on a port nothing here owns is not reported as gone. \
+ Port {port} answered: {observed}"
+ );
+ }
+
+ #[tokio::test]
+ async fn a_closed_port_we_are_bringing_up_is_not_gone() {
+ // #10520: our own backend has not bound its port yet, so the connect is refused
+ // exactly as for a dead one. Port 0 is safe here where it is not above, because
+ // ownership is read before anything connects.
+ let port = {
+ let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
+ let port = listener.local_addr().unwrap().port();
+ drop(listener);
+ port
+ };
+ assert!(
+ !super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || true).await,
+ "a backend of ours that is still starting was reported as gone"
+ );
+ }
+
+ #[tokio::test]
+ async fn ownership_taken_while_the_probe_ran_still_keeps_the_ladder() {
+ // A refusal is not instant on Windows, so without the second read a backend that
+ // starts during the probe is reported gone on a look taken a whole budget earlier.
+ let port = a_closed_port_below_the_ephemeral_range().await;
+ let asked = std::sync::atomic::AtomicUsize::new(0);
+ let gone = super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || {
+ // nobody owns it when the probe starts, somebody does by the time it answers
+ asked.fetch_add(1, std::sync::atomic::Ordering::SeqCst) > 0
+ })
+ .await;
+ assert_eq!(
+ asked.load(std::sync::atomic::Ordering::SeqCst),
+ 2,
+ "ownership was not re-read after the connect, so the answer rests on a stale look"
+ );
+ assert!(
+ !gone,
+ "a backend that became ours while the probe was in flight was reported as gone"
+ );
+ }
+
+ #[tokio::test]
+ async fn a_port_something_is_listening_on_is_not_gone() {
+ let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
+ let port = listener.local_addr().unwrap().port();
+ assert!(
+ !super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || false).await,
+ "a port that accepted a connection was reported as gone"
+ );
+ }
+
+ #[tokio::test]
+ async fn the_placeholder_port_is_never_an_answer() {
+ // Port 0 is the base the webview holds before server-port arrives.
+ assert!(!super::backend_is_gone(0, super::REFUSAL_PROBE_TIMEOUT, || false).await);
+ }
+
+ #[test]
+ fn a_connect_that_never_answers_is_unsettled_not_refused() {
+ // A filtered loopback handshake is #10520 itself, and no test here can drop a SYN,
+ // so the rule is checked on its own: only a refusal is proof, a spent budget is not.
+ assert_eq!(
+ super::classify_connect(None),
+ super::ConnectOutcome::Unsettled,
+ "a spent budget was reported as proof that nothing is listening"
+ );
+ assert_eq!(
+ super::classify_connect(Some(Err(std::io::Error::from(
+ std::io::ErrorKind::TimedOut
+ )))),
+ super::ConnectOutcome::Unsettled
+ );
+ assert_eq!(
+ super::classify_connect(Some(Err(std::io::Error::from(
+ std::io::ErrorKind::PermissionDenied
+ )))),
+ super::ConnectOutcome::Unsettled,
+ "a blocked connect is not evidence the port is empty"
+ );
+ assert_eq!(
+ super::classify_connect(Some(Err(std::io::Error::from(
+ std::io::ErrorKind::ConnectionRefused
+ )))),
+ super::ConnectOutcome::Refused
+ );
+ assert_eq!(
+ super::classify_connect(Some(Ok(()))),
+ super::ConnectOutcome::Accepted
+ );
+ }
+
+ #[test]
+ fn the_refusal_probe_cannot_eat_the_ladder_it_short_circuits() {
+ // It is spent before the first rung, so it is the whole cost the fast path adds to
+ // a backend that turns out to be alive.
+ assert!(super::REFUSAL_PROBE_TIMEOUT * 4 <= super::HEALTH_PROBE_TIMEOUT);
+ }
+
+ #[test]
+ fn the_fast_path_asks_for_absence_and_not_for_presence() {
+ // Two different questions. `check_backend_present` reports a backend of ours that
+ // has not bound its port yet as absent, so shortening the ladder on that answer
+ // would undo #10520.
+ let src = include_str!("../../frontend/src/features/auth/api.ts").replace("\r\n", "\n");
+ assert!(
+ src.contains("invoke<boolean>(\"check_backend_is_gone\""),
+ "the retry ladder no longer has a fast path for a backend that is provably gone"
+ );
+ let marker = "if (attempt === 0 && (await nativeBackendIsGone()))";
+ assert!(
+ src.contains(marker),
+ "the fast path stopped being gated on the first failure and a positive answer, \
+ so a slow backend can be abandoned before the ladder has run"
+ );
+ }
+
#[tokio::test]
async fn a_port_with_nothing_on_it_reads_as_death_not_a_stall() {
// The other half: a backend that really exited leaves a closed port, and that must
diff --git a/studio/src-tauri/src/main.rs b/studio/src-tauri/src/main.rs
index 9559ea280bb..06284894de3 100644
--- a/studio/src-tauri/src/main.rs
+++ b/studio/src-tauri/src/main.rs
@@ -2126,6 +2126,7 @@ fn main() {
commands::stop_server,
commands::check_health,
commands::check_backend_present,
+ commands::check_backend_is_gone,
commands::get_server_logs,
commands::open_logs_dir,
commands::open_models_dir,
diff --git a/studio/src-tauri/src/process.rs b/studio/src-tauri/src/process.rs
index 04585701e12..b617cdad35a 100644
--- a/studio/src-tauri/src/process.rs
+++ b/studio/src-tauri/src/process.rs
@@ -1189,6 +1189,38 @@ pub(crate) fn owned_backend_on_port_is_running(state: &BackendState, port: u16)
}
}
+/// Whether anything of ours COULD be on *port*, which is the question ABSENCE needs.
+///
+/// A handle we spawned names no port until a probe validates one, and for that whole window
+/// `owned_backend_on_port_is_running` calls our own starting backend somebody else's: right
+/// for presence, wrong here, since a refusal about a port we are about to bind proves nothing.
+pub(crate) fn owned_backend_could_bind_port(state: &BackendState, port: u16) -> bool {
+ let mut proc = match state.lock() {
+ Ok(guard) => guard,
+ Err(poisoned) => poisoned.into_inner(),
+ };
+ let handle = match proc.owned.as_mut() {
+ Some(handle) => handle,
+ None => return false,
+ };
+ match handle {
+ // A port it has not claimed yet is a port it may still claim.
+ OwnedBackendHandle::Spawned {
+ child,
+ reported_port,
+ ..
+ } => {
+ reported_port.is_none_or(|bound| bound == port)
+ && !matches!(child.try_wait(), Ok(Some(_)))
+ }
+ OwnedBackendHandle::Adopted {
+ port: owned_port,
+ pid,
+ ..
+ } => *owned_port == port && backend_pid_is_running(*pid),
+ }
+}
+
/// Whether *pid* still exists: false only when the pid is PROVABLY gone. A zombie has exited
/// but is unreaped, which `kill(pid, 0)` alone cannot see.
pub(crate) fn backend_pid_is_running(pid: u32) -> bool {
@@ -6354,6 +6386,69 @@ mod owned_backend_liveness_tests {
fn no_handle_at_all_is_not_a_managed_backend() {
let state = new_backend_state();
assert!(!owned_backend_on_port_is_running(&state, 8765));
+ assert!(!owned_backend_could_bind_port(&state, 8765));
+ }
+
+ /// While a handle has no port yet, presence reads every port as "not ours". Absence must
+ /// not agree, or a refusal during our own start becomes proof of death.
+ #[test]
+ fn a_child_that_has_not_reported_a_port_could_still_bind_the_one_asked_about() {
+ let state = new_backend_state();
+ {
+ let mut proc = state.lock().unwrap();
+ proc.owned = Some(OwnedBackendHandle::spawned(
+ spawn_owned(&LIVE_CHILD),
+ None,
+ 0,
+ 1,
+ ));
+ }
+ assert!(
+ !owned_backend_on_port_is_running(&state, 8765),
+ "presence is unchanged: a handle with no port names no port"
+ );
+ assert!(
+ owned_backend_could_bind_port(&state, 8765),
+ "a live backend of ours that has not bound a port yet was ruled out as the owner \
+ of the port it is starting on, which is the slow start #10520 exists to survive"
+ );
+ let mut proc = state.lock().unwrap();
+ if let Some(handle) = proc.owned.as_mut() {
+ if let Some(child) = handle.spawned_child_mut() {
+ let _ = child.start_kill();
+ }
+ }
+ }
+
+ #[test]
+ fn a_child_that_died_before_reporting_a_port_cannot_bind_anything() {
+ let mut child = spawn_owned(&DEAD_CHILD);
+ let _ = child.wait();
+ let state = new_backend_state();
+ {
+ let mut proc = state.lock().unwrap();
+ proc.owned = Some(OwnedBackendHandle::spawned(child, None, 0, 1));
+ }
+ assert!(
+ !owned_backend_could_bind_port(&state, 8765),
+ "an exited child kept the fast path switched off for every port"
+ );
+ }
+
+ #[test]
+ fn a_handle_that_reported_another_port_does_not_cover_this_one() {
+ let state = state_owning(spawn_owned(&LIVE_CHILD), 8765);
+ assert!(
+ !owned_backend_could_bind_port(&state, 8766),
+ "a backend that has told us its port was still treated as a candidate for others"
+ );
+ assert!(owned_backend_could_bind_port(&state, 8765));
+ let mut proc = state.lock().unwrap();
+ if let Some(handle) = proc.owned.as_mut() {
+ if let Some(child) = handle.spawned_child_mut() {
+ let _ = child.start_kill();
+ }
+ }
}
// The adopted half, where there is no child handle to wait on.