Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: report a backend that is provably gone without sleeping out the retry ladder (#11213)
Daniel Handdb2324e2423
diff --git a/studio/backend/tests/test_inference_worker_stderr_tail.py b/studio/backend/tests/test_inference_worker_stderr_tail.pyindex 282c0d36e65..2c2e6901290 100644--- a/studio/backend/tests/test_inference_worker_stderr_tail.py+++ b/studio/backend/tests/test_inference_worker_stderr_tail.py@@ -1241,3 +1241,89 @@ def test_two_paths_on_one_line_are_two_matches():spaced = _redact_worker_output("C:\\Program Files\\unsloth\\weights.gguf failed\n")assert spaced.strip() == ".../weights.gguf failed", spaced+++def test_the_mirror_module_does_not_pull_logging_into_a_spawned_worker():+ """Every inference worker imports this module before its entrypoint runs.++ A fresh spawn child has no ``logging`` yet, and importing it there was 4.2ms of the+ 5.3ms this module added to each worker spawn. Nothing on the spawn path needs it: the+ record marking runs later, after the worker has configured logging.+ """+ import ast++ source = (Path(_BACKEND_DIR) / "utils/worker_stderr.py").read_text(encoding = "utf-8")+ tree = ast.parse(source)+ top_level = [+ alias.name for node in tree.body if isinstance(node, ast.Import) for alias in node.names+ ]+ top_level += [node.module for node in tree.body if isinstance(node, ast.ImportFrom)]+ assert "logging" not in top_level, (+ "utils/worker_stderr.py imports logging at module scope again, which every "+ "spawned worker pays for before its entrypoint runs"+ )++ # And the same thing as a fact rather than as a reading of the source. Skipped rather+ # than failed where the interpreter already has logging for its own reasons, since the+ # claim is about what this module drags in, not about what a bare start-up loads.+ probe = subprocess.run(+ [+ sys.executable,+ "-c",+ "import sys\n"+ "preloaded = 'logging' in sys.modules\n"+ f"sys.path.insert(0, {_BACKEND_DIR!r})\n"+ "import utils.worker_stderr\n"+ "print(preloaded, 'logging' in sys.modules)\n",+ ],+ capture_output = True,+ text = True,+ timeout = 120,+ )+ assert probe.returncode == 0, probe.stderr+ preloaded, after = probe.stdout.split()+ if preloaded == "True":+ pytest.skip("this interpreter loads logging before any of our code runs")+ assert after == "False", "importing utils.worker_stderr imported logging"+++def test_a_second_formatter_class_cannot_wrap_a_handler_that_is_already_marked():+ """The marking class is built on first use, so there can be more than one of it.++ Two threads that both find the cache empty each build a class, and ``isinstance`` is+ false across the pair, so identity would wrap one handler twice and emit every+ continuation line with a doubled prefix. Rebuilding the class here is the same+ observation as that race, without depending on an interleaving.+ """+ import io+ import logging as logging_module++ from utils import worker_stderr+ from utils.worker_stderr import LOG_RECORD_CONTINUATION_PREFIX, LOG_RECORD_START_MARK++ stream = io.StringIO()+ handler = logging_module.StreamHandler(stream)+ handler.setFormatter(logging_module.Formatter("%(message)s"))++ original = worker_stderr._PREFIX_FORMATTER_CLASS+ try:+ assert worker_stderr._mark_handler(handler) is True+ first_class = worker_stderr._PREFIX_FORMATTER_CLASS+ worker_stderr._PREFIX_FORMATTER_CLASS = None+ assert (+ worker_stderr._mark_handler(handler) is False+ ), "a handler already marked by one formatter class was marked again by another"+ second_class = worker_stderr._prefix_formatter_class()+ assert first_class is not second_class, "the rebuild this test needs did not happen"++ logger = logging_module.getLogger(f"{__name__}.two-classes")+ logger.handlers = [handler]+ logger.propagate = False+ logger.setLevel(logging_module.DEBUG)+ logger.error("boom\n File x")+ emitted = stream.getvalue()+ finally:+ worker_stderr._PREFIX_FORMATTER_CLASS = original++ expected = LOG_RECORD_START_MARK + "boom\n" + LOG_RECORD_CONTINUATION_PREFIX + " File x\n"+ assert emitted == expected, emitteddiff --git a/studio/backend/utils/worker_stderr.py b/studio/backend/utils/worker_stderr.pyindex f4a02295664..65caca2752e 100644--- a/studio/backend/utils/worker_stderr.py+++ b/studio/backend/utils/worker_stderr.py@@ -7,12 +7,13 @@ from __future__ import annotationsimport atexitimport collections-import loggingimport osimport sysimport tempfileimport threading+# `logging` is deliberately NOT imported here; see _prefix_formatter_class().+__all__ = ["LOG_RECORD_CONTINUATION_PREFIX","LOG_RECORD_START_MARK",@@ -479,28 +480,55 @@ def install_worker_stderr_mirror(return True-class _EveryLineCarriesThePrefix(logging.Formatter):- def __init__(self, inner: "logging.Formatter") -> None:- super().__init__()- self._inner = inner+_PREFIX_FORMATTER_CLASS = None+# On the formatter, not the class: built lazily, so two threads racing the first call make two+# classes, `isinstance` is false across the pair, and the handler gets wrapped twice.+_MARKS_CONTINUATIONS = "_unsloth_marks_continuations"+++def _formatter_marks_continuations(formatter) -> bool:+ return getattr(formatter, _MARKS_CONTINUATIONS, False) is True- def format(self, record: "logging.LogRecord") -> str:- text = self._inner.format(record)- first, newline, rest = text.partition("\n")- # The first line too: a default-formatted single-line record has no shape to spot.- marked_first = (- first if first.startswith(LOG_RECORD_START_MARK) else LOG_RECORD_START_MARK + first- )- if not newline:- return marked_first- return (- marked_first- + "\n"- + "\n".join(LOG_RECORD_CONTINUATION_PREFIX + line for line in rest.split("\n"))- )- def __getattr__(self, name: str):- return getattr(self._inner, name)+def _prefix_formatter_class():+ """Built on first use, so importing this module does not import ``logging``.++ Every spawned worker imports this module before its entrypoint runs and a fresh spawn+ child has no ``logging`` yet, which was 4.2ms of the 5.3ms this file cost each spawn.+ """+ global _PREFIX_FORMATTER_CLASS+ if _PREFIX_FORMATTER_CLASS is not None:+ return _PREFIX_FORMATTER_CLASS+ import logging++ class _EveryLineCarriesThePrefix(logging.Formatter):+ # Read through `_formatter_marks_continuations`, never `isinstance`: see above.+ _unsloth_marks_continuations = True++ def __init__(self, inner: "logging.Formatter") -> None:+ super().__init__()+ self._inner = inner++ def format(self, record: "logging.LogRecord") -> str:+ text = self._inner.format(record)+ first, newline, rest = text.partition("\n")+ # The first line too: a default-formatted single-line record has no shape to spot.+ marked_first = (+ first if first.startswith(LOG_RECORD_START_MARK) else LOG_RECORD_START_MARK + first+ )+ if not newline:+ return marked_first+ return (+ marked_first+ + "\n"+ + "\n".join(LOG_RECORD_CONTINUATION_PREFIX + line for line in rest.split("\n"))+ )++ def __getattr__(self, name: str):+ return getattr(self._inner, name)++ _PREFIX_FORMATTER_CLASS = _EveryLineCarriesThePrefix+ return _PREFIX_FORMATTER_CLASS_UNHOOKED_SET_FORMATTER = None@@ -508,11 +536,14 @@ _UNHOOKED_ADD_HANDLER = Nonedef _mark_handler(handler) -> bool:+ import logging++ prefixed = _prefix_formatter_class()formatter = getattr(handler, "formatter", None)- if isinstance(formatter, _EveryLineCarriesThePrefix):+ if _formatter_marks_continuations(formatter):return Falsesetter = _UNHOOKED_SET_FORMATTER or type(handler).setFormatter- setter(handler, _EveryLineCarriesThePrefix(formatter or logging.Formatter()))+ setter(handler, prefixed(formatter or logging.Formatter()))return True@@ -521,12 +552,15 @@ def _install_continuation_hook() -> bool:global _UNHOOKED_SET_FORMATTER, _UNHOOKED_ADD_HANDLERif _UNHOOKED_SET_FORMATTER is not None:return False+ import logging++ prefixed = _prefix_formatter_class()unhooked_set = logging.Handler.setFormatterunhooked_add = logging.Logger.addHandlerdef setFormatter(self, fmt): # noqa: N802 -- matches logging's own spelling- if fmt is not None and not isinstance(fmt, _EveryLineCarriesThePrefix):- fmt = _EveryLineCarriesThePrefix(fmt)+ if fmt is not None and not _formatter_marks_continuations(fmt):+ fmt = prefixed(fmt)unhooked_set(self, fmt)def addHandler(self, hdlr): # noqa: N802 -- matches logging's own spelling@@ -546,6 +580,8 @@ def _install_continuation_hook() -> bool:def mark_log_record_continuations(logger_object = None, *, cover_later_handlers = True) -> int:"""Covers ``logging.lastResort`` and later handlers too: a miss hands a RECOVEREDrequest's traceback to the next caller as their crash."""+ import logging+root = logger_object if logger_object is not None else logging.getLogger()wrapped = 0for handler in list(getattr(root, "handlers", ())):diff --git a/studio/frontend/src/features/auth/api.ts b/studio/frontend/src/features/auth/api.tsindex a0899389302..fda7e6ab5ce 100644--- a/studio/frontend/src/features/auth/api.ts+++ b/studio/frontend/src/features/auth/api.ts@@ -94,6 +94,10 @@ async function fetchWithTauriNetworkRetry() {throw error;}+ // Tauri only, below the guard above. `fetch` cannot tell a refused port from a silent+ // one and the native side can, so ask it once on the FIRST failure rather than sleeping+ // out #10520's 10.5s ladder to be told what the refusal already proved.+ if (attempt === 0 && (await nativeBackendIsGone())) throw error;await wait(delays[attempt]);beforeRetry?.();}@@ -191,6 +195,51 @@ async function nativeBackendIsAlive(): Promise<boolean> {}}+/** `check_backend_is_gone` and NOT `check_backend_present`: presence reports a backend of ours that has not bound its port yet as absent. */+let nativeGoneInflight: { port: number; probe: Promise<boolean> } | null = null;++/**+ * Whether the retry ladder has anything left to wait for.+ *+ * Only ever answers true on positive proof, so every failure mode below returns false and+ * leaves the ladder exactly as long as it is today: the browser build, which has no native+ * side to ask; a port the webview has not been given yet; and a shell too old to carry the+ * command, whose rejected `invoke` is caught here.+ */+async function nativeBackendIsGone(): Promise<boolean> {+ if (!isTauri) {+ return false;+ }+ const port = getApiPort();+ if (port === null) {+ return false;+ }+ // Single flight, like the presence probe: a hub losing the backend fails every panel at once.+ if (nativeGoneInflight !== null && nativeGoneInflight.port === port) {+ return nativeGoneInflight.probe;+ }+ const probe = (async () => {+ try {+ const { invoke } = await import("@tauri-apps/api/core");+ return (+ (await invoke<boolean>("check_backend_is_gone", { port })) === true+ );+ } catch {+ return false;+ }+ })();+ const inflight = { port, probe };+ nativeGoneInflight = inflight;+ try {+ return await probe;+ } finally {+ // Identity, not the port: a probe for a newer port owns the slot now.+ if (nativeGoneInflight === inflight) {+ nativeGoneInflight = null;+ }+ }+}+async function asTransportFailure(err: unknown): Promise<unknown> {// fetch TypeError = offline | backend down | CORS/DNS. Tagged so callers tell "never reached"// from "rejected"; under Tauri the launcher is asked before claiming the backend is gone.diff --git a/studio/frontend/tests/tauri-backend-transport-failure.test.ts b/studio/frontend/tests/tauri-backend-transport-failure.test.tsindex be5cca65b99..1f5b8b3fa37 100644--- a/studio/frontend/tests/tauri-backend-transport-failure.test.ts+++ b/studio/frontend/tests/tauri-backend-transport-failure.test.ts@@ -24,6 +24,9 @@ function loadAuthApi(options: {port: number | null;getPort?: () => number | null;checkHealth?: (port: number) => boolean | Promise<boolean>;+ /** `check_backend_is_gone`. Default false: the ladder the tree had before the fast path. */+ checkGone?: (port: number) => boolean | Promise<boolean>;+ isTauri?: boolean;onInvoke?: (command: string, args: Record<string, unknown>) => void;}): AuthApi {const currentPort = options.getPort ?? (() => options.port);@@ -38,7 +41,7 @@ function loadAuthApi(options: {: `http://127.0.0.1:${port}${path}`;},getApiPort: () => currentPort(),- isTauri: true,+ isTauri: options.isTauri ?? true,},"@/lib/account-transition": { accountTransitionPending: () => false },"./session": {@@ -52,6 +55,9 @@ function loadAuthApi(options: {"@tauri-apps/api/core": {invoke: async (command: string, args: Record<string, unknown>) => {options.onInvoke?.(command, args);+ if (command === "check_backend_is_gone") {+ return options.checkGone?.(args.port as number) ?? false;+ }if (command !== "check_backend_present") {throw new Error(`unexpected command ${command}`);}@@ -104,7 +110,10 @@ test("a backend that accepts late is reached instead of declared not running", atry {listenAfter(server, port, SLOW_LOOPBACK_ACCEPT_DELAY_MS);- const authApi = loadAuthApi({ port });+ // The connects here are genuinely refused until 3s. `check_backend_is_gone` answers+ // false anyway, because a backend this app is bringing up owns the port before it binds+ // it -- which is why the fast path asks for absence rather than for presence.+ const authApi = loadAuthApi({ port, checkGone: () => false });const response = await authApi.authFetch("/api/models");assert.equal(response.status, 200);@@ -326,6 +335,8 @@ test("updating an existing install does not go through the changed path", asyncassert.ok(main.includes("commands::check_health,"));// check_health collapses a stalled probe onto false, the verdict this file exists to stop showing.assert.ok(main.includes("commands::check_backend_present,"));+ // The fast path's command. Unregistered, every invoke rejects and the ladder is the old one.+ assert.ok(main.includes("commands::check_backend_is_gone,"));const authApiSrc = await readFile(new URL("../src/features/auth/api.ts", import.meta.url),"utf8",@@ -441,3 +452,256 @@ test("a probe pending against the old port is not the answer about the new one",globalThis.fetch = originalFetch;}});++// The ladder the fast path short-circuits, as the source states it.+const LONG_LADDER_MS = 250 + 750 + 1500 + 3000 + 5000;+const LONG_LADDER_ATTEMPTS = 6;++test("a backend that is provably gone is reported without sleeping out the ladder", async () => {+ // A refused port with nothing of ours coming up on it: the native side has already+ // answered the question the remaining 10.5s of sleeping would ask five more times.+ const port = 61820;+ const asked: string[] = [];+ const originalFetch = globalThis.fetch;+ let attempts = 0;+ globalThis.fetch = async () => {+ attempts += 1;+ throw new TypeError("fetch failed");+ };++ try {+ const authApi = loadAuthApi({+ port,+ checkGone: () => true,+ checkHealth: () => false,+ onInvoke: (command) => asked.push(command),+ });+ const started = performance.now();+ const error = await authApi.authFetch("/api/models").then(+ () => null,+ (rejection: unknown) => rejection,+ );+ const elapsed = performance.now() - started;++ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);+ assert.equal(+ attempts,+ 1,+ `the request was sent ${attempts} times after the backend was proven gone`,+ );+ assert.ok(+ elapsed < LONG_LADDER_MS / 4,+ `reporting a dead backend took ${Math.round(elapsed)}ms of a ${LONG_LADDER_MS}ms ladder`,+ );+ assert.deepEqual(asked, ["check_backend_is_gone", "check_backend_present"]);+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("a backend that is merely slow still gets every rung and the busy message", async () => {+ // The other direction. `check_backend_is_gone` is false for a port we are bringing up and+ // for one whose handshake is filtered, and neither may lose a single retry.+ const port = 61821;+ const originalFetch = globalThis.fetch;+ let attempts = 0;+ let goneProbes = 0;+ globalThis.fetch = async () => {+ attempts += 1;+ throw new TypeError("fetch failed");+ };++ try {+ const authApi = loadAuthApi({+ port,+ checkGone: () => {+ goneProbes += 1;+ return false;+ },+ checkHealth: () => true,+ });+ const started = performance.now();+ const error = await authApi.authFetch("/api/models").then(+ () => null,+ (rejection: unknown) => rejection,+ );+ const elapsed = performance.now() - started;++ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_ANSWERING_MESSAGE);+ assert.equal(+ attempts,+ LONG_LADDER_ATTEMPTS,+ `a backend that may still be starting was given ${attempts} attempts`,+ );+ assert.ok(+ elapsed >= LONG_LADDER_MS,+ `the ladder finished in ${Math.round(elapsed)}ms, short of ${LONG_LADDER_MS}ms`,+ );+ assert.equal(+ goneProbes,+ 1,+ "the fast path probed on more than the first failure",+ );+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("a shell too old for the command keeps the ladder it has today", async () => {+ // The webview ships ahead of the shell it runs in. A rejected invoke is not an answer.+ const port = 61822;+ const originalFetch = globalThis.fetch;+ let attempts = 0;+ globalThis.fetch = async () => {+ attempts += 1;+ throw new TypeError("fetch failed");+ };++ try {+ const authApi = loadAuthApi({+ port,+ checkGone: () => {+ throw new Error("command not registered on this build");+ },+ checkHealth: () => false,+ });+ const error = await authApi.authFetch("/api/models").then(+ () => null,+ (rejection: unknown) => rejection,+ );++ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);+ assert.equal(attempts, LONG_LADDER_ATTEMPTS);+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("the browser build asks nothing and behaves exactly as it did", async () => {+ // `fetch` in a browser reports a refused connection and a timed-out one as the same+ // opaque TypeError, so there is nothing to be fast about and no native side to ask.+ const invoked: string[] = [];+ const originalFetch = globalThis.fetch;+ let attempts = 0;+ globalThis.fetch = async () => {+ attempts += 1;+ throw new TypeError("fetch failed");+ };++ try {+ const authApi = loadAuthApi({+ port: 61823,+ isTauri: false,+ checkGone: () => true,+ checkHealth: () => true,+ onInvoke: (command) => invoked.push(command),+ });+ const error = await authApi.authFetch("/api/models").then(+ () => null,+ (rejection: unknown) => rejection,+ );++ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);+ assert.equal(attempts, 1, "the browser build started retrying");+ assert.deepEqual(+ invoked,+ [],+ "the browser build reached for the native side",+ );+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("panels that all lose the backend at once share one absence probe", async () => {+ const originalFetch = globalThis.fetch;+ globalThis.fetch = async () => {+ throw new TypeError("fetch failed");+ };++ let probes = 0;+ let releaseProbe: (() => void) | null = null;+ const probeStarted = new Promise<void>((resolve) => {+ releaseProbe = resolve;+ });++ try {+ const authApi = loadAuthApi({+ port: 61824,+ checkGone: async () => {+ probes += 1;+ releaseProbe?.();+ await new Promise((resolve) => setTimeout(resolve, 50));+ return true;+ },+ checkHealth: () => false,+ });+ const call = () =>+ authApi.authFetch("/api/models").then(+ () => null,+ (rejection: unknown) => rejection,+ );++ const first = call();+ // Bounded: a build that never probes must fail on the count below, not hang here.+ await Promise.race([+ probeStarted,+ new Promise<void>((resolve) => {+ setTimeout(resolve, 5_000).unref();+ }),+ ]);+ const errors = await Promise.all([first, call(), call()]);++ assert.equal(probes, 1, "each lost panel opened its own absence probe");+ for (const error of errors) {+ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);+ }+ } finally {+ globalThis.fetch = originalFetch;+ }+});++test("a port nobody here owns that binds late is given up on, and that is the trade", async () => {+ // The other side of the fast path, pinned so it cannot change by accident. A refusal+ // proves nothing is listening NOW, not that nothing will bind later, so a backend the+ // shell did not start -- one attached to from outside Desktop, restarting -- is reported+ // gone rather than waited for. The owned case above keeps every rung; this one does not,+ // and the ownership answer is the whole difference between them.+ const port = await reserveLoopbackPort();+ const server = createServer((_request, response) => {+ response.writeHead(200, { "Content-Type": "application/json" });+ response.end(JSON.stringify({ ok: true }));+ });+ const originalFetch = globalThis.fetch;+ let attempts = 0;+ globalThis.fetch = async (input, init) => {+ attempts += 1;+ return await originalFetch(input, init);+ };++ try {+ listenAfter(server, port, SLOW_LOOPBACK_ACCEPT_DELAY_MS);+ // What a truthful native side answers for this port: refused, and owned by nobody here.+ const authApi = loadAuthApi({ port, checkGone: () => true, checkHealth: () => false });+ const error = await authApi.authFetch("/api/models").then(+ () => null,+ (rejection: unknown) => rejection,+ );++ assert.ok(error instanceof Error);+ assert.equal(error.message, authApi.BACKEND_NOT_RUNNING_MESSAGE);+ assert.equal(+ attempts,+ 1,+ `the ladder kept climbing for a port reported gone (${attempts} attempts)`,+ );+ } finally {+ globalThis.fetch = originalFetch;+ await new Promise<void>((resolve) => server.close(() => resolve()));+ }+});diff --git a/studio/src-tauri/src/commands.rs b/studio/src-tauri/src/commands.rsindex beff9a53c5d..a7d853a4d24 100644--- a/studio/src-tauri/src/commands.rs+++ b/studio/src-tauri/src/commands.rs@@ -3,6 +3,7 @@ use crate::install;use crate::process::{self, BackendState, ShutdownFlag};use crate::update;use log::{error, info, warn};+use std::net::{Ipv4Addr, SocketAddr};use std::time::{Duration, Instant};use tauri::{AppHandle, Emitter};@@ -36,6 +37,22 @@ const HEALTH_WATCHDOG_MAX_FAILURES_BUSY: u32 = 12;/// than retrying, and `backend/tests/test_health_answers_within_probe_budget.py` derives/// `_HEALTH_DETECT_BUDGET_S` from that number.const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(10);+/// How long one loopback connect gets to be REFUSED before silence starts meaning a filtered+/// handshake (#10520) rather than an empty port, in which case the full ladder is kept.+///+/// Per platform because the wait before a refusal is: 0.1ms worst on linux and 0.2ms on+/// macOS, but 2030.2ms on Windows, which retransmits the SYN first (as libuv documents when+/// it disables that per socket via SIO_TCP_INITIAL_RTO). The legal window is therefore+/// (2030, 2500] on Windows, since `the_refusal_probe_cannot_eat_the_ladder_it_short_circuits`+/// caps it at a quarter of HEALTH_PROBE_TIMEOUT, and this takes the end with the headroom.+/// Measurements per platform are in the PR description.+///+/// Spent BEFORE the first rung, so it is also the whole cost the fast path can add, and only+/// a port nobody here manages that neither answers nor refuses ever pays it.+#[cfg(windows)]+const REFUSAL_PROBE_TIMEOUT: Duration = Duration::from_millis(2_500);+#[cfg(not(windows))]+const REFUSAL_PROBE_TIMEOUT: Duration = Duration::from_millis(250);/// Budget for the single last-chance probe spent before a stalled backend is declared dead.////// Deliberately above HEALTH_WATCHDOG_INTERVAL, unlike the per-cycle budget: this one is not@@ -507,12 +524,87 @@ async fn backend_presence(}}+/// Whether the webview can stop retrying and tell the user the backend is gone.+///+/// Not `check_backend_present`: presence reports a backend of ours that has not bound its+/// port yet exactly as it reports one that was never there, and that first case is the slow+/// start the ladder exists to survive. Ownership is asked FIRST for the same reason.+#[tauri::command]+pub async fn check_backend_is_gone(+ state: tauri::State<'_, BackendState>,+ port: u16,+) -> Result<bool, String> {+ let state = state.inner();+ Ok(backend_is_gone(port, REFUSAL_PROBE_TIMEOUT, || {+ we_could_be_bringing_up(state, port)+ })+ .await)+}++async fn backend_is_gone(port: u16, budget: Duration, we_manage_it: impl Fn() -> bool) -> bool {+ // Port 0 is the placeholder base the webview holds before a validated port arrives.+ if port == 0 {+ return false;+ }+ // A backend we started may simply not have bound its port yet: #10520, keep waiting.+ if we_manage_it() {+ return false;+ }+ if !matches!(connect_outcome(port, budget).await, ConnectOutcome::Refused) {+ return false;+ }+ // Ownership was read BEFORE the connect, and a refusal is not instant everywhere: Windows+ // retransmits the SYN first, so that read can be 2s stale by the time it decides. Asking+ // again costs one mutex read and no network, and only a port nobody was bringing up at+ // either end is reported gone.+ !we_manage_it()+}++/// What one loopback connect established, as three answers rather than two.+#[derive(Debug, PartialEq, Eq)]+enum ConnectOutcome {+ /// The kernel answered that nothing holds the port. Proof, not an expiring budget.+ Refused,+ /// Something accepted, so an empty port is not what the webview ran into.+ Accepted,+ /// No answer inside the budget, or an error not naming a closed port: #10520 lands here.+ Unsettled,+}++async fn connect_outcome(port: u16, budget: Duration) -> ConnectOutcome {+ let target = SocketAddr::from((Ipv4Addr::LOCALHOST, port));+ let settled = tokio::time::timeout(budget, tokio::net::TcpStream::connect(target))+ .await+ .ok()+ .map(|attempt| attempt.map(|_stream| ()));+ classify_connect(settled)+}++/// `None` is a spent budget. Split from the connect so the rule is testable without a port+/// that behaves the way each branch needs.+fn classify_connect(settled: Option<std::io::Result<()>>) -> ConnectOutcome {+ match settled {+ Some(Ok(())) => ConnectOutcome::Accepted,+ Some(Err(err)) if err.kind() == std::io::ErrorKind::ConnectionRefused => {+ ConnectOutcome::Refused+ }+ Some(Err(_)) => ConnectOutcome::Unsettled,+ None => ConnectOutcome::Unsettled,+ }+}+/// Whether this app's own backend handle names *port*. The handle outlives the process it names, so the owner is checked too.fn we_manage_a_backend_on(state: &BackendState, port: u16) -> bool {// Handle and process state in one pass under the same lock, so they cannot disagree.process::owned_backend_on_port_is_running(state, port)}+/// Deliberately wider than presence: a live backend of ours that has not reported a port yet+/// may be about to bind THIS one, and a refusal about it is not proof of anything.+fn we_could_be_bringing_up(state: &BackendState, port: u16) -> bool {+ process::owned_backend_could_bind_port(state, port)+}+/// The rule `check_backend_present` applies, as a value so the test for it can actually fail.fn backend_is_present(liveness: &BackendLiveness, we_manage_it: bool) -> bool {// A healthy answer is presence whoever owns the port; a timeout or unhealthy reply counts only for a backend we manage.@@ -1850,6 +1942,183 @@ mod tests {);}+ /// Below the ephemeral range (32768 on Linux, 49152 elsewhere) and in its own window, so+ /// no sibling test can be handed this port back the way a dropped port-0 binding can be.+ async fn a_closed_port_below_the_ephemeral_range() -> u16 {+ for candidate in 20_064..20_128u16 {+ if let Ok(listener) = TcpListener::bind(("127.0.0.1", candidate)).await {+ drop(listener);+ return candidate;+ }+ }+ panic!("every port in the probe window was already bound")+ }++ /// Carried into the assertion message because the two failures need opposite fixes: an+ /// answer that is not `ConnectionRefused` indicts the classifier, a late one the budget.+ async fn describe_connect(port: u16, budget: Duration) -> String {+ let target =+ std::net::SocketAddr::from((std::net::Ipv4Addr::LOCALHOST, port));+ let started = std::time::Instant::now();+ let settled = tokio::time::timeout(budget, tokio::net::TcpStream::connect(target)).await;+ let elapsed = started.elapsed();+ match settled {+ Err(_) => format!("no answer inside {budget:?} (waited {elapsed:?})"),+ Ok(Ok(_)) => format!("accepted after {elapsed:?}"),+ Ok(Err(error)) => format!(+ "kind={:?} raw_os_error={:?} after {elapsed:?} ({error})",+ error.kind(),+ error.raw_os_error(),+ ),+ }+ }++ /// A budget under the platform's own wait before a refusal classifies every dead port as+ /// Unsettled, and the fast path silently stops existing. The floors are well under+ /// `REFUSAL_PROBE_TIMEOUT`, not equal to it, so only a real drop fails this.+ #[test]+ fn the_refusal_budget_clears_the_wait_this_platform_actually_takes() {+ // Not the budget itself: the ladder invariant below also caps it, and pinning both+ // ends would leave exactly one legal value.+ let floor = if cfg!(windows) {+ Duration::from_millis(2_100)+ } else {+ Duration::from_millis(50)+ };+ assert!(+ super::REFUSAL_PROBE_TIMEOUT >= floor,+ "REFUSAL_PROBE_TIMEOUT is {:?}, under the {floor:?} this platform needs to see a \+ refusal at all. Below the real wait every dead port reads as Unsettled and the \+ fast path never fires.",+ super::REFUSAL_PROBE_TIMEOUT,+ );+ }++ #[tokio::test]+ async fn a_closed_port_nobody_here_owns_is_gone() {+ let port = a_closed_port_below_the_ephemeral_range().await;+ let observed = describe_connect(port, super::REFUSAL_PROBE_TIMEOUT).await;+ assert!(+ super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || false).await,+ "a refused loopback connect on a port nothing here owns is not reported as gone. \+ Port {port} answered: {observed}"+ );+ }++ #[tokio::test]+ async fn a_closed_port_we_are_bringing_up_is_not_gone() {+ // #10520: our own backend has not bound its port yet, so the connect is refused+ // exactly as for a dead one. Port 0 is safe here where it is not above, because+ // ownership is read before anything connects.+ let port = {+ let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();+ let port = listener.local_addr().unwrap().port();+ drop(listener);+ port+ };+ assert!(+ !super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || true).await,+ "a backend of ours that is still starting was reported as gone"+ );+ }++ #[tokio::test]+ async fn ownership_taken_while_the_probe_ran_still_keeps_the_ladder() {+ // A refusal is not instant on Windows, so without the second read a backend that+ // starts during the probe is reported gone on a look taken a whole budget earlier.+ let port = a_closed_port_below_the_ephemeral_range().await;+ let asked = std::sync::atomic::AtomicUsize::new(0);+ let gone = super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || {+ // nobody owns it when the probe starts, somebody does by the time it answers+ asked.fetch_add(1, std::sync::atomic::Ordering::SeqCst) > 0+ })+ .await;+ assert_eq!(+ asked.load(std::sync::atomic::Ordering::SeqCst),+ 2,+ "ownership was not re-read after the connect, so the answer rests on a stale look"+ );+ assert!(+ !gone,+ "a backend that became ours while the probe was in flight was reported as gone"+ );+ }++ #[tokio::test]+ async fn a_port_something_is_listening_on_is_not_gone() {+ let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();+ let port = listener.local_addr().unwrap().port();+ assert!(+ !super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || false).await,+ "a port that accepted a connection was reported as gone"+ );+ }++ #[tokio::test]+ async fn the_placeholder_port_is_never_an_answer() {+ // Port 0 is the base the webview holds before server-port arrives.+ assert!(!super::backend_is_gone(0, super::REFUSAL_PROBE_TIMEOUT, || false).await);+ }++ #[test]+ fn a_connect_that_never_answers_is_unsettled_not_refused() {+ // A filtered loopback handshake is #10520 itself, and no test here can drop a SYN,+ // so the rule is checked on its own: only a refusal is proof, a spent budget is not.+ assert_eq!(+ super::classify_connect(None),+ super::ConnectOutcome::Unsettled,+ "a spent budget was reported as proof that nothing is listening"+ );+ assert_eq!(+ super::classify_connect(Some(Err(std::io::Error::from(+ std::io::ErrorKind::TimedOut+ )))),+ super::ConnectOutcome::Unsettled+ );+ assert_eq!(+ super::classify_connect(Some(Err(std::io::Error::from(+ std::io::ErrorKind::PermissionDenied+ )))),+ super::ConnectOutcome::Unsettled,+ "a blocked connect is not evidence the port is empty"+ );+ assert_eq!(+ super::classify_connect(Some(Err(std::io::Error::from(+ std::io::ErrorKind::ConnectionRefused+ )))),+ super::ConnectOutcome::Refused+ );+ assert_eq!(+ super::classify_connect(Some(Ok(()))),+ super::ConnectOutcome::Accepted+ );+ }++ #[test]+ fn the_refusal_probe_cannot_eat_the_ladder_it_short_circuits() {+ // It is spent before the first rung, so it is the whole cost the fast path adds to+ // a backend that turns out to be alive.+ assert!(super::REFUSAL_PROBE_TIMEOUT * 4 <= super::HEALTH_PROBE_TIMEOUT);+ }++ #[test]+ fn the_fast_path_asks_for_absence_and_not_for_presence() {+ // Two different questions. `check_backend_present` reports a backend of ours that+ // has not bound its port yet as absent, so shortening the ladder on that answer+ // would undo #10520.+ let src = include_str!("../../frontend/src/features/auth/api.ts").replace("\r\n", "\n");+ assert!(+ src.contains("invoke<boolean>(\"check_backend_is_gone\""),+ "the retry ladder no longer has a fast path for a backend that is provably gone"+ );+ let marker = "if (attempt === 0 && (await nativeBackendIsGone()))";+ assert!(+ src.contains(marker),+ "the fast path stopped being gated on the first failure and a positive answer, \+ so a slow backend can be abandoned before the ladder has run"+ );+ }+#[tokio::test]async fn a_port_with_nothing_on_it_reads_as_death_not_a_stall() {// The other half: a backend that really exited leaves a closed port, and that mustdiff --git a/studio/src-tauri/src/main.rs b/studio/src-tauri/src/main.rsindex 9559ea280bb..06284894de3 100644--- a/studio/src-tauri/src/main.rs+++ b/studio/src-tauri/src/main.rs@@ -2126,6 +2126,7 @@ fn main() {commands::stop_server,commands::check_health,commands::check_backend_present,+ commands::check_backend_is_gone,commands::get_server_logs,commands::open_logs_dir,commands::open_models_dir,diff --git a/studio/src-tauri/src/process.rs b/studio/src-tauri/src/process.rsindex 04585701e12..b617cdad35a 100644--- a/studio/src-tauri/src/process.rs+++ b/studio/src-tauri/src/process.rs@@ -1189,6 +1189,38 @@ pub(crate) fn owned_backend_on_port_is_running(state: &BackendState, port: u16)}}+/// Whether anything of ours COULD be on *port*, which is the question ABSENCE needs.+///+/// A handle we spawned names no port until a probe validates one, and for that whole window+/// `owned_backend_on_port_is_running` calls our own starting backend somebody else's: right+/// for presence, wrong here, since a refusal about a port we are about to bind proves nothing.+pub(crate) fn owned_backend_could_bind_port(state: &BackendState, port: u16) -> bool {+ let mut proc = match state.lock() {+ Ok(guard) => guard,+ Err(poisoned) => poisoned.into_inner(),+ };+ let handle = match proc.owned.as_mut() {+ Some(handle) => handle,+ None => return false,+ };+ match handle {+ // A port it has not claimed yet is a port it may still claim.+ OwnedBackendHandle::Spawned {+ child,+ reported_port,+ ..+ } => {+ reported_port.is_none_or(|bound| bound == port)+ && !matches!(child.try_wait(), Ok(Some(_)))+ }+ OwnedBackendHandle::Adopted {+ port: owned_port,+ pid,+ ..+ } => *owned_port == port && backend_pid_is_running(*pid),+ }+}+/// Whether *pid* still exists: false only when the pid is PROVABLY gone. A zombie has exited/// but is unreaped, which `kill(pid, 0)` alone cannot see.pub(crate) fn backend_pid_is_running(pid: u32) -> bool {@@ -6354,6 +6386,69 @@ mod owned_backend_liveness_tests {fn no_handle_at_all_is_not_a_managed_backend() {let state = new_backend_state();assert!(!owned_backend_on_port_is_running(&state, 8765));+ assert!(!owned_backend_could_bind_port(&state, 8765));+ }++ /// While a handle has no port yet, presence reads every port as "not ours". Absence must+ /// not agree, or a refusal during our own start becomes proof of death.+ #[test]+ fn a_child_that_has_not_reported_a_port_could_still_bind_the_one_asked_about() {+ let state = new_backend_state();+ {+ let mut proc = state.lock().unwrap();+ proc.owned = Some(OwnedBackendHandle::spawned(+ spawn_owned(&LIVE_CHILD),+ None,+ 0,+ 1,+ ));+ }+ assert!(+ !owned_backend_on_port_is_running(&state, 8765),+ "presence is unchanged: a handle with no port names no port"+ );+ assert!(+ owned_backend_could_bind_port(&state, 8765),+ "a live backend of ours that has not bound a port yet was ruled out as the owner \+ of the port it is starting on, which is the slow start #10520 exists to survive"+ );+ let mut proc = state.lock().unwrap();+ if let Some(handle) = proc.owned.as_mut() {+ if let Some(child) = handle.spawned_child_mut() {+ let _ = child.start_kill();+ }+ }+ }++ #[test]+ fn a_child_that_died_before_reporting_a_port_cannot_bind_anything() {+ let mut child = spawn_owned(&DEAD_CHILD);+ let _ = child.wait();+ let state = new_backend_state();+ {+ let mut proc = state.lock().unwrap();+ proc.owned = Some(OwnedBackendHandle::spawned(child, None, 0, 1));+ }+ assert!(+ !owned_backend_could_bind_port(&state, 8765),+ "an exited child kept the fast path switched off for every port"+ );+ }++ #[test]+ fn a_handle_that_reported_another_port_does_not_cover_this_one() {+ let state = state_owning(spawn_owned(&LIVE_CHILD), 8765);+ assert!(+ !owned_backend_could_bind_port(&state, 8766),+ "a backend that has told us its port was still treated as a candidate for others"+ );+ assert!(owned_backend_could_bind_port(&state, 8765));+ let mut proc = state.lock().unwrap();+ if let Some(handle) = proc.owned.as_mut() {+ if let Some(child) = handle.spawned_child_mut() {+ let _ = child.start_kill();+ }+ }}// The adopted half, where there is no child handle to wait on.