Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

98.9 KB
Raw Blame
use crate::diagnostics::{self, DiagnosticsState};
use crate::install;
use crate::process::{self, BackendState, ShutdownFlag};
use crate::update;
use log::{error, info, warn};
use std::net::{Ipv4Addr, SocketAddr};
use std::time::{Duration, Instant};
use tauri::{AppHandle, Emitter};

const BACKEND_STARTUP_GRACE_PERIOD: Duration = Duration::from_secs(5 * 60);
const HEALTH_WATCHDOG_INTERVAL: Duration = Duration::from_secs(15);
const HEALTH_WATCHDOG_MAX_FAILURES: u32 = 3;
/// Busy budget: a timed-out strike costs ~25s (sleep + probe), so 12 strikes is ~300s.
/// Only stalls after a busy answer get it; refused connections use the plain budget.
const HEALTH_WATCHDOG_MAX_FAILURES_BUSY: u32 = 12;
/// Generous: warm-thread imports hold the GIL for seconds. Must stay below HEALTH_WATCHDOG_INTERVAL.
/// Not the preflight budget, which stays 2s (test_health_answers_within_probe_budget.py).
const HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(10);
/// How long a loopback connect gets to be refused. Windows retransmits the SYN (~2030ms first),
/// so its window is (2030, 2500], capped at a quarter of HEALTH_PROBE_TIMEOUT.
#[cfg(windows)]
const REFUSAL_PROBE_TIMEOUT: Duration = Duration::from_millis(2_500);
#[cfg(not(windows))]
const REFUSAL_PROBE_TIMEOUT: Duration = Duration::from_millis(250);
/// Last-chance probe budget; above the interval on purpose since it runs at most once, before a kill.
const HEALTH_CONFIRM_PROBE_TIMEOUT: Duration = Duration::from_secs(30);

fn should_count_watchdog_failure(has_seen_healthy: bool, elapsed_since_start: Duration) -> bool {
    has_seen_healthy || elapsed_since_start >= BACKEND_STARTUP_GRACE_PERIOD
}

/// A warming answer clears the latch (adopted backends start latched); no answer leaves it alone.
fn watchdog_seen_healthy_after(previous: bool, alive: bool, warming_up: bool) -> bool {
    if !alive {
        return previous;
    }
    !warming_up
}

/// Only an answer updates the latch; silence keeps the last answer, since a stall gives no answer.
fn watchdog_inference_active_after(previous: bool, alive: bool, inference_active: bool) -> bool {
    if !alive {
        return previous;
    }
    inference_active
}

/// A timeout means something accepted and stalled; any other error means nothing is serving (death).
fn liveness_from_probe_error(error: &reqwest::Error) -> BackendLiveness {
    BackendLiveness {
        probe_timed_out: error.is_timeout(),
        ..BackendLiveness::default()
    }
}

/// A failed adopted check is a stall if the pre-probe answered, unless a different owner answered.
fn adopted_failure_is_a_stall(verified: bool, served_alive: bool, different_owner: bool) -> bool {
    !verified && served_alive && !different_owner
}

/// Keep busy stalls when ownership cannot be reverified; any verified live answer wins.
fn watchdog_confirm_keeps_backend(confirmed: &BackendLiveness) -> bool {
    confirmed.alive || (confirmed.inference_active && confirmed.probe_timed_out)
}

/// One long last-chance probe before a kill, for a generation that started between probes.
/// Only on a stall; a refused port still dies at three strikes.
fn watchdog_should_confirm_before_death(
    consecutive_failures: u32,
    budget: u32,
    probe_timed_out: bool,
) -> bool {
    consecutive_failures >= budget && probe_timed_out
}

/// Probes await up to ~40s; a restart (new generation) or stop may land meanwhile, so recheck
/// before killing or the replacement gets killed.
fn watchdog_may_still_act(
    current_generation: u64,
    watched_generation: u64,
    has_owned: bool,
    shutting_down: bool,
) -> bool {
    current_generation == watched_generation && has_owned && !shutting_down
}

fn watchdog_failure_budget(inference_active: bool, probe_timed_out: bool) -> u32 {
    if inference_active && probe_timed_out {
        HEALTH_WATCHDOG_MAX_FAILURES_BUSY
    } else {
        HEALTH_WATCHDOG_MAX_FAILURES
    }
}

async fn managed_install_ready_after_repair() -> bool {
    crate::preflight::managed_install_ready().await
}

fn should_emit_repair_failed(msg: &str) -> bool {
    !msg.contains("NEEDS_ELEVATION")
}

fn external_conflict_message(conflict: &crate::preflight::ExternalBackendConflict) -> String {
    match conflict.reason.as_str() {
        "desktop_owned_backend_active" => format!(
            "A desktop-owned Unsloth server for this install is already running on port {}. Quit the other desktop app instance, then try again.",
            conflict.port
        ),
        // Do not describe a backend from an unknown install as terminal-started.
        "ambiguous_root_external_backend_active" => format!(
            "An Unsloth server is already running on port {}, and this app cannot confirm which install it belongs to. Stop that server, then try again.",
            conflict.port
        ),
        _ => format!(
            "An Unsloth server for this install is already running from a terminal on port {}. Stop that server, or run `unsloth studio update` from that terminal before using desktop repair/update.",
            conflict.port
        ),
    }
}

fn owned_backend_port(state: &tauri::State<'_, BackendState>) -> Result<Option<u16>, String> {
    state
        .lock()
        .map(|proc| proc.owned_backend_port())
        .map_err(|e| e.to_string())
}

fn has_owned_backend(state: &tauri::State<'_, BackendState>) -> Result<bool, String> {
    state
        .lock()
        .map(|proc| proc.has_owned_backend())
        .map_err(|e| e.to_string())
}

async fn block_external_conflict(ignored_ports: &[u16]) -> Result<(), String> {
    if let Some(conflict) =
        crate::preflight::mutation_blocking_backend_ignoring(ignored_ports).await
    {
        return Err(external_conflict_message(&conflict));
    }
    Ok(())
}

#[tauri::command]
pub async fn desktop_preflight(
    app: AppHandle,
    state: tauri::State<'_, BackendState>,
    shutdown: tauri::State<'_, ShutdownFlag>,
    update_state: tauri::State<'_, update::UpdateState>,
    install_state: tauri::State<'_, install::InstallState>,
    diagnostics: tauri::State<'_, DiagnosticsState>,
) -> Result<crate::preflight::DesktopPreflightResult, String> {
    let started = Instant::now();
    // The installer phase does not hold the runtime gate; checked before and after the probe.
    let mutating = || {
        install::is_install_running(install_state.inner())
            || update::is_update_running(update_state.inner())
            || update::is_repair_running(update_state.inner())
    };
    let mutating_before = mutating();
    let (result, adopted_watchdog_generation) =
        crate::preflight::desktop_preflight_result_with_state(state.inner()).await?;
    let result = if mutating_before || mutating() {
        crate::preflight::busy_managed_environment(result)
    } else {
        result
    };
    diagnostics::record_preflight(&diagnostics, &result);

    info!(
        "desktop_preflight completed disposition={:?} port={:?} in {}ms",
        result.disposition,
        result.port,
        started.elapsed().as_millis()
    );

    if let Some((generation, newly_adopted)) = adopted_watchdog_generation {
        if newly_adopted {
            if let Some(port) = result.port {
                diagnostics::begin_adopted_backend_session(&diagnostics, port, generation);
            }
        }
        if process::claim_adopted_watchdog_if_current(state.inner(), generation) {
            shutdown.store(false, std::sync::atomic::Ordering::SeqCst);
            let watchdog_state = state.inner().clone();
            let watchdog_shutdown = shutdown.inner().clone();
            let watchdog_diagnostics = diagnostics.inner().clone();
            tokio::spawn(async move {
                health_watchdog(
                    app,
                    watchdog_state,
                    watchdog_shutdown,
                    watchdog_diagnostics,
                    generation,
                    true,
                )
                .await;
            });
        }
    }

    Ok(result)
}

/// Runs `unsloth -h` so a partial install (deps missing) reports false.
#[tauri::command]
pub async fn check_install_status() -> bool {
    let Some(bin) = process::find_unsloth_binary() else {
        return false;
    };

    let mut cmd = match process::build_managed_cli_command_tokio(&bin, &["-h"]) {
        Ok(cmd) => cmd,
        Err(e) => {
            warn!(
                "Install check: cannot run the managed CLI for {:?}: {}",
                bin, e
            );
            return false;
        }
    };
    cmd.stdout(std::process::Stdio::null())
        .stderr(std::process::Stdio::null());

    if let Err(error) = process::apply_managed_cli_context_tokio(&mut cmd) {
        warn!("Install check: no usable working directory: {}", error);
        return false;
    }

    #[cfg(windows)]
    {
        cmd.creation_flags(crate::process::CREATE_NO_WINDOW);
    }

    #[cfg(target_os = "linux")]
    crate::process::scrub_appimage_python_env_tokio(&mut cmd);

    // Inherited roots are dropped; the desktop's own root is passed instead, as in process.rs.
    cmd.env_remove("UNSLOTH_STUDIO_HOME");
    cmd.env_remove("STUDIO_HOME");
    if let Some(root) = crate::studio_home::chosen_root() {
        cmd.env(crate::studio_home::STUDIO_HOME_ENV, root);
    }

    let mut child = match process::with_studio_runtime_launch_guard(|| {
        cmd.spawn().map_err(|error| error.to_string())
    }) {
        Ok(c) => c,
        Err(e) => {
            warn!("Install check: failed to spawn {:?}: {}", bin, e);
            return false;
        }
    };

    match tokio::time::timeout(std::time::Duration::from_secs(10), child.wait()).await {
        Ok(Ok(status)) => {
            let ok = status.success();
            if !ok {
                warn!("Install check: `unsloth -h` exited with {}", status);
            }
            ok
        }
        Ok(Err(e)) => {
            warn!("Install check: wait failed: {}", e);
            false
        }
        Err(_) => {
            warn!("Install check: `unsloth -h` timed out after 10s");
            let _ = child.kill().await;
            false
        }
    }
}

/// Start the backend and spawn a health watchdog that emits `server-crashed` if it hangs.
#[tauri::command]
pub async fn start_server(
    app: AppHandle,
    state: tauri::State<'_, BackendState>,
    shutdown: tauri::State<'_, ShutdownFlag>,
    diagnostics: tauri::State<'_, DiagnosticsState>,
    port: u16,
) -> Result<(), String> {
    info!("start_server command called with port {}", port);

    let diagnostics_state = diagnostics.inner().clone();
    let generation = process::start_backend(&app, &state, port, &shutdown, &diagnostics_state)?;

    let watchdog_state = state.inner().clone();
    let watchdog_shutdown = shutdown.inner().clone();
    let watchdog_app = app.clone();
    tokio::spawn(async move {
        health_watchdog(
            watchdog_app,
            watchdog_state,
            watchdog_shutdown,
            diagnostics_state,
            generation,
            false,
        )
        .await;
    });

    Ok(())
}

/// Start the managed backend without reusing an existing backend.
#[tauri::command]
pub async fn start_managed_server(
    app: AppHandle,
    state: tauri::State<'_, BackendState>,
    shutdown: tauri::State<'_, ShutdownFlag>,
    diagnostics: tauri::State<'_, DiagnosticsState>,
    port: u16,
) -> Result<(), String> {
    info!("start_managed_server command called with port {}", port);

    let started = Instant::now();
    let diagnostics_state = diagnostics.inner().clone();
    let generation = process::start_backend(&app, &state, port, &shutdown, &diagnostics_state)?;

    info!(
        "start_managed_server spawned generation={} in {}ms",
        generation,
        started.elapsed().as_millis()
    );

    let watchdog_state = state.inner().clone();
    let watchdog_shutdown = shutdown.inner().clone();
    let watchdog_app = app.clone();
    tokio::spawn(async move {
        health_watchdog(
            watchdog_app,
            watchdog_state,
            watchdog_shutdown,
            diagnostics_state,
            generation,
            false,
        )
        .await;
    });

    Ok(())
}

/// Stop the backend this app owns or is attached to, if it can safely control it.
#[tauri::command]
pub async fn stop_server(
    state: tauri::State<'_, BackendState>,
    shutdown: tauri::State<'_, ShutdownFlag>,
    diagnostics: tauri::State<'_, DiagnosticsState>,
    port: Option<u16>,
) -> Result<(), String> {
    info!("stop_server command called");
    let state = state.inner().clone();
    let shutdown = shutdown.inner().clone();
    let diagnostics = diagnostics.inner().clone();
    tauri::async_runtime::spawn_blocking(move || {
        let attached_port = state
            .lock()
            .ok()
            .filter(|proc| !proc.has_owned_backend())
            .and_then(|proc| port.or(proc.port));
        if let Some(port) = attached_port {
            process::stop_attached_backend(port)?;
        }
        process::stop_backend(&state, &shutdown, Some(&diagnostics))
    })
    .await
    .map_err(|e| format!("stop backend task failed: {e}"))?
}

/// What one launcher probe learned about the backend process.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
struct BackendLiveness {
    alive: bool,
    /// Answered, but the background warm (ML-stack imports) is still in flight.
    warming_up: bool,
    /// On the adopted path this is the pre-probe reading; pair with `alive` for "answered now".
    inference_active: bool,
    /// Timed out rather than refused: a process still holds the port (stall, not death).
    probe_timed_out: bool,
    /// An HTTP RESPONSE, whatever its status. Weaker than `alive`, which also requires the payload to name this service.
    answered: bool,
}

/// Check if an Unsloth backend is running on the given port.
/// Expects JSON with status=="alive" (or "healthy") AND service=="Unsloth UI Backend".
#[tauri::command]
pub async fn check_health(port: u16) -> Result<bool, String> {
    match check_health_inner(port, HEALTH_PROBE_TIMEOUT).await {
        Ok(liveness) => Ok(liveness.alive),
        Err(e) => {
            info!("Health check on port {} failed: {}", port, e);
            Ok(false)
        }
    }
}

/// A stall arrives through the `Err` arm; a refused connection is not a timeout.
#[tauri::command]
pub async fn check_backend_present(
    state: tauri::State<'_, BackendState>,
    port: u16,
) -> Result<bool, String> {
    // Silence does not say which side went quiet, so ownership of the port settles it.
    let state = state.inner();
    backend_presence(port, || we_manage_a_backend_on(state, port)).await
}

/// *we_manage_it* is called AFTER the probe: an ownership answer taken before it died would report an exited backend as running.
async fn backend_presence(
    port: u16,
    we_manage_it: impl Fn() -> bool,
) -> Result<bool, String> {
    match check_health_inner(port, HEALTH_PROBE_TIMEOUT).await {
        Ok(liveness) => Ok(backend_is_present(&liveness, we_manage_it())),
        Err(e) => {
            info!("Backend presence check on port {} failed: {}", port, e);
            Ok(backend_is_present(
                &liveness_from_probe_error(&e),
                we_manage_it(),
            ))
        }
    }
}

/// Not `check_backend_present`: our own backend may not have bound its port yet. Ownership asked first.
#[tauri::command]
pub async fn check_backend_is_gone(
    state: tauri::State<'_, BackendState>,
    port: u16,
) -> Result<bool, String> {
    let state = state.inner();
    Ok(backend_is_gone(port, REFUSAL_PROBE_TIMEOUT, || {
        we_could_be_bringing_up(state, port)
    })
    .await)
}

async fn backend_is_gone(port: u16, budget: Duration, we_manage_it: impl Fn() -> bool) -> bool {
    // Port 0 is the placeholder base the webview holds before a validated port arrives.
    if port == 0 {
        return false;
    }
    // A backend we started may not have bound its port yet: keep waiting.
    if we_manage_it() {
        return false;
    }
    if !matches!(connect_outcome(port, budget).await, ConnectOutcome::Refused) {
        return false;
    }
    // Re-check ownership: Windows refusals can take ~2s, so the earlier read may be stale.
    !we_manage_it()
}

#[derive(Debug, PartialEq, Eq)]
enum ConnectOutcome {
    Refused,
    Accepted,
    /// No answer inside the budget, or an error not naming a closed port: #10520 lands here.
    Unsettled,
}

async fn connect_outcome(port: u16, budget: Duration) -> ConnectOutcome {
    let target = SocketAddr::from((Ipv4Addr::LOCALHOST, port));
    let settled = tokio::time::timeout(budget, tokio::net::TcpStream::connect(target))
        .await
        .ok()
        .map(|attempt| attempt.map(|_stream| ()));
    classify_connect(settled)
}

/// `None` is a spent budget.
fn classify_connect(settled: Option<std::io::Result<()>>) -> ConnectOutcome {
    match settled {
        Some(Ok(())) => ConnectOutcome::Accepted,
        Some(Err(err)) if err.kind() == std::io::ErrorKind::ConnectionRefused => {
            ConnectOutcome::Refused
        }
        Some(Err(_)) => ConnectOutcome::Unsettled,
        None => ConnectOutcome::Unsettled,
    }
}

/// Whether this app's own backend handle names *port*. The handle outlives the process it names, so the owner is checked too.
fn we_manage_a_backend_on(state: &BackendState, port: u16) -> bool {
    // Handle and process state in one pass under the same lock, so they cannot disagree.
    process::owned_backend_on_port_is_running(state, port)
}

/// Deliberately wider than presence: a live backend of ours that has not reported a port yet
/// may be about to bind THIS one, and a refusal about it is not proof of anything.
fn we_could_be_bringing_up(state: &BackendState, port: u16) -> bool {
    process::owned_backend_could_bind_port(state, port)
}

fn backend_is_present(liveness: &BackendLiveness, we_manage_it: bool) -> bool {
    // A healthy answer is presence whoever owns the port; a timeout or unhealthy reply counts only for a backend we manage.
    liveness.alive || ((liveness.probe_timed_out || liveness.answered) && we_manage_it)
}

/// Uses /api/liveness (health awaits hardware detection), falling back to /api/health on 404.
/// The budget is a parameter so the last-chance probe can use a wider one.
async fn check_health_inner(
    port: u16,
    budget: Duration,
) -> Result<BackendLiveness, reqwest::Error> {
    let client = crate::loopback_http::client(budget)?;
    let mut json = None;
    for path in ["/api/liveness", "/api/health"] {
        let resp = client
            .get(format!("http://127.0.0.1:{}{}", port, path))
            .send()
            .await?;
        if resp.status() == reqwest::StatusCode::NOT_FOUND && path == "/api/liveness" {
            continue;
        }
        if !resp.status().is_success() {
            // Not healthy, but not silence either: something answered on that port.
            return Ok(BackendLiveness {
                answered: true,
                ..BackendLiveness::default()
            });
        }
        json = match resp.json::<serde_json::Value>().await {
            Ok(value) => Some(value),
            Err(error) => {
                // Propagating the error made presence indistinguishable from a refusal.
                info!("Backend on port {} answered unparseable JSON: {}", port, error);
                return Ok(BackendLiveness {
                    answered: true,
                    ..BackendLiveness::default()
                });
            }
        };
        break;
    }
    let Some(json) = json else {
        return Ok(BackendLiveness {
            answered: true,
            ..BackendLiveness::default()
        });
    };

    // Liveness answers "alive", health "healthy"; accept either.
    let live = json
        .get("status")
        .and_then(|v| v.as_str())
        .map(|s| s == "alive" || s == "healthy")
        .unwrap_or(false);
    let correct_service = json
        .get("service")
        .and_then(|v| v.as_str())
        .map(|s| s == "Unsloth UI Backend")
        .unwrap_or(false);
    // `torch_warm_in_progress` covers the whole warm, not just hardware detection.
    let warming = json
        .get("torch_warm_in_progress")
        .and_then(|v| v.as_bool())
        .unwrap_or(false);
    // Fallback for older backends. A deferred warm sets hardware_detection_deferred and must not count
    // as warming, or the grace would never end.
    let detecting = json
        .get("hardware_detecting")
        .and_then(|v| v.as_bool())
        .unwrap_or(false);
    let deferred = json
        .get("hardware_detection_deferred")
        .and_then(|v| v.as_bool())
        .unwrap_or(false);

    // Absent on old backends, which read as not busy.
    let inference_active = json
        .get("inference_active")
        .and_then(|v| v.as_bool())
        .unwrap_or(false);

    let alive = live && correct_service;
    Ok(BackendLiveness {
        alive,
        warming_up: alive && (warming || (detecting && !deferred)),
        inference_active: alive && inference_active,
        probe_timed_out: false,
        answered: true,
    })
}

async fn check_watchdog_health(
    state: &BackendState,
    generation: u64,
    port: u16,
    has_adopted: bool,
    budget: Duration,
) -> BackendLiveness {
    if !has_adopted {
        return match check_health_inner(port, budget).await {
            Ok(liveness) => liveness,
            Err(error) => liveness_from_probe_error(&error),
        };
    }

    let snapshot = match process::owned_backend_snapshot(state) {
        Ok(Some(snapshot))
            if snapshot.is_adopted
                && snapshot.generation == generation
                && snapshot.port == Some(port) =>
        {
            snapshot
        }
        _ => return BackendLiveness::default(),
    };
    let Some(owner) = snapshot.owner else {
        return BackendLiveness::default();
    };
    // Classify the failure here first: the ownership probe folds every transport error into not-verified.
    let served = match check_health_inner(port, budget).await {
        Ok(liveness) => liveness,
        Err(error) => return liveness_from_probe_error(&error),
    };
    // HEALTH_PROBE_TIMEOUT, not the 2s default, or every request times out during the GIL stall.
    let ownership = crate::desktop_backend_owner::probe_owned_backend_state_with_timeout(
        owner,
        Some(port),
        false,
        budget,
    )
    .await;
    let verified = matches!(
        ownership,
        crate::desktop_backend_owner::OwnedBackendProbe::Verified(_)
    );
    // The one failure the probe is certain about: a complete answer naming another owner.
    let different_owner = crate::desktop_backend_owner::probe_saw_a_different_owner(&ownership);
    adopted_backend_liveness(verified, &served, different_owner)
}

/// `alive` and `warming_up` stay gated on verification so a foreign process cannot hold the grace open.
fn adopted_backend_liveness(
    verified: bool,
    served: &BackendLiveness,
    different_owner: bool,
) -> BackendLiveness {
    BackendLiveness {
        alive: verified,
        warming_up: verified && served.warming_up,
        // Not gated on verification: the pre-probe busy marker is the evidence the confirm probe needs.
        // A takeover (different_owner) still clears it.
        inference_active: served.inference_active && !different_owner,
        // Ownership's extra requests can time out on a saturated backend; the pre-probe answer
        // makes it a stall.
        probe_timed_out: adopted_failure_is_a_stall(verified, served.alive, different_owner),
        answered: served.answered || served.alive,
    }
}

/// Return buffered server logs.
#[tauri::command]
pub fn get_server_logs(state: tauri::State<'_, BackendState>) -> Vec<String> {
    match state.lock() {
        Ok(proc) => proc.logs.iter().cloned().collect(),
        Err(e) => {
            error!("Failed to lock state for logs: {}", e);
            vec![]
        }
    }
}

/// Validates the path first so callers get a clean error, not a raw OS failure.
fn open_existing_dir_with<E>(
    dir: &std::path::Path,
    opener: impl FnOnce(&std::path::Path) -> Result<(), E>,
) -> Result<(), String>
where
    E: std::fmt::Display,
{
    if !dir.is_dir() {
        return Err(format!("Directory does not exist: {}", dir.display()));
    }
    opener(dir).map_err(|error| format!("Failed to open directory: {error}"))
}

fn open_existing_dir(dir: &std::path::Path) -> Result<(), String> {
    open_existing_dir_with(dir, |path| crate::process::open_detached(path))
}

/// Open the Unsloth logs directory in the system file manager.
#[tauri::command]
pub fn open_logs_dir(webview: tauri::Webview) -> Result<(), String> {
    crate::native_intents::ensure_main_window(&webview)?;
    open_existing_dir(&diagnostics::logs_dir())
}

/// Open a models directory (resolved by the backend) in the system file manager.
#[tauri::command]
pub fn open_models_dir(webview: tauri::Webview, path: String) -> Result<(), String> {
    crate::native_intents::ensure_main_window(&webview)?;
    open_existing_dir(std::path::Path::new(&path))
}

/// Run the platform installer with --tauri, streaming progress.
/// Returns "NEEDS_ELEVATION" if system packages need elevated install (Linux only).
#[tauri::command]
pub async fn start_install(
    app: AppHandle,
    state: tauri::State<'_, install::InstallState>,
    backend_state: tauri::State<'_, BackendState>,
    update_state: tauri::State<'_, update::UpdateState>,
    diagnostics: tauri::State<'_, DiagnosticsState>,
    skip_long_paths: Option<bool>,
    chat_only: Option<bool>,
) -> Result<(), String> {
    if has_owned_backend(&backend_state)? {
        return Err(
            "The Unsloth backend is still running. Stop it before starting installation."
                .to_string(),
        );
    }
    // A repair's installer phase runs through run_install_for_repair, so one started here races it.
    if update::is_repair_running(update_state.inner()) {
        return Err("Cannot install while a repair is in progress.".to_string());
    }
    block_external_conflict(&[]).await?;

    let state = state.inner().clone();
    let diagnostics_state = diagnostics.inner().clone();
    let skip_long_paths = skip_long_paths.unwrap_or(false);
    let no_torch = chat_only.unwrap_or(false);
    tokio::task::spawn_blocking(move || {
        install::run_install(app, state, diagnostics_state, skip_long_paths, no_torch)
    })
    .await
    .map_err(|e| format!("Install task panicked: {e}"))?
}

/// Record that the user canceled a pending system-package elevation flow.
#[tauri::command]
pub fn cancel_pending_elevation(
    state: tauri::State<'_, install::InstallState>,
    diagnostics: tauri::State<'_, DiagnosticsState>,
) -> Result<(), String> {
    let _ = install::record_pending_elevation_canceled(&state, diagnostics.inner());
    Ok(())
}

/// Install system packages elevated (Linux); only packages the install script reported are allowed.
#[cfg(target_os = "linux")]
#[tauri::command]
pub fn install_system_packages(
    packages: Vec<String>,
    state: tauri::State<'_, install::InstallState>,
    diagnostics: tauri::State<'_, DiagnosticsState>,
) -> Result<(), String> {
    let allowed = state
        .lock()
        .map(|s| s.needed_packages.clone())
        .unwrap_or_default();
    for pkg in &packages {
        if !allowed.contains(pkg) {
            return Err(format!(
                "Package '{}' was not requested by the install script",
                pkg
            ));
        }
    }
    install::install_system_packages(&packages, &state, diagnostics.inner())
}

/// Non-Linux stub: elevation is handled by the scripts themselves.
#[cfg(not(target_os = "linux"))]
#[tauri::command]
pub fn install_system_packages(
    _packages: Vec<String>,
    _state: tauri::State<'_, install::InstallState>,
    _diagnostics: tauri::State<'_, DiagnosticsState>,
) -> Result<(), String> {
    Err("Elevated package install is only supported on Linux".to_string())
}

// Each prompt blocks while the webview stays live, so re-check the other activity after every answer.
fn backend_update_confirmed(
    mut training_active: impl FnMut() -> bool,
    mut downloads_active: impl FnMut() -> bool,
    mut confirm_training: impl FnMut() -> bool,
    mut confirm_downloads: impl FnMut() -> bool,
) -> bool {
    let (mut asked_training, mut asked_downloads) = (false, false);
    loop {
        if !asked_training && training_active() {
            if !confirm_training() {
                return false;
            }
            asked_training = true;
        } else if !asked_downloads && downloads_active() {
            if !confirm_downloads() {
                return false;
            }
            asked_downloads = true;
        } else {
            return true;
        }
    }
}

#[tauri::command]
pub async fn confirm_backend_update(app: AppHandle) -> bool {
    // blocking_show parks its thread until the user answers: keep it off the async workers.
    tauri::async_runtime::spawn_blocking(move || {
        backend_update_confirmed(
            || crate::training_is_active(&app),
            || crate::current_renderer_activity(&app).downloads,
            || crate::confirm_update_during_training(&app),
            || crate::confirm_update_during_downloads(&app),
        )
    })
    .await
    .unwrap_or(false)
}

/// Stop the server and run `unsloth studio update`. Does NOT restart; the frontend relaunches.
#[tauri::command]
pub async fn start_backend_update(
    app: AppHandle,
    backend_state: tauri::State<'_, BackendState>,
    shutdown: tauri::State<'_, ShutdownFlag>,
    update_state: tauri::State<'_, update::UpdateState>,
    install_state: tauri::State<'_, install::InstallState>,
    diagnostics: tauri::State<'_, DiagnosticsState>,
) -> Result<(), String> {
    info!("start_backend_update command called");

    if install_state
        .lock()
        .map(|s| s.child.is_some())
        .unwrap_or(false)
    {
        return Err("Cannot update while installation is in progress.".to_string());
    }
    // A repair holds no child handle between its update and its installer: invisible to the above.
    if update::is_repair_running(update_state.inner()) {
        return Err("Cannot update while a repair is in progress.".to_string());
    }

    if update_state
        .lock()
        .map(|s| s.child.is_some())
        .unwrap_or(false)
    {
        return Err("Update is already running.".to_string());
    }

    let owned_port = owned_backend_port(&backend_state)?;
    let has_owned = has_owned_backend(&backend_state)?;
    if has_owned {
        if let Some(port) = owned_port {
            block_external_conflict(&[port]).await?;
        }

        info!("Stopping backend before update...");
        process::stop_backend_for_mutation(&backend_state, &shutdown, Some(diagnostics.inner()))?;
        block_external_conflict(&[]).await?;
    } else {
        block_external_conflict(&[]).await?;
    }

    let state = update_state.inner().clone();
    let diagnostics_state = diagnostics.inner().clone();
    tokio::task::spawn_blocking(move || update::run_backend_update(app, state, diagnostics_state))
        .await
        .map_err(|e| format!("Update task panicked: {e}"))?
}

/// Whether a native path lease this app signs can be verified: the key is per process, so only a
/// backend THIS process spawned holds it; adopted ones advertise support with another key.
#[tauri::command]
pub async fn native_path_leases_usable(
    backend_state: tauri::State<'_, BackendState>,
) -> Result<bool, String> {
    Ok(
        matches!(process::owned_backend_snapshot(backend_state.inner())?,
            Some(snapshot) if !snapshot.is_adopted),
    )
}

/// Repair a stale managed Unsloth install.
/// `force_installer` skips `studio update`, which keeps a CPU-only torch; only the installer re-selects it.
#[tauri::command]
pub async fn start_managed_repair(
    app: AppHandle,
    backend_state: tauri::State<'_, BackendState>,
    shutdown: tauri::State<'_, ShutdownFlag>,
    update_state: tauri::State<'_, update::UpdateState>,
    install_state: tauri::State<'_, install::InstallState>,
    diagnostics: tauri::State<'_, DiagnosticsState>,
    force_installer: Option<bool>,
) -> Result<(), String> {
    let force_installer = force_installer.unwrap_or(false);
    info!(
        "start_managed_repair command called (force_installer={})",
        force_installer
    );

    if install_state
        .lock()
        .map(|s| s.child.is_some())
        .unwrap_or(false)
    {
        return Err("Cannot repair while installation is in progress.".to_string());
    }

    // Held to the end: process handles are empty between stop, update and installer, so duplicates raced.
    let _repair = update::RepairInFlight::claim(update_state.inner())?;

    let diagnostics_state = diagnostics.inner().clone();

    let owned_port = owned_backend_port(&backend_state)?;
    let has_owned = has_owned_backend(&backend_state)?;
    if has_owned {
        if let Some(port) = owned_port {
            block_external_conflict(&[port]).await?;
        }

        info!("Stopping backend before repair...");
        process::stop_backend_for_mutation(&backend_state, &shutdown, Some(&diagnostics_state))?;
        block_external_conflict(&[]).await?;
    } else {
        block_external_conflict(&[]).await?;
    }

    let repair_group_id = install::take_pending_repair_group_for_resume(&install_state)
        .unwrap_or_else(|| diagnostics::begin_repair_group(&diagnostics_state));

    let update_result = if force_installer {
        let _ = app.emit("repair-progress", "Running bundled installer...");
        Ok(())
    } else {
        let _ = app.emit("repair-progress", "Updating existing Unsloth install...");
        let update_app = app.clone();
        let update_state = update_state.inner().clone();
        let update_diagnostics = diagnostics_state.clone();
        let update_repair_group_id = repair_group_id.clone();
        tokio::task::spawn_blocking(move || {
            update::run_backend_update_for_repair(
                update_app,
                update_state,
                update_diagnostics,
                update_repair_group_id,
            )
        })
        .await
        .map_err(|e| format!("Repair update task panicked: {e}"))?
    };

    match update_result {
        Ok(()) if !force_installer && managed_install_ready_after_repair().await => {
            info!("Managed repair complete after update");
            diagnostics::finish_repair_group(&diagnostics_state, &repair_group_id, "success", None);
            let _ = app.emit("repair-complete", ());
            return Ok(());
        }
        Ok(()) if force_installer => {}
        Ok(()) => {
            warn!("Managed repair update finished, but preflight is still not ready; falling back to installer");
            let _ = app.emit(
                "repair-progress",
                "Update finished, but Unsloth is still not ready. Running bundled installer...",
            );
        }
        Err(msg) => {
            // A stop is the user quitting, not a broken install; running the installer would
            // half-build the venv.
            if msg == update::UPDATE_STOPPED {
                info!("Managed repair update stopped; skipping installer fallback");
                diagnostics::finish_repair_group(
                    &diagnostics_state,
                    &repair_group_id,
                    "canceled",
                    Some(msg.clone()),
                );
                return Err(msg);
            }
            if msg.to_ascii_lowercase().contains("already running") {
                error!("Managed repair update conflict: {}", msg);
                diagnostics::finish_repair_group(
                    &diagnostics_state,
                    &repair_group_id,
                    "failed",
                    Some(msg.clone()),
                );
                let _ = app.emit("repair-failed", &msg);
                return Err(msg);
            }

            warn!(
                "Managed repair update failed, falling back to bundled installer: {}",
                msg
            );
            let _ = app.emit(
                "repair-progress",
                "Update failed. Running bundled installer...",
            );
        }
    }

    if let Err(msg) = block_external_conflict(&[]).await {
        diagnostics::finish_repair_group(
            &diagnostics_state,
            &repair_group_id,
            "failed",
            Some(msg.clone()),
        );
        let _ = app.emit("repair-failed", &msg);
        return Err(msg);
    }

    let install_app = app.clone();
    let install_state = install_state.inner().clone();
    let install_diagnostics = diagnostics_state.clone();
    let install_repair_group_id = repair_group_id.clone();
    let install_result = tokio::task::spawn_blocking(move || {
        install::run_install_for_repair(
            install_app,
            install_state,
            install_diagnostics,
            install_repair_group_id,
            force_installer,
        )
    })
    .await
    .map_err(|e| format!("Repair install task panicked: {e}"))?;

    if let Err(msg) = install_result {
        diagnostics::finish_repair_group(
            &diagnostics_state,
            &repair_group_id,
            if msg == "NEEDS_ELEVATION" {
                "needs_elevation"
            } else {
                "failed"
            },
            Some(msg.clone()),
        );
        if should_emit_repair_failed(&msg) {
            error!("Managed repair installer failed: {}", msg);
            let _ = app.emit("repair-failed", &msg);
        }
        return Err(msg);
    }

    if managed_install_ready_after_repair().await {
        info!("Managed repair complete after installer");
        diagnostics::finish_repair_group(&diagnostics_state, &repair_group_id, "success", None);
        let _ = app.emit("repair-complete", ());
        return Ok(());
    }

    let msg = "Repair finished, but Unsloth install is still not desktop-ready.".to_string();
    error!("{}", msg);
    diagnostics::finish_repair_group(
        &diagnostics_state,
        &repair_group_id,
        "failed",
        Some(msg.clone()),
    );
    let _ = app.emit("repair-failed", &msg);
    Err(msg)
}

#[allow(clippy::items_after_test_module)]
#[cfg(test)]
mod tests {
    use std::fs;
    use std::sync::{Arc, Mutex};
    use std::time::{Duration, SystemTime, UNIX_EPOCH};
    use tokio::io::{AsyncReadExt, AsyncWriteExt};
    use tokio::net::TcpListener;

    const ROOT_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
    const OWNER_TOKEN: &str = "desktop-owner-token";

    fn ready_health(include_owner: bool) -> String {
        let owner = if include_owner {
            format!(
                r#", "desktop_owner":{{"kind":"tauri","token_sha256":"{}"}}"#,
                crate::desktop_backend_owner::token_sha256(OWNER_TOKEN)
            )
        } else {
            String::new()
        };
        // Only an owned backend can advertise lease support; ready_health(false)
        // stands in for a terminal-started server, which never can.
        let leases = if include_owner {
            r#""native_path_leases_supported":true,"#
        } else {
            ""
        };
        format!(
            r#"{{"status":"healthy","service":"Unsloth UI Backend","version":"2026.8.4","desktop_protocol_version":1,"desktop_manageability_version":1,"supports_desktop_auth":true,"supports_desktop_backend_ownership":true,{leases}"studio_root_id":"{ROOT_ID}"{owner}}}"#
        )
    }

    async fn command_test_backend(health_body: String) -> u16 {
        let mut listener = None;
        for port in 8888u16..=8908 {
            if let Ok(bound) = TcpListener::bind(("127.0.0.1", port)).await {
                listener = Some(bound);
                break;
            }
        }
        let listener = listener.expect("test needs a free desktop preflight port");
        let port = listener.local_addr().unwrap().port();
        tokio::spawn(async move {
            for _ in 0..2 {
                let Ok((mut stream, _)) = listener.accept().await else {
                    return;
                };
                let mut buffer = [0; 2048];
                let Ok(n) = stream.read(&mut buffer).await else {
                    return;
                };
                let request = String::from_utf8_lossy(&buffer[..n]);
                let (status, body) = if request.starts_with("GET /api/health ") {
                    ("200 OK", health_body.as_str())
                } else if request.starts_with("POST /api/auth/desktop-login ") {
                    ("401 Unauthorized", "")
                } else {
                    ("404 Not Found", "")
                };
                let response = format!(
                    "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
                    body.len()
                );
                let _ = stream.write_all(response.as_bytes()).await;
            }
        });
        port
    }

    #[test]
    fn existing_directory_helper_invokes_opener_and_surfaces_errors() {
        let nanos = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap()
            .as_nanos();
        let dir =
            std::env::temp_dir().join(format!("unsloth-open-dir-{}-{nanos}", std::process::id()));
        fs::create_dir_all(&dir).unwrap();
        let mut opened = false;
        super::open_existing_dir_with(&dir, |path| {
            opened = true;
            assert_eq!(path, dir);
            Ok::<_, &str>(())
        })
        .unwrap();
        assert!(opened);

        let error = super::open_existing_dir_with(&dir, |_| Err("opener failed")).unwrap_err();
        assert!(error.contains("opener failed"));
        let _ = fs::remove_dir_all(dir);
    }

    #[test]
    fn existing_directory_helper_rejects_missing_path_without_opening() {
        let missing = std::env::temp_dir().join("unsloth-definitely-missing-open-dir");
        let error = super::open_existing_dir_with(&missing, |_| {
            panic!("opener must not run for an invalid directory");
            #[allow(unreachable_code)]
            Ok::<_, &str>(())
        })
        .unwrap_err();
        assert!(error.contains("Directory does not exist"));
    }
    #[test]
    fn repair_elevation_is_not_a_terminal_repair_failure() {
        assert!(!super::should_emit_repair_failed("NEEDS_ELEVATION"));
        assert!(super::should_emit_repair_failed(
            "Installer exited with code 1"
        ));
    }

    #[test]
    fn keeping_training_at_the_update_prompt_declines_the_update() {
        assert!(!super::backend_update_confirmed(
            || true,
            || false,
            || false,
            || true
        ));
    }

    #[test]
    fn updating_anyway_during_training_confirms_the_update() {
        assert!(super::backend_update_confirmed(
            || true,
            || false,
            || true,
            || false
        ));
    }

    #[test]
    fn updating_without_training_does_not_ask() {
        let mut asked = false;
        assert!(super::backend_update_confirmed(
            || false,
            || false,
            || {
                asked = true;
                false
            },
            || false
        ));
        assert!(!asked);
    }

    #[test]
    fn keeping_downloads_at_the_update_prompt_declines_the_update() {
        assert!(!super::backend_update_confirmed(
            || false,
            || true,
            || true,
            || false
        ));
    }

    #[test]
    fn updating_anyway_during_downloads_confirms_the_update() {
        assert!(super::backend_update_confirmed(
            || false,
            || true,
            || false,
            || true
        ));
    }

    #[test]
    fn keeping_training_does_not_also_ask_about_downloads() {
        let mut asked = false;
        assert!(!super::backend_update_confirmed(
            || true,
            || true,
            || false,
            || {
                asked = true;
                true
            }
        ));
        assert!(!asked);
    }

    #[test]
    fn downloads_started_during_the_training_prompt_are_still_asked_about() {
        let downloading = std::cell::Cell::new(false);
        let mut asked = false;
        assert!(!super::backend_update_confirmed(
            || true,
            || downloading.get(),
            || {
                downloading.set(true);
                true
            },
            || {
                asked = true;
                false
            }
        ));
        assert!(asked);
    }

    #[test]
    fn training_started_during_the_downloads_prompt_is_still_asked_about() {
        let training = std::cell::Cell::new(false);
        let mut asked = false;
        assert!(!super::backend_update_confirmed(
            || training.get(),
            || true,
            || {
                asked = true;
                false
            },
            || {
                training.set(true);
                true
            }
        ));
        assert!(asked);
    }

    #[test]
    fn each_prompt_is_shown_at_most_once() {
        let (mut training_asks, mut download_asks) = (0, 0);
        assert!(super::backend_update_confirmed(
            || true,
            || true,
            || {
                training_asks += 1;
                true
            },
            || {
                download_asks += 1;
                true
            }
        ));
        assert_eq!((training_asks, download_asks), (1, 1));
    }

    #[tokio::test]
    async fn mutation_guard_blocks_second_external_backend_when_owned_child_is_ignored() {
        crate::desktop_backend_owner::install_test_owner(ROOT_ID, OWNER_TOKEN);
        let owned_port = command_test_backend(ready_health(true)).await;
        let external_port = command_test_backend(ready_health(false)).await;

        let err = super::block_external_conflict(&[owned_port])
            .await
            .expect_err("external non-owned backend should block mutation");

        assert!(err.contains(&format!("port {external_port}")));
        assert!(err.contains("Stop that server"));
    }

    /// Stub backend; `liveness: None` models a backend older than the route (404).
    async fn probe_test_backend(
        liveness: Option<String>,
        health: String,
    ) -> (u16, Arc<Mutex<Vec<String>>>) {
        let listener = TcpListener::bind(("127.0.0.1", 0))
            .await
            .expect("probe test needs a loopback port");
        let port = listener.local_addr().unwrap().port();
        let paths = Arc::new(Mutex::new(Vec::new()));
        let recorded = Arc::clone(&paths);
        tokio::spawn(async move {
            loop {
                let Ok((mut stream, _)) = listener.accept().await else {
                    return;
                };
                let mut buffer = [0; 2048];
                let Ok(n) = stream.read(&mut buffer).await else {
                    return;
                };
                let request = String::from_utf8_lossy(&buffer[..n]);
                let path = request
                    .split_whitespace()
                    .nth(1)
                    .unwrap_or_default()
                    .to_string();
                recorded.lock().unwrap().push(path.clone());
                let (status, body) = match path.as_str() {
                    "/api/liveness" => match liveness.as_deref() {
                        Some(body) => ("200 OK", body),
                        None => ("404 Not Found", ""),
                    },
                    "/api/health" => ("200 OK", health.as_str()),
                    _ => ("404 Not Found", ""),
                };
                let response = format!(
                    "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
                    body.len()
                );
                let _ = stream.write_all(response.as_bytes()).await;
            }
        });
        (port, paths)
    }

    #[tokio::test]
    async fn liveness_is_probed_instead_of_the_detection_gated_health_route() {
        // Health awaits hardware detection, so liveness must be the only route touched.
        let (port, paths) = probe_test_backend(
            Some(r#"{"status":"alive","service":"Unsloth UI Backend"}"#.to_string()),
            ready_health(false),
        )
        .await;

        let liveness = super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
            .await
            .unwrap();

        assert!(liveness.alive);
        assert!(!liveness.warming_up);
        assert_eq!(paths.lock().unwrap().as_slice(), ["/api/liveness"]);
    }

    #[tokio::test]
    async fn a_backend_without_the_liveness_route_still_validates_through_health() {
        let (port, paths) = probe_test_backend(None, ready_health(false)).await;

        let liveness = super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
            .await
            .unwrap();

        assert!(liveness.alive);
        assert_eq!(
            paths.lock().unwrap().as_slice(),
            ["/api/liveness", "/api/health"]
        );
    }

    #[tokio::test]
    async fn a_warming_backend_is_alive_but_not_finished_starting() {
        let (port, _) = probe_test_backend(
            Some(
                r#"{"status":"alive","service":"Unsloth UI Backend","hardware_detecting":true}"#
                    .to_string(),
            ),
            ready_health(false),
        )
        .await;

        let liveness = super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
            .await
            .unwrap();

        assert!(liveness.alive);
        assert!(
            liveness.warming_up,
            "an unsettled hardware verdict means the torch import is still in flight"
        );
    }

    #[tokio::test]
    async fn a_late_warm_stage_still_counts_as_warming_up() {
        // Hardware detection ends early; the grace must last until the whole warm finishes.
        let (port, _) = probe_test_backend(
            Some(
                r#"{"status":"alive","service":"Unsloth UI Backend","torch_warm_in_progress":true}"#
                    .to_string(),
            ),
            ready_health(false),
        )
        .await;

        let liveness = super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
            .await
            .unwrap();

        assert!(liveness.alive);
        assert!(
            liveness.warming_up,
            "a settled hardware verdict does not mean the warm is over; the imports that \
             hold the GIL longest run after it"
        );
    }

    #[tokio::test]
    async fn a_finished_warm_ends_the_startup_grace() {
        let (port, _) = probe_test_backend(
            Some(
                r#"{"status":"alive","service":"Unsloth UI Backend","torch_warm_in_progress":false}"#
                    .to_string(),
            ),
            ready_health(false),
        )
        .await;

        let liveness = super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
            .await
            .unwrap();

        assert!(liveness.alive);
        assert!(!liveness.warming_up);
    }

    #[tokio::test]
    async fn a_backend_predating_the_warm_field_still_gets_its_grace() {
        let (port, _) = probe_test_backend(
            Some(
                r#"{"status":"alive","service":"Unsloth UI Backend","hardware_detecting":true}"#
                    .to_string(),
            ),
            ready_health(false),
        )
        .await;

        let liveness = super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
            .await
            .unwrap();

        assert!(liveness.warming_up);
    }

    #[tokio::test]
    async fn a_deferred_warm_is_not_reported_as_still_warming_up() {
        let (port, _) = probe_test_backend(
            Some(
                r#"{"status":"alive","service":"Unsloth UI Backend","hardware_detecting":true,"hardware_detection_deferred":true}"#
                    .to_string(),
            ),
            ready_health(false),
        )
        .await;

        let liveness = super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
            .await
            .unwrap();

        assert!(liveness.alive);
        assert!(!liveness.warming_up);
    }

    #[tokio::test]
    async fn a_foreign_service_on_the_port_is_not_alive() {
        let (port, _) = probe_test_backend(
            Some(r#"{"status":"alive","service":"Some Other App"}"#.to_string()),
            ready_health(false),
        )
        .await;

        assert_eq!(
            super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
                .await
                .unwrap(),
            super::BackendLiveness {
                // It answered, and that is all `answered` says: not OUR backend.
                answered: true,
                ..super::BackendLiveness::default()
            }
        );
        assert!(!super::backend_is_present(
            &super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
                .await
                .unwrap(),
            false
        ));
    }

    #[test]
    fn the_probe_budget_fits_inside_one_watchdog_interval() {
        // A probe that outlives the interval would let the next tick start on top of it.
        assert!(super::HEALTH_PROBE_TIMEOUT < super::HEALTH_WATCHDOG_INTERVAL);
    }

    #[test]
    fn the_frontend_retry_ladder_outlives_one_probe_budget() {
        // The ladder lives in TypeScript and the budget here; this guard keeps them in step.
        let src = include_str!("../../frontend/src/features/auth/api.ts").replace("\r\n", "\n");
        let marker = "const TAURI_FETCH_RETRY_DELAYS_MS = [";
        let start = src
            .find(marker)
            .expect("the Tauri fetch retry ladder moved; update this guard")
            + marker.len();
        let ladder = &src[start..];
        let ladder = &ladder[..ladder.find(']').expect("unterminated retry ladder")];
        let total_ms: u64 = ladder
            .split(',')
            .map(str::trim)
            .filter(|delay| !delay.is_empty())
            .map(|delay| {
                delay
                    .parse::<u64>()
                    .expect("a retry delay stopped being a plain number of milliseconds")
            })
            .sum();
        assert!(
            total_ms >= super::HEALTH_PROBE_TIMEOUT.as_millis() as u64,
            "the webview gives up after {total_ms}ms while one native liveness probe is \
             allowed {}ms, so a backend the launcher still considers alive is reported to \
             the user as not running",
            super::HEALTH_PROBE_TIMEOUT.as_millis()
        );
        assert!(
            src.contains("invoke<boolean>(\"check_backend_present\""),
            "the transport-failure path no longer asks the native side before it tells the \
             user to relaunch"
        );
        // And not the health command, which reports a spent budget as a refused connection.
        assert!(
            !src.contains("invoke<boolean>(\"check_health\""),
            "the transport-failure path is back on check_health, which collapses a stalled \
             probe onto \"not running\""
        );
    }

    #[test]
    fn a_stalled_probe_is_not_reported_as_an_absent_backend() {
        let stalled = super::BackendLiveness {
            alive: false,
            probe_timed_out: true,
            ..Default::default()
        };
        let closed = super::BackendLiveness::default();
        let answered = super::BackendLiveness {
            alive: true,
            ..Default::default()
        };

        // Through the rule the command applies: an inlined copy asserts its own arithmetic.
        assert!(!stalled.alive, "a stall is not an answer");
        assert!(
            super::backend_is_present(&stalled, true),
            "a stalled probe must read as a backend that is still present"
        );
        assert!(
            !super::backend_is_present(&stalled, false),
            "a stall on a port this app does not manage proves nothing about our backend"
        );
        assert!(
            !super::backend_is_present(&closed, true),
            "a refused connection must still read as absent"
        );
        assert!(super::backend_is_present(&answered, false));
        assert!(
            !stalled.alive,
            "check_health still reports a stall as not usable"
        );
    }

    #[test]
    fn the_startup_grace_survives_the_mac_cold_start_timeline() {
        for (label, elapsed) in [
            ("no validated port yet", Duration::from_secs(15)),
            ("probe timeout 1/3", Duration::from_secs(30)),
            ("probe timeout 2/3", Duration::from_secs(47)),
            ("probe timeout 3/3", Duration::from_secs(64)),
        ] {
            assert!(
                !super::should_count_watchdog_failure(false, elapsed),
                "{label} at {}s was counted against a backend still inside the {}s startup grace",
                elapsed.as_secs(),
                super::BACKEND_STARTUP_GRACE_PERIOD.as_secs()
            );
        }
        assert!(!super::should_count_watchdog_failure(
            false,
            super::BACKEND_STARTUP_GRACE_PERIOD - Duration::from_millis(1)
        ));
        assert!(super::should_count_watchdog_failure(
            false,
            super::BACKEND_STARTUP_GRACE_PERIOD
        ));
    }

    #[test]
    fn the_adopted_ownership_probe_uses_the_watchdog_budget() {
        // The ownership probe must use HEALTH_PROBE_TIMEOUT or the warm-up read never runs.
        // Normalise CRLF: include_str! embeds Windows checkouts verbatim.
        let src = include_str!("commands.rs").replace("\r\n", "\n");
        let start = src
            .find("async fn check_watchdog_health")
            .expect("check_watchdog_health moved; update this guard");
        let body = &src[start..];
        let body = &body[..body.find("\n}\n").expect("could not find the function end")];
        assert!(
            body.contains("probe_owned_backend_state_with_timeout"),
            "the adopted path is back on the default-timeout ownership probe"
        );
        assert!(
            body.contains("HEALTH_PROBE_TIMEOUT"),
            "the adopted ownership probe no longer uses the watchdog's probe budget"
        );
    }

    #[test]
    fn an_adopted_backend_that_is_still_warming_gets_the_grace_back() {
        // Adopted watchdogs start latched; a cold-starting backend relaunched onto must still get the grace.
        let mut has_seen_healthy = true; // adopted: count_failures_immediately
        has_seen_healthy = super::watchdog_seen_healthy_after(has_seen_healthy, true, true);
        assert!(
            !has_seen_healthy,
            "a backend that answered \"still warming\" left the latch set, so the grace stayed off"
        );
        for elapsed in [15, 30, 47, 64] {
            assert!(
                !super::should_count_watchdog_failure(
                    has_seen_healthy,
                    Duration::from_secs(elapsed)
                ),
                "a stalled probe at {elapsed}s was counted against an adopted backend that had \
                 just reported it was still warming"
            );
        }
        // The grace is still bounded: a backend that never finishes warming is not immortal.
        assert!(super::should_count_watchdog_failure(
            has_seen_healthy,
            super::BACKEND_STARTUP_GRACE_PERIOD
        ));
    }

    #[test]
    fn the_latch_tracks_the_last_answer_and_ignores_silence() {
        assert!(super::watchdog_seen_healthy_after(false, true, false));
        assert!(!super::watchdog_seen_healthy_after(true, true, true));
        assert!(super::watchdog_seen_healthy_after(true, false, false));
        assert!(!super::watchdog_seen_healthy_after(false, false, false));
        assert!(super::watchdog_seen_healthy_after(false, true, false));
    }

    #[test]
    fn a_backend_that_stalls_while_generating_is_not_killed_at_three_strikes() {
        // A saturated host (0.28 tok/s) can miss probes while still streaming.
        let mut generating = false;
        generating = super::watchdog_inference_active_after(generating, true, true);
        assert!(generating);
        generating = super::watchdog_inference_active_after(generating, false, false);
        assert!(generating);
        assert_eq!(
            super::watchdog_failure_budget(generating, true),
            super::HEALTH_WATCHDOG_MAX_FAILURES_BUSY
        );
        assert!(super::HEALTH_WATCHDOG_MAX_FAILURES_BUSY > super::HEALTH_WATCHDOG_MAX_FAILURES);
    }

    #[test]
    fn a_dead_port_is_still_declared_dead_at_three_strikes() {
        assert_eq!(
            super::watchdog_failure_budget(true, false),
            super::HEALTH_WATCHDOG_MAX_FAILURES
        );
        assert_eq!(
            super::watchdog_failure_budget(false, true),
            super::HEALTH_WATCHDOG_MAX_FAILURES
        );
    }

    /// Accepted streams are parked, not dropped: dropping would answer with a reset.
    async fn stalling_test_backend() -> u16 {
        let listener = TcpListener::bind(("127.0.0.1", 0))
            .await
            .expect("probe test needs a loopback port");
        let port = listener.local_addr().unwrap().port();
        tokio::spawn(async move {
            let mut parked = Vec::new();
            while let Ok((stream, _)) = listener.accept().await {
                parked.push(stream);
            }
        });
        port
    }

    async fn answering_test_backend(status: &'static str, body: &'static str) -> u16 {
        let listener = TcpListener::bind(("127.0.0.1", 0))
            .await
            .expect("probe test needs a loopback port");
        let port = listener.local_addr().unwrap().port();
        tokio::spawn(async move {
            while let Ok((mut stream, _)) = listener.accept().await {
                let mut buffer = [0; 2048];
                let Ok(_) = stream.read(&mut buffer).await else {
                    return;
                };
                let response = format!(
                    "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
                    body.len()
                );
                let _ = stream.write_all(response.as_bytes()).await;
            }
        });
        port
    }

    #[tokio::test]
    async fn ownership_is_read_after_the_probe_not_before_it() {
        // The closure answers "we manage it" only once the request reached the server, so true means it ran AFTER the probe.
        let probe_arrived = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
        let listener = TcpListener::bind(("127.0.0.1", 0))
            .await
            .expect("probe test needs a loopback port");
        let port = listener.local_addr().unwrap().port();
        let seen = std::sync::Arc::clone(&probe_arrived);
        tokio::spawn(async move {
            while let Ok((mut stream, _)) = listener.accept().await {
                let mut buffer = [0; 2048];
                let Ok(_) = stream.read(&mut buffer).await else {
                    return;
                };
                seen.store(true, std::sync::atomic::Ordering::SeqCst);
                let response = "HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";
                let _ = stream.write_all(response.as_bytes()).await;
            }
        });

        let asked = std::sync::Arc::clone(&probe_arrived);
        assert_eq!(
            super::backend_presence(port, move || asked.load(std::sync::atomic::Ordering::SeqCst))
                .await,
            Ok(true),
            "ownership was read before the probe, so a backend that changed under it would \
             still have been reported as running"
        );
    }

    #[tokio::test]
    async fn a_managed_backend_that_answers_unhealthily_is_still_present() {
        let port = answering_test_backend("503 Service Unavailable", "").await;
        assert_eq!(
            super::backend_presence(port, || true).await,
            Ok(true),
            "a managed backend answering 503 was reported absent"
        );
        assert_eq!(super::backend_presence(port, || false).await, Ok(false));
    }

    #[tokio::test]
    async fn a_reply_this_build_cannot_parse_is_still_an_answer() {
        let port = answering_test_backend("200 OK", "not json at all").await;
        assert_eq!(
            super::backend_presence(port, || true).await,
            Ok(true),
            "a managed backend answering an unparseable body was reported absent"
        );
        assert_eq!(super::backend_presence(port, || false).await, Ok(false));

        let liveness = super::check_health_inner(port, super::HEALTH_PROBE_TIMEOUT)
            .await
            .expect("an answered probe is not a transport error");
        assert!(!liveness.alive, "an unparseable reply is not a healthy backend");
        assert!(liveness.answered, "the reply arrived, so the port is not silent");
    }

    #[tokio::test]
    async fn a_port_that_accepts_and_never_answers_reads_as_a_stall() {
        let port = stalling_test_backend().await;
        let client = crate::loopback_http::client(Duration::from_millis(250)).unwrap();

        let error = client
            .get(format!("http://127.0.0.1:{port}/api/liveness"))
            .send()
            .await
            .expect_err("a parked connection cannot answer");

        assert!(
            super::liveness_from_probe_error(&error).probe_timed_out,
            "a spent probe budget is not being classified as a stall, so a backend that is \
             merely busy gets the three-strike budget"
        );
    }

    #[tokio::test]
    async fn check_backend_present_reports_a_stalled_port_as_present() {
        // Costs HEALTH_PROBE_TIMEOUT in wall clock.
        let port = stalling_test_backend().await;
        assert_eq!(
            super::backend_presence(port, || true).await,
            Ok(true),
            "a backend holding the port and not answering was reported absent, which is the \
             relaunch prompt this command was added to prevent"
        );
    }

    #[tokio::test]
    async fn a_stalled_port_this_app_does_not_manage_is_not_our_backend() {
        let port = stalling_test_backend().await;
        assert_eq!(
            super::backend_presence(port, || false).await,
            Ok(false),
            "a stalled port with no managed backend behind it was reported as present"
        );
    }

    #[tokio::test]
    async fn check_backend_present_reports_a_closed_port_as_absent() {
        let port = {
            let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
            let port = listener.local_addr().unwrap().port();
            drop(listener);
            port
        };
        assert_eq!(
            super::backend_presence(port, || true).await,
            Ok(false),
            "a closed port must still read as an absent backend"
        );
    }

    /// Below the ephemeral range so no sibling test's port-0 binding can be handed this port.
    async fn a_closed_port_below_the_ephemeral_range() -> u16 {
        for candidate in 20_064..20_128u16 {
            if let Ok(listener) = TcpListener::bind(("127.0.0.1", candidate)).await {
                drop(listener);
                return candidate;
            }
        }
        panic!("every port in the probe window was already bound")
    }

    async fn describe_connect(port: u16, budget: Duration) -> String {
        let target =
            std::net::SocketAddr::from((std::net::Ipv4Addr::LOCALHOST, port));
        let started = std::time::Instant::now();
        let settled = tokio::time::timeout(budget, tokio::net::TcpStream::connect(target)).await;
        let elapsed = started.elapsed();
        match settled {
            Err(_) => format!("no answer inside {budget:?} (waited {elapsed:?})"),
            Ok(Ok(_)) => format!("accepted after {elapsed:?}"),
            Ok(Err(error)) => format!(
                "kind={:?} raw_os_error={:?} after {elapsed:?} ({error})",
                error.kind(),
                error.raw_os_error(),
            ),
        }
    }

    /// Floors are well under REFUSAL_PROBE_TIMEOUT so only a real drop fails this.
    #[test]
    fn the_refusal_budget_clears_the_wait_this_platform_actually_takes() {
        let floor = if cfg!(windows) {
            Duration::from_millis(2_100)
        } else {
            Duration::from_millis(50)
        };
        assert!(
            super::REFUSAL_PROBE_TIMEOUT >= floor,
            "REFUSAL_PROBE_TIMEOUT is {:?}, under the {floor:?} this platform needs to see a \
             refusal at all. Below the real wait every dead port reads as Unsettled and the \
             fast path never fires.",
            super::REFUSAL_PROBE_TIMEOUT,
        );
    }

    #[tokio::test]
    async fn a_closed_port_nobody_here_owns_is_gone() {
        let port = a_closed_port_below_the_ephemeral_range().await;
        let observed = describe_connect(port, super::REFUSAL_PROBE_TIMEOUT).await;
        assert!(
            super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || false).await,
            "a refused loopback connect on a port nothing here owns is not reported as gone. \
             Port {port} answered: {observed}"
        );
    }

    #[tokio::test]
    async fn a_closed_port_we_are_bringing_up_is_not_gone() {
        // Port 0 is safe here: ownership is read before anything connects.
        let port = {
            let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
            let port = listener.local_addr().unwrap().port();
            drop(listener);
            port
        };
        assert!(
            !super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || true).await,
            "a backend of ours that is still starting was reported as gone"
        );
    }

    #[tokio::test]
    async fn ownership_taken_while_the_probe_ran_still_keeps_the_ladder() {
        let port = a_closed_port_below_the_ephemeral_range().await;
        let asked = std::sync::atomic::AtomicUsize::new(0);
        let gone = super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || {
            asked.fetch_add(1, std::sync::atomic::Ordering::SeqCst) > 0
        })
        .await;
        assert_eq!(
            asked.load(std::sync::atomic::Ordering::SeqCst),
            2,
            "ownership was not re-read after the connect, so the answer rests on a stale look"
        );
        assert!(
            !gone,
            "a backend that became ours while the probe was in flight was reported as gone"
        );
    }

    #[tokio::test]
    async fn a_port_something_is_listening_on_is_not_gone() {
        let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
        let port = listener.local_addr().unwrap().port();
        assert!(
            !super::backend_is_gone(port, super::REFUSAL_PROBE_TIMEOUT, || false).await,
            "a port that accepted a connection was reported as gone"
        );
    }

    #[tokio::test]
    async fn the_placeholder_port_is_never_an_answer() {
        assert!(!super::backend_is_gone(0, super::REFUSAL_PROBE_TIMEOUT, || false).await);
    }

    #[test]
    fn a_connect_that_never_answers_is_unsettled_not_refused() {
        // No test can drop a SYN, so the rule is checked on its own.
        assert_eq!(
            super::classify_connect(None),
            super::ConnectOutcome::Unsettled,
            "a spent budget was reported as proof that nothing is listening"
        );
        assert_eq!(
            super::classify_connect(Some(Err(std::io::Error::from(
                std::io::ErrorKind::TimedOut
            )))),
            super::ConnectOutcome::Unsettled
        );
        assert_eq!(
            super::classify_connect(Some(Err(std::io::Error::from(
                std::io::ErrorKind::PermissionDenied
            )))),
            super::ConnectOutcome::Unsettled,
            "a blocked connect is not evidence the port is empty"
        );
        assert_eq!(
            super::classify_connect(Some(Err(std::io::Error::from(
                std::io::ErrorKind::ConnectionRefused
            )))),
            super::ConnectOutcome::Refused
        );
        assert_eq!(
            super::classify_connect(Some(Ok(()))),
            super::ConnectOutcome::Accepted
        );
    }

    #[test]
    fn the_refusal_probe_cannot_eat_the_ladder_it_short_circuits() {
        assert!(super::REFUSAL_PROBE_TIMEOUT * 4 <= super::HEALTH_PROBE_TIMEOUT);
    }

    #[test]
    fn the_fast_path_asks_for_absence_and_not_for_presence() {
        // `check_backend_present` reports our unbound backend as absent; the ladder must not use it.
        let src = include_str!("../../frontend/src/features/auth/api.ts").replace("\r\n", "\n");
        assert!(
            src.contains("invoke<boolean>(\"check_backend_is_gone\""),
            "the retry ladder no longer has a fast path for a backend that is provably gone"
        );
        let marker = "if (attempt === 0 && (await nativeBackendIsGone()))";
        assert!(
            src.contains(marker),
            "the fast path stopped being gated on the first failure and a positive answer, \
             so a slow backend can be abandoned before the ladder has run"
        );
    }

    #[tokio::test]
    async fn a_port_with_nothing_on_it_reads_as_death_not_a_stall() {
        let client = crate::loopback_http::client(Duration::from_secs(5)).unwrap();

        // Draw from below the ephemeral range: a freed port-0 port can be rebound by a sibling test,
        // and retrying a port that answered would consume another test's one-shot connection.
        let port = {
            let mut free = None;
            for candidate in 20_000..20_064u16 {
                if let Ok(listener) = TcpListener::bind(("127.0.0.1", candidate)).await {
                    drop(listener);
                    free = Some(candidate);
                    break;
                }
            }
            free.expect("every port in the probe window was already bound")
        };

        let error = client
            .get(format!("http://127.0.0.1:{port}/api/liveness"))
            .send()
            .await
            .expect_err("nothing is listening on this port");

        let liveness = super::liveness_from_probe_error(&error);
        assert!(!liveness.probe_timed_out, "a refused port read as a stall");
        assert_eq!(
            super::watchdog_failure_budget(true, liveness.probe_timed_out),
            super::HEALTH_WATCHDOG_MAX_FAILURES,
            "a dead port inherits the busy budget from the last answer it gave"
        );
    }

    #[tokio::test]
    async fn the_busy_marker_survives_the_wire() {
        // End to end, so the field name is checked against what main.py publishes.
        let (busy_port, _) = probe_test_backend(
            Some(
                r#"{"status":"alive","service":"Unsloth UI Backend","inference_active":true}"#
                    .to_string(),
            ),
            ready_health(false),
        )
        .await;
        let (idle_port, _) = probe_test_backend(
            Some(r#"{"status":"alive","service":"Unsloth UI Backend"}"#.to_string()),
            ready_health(false),
        )
        .await;

        assert!(
            super::check_health_inner(busy_port, super::HEALTH_PROBE_TIMEOUT)
                .await
                .unwrap()
                .inference_active
        );
        assert!(
            !super::check_health_inner(idle_port, super::HEALTH_PROBE_TIMEOUT)
                .await
                .unwrap()
                .inference_active
        );
    }

    #[test]
    fn both_watchdog_paths_classify_a_failed_probe() {
        // Normalise CRLF: include_str! embeds CRLF on Windows checkouts.
        let src = include_str!("commands.rs").replace("\r\n", "\n");
        let start = src
            .find("async fn check_watchdog_health")
            .expect("check_watchdog_health moved; update this guard");
        let body = &src[start..];
        let body = &body[..body.find("\n}\n").expect("could not find the function end")];

        assert_eq!(
            body.matches("liveness_from_probe_error").count(),
            2,
            "the owned and adopted branches must both classify a failed probe"
        );
    }

    #[test]
    fn a_backend_that_answers_idle_gives_the_wide_budget_back() {
        let generating = super::watchdog_inference_active_after(true, true, false);
        assert!(!generating);
        assert_eq!(
            super::watchdog_failure_budget(generating, true),
            super::HEALTH_WATCHDOG_MAX_FAILURES
        );
    }

    #[test]
    fn an_adopted_backend_that_answers_and_then_stalls_keeps_the_busy_budget() {
        assert!(super::adopted_failure_is_a_stall(false, true, false));
        assert_eq!(
            super::watchdog_failure_budget(
                true,
                super::adopted_failure_is_a_stall(false, true, false)
            ),
            super::HEALTH_WATCHDOG_MAX_FAILURES_BUSY
        );
    }

    #[test]
    fn a_stalled_adopted_backend_that_says_it_is_generating_survives_the_confirmation() {
        let served = super::BackendLiveness {
            alive: true,
            warming_up: false,
            inference_active: true,
            probe_timed_out: false,
            answered: true,
        };
        let confirmed = super::adopted_backend_liveness(false, &served, false);
        assert!(
            !confirmed.alive,
            "an unverified re-check is not a live answer"
        );
        assert!(super::watchdog_confirm_keeps_backend(&confirmed));

        let quiet =
            super::adopted_backend_liveness(false, &super::BackendLiveness::default(), false);
        assert!(!super::watchdog_confirm_keeps_backend(&quiet));

        let idle = super::adopted_backend_liveness(
            true,
            &super::BackendLiveness {
                alive: true,
                warming_up: false,
                inference_active: false,
                probe_timed_out: false,
                answered: true,
            },
            false,
        );
        assert!(super::watchdog_confirm_keeps_backend(&idle));

        let taken_over = super::adopted_backend_liveness(false, &served, true);
        assert!(!super::watchdog_confirm_keeps_backend(&taken_over));
    }

    #[test]
    fn a_port_another_backend_took_over_stays_on_the_normal_budget() {
        // A freed port rebound by another Unsloth backend is a takeover, not a stall.
        assert!(!super::adopted_failure_is_a_stall(false, true, true));
        assert_eq!(
            super::watchdog_failure_budget(
                true,
                super::adopted_failure_is_a_stall(false, true, true)
            ),
            super::HEALTH_WATCHDOG_MAX_FAILURES
        );
        assert!(!super::watchdog_should_confirm_before_death(
            super::HEALTH_WATCHDOG_MAX_FAILURES,
            super::HEALTH_WATCHDOG_MAX_FAILURES,
            super::adopted_failure_is_a_stall(false, true, true),
        ));
    }

    #[test]
    fn an_adopted_port_that_answered_nothing_is_not_called_a_stall() {
        assert!(!super::adopted_failure_is_a_stall(false, false, false));
        assert_eq!(
            super::watchdog_failure_budget(
                true,
                super::adopted_failure_is_a_stall(false, false, false)
            ),
            super::HEALTH_WATCHDOG_MAX_FAILURES
        );
        assert!(!super::adopted_failure_is_a_stall(true, true, false));
    }

    #[test]
    fn a_generation_that_starts_between_probes_still_gets_one_last_chance() {
        assert!(super::watchdog_should_confirm_before_death(
            super::HEALTH_WATCHDOG_MAX_FAILURES,
            super::HEALTH_WATCHDOG_MAX_FAILURES,
            true,
        ));
        assert!(!super::watchdog_should_confirm_before_death(
            super::HEALTH_WATCHDOG_MAX_FAILURES - 1,
            super::HEALTH_WATCHDOG_MAX_FAILURES,
            true,
        ));
        assert!(!super::watchdog_should_confirm_before_death(
            super::HEALTH_WATCHDOG_MAX_FAILURES,
            super::HEALTH_WATCHDOG_MAX_FAILURES,
            false,
        ));
    }

    #[test]
    fn the_last_chance_budget_is_wider_than_the_one_that_gave_up() {
        assert!(super::HEALTH_CONFIRM_PROBE_TIMEOUT > super::HEALTH_PROBE_TIMEOUT);
    }

    #[derive(Clone, Copy, Debug)]
    enum Probe {
        Answered { warming: bool, busy: bool },
        TimedOut,
        Refused,
    }

    fn answered(busy: bool) -> Probe {
        Probe::Answered {
            warming: false,
            busy,
        }
    }

    /// Mirrors health_watchdog's failure accounting (it needs an AppHandle and real sleeps) using the
    /// same helpers; returns the second the backend was declared dead.
    fn simulate_watchdog(probes: &[Probe], confirms: &[Probe]) -> Option<u64> {
        let interval = super::HEALTH_WATCHDOG_INTERVAL.as_secs();
        let probe_budget = super::HEALTH_PROBE_TIMEOUT.as_secs();
        let confirm_budget = super::HEALTH_CONFIRM_PROBE_TIMEOUT.as_secs();

        let mut failures: u32 = 0;
        let mut was_generating = false;
        let mut elapsed: u64 = 0;
        let mut next_confirm = 0usize;

        for probe in probes {
            elapsed += interval;
            let liveness = match *probe {
                Probe::Answered { warming, busy } => super::BackendLiveness {
                    alive: true,
                    warming_up: warming,
                    inference_active: busy,
                    probe_timed_out: false,
                    answered: true,
                },
                Probe::TimedOut => {
                    elapsed += probe_budget;
                    super::BackendLiveness {
                        probe_timed_out: true,
                        ..super::BackendLiveness::default()
                    }
                }
                Probe::Refused => super::BackendLiveness::default(),
            };

            if liveness.alive {
                was_generating = super::watchdog_inference_active_after(
                    was_generating,
                    liveness.alive,
                    liveness.inference_active,
                );
                failures = 0;
                continue;
            }

            let budget = super::watchdog_failure_budget(was_generating, liveness.probe_timed_out);
            failures += 1;
            if super::watchdog_should_confirm_before_death(
                failures,
                budget,
                liveness.probe_timed_out,
            ) {
                let confirm = confirms
                    .get(next_confirm)
                    .copied()
                    .unwrap_or(Probe::TimedOut);
                next_confirm += 1;
                let confirmed = match confirm {
                    Probe::Answered { warming, busy } => super::BackendLiveness {
                        alive: true,
                        warming_up: warming,
                        inference_active: busy,
                        probe_timed_out: false,
                        answered: true,
                    },
                    Probe::TimedOut => {
                        elapsed += confirm_budget;
                        super::BackendLiveness {
                            probe_timed_out: true,
                            ..Default::default()
                        }
                    }
                    Probe::Refused => super::BackendLiveness::default(),
                };
                if super::watchdog_confirm_keeps_backend(&confirmed) {
                    was_generating = confirmed.inference_active;
                    failures = 0;
                    continue;
                }
            }
            if failures >= budget {
                return Some(elapsed);
            }
        }
        None
    }

    #[test]
    fn simulated_timelines_kill_only_what_should_be_killed() {
        let plain = super::HEALTH_WATCHDOG_MAX_FAILURES as usize;
        let busy = super::HEALTH_WATCHDOG_MAX_FAILURES_BUSY as usize;
        let interval = super::HEALTH_WATCHDOG_INTERVAL.as_secs();
        let probe_budget = super::HEALTH_PROBE_TIMEOUT.as_secs();
        let confirm_budget = super::HEALTH_CONFIRM_PROBE_TIMEOUT.as_secs();
        let stall_cycle = interval + probe_budget;

        assert_eq!(simulate_watchdog(&vec![answered(false); 200], &[]), None);
        assert_eq!(simulate_watchdog(&vec![answered(true); 200], &[]), None);
        assert_eq!(
            simulate_watchdog(
                &vec![
                    Probe::Answered {
                        warming: true,
                        busy: false
                    };
                    200
                ],
                &[]
            ),
            None
        );

        let mut dead = vec![answered(false)];
        dead.extend(vec![Probe::Refused; plain]);
        assert_eq!(simulate_watchdog(&dead, &[]), Some(interval * 4));
        let mut died_generating = vec![answered(true)];
        died_generating.extend(vec![Probe::Refused; plain]);
        assert_eq!(simulate_watchdog(&died_generating, &[]), Some(interval * 4));

        let mut stalling = vec![answered(true)];
        stalling.extend(vec![Probe::TimedOut; busy - 1]);
        assert_eq!(simulate_watchdog(&stalling, &[]), None);

        let mut wedged = vec![answered(true)];
        wedged.extend(vec![Probe::TimedOut; busy + 4]);
        assert_eq!(
            simulate_watchdog(&wedged, &[Probe::TimedOut]),
            Some(interval + busy as u64 * stall_cycle + confirm_budget)
        );

        let mut started_between_probes = vec![answered(false)];
        started_between_probes.extend(vec![Probe::TimedOut; 40]);
        assert_eq!(
            simulate_watchdog(
                &started_between_probes,
                &[answered(true); 8] // one per spent budget
            ),
            None
        );

        let mut idle_confirmed = vec![answered(false)];
        idle_confirmed.extend(vec![Probe::TimedOut; plain * 2]);
        assert_eq!(
            simulate_watchdog(&idle_confirmed, &[answered(false), Probe::TimedOut]),
            Some(interval + (plain * 2) as u64 * stall_cycle + confirm_budget)
        );
        assert_eq!(
            simulate_watchdog(&started_between_probes, &[Probe::TimedOut]),
            Some(interval + plain as u64 * stall_cycle + confirm_budget)
        );
        assert_eq!(
            simulate_watchdog(&started_between_probes, &[answered(false)]),
            Some(interval + (plain * 2) as u64 * stall_cycle + confirm_budget)
        );

        let mut finished = vec![answered(true), answered(false)];
        finished.extend(vec![Probe::Refused; plain]);
        assert_eq!(simulate_watchdog(&finished, &[]), Some(interval * 5));

        let mut stalled_then_gone = vec![answered(true)];
        stalled_then_gone.extend(vec![Probe::TimedOut; 5]);
        stalled_then_gone.push(Probe::Refused);
        assert_eq!(
            simulate_watchdog(&stalled_then_gone, &[]),
            Some(interval + 5 * stall_cycle + interval)
        );

        let flapping: Vec<Probe> = (0..200)
            .map(|i| {
                if i % 2 == 0 {
                    answered(true)
                } else {
                    Probe::TimedOut
                }
            })
            .collect();
        assert_eq!(simulate_watchdog(&flapping, &[]), None);
    }

    /// Answers only after `delay`: a loop running behind, not gone.
    async fn slow_test_backend(delay: Duration, body: &'static str) -> u16 {
        let listener = TcpListener::bind(("127.0.0.1", 0))
            .await
            .expect("probe test needs a loopback port");
        let port = listener.local_addr().unwrap().port();
        tokio::spawn(async move {
            while let Ok((mut stream, _)) = listener.accept().await {
                tokio::spawn(async move {
                    let mut buffer = [0; 2048];
                    if stream.read(&mut buffer).await.is_err() {
                        return;
                    }
                    tokio::time::sleep(delay).await;
                    let response = format!(
                        "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
                        body.len()
                    );
                    let _ = stream.write_all(response.as_bytes()).await;
                });
            }
        });
        port
    }

    #[tokio::test]
    async fn a_wider_budget_reaches_a_backend_the_per_cycle_one_gives_up_on() {
        let port = slow_test_backend(
            Duration::from_millis(600),
            r#"{"status":"alive","service":"Unsloth UI Backend","inference_active":true}"#,
        )
        .await;

        let gave_up = super::check_health_inner(port, Duration::from_millis(150))
            .await
            .expect_err("a 150ms budget cannot outlast a 600ms answer");
        assert!(
            super::liveness_from_probe_error(&gave_up).probe_timed_out,
            "a slow-but-answering backend must read as a stall, not as a dead port"
        );

        let confirmed = super::check_health_inner(port, Duration::from_secs(5))
            .await
            .expect("a 5s budget outlasts a 600ms answer");
        assert!(confirmed.alive);
        assert!(
            confirmed.inference_active,
            "the wider probe must still parse the busy marker, or the reprieve never fires"
        );
    }

    #[test]
    fn conflict_message_only_blames_a_terminal_for_a_same_install_backend() {
        let message = |reason: &str| {
            super::external_conflict_message(&crate::preflight::ExternalBackendConflict {
                port: 8890,
                reason: reason.to_string(),
            })
        };

        assert!(message("same_root_external_backend_active").contains("from a terminal"));

        for reason in [
            "desktop_owned_backend_active",
            "ambiguous_root_external_backend_active",
        ] {
            let message = message(reason);
            assert!(!message.contains("from a terminal"), "{message}");
            assert!(!message.contains("unsloth studio update"), "{message}");
            assert!(message.contains("port 8890"), "{message}");
        }
    }

    #[test]
    fn watchdog_failure_policy_counts_only_after_health_or_grace_period() {
        for (has_seen_healthy, elapsed, expected) in [
            (
                false,
                super::BACKEND_STARTUP_GRACE_PERIOD - Duration::from_secs(1),
                false,
            ),
            (true, Duration::from_secs(1), true),
            (false, super::BACKEND_STARTUP_GRACE_PERIOD, true),
        ] {
            assert_eq!(
                super::should_count_watchdog_failure(has_seen_healthy, elapsed),
                expected
            );
        }
    }

    #[test]
    fn a_restart_during_the_last_chance_probe_is_not_declared_dead() {
        // A probe answer names a service, not a generation; anything but the same live generation must stop.
        assert!(super::watchdog_may_still_act(7, 7, true, false));
        assert!(!super::watchdog_may_still_act(8, 7, true, false));
        assert!(!super::watchdog_may_still_act(7, 7, false, false));
        assert!(!super::watchdog_may_still_act(7, 7, true, true));
    }

    #[test]
    fn the_watchdog_rereads_the_generation_after_the_confirm_probe() {
        // Up to 40s separates the generation check from the kill; without the re-read a restart gets killed.
        // Normalise CRLF: include_str! embeds CRLF on the Windows runner.
        let src = include_str!("commands.rs").replace("\r\n", "\n");
        let start = src
            .find("async fn health_watchdog")
            .expect("health_watchdog moved; update this guard");
        let body = &src[start..];
        let body = &body[..body.find("\n}\n").expect("could not find the function end")];

        let confirm = body
            .find("HEALTH_CONFIRM_PROBE_TIMEOUT")
            .expect("the last-chance probe is gone; update this guard");
        let guard = body
            .find("watchdog_may_still_act")
            .expect("the post-probe generation re-read is gone");
        let kill = body
            .rfind("stop_backend")
            .expect("the unresponsive-backend kill moved");
        assert!(
            confirm < guard && guard < kill,
            "the generation re-read must sit between the last-chance probe and the kill"
        );
    }
}

/// Periodic check for hung backends. Failures are ignored during the startup grace; afterwards 3
/// failures (12 when last seen generating and probes time out) emit `server-crashed`.
async fn health_watchdog(
    app: AppHandle,
    state: BackendState,
    shutdown: ShutdownFlag,
    diagnostics: DiagnosticsState,
    generation: u64,
    count_failures_immediately: bool,
) {
    use std::sync::atomic::Ordering;

    let started_at = Instant::now();
    let mut consecutive_failures: u32 = 0;
    let mut has_seen_healthy = count_failures_immediately;
    let mut was_generating = false;

    loop {
        tokio::time::sleep(HEALTH_WATCHDOG_INTERVAL).await;

        if shutdown.load(Ordering::SeqCst) {
            info!("Health watchdog: shutdown flag set, exiting");
            break;
        }

        let (port, has_owned, has_adopted, current_generation) = {
            let proc = match state.lock() {
                Ok(p) => p,
                Err(_) => break,
            };
            (
                proc.port,
                proc.has_owned_backend(),
                proc.has_adopted_backend(),
                proc.generation,
            )
        };

        if current_generation != generation {
            info!("Health watchdog: backend generation changed, exiting");
            break;
        }

        if !has_owned {
            info!("Health watchdog: backend stopped, exiting");
            break;
        }

        let should_count_failure =
            should_count_watchdog_failure(has_seen_healthy, started_at.elapsed());

        let Some(port) = port else {
            if has_adopted {
                diagnostics::record_backend_watchdog(
                    &diagnostics,
                    generation,
                    "adopted_port_missing",
                );
                error!("Health watchdog: adopted backend lost its port, declaring dead");
                process::clear_adopted_backend_if_current(
                    &state,
                    generation,
                    None,
                    "watchdog adopted port missing",
                );
                let _ = app.emit("server-crashed", ());
                break;
            }
            if !should_count_failure {
                info!("Health watchdog: backend has not reported a validated port yet");
                continue;
            }
            consecutive_failures += 1;
            warn!(
                "Health watchdog: missing validated port failure {}/{}",
                consecutive_failures, HEALTH_WATCHDOG_MAX_FAILURES
            );
            if consecutive_failures >= HEALTH_WATCHDOG_MAX_FAILURES {
                diagnostics::record_backend_watchdog(
                    &diagnostics,
                    generation,
                    "missing_validated_port",
                );
                error!("Health watchdog: backend never reported a validated port, killing and declaring dead");
                let _ = process::stop_backend(&state, &shutdown, Some(&diagnostics));
                let _ = app.emit("server-crashed", ());
                break;
            }
            continue;
        };

        let liveness =
            check_watchdog_health(&state, generation, port, has_adopted, HEALTH_PROBE_TIMEOUT)
                .await;
        if liveness.alive {
            // Only end the grace once the whole warm finished: warm imports hold the GIL and can miss probes.
            if liveness.warming_up {
                info!(
                    "Health watchdog: backend on port {} is alive but still warming up, holding the startup grace period",
                    port
                );
            }
            has_seen_healthy =
                watchdog_seen_healthy_after(has_seen_healthy, liveness.alive, liveness.warming_up);
            was_generating = watchdog_inference_active_after(
                was_generating,
                liveness.alive,
                liveness.inference_active,
            );
            consecutive_failures = 0;
        } else if !should_count_failure {
            info!(
                "Health watchdog: startup health check failed on port {} before grace period elapsed",
                port
            );
        } else {
            let budget = watchdog_failure_budget(was_generating, liveness.probe_timed_out);
            consecutive_failures += 1;
            warn!(
                "Health watchdog: failure {}/{} on port {}{}",
                consecutive_failures,
                budget,
                port,
                if budget > HEALTH_WATCHDOG_MAX_FAILURES {
                    " (backend last seen generating, probe timed out)"
                } else {
                    ""
                }
            );
            if watchdog_should_confirm_before_death(
                consecutive_failures,
                budget,
                liveness.probe_timed_out,
            ) {
                let confirmed = check_watchdog_health(
                    &state,
                    generation,
                    port,
                    has_adopted,
                    HEALTH_CONFIRM_PROBE_TIMEOUT,
                )
                .await;
                if watchdog_confirm_keeps_backend(&confirmed) {
                    warn!(
                        "Health watchdog: backend on port {} survived the last-chance probe, not declaring it dead",
                        port
                    );
                    has_seen_healthy = watchdog_seen_healthy_after(
                        has_seen_healthy,
                        confirmed.alive,
                        confirmed.warming_up,
                    );
                    was_generating = confirmed.inference_active;
                    consecutive_failures = 0;
                    continue;
                }
            }
            // Re-read before acting: a restart during the probes would make stop_backend kill the replacement.
            let (current_generation, still_owned) = {
                let proc = match state.lock() {
                    Ok(p) => p,
                    Err(_) => break,
                };
                (proc.generation, proc.has_owned_backend())
            };
            if !watchdog_may_still_act(
                current_generation,
                generation,
                still_owned,
                shutdown.load(Ordering::SeqCst),
            ) {
                info!(
                    "Health watchdog: backend changed while the health probe was in flight, exiting without declaring it dead"
                );
                break;
            }
            if consecutive_failures >= budget {
                diagnostics::record_backend_watchdog(
                    &diagnostics,
                    generation,
                    "unresponsive_health_check",
                );
                if has_adopted {
                    error!(
                        "Health watchdog: adopted backend unresponsive, clearing state and declaring dead"
                    );
                    process::clear_adopted_backend_if_current(
                        &state,
                        generation,
                        Some(port),
                        "watchdog health check failures",
                    );
                } else {
                    error!("Health watchdog: backend unresponsive, killing and declaring dead");
                    let _ = process::stop_backend(&state, &shutdown, Some(&diagnostics));
                }
                let _ = app.emit("server-crashed", ());
                break;
            }
        }
    }

    process::clear_adopted_watchdog_if_current(&state, generation);
}