Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

44.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""whisper.cpp rides on the llama update: a llama failure aborts, a whisper failure does not."""

from __future__ import annotations

import json
import sys
import time
from pathlib import Path

import pytest

_BACKEND = Path(__file__).resolve().parents[1]
if str(_BACKEND) not in sys.path:
    sys.path.insert(0, str(_BACKEND))

import utils.audio_cpp_update as aupd  # noqa: E402
import utils.llama_cpp_freshness as freshness  # noqa: E402
import utils.llama_cpp_update as upd  # noqa: E402
import utils.whisper_cpp_freshness as wfresh  # noqa: E402
import utils.whisper_cpp_update as wupd  # noqa: E402

MARKER = "UNSLOTH_PREBUILT_INFO.json"
WHISPER_MARKER = "UNSLOTH_WHISPER_PREBUILT_INFO.json"

# Pre-whisper fields; the combined payload must stay an exact superset for the UI.
LEGACY_STATUS_FIELDS = {
    "supported",
    "update_available",
    "stale",
    "installed_tag",
    "latest_tag",
    "published_repo",
    "installed_at_utc",
    "age_days",
    "source_build",
    "update_size_bytes",
    "job",
}
LEGACY_JOB_FIELDS = {
    "state",
    "message",
    "from_tag",
    "to_tag",
    "reload_required",
    "error",
    "progress",
    "started_at",
    "finished_at",
}


class _FakeInstallerPopen:
    """Stands in for the streamed llama installer process."""

    def __init__(
        self,
        cmd,
        *,
        returncode = 0,
        lines = None,
        on_start = None,
        **kwargs,
    ):
        if on_start is not None:
            on_start(list(cmd))
        self.pid = 515151
        self.returncode = returncode
        self.stdout = iter(lines or [])

    def poll(self):
        return self.returncode

    def wait(self):
        return self.returncode

    def kill(self):
        pass


def _patch_llama_installer(
    monkeypatch,
    *,
    returncode = 0,
    lines = None,
    on_start = None,
):
    def _popen(cmd, **kw):
        parts = [str(part) for part in cmd]
        # Resolvers run the same script; faking them would answer the drift probe with install output.
        is_installer = any("install_llama_prebuilt" in part for part in parts) and not any(
            part.startswith("--resolve-") for part in parts
        )
        return _FakeInstallerPopen(
            cmd,
            returncode = returncode if is_installer else 0,
            lines = lines if is_installer else None,
            on_start = on_start if is_installer else None,
        )

    monkeypatch.setattr(upd.subprocess, "Popen", _popen)


def _write_llama_install(dir_: Path, tag: str) -> str:
    """Create a fake llama prebuilt install and return the llama-server path."""
    bin_dir = dir_ / "build" / "bin"
    bin_dir.mkdir(parents = True, exist_ok = True)
    binary = bin_dir / "llama-server"
    binary.write_text("stub")
    (dir_ / MARKER).write_text(
        json.dumps(
            {
                "tag": tag,
                "release_tag": tag,
                "published_repo": "unslothai/llama.cpp",
                "installed_at_utc": "2020-01-01T00:00:00Z",
            }
        )
    )
    return str(binary)


def _write_whisper_install(
    dir_: Path,
    tag: str,
    backend: str = "cpu",
) -> str:
    """Create a fake whisper prebuilt install and return the whisper-server path."""
    bin_dir = dir_ / "build" / "bin"
    bin_dir.mkdir(parents = True, exist_ok = True)
    binary = bin_dir / "whisper-server"
    binary.write_text("stub")
    (dir_ / WHISPER_MARKER).write_text(
        json.dumps(
            {
                "release_tag": tag,
                "upstream_tag": tag.split("-")[0],
                "published_repo": "unslothai/whisper.cpp",
                "backend": backend,
                "installed_at_utc": "2020-01-01T00:00:00Z",
            }
        )
    )
    return str(binary)


@pytest.fixture(autouse = True)
def _clean_state(monkeypatch, tmp_path):
    freshness.reset_caches()
    wfresh.reset_caches()
    upd._reset_job_for_tests()
    upd._resolve_memo.clear()
    upd._backends_memo.clear()
    wupd._resolve_memo.clear()
    monkeypatch.setattr(freshness, "_cache_dir", lambda: tmp_path / ".llama_cache")
    monkeypatch.setattr(wfresh, "_cache_dir", lambda: tmp_path / ".whisper_cache")
    for var in (
        "LLAMA_SERVER_PATH",
        "UNSLOTH_LLAMA_CPP_PATH",
        "WHISPER_SERVER_PATH",
        "UNSLOTH_WHISPER_CPP_PATH",
    ):
        monkeypatch.delenv(var, raising = False)
    monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: None)
    monkeypatch.setattr(wfresh, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: None)
    # A dev box may hold a real managed audio.cpp; tests that want the phase opt in with _setup_audio.
    monkeypatch.setattr(
        aupd,
        "chained_phase_plan",
        lambda **_kw: {
            "status": None,
            "update_available": False,
            "skip_reason": "not_installed",
            "phase": None,
        },
    )
    yield
    freshness.reset_caches()
    wfresh.reset_caches()
    upd._reset_job_for_tests()
    upd._resolve_memo.clear()
    upd._backends_memo.clear()
    wupd._resolve_memo.clear()


def _setup_llama(
    monkeypatch,
    tmp_path,
    *,
    installed = "b9493",
    latest = "b9518",
):
    """Marker-managed llama install; behind when installed != latest."""
    install_dir = tmp_path / "llama.cpp"
    binary = _write_llama_install(install_dir, installed)
    monkeypatch.setattr(upd, "_find_binary", lambda: binary)
    monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")
    monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: latest)
    return install_dir


def _setup_whisper(
    monkeypatch,
    tmp_path,
    *,
    installed = "v1.9.1-unsloth.1",
    latest = "v1.9.2-unsloth.1",
):
    """Marker-managed whisper install; behind when latest is newer."""
    install_dir = tmp_path / "whisper.cpp"
    binary = _write_whisper_install(install_dir, installed)
    monkeypatch.setattr(wupd, "_find_binary", lambda: binary)
    monkeypatch.setattr(wupd, "_installer_script", lambda: tmp_path / "install_whisper_prebuilt.py")
    monkeypatch.setattr(wfresh, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: latest)
    return install_dir


def _patch_whisper_phase(
    monkeypatch,
    events,
    *,
    to_tag = "v1.9.2-unsloth.1",
    error = None,
):
    """Record whisper phase runs without touching a real installer."""

    def _run(phase, set_progress):
        events.append("whisper")
        if error is not None:
            raise RuntimeError(error)
        set_progress(0.5)
        return {
            "to_tag": to_tag,
            "reload_required": False,
            "message": f"Updated whisper.cpp to {to_tag}.",
        }

    monkeypatch.setattr(wupd, "run_chained_phase", _run)


_AUDIO_STATUS = {
    "installed_tag": "v0.8.2-audio8-perf-hotfix-unsloth.1",
    "latest_tag": "v0.9.0-unsloth.1",
    "update_size_bytes": None,
}


def _setup_audio(
    monkeypatch,
    events,
    *,
    error = None,
    reload_required = False,
):
    """An offered audio.cpp phase whose runs are recorded instead of installing."""
    plans = []

    def _plan(**kw):
        plans.append(("audio", kw))
        return {
            "status": dict(_AUDIO_STATUS),
            "update_available": True,
            "skip_reason": None,
            "phase": {"to_tag": "v0.9.0-unsloth.1"},
        }

    def _run(phase, set_progress):
        events.append("audio")
        if error is not None:
            raise RuntimeError(error)
        set_progress(0.5)
        return {
            "to_tag": "v0.9.0-unsloth.1",
            "reload_required": reload_required,
            "message": "Updated audio.cpp to v0.9.0-unsloth.1.",
        }

    monkeypatch.setattr(aupd, "chained_phase_plan", _plan)
    monkeypatch.setattr(aupd, "run_chained_phase", _run)
    return plans


def _wait_for_job():
    deadline = time.time() + 10
    while time.time() < deadline:
        with upd._job_lock:
            job = dict(upd._job)
        if job["state"] in ("success", "error"):
            return job
        time.sleep(0.05)
    with upd._job_lock:
        return dict(upd._job)


def test_status_payload_is_exact_superset_of_legacy_fields(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path)
    _setup_whisper(monkeypatch, tmp_path)
    st = upd.get_update_status(force_refresh = True)
    assert LEGACY_STATUS_FIELDS <= set(st)
    assert LEGACY_JOB_FIELDS <= set(st["job"])
    assert st["llama_update_available"] is True
    assert st["whisper"]["update_available"] is True
    assert st["whisper"]["latest_tag"] == "v1.9.2-unsloth.1"
    assert st["update_component"] == "llama"


def test_status_whisper_only_release_waits_for_llama(monkeypatch, tmp_path):
    # llama current, whisper behind: whisper.cpp alone never raises the update item.
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path)
    st = upd.get_update_status(force_refresh = True)
    assert st["llama_update_available"] is False
    assert st["whisper"]["update_available"] is False
    assert st["whisper"]["skip_reason"] == "waits_for_llama"
    assert st["update_available"] is False
    assert st["update_component"] is None
    assert st["installed_tag"] == "b9518"
    assert st["latest_tag"] == "b9518"
    assert st["whisper"]["installed_tag"] == "v1.9.1-unsloth.1"
    assert st["whisper"]["latest_tag"] == "v1.9.2-unsloth.1"


def test_status_whisper_current_does_not_flip_union(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path, installed = "v1.9.2-unsloth.1", latest = "v1.9.2-unsloth.1")
    st = upd.get_update_status(force_refresh = True)
    assert st["update_available"] is False
    assert st["whisper"]["skip_reason"] == "up_to_date"
    assert st["update_component"] is None


def test_status_survives_whisper_probe_failure(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path)

    def _boom(*, force_refresh = False):
        raise RuntimeError("probe exploded")

    monkeypatch.setattr(wupd, "chained_phase_plan", _boom)
    st = upd.get_update_status(force_refresh = True)
    assert st["update_available"] is True
    assert st["whisper"] is None


def test_status_skips_github_when_update_checks_disabled(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path)
    _setup_whisper(monkeypatch, tmp_path)

    def _network(*args, **kwargs):
        raise AssertionError("reached GitHub despite UNSLOTH_DISABLE_UPDATE_CHECK=1")

    monkeypatch.setattr(freshness, "_fetch_latest_release_tag", _network)
    monkeypatch.setattr(wfresh, "_fetch_latest_release_tag", _network)
    monkeypatch.setattr(upd, "_pending_backend_migration", _network)
    monkeypatch.setattr(wupd, "_resolve_prebuilt_for_host", _network)
    monkeypatch.setattr(wupd.sys, "platform", "darwin")
    monkeypatch.setenv("UNSLOTH_DISABLE_UPDATE_CHECK", "1")

    st = upd.get_update_status(force_refresh = True)
    assert st["update_available"] is False
    assert st["update_component"] is None
    assert st["backend_migration_available"] is False
    assert st["supported"] is True
    assert st["installed_tag"] == "b9493"
    assert st["latest_tag"] is None
    assert st["whisper"]["installed_tag"] == "v1.9.1-unsloth.1"
    assert st["whisper"]["update_available"] is False


def test_whisper_source_build_skips_probe_when_update_checks_disabled(monkeypatch, tmp_path):
    binary = tmp_path / "whisper.cpp" / "build" / "bin" / "whisper-server"
    binary.parent.mkdir(parents = True)
    binary.write_text("stub")
    monkeypatch.setattr(wupd, "_find_binary", lambda: str(binary))

    def _resolve(**kwargs):
        raise AssertionError("probed for a prebuilt despite UNSLOTH_DISABLE_UPDATE_CHECK=1")

    monkeypatch.setattr(wupd, "_resolve_prebuilt_for_host", _resolve)
    monkeypatch.setenv("UNSLOTH_DISABLE_UPDATE_CHECK", "1")
    st = wupd.get_update_status(force_refresh = True)
    assert st["update_available"] is False
    assert st["source_build"] is False


def test_whisper_plan_skips_local_link(monkeypatch, tmp_path):
    monkeypatch.setattr(wupd, "_find_binary", lambda: str(tmp_path / "whisper-server"))
    monkeypatch.setattr(wupd, "_active_install_is_local_link", lambda b: True)
    plan = wupd.chained_phase_plan()
    assert plan["update_available"] is False
    assert plan["skip_reason"] == "local_link"
    assert plan["phase"] is None


def test_whisper_plan_skips_source_build(monkeypatch, tmp_path):
    binary = tmp_path / "whisper.cpp" / "build" / "bin" / "whisper-server"
    binary.parent.mkdir(parents = True)
    binary.write_text("stub")
    monkeypatch.setattr(wupd, "_find_binary", lambda: str(binary))
    plan = wupd.chained_phase_plan()
    assert plan["skip_reason"] == "source_build"
    assert plan["phase"] is None


def test_whisper_update_targets_canonical_root_when_inner_marker_exists(tmp_path):
    install_dir = tmp_path / "whisper.cpp"
    binary = install_dir / "build" / "bin" / "whisper-server"
    binary.parent.mkdir(parents = True)
    binary.write_text("stub")
    (install_dir / WHISPER_MARKER).write_text("{}")
    (binary.parent / WHISPER_MARKER).write_text("{}")
    assert wupd._install_dir_for(str(binary)) == install_dir


def test_whisper_plan_skips_when_not_installed(monkeypatch):
    monkeypatch.setattr(wupd, "_find_binary", lambda: None)
    plan = wupd.chained_phase_plan()
    assert plan["skip_reason"] == "not_installed"
    assert plan["phase"] is None


def test_whisper_plan_eligible_when_behind(monkeypatch, tmp_path):
    install_dir = _setup_whisper(monkeypatch, tmp_path)
    script = tmp_path / "install_whisper_prebuilt.py"
    script.write_text("stub")
    plan = wupd.chained_phase_plan(force_refresh = True)
    assert plan["update_available"] is True
    assert plan["skip_reason"] is None
    assert plan["phase"]["install_dir"] == install_dir
    assert plan["phase"]["repo"] == "unslothai/whisper.cpp"
    assert plan["phase"]["backend"] == "cpu"
    # Unpinned, the installer's /releases/latest can lag and reinstall an older build in a loop.
    assert plan["phase"]["pin_release_tag"] == "v1.9.2-unsloth.1"


def test_whisper_plan_requires_a_repairable_pair_for_slim_installs(monkeypatch, tmp_path):
    install_dir = _setup_whisper(monkeypatch, tmp_path)
    marker_path = install_dir / WHISPER_MARKER
    marker = json.loads(marker_path.read_text())
    marker["install_kind"] = "slim"
    marker_path.write_text(json.dumps(marker))
    wfresh.reset_caches()
    monkeypatch.setattr(
        wupd,
        "_resolve_prebuilt_for_host",
        lambda **kwargs: {"prebuilt_available": False},
    )

    plan = wupd.chained_phase_plan(force_refresh = True)
    assert plan["update_available"] is False
    assert plan["skip_reason"] == "paired_llama_unavailable"

    repaired = wupd.chained_phase_plan(
        force_refresh = True,
        paired_llama_will_update = True,
    )
    assert repaired["update_available"] is True
    assert repaired["phase"] is not None


def test_whisper_phase_pins_installer_to_checked_release(monkeypatch, tmp_path):
    calls = []
    monkeypatch.setattr(
        wupd._flow,
        "stream_installer",
        lambda cmd, env, **kw: calls.append(cmd),
    )
    monkeypatch.setattr(wupd, "reset_caches", lambda **kw: None)
    monkeypatch.setattr(wupd, "latest_published_release", lambda repo, **kw: "v9")
    install_dir = tmp_path / "whisper.cpp"
    binary = _write_whisper_install(install_dir, "v9")
    monkeypatch.setattr(wupd, "_find_binary", lambda: binary)
    wupd.run_chained_phase(
        {
            "install_dir": install_dir,
            "repo": "unslothai/whisper.cpp",
            "asset": None,
            "backend": "cpu",
            "script": tmp_path / "install_whisper_prebuilt.py",
            "pin_release_tag": "v9",
        },
        lambda f: None,
    )
    cmd = calls[0]
    assert "--published-release-tag" in cmd
    assert cmd[cmd.index("--published-release-tag") + 1] == "v9"


def test_whisper_phase_exit_2_is_a_failed_phase(monkeypatch, tmp_path):
    def _raise_exit_2(cmd, env, **kw):
        raise wupd._flow.InstallerExit(2, "installer exited 2: incompatible release")

    monkeypatch.setattr(wupd._flow, "stream_installer", _raise_exit_2)
    install_dir = tmp_path / "whisper.cpp"
    binary = _write_whisper_install(install_dir, "v1")
    monkeypatch.setattr(wupd, "_find_binary", lambda: binary)
    with pytest.raises(wupd._flow.InstallerExit) as exc_info:
        wupd.run_chained_phase(
            {
                "install_dir": install_dir,
                "repo": "unslothai/whisper.cpp",
                "asset": None,
                "backend": "cpu",
                "script": tmp_path / "install_whisper_prebuilt.py",
                "pin_release_tag": None,
            },
            lambda f: None,
        )
    assert exc_info.value.returncode == 2


def test_llama_update_survives_unavailable_whisper_module(monkeypatch, tmp_path):
    import builtins

    llama_dir = _setup_llama(monkeypatch, tmp_path)
    monkeypatch.setattr(upd, "_whisper_chain_status", lambda **kw: None)
    _patch_llama_installer(
        monkeypatch,
        on_start = lambda cmd: _write_llama_install(llama_dir, "b9518"),
    )
    real_import = builtins.__import__

    def guarded_import(
        name,
        globals = None,
        locals = None,
        fromlist = (),
        level = 0,
    ):
        if name == "utils" and "whisper_cpp_update" in fromlist:
            raise AssertionError("whisper module was re-imported after its failed probe")
        return real_import(name, globals, locals, fromlist, level)

    monkeypatch.setattr(builtins, "__import__", guarded_import)

    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert job["phases"]["llama"]["state"] == "success"
    assert job["phases"]["whisper"]["state"] == "skipped"
    assert job["phases"]["whisper"]["reason"] == "unavailable"


def test_macos_status_uses_compatible_resolver_release(monkeypatch, tmp_path):
    _setup_whisper(
        monkeypatch,
        tmp_path,
        installed = "v1.9.1-unsloth.1",
        latest = "v1.9.2-unsloth.1",
    )
    monkeypatch.setattr(wupd.sys, "platform", "darwin")
    monkeypatch.setattr(
        wupd,
        "_resolve_prebuilt_for_host",
        lambda **kw: {
            "prebuilt_available": True,
            "release_tag": "v1.9.1-unsloth.1",
        },
    )

    status = wupd.get_update_status(force_refresh = True)
    assert status["latest_tag"] == "v1.9.1-unsloth.1"
    assert status["update_available"] is False
    assert status["stale"] is False


def test_whisper_phase_integrity_failure_is_not_swallowed(monkeypatch, tmp_path):
    def _raise_exit_1(cmd, env, **kw):
        raise wupd._flow.InstallerExit(1, "installer exited 1: checksum mismatch")

    monkeypatch.setattr(wupd._flow, "stream_installer", _raise_exit_1)
    install_dir = tmp_path / "whisper.cpp"
    binary = _write_whisper_install(install_dir, "v1")
    monkeypatch.setattr(wupd, "_find_binary", lambda: binary)
    with pytest.raises(wupd._flow.InstallerExit, match = "checksum mismatch"):
        wupd.run_chained_phase(
            {
                "install_dir": install_dir,
                "repo": "unslothai/whisper.cpp",
                "asset": None,
                "backend": "cpu",
                "script": tmp_path / "install_whisper_prebuilt.py",
                "pin_release_tag": None,
            },
            lambda f: None,
        )


def test_apply_runs_llama_then_whisper(monkeypatch, tmp_path):
    llama_dir = _setup_llama(monkeypatch, tmp_path)
    _setup_whisper(monkeypatch, tmp_path)
    (tmp_path / "install_whisper_prebuilt.py").write_text("stub")

    events = []
    _patch_llama_installer(
        monkeypatch,
        on_start = lambda cmd: (events.append("llama"), _write_llama_install(llama_dir, "b9518")),
    )
    _patch_whisper_phase(monkeypatch, events)

    res = upd.start_update()
    assert res["started"] is True, res
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert events == ["llama", "whisper"]
    assert job["phases"]["llama"]["state"] == "success"
    assert job["phases"]["llama"]["to_tag"] == "b9518"
    assert job["phases"]["whisper"]["state"] == "success"
    assert job["phases"]["whisper"]["to_tag"] == "v1.9.2-unsloth.1"
    assert job["from_tag"] == "b9493"
    assert job["to_tag"] == "b9518"
    assert "Updated llama.cpp to b9518." in job["message"]
    assert "Updated whisper.cpp to v1.9.2-unsloth.1." in job["message"]
    assert job["progress"] == 1.0
    assert LEGACY_JOB_FIELDS <= set(job)


def test_apply_llama_only_when_whisper_current(monkeypatch, tmp_path):
    llama_dir = _setup_llama(monkeypatch, tmp_path)
    _setup_whisper(monkeypatch, tmp_path, installed = "v1.9.2-unsloth.1", latest = "v1.9.2-unsloth.1")

    events = []
    _patch_llama_installer(
        monkeypatch,
        on_start = lambda cmd: (events.append("llama"), _write_llama_install(llama_dir, "b9518")),
    )
    _patch_whisper_phase(monkeypatch, events)

    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert events == ["llama"]
    assert job["phases"]["whisper"]["state"] == "skipped"
    assert job["phases"]["whisper"]["reason"] == "up_to_date"


def test_apply_whisper_only_release_starts_no_job(monkeypatch, tmp_path):
    # llama current + whisper behind: nothing to ride with, so the apply refuses as up to date.
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path)
    (tmp_path / "install_whisper_prebuilt.py").write_text("stub")

    events = []
    _patch_llama_installer(monkeypatch, on_start = lambda cmd: events.append("llama"))
    _patch_whisper_phase(monkeypatch, events)

    res = upd.start_update()
    assert res["started"] is False, res
    assert res["reason"] == "up_to_date"
    assert events == []


def test_a_failed_whisper_phase_can_be_retried_alone(monkeypatch, tmp_path):
    # llama updated but whisper failed: llama is now current, and Retry must still reach whisper.
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path)
    (tmp_path / "install_whisper_prebuilt.py").write_text("stub")
    failed = upd._flow.new_job()
    failed.update(state = "error", phases = {"whisper": {"state": upd._flow.PHASE_ERROR}})
    monkeypatch.setattr(upd, "_job", failed)

    st = upd.get_update_status(force_refresh = True)
    assert st["update_available"] is True
    assert st["update_component"] == "whisper"
    assert st["whisper"]["update_available"] is True

    events = []
    _patch_llama_installer(monkeypatch, on_start = lambda cmd: events.append("llama"))
    _patch_whisper_phase(monkeypatch, events)
    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert events == ["whisper"]


def test_apply_whisper_rides_with_audio(monkeypatch, tmp_path):
    # llama current, whisper and audio.cpp behind: whisper updates in the audio.cpp round.
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path)
    (tmp_path / "install_whisper_prebuilt.py").write_text("stub")

    events = []
    _patch_llama_installer(monkeypatch, on_start = lambda cmd: events.append("llama"))
    _patch_whisper_phase(monkeypatch, events)
    _setup_audio(monkeypatch, events)

    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert events == ["whisper", "audio"]
    assert job["phases"]["whisper"]["state"] == "success"


def test_whisper_reload_never_raises_job_reload_flag(monkeypatch, tmp_path):
    # A whisper update (riding with audio.cpp) that had to unload a warm sidecar reports
    # reload_required on its phase, but the JOB flag stays down: the chat
    # frontend resyncs (and clears the local checkpoint) off the job flag,
    # which must mean "the llama server changed", not "the sidecar restarted".
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path)
    _setup_audio(monkeypatch, [])
    (tmp_path / "install_whisper_prebuilt.py").write_text("stub")

    def _whisper_phase(phase, set_progress):
        return {
            "to_tag": "v1.9.2-unsloth.1",
            "reload_required": True,
            "message": "Updated whisper.cpp to v1.9.2-unsloth.1.",
        }

    monkeypatch.setattr(wupd, "run_chained_phase", _whisper_phase)
    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert job["phases"]["whisper"]["reload_required"] is True
    assert not job["reload_required"]


def test_apply_refuses_when_both_current(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path, installed = "v1.9.2-unsloth.1", latest = "v1.9.2-unsloth.1")
    res = upd.start_update()
    assert res["started"] is False
    assert res["reason"] == "up_to_date"


def test_apply_llama_failure_aborts_before_whisper(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path)
    _setup_whisper(monkeypatch, tmp_path)
    (tmp_path / "install_whisper_prebuilt.py").write_text("stub")

    events = []
    _patch_llama_installer(monkeypatch, returncode = 2, lines = ["boom: disk full\n"])
    _patch_whisper_phase(monkeypatch, events)

    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "error", job
    assert "boom" in (job["error"] or "")
    assert events == []
    assert job["phases"]["llama"]["state"] == "error"
    assert job["phases"]["whisper"]["state"] == "skipped"
    assert job["phases"]["whisper"]["reason"] == "aborted"
    assert job["message"] == "llama.cpp update failed."


def test_apply_whisper_failure_keeps_llama_partial_success(monkeypatch, tmp_path):
    llama_dir = _setup_llama(monkeypatch, tmp_path)
    _setup_whisper(monkeypatch, tmp_path)
    (tmp_path / "install_whisper_prebuilt.py").write_text("stub")

    import threading
    from types import ModuleType

    class _FakeBackend:
        def __init__(self):
            self._serial_load_lock = threading.Lock()
            self._llama_update_in_progress = False
            self.is_active = True

        def unload_model(self):
            self.is_active = False

    backend = _FakeBackend()
    routes_pkg = ModuleType("routes")
    routes_pkg.__path__ = []
    inference_mod = ModuleType("routes.inference")
    inference_mod.get_llama_cpp_backend = lambda: backend
    monkeypatch.setitem(sys.modules, "routes", routes_pkg)
    monkeypatch.setitem(sys.modules, "routes.inference", inference_mod)

    events = []
    _patch_llama_installer(
        monkeypatch,
        on_start = lambda cmd: (events.append("llama"), _write_llama_install(llama_dir, "b9518")),
    )
    _patch_whisper_phase(monkeypatch, events, error = "whisper installer exploded")

    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "error", job
    assert events == ["llama", "whisper"]
    assert "Updated llama.cpp to b9518." in job["message"]
    assert "whisper.cpp update failed." in job["message"]
    assert "whisper installer exploded" in (job["error"] or "")
    assert job["reload_required"] is True
    assert job["to_tag"] == "b9518"
    assert job["phases"]["llama"]["state"] == "success"
    assert job["phases"]["whisper"]["state"] == "error"


def test_apply_skips_whisper_local_link_silently(monkeypatch, tmp_path):
    llama_dir = _setup_llama(monkeypatch, tmp_path)
    _setup_whisper(monkeypatch, tmp_path)
    monkeypatch.setattr(wupd, "_active_install_is_local_link", lambda b: True)

    events = []
    _patch_llama_installer(
        monkeypatch,
        on_start = lambda cmd: (events.append("llama"), _write_llama_install(llama_dir, "b9518")),
    )
    _patch_whisper_phase(monkeypatch, events)

    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert events == ["llama"]
    assert job["phases"]["whisper"]["state"] == "skipped"
    assert job["phases"]["whisper"]["reason"] == "local_link"
    assert job["message"] == "Updated llama.cpp to b9518."


def test_chained_progress_windows(monkeypatch, tmp_path):
    llama_dir = _setup_llama(monkeypatch, tmp_path)
    _setup_whisper(monkeypatch, tmp_path)
    (tmp_path / "install_whisper_prebuilt.py").write_text("stub")

    seen = {}

    def _whisper_phase(phase, set_progress):
        with upd._job_lock:
            seen["at_whisper_start"] = upd._job["progress"]
        set_progress(0.5)
        with upd._job_lock:
            seen["mid_whisper"] = upd._job["progress"]
        return {"to_tag": "v1.9.2-unsloth.1", "reload_required": False, "message": "ok"}

    monkeypatch.setattr(wupd, "run_chained_phase", _whisper_phase)
    _patch_llama_installer(
        monkeypatch,
        lines = ["Downloading app.tar.gz: 100.0% (35.0 MiB/35.0 MiB) at 9.0 MiB/s\n"],
        on_start = lambda cmd: _write_llama_install(llama_dir, "b9518"),
    )

    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert seen["at_whisper_start"] == pytest.approx(0.7)
    assert seen["mid_whisper"] == pytest.approx(0.7 + 0.5 * 0.3)
    assert job["progress"] == 1.0


def _slim_phase(**over):
    phase = {
        "install_dir": "d",
        "repo": "unslothai/whisper.cpp",
        "asset": None,
        "backend": "cpu",
        "script": "s",
        "pin_release_tag": None,
    }
    phase.update(over)
    return phase


def _llama_installed_as(monkeypatch, backend, asset, *, whisper_kind):
    monkeypatch.setattr(wupd, "_installed_llama_bundle", lambda: (backend, asset))
    monkeypatch.setattr(wupd, "_find_binary", lambda: "whisper-server")
    marker = {"backend": "rocm"}
    if whisper_kind is not None:
        marker["install_kind"] = whisper_kind
    monkeypatch.setattr(wupd, "read_install_marker", lambda _b: dict(marker))


def test_a_migrated_llama_re_pairs_the_chained_phase(monkeypatch):
    """The plan named the backend whisper was installed against, and the llama phase has
    since replaced it: a slim install would hardlink a ggml that is no longer there."""
    seen = {}
    monkeypatch.setattr(
        wupd,
        "_resolve_prebuilt_for_host",
        lambda **kw: seen.update(kw) or {"prebuilt_available": True},
    )
    monkeypatch.setattr(wupd, "run_chained_phase", lambda phase, _sp: dict(phase))
    _llama_installed_as(
        monkeypatch, "vulkan", "llama-b1-bin-win-vulkan-x64.zip", whisper_kind = "slim"
    )

    installed = wupd.run_chained_phase_after_llama(_slim_phase(backend = "rocm"), lambda _f: None)
    assert seen["backend"] == "vulkan"
    assert installed["backend"] == "vulkan"
    assert installed["asset"] == "llama-b1-bin-win-vulkan-x64.zip"


def test_a_self_contained_install_keeps_its_own_backend(monkeypatch):
    """The control: only a slim install rides llama's ggml, so only it follows the
    migration. A fat one carries its own and llama's backend is not its backend."""
    monkeypatch.setattr(
        wupd, "_resolve_prebuilt_for_host", lambda **_kw: {"prebuilt_available": True}
    )
    monkeypatch.setattr(wupd, "run_chained_phase", lambda phase, _sp: dict(phase))
    _llama_installed_as(monkeypatch, "vulkan", "vulkan.zip", whisper_kind = "fat")

    installed = wupd.run_chained_phase_after_llama(_slim_phase(backend = "rocm"), lambda _f: None)
    assert installed["backend"] == "rocm"


def test_an_ordinary_update_leaves_the_phase_alone(monkeypatch):
    """No migration, no re-pair: the backend the plan named is the one llama still has."""
    monkeypatch.setattr(
        wupd, "_resolve_prebuilt_for_host", lambda **_kw: {"prebuilt_available": True}
    )
    monkeypatch.setattr(wupd, "run_chained_phase", lambda phase, _sp: dict(phase))
    _llama_installed_as(monkeypatch, "rocm", "rocm-new.zip", whisper_kind = "slim")

    installed = wupd.run_chained_phase_after_llama(
        _slim_phase(backend = "rocm", asset = "rocm-old.zip"), lambda _f: None
    )
    assert (installed["backend"], installed["asset"]) == ("rocm", "rocm-old.zip")


def test_a_slim_pairing_gap_skips_instead_of_failing_the_job(monkeypatch):
    """The state the pipeline was in for ten days: no whisper release paired to the
    llama being installed, so the phase exits 2 and fails a job llama already won."""
    monkeypatch.setattr(
        wupd,
        "_resolve_prebuilt_for_host",
        lambda **_kw: {"prebuilt_available": False, "unavailable_reason": "incompatible"},
    )

    def must_not_install(*_a, **_kw):
        raise AssertionError("the install cannot succeed, so it must not be attempted")

    monkeypatch.setattr(wupd, "run_chained_phase", must_not_install)

    assert wupd.run_chained_phase_after_llama(_slim_phase(), lambda _f: None) == {
        "skipped": True,
        "skip_reason": "paired_llama_unavailable",
    }


def test_the_pre_flight_probes_the_release_that_would_be_installed(monkeypatch):
    """Probe the release that will be installed, since probing another can skip a valid update or
    wave through a doomed one; latest can lag the published_at pin (issue 6219)."""
    seen = {}

    def spy(**kw):
        seen.update(kw)
        return {"prebuilt_available": True}

    monkeypatch.setattr(wupd, "_resolve_prebuilt_for_host", spy)
    monkeypatch.setattr(wupd, "run_chained_phase", lambda phase, _sp: {"to_tag": "t"})

    phase = _slim_phase(repo = "unslothai/whisper.cpp", pin_release_tag = "v1.9.2-unsloth.12")
    wupd.run_chained_phase_after_llama(phase, lambda _f: None)
    assert seen["published_repo"] == "unslothai/whisper.cpp"
    assert seen["published_release_tag"] == "v1.9.2-unsloth.12"


def test_a_workable_pairing_still_installs(monkeypatch):
    monkeypatch.setattr(
        wupd, "_resolve_prebuilt_for_host", lambda **_kw: {"prebuilt_available": True}
    )
    monkeypatch.setattr(
        wupd, "run_chained_phase", lambda phase, _sp: {"to_tag": "v1.9.2-unsloth.12"}
    )

    result = wupd.run_chained_phase_after_llama(_slim_phase(), lambda _f: None)
    assert result == {"to_tag": "v1.9.2-unsloth.12"}


def test_a_legacy_fat_install_is_pre_flighted_too(monkeypatch):
    """The pre-flight checks the target release, since a fat marker omits install_kind by design."""
    monkeypatch.setattr(
        wupd,
        "_resolve_prebuilt_for_host",
        lambda **_kw: {"prebuilt_available": False, "unavailable_reason": "incompatible"},
    )

    def must_not_install(*_a, **_kw):
        raise AssertionError("the install cannot succeed, so it must not be attempted")

    monkeypatch.setattr(wupd, "run_chained_phase", must_not_install)

    # No install_kind at all: the legacy fat marker, read because llama moved backends.
    _llama_installed_as(monkeypatch, "vulkan", "vulkan.zip", whisper_kind = None)
    assert wupd.run_chained_phase_after_llama(_slim_phase(), lambda _f: None) == {
        "skipped": True,
        "skip_reason": "paired_llama_unavailable",
    }


def test_a_fat_target_release_is_never_reported_incompatible(monkeypatch):
    """Why probing unconditionally costs nothing: _slim_release_incompatibility explains
    only a slim pairing failure, so a fat target can never be reported incompatible."""
    monkeypatch.setattr(
        wupd,
        "_resolve_prebuilt_for_host",
        lambda **_kw: {"prebuilt_available": True, "install_kind": "fat"},
    )
    monkeypatch.setattr(wupd, "run_chained_phase", lambda phase, _sp: {"to_tag": "v1"})

    assert wupd.run_chained_phase_after_llama(_slim_phase(), lambda _f: None) == {"to_tag": "v1"}


@pytest.mark.parametrize(
    "resolved",
    [
        # --resolve-prebuilt maps an unreachable API to exit 0 with no prebuilt.
        {"prebuilt_available": False, "unavailable_reason": "unresolved"},
        {"prebuilt_available": False},
        None,
    ],
)
def test_a_pre_flight_that_cannot_answer_still_attempts_the_install(monkeypatch, resolved):
    """A pre-flight that cannot answer must attempt the install, not skip it as a pairing gap."""
    monkeypatch.setattr(wupd, "_resolve_prebuilt_for_host", lambda **_kw: resolved)
    monkeypatch.setattr(wupd, "run_chained_phase", lambda phase, _sp: {"to_tag": "v1"})

    assert wupd.run_chained_phase_after_llama(_slim_phase(), lambda _f: None) == {"to_tag": "v1"}


def test_a_probe_that_raises_still_attempts_the_install(monkeypatch):
    def boom(**_kw):
        raise RuntimeError("resolver blew up")

    monkeypatch.setattr(wupd, "_resolve_prebuilt_for_host", boom)
    monkeypatch.setattr(wupd, "run_chained_phase", lambda phase, _sp: {"to_tag": "v1"})

    assert wupd.run_chained_phase_after_llama(_slim_phase(), lambda _f: None) == {"to_tag": "v1"}


def test_an_incompatible_release_that_slips_past_the_pre_flight_still_errors(monkeypatch):
    """The pre-flight narrows when exit 2 happens; it must never swallow it."""
    monkeypatch.setattr(
        wupd, "_resolve_prebuilt_for_host", lambda **_kw: {"prebuilt_available": True}
    )

    def exit_2(*_a, **_kw):
        raise wupd._flow.InstallerExit(2, "installer exited 2: incompatible release")

    monkeypatch.setattr(wupd, "run_chained_phase", exit_2)

    with pytest.raises(wupd._flow.InstallerExit):
        wupd.run_chained_phase_after_llama(_slim_phase(), lambda _f: None)


# --- audio.cpp: the third phase ---


def test_status_union_audio_only_surfaces_update(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path, installed = "v1.9.2-unsloth.1", latest = "v1.9.2-unsloth.1")
    _setup_audio(monkeypatch, [])
    st = upd.get_update_status(force_refresh = True)
    assert LEGACY_STATUS_FIELDS <= set(st)
    assert st["update_available"] is True
    assert st["llama_update_available"] is False
    assert st["update_component"] == "audio"
    assert st["audio"] == {**_AUDIO_STATUS, "update_available": True, "skip_reason": None}
    # The top-level version fields keep their llama meaning.
    assert st["installed_tag"] == "b9518"


@pytest.mark.parametrize("force_refresh", [True, False])
def test_status_force_refresh_reaches_audio(monkeypatch, tmp_path, force_refresh):
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path, installed = "v1.9.2-unsloth.1", latest = "v1.9.2-unsloth.1")
    plans = _setup_audio(monkeypatch, [])
    upd.get_update_status(force_refresh = force_refresh)
    assert plans == [("audio", {"force_refresh": force_refresh})]


def test_status_llama_keeps_the_card_name_over_audio(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path)
    _setup_audio(monkeypatch, [])
    st = upd.get_update_status(force_refresh = True)
    assert st["update_component"] == "llama"
    assert st["audio"]["update_available"] is True


def test_status_whisper_rides_beside_audio(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path)
    _setup_audio(monkeypatch, [])
    st = upd.get_update_status(force_refresh = True)
    assert st["update_component"] == "audio"
    assert st["whisper"]["update_available"] is True


def test_status_survives_audio_probe_failure(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path)

    def _boom(**_kw):
        raise RuntimeError("probe exploded")

    monkeypatch.setattr(aupd, "chained_phase_plan", _boom)
    st = upd.get_update_status(force_refresh = True)
    assert st["audio"] is None
    assert st["update_available"] is True
    assert st["update_component"] == "llama"


def test_status_skipped_audio_does_not_flip_union(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path, installed = "v1.9.2-unsloth.1", latest = "v1.9.2-unsloth.1")
    st = upd.get_update_status(force_refresh = True)
    assert st["update_available"] is False
    assert st["update_component"] is None
    assert st["audio"]["skip_reason"] == "not_installed"


def test_apply_audio_only_runs_only_the_audio_phase(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_whisper(monkeypatch, tmp_path, installed = "v1.9.2-unsloth.1", latest = "v1.9.2-unsloth.1")
    events = []
    _patch_llama_installer(monkeypatch, on_start = lambda cmd: events.append("llama"))
    _patch_whisper_phase(monkeypatch, events)
    _setup_audio(monkeypatch, events)

    res = upd.start_update()
    assert res["started"] is True, res
    assert "audio.cpp" in res["job"]["message"]
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert events == ["audio"]
    assert job["phases"]["llama"]["state"] == "skipped"
    assert job["phases"]["whisper"]["state"] == "skipped"
    assert job["phases"]["audio"]["state"] == "success"
    assert job["phases"]["audio"]["to_tag"] == "v0.9.0-unsloth.1"
    # The job-level to_tag stays the llama.cpp build, as for a whisper-only round.
    assert job["to_tag"] is None
    assert "Updated audio.cpp to v0.9.0-unsloth.1." in job["message"]
    assert job["progress"] == 1.0


def test_apply_runs_llama_then_whisper_then_audio(monkeypatch, tmp_path):
    llama_dir = _setup_llama(monkeypatch, tmp_path)
    _setup_whisper(monkeypatch, tmp_path)
    (tmp_path / "install_whisper_prebuilt.py").write_text("stub")
    events = []
    _patch_llama_installer(
        monkeypatch,
        on_start = lambda cmd: (events.append("llama"), _write_llama_install(llama_dir, "b9518")),
    )
    _patch_whisper_phase(monkeypatch, events)
    _setup_audio(monkeypatch, events)

    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert events == ["llama", "whisper", "audio"]
    # The legacy to_tag still names the llama build.
    assert job["to_tag"] == "b9518"


def test_apply_llama_failure_aborts_audio(monkeypatch, tmp_path):
    _setup_llama(monkeypatch, tmp_path)
    events = []
    _patch_llama_installer(monkeypatch, returncode = 2, lines = ["boom: disk full\n"])
    _setup_audio(monkeypatch, events)

    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "error", job
    assert events == []
    assert job["phases"]["audio"]["state"] == "skipped"
    assert job["phases"]["audio"]["reason"] == "aborted"


def test_apply_audio_failure_keeps_llama_partial_success(monkeypatch, tmp_path):
    llama_dir = _setup_llama(monkeypatch, tmp_path)
    events = []
    _patch_llama_installer(
        monkeypatch,
        on_start = lambda cmd: (events.append("llama"), _write_llama_install(llama_dir, "b9518")),
    )
    _setup_audio(monkeypatch, events, error = "sha256 mismatch")

    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "error", job
    assert events == ["llama", "audio"]
    assert "Updated llama.cpp to b9518." in job["message"]
    assert "audio.cpp update failed." in job["message"]
    assert "sha256 mismatch" in (job["error"] or "")
    assert job["phases"]["llama"]["state"] == "success"
    assert job["phases"]["audio"]["state"] == "error"


def test_audio_unload_raises_the_job_reload_flag(monkeypatch, tmp_path):
    # Unlike the dictation sidecars, an audio.cpp model in the main slot is a server model change.
    _setup_llama(monkeypatch, tmp_path, installed = "b9518", latest = "b9518")
    _setup_audio(monkeypatch, [], reload_required = True)
    assert upd.start_update()["started"] is True
    job = _wait_for_job()
    assert job["state"] == "success", job
    assert job["reload_required"] is True


def test_backend_switch_never_plans_audio(monkeypatch, tmp_path):
    plans = _setup_audio(monkeypatch, [])
    monkeypatch.setattr(
        upd,
        "_plan_llama_phase",
        lambda backend_request = None: {
            "skip_reason": "already_selected",
            "refusal": {
                "started": False,
                "reason": "already_selected",
                "message": "llama.cpp is already set to cpu.",
            },
        },
    )
    monkeypatch.setattr(upd, "_repair_pairing_plan_or_empty", lambda *a, **k: {})
    monkeypatch.setattr(upd, "_env_backend_override", lambda: None)
    res = upd.start_backend_switch("cpu")
    assert res["started"] is False
    assert res["reason"] == "already_selected"
    assert plans == []


def test_update_checks_disabled_refuses_before_planning_audio(monkeypatch, tmp_path):
    plans = _setup_audio(monkeypatch, [])
    monkeypatch.setenv("UNSLOTH_DISABLE_UPDATE_CHECK", "1")
    res = upd.start_update()
    assert res["started"] is False
    assert res["reason"] == "update_checks_disabled"
    assert plans == []