Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

103 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

"""Imports by path: kaggle_t4_ci and kaggle_studio_ci both ship report.py and build_kernel.py."""

from __future__ import annotations

import base64
import importlib.util
import io
import json
import os
import re
import subprocess
import sys
import tarfile
from pathlib import Path
from unittest import mock

import pytest

REPO_ROOT = Path(__file__).resolve().parents[2]
PAYLOAD_DIR = REPO_ROOT / "tests" / "kaggle" / "studio_gpu"
CI_DIR = REPO_ROOT / ".github" / "scripts" / "kaggle_studio_ci"
WORKFLOW = REPO_ROOT / ".github" / "workflows" / "kaggle-t4-studio-gpu-ci.yml"

sys.path.insert(0, str(PAYLOAD_DIR))

import gpu_assert  # noqa: E402
import studio_client  # noqa: E402


def _load(name: str, path: Path):
    spec = importlib.util.spec_from_file_location(name, path)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


run_studio_gpu = _load("studio_gpu_payload_main", PAYLOAD_DIR / "run_studio_gpu.py")
build_kernel = _load("studio_ci_build_kernel", CI_DIR / "build_kernel.py")
collect_evidence = _load("studio_ci_collect_evidence", CI_DIR / "collect_evidence.py")


@pytest.fixture(autouse = True)
def _no_caller_cuda_visible_devices(monkeypatch):
    # Samplers scope nvidia-smi by it; an empty value reads as no card visible.
    monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)


def test_compute_apps_parses_the_bare_csv_the_payload_asks_for():
    apps = gpu_assert.parse_compute_apps("1234, 2048\n5678, 96\n")
    assert apps == {1234: 2048, 5678: 96}


def test_compute_apps_parses_the_same_output_with_units_left_on():
    """Whether nounits is honoured varies by driver, and guessing wrong
    silently yields an empty dict, which reads as 'nothing on the GPU'."""
    assert gpu_assert.parse_compute_apps("1234, 2048 MiB\n") == {1234: 2048}


def test_compute_apps_skips_a_header_and_an_unreported_size():
    text = "pid, used_gpu_memory\n1234, [N/A]\n5678, 512\n"
    assert gpu_assert.parse_compute_apps(text) == {5678: 512}


def test_no_compute_apps_at_all_is_an_empty_dict_not_a_crash():
    assert gpu_assert.parse_compute_apps("") == {}


def test_listed_pids_are_counted_even_when_no_memory_is_attributed():
    """Windows (WDDM) and unified-memory parts list every CUDA process with [N/A].
    The GB10 lists a -ngl 0 server too, so this is neither "nothing on the card" nor
    proof of offload: it is the signal to fall back to the device delta."""
    text = "pid, used_gpu_memory\n1234, [N/A]\n5678, [N/A]\n"
    assert gpu_assert.parse_compute_apps(text) == {}
    assert gpu_assert.count_listed_pids(text) == 2


def test_listed_pids_ignore_the_header_and_junk_rows():
    assert gpu_assert.count_listed_pids("pid, used_gpu_memory\n[N/A], [N/A]\n\n") == 0
    assert gpu_assert.count_listed_pids("") == 0
    assert gpu_assert.count_listed_pids("1234, 2048\n5678, [N/A]\n") == 2


def test_an_all_unattributed_listing_reads_as_cannot_enumerate(monkeypatch):
    import run_studio_gpu  # the payload, on sys.path beside gpu_assert

    """So cli_run_gpu_failure takes its device-delta branch rather than concluding that
    no process appeared. Readable rows still come back as the usual dict."""

    class _P:
        def __init__(self, out):
            self.returncode = 0
            self.stdout = out

    monkeypatch.setattr(run_studio_gpu, "run", lambda *a, **k: _P("4242, [N/A]\n4243, [N/A]\n"))
    assert run_studio_gpu.nvidia_compute_apps() is None
    monkeypatch.setattr(run_studio_gpu, "run", lambda *a, **k: _P("4242, 512\n4243, [N/A]\n"))
    assert run_studio_gpu.nvidia_compute_apps() == {4242: 512}
    monkeypatch.setattr(run_studio_gpu, "run", lambda *a, **k: _P(""))
    assert run_studio_gpu.nvidia_compute_apps() == {}


def test_the_offload_line_is_read_from_the_most_recent_load():
    """A session loads more than once: the chat model, then the export."""
    log = (
        "load_tensors: offloaded 0/29 layers to GPU\n"
        "...\n"
        "load_tensors: offloaded 25/25 layers to GPU\n"
    )
    assert gpu_assert.offloaded_layers(log) == (25, 25)


def test_a_log_with_no_offload_line_reports_nothing_rather_than_zero():
    """None and (0, N) mean opposite things and must not be conflated."""
    assert gpu_assert.offloaded_layers("llama_model_loader: loaded meta data") is None


def test_the_device_model_buffer_is_read_in_mib():
    log = "load_tensors:        CUDA0 model buffer size =   1918.35 MiB\n"
    assert gpu_assert.cuda_buffer_mib(log) == pytest.approx(1918.35)


def test_a_cpu_only_log_reports_no_device_buffer():
    log = "load_tensors:   CPU model buffer size =   1918.35 MiB\n"
    assert gpu_assert.cuda_buffer_mib(log) is None


def test_a_cuda_bundle_is_recognised_from_its_marker(tmp_path):
    """The installer's marker has no install_kind key, so the fixture must use the real marker shape."""
    marker = tmp_path / "UNSLOTH_PREBUILT_INFO.json"
    marker.write_text(
        json.dumps(
            {
                "tag": "b10360",
                "asset": "app-b10360-mix-87da1a2-linux-x64-cuda13-older.tar.gz",
                "runtime_line": "cuda13",
                "coverage_class": "older",
            }
        )
    )
    assert gpu_assert.install_kind(marker) == "cuda13"
    assert gpu_assert.is_cuda_install(gpu_assert.install_kind(marker))


@pytest.mark.parametrize("kind", ["linux-cpu", "linux-vulkan", "linux-rocm", None, ""])
def test_everything_that_is_not_a_cuda_bundle_is_rejected(kind):
    """A CPU bundle on a box with a working NVIDIA driver is the regression
    this leg exists to see, so it must not read as 'close enough'."""
    assert not gpu_assert.is_cuda_install(kind)


def test_a_missing_or_unreadable_marker_is_not_a_cuda_install(tmp_path):
    assert gpu_assert.install_kind(tmp_path / "absent.json") is None
    broken = tmp_path / "UNSLOTH_PREBUILT_INFO.json"
    broken.write_text("{not json")
    assert gpu_assert.install_kind(broken) is None


def test_a_real_gguf_header_passes_and_a_truncated_one_does_not(tmp_path):
    good = tmp_path / "a.gguf"
    good.write_bytes(b"GGUF\x03\x00\x00\x00")
    bad = tmp_path / "b.gguf"
    bad.write_bytes(b"\x00\x00")
    assert gpu_assert.gguf_magic_ok(good)
    assert not gpu_assert.gguf_magic_ok(bad)
    assert not gpu_assert.gguf_magic_ok(tmp_path / "absent.gguf")


def _verdict(**kw):
    base = {
        "server_pid": None,
        "compute_apps": None,
        "log_text": "",
        "device_vram_delta_mib": None,
        "status": None,
    }
    base.update(kw)
    return gpu_assert.offload_verdict(**base)


def test_no_evidence_at_all_is_a_failure_and_not_a_pass():
    """No evidence at all must fail, not pass: a CPU fallback returns text just like the GPU path,
    so an unanswerable question was never answered in the affirmative."""
    verdict = _verdict()
    assert not verdict["passed"]
    assert any("no probe could show" in f for f in verdict["failures"])


def test_a_matching_process_holding_vram_is_enough_on_its_own():
    verdict = _verdict(server_pid = 42, compute_apps = {42: 2048})
    assert verdict["passed"]
    assert verdict["positives"]


def test_a_process_holding_only_a_cuda_context_is_not_enough():
    """Tens of MiB is what a bare context costs; the weights are elsewhere."""
    verdict = _verdict(server_pid = 42, compute_apps = {42: 30})
    assert not verdict["passed"]


def test_the_llama_cpp_offload_line_alone_is_enough():
    verdict = _verdict(log_text = "load_tensors: offloaded 25/25 layers to GPU")
    assert verdict["passed"]


def test_device_vram_growth_alone_is_enough():
    """The fallback probe for a container where nvidia-smi cannot see the
    process and llama.cpp's log did not reach us."""
    assert _verdict(device_vram_delta_mib = 1500.0)["passed"]


def test_a_small_vram_wobble_is_not_evidence():
    assert not _verdict(device_vram_delta_mib = 20.0)["passed"]


def test_zero_offloaded_layers_fails_however_much_else_looks_right():
    verdict = _verdict(
        server_pid = 42,
        compute_apps = {42: 4096},
        log_text = "load_tensors: offloaded 0/25 layers to GPU",
        device_vram_delta_mib = 4096.0,
    )
    assert not verdict["passed"]
    assert any("0/25" in f for f in verdict["failures"])


def test_a_declared_cpu_fallback_fails_however_much_else_looks_right():
    verdict = _verdict(
        server_pid = 42,
        compute_apps = {42: 4096},
        status = {"cpu_fallback_reason": "vulkan_startup_crash"},
    )
    assert not verdict["passed"]


def test_studio_reporting_zero_gpu_layers_fails():
    verdict = _verdict(server_pid = 42, compute_apps = {42: 4096}, status = {"gpu_layers": 0})
    assert not verdict["passed"]


def test_a_pid_that_is_not_among_the_gpu_processes_is_not_a_pass():
    """Something else on the box holding VRAM must not launder a CPU load."""
    verdict = _verdict(server_pid = 42, compute_apps = {99: 8000})
    assert not verdict["passed"]


def test_auto_mode_gpu_layers_of_minus_one_is_not_treated_as_zero():
    verdict = _verdict(server_pid = 42, compute_apps = {42: 4096}, status = {"gpu_layers": -1})
    assert verdict["passed"]


def test_health_is_not_ready_until_hardware_detection_settles():
    """Unsloth answers healthy while still detecting, and refuses to start a
    training run or an export in that window."""
    assert not studio_client.health_is_ready({"status": "healthy", "hardware_detecting": True})
    assert studio_client.health_is_ready({"status": "healthy"})
    # no startup hardware warmup: detection waits for the authenticated request, so the server is up
    deferred = {
        "status": "healthy",
        "hardware_detecting": True,
        "hardware_detection_deferred": True,
    }
    assert studio_client.health_is_ready(deferred)
    assert not studio_client.hardware_detected(deferred)
    assert studio_client.hardware_detected({"status": "healthy"})
    assert not studio_client.health_is_ready({"status": "starting"})
    assert not studio_client.health_is_ready("connection refused")


def test_wait_returns_as_soon_as_the_predicate_holds():
    values = iter([1, 2, 3])
    ok, last, reason = wait_helper(values, lambda v: v == 2)
    assert (ok, last, reason) == (True, 2, "")


def wait_helper(values, accept, **kw):
    clock = {"t": 0.0}

    def _now():
        return clock["t"]

    def _sleep(seconds):
        clock["t"] += seconds

    return studio_client.wait_for(
        probe = lambda: next(values),
        accept = accept,
        deadline_s = kw.pop("deadline_s", 100.0),
        interval_s = 1.0,
        now = _now,
        sleep = _sleep,
        **kw,
    )


def test_wait_times_out_rather_than_looping_forever():
    ok, _, reason = wait_helper(iter(range(1000)), lambda v: False, deadline_s = 5.0)
    assert not ok
    assert "timed out" in reason


def test_wait_gives_up_immediately_when_the_process_it_waits_on_is_dead():
    """Otherwise every crash at startup costs the full deadline and reports
    itself as slowness rather than as a crash."""
    ok, _, reason = wait_helper(iter(range(10)), lambda v: False, alive = lambda: False)
    assert not ok
    assert "exited" in reason


def test_a_probe_that_raises_is_retried_rather_than_fatal():
    """Polling a server that is not listening yet is the normal first case."""
    attempts = {"n": 0}

    def _probe():
        attempts["n"] += 1
        if attempts["n"] < 3:
            raise ConnectionRefusedError("not up yet")
        return "ready"

    clock = {"t": 0.0}
    ok, last, _ = studio_client.wait_for(
        probe = _probe,
        accept = lambda v: v == "ready",
        deadline_s = 60.0,
        interval_s = 1.0,
        now = lambda: clock["t"],
        sleep = lambda s: clock.__setitem__("t", clock["t"] + s),
    )
    assert ok and last == "ready" and attempts["n"] == 3


def test_a_running_job_is_not_terminal():
    for phase in (
        "idle",
        "loading_model",
        "loading_dataset",
        "configuring",
        "training",
        "finalizing",
    ):
        assert studio_client.training_verdict({"phase": phase}) == (False, "")


def test_a_completed_job_is_terminal_and_clean():
    assert studio_client.training_verdict({"phase": "completed"}) == (True, "")


@pytest.mark.parametrize("phase", ["error", "stopped"])
def test_a_failed_job_stops_the_poll_and_names_the_cause(phase):
    """Waiting out a twenty-minute deadline on a job that already died turns
    a legible failure into a timeout."""
    terminal, reason = studio_client.training_verdict({"phase": phase, "error": "boom"})
    assert terminal
    assert phase in reason and "boom" in reason


def test_steps_with_a_logged_loss_are_counted_and_nulls_are_not():
    status = {"metric_history": {"loss": [1.0, 0.9, None, 0.8]}}
    assert studio_client.trained_steps(status) == 3
    assert studio_client.trained_steps({}) == 0
    assert studio_client.trained_steps({"metric_history": {"loss": "nope"}}) == 0


def test_an_export_that_has_not_started_is_not_read_as_finished():
    """is_export_active is false before the job starts as well as after it
    ends, and there is no job id, so the sequence number is the only thing
    that distinguishes them."""
    status = {"last_op_seq": 7, "is_export_active": False, "last_op_status": "success"}
    assert studio_client.export_verdict(status, baseline_seq = 7) == (False, "")


def test_an_export_still_running_is_not_finished():
    status = {"last_op_seq": 8, "is_export_active": True, "last_op_status": "success"}
    assert studio_client.export_verdict(status, baseline_seq = 7) == (False, "")


def test_a_new_successful_export_is_finished_and_clean():
    status = {"last_op_seq": 8, "is_export_active": False, "last_op_status": "success"}
    assert studio_client.export_verdict(status, baseline_seq = 7) == (True, "")


@pytest.mark.parametrize("outcome", ["error", "cancelled"])
def test_a_new_failed_export_is_finished_and_names_the_cause(outcome):
    status = {
        "last_op_seq": 8,
        "is_export_active": False,
        "last_op_status": outcome,
        "last_op_error": "conversion wrote no .gguf",
    }
    done, reason = studio_client.export_verdict(status, baseline_seq = 7)
    assert done and outcome in reason and "no .gguf" in reason


def test_the_newest_gguf_is_found_recursively(tmp_path):
    import os
    import time

    old = tmp_path / "sub" / "old.gguf"
    old.parent.mkdir()
    old.write_bytes(b"GGUF")
    new = tmp_path / "new.gguf"
    new.write_bytes(b"GGUF")
    os.utime(old, (time.time() - 600, time.time() - 600))
    assert studio_client.newest_gguf(tmp_path) == new
    assert studio_client.newest_gguf(tmp_path / "absent") is None


def _adapter(
    tmp_path,
    *,
    config = True,
    weights = 100_000,
):
    root = tmp_path / "run"
    root.mkdir()
    if config:
        (root / "adapter_config.json").write_text("{}")
    if weights is not None:
        (root / "adapter_model.safetensors").write_bytes(b"x" * weights)
    return root


def test_a_real_adapter_directory_passes(tmp_path):
    ok, failures, detail = studio_client.adapter_verdict(_adapter(tmp_path))
    assert ok and not failures
    assert detail["adapter_weights"] == "adapter_model.safetensors"


def test_a_run_that_saved_nothing_fails_even_though_it_said_completed(tmp_path):
    ok, failures, _ = studio_client.adapter_verdict(_adapter(tmp_path, weights = None))
    assert not ok
    assert any("adapter weights" in f for f in failures)


def test_a_stub_sized_adapter_fails(tmp_path):
    ok, failures, _ = studio_client.adapter_verdict(_adapter(tmp_path, weights = 12))
    assert not ok
    assert any("byte floor" in f for f in failures)


def test_a_missing_config_fails(tmp_path):
    ok, failures, _ = studio_client.adapter_verdict(_adapter(tmp_path, config = False))
    assert not ok
    assert any("adapter_config.json" in f for f in failures)


def test_no_output_dir_at_all_fails_rather_than_passing_vacuously():
    ok, failures, _ = studio_client.adapter_verdict(None)
    assert not ok and failures


def test_an_output_dir_that_does_not_exist_fails(tmp_path):
    ok, failures, _ = studio_client.adapter_verdict(tmp_path / "never-created")
    assert not ok and failures


def _build(tmp_path, **kw):
    out = tmp_path / "kernel.ipynb"
    args = [
        sys.executable,
        str(CI_DIR / "build_kernel.py"),
        "--payload-dir",
        str(PAYLOAD_DIR),
        "--out",
        str(out),
        "--unsloth-ref",
        kw.pop("ref", "deadbeef"),
    ]
    for key, value in kw.items():
        args += [f"--{key.replace('_', '-')}", str(value)]
    proc = subprocess.run(args, capture_output = True, text = True)
    assert proc.returncode == 0, proc.stderr
    return json.loads(out.read_text())


def _payload_notebook(driver: dict) -> dict:
    import gzip
    import re

    source = "".join(driver["cells"][0]["source"])
    blob = re.search(r'PAYLOAD = "([^"]+)"', source).group(1)
    return json.loads(gzip.decompress(base64.b64decode(blob)).decode("utf-8"))


def test_the_built_kernel_is_valid_notebook_json_asking_for_a_gpu(tmp_path):
    driver = _build(tmp_path)
    assert driver["nbformat"] == 4
    assert driver["metadata"]["accelerator"] == "GPU"
    assert _payload_notebook(driver)["metadata"]["accelerator"] == "GPU"


def test_every_generated_cell_compiles(tmp_path):
    """The notebook leg lost a whole GPU session to a generated cell that had
    a syntax error, before a single assertion ran."""
    driver = _build(tmp_path)
    for index, cell in enumerate(driver["cells"]):
        compile("".join(cell["source"]), f"driver{index}", "exec")
    for index, cell in enumerate(_payload_notebook(driver)["cells"]):
        compile("".join(cell["source"]), f"payload{index}", "exec")


def test_no_generated_cell_reads_a_name_nothing_defines(tmp_path):
    """Cells share one namespace in execution order, so a name used in cell 4
    has to have been bound by cell 0 through 4 or by a builtin."""
    import builtins
    for cells in (_build(tmp_path)["cells"], _payload_notebook(_build(tmp_path))["cells"]):
        defined = set(dir(builtins))
        for index, cell in enumerate(cells):
            source = "".join(cell["source"])
            tree = compile(source, f"cell{index}", "exec", flags = 0x400, dont_inherit = True)
            del tree
            import ast

            parsed = ast.parse(source)
            loaded, stored = set(), set()
            for node in ast.walk(parsed):
                if isinstance(node, ast.Name):
                    (loaded if isinstance(node.ctx, ast.Load) else stored).add(node.id)
                elif isinstance(node, ast.alias):
                    stored.add((node.asname or node.name).split(".")[0])
                elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
                    stored.add(node.name)
                elif isinstance(node, ast.arg):
                    stored.add(node.arg)
                elif isinstance(node, ast.ExceptHandler) and node.name:
                    stored.add(node.name)
                elif isinstance(node, (ast.comprehension,)):
                    pass
            defined |= stored
            missing = sorted(loaded - defined)
            assert not missing, f"cell {index} reads undefined {missing}"


def test_the_ref_under_test_is_pinned_into_the_clone(tmp_path):
    source = "\n".join(
        "".join(c["source"]) for c in _payload_notebook(_build(tmp_path, ref = "c0ffee1"))["cells"]
    )
    assert "c0ffee1" in source
    assert "FETCH_HEAD" in source


def test_the_payload_never_puts_its_work_under_kaggle_working(tmp_path):
    """/kaggle/working is 19.5 GB and is also what `kernels output` ships
    home. A torch install landing there fills the disk mid-run."""
    cells = _payload_notebook(_build(tmp_path))["cells"]
    setup = "".join(cells[0]["source"])
    assert 'pathlib.Path.home() / "unsloth_studio_ci"' in setup
    assert "/tmp/unsloth_studio_ci" in setup
    for index, cell in enumerate(cells):
        source = "".join(cell["source"])
        for line in source.splitlines():
            if line.lstrip().startswith("#"):
                continue
            if "/kaggle/working" in line:
                assert "studio_gpu_out" in line, f"cell {index}: {line}"


def _payload_source(driver: dict) -> str:
    return "\n".join("".join(c["source"]) for c in _payload_notebook(driver)["cells"])


def test_payload_args_reach_the_payload(tmp_path):
    source = _payload_source(_build(tmp_path, payload_args = "--max-steps 3"))
    assert '["--max-steps", "3"]' in source


def test_the_builder_refuses_a_payload_directory_that_lost_a_file(tmp_path):
    """A rename that breaks the kernel should fail on the runner in seconds,
    not forty minutes into a GPU session."""
    stub = tmp_path / "payload"
    stub.mkdir()
    (stub / "run_studio_gpu.py").write_text("")
    proc = subprocess.run(
        [
            sys.executable,
            str(CI_DIR / "build_kernel.py"),
            "--payload-dir",
            str(stub),
            "--out",
            str(tmp_path / "k.ipynb"),
        ],
        capture_output = True,
        text = True,
    )
    assert proc.returncode != 0
    assert "missing" in proc.stderr


def test_every_file_the_builder_requires_is_actually_in_the_payload_dir():
    for name in build_kernel.PAYLOAD_FILES:
        assert (PAYLOAD_DIR / name).is_file(), name


def test_the_result_prefix_matches_the_shared_launcher():
    """The launcher is reused unchanged; it scrapes this exact prefix."""
    launcher = (REPO_ROOT / ".github" / "scripts" / "kaggle_t4_ci" / "launch.py").read_text(
        encoding = "utf-8"
    )
    assert f'RESULT_PREFIX = "{build_kernel.RESULT_PREFIX}"' in launcher
    payload = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")
    assert f'RESULT_PREFIX = "{build_kernel.RESULT_PREFIX}"' in payload


def _bundle(names: dict[str, bytes]) -> bytes:
    buf = io.BytesIO()
    with tarfile.open(fileobj = buf, mode = "w:gz") as tar:
        for name, data in names.items():
            info = tarfile.TarInfo(name)
            info.size = len(data)
            tar.addfile(info, io.BytesIO(data))
    return buf.getvalue()


def _chunk_lines(blob: bytes, size: int = 64) -> list[str]:
    encoded = base64.b64encode(blob).decode("ascii")
    chunks = [encoded[i : i + size] for i in range(0, len(encoded), size)]
    return [
        f"{collect_evidence.EVIDENCE_PREFIX}{i + 1}/{len(chunks)} {c}" for i, c in enumerate(chunks)
    ]


def test_a_bundle_survives_the_round_trip_through_stdout(tmp_path):
    blob = _bundle(
        {"studio_gpu_report.json": b'{"passed": true}', "playwright/01-chat.png": b"\x89PNG"}
    )
    log = tmp_path / "kernel.log"
    log.write_text("noise\n" + "\n".join(_chunk_lines(blob)) + "\nmore noise\n")
    outdir = tmp_path / "out"
    proc = subprocess.run(
        [
            sys.executable,
            str(CI_DIR / "collect_evidence.py"),
            "--evidence",
            str(log),
            "--outdir",
            str(outdir),
        ],
        capture_output = True,
        text = True,
    )
    assert proc.returncode == 0, proc.stderr
    assert (outdir / "studio_gpu_report.json").read_bytes() == b'{"passed": true}'
    assert (outdir / "playwright" / "01-chat.png").is_file()


def test_a_truncated_log_is_refused_rather_than_half_unpacked(tmp_path):
    """A bundle reassembled from a truncated log decodes to something, and
    that something is not the evidence."""
    blob = _bundle({"a.txt": b"x" * 4000})
    lines = _chunk_lines(blob)
    log = tmp_path / "kernel.log"
    log.write_text("\n".join(lines[:-1]))
    outdir = tmp_path / "out"
    proc = subprocess.run(
        [
            sys.executable,
            str(CI_DIR / "collect_evidence.py"),
            "--evidence",
            str(log),
            "--outdir",
            str(outdir),
        ],
        capture_output = True,
        text = True,
    )
    assert proc.returncode == 0
    assert "incomplete" in proc.stdout
    assert not outdir.exists() or not list(outdir.iterdir())


def test_a_run_that_emitted_no_bundle_is_not_an_error(tmp_path):
    log = tmp_path / "kernel.log"
    log.write_text("nothing to see")
    proc = subprocess.run(
        [
            sys.executable,
            str(CI_DIR / "collect_evidence.py"),
            "--evidence",
            str(log),
            "--outdir",
            str(tmp_path / "out"),
        ],
        capture_output = True,
        text = True,
    )
    assert proc.returncode == 0
    assert "no evidence bundle" in proc.stdout


@pytest.mark.parametrize("name", ["/etc/passwd", "../../escape.txt"])
def test_a_bundle_cannot_write_outside_the_output_directory(name):
    assert not collect_evidence.is_safe_member(name)
    assert collect_evidence.is_safe_member("playwright/01.png")


def test_chunks_are_reassembled_in_order_regardless_of_the_order_seen():
    chunks, total = collect_evidence.collect_chunks(
        [
            f"{collect_evidence.EVIDENCE_PREFIX}2/2 QQ",
            f"{collect_evidence.EVIDENCE_PREFIX}1/2 PP",
        ]
    )
    assert total == 2
    assert "".join(chunks[i] for i in (1, 2)) == "PPQQ"


def test_a_truncated_duplicate_does_not_overwrite_the_complete_chunk(tmp_path):
    """A log copy of a chunk can be cut short; it must not overwrite the complete notebook copy."""
    blob = _bundle({"studio_gpu_report.json": b'{"passed": true}', "shot.png": b"\x89PNG"})
    lines = _chunk_lines(blob)
    assert len(lines) > 2, "need a chunk that is neither first nor last"

    evidence = tmp_path / "kaggle_evidence" / "unsloth-t4-ci-deadbeef"
    evidence.mkdir(parents = True)
    (evidence / "studio_gpu_output.ipynb").write_text(
        json.dumps({"cells": [{"outputs": [{"text": "\n".join(lines)}]}]}), encoding = "utf-8"
    )
    (evidence / "kernel.log").write_text("\n".join(lines[:-2] + [lines[-2][:-9]]), encoding = "utf-8")

    outdir = tmp_path / "studio_evidence"
    proc = subprocess.run(
        [
            sys.executable,
            str(CI_DIR / "collect_evidence.py"),
            "--evidence",
            str(tmp_path / "kaggle_evidence"),
            "--outdir",
            str(outdir),
        ],
        capture_output = True,
        text = True,
    )
    assert proc.returncode == 0, proc.stderr
    assert (outdir / "studio_gpu_report.json").read_bytes() == b'{"passed": true}'
    assert (outdir / "shot.png").read_bytes() == b"\x89PNG"


def test_a_complete_later_copy_repairs_a_truncated_earlier_one():
    """The cut can land in either copy, so a later sighting that EXTENDS what
    is held is taken."""
    prefix = collect_evidence.EVIDENCE_PREFIX
    chunks, total = collect_evidence.collect_chunks(
        [f"{prefix}1/2 PP\n{prefix}2/2 QQ", f"{prefix}1/2 PPPP\n{prefix}2/2 QQ"]
    )
    assert total == 2
    assert "".join(chunks[i] for i in (1, 2)) == "PPPPQQ"


def _workflow() -> dict:
    yaml = pytest.importorskip("yaml")
    return yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8"))


def _triggers(wf: dict) -> dict:
    """The `on:` block. YAML 1.1 reads a bare `on` as the boolean True."""
    return wf.get("on", wf.get(True))


def test_the_workflow_parses_and_gates_the_expensive_job_on_the_cheap_one():
    jobs = _workflow()["jobs"]
    assert jobs["studio-gpu"]["needs"] == "gate"
    assert "needs.gate.outputs.should_run == 'true'" in jobs["studio-gpu"]["if"]
    assert "fork != true" in jobs["gate"]["if"]
    for job in jobs.values():
        assert job["timeout-minutes"] >= 1


def test_the_workflow_never_cancels_a_run_that_may_hold_a_kernel():
    wf = _workflow()
    assert wf["concurrency"]["cancel-in-progress"] is False
    assert wf["jobs"]["studio-gpu"]["concurrency"]["cancel-in-progress"] is False


def test_the_two_kaggle_legs_fit_the_account_side_by_side():
    """Pushes from both legs must sum within the 2-kernel per-account cap, not depend on a shared group"""
    yaml = pytest.importorskip("yaml")
    # Read gate.py as text: both kaggle CI dirs ship a `report` module, and importing either would
    # decide `import report` for every later test in the process.
    gate_src = (REPO_ROOT / ".github" / "scripts" / "kaggle_t4_ci" / "gate.py").read_text(
        encoding = "utf-8"
    )
    caps = re.findall(r"^MAX_CONCURRENT_GPU_KERNELS = (\d+)$", gate_src, re.MULTILINE)
    assert len(caps) == 1, caps
    MAX_CONCURRENT_GPU_KERNELS = int(caps[0])

    notebook_text = (REPO_ROOT / ".github" / "workflows" / "kaggle-t4-notebook-ci.yml").read_text(
        encoding = "utf-8"
    )
    notebook = yaml.safe_load(notebook_text)
    studio_group = _workflow()["jobs"]["studio-gpu"]["concurrency"]["group"]
    notebook_group = notebook["jobs"]["t4-smoke"]["concurrency"]["group"]

    assert studio_group != notebook_group, (
        "the two legs share a concurrency group again, so Unsloth waits out the "
        "whole notebook job for a Kaggle session that is free"
    )
    # Not keyed on the ref: one account, so branches of the same workflow must not overlap.
    for group in (studio_group, notebook_group):
        assert "github.ref" not in group, group

    pushes = {}
    for label, text in (
        ("studio", WORKFLOW.read_text(encoding = "utf-8")),
        ("notebook", notebook_text),
    ):
        found = {int(k) for k in re.findall(r"--kernels (\d+)", text)}
        assert len(found) == 1, (label, found)
        pushes[label] = found.pop()
    assert sum(pushes.values()) <= MAX_CONCURRENT_GPU_KERNELS, pushes


def test_the_workflow_is_never_preempted_by_the_capacity_sweeper():
    """Cancelling it orphans a Kaggle kernel that then bills to its ceiling."""
    preempt = json.loads((REPO_ROOT / ".github" / "ci-preempt.json").read_text(encoding = "utf-8"))
    assert WORKFLOW.name in preempt["never"]
    for machines in preempt["heavy"].values():
        assert WORKFLOW.name not in machines


def test_the_paths_filter_covers_everything_the_payload_depends_on():
    """A change to the payload, the builder or the shared launcher has to be
    able to trigger the job that exercises it."""
    triggers = _triggers(_workflow())
    for event in ("pull_request", "push"):
        paths = triggers[event]["paths"]
        assert "tests/kaggle/studio_gpu/**" in paths
        assert ".github/scripts/kaggle_studio_ci/**" in paths
        assert ".github/scripts/kaggle_t4_ci/**" in paths
        assert "tests/studio/playwright_chat_ui.py" in paths
        assert WORKFLOW.name in " ".join(paths)


def test_the_paths_filter_is_not_the_whole_of_studio():
    """studio/** would roughly double the eligible stream for coverage this
    job cannot see, and the budget arithmetic in the header depends on it."""
    triggers = _triggers(_workflow())
    for event in ("pull_request", "push"):
        assert "studio/**" not in triggers[event]["paths"]


def test_the_sampling_rate_matches_the_arithmetic_in_the_header():
    """The header states 5%, ~38 launches and ~28 GPU-h a week. If the flag
    and the prose disagree, one of them is a lie to whoever reads it next."""
    source = WORKFLOW.read_text(encoding = "utf-8")
    assert "--percent 5" in source
    assert "x sampling rate            0.05" in source
    assert "= launches                 ~38 / week" in source
    assert "= EXPECTED SPEND           ~28 GPU-h / week" in source


def test_studio_is_sampled_harder_than_the_notebook_leg():
    """Studio is sampled harder on launches, hours and budget share, but lower on percentage, by design."""
    studio = WORKFLOW.read_text(encoding = "utf-8")
    notebook = (REPO_ROOT / ".github" / "workflows" / "kaggle-t4-notebook-ci.yml").read_text(
        encoding = "utf-8"
    )

    assert "--percent 5" in studio and "--percent 15" in notebook
    assert "= EXPECTED SPEND           ~28 GPU-h / week" in studio
    assert "busy week   231 x 0.15 x 0.25 =   8.7 GPU-h" in notebook
    assert (
        "0.75 GPU-h" in studio and "TOTAL, expected                             ~0.25 h" in notebook
    )

    assert "The split is Unsloth 35, this leg 15" in notebook
    assert "--reserve-hours 20" in notebook and "--reserve-hours 10" in studio

    # The Unsloth block restates the notebook reserve in prose; assert the sentence still agrees.
    assert "reserve-hours is 10 rather than the notebook leg's 20" in studio

    assert "kaggle-t4-studio-gpu-ci.yml" in notebook
    assert "kaggle-t4-notebook-ci.yml" in studio


def test_the_two_legs_together_fit_inside_the_ci_allowance():
    """Re-derived from the two headers rather than trusting either total."""
    import re

    studio = WORKFLOW.read_text(encoding = "utf-8")
    notebook = (REPO_ROOT / ".github" / "workflows" / "kaggle-t4-notebook-ci.yml").read_text(
        encoding = "utf-8"
    )

    def rate(text):
        # The invoked flag, not the prose that quotes it.
        found = re.findall(r"^\s+--percent (\d+) \\$", text, re.M)
        assert len(found) == 1, found
        return int(found[0]) / 100.0

    studio_spend = 760 * rate(studio) * 0.75
    notebook_spend = 231 * rate(notebook) * 0.25  # busy week, the pessimistic end
    assert studio_spend > notebook_spend
    assert studio_spend + notebook_spend <= 50.0
    # With margin: the quota read only sees spend after the hours are gone.
    assert studio_spend + notebook_spend <= 40.0


def test_the_reserve_leaves_ci_the_fifty_hours_it_is_allowed():
    source = WORKFLOW.read_text(encoding = "utf-8")
    assert "--reserve-hours 10" in source
    assert "--budget-hours 4" in source
    assert "--budget-hours 2" not in source


def test_the_budget_hours_flag_covers_the_reaper_window():
    """budget-hours is the worst case one invocation can cost and must cover the collector's reaper
    window, since nothing waits on a kernel that ignores --kernel-timeout-sec."""
    source = WORKFLOW.read_text(encoding = "utf-8")
    assert "--kernel-timeout-sec 4200" in source
    ceiling_hours = 4200 / 3600
    budgets = {int(b) for b in re.findall(r"--budget-hours (\d+)", source)}
    assert len(budgets) == 1, budgets
    assert budgets.pop() >= ceiling_hours


def test_the_opt_in_label_the_summary_names_is_the_one_the_gate_reads():
    source = WORKFLOW.read_text(encoding = "utf-8")
    assert "--label-name kaggle-studio-gpu-ci" in source
    assert "kaggle-studio-gpu-ci" in (CI_DIR / "report.py").read_text(encoding = "utf-8")


def test_skipping_the_ui_driver_is_explicit_and_warns():
    source = WORKFLOW.read_text(encoding = "utf-8")
    assert "::warning title=UI driver disabled" in source
    assert source.count("--skip-ui") == 1


def test_the_gate_and_launcher_are_the_shared_ones():
    """Reused, not forked. A second copy of the quota and concurrency policy
    is a second copy that can drift out of agreement with the account."""
    source = WORKFLOW.read_text(encoding = "utf-8")
    assert ".github/scripts/kaggle_t4_ci/gate.py" in source
    assert ".github/scripts/kaggle_t4_ci/launch.py" in source
    assert not (CI_DIR / "gate.py").exists()
    assert not (CI_DIR / "launch.py").exists()


_PASSING = [{"label": "studio-gpu", "passed": True, "assertions": []}]
_FAILING = [{"label": "studio-gpu", "passed": False, "failures": ["x"], "assertions": []}]
_LEG = [{"label": "control", "passed": False, "steps": []}]


@pytest.mark.parametrize(
    ("verdict", "reports", "expected_exit"),
    [
        ("pass", _PASSING, 0),
        ("partial", [], 0),
        ("infra", [], 0),
        ("fail", _FAILING, 1),
        # The kernel verdict covers the training legs too; a failing leg must not mark Studio FAIL.
        ("fail", _PASSING + _LEG, 0),
    ],
)
def test_only_a_real_assertion_failure_turns_the_job_red(tmp_path, verdict, reports, expected_exit):
    evidence = tmp_path / "evidence"
    evidence.mkdir()
    (evidence / "launch_result.json").write_text(
        json.dumps(
            {
                "verdict": verdict,
                "reason": "test",
                "slug": "u/s",
                "kernel_state": "COMPLETE",
                "reports": reports,
            }
        )
    )
    proc = subprocess.run(
        [sys.executable, str(CI_DIR / "report.py"), "--evidence", str(evidence)],
        capture_output = True,
        text = True,
    )
    assert proc.returncode == expected_exit, proc.stdout


def test_a_run_that_reported_nothing_still_names_its_cause(tmp_path):
    evidence = tmp_path / "evidence"
    evidence.mkdir()
    proc = subprocess.run(
        [sys.executable, str(CI_DIR / "report.py"), "--evidence", str(evidence)],
        capture_output = True,
        text = True,
    )
    assert proc.returncode == 0
    assert "nothing is known about the code" in proc.stdout


def test_the_summary_says_what_each_assertion_is_worth(tmp_path):
    """A tick nobody can interpret is a tick nobody should trust."""
    report = _load("studio_ci_report", CI_DIR / "report.py")
    lines = "\n".join(
        report.render(
            {
                "label": "studio-gpu",
                "seconds": 2000,
                "environment": {"gpu_name": "Tesla T4", "llama_cpp_install_kind": "linux-cuda"},
                "config": {"chat_model": "m", "max_steps": 8},
                "assertions": [
                    {
                        "name": "gpu_inference",
                        "passed": True,
                        "evidence": ["nvidia-smi compute-apps: pid 42 holds 2048 MiB"],
                    },
                    {
                        "name": "lora_training",
                        "passed": False,
                        "output_dir": "/x",
                        "phase": "completed",
                        "steps_with_loss": 0,
                    },
                ],
                "failures": ["lora_training: logged a loss for only 0 of 8 steps"],
            }
        )
    )
    assert "not on a CPU fallback" in lines
    assert "AND left an adapter on disk" in lines
    assert "2048 MiB" in lines
    assert "**FAIL**" in lines


def test_the_reporter_survives_the_shared_module_being_unavailable(monkeypatch, tmp_path):
    """kaggle_t4_ci/report.py is owned elsewhere and under active change; the
    verdict must not depend on being able to import it."""
    report = _load("studio_ci_report_2", CI_DIR / "report.py")
    monkeypatch.setattr(report, "_SHARED", tmp_path / "gone.py")
    assert report._load_shared() is None


class _RecordingStudio(studio_client.Studio):
    """An Unsloth whose HTTP layer is a script, so login can be driven off-box."""

    def __init__(self, responses):
        super().__init__(base_url = "http://127.0.0.1:0")
        self._responses = list(responses)
        self.calls = []

    def request(
        self,
        method,
        path,
        body = None,
        **kw,
    ):
        self.calls.append((method, path, body, kw.get("auth", True)))
        return self._responses.pop(0)


def _login_ok(must_change):
    return (200, {"access_token": "bootstrap-token", "must_change_password": must_change})


def test_login_retires_a_bootstrap_password_studio_says_must_change():
    """A bootstrap password left unretired makes load and train return 403, so login must retire it."""
    studio = _RecordingStudio(
        [
            _login_ok(True),
            (200, {"access_token": "post-change-token"}),
        ]
    )
    studio.login("bootstrap-secret")

    assert [c[1] for c in studio.calls] == ["/api/auth/login", "/api/auth/change-password"]
    assert studio.token == "post-change-token"

    change_body = studio.calls[1][2]
    assert change_body["current_password"] == "bootstrap-secret"
    assert (
        change_body["new_password"] != "bootstrap-secret"
    ), "the route rejects an unchanged password"
    assert not any(
        ch.isspace() for ch in change_body["new_password"]
    ), "the route rejects whitespace"
    # Authenticated, or change-password returns 401 rather than 403.
    assert studio.calls[1][3] is True


def test_login_leaves_a_password_alone_when_studio_does_not_ask():
    """An account already past the gate must not have its password rotated as a
    side effect of logging in."""
    studio = _RecordingStudio([_login_ok(False)])
    studio.login("already-changed")
    assert [c[1] for c in studio.calls] == ["/api/auth/login"]
    assert studio.token == "bootstrap-token"


def test_a_refused_password_change_is_raised_rather_than_carried_on_with():
    """Continuing with a token that cannot act is how the first run produced four
    unmeasured assertions and no explanation."""
    studio = _RecordingStudio([_login_ok(True), (400, {"detail": "nope"})])
    with pytest.raises(studio_client.StudioError) as excinfo:
        studio.login("bootstrap-secret")
    assert "400" in str(excinfo.value)
    assert "bootstrap-secret" not in str(excinfo.value), "no credential in the error"


def test_a_password_change_without_a_token_is_refused():
    studio = _RecordingStudio([_login_ok(True), (200, {"detail": "ok"})])
    with pytest.raises(studio_client.StudioError) as excinfo:
        studio.login("bootstrap-secret")
    assert "no access_token" in str(excinfo.value)


def test_the_session_remembers_which_password_is_current():
    """The UI driver needs the password the session is currently authenticated by, not the seeded file."""
    changed = _RecordingStudio([_login_ok(True), (200, {"access_token": "t2"})])
    changed.login("bootstrap-secret")
    assert changed.password not in (None, "", "bootstrap-secret")
    assert changed.password == changed.calls[1][2]["new_password"]

    untouched = _RecordingStudio([_login_ok(False)])
    untouched.login("already-changed")
    assert untouched.password == "already-changed"


def test_the_ui_driver_gets_a_freshly_seeded_account():
    """The API and UI driver need opposite auth states; start_server() removes auth to reseed it."""
    source = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")
    body = source[source.index("def assert_chat_ui") :]
    body = body[: body.index("\n    def ")] if "\n    def " in body else body
    assert (
        "self.stop_server()" in body and "self.start_server()" in body
    ), "the driver needs a re-seeded account, which only a restart provides"
    assert "self.remember_bootstrap()" in body
    assert (
        "self.studio.password" not in body
    ), "the retired password is exactly what the driver cannot use"
    assert body.index("self.start_server()") < body.index(
        '"STUDIO_OLD_PW"'
    ), "the password must be read after the restart, or it is the old one"


def test_the_driver_subprocess_timeout_does_not_track_the_ui_wall_budget():
    """The driver's subprocess backstop must be keyed to the total budget, not ui_wall_timeout + 300."""
    source = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")
    body = source[source.index("def assert_chat_ui") :]
    body = body[: body.index("\n    def ")] if "\n    def " in body else body
    call = body[body.index("subprocess.run(") :]
    call = call[: call.index("\n            )")]
    assert "UI_DRIVER_PROC_TIMEOUT_S" in call, call
    assert "ui_wall_timeout" not in call, call

    module = _load_payload()
    assert "STUDIO_UI_TOTAL_TIMEOUT_S" in body, body
    assert module.UI_DRIVER_PROC_TIMEOUT_S > module.UI_DRIVER_TOTAL_TIMEOUT_S
    assert module.UI_DRIVER_TOTAL_TIMEOUT_S >= 6 * 600
    assert module.UI_DRIVER_PROC_TIMEOUT_S < 120 * 60


# install_llama_prebuilt.py resolves a linux-cuda bundle on x64 CUDA hosts; it must actually
# be installed under STUDIO_HOME or the export assertion fails.
def _load_payload():
    """Loads the payload under a private module name so it cannot collide with the copy on sys.path."""
    spec = importlib.util.spec_from_file_location(
        "_studio_gpu_payload_under_test", PAYLOAD_DIR / "run_studio_gpu.py"
    )
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def _session(module, tmp_path, **overrides):
    # Built from the real parser so new flags do not break these tests with AttributeError.
    args = module.parse_args(
        [
            "--outdir",
            str(tmp_path / "out"),
            "--repo-root",
            str(tmp_path / "repo"),
            "--studio-home",
            str(tmp_path / "home"),
        ]
    )
    for key, value in overrides.items():
        setattr(args, key, value)
    session = module.Payload.__new__(module.Payload)
    session.repo_root = Path(args.repo_root)
    session.studio_home = Path(args.studio_home)
    session.args = args
    session.assertions = []
    session.failures = []
    session.secrets = set()
    return session


def test_the_llama_cpp_install_actually_invokes_the_installer(tmp_path, monkeypatch):
    """run() already sets capture_output and text, so passing either again is a TypeError on hardware."""
    module = _load_payload()
    session = _session(module, tmp_path)
    installer = session.repo_root / "studio" / "install_llama_prebuilt.py"
    installer.parent.mkdir(parents = True)
    installer.write_text("")
    install_dir = session.studio_home / "llama.cpp"
    install_dir.mkdir(parents = True)
    (install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text(
        json.dumps({"asset": "app-b1-linux-x64-cuda13-older.tar.gz", "runtime_line": "cuda13"})
    )

    seen = {}

    def fake_run(cmd, **kw):
        seen["cmd"] = cmd
        seen["kw"] = kw
        return subprocess.CompletedProcess(cmd, 0, "linux_cuda_selection: ok", "")

    monkeypatch.setattr(module, "run", fake_run)
    assert session.install_llama_cpp() is True

    assert str(installer) in seen["cmd"]
    assert "--install-dir" in seen["cmd"]
    assert str(install_dir) in seen["cmd"]
    assert "capture_output" not in seen["kw"]
    assert "text" not in seen["kw"]
    assert seen["kw"]["timeout"] == module.LLAMA_CPP_INSTALL_TIMEOUT_S

    recorded = [entry for entry in session.assertions if entry["name"] == "llama_cpp_install"]
    assert len(recorded) == 1
    assert recorded[0]["llama_cpp_install_kind"] == "cuda13"
    assert "install_kind_before" in recorded[0]


def test_a_successful_installer_that_picks_a_cpu_bundle_is_still_a_failure(tmp_path, monkeypatch):
    """A CPU bundle picked on a CUDA box is a selection bug, separate from an installer that failed."""
    module = _load_payload()
    session = _session(module, tmp_path)
    installer = session.repo_root / "studio" / "install_llama_prebuilt.py"
    installer.parent.mkdir(parents = True)
    installer.write_text("")
    install_dir = session.studio_home / "llama.cpp"
    install_dir.mkdir(parents = True)
    (install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text(
        json.dumps({"asset": "app-b1-linux-x64-cpu.tar.gz", "runtime_line": "cpu"})
    )

    monkeypatch.setattr(
        module, "run", lambda cmd, **kw: subprocess.CompletedProcess(cmd, 0, "", "")
    )
    assert session.install_llama_cpp() is False
    recorded = [e for e in session.assertions if e["name"] == "llama_cpp_install"][0]
    assert any("succeeded but selected" in f for f in recorded["failures"])


def test_a_failed_llama_cpp_install_does_not_stop_the_run():
    """A failed llama.cpp install must not stop the run, so the other assertions still execute and
    report."""
    source = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")
    body = source[source.index("def execute(self)") :]
    body = body[: body.index("\n    def ")]
    assert "self.install_llama_cpp()" in body
    assert "if not self.install_llama_cpp()" not in body, (
        "a llama.cpp install failure now aborts the run, so a box that cannot "
        "install the bundle reports nothing about inference, training or the UI"
    )
    # Before the server, so the export route never sees llama.cpp appear underneath it.
    assert body.index("self.install_llama_cpp()") < body.index("self.start_server()")


def test_the_llama_cpp_marker_falls_back_to_the_canonical_location(tmp_path, monkeypatch):
    """The llama.cpp marker falls back to ~/.unsloth/llama.cpp, where install.sh --local actually
    puts it."""
    module = _load_payload()
    fake_home = tmp_path / "home"
    canonical = fake_home / ".unsloth" / "llama.cpp"
    canonical.mkdir(parents = True)
    (canonical / "UNSLOTH_PREBUILT_INFO.json").write_text(
        json.dumps({"asset": "app-b1-linux-x64-cuda13-older.tar.gz", "runtime_line": "cuda13"})
    )
    monkeypatch.setattr(module.Path, "home", staticmethod(lambda: fake_home))

    studio_home = tmp_path / "studio_home"
    studio_home.mkdir()
    found = module.llama_cpp_marker(studio_home)
    assert found is not None, "the canonical ~/.unsloth/llama.cpp was not consulted"
    assert module.install_kind(found) == "cuda13"


def test_an_explicit_studio_home_install_wins_over_the_canonical_one(tmp_path, monkeypatch):
    """Most specific first. A caller who installed into STUDIO_HOME on purpose
    must not be answered with whatever is in the shared location."""
    module = _load_payload()
    fake_home = tmp_path / "home"
    canonical = fake_home / ".unsloth" / "llama.cpp"
    canonical.mkdir(parents = True)
    (canonical / "UNSLOTH_PREBUILT_INFO.json").write_text(
        json.dumps({"asset": "app-b1-linux-x64-cpu.tar.gz", "runtime_line": "cpu"})
    )
    monkeypatch.setattr(module.Path, "home", staticmethod(lambda: fake_home))

    studio_home = tmp_path / "studio_home"
    (studio_home / "llama.cpp").mkdir(parents = True)
    (studio_home / "llama.cpp" / "UNSLOTH_PREBUILT_INFO.json").write_text(
        json.dumps({"asset": "app-b1-linux-x64-cuda13-older.tar.gz", "runtime_line": "cuda13"})
    )
    assert module.install_kind(module.llama_cpp_marker(studio_home)) == "cuda13"


def test_no_llama_cpp_anywhere_is_still_reported_as_absent(tmp_path, monkeypatch):
    """The fallback must not turn a real absence into a guess."""
    module = _load_payload()
    fake_home = tmp_path / "home"
    fake_home.mkdir()
    monkeypatch.setattr(module.Path, "home", staticmethod(lambda: fake_home))
    studio_home = tmp_path / "studio_home"
    studio_home.mkdir()
    assert module.llama_cpp_marker(studio_home) is None
    assert module.install_kind(None) is None


def test_the_marker_never_carried_an_install_kind(tmp_path):
    """install_kind is printed to stdout, never written to the marker, so reading it there yields None."""
    module = _load_payload()
    marker = tmp_path / "UNSLOTH_PREBUILT_INFO.json"
    marker.write_text(
        json.dumps(
            {
                "requested_tag": "b10360",
                "tag": "b10360",
                "release_tag": "b10360-mix-87da1a2",
                "asset": "app-b10360-mix-87da1a2-linux-x64-cuda13-older.tar.gz",
                "runtime_line": "cuda13",
                "coverage_class": "older",
                "bundle_profile": "mix",
            }
        )
    )
    kind = module.install_kind(marker)
    assert kind is not None, (
        "a marker with no install_kind key must still say what was installed; "
        "answering None here is what failed six hardware runs"
    )
    assert module.is_cuda_install(kind)


def test_a_non_cuda_runtime_line_is_not_a_cuda_install():
    """The assertion still has to be able to fail."""
    module = _load_payload()
    for kind in ("cpu", "vulkan", "rocm", "metal", None, ""):
        assert not module.is_cuda_install(kind), f"{kind!r} classified as CUDA"


def test_a_future_cuda_major_is_recognised():
    """Match the CUDA runtime line's shape, not a fixed list of names: a fixed set fails closed on
    the next CUDA major and reports a working install as not-CUDA."""
    module = _load_payload()
    assert module.is_cuda_install("cuda14")
    assert module.is_cuda_install("app-bX-linux-x64-cuda99-portable.tar.gz")
    assert not module.is_cuda_install("cudart")


class _FakeStudio:
    """Records calls and answers /api/inference/status from a script."""

    def __init__(
        self,
        status_body,
        *,
        unload_raises = False,
    ):
        self.status_body = status_body
        self.unload_raises = unload_raises
        self.posts = []

    def get(self, path, **kw):
        if path == "/api/inference/status":
            return 200, self.status_body
        return 404, {}

    def post(self, path, body, **kw):
        self.posts.append((path, body))
        if self.unload_raises:
            raise studio_client.StudioError("unload refused")
        return 200, {}


def _baseline_session(
    module,
    tmp_path,
    status_body,
    readings,
    *,
    unload_raises = False,
):
    session = _session(module, tmp_path, load_timeout = 30.0)
    session.studio = _FakeStudio(status_body, unload_raises = unload_raises)
    seq = list(readings)
    session._readings = seq
    return session


def test_the_baseline_waits_for_the_old_model_to_actually_leave(tmp_path, monkeypatch):
    """Sample the baseline only after the old model unloads, or the load reads as a negative delta."""
    module = _load_payload()
    session = _baseline_session(
        module, tmp_path, {"model_identifier": "chat.gguf"}, [4000.0, 2600.0, 1000.0, 996.0, 996.0]
    )
    monkeypatch.setattr(module, "VRAM_SETTLE_POLL_S", 0.0)
    monkeypatch.setattr(module, "nvidia_used_mib", lambda: session._readings.pop(0))
    assert session.settled_baseline() == 996.0
    assert session.studio.posts[0][0] == "/api/inference/unload"
    assert session.studio.posts[0][1]["model_path"] == "chat.gguf"
    assert session.studio.posts[0][1]["force_cancel_active"] is True


def test_nothing_loaded_means_nothing_to_unload(tmp_path, monkeypatch):
    module = _load_payload()
    session = _baseline_session(module, tmp_path, {}, [500.0, 500.0])
    monkeypatch.setattr(module, "VRAM_SETTLE_POLL_S", 0.0)
    monkeypatch.setattr(module, "nvidia_used_mib", lambda: session._readings.pop(0))
    assert session.settled_baseline() == 500.0
    assert session.studio.posts == []


def test_a_refused_unload_does_not_abort_the_probe(tmp_path, monkeypatch):
    """The delta is evidence, not the assertion. Losing it must not lose the
    load result that the probe is actually there to record."""
    module = _load_payload()
    session = _baseline_session(
        module, tmp_path, {"model_identifier": "chat.gguf"}, [900.0, 900.0], unload_raises = True
    )
    monkeypatch.setattr(module, "VRAM_SETTLE_POLL_S", 0.0)
    monkeypatch.setattr(module, "nvidia_used_mib", lambda: session._readings.pop(0))
    assert session.settled_baseline() == 900.0


def test_driver_jitter_is_not_mistaken_for_a_release(tmp_path, monkeypatch):
    """A few MiB of wobble must settle immediately, not burn the full budget
    and return whatever the last sample happened to be."""
    module = _load_payload()
    session = _baseline_session(module, tmp_path, {}, [1000.0, 995.0, 1000.0])
    monkeypatch.setattr(module, "VRAM_SETTLE_POLL_S", 0.0)
    monkeypatch.setattr(module, "nvidia_used_mib", lambda: session._readings.pop(0))
    assert session.settled_baseline() == 995.0


def test_a_card_that_never_settles_is_bounded(tmp_path, monkeypatch):
    """Something else holding the card must not hang the run forever."""
    module = _load_payload()
    session = _session(module, tmp_path, load_timeout = 30.0)
    session.studio = _FakeStudio({})
    monkeypatch.setattr(module, "VRAM_SETTLE_POLL_S", 0.0)
    calls = [0]

    def _falling():
        calls[0] += 1
        return 10000.0 - calls[0] * 100.0

    monkeypatch.setattr(module, "nvidia_used_mib", _falling)
    session.settled_baseline()
    assert calls[0] <= module.VRAM_SETTLE_SAMPLES + 1


def test_no_nvidia_smi_at_all_is_not_a_crash(tmp_path, monkeypatch):
    module = _load_payload()
    session = _session(module, tmp_path, load_timeout = 30.0)
    session.studio = _FakeStudio({})
    monkeypatch.setattr(module, "VRAM_SETTLE_POLL_S", 0.0)
    monkeypatch.setattr(module, "nvidia_used_mib", lambda: None)
    assert session.settled_baseline() is None


def test_the_status_fields_read_are_fields_the_response_really_has(tmp_path):
    """Status fields read must exist on InferenceStatusResponse, so a guessed name fails here."""
    import re

    model_src = (REPO_ROOT / "studio" / "backend" / "models" / "inference.py").read_text(
        encoding = "utf-8"
    )
    block = model_src.split("class InferenceStatusResponse", 1)[1]
    block = block.split("\nclass ", 1)[0]
    declared = set(re.findall(r"^    (\w+):", block, re.MULTILINE))
    payload_src = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")
    fn = payload_src.split("def settled_baseline", 1)[1].split("\n    def ", 1)[0]
    read = set(re.findall(r'body\.get\("(\w+)"\)', fn))
    assert read, "settled_baseline reads no status field at all"
    assert read <= declared, (
        f"settled_baseline reads {sorted(read - declared)}, which "
        f"InferenceStatusResponse does not declare"
    )


def test_the_display_name_is_only_a_fallback(tmp_path, monkeypatch):
    """model_identifier is the loadable one and is what /unload wants."""
    module = _load_payload()
    session = _baseline_session(
        module,
        tmp_path,
        {"model_identifier": "loadable.gguf", "active_model": "Pretty Name"},
        [3000.0, 500.0, 500.0],
    )
    monkeypatch.setattr(module, "VRAM_SETTLE_POLL_S", 0.0)
    monkeypatch.setattr(module, "nvidia_used_mib", lambda: session._readings.pop(0))
    session.settled_baseline()
    assert session.studio.posts[0][1]["model_path"] == "loadable.gguf"


def test_a_loaded_list_alone_is_enough_to_unload(tmp_path, monkeypatch):
    module = _load_payload()
    session = _baseline_session(
        module, tmp_path, {"loaded": ["only-here.gguf"]}, [3000.0, 500.0, 500.0]
    )
    monkeypatch.setattr(module, "VRAM_SETTLE_POLL_S", 0.0)
    monkeypatch.setattr(module, "nvidia_used_mib", lambda: session._readings.pop(0))
    session.settled_baseline()
    assert session.studio.posts[0][1]["model_path"] == "only-here.gguf"


# The launcher collects each kernel into its own subdirectory, so the walk must recurse.
def test_the_bundle_is_found_in_the_per_kernel_directory_the_launcher_writes(tmp_path):
    """launch.py::fetch_evidence writes kaggle_evidence/<slug>/..., and the
    workflow passes kaggle_evidence. A top-level glob reported every real run
    as having emitted no evidence at all."""
    blob = _bundle({"studio_gpu_report.json": b'{"passed": false}'})
    evidence = tmp_path / "kaggle_evidence" / "unsloth-t4-ci-deadbeef"
    evidence.mkdir(parents = True)
    (evidence / "kernel.log").write_text("\n".join(_chunk_lines(blob)), encoding = "utf-8")
    outdir = tmp_path / "studio_evidence"
    proc = subprocess.run(
        [
            sys.executable,
            str(CI_DIR / "collect_evidence.py"),
            "--evidence",
            str(tmp_path / "kaggle_evidence"),
            "--outdir",
            str(outdir),
        ],
        capture_output = True,
        text = True,
    )
    assert proc.returncode == 0, proc.stderr
    assert (outdir / "studio_gpu_report.json").read_bytes() == b'{"passed": false}'


def test_a_nested_executed_notebook_is_read_too(tmp_path):
    blob = _bundle({"studio.log": b"a log"})
    nb = {
        "cells": [{"outputs": [{"text": "\n".join(_chunk_lines(blob))}]}],
    }
    nested = tmp_path / "evidence" / "unsloth-t4-ci-1234"
    nested.mkdir(parents = True)
    (nested / "studio_gpu_output.ipynb").write_text(json.dumps(nb), encoding = "utf-8")
    chunks, total = collect_evidence.collect_chunks(
        collect_evidence.iter_text(tmp_path / "evidence")
    )
    assert total and len(chunks) == total


def test_evidence_that_is_json_but_not_a_notebook_does_not_raise(tmp_path):
    """Evidence that is valid JSON but not a notebook, such as [], a string cell or a numeric
    output, must not raise, since a raise after the verdict was posted fails the job."""
    evidence = tmp_path / "evidence" / "unsloth-t4-ci-1234"
    evidence.mkdir(parents = True)
    (evidence / "a_output.ipynb").write_text("[]", encoding = "utf-8")
    (evidence / "b_output.ipynb").write_text(
        json.dumps({"cells": ["not a cell", {"outputs": [7, {"text": ["x", 1]}]}]}),
        encoding = "utf-8",
    )
    blob = _bundle({"studio.log": b"a log"})
    (evidence / "c_output.ipynb").write_text(
        json.dumps({"cells": [{"outputs": [{"text": "\n".join(_chunk_lines(blob))}]}]}),
        encoding = "utf-8",
    )
    streams = list(collect_evidence.iter_text(tmp_path / "evidence"))
    assert "x1" in streams
    chunks, total = collect_evidence.collect_chunks(iter(streams))
    assert total and len(chunks) == total


def test_the_evidence_unpack_is_best_effort_on_the_gpu_job():
    """The notebook and scheduled unpacks already are; this one decided the
    colour of a job whose verdict had already been posted."""
    yaml = pytest.importorskip("yaml")
    workflow = yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8"))
    steps = [
        s
        for job in workflow["jobs"].values()
        for s in job.get("steps", [])
        if "collect_evidence.py" in (s.get("run") or "")
    ]
    assert steps, "no unpack step"
    for step in steps:
        assert step.get("continue-on-error") is True, step.get("name")


def test_a_nan_loss_is_not_a_trained_step():
    """A T4 has no bf16, so training runs in fp16; a diverged fp16 run logs NaN every step yet still
    completes and saves an adapter, so NaN losses are not trained steps."""
    status = {"metric_history": {"loss": [float("nan")] * 8}}
    assert studio_client.trained_steps(status) == 0
    assert len(studio_client.nonfinite_losses(status)) == 8


def test_an_infinite_loss_is_not_a_trained_step():
    status = {"metric_history": {"loss": [1.4, float("inf"), 0.9, None]}}
    assert studio_client.trained_steps(status) == 2
    assert studio_client.nonfinite_losses(status) == [float("inf")]


def test_real_losses_still_count():
    status = {"metric_history": {"loss": [2.0, 1.5, 1.1]}}
    assert studio_client.trained_steps(status) == 3
    assert studio_client.nonfinite_losses(status) == []


def test_studio_is_launched_from_the_interpreter_running_the_payload(tmp_path, monkeypatch):
    """Studio must be launched from the interpreter running the payload, since the venv bin is not
    on PATH and a global unsloth on PATH would win shutil.which()."""
    module = _load_payload()
    venv_bin = tmp_path / "venv" / "bin"
    venv_bin.mkdir(parents = True)
    own = venv_bin / "unsloth"
    own.write_text("#!/bin/sh\n", encoding = "utf-8")
    own.chmod(0o755)

    stray_bin = tmp_path / "stray"
    stray_bin.mkdir()
    stray = stray_bin / "unsloth"
    stray.write_text("#!/bin/sh\n", encoding = "utf-8")
    stray.chmod(0o755)
    monkeypatch.setenv("PATH", str(stray_bin))

    session = _session(module, tmp_path, port = 18902)
    monkeypatch.setattr(module.sys, "executable", str(venv_bin / "python"))
    assert session.studio_command()[0] == str(own)


def test_without_a_console_script_the_same_interpreter_runs_the_module(tmp_path, monkeypatch):
    module = _load_payload()
    venv_bin = tmp_path / "venv" / "bin"
    venv_bin.mkdir(parents = True)
    stray_bin = tmp_path / "stray"
    stray_bin.mkdir()
    stray = stray_bin / "unsloth"
    stray.write_text("#!/bin/sh\n", encoding = "utf-8")
    stray.chmod(0o755)
    monkeypatch.setenv("PATH", str(stray_bin))

    session = _session(module, tmp_path, port = 18902)
    monkeypatch.setattr(module.sys, "executable", str(venv_bin / "python"))
    command = session.studio_command()
    assert command[0] == str(venv_bin / "python")
    assert command[1] == "-c"


def test_the_payload_never_reads_a_pid_the_status_response_does_not_declare():
    """InferenceStatusResponse declares neither llama_server_pid nor pid, and FastAPI drops
    undeclared fields, so reading them was always None and the process VRAM probe never fired."""
    import re

    model_src = (REPO_ROOT / "studio" / "backend" / "models" / "inference.py").read_text(
        encoding = "utf-8"
    )
    declared = set()
    for cls in ("class InferenceStatusResponse", "class _InferenceRuntimeFields"):
        block = model_src.split(cls, 1)[1].split("\nclass ", 1)[0]
        declared |= set(re.findall(r"^    (\w+):", block, re.MULTILINE))
    payload_src = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")
    fn = payload_src.split("def load_model", 1)[1].split("\n    def ", 1)[0]
    read = set(re.findall(r'status_body\.get\("(\w+)"\)', fn))
    assert read <= declared, (
        f"load_model reads {sorted(read - declared)}, which the inference status "
        f"response does not declare"
    )


def test_a_discovered_llama_server_pid_is_enough_evidence():
    verdict = gpu_assert.offload_verdict(
        server_pid = None,
        server_pids = [4242],
        compute_apps = {4242: 2048},
        log_text = "",
        device_vram_delta_mib = None,
        status = {},
    )
    assert verdict["passed"]
    assert any("4242" in p for p in verdict["positives"])


def test_a_discovered_pid_holding_nothing_is_still_not_evidence():
    verdict = gpu_assert.offload_verdict(
        server_pid = None,
        server_pids = [4242],
        compute_apps = {4242: 12},
        log_text = "",
        device_vram_delta_mib = None,
        status = {},
    )
    assert not verdict["passed"]


def test_the_payload_can_find_a_llama_server_in_the_process_table(tmp_path):
    module = _load_payload()
    proc = subprocess.Popen(
        [sys.executable, "-c", "import time; time.sleep(30)"],
        executable = sys.executable,
    )
    try:
        assert isinstance(module.llama_server_pids(), list)
    finally:
        proc.kill()
        proc.wait()


def test_an_earlier_loads_offload_line_is_not_evidence_for_the_next(tmp_path):
    """The exported-model reload used to inherit the chat model's
    `offloaded 29/29` when its own load logged nothing, so the GPU check
    passed on evidence from a different model."""
    module = _load_payload()
    server_log = tmp_path / "studio.log"
    home = tmp_path / "home"
    home.mkdir()
    server_log.write_text("load_tensors: offloaded 29/29 layers to GPU\n", encoding = "utf-8")

    marks = module.log_marks(server_log, home)
    with open(server_log, "a", encoding = "utf-8") as fh:
        fh.write("llama_server: listening\n")

    scoped = module.studio_log_text(server_log, home, since = marks)
    assert "offloaded" not in scoped
    assert gpu_assert.offloaded_layers(scoped) is None
    assert "offloaded" in module.studio_log_text(server_log, home)


def test_a_log_that_was_rotated_under_us_is_read_whole(tmp_path):
    module = _load_payload()
    server_log = tmp_path / "studio.log"
    home = tmp_path / "home"
    home.mkdir()
    server_log.write_text("x" * 500, encoding = "utf-8")
    marks = module.log_marks(server_log, home)
    server_log.write_text("offloaded 7/7 layers to GPU\n", encoding = "utf-8")
    assert "offloaded" in module.studio_log_text(server_log, home, since = marks)


def test_the_restart_that_reseeds_the_account_keeps_the_earlier_log(tmp_path, monkeypatch):
    """assert_chat_ui() restarts Unsloth, and a truncating open threw away the
    backend log of every GPU assertion before the evidence was packaged."""
    module = _load_payload()
    session = _session(
        module,
        tmp_path,
        port = 18902,
        health_deadline = 0.0,
    )
    session.outdir = tmp_path / "out"
    session.outdir.mkdir()
    session.server_log = session.outdir / "studio.log"
    session.server_log.write_text("the first session's traceback\n", encoding = "utf-8")
    session.proc = None
    session.base_url = "http://127.0.0.1:18902"
    session.studio = _FakeStudio({})
    monkeypatch.setattr(module.subprocess, "Popen", lambda *a, **kw: _DeadProc())
    monkeypatch.setattr(module, "wait_for", lambda **kw: (True, {}, ""))
    session.start_server()
    assert "the first session's traceback" in session.server_log.read_text()


class _DeadProc:
    def poll(self):
        return None

    def terminate(self):
        return None

    def wait(self, timeout = None):
        return 0


def _export_session(module, tmp_path, studio):
    session = _session(
        module,
        tmp_path,
        export_timeout = 1.0,
        export_deadline = 5.0,
        quantization = "q8_0",
        gpu_layers = 99,
    )
    session.outdir = tmp_path / "out"
    session.outdir.mkdir()
    session.server_log = session.outdir / "studio.log"
    session.studio = studio
    session.proc = _DeadProc()
    return session


class _ExportStudio:
    """load-checkpoint fine, the gguf POST times out in transport, and the
    export finishes on the backend regardless."""

    def __init__(self, marker: Path):
        self.marker = marker
        self.polls = 0

    def expect(
        self,
        method,
        path,
        body = None,
        **kw,
    ):
        if path == "/api/export/export/gguf":
            raise TimeoutError("the read timed out")
        return {}

    def get(self, path, **kw):
        if path == "/api/export/status":
            self.polls += 1
            if self.polls < 2:
                return 200, {"last_op_seq": 1, "is_export_active": True}
            return 200, {
                "last_op_seq": 2,
                "is_export_active": False,
                "last_op_status": "success",
                "last_op_output_path": str(self.marker.parent),
            }
        return 200, {}


def test_an_export_that_outlives_its_http_request_is_still_polled(tmp_path, monkeypatch):
    """The route blocks for the whole export and the transport timeout is
    shorter than the export deadline, so a slow quantize raised an unhandled
    TimeoutError and crashed the payload instead of waiting for the result."""
    module = _load_payload()
    exports = tmp_path / "exports"
    exports.mkdir()
    gguf = exports / "model.Q8_0.gguf"
    gguf.write_bytes(b"GGUF" + b"\x00" * 64)
    session = _export_session(module, tmp_path, _ExportStudio(gguf))
    session.args.chat_timeout = 1.0
    session.args.load_timeout = 1.0
    monkeypatch.setattr(module, "llama_cpp_marker", lambda home: None)
    monkeypatch.setattr(module, "install_kind", lambda marker: "cuda12")
    monkeypatch.setattr(
        module.Payload,
        "load_model",
        lambda self, path, *, variant, label: {"failures": [], "evidence": []},
    )
    monkeypatch.setattr(
        module.Payload,
        "chat",
        lambda self, messages, **kw: (200, {"choices": [{"message": {"content": "hi"}}]}),
    )
    ok = session.assert_gguf_export(str(tmp_path / "adapter"))
    detail = session.assertions[-1]
    assert "export_request_timeout" in detail, detail
    assert ok, detail.get("failures")


def test_the_capped_bundle_still_carries_the_logs(tmp_path, monkeypatch):
    """The size-capped fallback bundle must still carry studio.log and the driver log, not just the
    report, since those logs matter most in failing runs."""
    module = _load_payload()
    session = _session(module, tmp_path)
    session.outdir = tmp_path / "out"
    session.art_dir = session.outdir / "playwright"
    session.art_dir.mkdir(parents = True)
    session.server_log = session.outdir / "studio.log"
    (session.outdir / "studio_gpu_report.json").write_text('{"passed": false}', encoding = "utf-8")
    session.server_log.write_text("backend traceback\n", encoding = "utf-8")
    (session.outdir / "playwright_chat_ui.log").write_text("driver log\n", encoding = "utf-8")
    (session.art_dir / "01.png").write_bytes(os.urandom(3_000_000))

    printed: list[str] = []
    monkeypatch.setattr("builtins.print", lambda *a, **kw: printed.append(" ".join(map(str, a))))
    session.emit_evidence(False)
    encoded = "".join(
        line.split(" ")[-1] for line in printed if line.startswith(module.EVIDENCE_PREFIX)
    )
    blob = base64.b64decode(encoded)
    assert len(blob) <= module.MAX_EVIDENCE_BYTES
    with tarfile.open(fileobj = io.BytesIO(blob), mode = "r:gz") as tar:
        names = tar.getnames()
        assert "studio.log" in names
        assert "playwright_chat_ui.log" in names
        assert "studio_gpu_report.json" in names


def test_a_crash_while_packaging_the_evidence_does_not_publish_a_pass(tmp_path, monkeypatch):
    """extract_reports keeps the first report per label|model, so a pass printed before a crash
    would be published while the later correction goes unread."""
    module = _load_payload()
    session = _session(
        module,
        tmp_path,
        label = "studio-gpu",
        chat_model = "m",
        chat_variant = None,
        train_model = "t",
        max_steps = 8,
        quantization = "q8_0",
        gpu_layers = 99,
    )
    session.outdir = tmp_path / "out"
    session.outdir.mkdir()
    session.server_log = session.outdir / "studio.log"
    session.proc = None
    session.started = 0.0

    def _boom(passed):
        raise OSError("the artifact could not be read")

    monkeypatch.setattr(session, "emit_evidence", _boom)
    printed: list[str] = []
    monkeypatch.setattr("builtins.print", lambda *a, **kw: printed.append(" ".join(map(str, a))))
    code = session.finish()
    reports = [line for line in printed if line.startswith(module.RESULT_PREFIX)]
    assert len(reports) == 1
    published = json.loads(reports[0][len(module.RESULT_PREFIX) :])
    assert published["passed"] is False
    assert code == 1


def _cell_source(driver: dict, needle: str) -> str:
    for cell in _payload_notebook(driver)["cells"]:
        source = "".join(cell["source"])
        if needle in source:
            return source
    raise AssertionError(f"no generated cell contains {needle!r}")


def test_a_failing_install_under_test_reports_a_failure_rather_than_infra(tmp_path):
    """A failing install must report failure, not infra, or the workflow passes the regressions it
    guards."""
    driver = _build(tmp_path)
    payload_source = _payload_source(driver)
    fail_report_src = (
        "def fail_report" + payload_source.split("def fail_report", 1)[1].split("\n\ndef ", 1)[0]
    )

    emitted: list[str] = []
    namespace = {"json": json, "print": lambda *a, **kw: emitted.append("".join(map(str, a)))}
    exec(compile(fail_report_src, "<fail_report>", "exec"), namespace)

    class _Result:
        returncode = 1

    namespace.update(
        {
            "sh": lambda *a, **kw: _Result(),
            "REPO": tmp_path,
            "STUDIO_HOME": tmp_path,
            "pathlib": __import__("pathlib"),
        }
    )
    with pytest.raises(SystemExit):
        exec(compile(_cell_source(driver, "_install = sh("), "<install>", "exec"), namespace)

    reports = [line for line in emitted if line.startswith(build_kernel.RESULT_PREFIX)]
    assert len(reports) == 1, emitted
    report = json.loads(reports[0][len(build_kernel.RESULT_PREFIX) :])
    assert report["passed"] is False
    assert "install.sh" in report["failures"][0]


def test_an_install_that_leaves_no_interpreter_is_also_a_failure(tmp_path):
    driver = _build(tmp_path)
    payload_source = _payload_source(driver)
    fail_report_src = (
        "def fail_report" + payload_source.split("def fail_report", 1)[1].split("\n\ndef ", 1)[0]
    )
    emitted: list[str] = []
    namespace = {"json": json, "print": lambda *a, **kw: emitted.append("".join(map(str, a)))}
    exec(compile(fail_report_src, "<fail_report>", "exec"), namespace)

    class _Result:
        returncode = 0

    namespace.update({"sh": lambda *a, **kw: _Result(), "REPO": tmp_path, "STUDIO_HOME": tmp_path})
    with pytest.raises(SystemExit):
        exec(compile(_cell_source(driver, "_install = sh("), "<install>", "exec"), namespace)
    reports = [line for line in emitted if line.startswith(build_kernel.RESULT_PREFIX)]
    assert len(reports) == 1
    assert json.loads(reports[0][len(build_kernel.RESULT_PREFIX) :])["passed"] is False


def _run_verify_cell(driver: dict, tmp_path, *, probe: dict, host_gpus: list[str]):
    """Runs the generated probe cell against a fake venv and nvidia-smi; returns the reports it
    published."""
    payload_source = _payload_source(driver)
    fail_report_src = (
        "def fail_report" + payload_source.split("def fail_report", 1)[1].split("\n\ndef ", 1)[0]
    )
    emitted: list[str] = []
    namespace = {"json": json, "print": lambda *a, **kw: emitted.append("".join(map(str, a)))}
    exec(compile(fail_report_src, "<fail_report>", "exec"), namespace)

    class _FakeSubprocess:
        @staticmethod
        def run(cmd, **kw):
            if "nvidia-smi" in cmd[0]:
                return subprocess.CompletedProcess(cmd, 0, "\n".join(host_gpus) + "\n", "")
            return subprocess.CompletedProcess(cmd, 0, json.dumps(probe) + "\n", "")

    namespace.update(
        {
            "subprocess": _FakeSubprocess,
            "os": os,
            "VENV_PY": tmp_path / "python",
            "STUDIO_HOME": tmp_path,
        }
    )
    with pytest.raises(SystemExit):
        exec(compile(_cell_source(driver, 'probe["missing"]'), "<verify>", "exec"), namespace)
    return [line for line in emitted if line.startswith(build_kernel.RESULT_PREFIX)]


def test_a_venv_that_cannot_use_cuda_on_a_gpu_box_is_a_failure_not_infra(tmp_path):
    """`install.sh --local` resolving a CPU-only torch is the CUDA install
    regression this leg exists to catch, and it left no T4_SMOKE_REPORT at all,
    so the launcher filed it as `infra` and the reporter exited 0."""
    reports = _run_verify_cell(
        _build(tmp_path),
        tmp_path,
        probe = {"versions": {"torch": "2.9.0+cpu"}, "missing": [], "cuda": {"available": False}},
        host_gpus = ["Tesla T4"],
    )
    assert len(reports) == 1, reports
    report = json.loads(reports[0][len(build_kernel.RESULT_PREFIX) :])
    assert report["passed"] is False
    assert "cannot use CUDA" in report["failures"][0]


def test_a_session_kaggle_gave_no_gpu_at_all_stays_infra(tmp_path):
    """Nothing was learned about the code, so this one keeps the no-report path
    and must NOT turn a pull request red."""
    reports = _run_verify_cell(
        _build(tmp_path),
        tmp_path,
        probe = {"versions": {"torch": "2.9.0"}, "missing": [], "cuda": {"available": False}},
        host_gpus = [],
    )
    assert reports == []


def test_a_payload_that_hangs_past_the_driver_deadline_is_a_failure(tmp_path):
    """papermill killed mid-assertion emitted no report at all, so a hang in
    the code under test was filed as unavailable infrastructure."""
    driver = _build(tmp_path)
    runner = None
    for cell in driver["cells"]:
        source = "".join(cell["source"])
        if "papermill" in source:
            runner = source
    assert runner is not None

    emitted: list[str] = []

    class _FakeSubprocess:
        TimeoutExpired = subprocess.TimeoutExpired
        STDOUT = subprocess.STDOUT

        @staticmethod
        def run(cmd, **kw):
            kw["stdout"].write(
                f"{build_kernel.PAYLOAD_SENTINEL} exec /usr/bin/python run_studio_gpu.py\n".encode()
            )
            kw["stdout"].flush()
            raise subprocess.TimeoutExpired(cmd, 3900)

    namespace = {
        "WORK": tmp_path,
        "os": os,
        "sys": sys,
        "time": __import__("time"),
        "json": json,
        "subprocess": _FakeSubprocess,
        "print": lambda *a, **kw: emitted.append("".join(map(str, a))),
    }
    exec(compile(runner, "<runner>", "exec"), namespace)
    reports = [line for line in emitted if line.startswith(build_kernel.RESULT_PREFIX)]
    assert len(reports) == 1, emitted
    assert json.loads(reports[0][len(build_kernel.RESULT_PREFIX) :])["passed"] is False


def test_a_timeout_before_the_payload_started_is_still_infra(tmp_path):
    """A slow clone or install is Kaggle being slow, and must stay a no-report
    infra outcome rather than turning a pull request red."""
    driver = _build(tmp_path)
    runner = ["".join(c["source"]) for c in driver["cells"] if "papermill" in "".join(c["source"])][
        0
    ]
    emitted: list[str] = []

    class _FakeSubprocess:
        TimeoutExpired = subprocess.TimeoutExpired
        STDOUT = subprocess.STDOUT

        @staticmethod
        def run(cmd, **kw):
            kw["stdout"].write(b"cloning...\n")
            raise subprocess.TimeoutExpired(cmd, 3900)

    namespace = {
        "WORK": tmp_path,
        "os": os,
        "sys": sys,
        "time": __import__("time"),
        "json": json,
        "subprocess": _FakeSubprocess,
        "print": lambda *a, **kw: emitted.append("".join(map(str, a))),
    }
    exec(compile(runner, "<runner>", "exec"), namespace)
    assert not [line for line in emitted if line.startswith(build_kernel.RESULT_PREFIX)]


def test_the_gate_is_told_how_many_kernels_this_leg_actually_pushes():
    """This leg pushes ONE kernel and leaves the second T4 idle. The gate
    defaults to two, and refuses unless that many slots are free."""
    import re

    text = WORKFLOW.read_text(encoding = "utf-8")
    # Find every gate.py invocation (gate and recheck), not the first textual mention.
    invocations = re.findall(r"kaggle_t4_ci/gate\.py \\\n.*?(?=\n\s*\n|\Z)", text, re.DOTALL)
    assert invocations, "the workflow never runs the gate"
    for invocation in invocations:
        assert re.search(r"^\s+--kernels 1 \\?\s*$", invocation, re.MULTILINE), invocation
    assert "--expect 1" in text


def test_the_opt_in_label_can_actually_start_the_workflow():
    """GitHub's default pull_request activity types are opened, synchronize
    and reopened, so adding the advertised label started nothing."""
    triggers = _triggers(_workflow())
    assert "labeled" in triggers["pull_request"]["types"]
    for default in ("opened", "synchronize", "reopened"):
        assert default in triggers["pull_request"]["types"]


def test_an_unrelated_label_cannot_start_a_seventy_minute_kaggle_run():
    """GitHub has no per-label event filter, so labeled fires for every label while the gate reads
    the whole label list; the job if must screen out unrelated labels."""
    condition = " ".join(_workflow()["jobs"]["gate"]["if"].split())
    assert "github.event.action != 'labeled'" in condition
    assert "github.event.label.name == 'kaggle-studio-gpu-ci'" in condition
    assert "fork != true" in condition


class _LoadStudio:
    def __init__(self, status_body):
        self.status_body = status_body

    def expect(
        self,
        method,
        path,
        body = None,
        **kw,
    ):
        return {}

    def get(self, path, **kw):
        return 200, self.status_body

    def post(
        self,
        path,
        body = None,
        **kw,
    ):
        return 200, {}


def test_load_model_does_not_inherit_the_previous_loads_offload_line(tmp_path, monkeypatch):
    """The end-to-end shape of the scoping fix: a reload whose own load logged
    nothing must not pass on the chat model's `offloaded 29/29`."""
    module = _load_payload()
    session = _session(module, tmp_path, load_timeout = 5.0, gpu_layers = 99)
    session.outdir = tmp_path / "out"
    session.outdir.mkdir()
    session.server_log = session.outdir / "studio.log"
    session.studio_home.mkdir(parents = True, exist_ok = True)
    session.server_log.write_text("load_tensors: offloaded 29/29 layers to GPU\n", encoding = "utf-8")
    session.studio = _LoadStudio({})
    monkeypatch.setattr(module, "nvidia_used_mib", lambda: None)
    monkeypatch.setattr(module, "nvidia_compute_apps", lambda: {})
    monkeypatch.setattr(module, "llama_server_pids", lambda: [])
    monkeypatch.setattr(module.Payload, "settled_baseline", lambda self: None)

    detail = session.load_model("exported.gguf", variant = None, label = "exported")
    assert detail["positives"] == []
    assert detail["failures"], "silence must be a failure, not the previous load's evidence"


def test_a_box_without_nvidia_smi_reports_no_gpu_rather_than_crashing(tmp_path, monkeypatch):
    """environment() runs inside finish() on every path, so a missing nvidia-smi (FileNotFoundError)
    must read as no GPU, not end the run with no report at all."""
    module = _load_payload()
    monkeypatch.setenv("PATH", str(tmp_path))
    assert module.gpu_inventory() == []
    assert module.nvidia_used_mib() is None
    assert module.nvidia_compute_apps() is None


def test_the_kaggle_client_is_new_enough_to_read_the_only_credential_we_have():
    """Kaggle 1.7.4.5 cannot read KAGGLE_API_TOKEN, and the resulting OSError becomes a green skip."""
    packaging_version = pytest.importorskip("packaging.version")
    pins = re.findall(
        r"pip install [^\n]*'kaggle==([0-9][^']*)'", WORKFLOW.read_text(encoding = "utf-8")
    )
    assert pins, "no pinned kaggle client in the workflow"
    assert len(set(pins)) == 1, f"jobs disagree on the kaggle client: {pins}"
    assert packaging_version.Version(pins[0]) >= packaging_version.Version("2.2.0"), pins[0]


def test_no_studio_assertion_is_wired_to_a_constant_branch():
    """A constant test in an assert method is a branch that can never be taken or always taken."""
    import ast

    src = (
        Path(__file__).resolve().parents[2]
        / "tests"
        / "kaggle"
        / "studio_gpu"
        / "run_studio_gpu.py"
    ).read_text(encoding = "utf-8")
    offenders = []
    for func in ast.walk(ast.parse(src)):
        if not (isinstance(func, ast.FunctionDef) and func.name.startswith("assert_")):
            continue
        for node in ast.walk(func):
            if isinstance(node, ast.If) and isinstance(node.test, ast.Constant):
                offenders.append(f"{func.name}: if {ast.unparse(node.test)}")
    assert offenders == [], f"assertions wired to a constant: {offenders}"


def test_every_assertion_carries_its_own_wall_clock():
    """Each Studio assertion needs its own wall clock, or the longest payload's time cannot be
    attributed."""
    import importlib.util
    import types

    payload = PAYLOAD_DIR / "run_studio_gpu.py"
    spec = importlib.util.spec_from_file_location("_studio_payload_timing", payload)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)

    runner = types.SimpleNamespace(
        assertions = [],
        failures = [],
        started = module.time.time() - 5.0,
        record = None,
    )
    record = module.Payload.record.__get__(runner, module.Payload)
    module.log = lambda *a, **k: None
    record("first", True, {})
    record("second", True, {"failures": []})

    names = [a["name"] for a in runner.assertions]
    assert names == ["first", "second"]
    # The first entry measures from process start, capturing setup time.
    assert runner.assertions[0]["seconds_since_previous"] >= 5.0
    assert runner.assertions[1]["seconds_since_previous"] < 1.0
    assert runner.assertions[1]["at_seconds"] >= runner.assertions[0]["at_seconds"]


class TestAMixedListingIsNotProofOfCpu:
    """A readable row on one GPU must not hide an [N/A] server, which may still be on the card."""

    @staticmethod
    def _verdict(**kwargs):
        return run_studio_gpu.cli_run_gpu_failure(**kwargs)

    def test_an_appeared_but_unattributed_pid_defers_to_the_device_delta(self):
        """On a card this run OWNS: nothing was there before, so the total is ours."""
        failure, detail = self._verdict(
            apps_before = {},
            apps_after = {},
            baseline = 1000.0,
            settled = 1600.0,
            listed_before = set(),
            listed_after = {222},
        )
        assert failure is None, (
            "the device grew by 600 MiB and pid 222 is listed but unattributed, which is "
            f"the offloaded server, not an absence: {detail}"
        )
        assert detail["compute_apps_unattributed"] == [222]

    def test_an_unattributed_pid_with_no_device_growth_still_fails(self):
        failure, detail = self._verdict(
            apps_before = {},
            apps_after = {},
            baseline = 1000.0,
            settled = 1010.0,
            listed_before = set(),
            listed_after = {222},
        )
        assert failure is not None and "served from the CPU" in failure
        assert "222" in failure, "the message names the process it could not attribute"

    def test_an_unattributed_pid_with_no_device_reading_is_unmeasured(self):
        failure, detail = self._verdict(
            apps_before = {},
            apps_after = {111: 500},
            baseline = None,
            settled = None,
            listed_before = set(),
            listed_after = {111, 222},
        )
        assert failure is not None and "unmeasured" in failure

    def test_an_attributed_new_process_is_unaffected(self):
        """The ordinary path must not start deferring to the shared device counter."""
        failure, detail = self._verdict(
            apps_before = {111: 500},
            apps_after = {111: 500, 222: 2600},
            baseline = 1000.0,
            settled = 900.0,
            listed_before = {111},
            listed_after = {111, 222},
        )
        assert failure is None, "a co-tenant freeing memory must not sink an attributed pid"
        assert detail["process_vram_mib"] == 2600

    def test_a_pid_already_present_but_unattributed_is_not_new(self):
        """Only a pid that APPEARED counts; a co-tenant showing [N/A] throughout is theirs."""
        failure, detail = self._verdict(
            apps_before = {111: 500},
            apps_after = {111: 500},
            baseline = 1000.0,
            settled = 1010.0,
            listed_before = {111, 999},
            listed_after = {111, 999},
        )
        assert "compute_apps_unattributed" not in detail
        assert failure is not None and "no process appeared" in failure

    def test_the_old_callers_still_work(self):
        """The listed sets are optional, so a caller that has none keeps the old verdict."""
        failure, _ = self._verdict(
            apps_before = {},
            apps_after = {222: 2600},
            baseline = None,
            settled = None,
        )
        assert failure is None


class TestTheListingNamesItsPids:
    def test_listed_pids_includes_unattributed_rows(self):
        csv_text = "111, 500\n222, [N/A]\n"
        assert gpu_assert.parse_compute_apps(csv_text) == {111: 500}
        assert gpu_assert.listed_pids(csv_text) == {111, 222}

    def test_listed_pids_ignores_headers_and_junk(self):
        assert gpu_assert.listed_pids("pid, used_gpu_memory\n\nnotapid, 5\n333, 10\n") == {333}

    def test_the_sample_is_taken_once(self):
        """The compute-apps listing is sampled once: two nvidia-smi calls would describe two
        moments, so the attributed mapping and listed pids could disagree."""
        body = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")
        assert body.count("nvidia_compute_apps_listing()") >= 3
        assert "apps_before = attributed_apps(_listing_before)" in body
        assert "apps_after = attributed_apps(_listing_after)" in body


class TestASharedCardIsNotMeasuredByItsTotal:
    """A shared card's device total cannot prove this run used the GPU, so the fallback is refused."""

    @staticmethod
    def _verdict(**kwargs):
        return run_studio_gpu.cli_run_gpu_failure(**kwargs)

    def test_a_co_tenant_blocks_the_no_enumeration_fallback(self):
        failure, detail = self._verdict(
            apps_before = {111: 500},
            apps_after = None,
            baseline = 1000.0,
            settled = 9000.0,
            listed_before = {111},
            listed_after = None,
        )
        assert failure is not None and "unmeasured rather than proven" in failure
        assert detail["card_shared_before_launch"] is True

    def test_a_co_tenant_blocks_the_mixed_listing_fallback(self):
        failure, detail = self._verdict(
            apps_before = {111: 500},
            apps_after = {111: 500},
            baseline = 1000.0,
            settled = 9000.0,
            listed_before = {111},
            listed_after = {111, 222},
        )
        assert failure is not None and "unmeasured rather than proven" in failure
        assert detail["card_shared_before_launch"] is True

    def test_a_listed_but_unattributed_co_tenant_counts(self):
        """A co-tenant reporting [N/A] is still a co-tenant."""
        failure, _ = self._verdict(
            apps_before = {},
            apps_after = None,
            baseline = 1000.0,
            settled = 9000.0,
            listed_before = {999},
            listed_after = None,
        )
        assert failure is not None and "unmeasured rather than proven" in failure

    def test_an_empty_card_keeps_the_fallback(self):
        """The fallback exists for parts that report [N/A] for everything; it stays."""
        failure, _ = self._verdict(
            apps_before = {},
            apps_after = None,
            baseline = 1000.0,
            settled = 1600.0,
            listed_before = set(),
            listed_after = None,
        )
        assert failure is None

    def test_attribution_still_wins_over_the_rule(self):
        """A co-tenant does not sink a run whose own process WAS attributed."""
        failure, detail = self._verdict(
            apps_before = {111: 500},
            apps_after = {111: 500, 222: 2600},
            baseline = 1000.0,
            settled = 900.0,
            listed_before = {111},
            listed_after = {111, 222},
        )
        assert failure is None
        assert detail["process_vram_mib"] == 2600

    def test_unknown_co_tenancy_is_not_invented(self):
        """No listing at all is no evidence either way, so behaviour is unchanged."""
        failure, _ = self._verdict(
            apps_before = None,
            apps_after = None,
            baseline = 1000.0,
            settled = 1600.0,
        )
        assert failure is None


class TestTheSamplesAreScopedToTheVisibleCard:
    """nvidia-smi ignores CUDA_VISIBLE_DEVICES, so samples must be filtered to the visible card."""

    @staticmethod
    def _selector(value):
        env = dict(os.environ)
        if value is None:
            env.pop("CUDA_VISIBLE_DEVICES", None)
        else:
            env["CUDA_VISIBLE_DEVICES"] = value
        with mock.patch.dict(os.environ, env, clear = True):
            return run_studio_gpu.visible_device_selector()

    def test_unset_selects_every_card(self):
        assert self._selector(None) is None

    def test_a_pinned_card_selects_only_it(self):
        assert self._selector("1") == "1"

    def test_a_list_is_carried_through(self):
        assert self._selector("0, 1") == "0,1"

    def test_a_uuid_needs_no_mapping(self):
        """`nvidia-smi -i` takes the GPU-UUID form directly."""
        assert self._selector("GPU-84ccface-663f") == "GPU-84ccface-663f"

    def test_an_empty_value_is_no_cards_not_all_of_them(self):
        assert self._selector("") == ""

    @staticmethod
    def _scoped(value):
        env = dict(os.environ)
        if value is None:
            env.pop("CUDA_VISIBLE_DEVICES", None)
        else:
            env["CUDA_VISIBLE_DEVICES"] = value
        with mock.patch.dict(os.environ, env, clear = True):
            return run_studio_gpu._scoped(["nvidia-smi", "--query-gpu=memory.used"])

    def test_the_pin_reaches_the_command(self):
        assert self._scoped("1") == ["nvidia-smi", "-i", "1", "--query-gpu=memory.used"]

    def test_no_pin_leaves_the_command_alone(self):
        assert self._scoped(None) == ["nvidia-smi", "--query-gpu=memory.used"]

    def test_no_visible_card_takes_no_sample(self):
        assert self._scoped("") is None

    def test_both_samplers_are_scoped(self):
        """The pair is the point: scoping one ruler and not the other still compares
        a card-local reading against a box-wide one."""
        seen = []

        def fake_run(cmd, **kwargs):
            seen.append(cmd)
            return subprocess.CompletedProcess(cmd, 0, "7, 100\n", "")

        with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "7"}):
            with mock.patch.object(run_studio_gpu, "run", fake_run):
                run_studio_gpu.nvidia_used_mib()
                run_studio_gpu.nvidia_compute_apps_listing()
        assert len(seen) == 2
        for cmd in seen:
            assert cmd[1:3] == ["-i", "7"], f"unscoped sample: {cmd}"

    def test_neither_sampler_runs_without_a_visible_card(self):
        def explode(cmd, **kwargs):
            raise AssertionError(f"sampled with no visible card: {cmd}")

        with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": ""}):
            with mock.patch.object(run_studio_gpu, "run", explode):
                assert run_studio_gpu.nvidia_used_mib() is None
                assert run_studio_gpu.nvidia_compute_apps_listing() is None


class TestTheServerIsStoppedBeforeTheCliBaselineRegardlessOfSkipUi:
    """Stop the server before the CLI baseline even with --skip-ui, or its pid reads as a co-tenant."""

    def test_stop_server_precedes_the_cli_assertion_unconditionally(self):
        text = Path(run_studio_gpu.__file__).read_text(encoding = "utf-8")
        ui = text.index("if not self.args.skip_ui:\n            self.assert_chat_ui()")
        cli = text.index("        self.assert_cli_run()", ui)
        between = text[ui:cli]
        stop = between.rindex("self.stop_server()")
        line_start = between.rfind("\n", 0, stop) + 1
        assert (
            between[line_start:stop] == "        "
        ), "stop_server must not sit inside the skip_ui guard"

    def test_the_card_is_settled_between_the_stop_and_the_assertion(self):
        """Stopping is not the same as having stopped. Source order, because the two calls
        are orchestration in run() and there is no seam between them to observe."""
        text = Path(run_studio_gpu.__file__).read_text(encoding = "utf-8")
        cli = text.index("        self.assert_cli_run()")
        between = text[
            text.index("if not self.args.skip_ui:\n            self.assert_chat_ui()") : cli
        ]
        assert between.rindex("self.stop_server()") < between.rindex("wait_for_card_to_settle()")


class TestTheCardIsGivenTimeToSettleAfterAStop:
    """A stopped llama-server stays listed briefly, so the card must be polled until it settles."""

    @staticmethod
    def _settle(samples):
        """Drive wait_for_card_to_settle over `samples` of (used_mib, listed pids), the
        last one repeating. (None, None) is an nvidia-smi that did not answer. Returns the
        number of polls taken: two nvidia-smi calls each, the entry sample included."""
        taken = []

        def fake_run(cmd, **kwargs):
            used, pids = samples[min(len(taken) // 2, len(samples) - 1)]
            taken.append(cmd)
            if pids is None:
                return subprocess.CompletedProcess(cmd, 9, "", "no devices were found")
            if "--query-compute-apps=pid,used_gpu_memory" in cmd:
                # The [N/A] shape: every pid listed, none attributed.
                return subprocess.CompletedProcess(
                    cmd, 0, "".join(f"{p}, [N/A]\n" for p in pids), ""
                )
            return subprocess.CompletedProcess(cmd, 0, f"{used}\n", "")

        with mock.patch.object(run_studio_gpu, "run", fake_run):
            with mock.patch.object(run_studio_gpu.time, "sleep", lambda _s: None):
                run_studio_gpu.wait_for_card_to_settle()
        return len(taken) // 2

    def test_it_waits_for_the_stopped_server_to_leave_the_listing(self):
        """Two equal samples are not proof of a settled card when the driver has not begun
        giving the memory back, which is why the pid is what is waited on."""
        assert self._settle([(3400.0, [4242]), (3400.0, [4242]), (200.0, []), (200.0, [])]) == 5

    def test_it_waits_for_the_memory_to_come_back(self):
        """The pid can go while the driver is still returning the allocation."""
        assert self._settle([(3400.0, []), (1800.0, []), (200.0, []), (200.0, [])]) == 5

    def test_a_stalled_reclaim_with_no_pid_to_wait_on_is_not_settled(self):
        """The pid can also be gone BEFORE the first sample, leaving nothing to wait on. One
        flat interval is then a stall as easily as a finished reclaim, and taking it as
        settled hands assert_cli_run a baseline with the old model still in it."""
        assert self._settle([(3400.0, []), (3400.0, []), (200.0, []), (200.0, [])]) == 5

    def test_a_settled_card_costs_only_the_quiet_window(self):
        assert self._settle([(200.0, [])]) == run_studio_gpu.VRAM_SETTLE_QUIET_POLLS + 1

    def test_a_co_tenant_arriving_afterwards_does_not_hold_the_run(self):
        """Only the pids the card carried at entry are the stop's to wait for."""
        assert self._settle([(200.0, []), (2600.0, [777]), (2600.0, [777])]) == 3

    def test_an_unanswering_smi_is_not_waited_on(self):
        assert self._settle([(None, None)]) == 3

    def test_the_wait_is_bounded(self):
        """A co-tenant never leaves, and a card somebody else is draining never settles."""
        assert (
            self._settle([(3400.0 - 100.0 * i, [4242]) for i in range(60)])
            == run_studio_gpu.VRAM_SETTLE_SAMPLES + 1
        )

    def test_the_settled_listing_no_longer_reads_as_a_co_tenant(self):
        """The point of the wait, stated as the verdict it changes."""
        stale = run_studio_gpu.cli_run_gpu_failure(
            {},
            None,
            3400.0,
            3900.0,
            {4242},
            {4242, 5555},
        )[0]
        assert stale is not None and "already on the card" in stale
        settled = run_studio_gpu.cli_run_gpu_failure(
            {},
            None,
            200.0,
            700.0,
            set(),
            {5555},
        )[0]
        assert settled is None


class TestTheTwoRulersAreReadInTheRightOrder:
    """Per-process readings outrank the device total, which must never overrule an attributed pid."""

    @staticmethod
    def _verdict(**kwargs):
        return run_studio_gpu.cli_run_gpu_failure(**kwargs)

    def test_a_flat_device_total_does_not_disprove_an_attributed_process(self):
        """A flat device total cannot disprove an attributed pid, since the total is a shared counter."""
        failure, detail = self._verdict(
            apps_before = {},
            apps_after = {222: 2600},
            baseline = 1000.0,
            settled = 1050.0,
            listed_before = set(),
            listed_after = {222, 333},
        )
        assert failure is not None
        assert "served from the CPU" not in failure, failure
        assert "unmeasured rather than disproven" in failure
        assert detail["process_vram_mib"] == 2600
        assert detail["compute_apps_unattributed"] == [333]

    def test_a_co_tenant_that_becomes_readable_is_still_not_new(self):
        """pid 77 was on the card at launch and merely unattributable then. Keying the
        exclusion on the attributed map alone counted its 500 MiB as this launch's and
        passed a server that never left the CPU."""
        failure, detail = self._verdict(
            apps_before = {},
            apps_after = {77: 500},
            baseline = 1000.0,
            settled = 1000.0,
            listed_before = {77},
            listed_after = {77},
        )
        assert failure is not None and "no process appeared" in failure
        assert detail["compute_apps_appeared"] == {}

    def test_the_same_co_tenant_does_not_carry_the_mixed_listing_either(self):
        failure, detail = self._verdict(
            apps_before = {},
            apps_after = {77: 500},
            baseline = 1000.0,
            settled = 1000.0,
            listed_before = {77},
            listed_after = {77, 888},
        )
        assert failure is not None and "unmeasured rather than proven" in failure
        assert detail["card_shared_before_launch"] is True

    def test_an_attributed_launch_on_an_empty_card_still_passes(self):
        """The ordinary verdict is untouched: the exclusion only ever grows."""
        failure, detail = self._verdict(
            apps_before = {},
            apps_after = {222: 2600},
            baseline = 1000.0,
            settled = 3600.0,
            listed_before = set(),
            listed_after = {222},
        )
        assert failure is None
        assert detail["process_vram_mib"] == 2600