Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

26.3 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Cached, coalesced nvidia-smi reads, driven by a real fake ``nvidia-smi`` on PATH."""

from __future__ import annotations

import json
import os
import subprocess
import sys
import threading
import time
from pathlib import Path

import pytest

from core.inference.llama_cpp import LlamaCppBackend
from utils import gpu_memory_events
from utils.hardware import gpu_query, nvidia

_FAKE = r"""#!{python}
import json, os, sys, time
state = json.load(open({state!r}))
with open({calls!r}, "a") as f:
    f.write(" ".join(sys.argv[1:]) + "\n")
time.sleep(state.get("delay", 0))
if state.get("exit"):
    sys.stderr.write("NVIDIA-SMI has failed\n")
    sys.exit(state["exit"])
args = sys.argv[1:]
gpus = state["gpus"]
if args[:1] == ["-L"]:
    for i, g in enumerate(gpus):
        print(f"GPU {{i}}: {{g['name']}} (UUID: {{g['uuid']}})")
    sys.exit(0)
if args[:1] == ["topo"]:
    print("\tGPU0\tGPU1\nGPU0\t X \tNV18\nGPU1\tNV18\t X \n\nLegend:\n")
    sys.exit(0)
fields = [a.split("=", 1)[1].split(",") for a in args if a.startswith("--query-gpu=")][0]
nounits = any("nounits" in a for a in args if a.startswith("--format="))
for i, g in enumerate(gpus):
    total = g["total"]
    free = g["free"]
    vals = {{
        "index": str(i), "uuid": g["uuid"], "name": g["name"], "compute_cap": "10.0",
        "memory.total": str(total), "memory.free": str(free), "memory.used": str(total - free),
        "utilization.gpu": str(g.get("util", 0)), "temperature.gpu": "40",
        "power.draw": "100.0", "power.limit": "1000.0",
    }}
    out = []
    for name in fields:
        v = vals[name]
        if not nounits and name.startswith("memory."):
            v += " MiB"
        out.append(v)
    print(", ".join(out))
"""


class FakeSmi:
    def __init__(self, root: Path):
        self.root = root
        self.state_path = root / "state.json"
        self.calls_path = root / "calls.log"
        self.calls_path.write_text("")
        self.state = {
            "delay": 0,
            "exit": 0,
            "gpus": [
                {"name": "NVIDIA B200", "uuid": "GPU-aaaa", "total": 183359, "free": 180000},
                {"name": "NVIDIA B200", "uuid": "GPU-bbbb", "total": 183359, "free": 170000},
            ],
        }
        self.save()
        exe = root / "nvidia-smi"
        exe.write_text(
            _FAKE.format(
                python = sys.executable, state = str(self.state_path), calls = str(self.calls_path)
            )
        )
        exe.chmod(0o755)

    def save(self):
        self.state_path.write_text(json.dumps(self.state))

    def set(self, **kw):
        self.state.update(kw)
        self.save()

    def set_free(self, *free_mib):
        for gpu, free in zip(self.state["gpus"], free_mib):
            gpu["free"] = free
        self.save()

    def calls(self, needle: str = "") -> int:
        return sum(1 for line in self.calls_path.read_text().splitlines() if needle in line)

    def wait_for_call(
        self,
        needle: str,
        count: int,
        timeout: float = 30.0,
    ) -> None:
        """Wait for ``count`` matching children; each reads its state before logging, unlike a fixed sleep."""
        deadline = time.monotonic() + timeout
        while self.calls(needle) < count:
            assert time.monotonic() < deadline, f"no {needle!r} child started within {timeout}s"
            time.sleep(0.01)


@pytest.fixture
def smi(tmp_path, monkeypatch):
    fake = FakeSmi(tmp_path)
    monkeypatch.setenv("PATH", f"{tmp_path}{os.pathsep}{os.environ.get('PATH', '')}")
    monkeypatch.delenv("UNSLOTH_GPU_QUERY_CACHE", raising = False)
    monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)
    # No NVML child unless asked: it would read this host's real driver.
    monkeypatch.setenv("UNSLOTH_NVIDIA_LIBRARY_PROBE", "0")
    monkeypatch.setattr(nvidia, "_nvidia_smi_executable", lambda: "nvidia-smi")
    gpu_query.reset()
    yield fake
    gpu_query.reset()


@pytest.fixture
def llama_probe(monkeypatch):
    monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: False))
    monkeypatch.setattr(
        LlamaCppBackend, "_find_llama_server_binary", staticmethod(lambda: "/opt/llama-server")
    )
    monkeypatch.setattr(LlamaCppBackend, "_get_gpu_memory_nvml", staticmethod(lambda: []))
    monkeypatch.setattr(
        LlamaCppBackend, "_get_gpu_memory_amd_smi", staticmethod(lambda *a, **k: [])
    )
    monkeypatch.setattr(
        LlamaCppBackend, "_widen_integrated_cuda_rows", staticmethod(lambda rows: rows)
    )
    monkeypatch.setitem(sys.modules, "torch", None)
    return LlamaCppBackend._get_gpu_memory


def _free_by_index(result):
    return {d["index"]: round(d["vram_total_gb"] - d["vram_used_gb"], 2) for d in result["devices"]}


def test_classification():
    assert gpu_query.classify(["nvidia-smi", "-L"]) == gpu_query.STATIC
    assert gpu_query.classify(["nvidia-smi", "topo", "-m"]) == gpu_query.STATIC
    assert (
        gpu_query.classify(["nvidia-smi", "--query-gpu=index,name,memory.total", "--format=csv"])
        == gpu_query.STATIC
    )
    assert (
        gpu_query.classify(["nvidia-smi", "--query-gpu=index,compute_cap", "--format=csv"])
        == gpu_query.STATIC
    )
    live = ["nvidia-smi", "--query-gpu=index,memory.free,memory.total", "--format=csv"]
    assert gpu_query.classify(live) == gpu_query.CRITICAL
    with gpu_query.display_reads():
        assert gpu_query.classify(live) == gpu_query.DISPLAY
    assert gpu_query.classify(["nvidia-smi", "-q"]) == gpu_query.CRITICAL


def test_concurrent_identical_display_queries_start_one_child(smi):
    smi.set(delay = 0.6)
    results, errors = [], []

    def worker():
        try:
            with gpu_query.display_reads():
                results.append(nvidia.get_visible_gpu_utilization([0, 1]))
        except Exception as e:  # pragma: no cover - surfaced below
            errors.append(e)

    threads = [threading.Thread(target = worker) for _ in range(16)]
    for t in threads:
        t.start()
    for t in threads:
        t.join(10)
    assert not errors
    assert len(results) == 16
    assert all(r == results[0] for r in results)
    assert results[0]["available"] is True
    assert smi.calls("--query-gpu") == 1
    assert gpu_query.stats()["coalesced"] >= 1


def test_concurrent_fit_checks_each_run_the_cli(smi, llama_probe):
    smi.set(delay = 0.3)
    threads = [threading.Thread(target = llama_probe) for _ in range(4)]
    for t in threads:
        t.start()
    for t in threads:
        t.join(10)
    assert smi.calls("memory.free") == 4


def test_concurrent_fit_checks_never_answer_from_before_they_started(smi, llama_probe):
    lock = threading.Lock()
    current = [1000]
    stop = threading.Event()

    def writer():
        while not stop.is_set():
            with lock:
                current[0] += 1
                for gpu in smi.state["gpus"]:
                    gpu["free"] = current[0]
                tmp = smi.state_path.with_suffix(".tmp")
                tmp.write_text(json.dumps(smi.state))
                os.replace(tmp, smi.state_path)
            time.sleep(0.01)

    violations, errors = [], []

    def reader():
        for _ in range(5):
            with lock:
                floor = current[0]
            try:
                rows = llama_probe()
            except Exception as e:  # pragma: no cover - surfaced below
                errors.append(e)
                continue
            if not rows or rows[0][1] < floor:
                violations.append((floor, rows))

    w = threading.Thread(target = writer)
    w.start()
    readers = [threading.Thread(target = reader) for _ in range(16)]
    for t in readers:
        t.start()
    for t in readers:
        t.join(60)
    stop.set()
    w.join(5)
    assert not errors
    assert violations == []
    assert smi.calls("memory.free") == 80


def test_different_queries_are_not_merged(smi):
    nvidia.get_visible_gpu_utilization([0, 1])
    nvidia.get_primary_gpu_utilization()
    assert smi.calls("--query-gpu") == 2


def test_fit_checks_always_read_afresh(smi, llama_probe):
    """Another process's allocation raises no Studio event, so the next fit check must see it."""
    assert llama_probe() == [(0, 180000, 183359), (1, 170000, 183359)]
    smi.set_free(1000, 1000)
    assert llama_probe() == [(0, 1000, 183359), (1, 1000, 183359)]
    assert smi.calls("memory.free") == 2


def test_display_reads_are_served_stale_while_one_child_refreshes(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", "0.2")
    with gpu_query.display_reads():
        first = _free_by_index(nvidia.get_visible_gpu_utilization([0, 1]))
    smi.set(delay = 0.5)
    smi.set_free(2048, 2048)
    time.sleep(0.3)
    t0 = time.monotonic()
    with gpu_query.display_reads():
        stale = _free_by_index(nvidia.get_visible_gpu_utilization([0, 1]))
    assert time.monotonic() - t0 < 0.3
    assert stale == first
    deadline = time.monotonic() + 5
    while gpu_query.stats()["inflight"] and time.monotonic() < deadline:
        time.sleep(0.05)
    with gpu_query.display_reads():
        refreshed = _free_by_index(nvidia.get_visible_gpu_utilization([0, 1]))
    assert refreshed == {0: 2.0, 1: 2.0}


def test_display_ttl_does_not_leak_into_fit_checks(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", "60")
    with gpu_query.display_reads():
        nvidia.get_visible_gpu_utilization([0, 1])
    smi.set_free(512, 512)
    fresh = _free_by_index(nvidia.get_visible_gpu_utilization([0, 1]))
    assert fresh == {0: 0.5, 1: 0.5}


def test_static_inventory_is_cached_until_redetection(smi):
    first = nvidia.get_physical_gpu_inventory()
    assert [d["name"] for d in first["devices"]] == ["NVIDIA B200", "NVIDIA B200"]
    assert nvidia.get_physical_gpu_count() == 2
    for _ in range(5):
        nvidia.get_physical_gpu_inventory()
        nvidia.get_physical_gpu_count()
        LlamaCppBackend._cuda_compute_caps()
        LlamaCppBackend._probe_nvlink_topology()
    assert smi.calls("memory.total") == 1
    assert smi.calls("-L") == 1
    assert smi.calls("compute_cap") == 1
    assert smi.calls("topo") == 5
    gpu_memory_events.invalidate_gpu_memory("load")
    nvidia.get_physical_gpu_inventory()
    assert smi.calls("memory.total") == 1
    gpu_query.invalidate_static("redetect")
    nvidia.get_physical_gpu_inventory()
    assert smi.calls("memory.total") == 2


def test_an_answered_empty_inventory_replaces_the_cached_one(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_STATIC_TTL", "0")
    assert nvidia.get_physical_gpu_count() == 2
    smi.set(gpus = [])
    counts = []
    for _ in range(20):
        counts.append(nvidia.get_physical_gpu_count())
        if counts[-1] in (0, None):
            break
        time.sleep(0.1)
    assert counts[-1] in (0, None), counts


def test_a_static_read_after_redetection_does_not_join_an_older_child(smi):
    smi.set(delay = 0.6)
    first = threading.Thread(target = nvidia.get_physical_gpu_count)
    started = smi.calls("-L")
    first.start()
    smi.wait_for_call("-L", started + 1)
    smi.state["gpus"] = smi.state["gpus"][:1]
    smi.save()
    gpu_query.invalidate_static("redetect")
    assert nvidia.get_physical_gpu_count() == 1
    first.join(5)
    assert smi.calls("-L") == 2


def test_display_after_a_load_reads_afresh_inside_the_ttl(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", "3600")
    with gpu_query.display_reads():
        nvidia.get_visible_gpu_utilization([0, 1])

    @gpu_memory_events.invalidates_gpu_memory("test load")
    def load_model():
        smi.set_free(1024, 1024)

    load_model()
    with gpu_query.display_reads():
        after = _free_by_index(nvidia.get_visible_gpu_utilization([0, 1]))
    assert after == {0: 1.0, 1: 1.0}
    assert smi.calls("--query-gpu") == 2


def test_studio_load_unload_and_training_methods_invalidate():
    from core.inference import diffusion, inference, orchestrator, video
    from core.inference import native_audio, sd_cpp_backend
    from core.rag import embed_llama_server
    from core.inference import stt_registry
    from core.training import diffusion_training_service, training

    methods = [
        LlamaCppBackend.load_model,
        LlamaCppBackend.unload_model,
        LlamaCppBackend._kill_process,
        orchestrator.InferenceOrchestrator.load_model,
        orchestrator.InferenceOrchestrator.unload_model,
        training.TrainingBackend.start_training,
        training.TrainingBackend.stop_training,
        diffusion.DiffusionBackend.load_pipeline,
        diffusion.DiffusionBackend.unload,
        video.VideoBackend.load_pipeline,
        video.VideoBackend.unload,
        sd_cpp_backend.SdCppDiffusionBackend._run_load,
        sd_cpp_backend.SdCppDiffusionBackend.unload,
        embed_llama_server.LlamaServerBackend._spawn,
        embed_llama_server.LlamaServerBackend._kill_process,
        inference.InferenceBackend.load_model,
        inference.InferenceBackend.unload_model,
        native_audio.NativeAudioBackend.load_model,
        native_audio.NativeAudioBackend.unload_model,
        stt_registry.load,
        stt_registry.unload,
        diffusion_training_service.DiffusionTrainingService.start,
        diffusion_training_service.DiffusionTrainingService.stop,
    ]
    for method in methods:
        code = method.__code__
        assert (
            code is gpu_memory_events.invalidates_gpu_memory("x")(lambda: None).__code__
        ), method.__qualname__


def test_invalidation_happens_even_when_the_load_fails():
    before = gpu_memory_events.generation()

    @gpu_memory_events.invalidates_gpu_memory("failing load")
    def load():
        raise RuntimeError("boom")

    with pytest.raises(RuntimeError):
        load()
    assert gpu_memory_events.generation() == before + 2


def test_a_child_started_before_an_invalidation_is_not_joined(smi, llama_probe):
    """A display read arriving after a load must not share a CLI call that began before it."""
    smi.set(delay = 0.5)
    first_result = []

    def display_probe():
        with gpu_query.display_reads():
            return llama_probe()

    t = threading.Thread(target = lambda: first_result.append(display_probe()))
    started = smi.calls("memory.free")
    t.start()
    smi.wait_for_call("memory.free", started + 1)
    smi.set_free(64, 64)
    gpu_memory_events.invalidate_gpu_memory("load")
    second = display_probe()
    t.join(5)
    assert first_result == [[(0, 180000, 183359), (1, 170000, 183359)]]
    assert second == [(0, 64, 183359), (1, 64, 183359)]
    assert smi.calls("memory.free") == 2


def test_fresh_reads_always_run_the_cli(smi):
    for _ in range(3):
        with gpu_query.fresh_reads():
            nvidia.get_visible_gpu_utilization([0, 1])
    assert smi.calls("--query-gpu") == 3


def test_a_fit_check_waits_its_whole_timeout_on_a_slow_driver(smi):
    smi.set(delay = 1.5)
    argv = ["nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader,nounits"]
    with pytest.raises(subprocess.TimeoutExpired):
        gpu_query.run_nvidia_smi(argv, timeout = 0.5, capture_output = True, text = True)
    assert gpu_query.driver_slow()
    out = gpu_query.run_nvidia_smi(argv, timeout = 5, capture_output = True, text = True)
    assert out.stdout.split() == ["0,", "180000", "1,", "170000"]


def test_a_slow_cli_holds_the_caller_only_for_its_timeout(smi, monkeypatch):
    monkeypatch.setattr(gpu_query, "_background_timeout", lambda: 3.0)
    smi.set(delay = 2.0)
    t0 = time.monotonic()
    with pytest.raises(subprocess.TimeoutExpired):
        gpu_query.run_nvidia_smi(
            ["nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader,nounits"],
            timeout = 0.3,
            capture_output = True,
            text = True,
        )
    assert time.monotonic() - t0 < 1.0
    assert gpu_query.driver_slow()


def test_a_child_the_kernel_cannot_reap_does_not_hold_the_caller(monkeypatch):
    gpu_query.reset()
    monkeypatch.delenv("UNSLOTH_GPU_QUERY_CACHE", raising = False)
    monkeypatch.setenv("UNSLOTH_NVIDIA_LIBRARY_PROBE", "0")
    calls = []
    # Held until the test is done rather than for a flat 12 s: the worker stays stuck for as long
    # as the caller is being measured, and teardown does not then wait the sleep out.
    released = threading.Event()

    def unreapable(argv, **kwargs):
        calls.append(argv)
        released.wait(12.0)
        raise subprocess.TimeoutExpired(argv, kwargs.get("timeout"))

    monkeypatch.setattr(subprocess, "run", unreapable)
    # The caller's 5 s budget shrunk to 0.5 s: the bound under test is "timeout + 1 s reap grace,
    # never the child's 12 s", and it holds at any timeout, so the test need not spend 2 x 6 s.
    monkeypatch.setattr(gpu_query, "run_nvidia_smi", _with_timeout(gpu_query.run_nvidia_smi, 0.5))
    try:
        for _ in range(2):
            t0 = time.monotonic()
            out = nvidia.get_visible_gpu_utilization([0, 1])
            assert out["available"] is False
            assert time.monotonic() - t0 < 3.0
        assert len(calls) == 2
    finally:
        released.set()
        gpu_query.reset()


def test_slow_cli_answers_a_display_read_from_the_last_good_value(smi, monkeypatch):
    with gpu_query.display_reads():
        good = nvidia.get_visible_gpu_utilization([0, 1])
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", "0")
    monkeypatch.setattr(gpu_query, "_background_timeout", lambda: 4.0)
    smi.set(delay = 12.0)
    monkeypatch.setattr(gpu_query, "run_nvidia_smi", _with_timeout(gpu_query.run_nvidia_smi, 0.5))
    t0 = time.monotonic()
    with gpu_query.display_reads():
        out = nvidia.get_visible_gpu_utilization([0, 1])
    assert time.monotonic() - t0 < 2.0
    assert out == good


def test_slow_cli_never_shows_a_pre_load_reading_after_a_load(smi, monkeypatch):
    with gpu_query.display_reads():
        nvidia.get_visible_gpu_utilization([0, 1])
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", "0")
    gpu_memory_events.invalidate_gpu_memory("load")
    monkeypatch.setattr(gpu_query, "_background_timeout", lambda: 4.0)
    smi.set(delay = 12.0)
    monkeypatch.setattr(gpu_query, "run_nvidia_smi", _with_timeout(gpu_query.run_nvidia_smi, 0.5))
    t0 = time.monotonic()
    with gpu_query.display_reads():
        out = nvidia.get_visible_gpu_utilization([0, 1])
    assert time.monotonic() - t0 < 2.0
    assert out["available"] is False


def test_a_failed_answer_replaces_the_cached_inventory(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_STATIC_TTL", "0")
    assert nvidia.get_physical_gpu_count() == 2
    smi.set(exit = 6)  # nvidia-smi with no GPU: exit 6
    counts = []
    for _ in range(20):
        counts.append(nvidia.get_physical_gpu_count())
        if counts[-1] is None:
            break
        time.sleep(0.1)
    assert counts[-1] is None, counts


def test_an_expired_inventory_is_not_served_after_a_card_goes_away(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_STATIC_TTL", "0.2")
    assert nvidia.get_physical_gpu_count() == 2
    time.sleep(0.5)
    smi.state["gpus"] = smi.state["gpus"][:1]
    smi.save()
    assert nvidia.get_physical_gpu_count() == 1


def test_a_check_true_failure_replaces_the_cached_answer(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", "0")
    argv = ["nvidia-smi", "--query-gpu=index,memory.used", "--format=csv,noheader"]

    def run():
        with gpu_query.display_reads():
            return gpu_query.run_nvidia_smi(
                argv, capture_output = True, text = True, timeout = 5, check = True
            )

    run()
    smi.set(exit = 6)
    for _ in range(20):
        try:
            run()
        except subprocess.CalledProcessError:
            break
        time.sleep(0.1)
    with pytest.raises(subprocess.CalledProcessError):
        run()


@pytest.mark.parametrize("code, failed", [(6, False), (9, True)])
def test_no_devices_found_is_an_answer_not_a_failure(smi, code, failed):
    assert nvidia.get_backend_visible_gpu_info([0, 1], "0,1")["available"]
    smi.set(exit = code)  # 6: No devices were found; 9: driver not loaded
    gpu_query.invalidate_static("redetect")
    out = nvidia.get_backend_visible_gpu_info([0, 1], "0,1")
    assert out["available"] is False
    assert bool(out.get("probe_failed")) is failed


def test_a_read_after_a_hung_child_does_not_wait_on_it(smi, monkeypatch):
    argv = ["nvidia-smi", "--query-gpu=index,compute_cap", "--format=csv,noheader"]

    def run(timeout):
        return gpu_query.run_nvidia_smi(argv, capture_output = True, text = True, timeout = timeout)

    monkeypatch.setattr(gpu_query, "_background_timeout", lambda: 10.0)
    smi.set(delay = 8.0)
    with pytest.raises(subprocess.TimeoutExpired):
        run(0.5)
    smi.set(delay = 0)
    t0 = time.monotonic()
    assert run(0.5).stdout.split()[0] == "0,"
    assert time.monotonic() - t0 < 1.0


def test_display_reads_without_stale_wait_for_a_new_reading(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", "0.2")
    with gpu_query.display_reads(max_stale = 0):
        first = _free_by_index(nvidia.get_visible_gpu_utilization([0, 1]))
    time.sleep(0.4)
    smi.set_free(1000, 1000)
    with gpu_query.display_reads(max_stale = 0):
        after = _free_by_index(nvidia.get_visible_gpu_utilization([0, 1]))
    assert after != first
    assert smi.calls("--query-gpu") == 2


def test_display_reads_without_stale_still_survive_a_hung_cli(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", "0")
    with gpu_query.display_reads(max_stale = 0):
        good = nvidia.get_visible_gpu_utilization([0, 1])
    monkeypatch.setattr(gpu_query, "_background_timeout", lambda: 4.0)
    smi.set(delay = 12.0)
    monkeypatch.setattr(gpu_query, "run_nvidia_smi", _with_timeout(gpu_query.run_nvidia_smi, 0.5))
    t0 = time.monotonic()
    with gpu_query.display_reads(max_stale = 0):
        out = nvidia.get_visible_gpu_utilization([0, 1])
    assert time.monotonic() - t0 < 2.0
    assert out == good


def test_an_older_answer_does_not_overwrite_a_newer_empty_one(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_DISPLAY_TTL", "3600")
    argv = ["nvidia-smi", "--query-gpu=index,memory.used", "--format=csv,noheader"]

    def run():
        return gpu_query.run_nvidia_smi(argv, capture_output = True, text = True, timeout = 5)

    smi.set(delay = 1.0)
    older = threading.Thread(target = run)
    started = smi.calls("--query-gpu")
    older.start()
    smi.wait_for_call("--query-gpu", started + 1)
    smi.set(delay = 0, gpus = [])
    assert run().stdout == ""
    older.join(5)
    with gpu_query.display_reads():
        assert run().stdout == ""


def test_slow_cli_never_answers_a_fit_check_from_an_old_reading(smi, llama_probe, monkeypatch):
    """Another process can allocate between readings, so no earlier sample may stand in."""
    monkeypatch.setattr(gpu_query, "_background_timeout", lambda: 4.0)
    assert llama_probe() == [(0, 180000, 183359), (1, 170000, 183359)]
    smi.set_free(1000, 1000)
    smi.set(delay = 12.0)
    monkeypatch.setattr(gpu_query, "run_nvidia_smi", _with_timeout(gpu_query.run_nvidia_smi, 0.5))
    assert llama_probe() == []
    assert gpu_query.stats()["stale_served"] == 0


def test_a_fit_check_does_not_join_an_older_child(smi, monkeypatch):
    monkeypatch.setattr(gpu_query, "_background_timeout", lambda: 4.0)
    smi.set(delay = 3.0)
    argv = ["nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader,nounits"]
    kw = dict(timeout = 5, capture_output = True, text = True)
    first = threading.Thread(target = lambda: gpu_query.run_nvidia_smi(argv, **kw))
    started = smi.calls("--query-gpu=index,memory.free")
    first.start()
    smi.wait_for_call("--query-gpu=index,memory.free", started + 1)
    smi.set(delay = 0)
    smi.set_free(1000, 1000)
    out = gpu_query.run_nvidia_smi(argv, **kw)
    assert out.stdout.split() == ["0,", "1000", "1,", "1000"]
    first.join()
    assert smi.calls("--query-gpu=index,memory.free") == 2


def test_no_stale_fit_answer_across_a_load(smi, llama_probe, monkeypatch):
    monkeypatch.setattr(gpu_query, "_background_timeout", lambda: 4.0)
    llama_probe()
    gpu_memory_events.invalidate_gpu_memory("load")
    smi.set(delay = 12.0)
    monkeypatch.setattr(gpu_query, "run_nvidia_smi", _with_timeout(gpu_query.run_nvidia_smi, 0.5))
    assert llama_probe() == []


def test_a_hung_cli_leaves_llama_cpp_its_own_mig_aware_fallback(smi, llama_probe, monkeypatch):
    monkeypatch.setattr(gpu_query, "_background_timeout", lambda: 4.0)
    slice_rows = [(0, 5000, 10240)]
    monkeypatch.setattr(LlamaCppBackend, "_get_gpu_memory_nvml", staticmethod(lambda: slice_rows))
    monkeypatch.setenv("UNSLOTH_NVIDIA_LIBRARY_PROBE", "1")
    whole_gpu = [{"index": 0, "memory_total_mib": 81920, "memory_free_mib": 70000}]
    monkeypatch.setattr(gpu_query, "_nvml_rows", lambda timeout: whole_gpu, raising = False)
    smi.set(delay = 12.0)
    monkeypatch.setattr(gpu_query, "run_nvidia_smi", _with_timeout(gpu_query.run_nvidia_smi, 0.5))
    assert llama_probe() == slice_rows


def _with_timeout(fn, timeout):
    def wrapped(argv, **kwargs):
        kwargs["timeout"] = timeout
        return fn(argv, **kwargs)

    return wrapped


def test_failing_cli_is_not_cached(smi):
    smi.set(exit = 9)
    for _ in range(3):
        assert nvidia.get_visible_gpu_utilization([0, 1])["available"] is False
    assert smi.calls("--query-gpu") == 3
    smi.set(exit = 0)
    assert nvidia.get_visible_gpu_utilization([0, 1])["available"] is True


def test_missing_cli_is_reported_absent(monkeypatch, tmp_path):
    monkeypatch.setenv("PATH", str(tmp_path))
    monkeypatch.setattr(nvidia, "_nvidia_smi_executable", lambda: "nvidia-smi")
    gpu_query.reset()
    assert nvidia._query_gpu_inventory("test") is nvidia.NVIDIA_SMI_ABSENT
    assert nvidia.get_physical_gpu_count() is None


def test_cache_can_be_turned_off(smi, monkeypatch):
    monkeypatch.setenv("UNSLOTH_GPU_QUERY_CACHE", "0")
    for _ in range(3):
        nvidia.get_visible_gpu_utilization([0, 1])
        nvidia.get_physical_gpu_inventory()
    assert smi.calls("--query-gpu") == 6


def test_results_are_copies(smi):
    argv = ["nvidia-smi", "--query-gpu=index,memory.free", "--format=csv,noheader,nounits"]
    a = gpu_query.run_nvidia_smi(argv, timeout = 5, capture_output = True, text = True)
    a.stdout = "mutated"
    b = gpu_query.run_nvidia_smi(argv, timeout = 5, capture_output = True, text = True)
    assert b.stdout.startswith("0, 180000")