Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Kaggle GPU harness: an all-[N/A] listing is cannot attribute, not no GPU (#10472)
Daniel Han19ca5928dcef
diff --git a/tests/kaggle/studio_gpu/gpu_assert.py b/tests/kaggle/studio_gpu/gpu_assert.pyindex aaccb534526..1cde39515d6 100644--- a/tests/kaggle/studio_gpu/gpu_assert.py+++ b/tests/kaggle/studio_gpu/gpu_assert.py@@ -106,6 +106,58 @@ def parse_compute_apps(csv_text: str) -> dict[int, int]:return apps+def count_listed_pids(csv_text: str) -> int:+ """How many processes ``--query-compute-apps`` LISTED, whether or not it could say+ how much memory each holds.++ On Windows (WDDM) and on unified-memory parts such as the GB10, nvidia-smi lists+ every CUDA process but reports ``[N/A]`` for all of them, and it lists a+ ``-ngl 0`` server too, since a CUDA build creates a context regardless. So a+ listing with no readable figure is not "nothing on the card" and not proof of+ offload either: it is "cannot attribute", and the caller falls back to the+ device-wide delta, which does move on those parts (measured on a GB10: +140 MiB+ for a bare context, +428 MiB with a 270M model offloaded)."""+ n = 0+ for line in csv_text.splitlines():+ line = line.strip()+ if not line or line.lower().startswith("pid"):+ continue+ parts = [p.strip() for p in line.split(",")]+ if not parts:+ continue+ try:+ int(parts[0])+ except ValueError:+ continue+ n += 1+ return n+++def listed_pids(csv_text: str) -> set[int]:+ """Every pid ``--query-compute-apps`` LISTED, whether or not it carried a figure.++ count_listed_pids answers "how many", which is enough to tell an empty listing from+ an all-``[N/A]`` one. It is not enough for a MIXED listing: a readable row on one GPU+ and the newly launched server as ``[N/A]`` on another leaves the parsed mapping+ nonempty, so the caller saw a normal result and the server's pid simply was not in+ it. Naming the pids lets the caller notice that the process it cares about is the+ unattributed one, rather than concluding it never reached the card.+ """+ pids: set[int] = set()+ for line in csv_text.splitlines():+ line = line.strip()+ if not line or line.lower().startswith("pid"):+ continue+ parts = [p.strip() for p in line.split(",")]+ if not parts:+ continue+ try:+ pids.add(int(parts[0]))+ except ValueError:+ continue+ return pids++def offloaded_layers(log_text: str) -> tuple[int, int] | None:"""The last ``offloaded N/M layers to GPU`` in a llama.cpp log.diff --git a/tests/kaggle/studio_gpu/run_studio_gpu.py b/tests/kaggle/studio_gpu/run_studio_gpu.pyindex 45dd0190801..16de1964c3d 100644--- a/tests/kaggle/studio_gpu/run_studio_gpu.py+++ b/tests/kaggle/studio_gpu/run_studio_gpu.py@@ -86,6 +86,7 @@ from gpu_assert import ( # noqa: E402is_cuda_install,offload_verdict,parse_compute_apps,+ listed_pids,)from studio_client import ( # noqa: E402Studio,@@ -134,6 +135,10 @@ VRAM_SETTLE_POLL_S = 2.5VRAM_SETTLE_SAMPLES = 12# Driver readings jitter by a few MiB between samples; only a real drop counts.VRAM_SETTLE_TOLERANCE_MIB = 16.0+# Intervals the total must hold still before wait_for_card_to_settle believes it. One is a+# stall as easily as a finished reclaim, and reading a stall as settled hands assert_cli_run+# a baseline with the previous model still in it.+VRAM_SETTLE_QUIET_POLLS = 2CANARY = "__UNSLOTH_STUDIO__!!!"@@ -173,12 +178,46 @@ def run(cmd: list[str], **kw) -> subprocess.CompletedProcess:return subprocess.CompletedProcess(cmd, 127, "", f"{type(exc).__name__}: {exc}")+def visible_device_selector() -> str | None:+ """CUDA_VISIBLE_DEVICES as an ``nvidia-smi -i`` argument, or None if unset.++ nvidia-smi ignores CUDA_VISIBLE_DEVICES and reports every PHYSICAL card, so both+ samplers below were device-wide on a two-card host while the payload was pinned to+ one. An unrelated process starting on the HIDDEN card and taking 200 MiB was then+ enough to carry a CPU-served run past the memory assertion.++ The entries are passed to `-i` as written, which is the same reading of them that+ gpu_inventory() already uses to slice nvidia-smi's rows, and matches build_kernel.py+ setting the variable from the index it admitted the payload on. `-i` also accepts the+ GPU-UUID form, so a UUID entry needs no mapping here.++ Drivers old enough to take only ONE id reject the list, and both callers read that as+ "nvidia-smi did not answer" -- no sample rather than a device-wide one, which is the+ safe direction: it withdraws the fallback ruler instead of widening it.+ """+ raw = os.environ.get("CUDA_VISIBLE_DEVICES")+ if raw is None:+ return None+ return ",".join(part.strip() for part in raw.split(",") if part.strip())+++def _scoped(cmd: list[str]) -> list[str] | None:+ """`cmd` restricted to the visible cards, or None when none are visible at all."""+ selector = visible_device_selector()+ if selector is None:+ return cmd+ if not selector:+ # An empty CUDA_VISIBLE_DEVICES means no card is visible, not every card.+ return None+ return [cmd[0], "-i", selector, *cmd[1:]]++def nvidia_used_mib() -> float | None:- """Device-wide VRAM in use, summed over every visible GPU."""- proc = run(- ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],- timeout = 60,- )+ """VRAM in use, summed over the cards this process can actually see."""+ cmd = _scoped(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"])+ if cmd is None:+ return None+ proc = run(cmd, timeout = 60)if proc.returncode != 0:return Nonetotal = 0.0@@ -195,11 +234,31 @@ def nvidia_used_mib() -> float | None:return total if seen else None+def card_is_shared(apps_before: dict[int, int] | None, listed_before: set[int] | None) -> bool:+ """Was another CUDA process already on the card when this run launched?++ The device-wide total is the fallback ruler whenever attribution fails, and it is only+ a measurement of THIS process when this process owns the card. Under --studio-concurrent+ it does not: a training leg shares it, and that leg both allocates and frees inside the+ window -- one recorded run read the delta as -182.0 MiB while the server genuinely held+ 2.6 GB. Accepting a +200 MiB device rise there would pass a CPU-served run on memory+ somebody else allocated.++ Evidence, not configuration: the payload is not told whether it is the concurrent half,+ but a pid on the card before launch says so. Absence of evidence is left alone -- an+ ordinary single-payload run has an empty listing and keeps the fallback it needs, which+ is the whole point of the fallback on parts that report [N/A] for everything.+ """+ return bool(apps_before) or bool(listed_before)++def cli_run_gpu_failure(apps_before: dict[int, int] | None,apps_after: dict[int, int] | None,baseline: float | None,settled: float | None,+ listed_before: set[int] | None = None,+ listed_after: set[int] | None = None,) -> tuple[str | None, dict]:"""Did a model actually reach the card? Returns (failure or None, detail).@@ -230,6 +289,13 @@ def cli_run_gpu_failure(if apps_after is None:if baseline is None or settled is None:return "nvidia-smi did not answer, so GPU use is unmeasured", detail+ if card_is_shared(apps_before, listed_before):+ detail["card_shared_before_launch"] = True+ return (+ "nvidia-smi could not attribute any process, and another was already on "+ "the card before the launch -- the device total is a shared counter, so "+ "GPU use is unmeasured rather than proven"+ ), detailif settled - baseline < 200.0:return (f"device VRAM grew by {settled - baseline:.1f} MiB across the "@@ -239,11 +305,55 @@ def cli_run_gpu_failure(), detailreturn None, detail- before = apps_before or {}+ # A pid on the card before the launch is not this launch's whether or not the listing+ # could put a figure on it. Excluding only the ATTRIBUTED ones let a co-tenant that+ # read [N/A] before and a number after count as newly appeared, and pass the claim on+ # its memory.+ before = set(apps_before or {}) | set(listed_before or ())appeared = {pid: mib for pid, mib in apps_after.items() if pid not in before}detail["compute_apps_appeared"] = appearedgrew = sum(appeared.values())detail["process_vram_mib"] = grew+ # A pid that APPEARED but carries no figure is the mixed-listing case: the all-[N/A] guard+ # never fires, so defer to the device-wide delta rather than reporting CPU.+ if listed_before is not None and listed_after is not None:+ appeared_unattributed = sorted((listed_after - set(listed_before)) - set(apps_after))+ if appeared_unattributed:+ detail["compute_apps_unattributed"] = appeared_unattributed+ if baseline is None or settled is None:+ return (+ "nvidia-smi listed a new process but could not say how much "+ "memory it holds, and gave no device total either, so GPU use "+ "is unmeasured"+ ), detail+ if card_is_shared(apps_before, listed_before):+ detail["card_shared_before_launch"] = True+ return (+ f"nvidia-smi could not attribute the new process "+ f"{appeared_unattributed}, and another was already on the card "+ f"before the launch -- the device total is a shared counter, so "+ f"GPU use is unmeasured rather than proven"+ ), detail+ if settled - baseline < 200.0:+ if grew >= 200.0:+ # The device total is the WEAKER ruler and cannot overrule a direct+ # per-process reading: {222: 2600} beside an unattributable {333} is+ # not a CPU-served run however flat the total looks. Which of the two+ # is the server is unknown, so this is a hedge and not a pass.+ return (+ f"the device total moved {settled - baseline:.1f} MiB while "+ f"{sorted(appeared)} appeared holding {grew} MiB and "+ f"{appeared_unattributed} could not be attributed at all -- the "+ f"two rulers disagree, so GPU use is unmeasured rather than "+ f"disproven"+ ), detail+ return (+ f"device VRAM grew by {settled - baseline:.1f} MiB across the "+ f"launch and a served completion, and nvidia-smi could not "+ f"attribute the new process {appeared_unattributed} -- "+ f"`unsloth run` served from the CPU"+ ), detail+ return None, detailif grew < 200.0:return (f"no process appeared on the GPU holding more than {grew} MiB "@@ -254,14 +364,81 @@ def cli_run_gpu_failure(return None, detail-def nvidia_compute_apps() -> dict[int, int] | None:- proc = run(- ["nvidia-smi", "--query-compute-apps=pid,used_gpu_memory", "--format=csv,noheader,nounits"],- timeout = 60,+def nvidia_compute_apps_listing() -> tuple[dict[int, int], set[int]] | None:+ """(pids with a readable figure, every pid listed), or None if nvidia-smi did not answer.++ Both, because the interesting case is the difference: a pid that is listed but has no+ figure is on the card and unattributable, which is not the same as absent.+ """+ cmd = _scoped(+ ["nvidia-smi", "--query-compute-apps=pid,used_gpu_memory", "--format=csv,noheader,nounits"])+ if cmd is None:+ return None+ proc = run(cmd, timeout = 60)if proc.returncode != 0:return None- return parse_compute_apps(proc.stdout)+ return parse_compute_apps(proc.stdout), listed_pids(proc.stdout)+++def attributed_apps(listing: tuple[dict[int, int], set[int]] | None) -> dict[int, int] | None:+ """The readable figures from a listing, or None when it cannot attribute at all."""+ if listing is None:+ return None+ apps, listed = listing+ if not apps and listed:+ # Listed but none readable: WDDM and unified-memory parts report [N/A] for every+ # process, which attributes nothing, so the device-wide delta decides.+ return None+ return apps+++def nvidia_compute_apps() -> dict[int, int] | None:+ return attributed_apps(nvidia_compute_apps_listing())+++def wait_for_card_to_settle() -> None:+ """Block until the driver has finished reclaiming after a server was stopped.++ settled_baseline() already waits for the number to stop falling after an unload,+ but it talks to a LIVE backend and there is none left after stop_server(). Both+ halves of that stop are asynchronous: the allocation is returned after the process+ exits, and the pid keeps being listed for a moment after that. Sampled immediately,+ the dying llama-server is assert_cli_run's before-launch state -- on a part that+ reports [N/A] for every process it is the only evidence there is, so card_is_shared()+ calls the card shared and a run that did reach the GPU comes back "unmeasured rather+ than proven"; and its retained memory inflates the baseline, so the same run's device+ delta reads as "served from the CPU".++ Two things have to go quiet, because either alone is satisfied too early. The pid is+ waited ON: every pid the card carries here is the server just stopped or a genuine+ co-tenant, so none of THEM may be left -- one arriving afterwards is somebody else's.+ And the total must hold still across VRAM_SETTLE_QUIET_POLLS intervals, not one: if the+ pid is already gone at entry and reclaim has not begun, a single flat interval is a+ stall, not a settled card, and the baseline would carry the old model's allocation.++ A real co-tenant never leaves and spends the whole 30s budget. That is accepted rather+ than special-cased: a shared card is exactly the run where card_is_shared() withdraws+ the device-delta fallback anyway, so there is no verdict for the wait to protect there.+ """+ listing = nvidia_compute_apps_listing()+ resident = listing[1] if listing else set()+ previous_mib = nvidia_used_mib()+ quiet = 0+ for _ in range(VRAM_SETTLE_SAMPLES):+ time.sleep(VRAM_SETTLE_POLL_S)+ listing = nvidia_compute_apps_listing()+ current_mib = nvidia_used_mib()+ fell = (+ previous_mib is not None+ and current_mib is not None+ and current_mib < previous_mib - VRAM_SETTLE_TOLERANCE_MIB+ )+ previous_mib = current_mib+ quiet = 0 if fell else quiet + 1+ still_there = resident & (listing[1] if listing else set())+ if quiet >= VRAM_SETTLE_QUIET_POLLS and not still_there:+ returndef visible_device_indices() -> list[int] | None:@@ -2704,7 +2881,9 @@ class Payload:# The per-process reading is taken alongside the device one because the# device one is only valid when this payload owns the card, and under# --studio-concurrent it does not. See the verdict below.- apps_before = nvidia_compute_apps() or {}+ _listing_before = nvidia_compute_apps_listing()+ apps_before = attributed_apps(_listing_before) or {}+ listed_before = _listing_before[1] if _listing_before else Nonedetail["compute_apps_before"] = apps_beforehead = self.studio_command()[:1] or [sys.executable]@@ -2828,13 +3007,17 @@ class Payload:# after it.settled = nvidia_used_mib()detail["vram_after_mib"] = settled- apps_after = nvidia_compute_apps()+ _listing_after = nvidia_compute_apps_listing()+ apps_after = attributed_apps(_listing_after)+ listed_after = _listing_after[1] if _listing_after else Nonedetail["compute_apps"] = apps_afterfailure, measured = cli_run_gpu_failure(apps_before,apps_after,baseline,settled,+ listed_before,+ listed_after,)detail.update(measured)if failure:@@ -3196,6 +3379,12 @@ class Payload:# home's state is not a configuration anybody ships. assert_chat_ui# ends by stopping the server, so by here the port is free, the card is# empty, and the VRAM delta below measures this launch alone.+ # Stopped here, not only at the end of assert_chat_ui: with --skip-ui that driver never+ # runs, so a live llama-server read as a co-tenant. Idempotent if already stopped.+ # The wait is half the fix: a stopped server is still listed, and still resident, for+ # a moment afterwards, which is the same co-tenant to the assertion below.+ self.stop_server()+ wait_for_card_to_settle()self.assert_cli_run()# LAST of all, and the only thing here that touches the publicdiff --git a/tests/kaggle/test_studio_cli_run.py b/tests/kaggle/test_studio_cli_run.pyindex a95c6ca6c1b..a8afc702642 100644--- a/tests/kaggle/test_studio_cli_run.py+++ b/tests/kaggle/test_studio_cli_run.py@@ -202,6 +202,19 @@ def test_a_real_cpu_fallback_still_fails():assert failure and "served from the CPU" in failure+def test_a_unified_memory_part_that_cannot_attribute_is_judged_on_the_device_delta():+ """Measured on a GB10 (Windows, unified memory): nvidia-smi lists the server with+ [N/A], the device counter reads 132 MiB idle, 272 MiB with a bare CUDA context+ (-ngl 0) and 560 MiB with the 270M model offloaded. The verdict must pass the+ second and fail the first, with nothing to attribute per process."""+ failure, detail = _verdict()(None, None, 132.0, 560.0)+ assert failure is None, failure+ assert detail["vram_delta_mib"] == 428.0+ failure, detail = _verdict()(None, None, 132.0, 272.0)+ assert failure and "served from the CPU" in failure+ assert detail["vram_delta_mib"] == 140.0++def test_the_device_delta_is_the_fallback_only_when_processes_are_unreadable():"""An nvidia-smi that answers a total but cannot enumerate apps still gets averdict rather than a silent pass."""@@ -218,4 +231,11 @@ def test_the_before_sample_is_taken_before_the_launch():"""An `apps_before` read after the server started would contain the server,so nothing would ever have `appeared` and every run would fail."""body = _body()- assert body.index("apps_before = nvidia_compute_apps()") < body.index("subprocess.Popen")+ # Anchored on the sample, not one spelling of it: one nvidia_compute_apps_listing() call, so+ # the attributed mapping and the listed pids describe the same moment.+ assert body.index("_listing_before = nvidia_compute_apps_listing()") < body.index(+ "subprocess.Popen"+ )+ assert body.index("apps_before = attributed_apps(_listing_before)") < body.index(+ "subprocess.Popen"+ )diff --git a/tests/kaggle/test_studio_gpu_harness.py b/tests/kaggle/test_studio_gpu_harness.pyindex fc8461a0e83..4013b72a218 100644--- a/tests/kaggle/test_studio_gpu_harness.py+++ b/tests/kaggle/test_studio_gpu_harness.py@@ -29,6 +29,7 @@ import subprocessimport sysimport tarfilefrom pathlib import Path+from unittest import mockimport pytest@@ -50,6 +51,7 @@ def _load(name: str, path: Path):return module+run_studio_gpu = _load("studio_gpu_payload_main", PAYLOAD_DIR / "run_studio_gpu.py")build_kernel = _load("studio_ci_build_kernel", CI_DIR / "build_kernel.py")collect_evidence = _load("studio_ci_collect_evidence", CI_DIR / "collect_evidence.py")@@ -77,6 +79,40 @@ def test_no_compute_apps_at_all_is_an_empty_dict_not_a_crash():assert gpu_assert.parse_compute_apps("") == {}+def test_listed_pids_are_counted_even_when_no_memory_is_attributed():+ """Windows (WDDM) and unified-memory parts list every CUDA process with [N/A].+ The GB10 lists a -ngl 0 server too, so this is neither "nothing on the card" nor+ proof of offload: it is the signal to fall back to the device delta."""+ text = "pid, used_gpu_memory\n1234, [N/A]\n5678, [N/A]\n"+ assert gpu_assert.parse_compute_apps(text) == {}+ assert gpu_assert.count_listed_pids(text) == 2+++def test_listed_pids_ignore_the_header_and_junk_rows():+ assert gpu_assert.count_listed_pids("pid, used_gpu_memory\n[N/A], [N/A]\n\n") == 0+ assert gpu_assert.count_listed_pids("") == 0+ assert gpu_assert.count_listed_pids("1234, 2048\n5678, [N/A]\n") == 2+++def test_an_all_unattributed_listing_reads_as_cannot_enumerate(monkeypatch):+ import run_studio_gpu # the payload, on sys.path beside gpu_assert++ """So cli_run_gpu_failure takes its device-delta branch rather than concluding that+ no process appeared. Readable rows still come back as the usual dict."""++ class _P:+ def __init__(self, out):+ self.returncode = 0+ self.stdout = out++ monkeypatch.setattr(run_studio_gpu, "run", lambda *a, **k: _P("4242, [N/A]\n4243, [N/A]\n"))+ assert run_studio_gpu.nvidia_compute_apps() is None+ monkeypatch.setattr(run_studio_gpu, "run", lambda *a, **k: _P("4242, 512\n4243, [N/A]\n"))+ assert run_studio_gpu.nvidia_compute_apps() == {4242: 512}+ monkeypatch.setattr(run_studio_gpu, "run", lambda *a, **k: _P(""))+ assert run_studio_gpu.nvidia_compute_apps() == {}++# ------------------------------------------------------------ llama.cpp log@@ -2478,3 +2514,466 @@ def test_every_assertion_carries_its_own_wall_clock():# And an absolute position, so a reader can line the report up against the# driver's own interval for the payload.assert runner.assertions[1]["at_seconds"] >= runner.assertions[0]["at_seconds"]+++class TestAMixedListingIsNotProofOfCpu:+ """A readable row on one GPU keeps the mapping nonempty, hiding an [N/A] server.++ The all-[N/A] guard fires only when NOTHING parsed. On a box with a TCC card whose+ process reports a figure and a WDDM or unified-memory card where the newly launched+ server reports [N/A], the mapping is nonempty, the guard stays quiet, and the server's+ pid is simply missing -- so nothing "appeared" and the harness failed a run whose model+ was on the card the whole time.+ """++ @staticmethod+ def _verdict(**kwargs):+ return run_studio_gpu.cli_run_gpu_failure(**kwargs)++ def test_an_appeared_but_unattributed_pid_defers_to_the_device_delta(self):+ """On a card this run OWNS: nothing was there before, so the total is ours."""+ failure, detail = self._verdict(+ apps_before = {},+ apps_after = {},+ baseline = 1000.0,+ settled = 1600.0,+ listed_before = set(),+ listed_after = {222},+ )+ assert failure is None, (+ "the device grew by 600 MiB and pid 222 is listed but unattributed, which is "+ f"the offloaded server, not an absence: {detail}"+ )+ assert detail["compute_apps_unattributed"] == [222]++ def test_an_unattributed_pid_with_no_device_growth_still_fails(self):+ failure, detail = self._verdict(+ apps_before = {},+ apps_after = {},+ baseline = 1000.0,+ settled = 1010.0,+ listed_before = set(),+ listed_after = {222},+ )+ assert failure is not None and "served from the CPU" in failure+ assert "222" in failure, "the message names the process it could not attribute"++ def test_an_unattributed_pid_with_no_device_reading_is_unmeasured(self):+ failure, detail = self._verdict(+ apps_before = {},+ apps_after = {111: 500},+ baseline = None,+ settled = None,+ listed_before = set(),+ listed_after = {111, 222},+ )+ assert failure is not None and "unmeasured" in failure++ def test_an_attributed_new_process_is_unaffected(self):+ """The ordinary path must not start deferring to the shared device counter."""+ failure, detail = self._verdict(+ apps_before = {111: 500},+ apps_after = {111: 500, 222: 2600},+ baseline = 1000.0,+ settled = 900.0,+ listed_before = {111},+ listed_after = {111, 222},+ )+ assert failure is None, "a co-tenant freeing memory must not sink an attributed pid"+ assert detail["process_vram_mib"] == 2600++ def test_a_pid_already_present_but_unattributed_is_not_new(self):+ """Only a pid that APPEARED counts; a co-tenant showing [N/A] throughout is theirs."""+ failure, detail = self._verdict(+ apps_before = {111: 500},+ apps_after = {111: 500},+ baseline = 1000.0,+ settled = 1010.0,+ listed_before = {111, 999},+ listed_after = {111, 999},+ )+ assert "compute_apps_unattributed" not in detail+ assert failure is not None and "no process appeared" in failure++ def test_the_old_callers_still_work(self):+ """The listed sets are optional, so a caller that has none keeps the old verdict."""+ failure, _ = self._verdict(+ apps_before = {},+ apps_after = {222: 2600},+ baseline = None,+ settled = None,+ )+ assert failure is None+++class TestTheListingNamesItsPids:+ def test_listed_pids_includes_unattributed_rows(self):+ csv_text = "111, 500\n222, [N/A]\n"+ assert gpu_assert.parse_compute_apps(csv_text) == {111: 500}+ assert gpu_assert.listed_pids(csv_text) == {111, 222}++ def test_listed_pids_ignores_headers_and_junk(self):+ assert gpu_assert.listed_pids("pid, used_gpu_memory\n\nnotapid, 5\n333, 10\n") == {333}++ def test_the_sample_is_taken_once(self):+ """Two nvidia-smi calls would describe two different moments, so the attributed+ mapping and the listed pids could disagree about which processes exist.+ """+ body = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")+ assert body.count("nvidia_compute_apps_listing()") >= 3+ assert "apps_before = attributed_apps(_listing_before)" in body+ assert "apps_after = attributed_apps(_listing_after)" in body+++class TestASharedCardIsNotMeasuredByItsTotal:+ """The device total only measures THIS process when THIS process owns the card.++ Under --studio-concurrent a training leg shares it and both allocates and frees inside+ the window: one recorded run read the delta as -182.0 MiB while the server genuinely+ held 2.6 GB. Accepting a +200 MiB rise there would pass a CPU-served run on memory+ somebody else allocated, so a fallback to the total is refused when anything was on the+ card before the launch.+ """++ @staticmethod+ def _verdict(**kwargs):+ return run_studio_gpu.cli_run_gpu_failure(**kwargs)++ def test_a_co_tenant_blocks_the_no_enumeration_fallback(self):+ failure, detail = self._verdict(+ apps_before = {111: 500},+ apps_after = None,+ baseline = 1000.0,+ settled = 9000.0,+ listed_before = {111},+ listed_after = None,+ )+ assert failure is not None and "unmeasured rather than proven" in failure+ assert detail["card_shared_before_launch"] is True++ def test_a_co_tenant_blocks_the_mixed_listing_fallback(self):+ failure, detail = self._verdict(+ apps_before = {111: 500},+ apps_after = {111: 500},+ baseline = 1000.0,+ settled = 9000.0,+ listed_before = {111},+ listed_after = {111, 222},+ )+ assert failure is not None and "unmeasured rather than proven" in failure+ assert detail["card_shared_before_launch"] is True++ def test_a_listed_but_unattributed_co_tenant_counts(self):+ """A co-tenant reporting [N/A] is still a co-tenant."""+ failure, _ = self._verdict(+ apps_before = {},+ apps_after = None,+ baseline = 1000.0,+ settled = 9000.0,+ listed_before = {999},+ listed_after = None,+ )+ assert failure is not None and "unmeasured rather than proven" in failure++ def test_an_empty_card_keeps_the_fallback(self):+ """The fallback exists for parts that report [N/A] for everything; it stays."""+ failure, _ = self._verdict(+ apps_before = {},+ apps_after = None,+ baseline = 1000.0,+ settled = 1600.0,+ listed_before = set(),+ listed_after = None,+ )+ assert failure is None++ def test_attribution_still_wins_over_the_rule(self):+ """A co-tenant does not sink a run whose own process WAS attributed."""+ failure, detail = self._verdict(+ apps_before = {111: 500},+ apps_after = {111: 500, 222: 2600},+ baseline = 1000.0,+ settled = 900.0,+ listed_before = {111},+ listed_after = {111, 222},+ )+ assert failure is None+ assert detail["process_vram_mib"] == 2600++ def test_unknown_co_tenancy_is_not_invented(self):+ """No listing at all is no evidence either way, so behaviour is unchanged."""+ failure, _ = self._verdict(+ apps_before = None,+ apps_after = None,+ baseline = 1000.0,+ settled = 1600.0,+ )+ assert failure is None+++class TestTheSamplesAreScopedToTheVisibleCard:+ """nvidia-smi ignores CUDA_VISIBLE_DEVICES and answers for every PHYSICAL card.++ Kaggle offers a two-GPU T4 session and build_kernel.py pins each payload to one card,+ so both samplers were reading the whole box. An unrelated process starting on the+ HIDDEN card and taking 200 MiB was then enough evidence for the device-wide delta to+ carry a CPU-served run past the memory assertion.+ """++ @staticmethod+ def _selector(value):+ env = dict(os.environ)+ if value is None:+ env.pop("CUDA_VISIBLE_DEVICES", None)+ else:+ env["CUDA_VISIBLE_DEVICES"] = value+ with mock.patch.dict(os.environ, env, clear = True):+ return run_studio_gpu.visible_device_selector()++ def test_unset_selects_every_card(self):+ assert self._selector(None) is None++ def test_a_pinned_card_selects_only_it(self):+ assert self._selector("1") == "1"++ def test_a_list_is_carried_through(self):+ assert self._selector("0, 1") == "0,1"++ def test_a_uuid_needs_no_mapping(self):+ """`nvidia-smi -i` takes the GPU-UUID form directly."""+ assert self._selector("GPU-84ccface-663f") == "GPU-84ccface-663f"++ def test_an_empty_value_is_no_cards_not_all_of_them(self):+ assert self._selector("") == ""++ @staticmethod+ def _scoped(value):+ env = dict(os.environ)+ if value is None:+ env.pop("CUDA_VISIBLE_DEVICES", None)+ else:+ env["CUDA_VISIBLE_DEVICES"] = value+ with mock.patch.dict(os.environ, env, clear = True):+ return run_studio_gpu._scoped(["nvidia-smi", "--query-gpu=memory.used"])++ def test_the_pin_reaches_the_command(self):+ assert self._scoped("1") == ["nvidia-smi", "-i", "1", "--query-gpu=memory.used"]++ def test_no_pin_leaves_the_command_alone(self):+ assert self._scoped(None) == ["nvidia-smi", "--query-gpu=memory.used"]++ def test_no_visible_card_takes_no_sample(self):+ assert self._scoped("") is None++ def test_both_samplers_are_scoped(self):+ """The pair is the point: scoping one ruler and not the other still compares+ a card-local reading against a box-wide one."""+ seen = []++ def fake_run(cmd, **kwargs):+ seen.append(cmd)+ return subprocess.CompletedProcess(cmd, 0, "7, 100\n", "")++ with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "7"}):+ with mock.patch.object(run_studio_gpu, "run", fake_run):+ run_studio_gpu.nvidia_used_mib()+ run_studio_gpu.nvidia_compute_apps_listing()+ assert len(seen) == 2+ for cmd in seen:+ assert cmd[1:3] == ["-i", "7"], f"unscoped sample: {cmd}"++ def test_neither_sampler_runs_without_a_visible_card(self):+ def explode(cmd, **kwargs):+ raise AssertionError(f"sampled with no visible card: {cmd}")++ with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": ""}):+ with mock.patch.object(run_studio_gpu, "run", explode):+ assert run_studio_gpu.nvidia_used_mib() is None+ assert run_studio_gpu.nvidia_compute_apps_listing() is None+++class TestTheServerIsStoppedBeforeTheCliBaselineRegardlessOfSkipUi:+ """assert_chat_ui ends by stopping the Studio server, so with --skip-ui nothing did, and+ the still-running llama-server sat in assert_cli_run's before-launch listing. On parts that+ report [N/A] per process that pid read as a co-tenant, the device-delta fallback was+ refused, and a GPU-backed run came back "unmeasured rather than proven"."""++ def test_stop_server_precedes_the_cli_assertion_unconditionally(self):+ text = Path(run_studio_gpu.__file__).read_text(encoding = "utf-8")+ ui = text.index("if not self.args.skip_ui:\n self.assert_chat_ui()")+ cli = text.index(" self.assert_cli_run()", ui)+ between = text[ui:cli]+ stop = between.rindex("self.stop_server()")+ # At the method's own indentation, i.e. not inside the skip_ui branch.+ line_start = between.rfind("\n", 0, stop) + 1+ assert (+ between[line_start:stop] == " "+ ), "stop_server must not sit inside the skip_ui guard"++ def test_the_card_is_settled_between_the_stop_and_the_assertion(self):+ """Stopping is not the same as having stopped. Source order, because the two calls+ are orchestration in run() and there is no seam between them to observe."""+ text = Path(run_studio_gpu.__file__).read_text(encoding = "utf-8")+ cli = text.index(" self.assert_cli_run()")+ between = text[+ text.index("if not self.args.skip_ui:\n self.assert_chat_ui()") : cli+ ]+ assert between.rindex("self.stop_server()") < between.rindex("wait_for_card_to_settle()")+++class TestTheCardIsGivenTimeToSettleAfterAStop:+ """A terminated llama-server keeps its allocation, and keeps being LISTED, for a moment+ after it exits. Sampled straight away that pid is assert_cli_run's before-launch state:+ card_is_shared() withdraws the device-delta fallback and a GPU-backed run reports+ "unmeasured rather than proven", which is the false red this PR set out to remove.+ """++ @staticmethod+ def _settle(samples):+ """Drive wait_for_card_to_settle over `samples` of (used_mib, listed pids), the+ last one repeating. (None, None) is an nvidia-smi that did not answer. Returns the+ number of polls taken: two nvidia-smi calls each, the entry sample included."""+ taken = []++ def fake_run(cmd, **kwargs):+ used, pids = samples[min(len(taken) // 2, len(samples) - 1)]+ taken.append(cmd)+ if pids is None:+ return subprocess.CompletedProcess(cmd, 9, "", "no devices were found")+ if "--query-compute-apps=pid,used_gpu_memory" in cmd:+ # The [N/A] shape: every pid listed, none of them attributed.+ return subprocess.CompletedProcess(+ cmd, 0, "".join(f"{p}, [N/A]\n" for p in pids), ""+ )+ return subprocess.CompletedProcess(cmd, 0, f"{used}\n", "")++ with mock.patch.object(run_studio_gpu, "run", fake_run):+ with mock.patch.object(run_studio_gpu.time, "sleep", lambda _s: None):+ run_studio_gpu.wait_for_card_to_settle()+ return len(taken) // 2++ def test_it_waits_for_the_stopped_server_to_leave_the_listing(self):+ """Two equal samples are not proof of a settled card when the driver has not begun+ giving the memory back, which is why the pid is what is waited on."""+ assert self._settle([(3400.0, [4242]), (3400.0, [4242]), (200.0, []), (200.0, [])]) == 5++ def test_it_waits_for_the_memory_to_come_back(self):+ """The pid can go while the driver is still returning the allocation."""+ assert self._settle([(3400.0, []), (1800.0, []), (200.0, []), (200.0, [])]) == 5++ def test_a_stalled_reclaim_with_no_pid_to_wait_on_is_not_settled(self):+ """The pid can also be gone BEFORE the first sample, leaving nothing to wait on. One+ flat interval is then a stall as easily as a finished reclaim, and taking it as+ settled hands assert_cli_run a baseline with the old model still in it."""+ assert self._settle([(3400.0, []), (3400.0, []), (200.0, []), (200.0, [])]) == 5++ def test_a_settled_card_costs_only_the_quiet_window(self):+ assert self._settle([(200.0, [])]) == run_studio_gpu.VRAM_SETTLE_QUIET_POLLS + 1++ def test_a_co_tenant_arriving_afterwards_does_not_hold_the_run(self):+ """Only the pids the card carried at entry are the stop's to wait for."""+ assert self._settle([(200.0, []), (2600.0, [777]), (2600.0, [777])]) == 3++ def test_an_unanswering_smi_is_not_waited_on(self):+ assert self._settle([(None, None)]) == 3++ def test_the_wait_is_bounded(self):+ """A co-tenant never leaves, and a card somebody else is draining never settles."""+ assert (+ self._settle([(3400.0 - 100.0 * i, [4242]) for i in range(60)])+ == run_studio_gpu.VRAM_SETTLE_SAMPLES + 1+ )++ def test_the_settled_listing_no_longer_reads_as_a_co_tenant(self):+ """The point of the wait, stated as the verdict it changes."""+ stale = run_studio_gpu.cli_run_gpu_failure(+ {},+ None,+ 3400.0,+ 3900.0,+ {4242},+ {4242, 5555},+ )[0]+ assert stale is not None and "already on the card" in stale+ settled = run_studio_gpu.cli_run_gpu_failure(+ {},+ None,+ 200.0,+ 700.0,+ set(),+ {5555},+ )[0]+ assert settled is None+++class TestTheTwoRulersAreReadInTheRightOrder:+ """Which pids count as new, and which reading is allowed to overrule which.++ Both of these are the mixed listing, where one new pid carries a figure and another+ does not, and both come out of splitting "listed" from "attributed": the split is only+ half applied unless the exclusion uses it too, and the weaker ruler must not be allowed+ to contradict the stronger one.+ """++ @staticmethod+ def _verdict(**kwargs):+ return run_studio_gpu.cli_run_gpu_failure(**kwargs)++ def test_a_flat_device_total_does_not_disprove_an_attributed_process(self):+ """2.6 GiB read off pid 222 is not a CPU-served run because pid 333 was also new+ and unreadable. The total is a shared counter and cannot outrank a direct reading;+ which of the two is the server is still unknown, so this hedges rather than passes.+ """+ failure, detail = self._verdict(+ apps_before = {},+ apps_after = {222: 2600},+ baseline = 1000.0,+ settled = 1050.0,+ listed_before = set(),+ listed_after = {222, 333},+ )+ assert failure is not None+ assert "served from the CPU" not in failure, failure+ assert "unmeasured rather than disproven" in failure+ assert detail["process_vram_mib"] == 2600+ assert detail["compute_apps_unattributed"] == [333]++ def test_a_co_tenant_that_becomes_readable_is_still_not_new(self):+ """pid 77 was on the card at launch and merely unattributable then. Keying the+ exclusion on the attributed map alone counted its 500 MiB as this launch's and+ passed a server that never left the CPU."""+ failure, detail = self._verdict(+ apps_before = {},+ apps_after = {77: 500},+ baseline = 1000.0,+ settled = 1000.0,+ listed_before = {77},+ listed_after = {77},+ )+ assert failure is not None and "no process appeared" in failure+ assert detail["compute_apps_appeared"] == {}++ def test_the_same_co_tenant_does_not_carry_the_mixed_listing_either(self):+ failure, detail = self._verdict(+ apps_before = {},+ apps_after = {77: 500},+ baseline = 1000.0,+ settled = 1000.0,+ listed_before = {77},+ listed_after = {77, 888},+ )+ assert failure is not None and "unmeasured rather than proven" in failure+ assert detail["card_shared_before_launch"] is True++ def test_an_attributed_launch_on_an_empty_card_still_passes(self):+ """The ordinary verdict is untouched: the exclusion only ever grows."""+ failure, detail = self._verdict(+ apps_before = {},+ apps_after = {222: 2600},+ baseline = 1000.0,+ settled = 3600.0,+ listed_before = set(),+ listed_after = {222},+ )+ assert failure is None+ assert detail["process_vram_mib"] == 2600