Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Kaggle GPU harness: an all-[N/A] listing is cannot attribute, not no GPU (#10472)

Daniel Han19ca5928dcef
diff --git a/tests/kaggle/studio_gpu/gpu_assert.py b/tests/kaggle/studio_gpu/gpu_assert.py
index aaccb534526..1cde39515d6 100644
--- a/tests/kaggle/studio_gpu/gpu_assert.py
+++ b/tests/kaggle/studio_gpu/gpu_assert.py
@@ -106,6 +106,58 @@ def parse_compute_apps(csv_text: str) -> dict[int, int]:
return apps
+def count_listed_pids(csv_text: str) -> int:
+ """How many processes ``--query-compute-apps`` LISTED, whether or not it could say
+ how much memory each holds.
+
+ On Windows (WDDM) and on unified-memory parts such as the GB10, nvidia-smi lists
+ every CUDA process but reports ``[N/A]`` for all of them, and it lists a
+ ``-ngl 0`` server too, since a CUDA build creates a context regardless. So a
+ listing with no readable figure is not "nothing on the card" and not proof of
+ offload either: it is "cannot attribute", and the caller falls back to the
+ device-wide delta, which does move on those parts (measured on a GB10: +140 MiB
+ for a bare context, +428 MiB with a 270M model offloaded)."""
+ n = 0
+ for line in csv_text.splitlines():
+ line = line.strip()
+ if not line or line.lower().startswith("pid"):
+ continue
+ parts = [p.strip() for p in line.split(",")]
+ if not parts:
+ continue
+ try:
+ int(parts[0])
+ except ValueError:
+ continue
+ n += 1
+ return n
+
+
+def listed_pids(csv_text: str) -> set[int]:
+ """Every pid ``--query-compute-apps`` LISTED, whether or not it carried a figure.
+
+ count_listed_pids answers "how many", which is enough to tell an empty listing from
+ an all-``[N/A]`` one. It is not enough for a MIXED listing: a readable row on one GPU
+ and the newly launched server as ``[N/A]`` on another leaves the parsed mapping
+ nonempty, so the caller saw a normal result and the server's pid simply was not in
+ it. Naming the pids lets the caller notice that the process it cares about is the
+ unattributed one, rather than concluding it never reached the card.
+ """
+ pids: set[int] = set()
+ for line in csv_text.splitlines():
+ line = line.strip()
+ if not line or line.lower().startswith("pid"):
+ continue
+ parts = [p.strip() for p in line.split(",")]
+ if not parts:
+ continue
+ try:
+ pids.add(int(parts[0]))
+ except ValueError:
+ continue
+ return pids
+
+
def offloaded_layers(log_text: str) -> tuple[int, int] | None:
"""The last ``offloaded N/M layers to GPU`` in a llama.cpp log.
diff --git a/tests/kaggle/studio_gpu/run_studio_gpu.py b/tests/kaggle/studio_gpu/run_studio_gpu.py
index 45dd0190801..16de1964c3d 100644
--- a/tests/kaggle/studio_gpu/run_studio_gpu.py
+++ b/tests/kaggle/studio_gpu/run_studio_gpu.py
@@ -86,6 +86,7 @@ from gpu_assert import ( # noqa: E402
is_cuda_install,
offload_verdict,
parse_compute_apps,
+ listed_pids,
)
from studio_client import ( # noqa: E402
Studio,
@@ -134,6 +135,10 @@ VRAM_SETTLE_POLL_S = 2.5
VRAM_SETTLE_SAMPLES = 12
# Driver readings jitter by a few MiB between samples; only a real drop counts.
VRAM_SETTLE_TOLERANCE_MIB = 16.0
+# Intervals the total must hold still before wait_for_card_to_settle believes it. One is a
+# stall as easily as a finished reclaim, and reading a stall as settled hands assert_cli_run
+# a baseline with the previous model still in it.
+VRAM_SETTLE_QUIET_POLLS = 2
CANARY = "__UNSLOTH_STUDIO__!!!"
@@ -173,12 +178,46 @@ def run(cmd: list[str], **kw) -> subprocess.CompletedProcess:
return subprocess.CompletedProcess(cmd, 127, "", f"{type(exc).__name__}: {exc}")
+def visible_device_selector() -> str | None:
+ """CUDA_VISIBLE_DEVICES as an ``nvidia-smi -i`` argument, or None if unset.
+
+ nvidia-smi ignores CUDA_VISIBLE_DEVICES and reports every PHYSICAL card, so both
+ samplers below were device-wide on a two-card host while the payload was pinned to
+ one. An unrelated process starting on the HIDDEN card and taking 200 MiB was then
+ enough to carry a CPU-served run past the memory assertion.
+
+ The entries are passed to `-i` as written, which is the same reading of them that
+ gpu_inventory() already uses to slice nvidia-smi's rows, and matches build_kernel.py
+ setting the variable from the index it admitted the payload on. `-i` also accepts the
+ GPU-UUID form, so a UUID entry needs no mapping here.
+
+ Drivers old enough to take only ONE id reject the list, and both callers read that as
+ "nvidia-smi did not answer" -- no sample rather than a device-wide one, which is the
+ safe direction: it withdraws the fallback ruler instead of widening it.
+ """
+ raw = os.environ.get("CUDA_VISIBLE_DEVICES")
+ if raw is None:
+ return None
+ return ",".join(part.strip() for part in raw.split(",") if part.strip())
+
+
+def _scoped(cmd: list[str]) -> list[str] | None:
+ """`cmd` restricted to the visible cards, or None when none are visible at all."""
+ selector = visible_device_selector()
+ if selector is None:
+ return cmd
+ if not selector:
+ # An empty CUDA_VISIBLE_DEVICES means no card is visible, not every card.
+ return None
+ return [cmd[0], "-i", selector, *cmd[1:]]
+
+
def nvidia_used_mib() -> float | None:
- """Device-wide VRAM in use, summed over every visible GPU."""
- proc = run(
- ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],
- timeout = 60,
- )
+ """VRAM in use, summed over the cards this process can actually see."""
+ cmd = _scoped(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"])
+ if cmd is None:
+ return None
+ proc = run(cmd, timeout = 60)
if proc.returncode != 0:
return None
total = 0.0
@@ -195,11 +234,31 @@ def nvidia_used_mib() -> float | None:
return total if seen else None
+def card_is_shared(apps_before: dict[int, int] | None, listed_before: set[int] | None) -> bool:
+ """Was another CUDA process already on the card when this run launched?
+
+ The device-wide total is the fallback ruler whenever attribution fails, and it is only
+ a measurement of THIS process when this process owns the card. Under --studio-concurrent
+ it does not: a training leg shares it, and that leg both allocates and frees inside the
+ window -- one recorded run read the delta as -182.0 MiB while the server genuinely held
+ 2.6 GB. Accepting a +200 MiB device rise there would pass a CPU-served run on memory
+ somebody else allocated.
+
+ Evidence, not configuration: the payload is not told whether it is the concurrent half,
+ but a pid on the card before launch says so. Absence of evidence is left alone -- an
+ ordinary single-payload run has an empty listing and keeps the fallback it needs, which
+ is the whole point of the fallback on parts that report [N/A] for everything.
+ """
+ return bool(apps_before) or bool(listed_before)
+
+
def cli_run_gpu_failure(
apps_before: dict[int, int] | None,
apps_after: dict[int, int] | None,
baseline: float | None,
settled: float | None,
+ listed_before: set[int] | None = None,
+ listed_after: set[int] | None = None,
) -> tuple[str | None, dict]:
"""Did a model actually reach the card? Returns (failure or None, detail).
@@ -230,6 +289,13 @@ def cli_run_gpu_failure(
if apps_after is None:
if baseline is None or settled is None:
return "nvidia-smi did not answer, so GPU use is unmeasured", detail
+ if card_is_shared(apps_before, listed_before):
+ detail["card_shared_before_launch"] = True
+ return (
+ "nvidia-smi could not attribute any process, and another was already on "
+ "the card before the launch -- the device total is a shared counter, so "
+ "GPU use is unmeasured rather than proven"
+ ), detail
if settled - baseline < 200.0:
return (
f"device VRAM grew by {settled - baseline:.1f} MiB across the "
@@ -239,11 +305,55 @@ def cli_run_gpu_failure(
), detail
return None, detail
- before = apps_before or {}
+ # A pid on the card before the launch is not this launch's whether or not the listing
+ # could put a figure on it. Excluding only the ATTRIBUTED ones let a co-tenant that
+ # read [N/A] before and a number after count as newly appeared, and pass the claim on
+ # its memory.
+ before = set(apps_before or {}) | set(listed_before or ())
appeared = {pid: mib for pid, mib in apps_after.items() if pid not in before}
detail["compute_apps_appeared"] = appeared
grew = sum(appeared.values())
detail["process_vram_mib"] = grew
+ # A pid that APPEARED but carries no figure is the mixed-listing case: the all-[N/A] guard
+ # never fires, so defer to the device-wide delta rather than reporting CPU.
+ if listed_before is not None and listed_after is not None:
+ appeared_unattributed = sorted((listed_after - set(listed_before)) - set(apps_after))
+ if appeared_unattributed:
+ detail["compute_apps_unattributed"] = appeared_unattributed
+ if baseline is None or settled is None:
+ return (
+ "nvidia-smi listed a new process but could not say how much "
+ "memory it holds, and gave no device total either, so GPU use "
+ "is unmeasured"
+ ), detail
+ if card_is_shared(apps_before, listed_before):
+ detail["card_shared_before_launch"] = True
+ return (
+ f"nvidia-smi could not attribute the new process "
+ f"{appeared_unattributed}, and another was already on the card "
+ f"before the launch -- the device total is a shared counter, so "
+ f"GPU use is unmeasured rather than proven"
+ ), detail
+ if settled - baseline < 200.0:
+ if grew >= 200.0:
+ # The device total is the WEAKER ruler and cannot overrule a direct
+ # per-process reading: {222: 2600} beside an unattributable {333} is
+ # not a CPU-served run however flat the total looks. Which of the two
+ # is the server is unknown, so this is a hedge and not a pass.
+ return (
+ f"the device total moved {settled - baseline:.1f} MiB while "
+ f"{sorted(appeared)} appeared holding {grew} MiB and "
+ f"{appeared_unattributed} could not be attributed at all -- the "
+ f"two rulers disagree, so GPU use is unmeasured rather than "
+ f"disproven"
+ ), detail
+ return (
+ f"device VRAM grew by {settled - baseline:.1f} MiB across the "
+ f"launch and a served completion, and nvidia-smi could not "
+ f"attribute the new process {appeared_unattributed} -- "
+ f"`unsloth run` served from the CPU"
+ ), detail
+ return None, detail
if grew < 200.0:
return (
f"no process appeared on the GPU holding more than {grew} MiB "
@@ -254,14 +364,81 @@ def cli_run_gpu_failure(
return None, detail
-def nvidia_compute_apps() -> dict[int, int] | None:
- proc = run(
- ["nvidia-smi", "--query-compute-apps=pid,used_gpu_memory", "--format=csv,noheader,nounits"],
- timeout = 60,
+def nvidia_compute_apps_listing() -> tuple[dict[int, int], set[int]] | None:
+ """(pids with a readable figure, every pid listed), or None if nvidia-smi did not answer.
+
+ Both, because the interesting case is the difference: a pid that is listed but has no
+ figure is on the card and unattributable, which is not the same as absent.
+ """
+ cmd = _scoped(
+ ["nvidia-smi", "--query-compute-apps=pid,used_gpu_memory", "--format=csv,noheader,nounits"]
)
+ if cmd is None:
+ return None
+ proc = run(cmd, timeout = 60)
if proc.returncode != 0:
return None
- return parse_compute_apps(proc.stdout)
+ return parse_compute_apps(proc.stdout), listed_pids(proc.stdout)
+
+
+def attributed_apps(listing: tuple[dict[int, int], set[int]] | None) -> dict[int, int] | None:
+ """The readable figures from a listing, or None when it cannot attribute at all."""
+ if listing is None:
+ return None
+ apps, listed = listing
+ if not apps and listed:
+ # Listed but none readable: WDDM and unified-memory parts report [N/A] for every
+ # process, which attributes nothing, so the device-wide delta decides.
+ return None
+ return apps
+
+
+def nvidia_compute_apps() -> dict[int, int] | None:
+ return attributed_apps(nvidia_compute_apps_listing())
+
+
+def wait_for_card_to_settle() -> None:
+ """Block until the driver has finished reclaiming after a server was stopped.
+
+ settled_baseline() already waits for the number to stop falling after an unload,
+ but it talks to a LIVE backend and there is none left after stop_server(). Both
+ halves of that stop are asynchronous: the allocation is returned after the process
+ exits, and the pid keeps being listed for a moment after that. Sampled immediately,
+ the dying llama-server is assert_cli_run's before-launch state -- on a part that
+ reports [N/A] for every process it is the only evidence there is, so card_is_shared()
+ calls the card shared and a run that did reach the GPU comes back "unmeasured rather
+ than proven"; and its retained memory inflates the baseline, so the same run's device
+ delta reads as "served from the CPU".
+
+ Two things have to go quiet, because either alone is satisfied too early. The pid is
+ waited ON: every pid the card carries here is the server just stopped or a genuine
+ co-tenant, so none of THEM may be left -- one arriving afterwards is somebody else's.
+ And the total must hold still across VRAM_SETTLE_QUIET_POLLS intervals, not one: if the
+ pid is already gone at entry and reclaim has not begun, a single flat interval is a
+ stall, not a settled card, and the baseline would carry the old model's allocation.
+
+ A real co-tenant never leaves and spends the whole 30s budget. That is accepted rather
+ than special-cased: a shared card is exactly the run where card_is_shared() withdraws
+ the device-delta fallback anyway, so there is no verdict for the wait to protect there.
+ """
+ listing = nvidia_compute_apps_listing()
+ resident = listing[1] if listing else set()
+ previous_mib = nvidia_used_mib()
+ quiet = 0
+ for _ in range(VRAM_SETTLE_SAMPLES):
+ time.sleep(VRAM_SETTLE_POLL_S)
+ listing = nvidia_compute_apps_listing()
+ current_mib = nvidia_used_mib()
+ fell = (
+ previous_mib is not None
+ and current_mib is not None
+ and current_mib < previous_mib - VRAM_SETTLE_TOLERANCE_MIB
+ )
+ previous_mib = current_mib
+ quiet = 0 if fell else quiet + 1
+ still_there = resident & (listing[1] if listing else set())
+ if quiet >= VRAM_SETTLE_QUIET_POLLS and not still_there:
+ return
def visible_device_indices() -> list[int] | None:
@@ -2704,7 +2881,9 @@ class Payload:
# The per-process reading is taken alongside the device one because the
# device one is only valid when this payload owns the card, and under
# --studio-concurrent it does not. See the verdict below.
- apps_before = nvidia_compute_apps() or {}
+ _listing_before = nvidia_compute_apps_listing()
+ apps_before = attributed_apps(_listing_before) or {}
+ listed_before = _listing_before[1] if _listing_before else None
detail["compute_apps_before"] = apps_before
head = self.studio_command()[:1] or [sys.executable]
@@ -2828,13 +3007,17 @@ class Payload:
# after it.
settled = nvidia_used_mib()
detail["vram_after_mib"] = settled
- apps_after = nvidia_compute_apps()
+ _listing_after = nvidia_compute_apps_listing()
+ apps_after = attributed_apps(_listing_after)
+ listed_after = _listing_after[1] if _listing_after else None
detail["compute_apps"] = apps_after
failure, measured = cli_run_gpu_failure(
apps_before,
apps_after,
baseline,
settled,
+ listed_before,
+ listed_after,
)
detail.update(measured)
if failure:
@@ -3196,6 +3379,12 @@ class Payload:
# home's state is not a configuration anybody ships. assert_chat_ui
# ends by stopping the server, so by here the port is free, the card is
# empty, and the VRAM delta below measures this launch alone.
+ # Stopped here, not only at the end of assert_chat_ui: with --skip-ui that driver never
+ # runs, so a live llama-server read as a co-tenant. Idempotent if already stopped.
+ # The wait is half the fix: a stopped server is still listed, and still resident, for
+ # a moment afterwards, which is the same co-tenant to the assertion below.
+ self.stop_server()
+ wait_for_card_to_settle()
self.assert_cli_run()
# LAST of all, and the only thing here that touches the public
diff --git a/tests/kaggle/test_studio_cli_run.py b/tests/kaggle/test_studio_cli_run.py
index a95c6ca6c1b..a8afc702642 100644
--- a/tests/kaggle/test_studio_cli_run.py
+++ b/tests/kaggle/test_studio_cli_run.py
@@ -202,6 +202,19 @@ def test_a_real_cpu_fallback_still_fails():
assert failure and "served from the CPU" in failure
+def test_a_unified_memory_part_that_cannot_attribute_is_judged_on_the_device_delta():
+ """Measured on a GB10 (Windows, unified memory): nvidia-smi lists the server with
+ [N/A], the device counter reads 132 MiB idle, 272 MiB with a bare CUDA context
+ (-ngl 0) and 560 MiB with the 270M model offloaded. The verdict must pass the
+ second and fail the first, with nothing to attribute per process."""
+ failure, detail = _verdict()(None, None, 132.0, 560.0)
+ assert failure is None, failure
+ assert detail["vram_delta_mib"] == 428.0
+ failure, detail = _verdict()(None, None, 132.0, 272.0)
+ assert failure and "served from the CPU" in failure
+ assert detail["vram_delta_mib"] == 140.0
+
+
def test_the_device_delta_is_the_fallback_only_when_processes_are_unreadable():
"""An nvidia-smi that answers a total but cannot enumerate apps still gets a
verdict rather than a silent pass."""
@@ -218,4 +231,11 @@ def test_the_before_sample_is_taken_before_the_launch():
"""An `apps_before` read after the server started would contain the server,
so nothing would ever have `appeared` and every run would fail."""
body = _body()
- assert body.index("apps_before = nvidia_compute_apps()") < body.index("subprocess.Popen")
+ # Anchored on the sample, not one spelling of it: one nvidia_compute_apps_listing() call, so
+ # the attributed mapping and the listed pids describe the same moment.
+ assert body.index("_listing_before = nvidia_compute_apps_listing()") < body.index(
+ "subprocess.Popen"
+ )
+ assert body.index("apps_before = attributed_apps(_listing_before)") < body.index(
+ "subprocess.Popen"
+ )
diff --git a/tests/kaggle/test_studio_gpu_harness.py b/tests/kaggle/test_studio_gpu_harness.py
index fc8461a0e83..4013b72a218 100644
--- a/tests/kaggle/test_studio_gpu_harness.py
+++ b/tests/kaggle/test_studio_gpu_harness.py
@@ -29,6 +29,7 @@ import subprocess
import sys
import tarfile
from pathlib import Path
+from unittest import mock
import pytest
@@ -50,6 +51,7 @@ def _load(name: str, path: Path):
return module
+run_studio_gpu = _load("studio_gpu_payload_main", PAYLOAD_DIR / "run_studio_gpu.py")
build_kernel = _load("studio_ci_build_kernel", CI_DIR / "build_kernel.py")
collect_evidence = _load("studio_ci_collect_evidence", CI_DIR / "collect_evidence.py")
@@ -77,6 +79,40 @@ def test_no_compute_apps_at_all_is_an_empty_dict_not_a_crash():
assert gpu_assert.parse_compute_apps("") == {}
+def test_listed_pids_are_counted_even_when_no_memory_is_attributed():
+ """Windows (WDDM) and unified-memory parts list every CUDA process with [N/A].
+ The GB10 lists a -ngl 0 server too, so this is neither "nothing on the card" nor
+ proof of offload: it is the signal to fall back to the device delta."""
+ text = "pid, used_gpu_memory\n1234, [N/A]\n5678, [N/A]\n"
+ assert gpu_assert.parse_compute_apps(text) == {}
+ assert gpu_assert.count_listed_pids(text) == 2
+
+
+def test_listed_pids_ignore_the_header_and_junk_rows():
+ assert gpu_assert.count_listed_pids("pid, used_gpu_memory\n[N/A], [N/A]\n\n") == 0
+ assert gpu_assert.count_listed_pids("") == 0
+ assert gpu_assert.count_listed_pids("1234, 2048\n5678, [N/A]\n") == 2
+
+
+def test_an_all_unattributed_listing_reads_as_cannot_enumerate(monkeypatch):
+ import run_studio_gpu # the payload, on sys.path beside gpu_assert
+
+ """So cli_run_gpu_failure takes its device-delta branch rather than concluding that
+ no process appeared. Readable rows still come back as the usual dict."""
+
+ class _P:
+ def __init__(self, out):
+ self.returncode = 0
+ self.stdout = out
+
+ monkeypatch.setattr(run_studio_gpu, "run", lambda *a, **k: _P("4242, [N/A]\n4243, [N/A]\n"))
+ assert run_studio_gpu.nvidia_compute_apps() is None
+ monkeypatch.setattr(run_studio_gpu, "run", lambda *a, **k: _P("4242, 512\n4243, [N/A]\n"))
+ assert run_studio_gpu.nvidia_compute_apps() == {4242: 512}
+ monkeypatch.setattr(run_studio_gpu, "run", lambda *a, **k: _P(""))
+ assert run_studio_gpu.nvidia_compute_apps() == {}
+
+
# ------------------------------------------------------------ llama.cpp log
@@ -2478,3 +2514,466 @@ def test_every_assertion_carries_its_own_wall_clock():
# And an absolute position, so a reader can line the report up against the
# driver's own interval for the payload.
assert runner.assertions[1]["at_seconds"] >= runner.assertions[0]["at_seconds"]
+
+
+class TestAMixedListingIsNotProofOfCpu:
+ """A readable row on one GPU keeps the mapping nonempty, hiding an [N/A] server.
+
+ The all-[N/A] guard fires only when NOTHING parsed. On a box with a TCC card whose
+ process reports a figure and a WDDM or unified-memory card where the newly launched
+ server reports [N/A], the mapping is nonempty, the guard stays quiet, and the server's
+ pid is simply missing -- so nothing "appeared" and the harness failed a run whose model
+ was on the card the whole time.
+ """
+
+ @staticmethod
+ def _verdict(**kwargs):
+ return run_studio_gpu.cli_run_gpu_failure(**kwargs)
+
+ def test_an_appeared_but_unattributed_pid_defers_to_the_device_delta(self):
+ """On a card this run OWNS: nothing was there before, so the total is ours."""
+ failure, detail = self._verdict(
+ apps_before = {},
+ apps_after = {},
+ baseline = 1000.0,
+ settled = 1600.0,
+ listed_before = set(),
+ listed_after = {222},
+ )
+ assert failure is None, (
+ "the device grew by 600 MiB and pid 222 is listed but unattributed, which is "
+ f"the offloaded server, not an absence: {detail}"
+ )
+ assert detail["compute_apps_unattributed"] == [222]
+
+ def test_an_unattributed_pid_with_no_device_growth_still_fails(self):
+ failure, detail = self._verdict(
+ apps_before = {},
+ apps_after = {},
+ baseline = 1000.0,
+ settled = 1010.0,
+ listed_before = set(),
+ listed_after = {222},
+ )
+ assert failure is not None and "served from the CPU" in failure
+ assert "222" in failure, "the message names the process it could not attribute"
+
+ def test_an_unattributed_pid_with_no_device_reading_is_unmeasured(self):
+ failure, detail = self._verdict(
+ apps_before = {},
+ apps_after = {111: 500},
+ baseline = None,
+ settled = None,
+ listed_before = set(),
+ listed_after = {111, 222},
+ )
+ assert failure is not None and "unmeasured" in failure
+
+ def test_an_attributed_new_process_is_unaffected(self):
+ """The ordinary path must not start deferring to the shared device counter."""
+ failure, detail = self._verdict(
+ apps_before = {111: 500},
+ apps_after = {111: 500, 222: 2600},
+ baseline = 1000.0,
+ settled = 900.0,
+ listed_before = {111},
+ listed_after = {111, 222},
+ )
+ assert failure is None, "a co-tenant freeing memory must not sink an attributed pid"
+ assert detail["process_vram_mib"] == 2600
+
+ def test_a_pid_already_present_but_unattributed_is_not_new(self):
+ """Only a pid that APPEARED counts; a co-tenant showing [N/A] throughout is theirs."""
+ failure, detail = self._verdict(
+ apps_before = {111: 500},
+ apps_after = {111: 500},
+ baseline = 1000.0,
+ settled = 1010.0,
+ listed_before = {111, 999},
+ listed_after = {111, 999},
+ )
+ assert "compute_apps_unattributed" not in detail
+ assert failure is not None and "no process appeared" in failure
+
+ def test_the_old_callers_still_work(self):
+ """The listed sets are optional, so a caller that has none keeps the old verdict."""
+ failure, _ = self._verdict(
+ apps_before = {},
+ apps_after = {222: 2600},
+ baseline = None,
+ settled = None,
+ )
+ assert failure is None
+
+
+class TestTheListingNamesItsPids:
+ def test_listed_pids_includes_unattributed_rows(self):
+ csv_text = "111, 500\n222, [N/A]\n"
+ assert gpu_assert.parse_compute_apps(csv_text) == {111: 500}
+ assert gpu_assert.listed_pids(csv_text) == {111, 222}
+
+ def test_listed_pids_ignores_headers_and_junk(self):
+ assert gpu_assert.listed_pids("pid, used_gpu_memory\n\nnotapid, 5\n333, 10\n") == {333}
+
+ def test_the_sample_is_taken_once(self):
+ """Two nvidia-smi calls would describe two different moments, so the attributed
+ mapping and the listed pids could disagree about which processes exist.
+ """
+ body = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")
+ assert body.count("nvidia_compute_apps_listing()") >= 3
+ assert "apps_before = attributed_apps(_listing_before)" in body
+ assert "apps_after = attributed_apps(_listing_after)" in body
+
+
+class TestASharedCardIsNotMeasuredByItsTotal:
+ """The device total only measures THIS process when THIS process owns the card.
+
+ Under --studio-concurrent a training leg shares it and both allocates and frees inside
+ the window: one recorded run read the delta as -182.0 MiB while the server genuinely
+ held 2.6 GB. Accepting a +200 MiB rise there would pass a CPU-served run on memory
+ somebody else allocated, so a fallback to the total is refused when anything was on the
+ card before the launch.
+ """
+
+ @staticmethod
+ def _verdict(**kwargs):
+ return run_studio_gpu.cli_run_gpu_failure(**kwargs)
+
+ def test_a_co_tenant_blocks_the_no_enumeration_fallback(self):
+ failure, detail = self._verdict(
+ apps_before = {111: 500},
+ apps_after = None,
+ baseline = 1000.0,
+ settled = 9000.0,
+ listed_before = {111},
+ listed_after = None,
+ )
+ assert failure is not None and "unmeasured rather than proven" in failure
+ assert detail["card_shared_before_launch"] is True
+
+ def test_a_co_tenant_blocks_the_mixed_listing_fallback(self):
+ failure, detail = self._verdict(
+ apps_before = {111: 500},
+ apps_after = {111: 500},
+ baseline = 1000.0,
+ settled = 9000.0,
+ listed_before = {111},
+ listed_after = {111, 222},
+ )
+ assert failure is not None and "unmeasured rather than proven" in failure
+ assert detail["card_shared_before_launch"] is True
+
+ def test_a_listed_but_unattributed_co_tenant_counts(self):
+ """A co-tenant reporting [N/A] is still a co-tenant."""
+ failure, _ = self._verdict(
+ apps_before = {},
+ apps_after = None,
+ baseline = 1000.0,
+ settled = 9000.0,
+ listed_before = {999},
+ listed_after = None,
+ )
+ assert failure is not None and "unmeasured rather than proven" in failure
+
+ def test_an_empty_card_keeps_the_fallback(self):
+ """The fallback exists for parts that report [N/A] for everything; it stays."""
+ failure, _ = self._verdict(
+ apps_before = {},
+ apps_after = None,
+ baseline = 1000.0,
+ settled = 1600.0,
+ listed_before = set(),
+ listed_after = None,
+ )
+ assert failure is None
+
+ def test_attribution_still_wins_over_the_rule(self):
+ """A co-tenant does not sink a run whose own process WAS attributed."""
+ failure, detail = self._verdict(
+ apps_before = {111: 500},
+ apps_after = {111: 500, 222: 2600},
+ baseline = 1000.0,
+ settled = 900.0,
+ listed_before = {111},
+ listed_after = {111, 222},
+ )
+ assert failure is None
+ assert detail["process_vram_mib"] == 2600
+
+ def test_unknown_co_tenancy_is_not_invented(self):
+ """No listing at all is no evidence either way, so behaviour is unchanged."""
+ failure, _ = self._verdict(
+ apps_before = None,
+ apps_after = None,
+ baseline = 1000.0,
+ settled = 1600.0,
+ )
+ assert failure is None
+
+
+class TestTheSamplesAreScopedToTheVisibleCard:
+ """nvidia-smi ignores CUDA_VISIBLE_DEVICES and answers for every PHYSICAL card.
+
+ Kaggle offers a two-GPU T4 session and build_kernel.py pins each payload to one card,
+ so both samplers were reading the whole box. An unrelated process starting on the
+ HIDDEN card and taking 200 MiB was then enough evidence for the device-wide delta to
+ carry a CPU-served run past the memory assertion.
+ """
+
+ @staticmethod
+ def _selector(value):
+ env = dict(os.environ)
+ if value is None:
+ env.pop("CUDA_VISIBLE_DEVICES", None)
+ else:
+ env["CUDA_VISIBLE_DEVICES"] = value
+ with mock.patch.dict(os.environ, env, clear = True):
+ return run_studio_gpu.visible_device_selector()
+
+ def test_unset_selects_every_card(self):
+ assert self._selector(None) is None
+
+ def test_a_pinned_card_selects_only_it(self):
+ assert self._selector("1") == "1"
+
+ def test_a_list_is_carried_through(self):
+ assert self._selector("0, 1") == "0,1"
+
+ def test_a_uuid_needs_no_mapping(self):
+ """`nvidia-smi -i` takes the GPU-UUID form directly."""
+ assert self._selector("GPU-84ccface-663f") == "GPU-84ccface-663f"
+
+ def test_an_empty_value_is_no_cards_not_all_of_them(self):
+ assert self._selector("") == ""
+
+ @staticmethod
+ def _scoped(value):
+ env = dict(os.environ)
+ if value is None:
+ env.pop("CUDA_VISIBLE_DEVICES", None)
+ else:
+ env["CUDA_VISIBLE_DEVICES"] = value
+ with mock.patch.dict(os.environ, env, clear = True):
+ return run_studio_gpu._scoped(["nvidia-smi", "--query-gpu=memory.used"])
+
+ def test_the_pin_reaches_the_command(self):
+ assert self._scoped("1") == ["nvidia-smi", "-i", "1", "--query-gpu=memory.used"]
+
+ def test_no_pin_leaves_the_command_alone(self):
+ assert self._scoped(None) == ["nvidia-smi", "--query-gpu=memory.used"]
+
+ def test_no_visible_card_takes_no_sample(self):
+ assert self._scoped("") is None
+
+ def test_both_samplers_are_scoped(self):
+ """The pair is the point: scoping one ruler and not the other still compares
+ a card-local reading against a box-wide one."""
+ seen = []
+
+ def fake_run(cmd, **kwargs):
+ seen.append(cmd)
+ return subprocess.CompletedProcess(cmd, 0, "7, 100\n", "")
+
+ with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "7"}):
+ with mock.patch.object(run_studio_gpu, "run", fake_run):
+ run_studio_gpu.nvidia_used_mib()
+ run_studio_gpu.nvidia_compute_apps_listing()
+ assert len(seen) == 2
+ for cmd in seen:
+ assert cmd[1:3] == ["-i", "7"], f"unscoped sample: {cmd}"
+
+ def test_neither_sampler_runs_without_a_visible_card(self):
+ def explode(cmd, **kwargs):
+ raise AssertionError(f"sampled with no visible card: {cmd}")
+
+ with mock.patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": ""}):
+ with mock.patch.object(run_studio_gpu, "run", explode):
+ assert run_studio_gpu.nvidia_used_mib() is None
+ assert run_studio_gpu.nvidia_compute_apps_listing() is None
+
+
+class TestTheServerIsStoppedBeforeTheCliBaselineRegardlessOfSkipUi:
+ """assert_chat_ui ends by stopping the Studio server, so with --skip-ui nothing did, and
+ the still-running llama-server sat in assert_cli_run's before-launch listing. On parts that
+ report [N/A] per process that pid read as a co-tenant, the device-delta fallback was
+ refused, and a GPU-backed run came back "unmeasured rather than proven"."""
+
+ def test_stop_server_precedes_the_cli_assertion_unconditionally(self):
+ text = Path(run_studio_gpu.__file__).read_text(encoding = "utf-8")
+ ui = text.index("if not self.args.skip_ui:\n self.assert_chat_ui()")
+ cli = text.index(" self.assert_cli_run()", ui)
+ between = text[ui:cli]
+ stop = between.rindex("self.stop_server()")
+ # At the method's own indentation, i.e. not inside the skip_ui branch.
+ line_start = between.rfind("\n", 0, stop) + 1
+ assert (
+ between[line_start:stop] == " "
+ ), "stop_server must not sit inside the skip_ui guard"
+
+ def test_the_card_is_settled_between_the_stop_and_the_assertion(self):
+ """Stopping is not the same as having stopped. Source order, because the two calls
+ are orchestration in run() and there is no seam between them to observe."""
+ text = Path(run_studio_gpu.__file__).read_text(encoding = "utf-8")
+ cli = text.index(" self.assert_cli_run()")
+ between = text[
+ text.index("if not self.args.skip_ui:\n self.assert_chat_ui()") : cli
+ ]
+ assert between.rindex("self.stop_server()") < between.rindex("wait_for_card_to_settle()")
+
+
+class TestTheCardIsGivenTimeToSettleAfterAStop:
+ """A terminated llama-server keeps its allocation, and keeps being LISTED, for a moment
+ after it exits. Sampled straight away that pid is assert_cli_run's before-launch state:
+ card_is_shared() withdraws the device-delta fallback and a GPU-backed run reports
+ "unmeasured rather than proven", which is the false red this PR set out to remove.
+ """
+
+ @staticmethod
+ def _settle(samples):
+ """Drive wait_for_card_to_settle over `samples` of (used_mib, listed pids), the
+ last one repeating. (None, None) is an nvidia-smi that did not answer. Returns the
+ number of polls taken: two nvidia-smi calls each, the entry sample included."""
+ taken = []
+
+ def fake_run(cmd, **kwargs):
+ used, pids = samples[min(len(taken) // 2, len(samples) - 1)]
+ taken.append(cmd)
+ if pids is None:
+ return subprocess.CompletedProcess(cmd, 9, "", "no devices were found")
+ if "--query-compute-apps=pid,used_gpu_memory" in cmd:
+ # The [N/A] shape: every pid listed, none of them attributed.
+ return subprocess.CompletedProcess(
+ cmd, 0, "".join(f"{p}, [N/A]\n" for p in pids), ""
+ )
+ return subprocess.CompletedProcess(cmd, 0, f"{used}\n", "")
+
+ with mock.patch.object(run_studio_gpu, "run", fake_run):
+ with mock.patch.object(run_studio_gpu.time, "sleep", lambda _s: None):
+ run_studio_gpu.wait_for_card_to_settle()
+ return len(taken) // 2
+
+ def test_it_waits_for_the_stopped_server_to_leave_the_listing(self):
+ """Two equal samples are not proof of a settled card when the driver has not begun
+ giving the memory back, which is why the pid is what is waited on."""
+ assert self._settle([(3400.0, [4242]), (3400.0, [4242]), (200.0, []), (200.0, [])]) == 5
+
+ def test_it_waits_for_the_memory_to_come_back(self):
+ """The pid can go while the driver is still returning the allocation."""
+ assert self._settle([(3400.0, []), (1800.0, []), (200.0, []), (200.0, [])]) == 5
+
+ def test_a_stalled_reclaim_with_no_pid_to_wait_on_is_not_settled(self):
+ """The pid can also be gone BEFORE the first sample, leaving nothing to wait on. One
+ flat interval is then a stall as easily as a finished reclaim, and taking it as
+ settled hands assert_cli_run a baseline with the old model still in it."""
+ assert self._settle([(3400.0, []), (3400.0, []), (200.0, []), (200.0, [])]) == 5
+
+ def test_a_settled_card_costs_only_the_quiet_window(self):
+ assert self._settle([(200.0, [])]) == run_studio_gpu.VRAM_SETTLE_QUIET_POLLS + 1
+
+ def test_a_co_tenant_arriving_afterwards_does_not_hold_the_run(self):
+ """Only the pids the card carried at entry are the stop's to wait for."""
+ assert self._settle([(200.0, []), (2600.0, [777]), (2600.0, [777])]) == 3
+
+ def test_an_unanswering_smi_is_not_waited_on(self):
+ assert self._settle([(None, None)]) == 3
+
+ def test_the_wait_is_bounded(self):
+ """A co-tenant never leaves, and a card somebody else is draining never settles."""
+ assert (
+ self._settle([(3400.0 - 100.0 * i, [4242]) for i in range(60)])
+ == run_studio_gpu.VRAM_SETTLE_SAMPLES + 1
+ )
+
+ def test_the_settled_listing_no_longer_reads_as_a_co_tenant(self):
+ """The point of the wait, stated as the verdict it changes."""
+ stale = run_studio_gpu.cli_run_gpu_failure(
+ {},
+ None,
+ 3400.0,
+ 3900.0,
+ {4242},
+ {4242, 5555},
+ )[0]
+ assert stale is not None and "already on the card" in stale
+ settled = run_studio_gpu.cli_run_gpu_failure(
+ {},
+ None,
+ 200.0,
+ 700.0,
+ set(),
+ {5555},
+ )[0]
+ assert settled is None
+
+
+class TestTheTwoRulersAreReadInTheRightOrder:
+ """Which pids count as new, and which reading is allowed to overrule which.
+
+ Both of these are the mixed listing, where one new pid carries a figure and another
+ does not, and both come out of splitting "listed" from "attributed": the split is only
+ half applied unless the exclusion uses it too, and the weaker ruler must not be allowed
+ to contradict the stronger one.
+ """
+
+ @staticmethod
+ def _verdict(**kwargs):
+ return run_studio_gpu.cli_run_gpu_failure(**kwargs)
+
+ def test_a_flat_device_total_does_not_disprove_an_attributed_process(self):
+ """2.6 GiB read off pid 222 is not a CPU-served run because pid 333 was also new
+ and unreadable. The total is a shared counter and cannot outrank a direct reading;
+ which of the two is the server is still unknown, so this hedges rather than passes.
+ """
+ failure, detail = self._verdict(
+ apps_before = {},
+ apps_after = {222: 2600},
+ baseline = 1000.0,
+ settled = 1050.0,
+ listed_before = set(),
+ listed_after = {222, 333},
+ )
+ assert failure is not None
+ assert "served from the CPU" not in failure, failure
+ assert "unmeasured rather than disproven" in failure
+ assert detail["process_vram_mib"] == 2600
+ assert detail["compute_apps_unattributed"] == [333]
+
+ def test_a_co_tenant_that_becomes_readable_is_still_not_new(self):
+ """pid 77 was on the card at launch and merely unattributable then. Keying the
+ exclusion on the attributed map alone counted its 500 MiB as this launch's and
+ passed a server that never left the CPU."""
+ failure, detail = self._verdict(
+ apps_before = {},
+ apps_after = {77: 500},
+ baseline = 1000.0,
+ settled = 1000.0,
+ listed_before = {77},
+ listed_after = {77},
+ )
+ assert failure is not None and "no process appeared" in failure
+ assert detail["compute_apps_appeared"] == {}
+
+ def test_the_same_co_tenant_does_not_carry_the_mixed_listing_either(self):
+ failure, detail = self._verdict(
+ apps_before = {},
+ apps_after = {77: 500},
+ baseline = 1000.0,
+ settled = 1000.0,
+ listed_before = {77},
+ listed_after = {77, 888},
+ )
+ assert failure is not None and "unmeasured rather than proven" in failure
+ assert detail["card_shared_before_launch"] is True
+
+ def test_an_attributed_launch_on_an_empty_card_still_passes(self):
+ """The ordinary verdict is untouched: the exclusion only ever grows."""
+ failure, detail = self._verdict(
+ apps_before = {},
+ apps_after = {222: 2600},
+ baseline = 1000.0,
+ settled = 3600.0,
+ listed_before = set(),
+ listed_after = {222},
+ )
+ assert failure is None
+ assert detail["process_vram_mib"] == 2600