Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop the dense-quant probes pinning a CUDA context on every card of a multi-GPU host (#11954)
Leo Borcherding5592fc5af917
diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.pyindex db486cad475..61db8a0f083 100644--- a/studio/backend/core/inference/diffusion_transformer_quant.py+++ b/studio/backend/core/inference/diffusion_transformer_quant.py@@ -634,10 +634,12 @@ def _strip_paths(text: str) -> str:_SMOKE_CACHE: dict[tuple[str, str], bool] = {}-def _smoke_cache_device_key(device: str) -> str:- """``device`` qualified with the current CUDA index, so each card is validated on its own."""+def _smoke_cache_device_key(device: str, ordinal: Optional[int] = None) -> str:+ """``device`` qualified with the current CUDA index, or ``ordinal`` when given."""if device != "cuda":return device+ if ordinal is not None:+ return f"cuda:{ordinal}"try:import torchreturn f"cuda:{torch.cuda.current_device()}"@@ -1082,10 +1084,11 @@ def dense_quant_host_capable(target: Any) -> bool:# import fails, so an ABI skew would advertise a fast path every load then falls back from.if torchao_unavailable_reason() is not None:return False- cap = _capability()+ ordinal = getattr(target, "ordinal", None)+ cap = _capability(ordinal)if cap is None:return False- card = _smoke_cache_device_key(str(getattr(target, "device", "cuda")))+ card = _smoke_cache_device_key(str(getattr(target, "device", "cuda")), ordinal)for floor, schemes in _AUTO_LADDER:if cap >= floor:return any(_SMOKE_CACHE.get((scheme, card), True) for scheme in without_nvfp4(schemes))@@ -1183,11 +1186,12 @@ def auto_scheme_candidates_cached(target: Any, family: Optional[str] = None) ->published ladder sharpens as loads record verdicts instead of paying for them here."""if not dense_transformer_supported(target):return ()- cap = _capability()+ ordinal = getattr(target, "ordinal", None)+ cap = _capability(ordinal)if cap is None:return ()device = str(getattr(target, "device", "cuda"))- card = _smoke_cache_device_key(device)+ card = _smoke_cache_device_key(device, ordinal)return tuple(schemefor scheme in _auto_scheme_order(family, device, cap)@@ -1196,10 +1200,10 @@ def auto_scheme_candidates_cached(target: Any, family: Optional[str] = None) ->)-def _capability() -> Optional[tuple[int, int]]:+def _capability(ordinal: Optional[int] = None) -> Optional[tuple[int, int]]:try:import torch- major, minor = torch.cuda.get_device_capability()+ major, minor = torch.cuda.get_device_capability(ordinal)return (int(major), int(minor))except Exception:return Nonediff --git a/studio/backend/main.py b/studio/backend/main.pyindex c1e5b4adcf6..b41c105be94 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -2282,10 +2282,7 @@ def _probe_dense_quant_supported() -> bool:sharpens, since an unprobed scheme counts as usable and a later load can record a kernelfailure in ``_SMOKE_CACHE``."""try:- from core.inference.diffusion_device import (- diffusion_device_scope,- resolve_diffusion_device_target,- )+ from core.inference.diffusion_device import resolve_diffusion_device_targetfrom core.inference.diffusion_transformer_quant import dense_quant_host_capableimport torch@@ -2293,10 +2290,10 @@ def _probe_dense_quant_supported() -> bool:count = torch.cuda.device_count() if torch.cuda.is_available() else 0if count <= 1:return bool(dense_quant_host_capable(resolve_diffusion_device_target()))+ # No device scope: cudaSetDevice pins a primary context on every card (CUDA 12).for ordinal in range(count):- with diffusion_device_scope(ordinal):- if not dense_quant_host_capable(resolve_diffusion_device_target(ordinal = ordinal)):- return False+ if not dense_quant_host_capable(resolve_diffusion_device_target(ordinal = ordinal)):+ return Falsereturn Trueexcept Exception: # noqa: BLE001 -- a capability probe must never fail a status requestreturn False@@ -2311,10 +2308,7 @@ def _probe_dense_quant_schemes() -> list[str]:the load-time helper runs ``_scheme_supported``, which spawns the smoke probe or allocates inthis process. Like the capability bit, it sharpens as loads record verdicts in ``_SMOKE_CACHE``."""try:- from core.inference.diffusion_device import (- diffusion_device_scope,- resolve_diffusion_device_target,- )+ from core.inference.diffusion_device import resolve_diffusion_device_targetfrom core.inference.diffusion_transformer_quant import auto_scheme_candidates_cachedimport torch@@ -2324,10 +2318,9 @@ def _probe_dense_quant_schemes() -> list[str]:return list(auto_scheme_candidates_cached(resolve_diffusion_device_target()))common: Optional[list[str]] = Nonefor ordinal in range(count):- with diffusion_device_scope(ordinal):- schemes = list(- auto_scheme_candidates_cached(resolve_diffusion_device_target(ordinal = ordinal))- )+ schemes = list(+ auto_scheme_candidates_cached(resolve_diffusion_device_target(ordinal = ordinal))+ )common = schemes if common is None else [s for s in common if s in schemes]return common or []except Exception: # noqa: BLE001 -- a capability probe must never fail a status requestdiff --git a/studio/backend/tests/test_dense_quant_capability.py b/studio/backend/tests/test_dense_quant_capability.pyindex 4e1dc0c51a7..4616d78e45f 100644--- a/studio/backend/tests/test_dense_quant_capability.py+++ b/studio/backend/tests/test_dense_quant_capability.py@@ -22,33 +22,42 @@ def _src(name: str) -> str:return ast.get_source_segment(src, node)-def _run(monkeypatch, *, device_count, capable_by_ordinal):- """Run an uncached copy of ``_dense_quant_supported`` with mocked dependencies."""- scoped: list = []-- fake_torch = types.SimpleNamespace(- cuda = types.SimpleNamespace(- is_available = lambda: device_count > 0,- device_count = lambda: device_count,- )+def _cuda_namespace(device_count, switched):+ """A ``torch.cuda`` whose context-pinning calls record, then raise."""++ def _forbid(name):+ def _call(*_a, **_k):+ switched.append(name)+ raise AssertionError(f"torch.cuda.{name} would pin a context")++ return _call++ return types.SimpleNamespace(+ is_available = lambda: device_count > 0,+ device_count = lambda: device_count,+ device = _forbid("device"),+ set_device = _forbid("set_device"),+ _exchange_device = _forbid("_exchange_device"),+ _maybe_exchange_device = _forbid("_maybe_exchange_device"),+ mem_get_info = _forbid("mem_get_info"),)- class _Scope:- def __init__(self, ordinal):- self.ordinal = ordinal-- def __enter__(self):- scoped.append(self.ordinal)- return None- def __exit__(self, *exc):- return False+def _run(monkeypatch, *, device_count, capable_by_ordinal):+ """Run an uncached copy of ``_probe_dense_quant_supported`` with mocked dependencies."""+ switched: list = []+ asked: list = []+ fake_torch = types.SimpleNamespace(cuda = _cuda_namespace(device_count, switched))fake_device = types.ModuleType("core.inference.diffusion_device")- fake_device.diffusion_device_scope = _Scopefake_device.resolve_diffusion_device_target = lambda ordinal = None: ordinalfake_quant = types.ModuleType("core.inference.diffusion_transformer_quant")- fake_quant.dense_quant_host_capable = lambda target: capable_by_ordinal[target]++ def _capable(target):+ asked.append(target)+ return capable_by_ordinal[target]++ fake_quant.dense_quant_host_capable = _capablemonkeypatch.setitem(sys.modules, "torch", fake_torch)monkeypatch.setitem(sys.modules, "core.inference.diffusion_device", fake_device)@@ -56,35 +65,41 @@ def _run(monkeypatch, *, device_count, capable_by_ordinal):namespace: dict = {}exec(_src("_probe_dense_quant_supported"), namespace) # noqa: S102 -- the real body- return namespace["_probe_dense_quant_supported"](), scoped+ return namespace["_probe_dense_quant_supported"](), switched, askeddef test_a_single_capable_gpu_reports_capable(monkeypatch):- result, scoped = _run(monkeypatch, device_count = 1, capable_by_ordinal = {None: True})+ result, switched, asked = _run(monkeypatch, device_count = 1, capable_by_ordinal = {None: True})assert result is True- assert scoped == []+ assert switched == []+ assert asked == [None]def test_a_single_incapable_gpu_reports_incapable(monkeypatch):- result, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = {None: False})+ result, _, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = {None: False})assert result is Falsedef test_every_visible_card_must_be_capable(monkeypatch):- result, scoped = _run(monkeypatch, device_count = 2, capable_by_ordinal = {0: True, 1: False})+ result, switched, asked = _run(+ monkeypatch, device_count = 2, capable_by_ordinal = {0: True, 1: False}+ )assert result is False- # Probe each ordinal under its own device scope.- assert scoped == [0, 1]+ assert asked == [0, 1]+ assert switched == []def test_a_homogeneous_multi_gpu_host_still_reports_capable(monkeypatch):- result, scoped = _run(monkeypatch, device_count = 2, capable_by_ordinal = {0: True, 1: True})+ result, switched, asked = _run(+ monkeypatch, device_count = 2, capable_by_ordinal = {0: True, 1: True}+ )assert result is True- assert scoped == [0, 1]+ assert asked == [0, 1]+ assert switched == []def test_no_gpu_reports_incapable(monkeypatch):- result, _ = _run(monkeypatch, device_count = 0, capable_by_ordinal = {None: False})+ result, _, _ = _run(monkeypatch, device_count = 0, capable_by_ordinal = {None: False})assert result is False@@ -95,15 +110,22 @@ def test_a_probe_failure_reports_incapable(monkeypatch):def __getitem__(self, key):raise RuntimeError("driver went away")- result, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = _Boom())+ result, _, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = _Boom())assert result is False[email protected]("needle", ["diffusion_device_scope", "device_count"])+ "needle", ["device_count", "resolve_diffusion_device_target(ordinal = ordinal)"]+)def test_the_wiring_stays_in_place(needle):assert needle in _src("_probe_dense_quant_supported")[email protected]("name", ["_probe_dense_quant_supported", "_probe_dense_quant_schemes"])+def test_the_probes_never_switch_the_current_device(name):+ assert "diffusion_device_scope" not in _src(name)++def test_the_capability_is_published_and_never_memoised():"""`/api/system` carries the bit, and the probe must not be pinned.@@ -138,10 +160,10 @@ def test_the_polled_route_never_imports_the_ml_stack():def test_the_probe_follows_the_smoke_cache_as_it_warms(monkeypatch):"""A verdict the loader has since paid for must reach the next resolution."""answers = {None: True}- result, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = answers)+ result, _, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = answers)assert result is Trueanswers[None] = False- result, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = answers)+ result, _, _ = _run(monkeypatch, device_count = 1, capable_by_ordinal = answers)assert result is False@@ -169,31 +191,19 @@ def test_the_ladder_is_read_off_the_same_refresh_as_the_bit():def _run_schemes(monkeypatch, *, device_count, schemes_by_ordinal):"""Run an uncached copy of ``_probe_dense_quant_schemes`` with mocked dependencies."""- scoped: list = []-- fake_torch = types.SimpleNamespace(- cuda = types.SimpleNamespace(- is_available = lambda: device_count > 0,- device_count = lambda: device_count,- )- )-- class _Scope:- def __init__(self, ordinal):- self.ordinal = ordinal-- def __enter__(self):- scoped.append(self.ordinal)- return None-- def __exit__(self, *exc):- return False+ switched: list = []+ asked: list = []+ fake_torch = types.SimpleNamespace(cuda = _cuda_namespace(device_count, switched))fake_device = types.ModuleType("core.inference.diffusion_device")- fake_device.diffusion_device_scope = _Scopefake_device.resolve_diffusion_device_target = lambda ordinal = None: ordinalfake_quant = types.ModuleType("core.inference.diffusion_transformer_quant")- fake_quant.auto_scheme_candidates_cached = lambda target: schemes_by_ordinal[target]++ def _ladder(target):+ asked.append(target)+ return schemes_by_ordinal[target]++ fake_quant.auto_scheme_candidates_cached = _laddermonkeypatch.setitem(sys.modules, "torch", fake_torch)monkeypatch.setitem(sys.modules, "core.inference.diffusion_device", fake_device)@@ -201,34 +211,36 @@ def _run_schemes(monkeypatch, *, device_count, schemes_by_ordinal):namespace: dict = {"Optional": Optional}exec(_src("_probe_dense_quant_schemes"), namespace) # noqa: S102 -- the real body- return namespace["_probe_dense_quant_schemes"](), scoped+ return namespace["_probe_dense_quant_schemes"](), switched, askeddef test_an_ada_host_publishes_fp8_first(monkeypatch):- result, scoped = _run_schemes(+ result, switched, asked = _run_schemes(monkeypatch, device_count = 1, schemes_by_ordinal = {None: ("fp8", "int8")})assert result == ["fp8", "int8"]- assert scoped == []+ assert switched == []+ assert asked == [None]def test_an_ampere_host_publishes_int8(monkeypatch):- result, _ = _run_schemes(monkeypatch, device_count = 1, schemes_by_ordinal = {None: ("int8",)})+ result, _, _ = _run_schemes(monkeypatch, device_count = 1, schemes_by_ordinal = {None: ("int8",)})assert result == ["int8"]def test_a_mixed_host_publishes_only_what_every_card_runs(monkeypatch):- result, scoped = _run_schemes(+ result, switched, asked = _run_schemes(monkeypatch,device_count = 2,schemes_by_ordinal = {0: ("fp8", "int8"), 1: ("int8",)},)assert result == ["int8"]- assert scoped == [0, 1]+ assert asked == [0, 1]+ assert switched == []def test_an_unsupported_host_publishes_nothing(monkeypatch):- result, _ = _run_schemes(monkeypatch, device_count = 0, schemes_by_ordinal = {None: ()})+ result, _, _ = _run_schemes(monkeypatch, device_count = 0, schemes_by_ordinal = {None: ()})assert result == []@@ -237,7 +249,7 @@ def test_a_scheme_probe_failure_publishes_nothing(monkeypatch):def __getitem__(self, key):raise RuntimeError("driver went away")- result, _ = _run_schemes(monkeypatch, device_count = 1, schemes_by_ordinal = _Boom())+ result, _, _ = _run_schemes(monkeypatch, device_count = 1, schemes_by_ordinal = _Boom())assert result == []@@ -275,11 +287,56 @@ def test_the_polled_ladder_never_runs_the_allocating_smoke_probe(monkeypatch):monkeypatch.setattr(tq, "_scheme_supported", _never)monkeypatch.setattr(tq, "dense_transformer_supported", lambda _target: True)- monkeypatch.setattr(tq, "_capability", lambda: (8, 9))- monkeypatch.setattr(tq, "_smoke_cache_device_key", lambda _device: "cuda:0")+ monkeypatch.setattr(tq, "_capability", lambda ordinal = None: (8, 9))+ monkeypatch.setattr(tq, "_smoke_cache_device_key", lambda _device, ordinal = None: "cuda:0")monkeypatch.setattr(tq, "_SMOKE_CACHE", {("fp8", "cuda:0"): False})# An unprobed scheme counts as usable; a probed failure does not.assert tq.auto_scheme_candidates_cached(object()) == ("int8",)monkeypatch.setattr(tq, "_SMOKE_CACHE", {})assert tq.auto_scheme_candidates_cached(object()) == ("int8", "fp8")assert "auto_scheme_candidates_cached" in _src("_probe_dense_quant_schemes")+++def test_the_cached_readers_ask_the_target_card_without_switching(monkeypatch):+ from core.inference import diffusion_speed+ from core.inference import diffusion_transformer_quant as tq++ asked: list = []+ forbidden: list = []++ def _forbid(name):+ def _call(*_a, **_k):+ forbidden.append(name)+ raise AssertionError(f"torch.cuda.{name} would pin a context")++ return _call++ torch = types.ModuleType("torch")+ torch.cuda = types.SimpleNamespace(+ is_available = lambda: True,+ device_count = lambda: 2,+ get_device_capability = lambda device = None: asked.append(device) or (8, 9),+ current_device = lambda: 0,+ device = _forbid("device"),+ set_device = _forbid("set_device"),+ _exchange_device = _forbid("_exchange_device"),+ mem_get_info = _forbid("mem_get_info"),+ )+ monkeypatch.setitem(sys.modules, "torch", torch)+ monkeypatch.setattr(diffusion_speed, "compile_eligible", lambda target, **kw: True)+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda _target: True)+ monkeypatch.setattr(tq, "_TORCHAO_UNAVAILABLE", (None,))+ monkeypatch.setattr(tq, "_SMOKE_CACHE", {("fp8", "cuda:1"): False})++ card1 = types.SimpleNamespace(device = "cuda", ordinal = 1)+ assert tq.dense_quant_host_capable(card1) is True+ assert tq.auto_scheme_candidates_cached(card1) == ("int8",)+ assert asked == [1, 1]+ card0 = types.SimpleNamespace(device = "cuda", ordinal = 0)+ assert tq.auto_scheme_candidates_cached(card0) == ("int8", "fp8")+ assert asked == [1, 1, 0]+ del asked[:]+ bare = types.SimpleNamespace(device = "cuda")+ assert tq.auto_scheme_candidates_cached(bare) == ("int8", "fp8")+ assert asked == [None]+ assert forbidden == []diff --git a/studio/backend/tests/test_diffusion_transformer_quant.py b/studio/backend/tests/test_diffusion_transformer_quant.pyindex 8645ebfec5b..8cfa90952fe 100644--- a/studio/backend/tests/test_diffusion_transformer_quant.py+++ b/studio/backend/tests/test_diffusion_transformer_quant.py@@ -2643,9 +2643,9 @@ def _capable_host(monkeypatch.setattr(diffusion_speed, "compile_eligible", lambda target, **kw: True)monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: True)- monkeypatch.setattr(tq, "_capability", lambda: cap)+ monkeypatch.setattr(tq, "_capability", lambda ordinal = None: cap)monkeypatch.setattr(tq, "_TORCHAO_UNAVAILABLE", (torchao_reason,))- monkeypatch.setattr(tq, "_smoke_cache_device_key", lambda device: "cuda:0")+ monkeypatch.setattr(tq, "_smoke_cache_device_key", lambda device, ordinal = None: "cuda:0")monkeypatch.setattr(tq, "_SMOKE_CACHE", {})diff --git a/studio/backend/tests/test_nvfp4_diffusion_flag.py b/studio/backend/tests/test_nvfp4_diffusion_flag.pyindex fde4ba70ff8..f376a561135 100644--- a/studio/backend/tests/test_nvfp4_diffusion_flag.py+++ b/studio/backend/tests/test_nvfp4_diffusion_flag.py@@ -51,7 +51,7 @@ def test_a_video_family_without_a_usable_hosted_seed_spends_no_smoke_probe(monketq, "_smoke_probe", lambda s, d, unproven_ok = False: calls.append(s) or unproven_ok)monkeypatch.setattr(tq, "dense_transformer_supported", lambda t: True)- monkeypatch.setattr(tq, "_capability", lambda: (10, 0))+ monkeypatch.setattr(tq, "_capability", lambda ordinal = None: (10, 0))monkeypatch.setattr(torch.cuda, "is_available", lambda: True)target = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16, ordinal = 0)for fam in _FAMILIES:@@ -110,7 +110,7 @@ def _blackwell(monkeypatch, *, supported = ("int8", "fp8", "nvfp4", "mxfp8")):import torchmonkeypatch.setattr(torch.cuda, "is_available", lambda: True)- monkeypatch.setattr(tq, "_capability", lambda: (10, 0))+ monkeypatch.setattr(tq, "_capability", lambda ordinal = None: (10, 0))monkeypatch.setattr(tq, "dense_transformer_supported", lambda target: True)monkeypatch.setattr(tq, "_is_consumer_gpu", lambda device = None: False)monkeypatch.setattr(tq, "_child_probe_table", lambda device: None)diff --git a/studio/backend/tests/test_system_poll_no_cuda_context.py b/studio/backend/tests/test_system_poll_no_cuda_context.pyindex 35eed8614f6..e31a1a25cf5 100644--- a/studio/backend/tests/test_system_poll_no_cuda_context.py+++ b/studio/backend/tests/test_system_poll_no_cuda_context.py@@ -233,6 +233,140 @@ def test_the_probe_would_catch_a_regression():assert mib > 0, "mem_get_info created no context here, so the assertions above prove nothing"+def _cuda_device_count() -> int:+ try:+ import torch+ return int(torch.cuda.device_count()) if torch.cuda.is_available() else 0+ except Exception:+ return 0+++def _is_hip() -> bool:+ try:+ import torch+ return bool(getattr(torch.version, "hip", None))+ except Exception:+ return False+++needs_two_nvidia = pytest.mark.skipif(+ _cuda_device_count() < 2 or _is_hip(),+ reason = "needs two CUDA devices on NVIDIA to observe a per-card primary context",+)++_PROBE_BY_GPU = textwrap.dedent(+ """+ import os, subprocess+ def _held_by_gpu():+ try:+ out = subprocess.run(+ ["nvidia-smi", "--query-compute-apps=gpu_uuid,pid,used_memory",+ "--format=csv,noheader,nounits"],+ capture_output = True, text = True, timeout = 60,+ )+ except Exception:+ return None+ if out.returncode != 0:+ return None+ me = str(os.getpid())+ held = {}+ for line in out.stdout.splitlines():+ parts = [p.strip() for p in line.split(",")]+ if len(parts) >= 3 and parts[1] == me:+ try:+ held[parts[0]] = held.get(parts[0], 0) + int(parts[2])+ except ValueError:+ return None+ return held+ def _primary_contexts():+ import torch+ has = getattr(torch._C, "_cuda_hasPrimaryContext", None)+ if has is None:+ return None+ return [i for i in range(torch.cuda.device_count()) if has(i)]+ """+)++_BY_GPU_CHILD = textwrap.dedent(+ """+ import sys+ sys.path.insert(0, {backend!r})+ import torch+ assert sum(torch.cuda.memory_reserved(i) for i in range(torch.cuda.device_count())) == 0+ visited = None+ {call}+ print("VISITED:" + repr(visited))+ print("HELD:" + repr(_held_by_gpu()))+ print("PRIMARY:" + repr(_primary_contexts()))+ """+)++# Probe bodies exec'd from main.py source: importing main builds the app.+_DENSE_QUANT_PROBES = textwrap.dedent(+ """+ import ast+ from typing import Optional+ src = open({main!r}, encoding = "utf-8").read()+ tree = ast.parse(src)+ ns = {{"Optional": Optional}}+ for name in ("_probe_dense_quant_supported", "_probe_dense_quant_schemes"):+ node = next(n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == name)+ exec(ast.get_source_segment(src, node), ns)+ from core.inference import diffusion_transformer_quant as tq+ visited = {{"bit": [], "ladder": []}}+ _capable, _ladder = tq.dense_quant_host_capable, tq.auto_scheme_candidates_cached+ def _bit(t):+ visited["bit"].append(getattr(t, "ordinal", None))+ return _capable(t)+ def _schemes(t):+ visited["ladder"].append(getattr(t, "ordinal", None))+ return _ladder(t)+ tq.dense_quant_host_capable = _bit+ tq.auto_scheme_candidates_cached = _schemes+ ns["_probe_dense_quant_supported"]()+ ns["_probe_dense_quant_schemes"]()+ """+).format(main = str(_BACKEND_DIR / "main.py"))+++def _run_child_by_gpu(call: str):+ """Run ``call`` in a fresh interpreter; return (held_by_gpu | None, primary | None, visited)."""+ src = _PROBE_BY_GPU + _BY_GPU_CHILD.format(backend = str(_BACKEND_DIR), call = call)+ proc = subprocess.run([sys.executable, "-c", src], capture_output = True, text = True, timeout = 600)+ assert proc.returncode == 0, f"child failed:\n{proc.stdout}\n{proc.stderr}"+ out: dict = {}+ for line in proc.stdout.splitlines():+ for key in ("VISITED", "HELD", "PRIMARY"):+ if line.startswith(key + ":"):+ out[key] = eval(line[len(key) + 1 :])+ assert set(out) == {"VISITED", "HELD", "PRIMARY"}, proc.stdout+ return out["HELD"], out["PRIMARY"], out["VISITED"]+++@needs_two_nvidia+def test_the_dense_quant_probes_pin_no_context_on_any_card():+ held, primary, visited = _run_child_by_gpu(_DENSE_QUANT_PROBES)+ # A swallowed import error also answers "incapable"; prove the loop visited every card.+ assert visited["ladder"] == list(range(_cuda_device_count())), visited+ assert visited["bit"][:1] == [0], visited+ if held is None and primary is None:+ pytest.skip("no per-PID nvidia-smi accounting and no torch primary-context flag")+ if held is not None:+ assert not any(held.values()), f"the dense-quant probes pinned a context: {held}"+ if primary is not None:+ assert primary == [], f"the dense-quant probes pinned a primary context on {primary}"+++@needs_two_nvidia+def test_the_multi_gpu_probe_would_catch_a_regression():+ # Negative control: the pre-fix scoped call must register a context.+ held, primary, _ = _run_child_by_gpu("torch.cuda.device(1).__enter__()")+ if held is None and primary is None:+ pytest.skip("no per-PID nvidia-smi accounting and no torch primary-context flag")+ pinned = bool(held and any(held.values())) or bool(primary and 1 in primary)+ assert pinned, "entering torch.cuda.device(1) pinned nothing, so the test above proves nothing"++def _summary_torch(*, properties_total, driver_free, driver_total):props = types.SimpleNamespace(total_memory = properties_total, name = "Test GPU")