Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Ask the spoof-survival question in a subprocess, not in the worker (#10866)
Daniel Han1a6c33b45323
diff --git a/tests/python/test_accelerator_skip_guards.py b/tests/python/test_accelerator_skip_guards.pyindex eb254424070..0368a219741 100644--- a/tests/python/test_accelerator_skip_guards.py+++ b/tests/python/test_accelerator_skip_guards.py@@ -27,6 +27,10 @@ Scanned with ast rather than grep so a reformatted decorator, a multi-linefrom __future__ import annotationsimport ast+import json+import subprocess+import sys+import textwrapfrom pathlib import Pathimport pytest@@ -105,16 +109,13 @@ def test_no_skip_guard_reads_a_spoofable_accelerator_probe():)-def test_the_recorded_answer_survives_the_spoof():- """The property the helper exists for, asserted rather than assumed.-- Primed by conftest before collection, so by the time this runs the aggressive- spoof may or may not have been applied depending on what else is in the- session. Apply it here and check the recorded answer does not move.+_SURVIVES_THE_SPOOF_PROBE = textwrap.dedent("""- import sys+ import json, sys++ sys.path.insert(0, {tests_root!r})+ sys.path.insert(0, {shared_dir!r})- sys.path.insert(0, str(_TESTS_ROOT))from real_accelerator import has_real_acceleratorbefore = has_real_accelerator()@@ -125,13 +126,87 @@ def test_the_recorded_answer_survives_the_spoof():import torch- assert torch.cuda.is_available() is True, (+ print("SPOOF_PROBE " + json.dumps({{+ "before": before,+ "after": has_real_accelerator(),+ "spoof_patched_is_available": bool(torch.cuda.is_available()),+ }}))+ """+)+++def test_the_recorded_answer_survives_the_spoof():+ """The property the helper exists for, asserted rather than assumed.++ In a SUBPROCESS, and that is not incidental. `spoof.apply()` sets+ `torch.cuda.is_available` to return True and never restores it, so calling it+ in-process poisons the rest of the xdist worker: every later test on that+ worker sees a machine with a CUDA card that is not there. peft's+ `infer_device()` is one of the things that reads it, so+ tests/test_save_lora_without_vllm.py then asks safetensors to load onto CUDA+ and dies with++ NotImplementedError: Could not run 'aten::empty_strided' with arguments+ from the 'CUDA' backend++ which is the exact class of cross-test damage this file exists to prevent. An+ earlier version of this test did apply the spoof in-process and caused that+ failure; it reproduces deterministically with just this file and that one, in+ that order.++ Same subprocess pattern as tests/vllm_compat/test_unsloth_zoo_imports.py+ (#10855), for the same reason: a question about global state has to be asked+ somewhere the answer cannot leak back.+ """+ proc = subprocess.run(+ [+ sys.executable,+ "-c",+ _SURVIVES_THE_SPOOF_PROBE.format(+ tests_root = str(_TESTS_ROOT),+ shared_dir = str(_TESTS_ROOT / "_shared"),+ ),+ ],+ capture_output = True,+ text = True,+ timeout = 300,+ cwd = str(_TESTS_ROOT.parent),+ )+ combined = proc.stdout + proc.stderr+ assert proc.returncode == 0, f"the spoof probe failed to run:\n{combined[-4000:]}"++ marker = "SPOOF_PROBE "+ line = next((l for l in proc.stdout.splitlines() if l.startswith(marker)), None)+ assert line is not None, f"probe produced no verdict:\n{combined[-4000:]}"+ verdict = json.loads(line[len(marker) :])++ assert verdict["spoof_patched_is_available"] is True, ("the spoof no longer patches torch.cuda.is_available, so this test is not ""checking anything; re-point it at whatever it patches now")- assert has_real_accelerator() is before, (+ assert verdict["after"] == verdict["before"], ("has_real_accelerator() moved after the spoof was applied, which is the whole "- "thing it is supposed to be immune to"+ f"thing it is supposed to be immune to: {verdict}"+ )+++def test_this_file_never_applies_the_spoof_in_process():+ """The regression guard for the bug the test above used to be.++ Applying the spoof in-process is invisible here and fails somewhere else+ entirely, on whichever test the xdist scheduler happens to put next on the+ same worker. So pin it structurally rather than trusting it to stay fixed.+ """+ tree = ast.parse(Path(__file__).read_text(encoding = "utf-8"))+ offenders = [+ f"line {node.lineno}"+ for node in ast.walk(tree)+ if isinstance(node, ast.Call) and _dotted(node.func)[-2:] == ("spoof", "apply")+ ]+ assert not offenders, (+ "this file calls spoof.apply() in-process. That leaves torch.cuda.is_available "+ "patched for every later test on this xdist worker. Ask it in a subprocess "+ f"instead, as _SURVIVES_THE_SPOOF_PROBE does: {offenders}")