Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Hermetic tests for the on-demand FlashInfer install behind NVFP4: installer, torch probe and venv are fakes."""
from __future__ import annotations
import os
import pathlib
import re
import sys
import threading
import time
import types
import pytest
from core.inference import diffusion_nvfp4_dispatch as dispatch
from core.inference import diffusion_nvfp4_install as inst
from core.inference import diffusion_nvfp4_ops as ops
_INFERENCE_DIR = pathlib.Path(inst.__file__).resolve().parent
# The default fake host (CUDA 13.0 torch, sm_100, no cutlass-dsl) takes the newest release and its sm100a provider.
V = inst.FLASHINFER_VERSION
FALLBACK = inst.FLASHINFER_RELEASES[-1].version
PROVIDER = "flashinfer-jit-cache-sm100a"
_BASE_DISTS = {
"torch": "2.12.1+cu130",
"triton": "3.7.1",
"nvidia-cublas": "13.1.0.3",
"numpy": "2.3.5",
"packaging": "25.0",
}
_ADDED_BY_INSTALL = {
"flashinfer-python": V,
"apache-tvm-ffi": "0.1.9",
"nvidia-cutlass-dsl": "4.4.2",
}
class _Result:
def __init__(
self,
returncode = 0,
stdout = "",
):
self.returncode = returncode
self.stdout = stdout
class _FakeEnv:
"""The venv as the installer sees it: a distribution table, a torch probe, and a subprocess."""
def __init__(
self,
*,
cuda = "13.0",
fail_install = False,
install_moves = None,
install_delay = 0.0,
concurrent_add = None,
):
self.dists = dict(_BASE_DISTS)
self.cuda = cuda
self.torch_version = self.dists["torch"]
self.importable = False
self.fail_install = fail_install
self.fail_verify = False
self.install_moves = install_moves or {}
self.install_delay = install_delay
self.concurrent_add = concurrent_add or {}
self.commands: list[list[str]] = []
self.constraints_seen: list[str] = []
self.lock = threading.Lock()
def probe(self, timeout = 60):
return {"torch_version": self.torch_version, "cuda_version": self.cuda}
def run(self, cmd, **kwargs):
cmd = [str(c) for c in cmd]
with self.lock:
self.commands.append(cmd)
if cmd[1:3] == ["-c", cmd[2]] and "import flashinfer" in cmd[2]:
good = self.importable and not self.fail_verify
providers = ",".join(
sorted(n for n in self.dists if n.startswith("flashinfer-jit-cache-"))
)
got = f"providers={providers}\n{self.dists.get('flashinfer-python', V)}"
return _Result(0 if good else 1, got if good else "ImportError")
if "uninstall" in cmd:
for name in cmd[cmd.index("uninstall") + 1 :]:
self.dists.pop(name, None)
self.importable = "flashinfer-python" in self.dists
return _Result(0)
if "install" in cmd:
if "-c" in cmd:
path = cmd[cmd.index("-c") + 1]
with open(path, encoding = "utf-8") as fh:
self.constraints_seen.append(fh.read())
if self.install_delay:
time.sleep(self.install_delay)
self.dists.update(self.concurrent_add)
if self.fail_install and any(a.startswith("flashinfer-jit-cache") for a in cmd):
return _Result(1, "ERROR: no matching distribution")
if any(a.startswith("flashinfer-python==") for a in cmd):
self.dists.update(_ADDED_BY_INSTALL)
spec = next(a for a in cmd if a.startswith("flashinfer-python=="))
self.dists["flashinfer-python"] = spec.split("==", 1)[1]
for name, version in self.install_moves.items():
self.dists[name] = version
if name == "torch":
self.torch_version = version
self.importable = not self.install_moves
# uv reports an upgrade or downgrade as `- name==old` then `+ name==new`.
return _Result(
0,
f"Installed {len(_ADDED_BY_INSTALL)} packages in 1.2s\n"
+ "\n".join(
f" + {n}=={v}"
for n, v in {
**_ADDED_BY_INSTALL,
"flashinfer-python": self.dists["flashinfer-python"],
**self.install_moves,
}.items()
),
)
elif any(a.startswith("flashinfer-jit-cache") and "==" in a for a in cmd):
specs = [a for a in cmd if a.startswith("flashinfer-jit-cache") and "==" in a]
for spec in specs:
name, version = spec.split("==", 1)
self.dists[name] = version
return _Result(
0,
f"Installed {len(specs)} package(s) in 3.1s\n"
+ "\n".join(f" + {spec}" for spec in specs),
)
else:
for spec in cmd:
if "==" in spec:
name, version = spec.split("==", 1)
self.dists[name] = version
if name == "torch":
self.torch_version = version
return _Result(0)
return _Result(1, "unexpected command")
def installs(self):
return [c for c in self.commands if "install" in c and "uninstall" not in c]
def _fake_torch(*, cuda = "13.0", hip = None):
torch = types.ModuleType("torch")
torch.version = types.SimpleNamespace(cuda = cuda, hip = hip)
return torch
@pytest.fixture(autouse = True)
def _clean(monkeypatch, tmp_path):
monkeypatch.setattr(inst, "_env_lock_path", lambda: str(tmp_path / "install.lock"))
monkeypatch.delenv(inst.FLASHINFER_INSTALL_ENV, raising = False)
monkeypatch.delenv(ops.NVFP4_BACKEND_ENV, raising = False)
for name in (
*inst._CUSTOM_INDEX_ENVS,
"UV_INDEX",
"UV_EXTRA_INDEX_URL",
*inst._PIP_EXTRA_INDEX_ENVS,
"UV_OFFLINE",
"UV_CONFIG_FILE",
"ALL_PROXY",
"all_proxy",
):
monkeypatch.delenv(name, raising = False)
monkeypatch.setenv("UV_NO_CONFIG", "1")
inst.reset_install_state()
yield
inst.reset_install_state()
@pytest.fixture
def env(monkeypatch):
"""An eligible host (Linux, CUDA 13.0 torch, sm_100) with flashinfer absent."""
fake = _FakeEnv()
monkeypatch.setattr(sys, "platform", "linux")
monkeypatch.setitem(sys.modules, "torch", _fake_torch())
monkeypatch.setattr(ops, "_device_capability", lambda device = None: (10, 0))
monkeypatch.setattr(
inst,
"_import_flashinfer",
lambda: (True, fake.dists.get("flashinfer-python", V))
if fake.importable
else (False, "ModuleNotFoundError: flashinfer"),
)
monkeypatch.setattr(inst, "_dist_version", lambda name: fake.dists.get(name))
monkeypatch.setattr(inst, "installed_distributions", lambda: dict(fake.dists))
monkeypatch.setattr(inst, "_torch_probe", fake.probe)
monkeypatch.setattr(inst, "_reachable", lambda url: True)
monkeypatch.setattr(inst, "_uv_executable", lambda: "uv")
monkeypatch.setattr(inst, "_nvcc_available", lambda: False)
import utils.utils as uu
monkeypatch.setattr(uu, "hf_env_offline", lambda: False)
return fake
def _steps(installs):
"""``(main, jit)`` by content, whichever order the transaction runs them in."""
main = next(c for c in installs if any(a.startswith("flashinfer-python==") for a in c))
jit = next(c for c in installs if any(a.startswith("flashinfer-jit-cache==") for a in c))
return main, jit
def _ensure(fake, device = 0):
return inst.ensure_flashinfer_for_nvfp4(device, run = fake.run)
def test_installs_pinned_flashinfer_and_matching_jit_cache_when_missing(env):
ok, reason = _ensure(env)
assert ok, reason
installs = env.installs()
assert len(installs) == 2
main, jit = _steps(installs)
assert f"flashinfer-python=={V}" in main and "--only-binary" in main
assert "--no-deps" not in main # tvm-ffi and friends are real import deps
assert f"flashinfer-jit-cache=={V}+cu130" in jit and "--no-deps" in jit
assert jit[jit.index("--index-url") + 1] == "https://flashinfer.ai/whl/cu130"
assert jit[jit.index("--index") + 1] == "https://flashinfer.ai/whl/cu130"
assert env.dists["flashinfer-jit-cache"] == f"{V}+cu130"
pins = env.constraints_seen[0].splitlines()
for name, version in _BASE_DISTS.items():
assert f"{name}=={version}" in pins
assert inst.last_install_reason() is None
def test_constraints_file_is_deleted_after_the_install(env, monkeypatch):
written = []
real = inst._write_constraints
def _spy(pins):
path = real(pins)
written.append(path)
return path
monkeypatch.setattr(inst, "_write_constraints", _spy)
_ensure(env)
assert written and not os.path.exists(written[0])
def test_already_installed_runs_nothing(env):
env.importable = True
ok, reason = _ensure(env)
assert ok and "already installed" in reason
assert env.commands == []
def test_installed_but_broken_is_not_repaired(env):
env.dists["flashinfer-python"] = "0.5.3"
ok, reason = _ensure(env)
assert not ok and "does not import" in reason
assert env.commands == []
@pytest.mark.parametrize(
"setup, needle",
[
(lambda mp: mp.setattr(sys, "platform", "darwin"), "Linux wheels only"),
(lambda mp: mp.setattr(sys, "platform", "win32"), "Linux wheels only"),
(lambda mp: mp.setitem(sys.modules, "torch", _fake_torch(cuda = None, hip = "6.4")), "ROCm"),
(lambda mp: mp.setitem(sys.modules, "torch", _fake_torch(cuda = None)), "CUDA build"),
(lambda mp: mp.setattr(ops, "_device_capability", lambda device = None: (9, 0)), "sm_90"),
(lambda mp: mp.setattr(ops, "_device_capability", lambda device = None: (8, 9)), "sm_89"),
(lambda mp: mp.setattr(ops, "_device_capability", lambda device = None: None), "capability"),
],
)
def test_never_installs_on_an_ineligible_host(env, monkeypatch, setup, needle):
setup(monkeypatch)
ok, reason = _ensure(env)
assert not ok and needle in reason
assert env.commands == []
def test_torch_version_change_rolls_back_and_refuses(env):
env.install_moves = {"torch": "2.13.0+cu130"}
ok, reason = _ensure(env)
assert not ok and "rolled back" in reason and "torch 2.12.1+cu130 -> 2.13.0+cu130" in reason
assert env.dists == _BASE_DISTS
uninstall = [c for c in env.commands if "uninstall" in c]
assert uninstall and set(_ADDED_BY_INSTALL) <= set(uninstall[0])
assert any("torch==2.12.1+cu130" in c and "--no-deps" in c for c in env.installs())
assert inst.last_install_reason() == reason
def test_torch_probe_drift_without_metadata_drift_also_rolls_back(env, monkeypatch):
probes = iter(
[
{"torch_version": "2.12.1+cu130", "cuda_version": "13.0"},
{"torch_version": "2.12.1+cu130", "cuda_version": "12.8"},
]
)
monkeypatch.setattr(inst, "_torch_probe", lambda timeout = 60: next(probes))
ok, reason = _ensure(env)
assert not ok and "rolled back" in reason and "torch reads differently" in reason
assert env.dists == _BASE_DISTS
def test_failed_jit_cache_step_rolls_back_the_first_step(env):
env.fail_install = True
ok, reason = _ensure(env)
assert not ok and "rolled back" in reason and "installer failed" in reason
assert env.dists == _BASE_DISTS
def test_rollback_leaves_packages_another_installer_added_meanwhile(env):
env.fail_verify = True
env.concurrent_add = {"requests": "2.32.3", "rich": "14.0.0"}
ok, reason = _ensure(env)
assert not ok and "rolled back" in reason
for name in _ADDED_BY_INSTALL:
assert name not in env.dists
assert env.dists.get("requests") == "2.32.3" and env.dists.get("rich") == "14.0.0", (
reason,
env.commands,
)
uninstall = next(c for c in env.commands if "uninstall" in c)
assert "requests" not in uninstall and "rich" not in uninstall
def test_reported_installs_reads_uv_and_pip_output():
uv_out = "Resolved 5 packages\nInstalled 2 packages in 9ms\n + Apache_TVM_FFI==0.1.9\n + flashinfer-python==0.6.6\n ~ numpy==2.3.5"
assert inst._reported_installs(uv_out) == {"apache-tvm-ffi", "flashinfer-python"}
pip_out = (
"Collecting x\nSuccessfully installed apache-tvm-ffi-0.1.9 flashinfer-jit-cache-0.6.6+cu130"
)
assert inst._reported_installs(pip_out) == {"apache-tvm-ffi", "flashinfer-jit-cache"}
assert inst._reported_installs("ERROR: no matching distribution") == set()
def test_rollback_without_installer_output_follows_the_flashinfer_dependency_tree(monkeypatch):
import importlib.metadata as md
requires = {
"flashinfer-python": [
"apache-tvm-ffi>=0.1",
"nvidia-cutlass-dsl; python_version >= '3'",
"pytest; extra == 'test'",
],
"apache-tvm-ffi": [],
"nvidia-cutlass-dsl": [],
}
def _distribution(name):
if name not in requires:
raise md.PackageNotFoundError(name)
return types.SimpleNamespace(requires = requires[name])
monkeypatch.setattr(md, "distribution", _distribution)
before = dict(_BASE_DISTS)
after = {**before, **_ADDED_BY_INSTALL, "pytest": "8.0.0", "requests": "2.32.3"}
monkeypatch.setattr(inst, "installed_distributions", lambda: dict(after))
commands = []
def run(cmd, **kwargs):
commands.append([str(c) for c in cmd])
return _Result(0)
inst._rollback(run, "uv", before, None, reported = set())
assert len(commands) == 1
removed = set(commands[0][commands[0].index(sys.executable) + 1 :])
assert removed == set(_ADDED_BY_INSTALL)
def test_failed_import_after_install_rolls_back(env, monkeypatch):
real_run = env.run
def _run(cmd, **kwargs):
result = real_run(cmd, **kwargs)
if "import flashinfer" in " ".join(map(str, cmd)):
return _Result(1, "ImportError: libcudart.so.13")
return result
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = _run)
assert not ok and "import flashinfer failed" in reason
assert env.dists == _BASE_DISTS
def test_offline_refuses_with_a_reason_and_is_not_memoised(env, monkeypatch):
import utils.utils as uu
monkeypatch.setattr(uu, "hf_env_offline", lambda: True)
ok, reason = _ensure(env)
assert not ok and "offline" in reason
assert env.commands == []
assert inst.last_install_reason() == reason
monkeypatch.setattr(uu, "hf_env_offline", lambda: False)
assert _ensure(env)[0]
def test_uv_offline_refuses_before_any_index_probe(env, monkeypatch):
probes: list = []
monkeypatch.setattr(inst, "_reachable", lambda url: probes.append(url) or True)
monkeypatch.setenv("UV_OFFLINE", "1")
ok, reason = _ensure(env)
assert not ok and "offline" in reason
assert probes == [] and env.commands == []
monkeypatch.delenv("UV_OFFLINE")
assert _ensure(env)[0]
def test_another_process_holding_the_env_lock_blocks_the_install(env, monkeypatch):
filelock = pytest.importorskip("filelock")
monkeypatch.setattr(inst, "ENV_LOCK_TIMEOUT_S", 0.05)
real = inst._env_install_lock
monkeypatch.setattr(inst, "_env_install_lock", lambda timeout = 0.05: real(timeout))
# A second FileLock on its own handle, held from a thread, stands in for another process.
other = filelock.FileLock(inst._env_lock_path(), thread_local = False)
import threading
held, release = threading.Event(), threading.Event()
def _hold():
with other:
held.set()
release.wait(5)
t = threading.Thread(target = _hold)
t.start()
held.wait(5)
try:
ok, reason = _ensure(env)
finally:
release.set()
t.join()
assert not ok and "another process" in reason
assert env.commands == []
assert _ensure(env)[0]
def test_unreachable_index_refuses_without_running_the_installer(env, monkeypatch):
monkeypatch.setattr(inst, "_reachable", lambda url: False)
ok, reason = _ensure(env)
assert not ok and "is not reachable" in reason
assert env.installs() == []
monkeypatch.setattr(inst, "_reachable", lambda url: True)
assert _ensure(env)[0]
@pytest.mark.parametrize("value", ["0", "off", "false", "no"])
def test_opt_out_env_refuses(env, monkeypatch, value):
monkeypatch.setenv(inst.FLASHINFER_INSTALL_ENV, value)
ok, reason = _ensure(env)
assert not ok and inst.FLASHINFER_INSTALL_ENV in reason
assert env.commands == []
def test_explicit_torchao_backend_never_installs(env, monkeypatch):
monkeypatch.setenv(ops.NVFP4_BACKEND_ENV, "torchao")
ok, reason = _ensure(env)
assert not ok and "torchao" in reason
assert env.commands == []
def test_concurrent_loads_install_once(env):
env.install_delay = 0.3
results = []
barrier = threading.Barrier(4)
def _worker():
barrier.wait()
results.append(_ensure(env))
threads = [threading.Thread(target = _worker) for _ in range(4)]
for t in threads:
t.start()
for t in threads:
t.join(10)
assert len(results) == 4 and all(ok for ok, _ in results)
assert sum(1 for c in env.installs() if f"flashinfer-python=={V}" in c) == 1
def test_a_failed_install_is_attempted_once_per_process(env):
env.fail_install = True
assert not _ensure(env)[0]
count = len(env.installs())
assert not _ensure(env)[0]
assert len(env.installs()) == count
def test_no_jit_cache_and_no_nvcc_refuses(env):
env.cuda = "12.6"
ok, reason = _ensure(env)
assert not ok and "no nvcc" in reason and "12.6" in reason
assert env.installs() == []
def test_no_jit_cache_with_nvcc_installs_flashinfer_only(env, monkeypatch):
env.cuda = "14.0"
monkeypatch.setattr(inst, "_nvcc_available", lambda: True)
ok, reason = _ensure(env)
assert ok, reason
assert [c for c in env.installs() if any("jit-cache" in a for a in c)] == []
def test_stray_jit_cache_of_another_version_refuses(env):
env.dists["flashinfer-jit-cache"] = "0.6.4+cu130"
ok, reason = _ensure(env)
assert not ok and "flashinfer-jit-cache 0.6.4+cu130" in reason
assert env.installs() == []
def test_success_forgets_the_dispatch_availability_memo(env, monkeypatch):
monkeypatch.setattr(dispatch, "_AVAILABLE", (False, "ModuleNotFoundError"))
assert _ensure(env)[0]
assert dispatch._AVAILABLE is None
@pytest.mark.parametrize(
"cuda, tag",
[
("12.8", None), # the newest release has no cu128 jit-cache; select_release falls back
("12.9", "cu129"),
("13.0", "cu130"),
("13.2", "cu130"),
("12.6", None),
("14.0", None),
(None, None),
("garbage", None),
],
)
def test_jit_cache_tag_matches_the_running_cuda(cuda, tag):
assert inst.jit_cache_tag(cuda) == tag
def test_status_fields_carry_the_reason_only_for_torchao(env, monkeypatch):
monkeypatch.setenv(inst.FLASHINFER_INSTALL_ENV, "0")
_ensure(env)
assert inst.nvfp4_backend_fields("torchao")["transformer_quant_backend_reason"]
assert inst.nvfp4_backend_fields("flashinfer") == {
"transformer_quant_backend": "flashinfer",
"transformer_quant_backend_reason": None,
}
assert inst.nvfp4_backend_fields(None)["transformer_quant_backend_reason"] is None
def test_never_raises(env, monkeypatch):
def _boom(device):
raise RuntimeError("probe exploded")
monkeypatch.setattr(inst, "_host_refusal", _boom)
ok, reason = _ensure(env)
assert not ok and "probe exploded" in reason
def _load_pipeline_body(path: pathlib.Path) -> str:
source = path.read_text(encoding = "utf-8")
start = source.index(" def load_pipeline(")
return source[start : source.index("\n def ", start + 10)]
def _flat(text: str) -> str:
"""Whitespace-insensitive, so a formatter re-wrapping the condition does not fail the test."""
return re.sub(r"[\s,]+", " ", text).replace("( ", "(").replace(" )", ")")
def test_image_loader_installs_before_the_load_locks():
body = _load_pipeline_body(_INFERENCE_DIR / "diffusion.py")
call = body.index("ensure_flashinfer_for_nvfp4(")
assert call < body.index("with self._lock:")
assert _flat(
"TQ_NVFP4 in (normalize_transformer_quant(transformer_quant), _pipeline_prequant_planned)"
) in _flat(body)
def test_video_loader_installs_after_teardown_and_outside_the_locks():
body = _load_pipeline_body(_INFERENCE_DIR / "video.py")
teardown = body.index("self._teardown_state_locked()")
modular = body.index("ensure_flashinfer_for_nvfp4(")
conventional = body.index("self._install_flashinfer_for_seed(")
assert body.count("ensure_flashinfer_for_nvfp4(") == 1
for call, indent in ((modular, 16), (conventional, 8)):
assert teardown < call
line = body[body.rindex("\n", 0, call) + 1 : call]
assert line == " " * indent + "_nvfp4_install_outcome = "
assert _flat(
'"nvfp4" in (normalize_transformer_quant(transformer_quant), _video_auto_denoiser_planned)'
) in _flat(body)
def test_video_loader_installs_only_for_a_seed_the_live_memory_plan_kept():
from core.inference.video import VideoBackend
body = _load_pipeline_body(_INFERENCE_DIR / "video.py")
conventional = body.index("self._install_flashinfer_for_seed(")
injection = body.index("denoiser_prequant_pipe_kwargs(")
assert body.rindex("denoiser_seed_scheme = None", 0, injection) < conventional < injection
assert _flat("_nvfp4_install_wanted denoiser_seed_scheme device") in _flat(
body[conventional : body.index(")", conventional)]
)
modular = body.index("ensure_flashinfer_for_nvfp4(")
assert (
body.index("if fam.modular_workflow:")
< modular
< body.index("self._load_h3_modular_pipeline(")
)
calls = []
import core.inference.diffusion_nvfp4_install as inst_mod
original = inst_mod.ensure_flashinfer_for_nvfp4
inst_mod.ensure_flashinfer_for_nvfp4 = lambda device, **kw: calls.append(device) or (True, "ok")
try:
assert VideoBackend._install_flashinfer_for_seed(True, None, "cuda") is None
assert VideoBackend._install_flashinfer_for_seed(True, "int8", "cuda") is None
assert VideoBackend._install_flashinfer_for_seed(False, "nvfp4", "cuda") is None
assert calls == []
assert VideoBackend._install_flashinfer_for_seed(True, "nvfp4", "cuda") == (True, "ok")
assert calls == ["cuda"]
finally:
inst_mod.ensure_flashinfer_for_nvfp4 = original
def test_image_loader_gates_the_install_on_the_seed_plan_it_will_rerun():
import inspect
from core.inference import diffusion
body = _load_pipeline_body(_INFERENCE_DIR / "diffusion.py")
call = body.index("ensure_flashinfer_for_nvfp4(")
assert "self._seed_plan_stays_resident(" in body[:call]
locked = body[body.index("with self._lock:") :]
assert "self._seeded_pipeline_plan(" in locked
helper = inspect.getsource(diffusion.DiffusionBackend._seed_plan_stays_resident)
assert "self._seeded_pipeline_plan(" in helper
class _FakePlan:
def __init__(self, offload_policy):
self.offload_policy = offload_policy
def _seed_gate(
monkeypatch,
*,
free_mib,
reserved_mib,
need_mib,
total_mib = 100_000,
):
from core.inference import diffusion
from core.inference.diffusion_memory import DeviceMemory
seen = []
monkeypatch.setattr(
diffusion,
"snapshot_device_memory",
lambda target: DeviceMemory("cuda", "cuda", "discrete_vram", free_mib, total_mib),
)
torch = types.ModuleType("torch")
torch.cuda = types.SimpleNamespace(memory_reserved = lambda: reserved_mib * 1024 * 1024)
monkeypatch.setitem(sys.modules, "torch", torch)
monkeypatch.setattr(
diffusion,
"estimate_dense_quant",
lambda fam, scheme, base_repo = None, prequant_available = False: types.SimpleNamespace(
steady_transformer_mib = need_mib, companions_mib = 0, text_encoders_mib = 0
),
)
def _plan_memory(self, target, single_file_path, base, fam, memory_mode, cpu_offload, **kw):
seen.append(kw)
memory = kw["device_memory_override"]
return _FakePlan("none" if memory.free_mib >= need_mib else "model")
monkeypatch.setattr(diffusion.DiffusionBackend, "_plan_memory", _plan_memory)
backend = object.__new__(diffusion.DiffusionBackend)
ok = backend._seed_plan_stays_resident(
"nvfp4",
types.SimpleNamespace(device = "cuda"),
"org/base",
object(),
None,
False,
repo_id = "org/base",
base_local_dir = None,
fetch_base = None,
)
return ok, seen
def test_seed_gate_refuses_when_live_memory_would_offload_the_seed(monkeypatch):
ok, seen = _seed_gate(monkeypatch, free_mib = 10_000, reserved_mib = 0, need_mib = 40_000)
assert ok is False
assert seen and seen[0]["transformer_resident_override_mib"] == 40_000
def test_seed_gate_credits_what_the_teardown_frees(monkeypatch):
ok, _ = _seed_gate(monkeypatch, free_mib = 10_000, reserved_mib = 35_000, need_mib = 40_000)
assert ok is True
def test_seed_gate_unanswerable_keeps_the_old_answer(monkeypatch):
from core.inference import diffusion
monkeypatch.setattr(
diffusion,
"snapshot_device_memory",
lambda target: (_ for _ in ()).throw(RuntimeError("probe")),
)
backend = object.__new__(diffusion.DiffusionBackend)
assert (
backend._seed_plan_stays_resident(
"nvfp4",
types.SimpleNamespace(device = "cuda"),
"b",
object(),
None,
False,
repo_id = "b",
base_local_dir = None,
fetch_base = None,
)
is True
)
def test_a_configured_mirror_skips_the_pypi_probe_but_not_the_jit_cache_one(env, monkeypatch):
probed = []
monkeypatch.setattr(inst, "_reachable", lambda url: probed.append(url) or True)
monkeypatch.setenv("UV_INDEX_URL", "https://mirror.example/simple")
assert _ensure(env)[0]
assert probed == [f"https://flashinfer.ai/whl/cu130/{PROVIDER}/"]
def test_a_pip_only_mirror_is_handed_to_uv(env, monkeypatch):
monkeypatch.setattr(inst, "_reachable", lambda url: True)
monkeypatch.setenv("PIP_INDEX_URL", "https://pip-mirror.example/simple")
assert _ensure(env)[0]
installs = [c for c in env.commands if c[:3] == ["uv", "pip", "install"]]
assert installs and "https://pip-mirror.example/simple" in _steps(installs)[0]
monkeypatch.setenv("UV_INDEX_URL", "https://uv-mirror.example/simple")
assert "--index-url" not in inst._installer_prefix("uv")
assert "--index-url" not in inst._installer_prefix(None)
def test_a_uv_only_mirror_is_handed_to_the_pip_fallback(env, monkeypatch):
monkeypatch.setattr(inst, "_uv_executable", lambda: None)
for name in ("PIP_INDEX_URL", "UV_DEFAULT_INDEX"):
monkeypatch.delenv(name, raising = False)
monkeypatch.setenv("UV_INDEX_URL", "https://uv-mirror.example/simple")
main = inst._installer_prefix(None)
assert main[main.index("--index-url") + 1] == "https://uv-mirror.example/simple"
monkeypatch.setenv("UV_DEFAULT_INDEX", "https://uv-default.example/simple")
main = inst._installer_prefix(None)
assert main[main.index("--index-url") + 1] == "https://uv-default.example/simple"
jit = inst._installer_prefix(None, "https://flashinfer.ai/whl/cu130")
assert jit.count("--index-url") == 1 and jit[-1] == "https://flashinfer.ai/whl/cu130"
monkeypatch.setenv("PIP_INDEX_URL", "https://pip-mirror.example/simple")
assert "--index-url" not in inst._installer_prefix(None)
def test_a_pip_only_mirror_never_gives_uv_two_index_urls(env, monkeypatch):
monkeypatch.setenv("PIP_INDEX_URL", "https://pip-mirror.example/simple")
assert _ensure(env)[0]
main, jit = _steps(env.installs())
assert main.count("--index-url") == 1
assert main[main.index("--index-url") + 1] == "https://pip-mirror.example/simple"
assert jit.count("--index-url") == 1
assert jit[jit.index("--index-url") + 1] == "https://flashinfer.ai/whl/cu130"
pip_jit = inst._installer_prefix(None, "https://flashinfer.ai/whl/cu130")
assert pip_jit.count("--index-url") == 1
def test_status_reason_reports_a_transient_preflight_failure(monkeypatch):
torch = pytest.importorskip("torch")
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device = None: (10, 0))
monkeypatch.setattr(torch.cuda, "get_device_name", lambda device = None: "stub B200")
ops.reset_preflight_cache()
def _oom(dev):
raise torch.cuda.OutOfMemoryError("CUDA out of memory. Tried to allocate 64.00 MiB")
class _Backend:
pass
image = _Backend()
inst.record_install_reason(image, True, "already installed", 0)
monkeypatch.setattr(ops, "_preflight_probe", _oom)
assert ops.nvfp4_preflight(0)["ok"] is False
assert 0 not in ops._PREFLIGHT
reason = inst.nvfp4_backend_fields("torchao", owner = image)["transformer_quant_backend_reason"]
assert reason and "out of memory" in reason.lower()
assert (
"out of memory"
in inst.nvfp4_backend_fields("torchao")["transformer_quant_backend_reason"].lower()
)
monkeypatch.setattr(ops, "_preflight_probe", lambda dev: False)
ops.nvfp4_preflight(0)
assert (
inst.nvfp4_backend_fields("torchao", owner = image)["transformer_quant_backend_reason"]
== "flashinfer preflight failed: mm_fp4 produced a non-finite result"
)
ops.reset_preflight_cache()
def test_status_reason_falls_back_to_a_cached_preflight_failure(monkeypatch):
monkeypatch.setitem(ops._PREFLIGHT, 0, {"ok": False, "reason": "JIT build failed"})
fields = inst.nvfp4_backend_fields("torchao")
assert (
fields["transformer_quant_backend_reason"]
== "flashinfer preflight failed: JIT build failed"
)
class _Logger:
def __init__(self):
self.records = []
def info(self, fmt, *args):
self.records.append(("info", fmt % args))
def warning(self, fmt, *args):
self.records.append(("warning", fmt % args))
def test_policy_refusals_log_at_info_and_failures_at_warning(env, monkeypatch):
logger = _Logger()
monkeypatch.setattr(ops, "_device_capability", lambda device = None: (9, 0))
inst.ensure_flashinfer_for_nvfp4(0, logger = logger, run = env.run)
assert logger.records[-1][0] == "info"
monkeypatch.setattr(ops, "_device_capability", lambda device = None: (10, 0))
env.fail_install = True
inst.ensure_flashinfer_for_nvfp4(0, logger = logger, run = env.run)
assert logger.records[-1][0] == "warning" and "rolled back" in logger.records[-1][1]
def test_status_callback_sees_the_install_and_its_outcome(env):
seen = []
ok, _ = inst.ensure_flashinfer_for_nvfp4(0, status_cb = seen.append, run = env.run)
assert ok
assert seen[0].startswith(f"installing flashinfer-python {V}") and "cu130" in seen[0]
assert seen[-1] == f"installed flashinfer {V} for NVFP4"
def test_local_only_load_never_installs(env):
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = env.run, local_files_only = True)
assert not ok and "local-only" in reason
assert env.installs() == []
def test_status_reason_is_bound_to_the_backend_that_loaded():
class _Backend:
pass
image, video = _Backend(), _Backend()
inst.reset_install_state()
inst.record_install_reason(image, False, "offline: flashinfer is not downloaded", 0)
inst.record_install_reason(video, True, "installed", 1)
assert inst.nvfp4_backend_fields("torchao", owner = image)[
"transformer_quant_backend_reason"
] == ("offline: flashinfer is not downloaded")
assert (
inst.nvfp4_backend_fields("flashinfer", owner = image)["transformer_quant_backend_reason"]
is None
)
inst.reset_install_state()
def test_status_preflight_reason_is_the_loaded_device_only(monkeypatch):
class _Backend:
pass
image, video = _Backend(), _Backend()
inst.reset_install_state()
monkeypatch.setitem(ops._PREFLIGHT, 0, {"ok": False, "reason": "JIT build failed"})
monkeypatch.setitem(ops._PREFLIGHT, 1, {"ok": False, "reason": "sm_120 unsupported"})
inst.record_install_reason(image, True, "already installed", 0)
inst.record_install_reason(video, True, "already installed", "cuda:1")
assert (
inst.nvfp4_backend_fields("torchao", owner = image)["transformer_quant_backend_reason"]
== "flashinfer preflight failed: JIT build failed"
)
assert (
inst.nvfp4_backend_fields("torchao", owner = video)["transformer_quant_backend_reason"]
== "flashinfer preflight failed: sm_120 unsupported"
)
inst.reset_install_state()
def test_image_loader_skips_the_install_when_the_plan_declined_the_seed():
body = _load_pipeline_body(_INFERENCE_DIR / "diffusion.py")
gate = body[: body.index("ensure_flashinfer_for_nvfp4(")].rsplit("if (", 1)[-1]
declined = gate.index("_pipeline_prequant_planned != PIPELINE_SEED_DECLINED")
assert declined < gate.index("self._nvfp4_checkpoint_will_load(")
def test_image_loader_binds_the_reason_only_after_the_resident_model_is_unloaded():
import inspect
from core.inference import diffusion
src = inspect.getsource(diffusion.DiffusionBackend)
ensure_at = src.index("ensure_flashinfer_for_nvfp4(")
call = src[ensure_at : src.index(")", ensure_at)]
assert "owner" not in call
unload_at = src.index("self._unload_locked()", ensure_at)
assert src.index("record_install_reason(self", ensure_at) > unload_at
def test_video_loader_binds_the_reason_only_at_the_commit():
import inspect
from core.inference import video
src = inspect.getsource(video.VideoBackend)
ensure_at = src.index("ensure_flashinfer_for_nvfp4(")
call = src[ensure_at : src.index(")", ensure_at)]
assert "owner" not in call
commits = [m.start() for m in re.finditer(r"self\._state = _VideoLoadState\(", src)]
binds = [m.start() for m in re.finditer(r"record_install_reason\(self", src)]
assert len(binds) == 2
for bind in binds:
commit = min(c for c in commits if c > bind)
cancel = src.rindex("Video load was cancelled or superseded.", 0, bind)
assert src.index("self._state = _VideoLoadState(", cancel) == commit
def test_every_loader_commit_binds_a_reason_even_when_the_install_gate_skipped():
import inspect
from core.inference import diffusion, video
for cls in (diffusion.DiffusionBackend, video.VideoBackend):
src = inspect.getsource(cls)
assert "if _nvfp4_install_outcome is not None" not in src
binds = re.findall(
r"record_install_reason\(self, \*\(_nvfp4_install_outcome or \(True, None\)\)", src
)
assert len(binds) == (1 if cls is diffusion.DiffusionBackend else 2)
def test_install_gates_skip_kinds_the_dense_quant_path_cannot_reach():
import inspect
from core.inference import diffusion, video
img = inspect.getsource(diffusion.DiffusionBackend)
gate = img[: img.index("ensure_flashinfer_for_nvfp4(")].rsplit("if ", 1)[-1]
# One exception, by kind: a ComfyUI nvfp4 single file (it never reaches the dense quant path, but its own codes
# run on the FlashInfer Linear). Every other kind still has to pass the dense-quant clause.
comfy, dense = gate.split(") or (", 1)
assert comfy.lstrip("( \n").startswith('kind == "single_file"')
assert "_comfy_single_file_holds_nvfp4(" in comfy
assert comfy.index("comfy_nvfp4_runtime_possible(") < comfy.index(
"_comfy_single_file_holds_nvfp4("
)
assert comfy.index("_has_active_lora(loras)") < comfy.index("_comfy_single_file_holds_nvfp4(")
assert dense.lstrip("( \n").startswith("dense_quant_supported_kind(kind)")
vid = inspect.getsource(video.VideoBackend)
gate = vid[vid.index("_nvfp4_install_wanted = (") + len("_nvfp4_install_wanted = (") :]
assert gate.lstrip("( \n").startswith('kind == "pipeline"')
body = _load_pipeline_body(_INFERENCE_DIR / "video.py")
modular = body.index("ensure_flashinfer_for_nvfp4(")
assert "if _nvfp4_install_wanted:" in body[body.index("if fam.modular_workflow:") : modular]
hop = body[body.index("self._install_flashinfer_for_seed(") :]
assert hop[: hop.index(")")].split("(", 1)[1].split(",")[0].strip() == "_nvfp4_install_wanted"
def test_jit_cache_step_drops_extra_index_sources_from_the_environment(env, monkeypatch):
extra = {
"UV_INDEX": "https://corp.example/simple",
"UV_EXTRA_INDEX_URL": "https://corp.example/extra",
"PIP_EXTRA_INDEX_URL": "https://corp.example/pip-extra",
"UV_FIND_LINKS": "https://corp.example/links",
}
for name, value in extra.items():
monkeypatch.setenv(name, value)
seen = []
real = env.run
def _run(cmd, **kwargs):
seen.append(([str(c) for c in cmd], kwargs.get("env")))
return real(cmd, **kwargs)
assert inst.ensure_flashinfer_for_nvfp4(0, run = _run)[0]
envs = {
"main" if any(c.startswith("flashinfer-python==") for c in cmd) else "jit": child
for cmd, child in seen
if "install" in cmd and "uninstall" not in cmd
}
for name, value in extra.items():
assert envs["main"].get(name) == value
assert name not in envs["jit"]
def test_pip_jit_cache_step_reads_no_pip_conf_but_keeps_its_transport(env, monkeypatch):
monkeypatch.setattr(inst, "_uv_executable", lambda: None)
monkeypatch.setenv("PIP_TIMEOUT", "30")
lines = [
"global.extra-index-url='https://corp.example/extra'",
"install.find-links='https://corp.example/links'",
"global.proxy='http://proxy.corp.example:3128'",
"global.cert='/etc/ssl/corp.pem'",
"global.trusted-host='a.corp.example\\nb.corp.example'",
"global.timeout='120'",
":env:.timeout='30'",
]
seen = []
config_run = _pip_config(env, lines)
def _run(cmd, **kwargs):
seen.append(([str(c) for c in cmd], kwargs.get("env")))
return config_run(cmd, **kwargs)
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = _run)
assert ok, reason
installs = {
"main" if any(c.startswith("flashinfer-python==") for c in cmd) else "jit": (cmd, child)
for cmd, child in seen
if "install" in cmd and "uninstall" not in cmd and "config" not in cmd
}
main_cmd, main_env = installs["main"]
jit_cmd, jit_env = installs["jit"]
assert main_env is None or main_env.get("PIP_CONFIG_FILE") != os.devnull
assert jit_env["PIP_CONFIG_FILE"] == os.devnull
assert jit_cmd[jit_cmd.index("--index-url") + 1] == "https://flashinfer.ai/whl/cu130"
assert jit_env["PIP_PROXY"] == "http://proxy.corp.example:3128"
assert jit_env["PIP_CERT"] == "/etc/ssl/corp.pem"
assert jit_env["PIP_TRUSTED_HOST"].split() == ["a.corp.example", "b.corp.example"]
assert jit_env["PIP_TIMEOUT"] == "30"
assert not any(k in jit_env for k in ("PIP_EXTRA_INDEX_URL", "PIP_FIND_LINKS"))
def test_uv_jit_cache_step_leaves_pip_config_alone(env, monkeypatch):
seen = []
real = env.run
def _run(cmd, **kwargs):
seen.append(([str(c) for c in cmd], kwargs.get("env")))
return real(cmd, **kwargs)
assert inst.ensure_flashinfer_for_nvfp4(0, run = _run)[0]
jit_env = next(
child for cmd, child in seen if any(c.startswith("flashinfer-jit-cache==") for c in cmd)
)
assert jit_env.get("PIP_CONFIG_FILE") == os.environ.get("PIP_CONFIG_FILE")
def _hold_env_lock(release_after):
filelock = pytest.importorskip("filelock")
other = filelock.FileLock(inst._env_lock_path(), thread_local = False)
held = threading.Event()
def _hold():
with other:
held.set()
time.sleep(release_after)
t = threading.Thread(target = _hold)
t.start()
held.wait(5)
return t
def _track_imports(env, monkeypatch):
stamps = []
monkeypatch.setattr(
inst,
"_import_flashinfer",
lambda: stamps.append(time.monotonic())
or ((True, "0.6.6") if env.importable else (False, "ModuleNotFoundError: flashinfer")),
)
monkeypatch.delitem(sys.modules, "flashinfer", raising = False)
return stamps
def test_an_in_flight_install_by_another_process_is_waited_for_before_importing(env, monkeypatch):
env.dists["flashinfer-python"] = "0.6.6"
env.importable = True
stamps = _track_imports(env, monkeypatch)
start = time.monotonic()
t = _hold_env_lock(0.5)
try:
ok, reason = _ensure(env)
finally:
t.join()
assert ok and "already installed" in reason
assert stamps and stamps[0] - start >= 0.45
assert env.commands == []
def test_a_complete_install_still_holding_the_lock_is_waited_for_in_case_it_rolls_back(
env, monkeypatch
):
env.dists.update({"flashinfer-python": "0.6.6", "flashinfer-jit-cache": "0.6.6+cu130"})
env.importable = True
stamps = _track_imports(env, monkeypatch)
filelock = pytest.importorskip("filelock")
other = filelock.FileLock(inst._env_lock_path(), thread_local = False)
held, rolled_back = threading.Event(), []
def _hold():
with other:
held.set()
time.sleep(0.5)
env.dists.pop("flashinfer-python")
env.dists.pop("flashinfer-jit-cache")
env.importable = False
rolled_back.append(len(stamps))
t = threading.Thread(target = _hold)
t.start()
held.wait(5)
try:
ok, reason = _ensure(env)
finally:
t.join()
assert rolled_back == [0] and stamps
assert "already installed" not in reason
def test_a_user_flashinfer_never_waits_on_the_lock(env, monkeypatch):
env.dists["flashinfer-python"] = "0.5.3"
env.importable = True
stamps = _track_imports(env, monkeypatch)
start = time.monotonic()
t = _hold_env_lock(1.0)
try:
ok, _ = _ensure(env)
finally:
t.join()
assert ok and stamps[0] - start < 0.5
assert env.commands == []
def test_a_timed_out_wait_for_an_in_flight_install_does_not_import(env, monkeypatch):
env.dists["flashinfer-python"] = "0.6.6"
env.importable = True
real = inst._env_install_lock
monkeypatch.setattr(inst, "_env_install_lock", lambda timeout = 0.05: real(timeout))
stamps = _track_imports(env, monkeypatch)
t = _hold_env_lock(0.5)
try:
ok, reason = _ensure(env)
finally:
t.join()
assert not ok and "another process" in reason
assert stamps == [] and env.commands == []
ok, reason = _ensure(env)
assert ok and "already installed" in reason
@pytest.mark.parametrize("uv", ["uv", None])
@pytest.mark.parametrize("name", ["UV_INDEX", "UV_EXTRA_INDEX_URL", "UV_FIND_LINKS"])
def test_a_uv_index_skips_the_pypi_probe_only_when_uv_installs(env, monkeypatch, name, uv):
probed = []
monkeypatch.setattr(inst, "_reachable", lambda url: probed.append(url) or True)
monkeypatch.setattr(inst, "_uv_executable", lambda: uv)
monkeypatch.setenv(name, "https://uv-mirror.example/simple")
assert _ensure(env)[0]
assert (inst._PYPI_PROBE_URL in probed) == (uv is None)
@pytest.mark.parametrize(
"cuda, cache, nvcc",
[
("13.0", f"{V}+cu129", False), # left over from a cu129 torch
("13.0", V, False), # no local tag: the CUDA it was built for cannot be told
("14.0", f"{V}+cu130", True),
],
)
def test_a_jit_cache_built_for_another_cuda_refuses(env, monkeypatch, cuda, cache, nvcc):
env.cuda = cuda
env.dists["flashinfer-jit-cache"] = cache
monkeypatch.setattr(inst, "_nvcc_available", lambda: nvcc)
ok, reason = _ensure(env)
assert not ok and f"flashinfer-jit-cache {cache}" in reason and "does not match CUDA" in reason
assert env.installs() == []
def test_a_jit_cache_of_the_running_cuda_is_kept(env):
env.dists["flashinfer-jit-cache"] = f"{V}+cu130"
ok, reason = _ensure(env)
assert ok, reason
@pytest.mark.parametrize("policy", ["local_files_only", "opt_out"])
def test_a_load_that_declined_installs_does_not_wait_on_another_install(env, monkeypatch, policy):
env.dists["flashinfer-python"] = "0.6.6"
env.importable = True
stamps = _track_imports(env, monkeypatch)
kwargs = {}
if policy == "local_files_only":
kwargs["local_files_only"] = True
else:
monkeypatch.setenv(inst.FLASHINFER_INSTALL_ENV, "0")
t = _hold_env_lock(2.0)
start = time.monotonic()
try:
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = env.run, **kwargs)
elapsed = time.monotonic() - start
finally:
t.join()
assert elapsed < 1.0
assert not ok and "another process is still installing" in reason
assert stamps == []
assert env.commands == []
def test_a_load_that_declined_installs_still_imports_a_finished_install(env, monkeypatch):
env.dists.update({"flashinfer-python": "0.6.6", "flashinfer-jit-cache": "0.6.6+cu130"})
env.importable = True
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = env.run, local_files_only = True)
assert ok and "already installed" in reason
def _pip_config(env, lines):
"""``env.run`` that also answers ``pip config list`` with ``lines``, as pip prints them."""
calls = []
def _run(cmd, **kwargs):
cmd = [str(c) for c in cmd]
if cmd[1:] == ["-m", "pip", "config", "list"]:
calls.append(cmd)
return _Result(0, "".join(f"{line}\n" for line in lines))
return env.run(cmd, **kwargs)
_run.calls = calls
return _run
def test_a_pip_conf_mirror_skips_the_pypi_probe_for_the_pip_fallback(env, monkeypatch):
probed = []
monkeypatch.setattr(inst, "_reachable", lambda url: probed.append(url) or True)
monkeypatch.setattr(inst, "_uv_executable", lambda: None)
run = _pip_config(env, ["global.index-url='https://corp.example/simple'"])
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = run)
assert ok, reason
assert probed == [f"https://flashinfer.ai/whl/cu130/{PROVIDER}/"]
@pytest.mark.parametrize(
"lines, envvar",
[
([":env:.extra-index-url='https://corp.example/simple'"], None),
(["global.extra-index-url='https://corp.example/simple'"], None),
(["install.find-links='https://corp.example/links'"], None),
([], "PIP_EXTRA_INDEX_URL"),
([], "PIP_FIND_LINKS"),
],
)
def test_a_pip_extra_index_skips_the_pypi_probe_for_the_pip_fallback(
env, monkeypatch, lines, envvar
):
probed = []
monkeypatch.setattr(
inst, "_reachable", lambda url: probed.append(url) or url != inst._PYPI_PROBE_URL
)
monkeypatch.setattr(inst, "_uv_executable", lambda: None)
if envvar:
monkeypatch.setenv(envvar, "https://corp.example/simple")
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = _pip_config(env, lines))
assert ok, reason
assert inst._PYPI_PROBE_URL not in probed
def test_a_pip_conf_mirror_is_not_overridden_by_a_uv_one(env, monkeypatch):
monkeypatch.setattr(inst, "_uv_executable", lambda: None)
monkeypatch.setenv("UV_INDEX_URL", "https://uv-mirror.example/simple")
run = _pip_config(env, ["install.index-url='https://corp.example/simple'"])
assert inst.ensure_flashinfer_for_nvfp4(0, run = run)[0]
main = _steps(env.installs())[0]
assert "--index-url" not in main # pip reads its own pip.conf mirror
def test_pip_config_is_only_read_when_pip_installs(env):
run = _pip_config(env, [":env:.no-index='1'"])
assert inst.ensure_flashinfer_for_nvfp4(0, run = run)[0]
assert run.calls == [] # uv ignores pip's settings
@pytest.mark.parametrize(
"line, needle",
[
(":env:.no-index='1'", "no-index"),
("global.no-index='true'", "no-index"),
(":env:.no-deps='yes'", "no-deps"),
("install.target='/opt/elsewhere'", "target directory"),
(":env:.prefix='/opt/elsewhere'", "prefix"),
("install.root='/chroot'", "another root"),
(":env:.force-reinstall='1'", "force-reinstall"),
("install.force-reinstall='true'", "force-reinstall"),
(":env:.ignore-installed='yes'", "ignore installed"),
("global.ignore-installed='true'", "ignore installed"),
],
)
def test_pip_settings_it_cannot_honor_refuse_before_installing(env, monkeypatch, line, needle):
probed = []
monkeypatch.setattr(inst, "_reachable", lambda url: probed.append(url) or True)
monkeypatch.setattr(inst, "_uv_executable", lambda: None)
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = _pip_config(env, [line]))
assert not ok and needle in reason
assert env.installs() == [] and probed == []
assert inst.ensure_flashinfer_for_nvfp4(0, run = _pip_config(env, []))[0]
def test_pip_require_virtualenv_refuses_only_outside_a_venv(env, monkeypatch):
monkeypatch.setattr(inst, "_uv_executable", lambda: None)
run = _pip_config(env, [":env:.require-virtualenv='1'"])
monkeypatch.setattr(sys, "base_prefix", sys.prefix)
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = run)
assert not ok and "virtualenv" in reason and env.installs() == []
monkeypatch.setattr(sys, "base_prefix", sys.prefix + "-base")
assert inst.ensure_flashinfer_for_nvfp4(0, run = run)[0]
def _uv_toml(monkeypatch, tmp_path, text):
path = tmp_path / "uv.toml"
path.write_text(text, encoding = "utf-8")
monkeypatch.setenv("UV_CONFIG_FILE", str(path))
return path
@pytest.mark.parametrize(
"text, needle",
[
("offline = true\n", "offline"),
("no-index = true\n", "no-index"),
("[pip]\nno-index = true\n", "no-index"),
("[pip]\nno-deps = true\n", "no-deps"),
('[pip]\ntarget = "/opt/elsewhere"\n', "target directory"),
('[pip]\nprefix = "/opt/elsewhere"\n', "prefix"),
("reinstall = true\n", "reinstall"),
("[pip]\nreinstall = true\n", "reinstall"),
('[pip]\nreinstall-package = ["torch"]\n', "reinstall"),
],
)
def test_uv_config_settings_it_cannot_honor_refuse_before_installing(
env, monkeypatch, tmp_path, text, needle
):
probed = []
monkeypatch.setattr(inst, "_reachable", lambda url: probed.append(url) or True)
_uv_toml(monkeypatch, tmp_path, text)
ok, reason = _ensure(env)
assert not ok and needle in reason
assert env.installs() == [] and probed == []
monkeypatch.delenv("UV_CONFIG_FILE")
assert _ensure(env)[0]
def test_uv_offline_env_false_overrides_a_config_offline(env, monkeypatch, tmp_path):
_uv_toml(monkeypatch, tmp_path, "offline = true\n")
monkeypatch.setenv("UV_OFFLINE", "0")
assert _ensure(env)[0]
def test_pip_section_overrides_the_top_level_in_uv_config(env, monkeypatch, tmp_path):
_uv_toml(monkeypatch, tmp_path, "no-index = true\n[pip]\nno-index = false\n")
assert _ensure(env)[0]
@pytest.mark.parametrize(
"text",
[
'index-url = "https://corp.example/simple"\n',
'[[index]]\nurl = "https://corp.example/simple"\n',
'[pip]\nextra-index-url = ["https://corp.example/simple"]\n',
"this is not toml = = \n",
],
)
def test_a_uv_config_mirror_skips_the_pypi_probe(env, monkeypatch, tmp_path, text):
probed = []
monkeypatch.setattr(inst, "_reachable", lambda url: probed.append(url) or True)
_uv_toml(monkeypatch, tmp_path, text)
assert _ensure(env)[0]
assert probed == [f"https://flashinfer.ai/whl/cu130/{PROVIDER}/"]
def test_uv_config_discovery_finds_the_project_file_and_honors_no_config(
env, monkeypatch, tmp_path
):
project = tmp_path / "proj" / "sub"
project.mkdir(parents = True)
(tmp_path / "proj" / "pyproject.toml").write_text(
'[project]\nname = "x"\n[tool.uv]\nindex-url = "https://corp.example/simple"\n',
encoding = "utf-8",
)
(project / "pyproject.toml").write_text('[project]\nname = "y"\n', encoding = "utf-8")
monkeypatch.chdir(project)
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg"))
monkeypatch.setenv("XDG_CONFIG_DIRS", str(tmp_path / "xdg-sys"))
monkeypatch.delenv("UV_NO_CONFIG")
config = inst._installer_config("uv", env.run)
assert config["mirror"] and config["own_index"]
monkeypatch.setenv("PIP_INDEX_URL", "https://pip-mirror.example/simple")
assert "--index-url" not in inst._installer_prefix("uv", own_index = config["own_index"])
monkeypatch.setenv("UV_NO_CONFIG", "1")
assert not inst._installer_config("uv", env.run)["own_index"]
monkeypatch.delenv("UV_NO_CONFIG")
monkeypatch.chdir(tmp_path)
(tmp_path / "xdg" / "uv").mkdir(parents = True)
(tmp_path / "xdg" / "uv" / "uv.toml").write_text("offline = true\n", encoding = "utf-8")
assert "offline" in inst._installer_config("uv", env.run)["refusal"]
@pytest.mark.parametrize("uv", ["uv", None])
def test_a_proxy_the_probe_cannot_use_skips_the_probes(env, monkeypatch, uv):
monkeypatch.setattr(inst, "_reachable", lambda url: False)
monkeypatch.setattr(inst, "_uv_executable", lambda: uv)
monkeypatch.setenv("ALL_PROXY", "socks5h://proxy.example:1080")
assert inst.ensure_flashinfer_for_nvfp4(0, run = _pip_config(env, []))[0]
def test_pips_own_proxy_setting_skips_the_probes(env, monkeypatch):
monkeypatch.setattr(inst, "_reachable", lambda url: False)
monkeypatch.setattr(inst, "_uv_executable", lambda: None)
run = _pip_config(env, ["global.proxy='http://proxy.example:3128'"])
assert inst.ensure_flashinfer_for_nvfp4(0, run = run)[0]
def test_reachability_counts_a_tls_failure_as_a_route(monkeypatch):
import ssl
import urllib.error
import urllib.request
def _raise(exc):
def _urlopen(*args, **kwargs):
raise exc
return _urlopen
cases = [
(urllib.error.URLError(ssl.SSLCertVerificationError(1, "self-signed certificate")), True),
(urllib.error.URLError(OSError(101, "Network is unreachable")), False),
(urllib.error.HTTPError("https://x", 404, "Not Found", {}, None), False),
(TimeoutError("timed out"), False),
]
for exc, expected in cases:
monkeypatch.setattr(urllib.request, "urlopen", _raise(exc))
assert inst._reachable("https://pypi.org/simple/flashinfer-python/") is expected
def test_an_install_killed_between_its_steps_completes_on_the_next_load(env):
# Killed between steps: the next load must finish, not treat jit-cache-less flashinfer as done.
real_run = env.run
calls = {"n": 0}
def dies_on_second_install(cmd, **kwargs):
if "install" in cmd and "uninstall" not in cmd:
calls["n"] += 1
if calls["n"] == 2:
raise SystemExit("killed mid-install")
return real_run(cmd, **kwargs)
with pytest.raises(SystemExit):
inst.ensure_flashinfer_for_nvfp4(0, run = dies_on_second_install)
inst.reset_install_state()
env.importable = "flashinfer-python" in env.dists
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = env.run)
assert ok, reason
assert env.dists.get("flashinfer-python") == V
assert env.dists.get("flashinfer-jit-cache") == f"{V}+cu130", reason
assert env.dists.get(PROVIDER) == f"{V}+cu130", reason
@pytest.mark.parametrize("model", ["DiffusionStatusResponse", "VideoStatusResponse"])
def test_the_status_reason_hides_host_paths_from_api_key_callers(model):
from hub.utils.host_paths import redact_host_paths
from models import inference as models
secret = "/home/alice/.unsloth/studio/venv"
reason = f"flashinfer install rolled back: the installer failed: Using Python environment at: {secret} error"
status = getattr(models, model)(
loaded = True, transformer_quant_backend = "torchao", transformer_quant_backend_reason = reason
)
seen = redact_host_paths(status, via_api_key = True)
seen = seen if isinstance(seen, dict) else seen.model_dump()
assert secret not in str(seen["transformer_quant_backend_reason"])
assert "rolled back" in seen["transformer_quant_backend_reason"]
own = redact_host_paths(status, via_api_key = False)
own = own if isinstance(own, dict) else own.model_dump()
assert secret in own["transformer_quant_backend_reason"]
def test_a_concurrent_upgrade_by_another_installer_is_not_reverted(env):
# A concurrent terminal upgrade fails drift but must not be downgraded by the rollback.
env.concurrent_add = {"packaging": "26.0"}
ok, reason = _ensure(env)
assert not ok and "rolled back" in reason
assert env.dists["packaging"] == "26.0", reason
assert not [c for c in env.commands if "packaging==25.0" in c]
def test_a_later_step_that_dies_unreported_is_still_rolled_back(env):
real_run = env.run
def run(cmd, **kwargs):
result = real_run(cmd, **kwargs)
steps = [c for c in env.installs() if any(a.startswith("flashinfer") for a in c)]
if "install" in cmd and "uninstall" not in cmd and len(steps) == 2:
return _Result(1, "")
return result
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = run)
assert not ok and "rolled back" in reason
assert "flashinfer-python" not in env.dists, reason
assert "flashinfer-jit-cache" not in env.dists, reason
def test_a_failed_install_does_not_quote_index_credentials(env):
real_run = env.run
def run(cmd, **kwargs):
if "install" in cmd and "uninstall" not in cmd:
return _Result(
1,
"error: https://__token__:[email protected]/simple/ ?token=SECRET2 returned 401",
)
return real_run(cmd, **kwargs)
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = run)
assert not ok and "the installer failed" in reason
assert "SECRET" not in reason
def test_an_unreported_failure_does_not_revert_a_concurrent_upgrade(env):
env.concurrent_add = {"packaging": "26.0"}
real_run = env.run
def run(cmd, **kwargs):
result = real_run(cmd, **kwargs)
if (
"install" in cmd
and "uninstall" not in cmd
and any(a.startswith("flashinfer-python==") for a in cmd)
):
return _Result(1, "")
return result
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = run)
assert not ok and "rolled back" in reason
assert "flashinfer-python" not in env.dists, reason
assert env.dists["packaging"] == "26.0", reason
assert not [c for c in env.commands if "packaging==25.0" in c]
def test_cutlass_dsl_is_held_to_the_flash4_range_when_absent_on_the_fallback(env):
# An unconstrained resolve lands cutlass-dsl 4.8.0, which the hub FA4 build cannot load. Only the fallback
# release accepts the FA4 range; a cu128 torch has no newer prebuilt jit-cache, so it takes the fallback.
env.cuda = "12.8"
ok, reason = _ensure(env)
assert ok, reason
assert "nvidia-cutlass-dsl>=4.4,<4.6" in env.constraints_seen[0].splitlines()
assert env.dists["flashinfer-python"] == FALLBACK
def test_the_newest_release_does_not_hold_cutlass_dsl(env):
# Its own floor (>= 4.8) is above the FA4 range, so a hold there would make every fresh install unresolvable.
ok, reason = _ensure(env)
assert ok, reason
assert env.dists["flashinfer-python"] == V
assert not any("nvidia-cutlass-dsl" in line for line in env.constraints_seen[0].splitlines())
def test_an_installed_cutlass_dsl_is_pinned_not_ranged(env):
# 4.4.2 is below the newest release's floor and is never moved, so the fallback is installed beside it.
env.dists["nvidia-cutlass-dsl"] = "4.4.2"
ok, reason = _ensure(env)
assert ok, reason
pins = env.constraints_seen[0].splitlines()
assert "nvidia-cutlass-dsl==4.4.2" in pins and "nvidia-cutlass-dsl>=4.4,<4.6" not in pins
assert env.dists["flashinfer-python"] == FALLBACK
assert PROVIDER not in env.dists
@pytest.mark.parametrize(
"cuda, capability, dists, version, tag",
[
("13.0", (10, 0), {}, V, "cu130"),
("12.9", (12, 0), {}, V, "cu129"),
("13.2", (10, 3), {}, V, "cu130"),
("12.8", (10, 0), {}, FALLBACK, "cu128"),
("13.0", (10, 0), {"nvidia-cutlass-dsl": "4.5.0"}, FALLBACK, "cu130"),
("13.0", (10, 0), {"nvidia-cutlass-dsl": "4.6.2a0"}, FALLBACK, "cu130"),
("13.0", (10, 0), {"apache-tvm-ffi": "0.1.9"}, FALLBACK, "cu130"),
("13.0", (10, 0), {"nvidia-cudnn-frontend": "1.20.0"}, FALLBACK, "cu130"),
(
"13.0",
(10, 0),
{"nvidia-cutlass-dsl": "4.8.1", "apache-tvm-ffi": "0.1.15.post2"},
V,
"cu130",
),
# No prebuilt anywhere: the newest that fits, JIT-compiled (the caller needs nvcc).
("12.6", (10, 0), {}, V, None),
("14.0", (10, 0), {}, V, None),
],
)
def test_select_release(cuda, capability, dists, version, tag):
release, got_tag, _ = inst.select_release(cuda, capability, dists)
assert (release.version, got_tag) == (version, tag)
def test_select_release_names_the_held_dependency_when_nothing_fits(monkeypatch):
newest = inst.FLASHINFER_RELEASES[0]
monkeypatch.setattr(inst, "FLASHINFER_RELEASES", (newest,))
release, tag, why = inst.select_release("13.0", (10, 0), {"nvidia-cutlass-dsl": "4.4.2"})
assert release is None and tag is None
assert "nvidia-cutlass-dsl>=4.8 (installed 4.4.2)" in why
@pytest.mark.parametrize(
"capability, provider",
[
((10, 0), "flashinfer-jit-cache-sm100a"),
((10, 3), "flashinfer-jit-cache-sm103a"),
((12, 0), "flashinfer-jit-cache-sm120f"),
],
)
def test_every_nvfp4_capability_has_a_jit_cache_provider(capability, provider):
assert tuple(capability) in ops.NVFP4_FLASHINFER_CAPS
assert inst.jit_cache_provider(inst.FLASHINFER_RELEASES[0], capability) == provider
assert set(inst.FLASHINFER_JIT_CACHE_ARCH) == set(ops.NVFP4_FLASHINFER_CAPS)
def test_the_newest_release_installs_the_shim_and_this_devices_provider_only(env):
ok, reason = _ensure(env)
assert ok, reason
jit = next(c for c in env.installs() if any(a.startswith("flashinfer-jit-cache") for a in c))
specs = [a for a in jit if a.startswith("flashinfer-")]
assert specs == [f"{PROVIDER}=={V}+cu130", f"flashinfer-jit-cache=={V}+cu130"]
assert "--no-deps" in jit # the shim requires every arch's provider
def test_a_split_release_whose_provider_is_not_loaded_rolls_back(env, monkeypatch):
real_run = env.run
def no_providers(cmd, **kwargs):
res = real_run(cmd, **kwargs)
if "-c" in cmd and "import flashinfer" in str(cmd[-1]) and res.returncode == 0:
res.stdout = f"providers=\n{V}"
return res
ok, reason = inst.ensure_flashinfer_for_nvfp4(0, run = no_providers)
assert (
not ok
and "rolled back" in reason
and f"did not load the {PROVIDER} prebuilt kernels" in reason
)
assert PROVIDER not in env.dists and "flashinfer-python" not in env.dists
def test_a_stray_provider_of_another_version_refuses(env):
env.dists[PROVIDER] = "0.7.2+cu130"
ok, reason = _ensure(env)
assert not ok and f"{PROVIDER} 0.7.2+cu130" in reason
assert env.installs() == []
def test_the_fallback_installs_the_single_jit_cache(env):
env.cuda = "12.8"
ok, reason = _ensure(env)
assert ok, reason
jit = next(c for c in env.installs() if any(a.startswith("flashinfer-jit-cache") for a in c))
assert [a for a in jit if a.startswith("flashinfer-")] == [
f"flashinfer-jit-cache=={FALLBACK}+cu128"
]
def test_an_installed_fallback_is_kept_not_upgraded(env):
env.dists.update({"flashinfer-python": FALLBACK, "flashinfer-jit-cache": f"{FALLBACK}+cu130"})
env.importable = True
ok, reason = _ensure(env)
assert ok and FALLBACK in reason
assert env.installs() == []
def test_the_verify_child_runs_the_shipped_snippet():
import ast
ast.parse(inst._VERIFY_CODE)
assert inst._VERIFY_CODE.rstrip().endswith("'unknown')))")
def _two_arch_host(monkeypatch, caps):
torch = sys.modules["torch"]
torch.cuda = types.SimpleNamespace(device_count = lambda: len(caps))
monkeypatch.setattr(
ops,
"_device_capability",
lambda device = None: caps[device if isinstance(device, int) else 0],
)
def test_a_mixed_arch_host_installs_every_visible_nvfp4_provider(env, monkeypatch):
# A later load on the other card finds flashinfer importable and never installs again, so both go in now.
_two_arch_host(monkeypatch, [(10, 0), (8, 0), (12, 0)])
ok, reason = _ensure(env)
assert ok, reason
jit = next(c for c in env.installs() if any(a.startswith("flashinfer-jit-cache") for a in c))
assert [a for a in jit if a.startswith("flashinfer-")] == [
f"{PROVIDER}=={V}+cu130",
f"flashinfer-jit-cache-sm120f=={V}+cu130",
f"flashinfer-jit-cache=={V}+cu130",
]
def test_a_mixed_arch_install_missing_one_provider_rolls_back(env, monkeypatch):
_two_arch_host(monkeypatch, [(12, 0), (10, 0)])
real_run = env.run
def one_provider(cmd, **kwargs):
res = real_run(cmd, **kwargs)
if "-c" in cmd and "import flashinfer" in str(cmd[-1]) and res.returncode == 0:
res.stdout = f"providers=flashinfer-jit-cache-sm120f\n{V}"
return res
ok, reason = inst.ensure_flashinfer_for_nvfp4(1, run = one_provider)
assert not ok and f"did not load the {PROVIDER} prebuilt kernels" in reason
assert "flashinfer-jit-cache-sm120f" not in env.dists and PROVIDER not in env.dists