Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: FLUX.3 upsampler pin follows torch's vendor, single-GPU hosts launch as before, non-blocking Vulkan probe (#14491)

Daniel Han3f74ad129f89
diff --git a/studio/backend/core/inference/flux3_prompt_upsampler.py b/studio/backend/core/inference/flux3_prompt_upsampler.py
index 443ce4c5d72..2fd235042a8 100644
--- a/studio/backend/core/inference/flux3_prompt_upsampler.py
+++ b/studio/backend/core/inference/flux3_prompt_upsampler.py
@@ -938,13 +938,113 @@ def _user_device_pick(binary: Optional[str] = None) -> bool:
return True
-def _prefetch_vulkan_rows(binary: str) -> None:
+def _vulkan_pin_possible(binary: str, torch) -> bool:
+ """Whether a Vulkan pin is worth the probe: a torch card, a Vulkan build, nothing the user picked."""
+ from core.inference.llama_cpp import LlamaCppBackend
+ return (
+ torch.cuda.device_count() >= 1
+ and not _user_device_pick()
+ and not str(os.environ.get("GGML_BACKEND_PATH", "")).strip()
+ and bool(LlamaCppBackend._is_vulkan_backend(binary))
+ and not (
+ (os.environ.get("HIP_VISIBLE_DEVICES") or "").strip()
+ and not LlamaCppBackend._torch_is_rocm(torch)
+ )
+ )
+
+
+def _prefetch_vulkan_rows(binary: str, cancel: Optional[threading.Event] = None) -> None:
+ """Probe (and read the card lookup's inventory) before the render slot, waited for cancellably: the probe can take
+ its 15 s timeout, which a blocking unload behind this warm start must not sit out."""
try:
- from core.inference.llama_cpp import LlamaCppBackend
- if not _user_device_pick() and LlamaCppBackend._is_vulkan_backend(binary):
- _vulkan_rows(binary)
+ import torch
+ if not _vulkan_pin_possible(binary, torch):
+ return
except Exception: # noqa: BLE001
- pass
+ return
+
+ def _work() -> None:
+ try:
+ # An empty answer (cached or fresh) is retried once, as the launch it stands in for did.
+ rows = _vulkan_rows(binary, retry_empty = True) or _vulkan_rows(binary, retry_empty = True)
+ # One row on one torch card launches unpinned, so it needs no card lookup.
+ if rows and (len(rows) > 1 or torch.cuda.device_count() > 1):
+ ordinal = _resident_cuda_ordinal()
+ if ordinal is not None and ordinal >= 0:
+ _vulkan_card_name(ordinal, torch)
+ except Exception: # noqa: BLE001
+ pass
+
+ # A maintenance pass that cancelled this start before the probe began must not find one running.
+ if cancel is not None and cancel.is_set():
+ raise UpsamplerCancelledError("Cancelled.")
+ worker = threading.Thread(target = _work, name = "flux3-pup-vulkan-probe", daemon = True)
+ # Registered once started: joining a thread that has not started raises.
+ worker.start()
+ _vulkan_probe_workers[:] = [w for w in _vulkan_probe_workers if w.is_alive()] + [worker]
+ while worker.is_alive():
+ worker.join(0.1)
+ if cancel is not None and cancel.is_set():
+ raise UpsamplerCancelledError("Cancelled.")
+
+
+_vulkan_probe_workers: list = []
+
+
+def _wait_for_vulkan_probe(timeout: float = 20.0) -> None:
+ """Join prefetches cancelled warm starts abandoned: their probes can still hold the llama.cpp tree's libraries."""
+ deadline = time.monotonic() + timeout
+ for worker in list(_vulkan_probe_workers):
+ worker.join(max(0.0, deadline - time.monotonic()))
+
+
+def _vulkan_rows_came_back_empty(binary: str) -> bool:
+ """A pinnable multi-GPU host whose cached Vulkan probe listed nothing: a warm start would stay unpinned for good.
+ One torch card warm-starts unpinned as before."""
+ try:
+ import torch
+ return (
+ torch.cuda.device_count() > 1
+ and _vulkan_pin_possible(binary, torch)
+ and not _vulkan_rows(binary)
+ )
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _vulkan_card_name(ordinal: int, torch) -> tuple:
+ """``physical_card_name`` on ROCm. A CUDA torch reads no HIP/ROCR mask and its cards are not amd-smi's, so that
+ lookup would place it by an AMD card's position: name it in torch order, unplaced under a CUDA mask (Vulkan still
+ lists the cards it hides)."""
+ from core.inference.llama_cpp import LlamaCppBackend
+ from core.inference.sd_cpp_backend import physical_card_name
+
+ if LlamaCppBackend._torch_is_rocm(torch):
+ return physical_card_name(ordinal)
+ if not torch.cuda.is_available() or not 0 <= ordinal < torch.cuda.device_count():
+ return None, None
+ names = [
+ (torch.cuda.get_device_name(i) or "").strip() for i in range(torch.cuda.device_count())
+ ]
+ name = names[ordinal]
+ if not name:
+ return None, None
+ if (os.environ.get("CUDA_VISIBLE_DEVICES") or "").strip():
+ return name, None
+ # Between identical cards only PCI order matches Vulkan's; FASTEST_FIRST (CUDA's default) may not.
+ if names.count(name) > 1 and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID":
+ return name, None
+ return name, names[:ordinal].count(name)
+
+
+def _vulkan_row_is_another_vendor(row_name: str, torch) -> bool:
+ """A CUDA torch's card is NVIDIA's and a ROCm torch's AMD's: a Vulkan row naming neither is another card."""
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ name = str(row_name or "").lower()
+ if LlamaCppBackend._torch_is_rocm(torch):
+ return not any(tag in name for tag in ("amd", "radeon", "instinct"))
+ return "nvidia" not in name
def _vulkan_device_args(
@@ -952,55 +1052,54 @@ def _vulkan_device_args(
ordinal: Optional[int],
retry_empty: bool = False,
) -> Optional[list]:
- """``--device Vulkan<i>`` for the image model's card (Vulkan ignores visibility masks); [] unpinned, None unknown."""
+ """``--device Vulkan<i>`` for the image model's card (Vulkan ignores visibility masks); [] unpinned, None unknown.
+ One torch card and one Vulkan device launch as before Vulkan pinning; non-Vulkan builds never probe."""
if ordinal is None or ordinal < 0:
return []
try:
- from core.inference.llama_cpp import LlamaCppBackend
-
import torch
- # The user's own device pick wins (a Vulkan mask renumbers what the probe lists).
- if _user_device_pick():
- return []
- # Backends loaded from GGML_BACKEND_PATH are not the ones beside the binary the probe would list.
- if str(os.environ.get("GGML_BACKEND_PATH", "")).strip():
- return None if torch.cuda.device_count() > 1 else []
- if not LlamaCppBackend._is_vulkan_backend(binary):
+ count = torch.cuda.device_count()
+ if count < 1 or _user_device_pick():
return []
- # The card lookup reads HIP masks, which only a ROCm torch applies: elsewhere one could name the wrong card.
- if (
- os.environ.get("HIP_VISIBLE_DEVICES") or ""
- ).strip() and not LlamaCppBackend._torch_is_rocm(torch):
- return None if torch.cuda.device_count() > 1 else []
- # Vulkan sees every physical GPU (an iGPU, cards a mask hides), so torch's count says nothing here.
+ if not _vulkan_pin_possible(binary, torch):
+ # Backends from GGML_BACKEND_PATH are not the ones the probe lists; a HIP mask on a CUDA torch steers nothing.
+ from core.inference.llama_cpp import LlamaCppBackend
+ if count == 1 or not (
+ str(os.environ.get("GGML_BACKEND_PATH", "")).strip()
+ or LlamaCppBackend._is_vulkan_backend(binary)
+ ):
+ return []
+ return None
+ # Vulkan also lists an iGPU and cards a mask hides (ggml's default takes discrete AND integrated GPUs).
rows = _vulkan_rows(binary, retry_empty = retry_empty)
if not rows:
- return None if torch.cuda.device_count() > 1 else []
+ return None if count > 1 else []
+ if len(rows) == 1 and count == 1:
+ # Nothing to choose, so no name match (a renamed card never declines a single-GPU host),
+ # unless that one device is plainly another vendor's: then it is not the image model's card.
+ return None if _vulkan_row_is_another_vendor(rows[0]["name"], torch) else []
except Exception: # noqa: BLE001
return []
try:
from core.inference.sd_cpp_backend import (
_normalized_card_name,
_without_driver_tag,
- physical_card_name,
sd_cpp_device_named,
)
# By name, as the image engine pins its Vulkan build: no torch ordinal maps onto Vulkan's index.
listing = "".join(f"Vulkan{row['index']}\t{row['name']}\n" for row in rows)
- name, position = physical_card_name(ordinal)
+ name, position = _vulkan_card_name(ordinal, torch)
device = sd_cpp_device_named(binary, name, position = position, listing = listing)
if not device:
return None
# Exact model match: containment takes an `RX 7600 XT` for an `RX 7600`.
- import torch
-
matched = next((row["name"] for row in rows if f"Vulkan{row['index']}" == device), "")
exact = bool(name) and _normalized_card_name(
_without_driver_tag(matched)
) == _normalized_card_name(name)
- if not exact and (len(rows) > 1 or torch.cuda.device_count() > 1):
+ if not exact:
return None
return [] if len(rows) == 1 else ["--device", device]
except Exception: # noqa: BLE001
@@ -1194,10 +1293,14 @@ class PromptUpsamplerSidecar:
self._update_in_progress = True
try:
self.cancel_pending_load()
- with self._request_lock, self._lock:
- was_active = self.loaded
- self._release_locked()
- yield was_active
+ _wait_for_vulkan_probe()
+ with self._request_lock:
+ # Again once no warm start can still be about to start one.
+ _wait_for_vulkan_probe()
+ with self._lock:
+ was_active = self.loaded
+ self._release_locked()
+ yield was_active
finally:
self._update_in_progress = False
@@ -1258,7 +1361,8 @@ class PromptUpsamplerSidecar:
if layers == 0:
cmd.append("--no-mmproj-offload")
if layers:
- cmd += _vulkan_device_args(binary, ordinal, retry_empty = True) or []
+ # A warm start probed just before its slot: re-probing an empty answer here would run it twice.
+ cmd += _vulkan_device_args(binary, ordinal, retry_empty = not self._warming) or []
sock.close()
if is_process_shutting_down():
raise UpsamplerUnavailableError("Unsloth is shutting down.")
@@ -1504,7 +1608,7 @@ class PromptUpsamplerSidecar:
if _user_device_pick(binary) and _cuda_device_count() > 1:
return "placement"
# The Vulkan probe can take its full timeout, so it runs before the slot that renders queue behind.
- _prefetch_vulkan_rows(binary)
+ _prefetch_vulkan_rows(binary, self._load_cancel)
# The engine's generation slot, held until the server is up: a render or model transition that arrives
# meanwhile queues behind the start instead of overlapping it (a load cancels it first through
# stop_for_handover). Not taken while one runs or a load is in flight.
@@ -1512,6 +1616,9 @@ class PromptUpsamplerSidecar:
if slot is None:
return why
slot_held = slot
+ # An empty (maybe transient) probe would leave the server unpinned for good; a real launch re-probes.
+ if _vulkan_rows_came_back_empty(binary):
+ return "placement"
ordinal = _resident_cuda_ordinal()
# Unpinnable on a multi-GPU host (the native engine records its card elsewhere): the server would split
# over every visible card and the probe below would read the wrong one. Its first enhanced run starts it.
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index 05a8142c3cc..e4d04210ffa 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -748,8 +748,13 @@ def sd_cpp_vulkan_device_is_uma(
_GGML_DEVICE_PREFIXES: tuple[str, ...] = ("CUDA", "ROCM", "VULKAN", "SYCL", "METAL", "OPENCL")
+_TRADEMARK_RE = re.compile(r"\((?:tm|r)\)|[\u2122\u00ae]", re.IGNORECASE)
+
+
def _normalized_card_name(text: str) -> str:
- """Letters and digits only, lowercased: `AMD Radeon RX 7900 XTX (RADV NAVI31)` then contains `AMD Radeon RX 7900 XTX`."""
+ """Letters and digits only, lowercased, trademark marks dropped (HIP's `AMD Radeon(TM) Graphics` is RADV's
+ `AMD Radeon Graphics`): `AMD Radeon RX 7900 XTX (RADV NAVI31)` then contains `AMD Radeon RX 7900 XTX`."""
+ text = _TRADEMARK_RE.sub(" ", text)
return "".join(character for character in text.lower() if character.isalnum())
@@ -803,14 +808,14 @@ def _physical_index_of(ordinal: int, env: Optional[dict] = None) -> "tuple[Optio
if opaque is not None:
return None, True
visible: "Optional[list[int]]" = None
- for raw in (rocr, hip):
+ for is_hip, raw in ((False, rocr), (True, hip)):
if raw is None:
continue
entries = _mask_entries(raw)
if entries is None:
return None, True
# clr keeps the first of a repeated HIP entry; ROCr stops at one (utils/hardware/amd.py).
- if raw is hip:
+ if is_hip:
entries = list(dict.fromkeys(entries))
elif len(set(entries)) < len(entries):
entries = entries[: next(i for i, e in enumerate(entries) if e in entries[:i])]
diff --git a/studio/backend/tests/test_flux3_prompt_upsampler.py b/studio/backend/tests/test_flux3_prompt_upsampler.py
index 7ebb1d6eb11..cd3e45ad77e 100644
--- a/studio/backend/tests/test_flux3_prompt_upsampler.py
+++ b/studio/backend/tests/test_flux3_prompt_upsampler.py
@@ -1088,11 +1088,11 @@ def test_vulkan_sole_device_must_be_the_image_models_card(spawn_argv, warmable,
state, run = spawn_argv
sidecar, launched, _memory, model, _mmproj = warmable
- # torch holds the image model on the 7900 XTX, but the only Vulkan device is the iGPU.
+ # torch holds the image model on the 7900 XTX, but Vulkan lists only the iGPU and an RX 7600.
monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
state["names"] = ["AMD Radeon RX 7900 XTX"]
state["ordinal"] = 0
- state["rows"] = [_VULKAN_ROWS[0]]
+ state["rows"] = _VULKAN_ROWS[:2]
assert "--device" not in run()[0]
assert sidecar.warm(binary = "b", model = model, mmproj = None) == "placement"
@@ -2124,7 +2124,10 @@ def test_the_vulkan_probe_runs_before_the_render_slot(warmable, monkeypatch):
# A slow probe under the slot would hold renders and handovers until its timeout.
from core.inference import llama_cpp
+ import torch
+
sidecar, _launched, _memory, model, _mmproj = warmable
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
order = []
take = pup._take_render_slot
monkeypatch.setattr(pup, "_take_render_slot", lambda: order.append("slot") or take())
@@ -2135,7 +2138,8 @@ def test_the_vulkan_probe_runs_before_the_render_slot(warmable, monkeypatch):
pup, "_vulkan_rows", lambda b, retry_empty = False: order.append("probe") or []
)
sidecar.warm(binary = "b", model = model, mmproj = None)
- assert order[:2] == ["probe", "slot"]
+ # An empty answer is retried once, still before the slot.
+ assert order[: order.index("slot")] == ["probe", "probe"]
sidecar._cancel_idle_locked()
@@ -2240,3 +2244,421 @@ def test_a_user_main_gpu_is_a_device_pick(monkeypatch):
monkeypatch.setenv("LLAMA_ARG_MAIN_GPU", "1")
assert pup._user_device_pick() is True
assert pup._vulkan_device_args("b", 0) == []
+
+
+from core.inference import sd_cpp_backend as _sd_cpp_backend # noqa: E402
+
+_REAL_PHYSICAL_POSITION_OF = _sd_cpp_backend._physical_position_of
+
+
+def _real_card_lookup(monkeypatch, amd_rows, hip_by_row):
+ """Undo the fixture's position stub: the real lookup over a mocked amd-smi inventory and HIP id map."""
+ from core.inference import sd_cpp_backend
+ from utils.hardware import amd
+
+ monkeypatch.setattr(sd_cpp_backend, "_physical_position_of", _REAL_PHYSICAL_POSITION_OF)
+ inventory = {
+ "devices": [{"index": i, "vendor": "amd", "name": name} for i, name in enumerate(amd_rows)]
+ }
+ monkeypatch.setattr(sd_cpp_backend, "_card_lookup_inventory", lambda: inventory)
+ monkeypatch.setattr(amd, "get_hip_id_by_gpu_index", lambda: dict(hip_by_row))
+
+
+_MIXED_ROWS = [
+ {"index": 0, "name": "AMD Radeon Graphics (RADV RAPHAEL)"},
+ {"index": 1, "name": "AMD Radeon RX 7600 (RADV NAVI33)"},
+ {"index": 2, "name": "NVIDIA GeForce RTX 4090"},
+ {"index": 3, "name": "NVIDIA GeForce RTX 4090"},
+]
+
+
[email protected]("ordinal, device", [(0, "Vulkan2"), (1, "Vulkan3")])
+def test_a_cuda_torch_never_takes_an_amd_cards_position(spawn_argv, monkeypatch, ordinal, device):
+ # Two RTX 4090s on a CUDA torch beside AMD cards amd-smi reads: HIP device 1 (the RX 7600) is no torch cuda:1.
+ import torch
+
+ state, run = spawn_argv
+ _real_card_lookup(monkeypatch, ["AMD Radeon Graphics", "AMD Radeon RX 7600"], {0: 0, 1: 1})
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ state["names"] = ["NVIDIA GeForce RTX 4090", "NVIDIA GeForce RTX 4090"]
+ state["rows"] = _MIXED_ROWS
+ state["ordinal"] = ordinal
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+ argv, env = run()
+ assert argv[-2:] == ["--device", device] and env == {"CUDA_VISIBLE_DEVICES": str(ordinal)}
+
+
[email protected]("order", [None, "FASTEST_FIRST"])
+def test_identical_cuda_cards_are_not_told_apart_outside_pci_order(
+ spawn_argv, warmable, monkeypatch, order
+):
+ # CUDA's default FASTEST_FIRST may not match Vulkan's (PCI) order, so torch's position among twins is no pin.
+ import torch
+
+ state, run = spawn_argv
+ sidecar, launched, _memory, model, _mmproj = warmable
+ _real_card_lookup(monkeypatch, ["AMD Radeon Graphics", "AMD Radeon RX 7600"], {0: 0, 1: 1})
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)
+ state["names"] = ["NVIDIA GeForce RTX 4090", "NVIDIA GeForce RTX 4090"]
+ state["rows"] = _MIXED_ROWS
+ state["ordinal"] = 1
+ if order is None:
+ monkeypatch.delenv("CUDA_DEVICE_ORDER", raising = False)
+ else:
+ monkeypatch.setenv("CUDA_DEVICE_ORDER", order)
+ assert pup._vulkan_device_args("b", 1) is None
+ assert "--device" not in run()[0]
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "placement"
+ # A card with no twin needs no order to be found.
+ state["names"] = ["NVIDIA GeForce RTX 4090", "AMD Radeon RX 7600"]
+ assert pup._vulkan_device_args("b", 0) == ["--device", "Vulkan2"]
+
+
+ "var, value", [("CUDA_VISIBLE_DEVICES", "1,0"), ("ROCR_VISIBLE_DEVICES", "1,0")]
+)
+def test_a_cuda_torch_mask_never_steers_the_vulkan_pin_onto_a_guess(
+ spawn_argv, warmable, monkeypatch, var, value
+):
+ # CVD reorders the identical pair (the order Vulkan cannot see); ROCR is not read by a CUDA torch at all.
+ import torch
+
+ state, run = spawn_argv
+ sidecar, launched, _memory, model, _mmproj = warmable
+ _real_card_lookup(monkeypatch, ["AMD Radeon Graphics", "AMD Radeon RX 7600"], {0: 0, 1: 1})
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)
+ state["names"] = ["NVIDIA GeForce RTX 4090", "NVIDIA GeForce RTX 4090"]
+ state["rows"] = _MIXED_ROWS
+ state["ordinal"] = 1
+ monkeypatch.setenv(var, value)
+ expected = None if var == "CUDA_VISIBLE_DEVICES" else ["--device", "Vulkan3"]
+ assert pup._vulkan_device_args("b", 1) == expected
+ if expected is None:
+ assert "--device" not in run()[0]
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "placement"
+
+
[email protected]("vulkan", [False, True])
+def test_one_torch_card_and_one_vulkan_device_launch_as_before(
+ spawn_argv, warmable, monkeypatch, vulkan
+):
+ # The pre-pinning launch: no --device, no mask, a start. A non-Vulkan build never probes; one Vulkan row (even a
+ # renamed `Radeon(TM)` card) is nothing to choose between, so no name match can decline it.
+ import torch
+
+ from core.inference import llama_cpp
+
+ state, run = spawn_argv
+ sidecar, launched, _memory, model, _mmproj = warmable
+ calls = []
+ monkeypatch.setattr(
+ llama_cpp.LlamaCppBackend,
+ "_run_vulkan_probe",
+ staticmethod(
+ lambda b = None: calls.append(b)
+ or [{"index": 0, "name": "AMD Radeon Graphics (RADV RAPHAEL)"}]
+ ),
+ )
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
+ state["vulkan"] = vulkan
+ state["names"] = ["AMD Radeon(TM) Graphics"]
+ state["ordinal"] = 0
+ monkeypatch.setattr(pup, "_resident_cuda_ordinal", lambda: 0)
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "started"
+ sidecar._cancel_idle_locked()
+ argv, env = run()
+ assert "--device" not in argv and env == {}
+ assert bool(calls) is vulkan
+
+
[email protected]("cards", [1, 2])
+def test_a_warm_start_retries_an_empty_vulkan_answer_once(spawn_argv, warmable, monkeypatch, cards):
+ # The warm start retries an empty answer once, as the launch it stands in for did; its real spawn adds none.
+ import torch
+
+ from core.inference import llama_cpp
+
+ state, _run = spawn_argv
+ sidecar, launched, _memory, model, _mmproj = warmable
+ calls = []
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: cards)
+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: cards)
+ monkeypatch.setattr(pup, "_resident_cuda_ordinal", lambda: 0)
+ monkeypatch.setattr(pup, "_VULKAN_ROWS_TTL_S", 120.0)
+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})
+ monkeypatch.setattr(
+ llama_cpp.LlamaCppBackend,
+ "_run_vulkan_probe",
+ staticmethod(lambda b = None: calls.append(b) or []),
+ )
+ monkeypatch.setattr(
+ sidecar, "_spawn_server", pup.PromptUpsamplerSidecar._spawn_server.__get__(sidecar)
+ )
+ # Several cards: declined, not started unpinned; the first enhanced run re-probes (and pins, or launches as
+ # before). One card goes on to the spawn unpinned, as before pinning.
+ out = sidecar.warm(binary = "/x/llama-server", model = model, mmproj = None)
+ assert (out == "placement") is (cards > 1)
+ assert len(calls) == 2
+
+
+def test_maintenance_waits_for_every_abandoned_vulkan_probe(monkeypatch):
+ # Two warm starts cancelled within one probe's timeout leave two probes holding the llama.cpp libraries.
+ import torch
+
+ from core.inference import llama_cpp
+
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ monkeypatch.setattr(
+ llama_cpp.LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: True)
+ )
+ for var in (
+ "HIP_VISIBLE_DEVICES",
+ "GGML_BACKEND_PATH",
+ "LLAMA_ARG_DEVICE",
+ "LLAMA_ARG_MAIN_GPU",
+ ):
+ monkeypatch.delenv(var, raising = False)
+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})
+ monkeypatch.setattr(pup, "_vulkan_probe_workers", [], raising = False)
+ releases = [threading.Event(), threading.Event()]
+ entered = []
+
+ def _probe(b = None):
+ release = releases[len(entered)]
+ entered.append(b)
+ release.wait(10)
+ return []
+
+ monkeypatch.setattr(llama_cpp.LlamaCppBackend, "_run_vulkan_probe", staticmethod(_probe))
+ try:
+ for _ in releases:
+ cancel = threading.Event()
+ threading.Timer(0.2, cancel.set).start()
+ with pytest.raises(pup.UpsamplerCancelledError):
+ pup._prefetch_vulkan_rows("b", cancel)
+ releases[1].set()
+ waiter = threading.Thread(target = pup._wait_for_vulkan_probe, args = (10,))
+ waiter.start()
+ waiter.join(0.5)
+ assert waiter.is_alive()
+ releases[0].set()
+ waiter.join(5)
+ assert not waiter.is_alive()
+ finally:
+ for release in releases:
+ release.set()
+
+
+ "name, row",
+ [
+ ("AMD Radeon(TM) Graphics", "AMD Radeon Graphics (RADV RAPHAEL)"),
+ ("AMD Radeon(TM) 8060S Graphics", "AMD Radeon 8060S Graphics (RADV GFX1151)"),
+ ],
+)
+def test_trademark_marks_do_not_hide_the_card(spawn_argv, warmable, monkeypatch, name, row):
+ import torch
+
+ state, run = spawn_argv
+ sidecar, launched, _memory, model, _mmproj = warmable
+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)
+ state["names"] = [name, "AMD Radeon RX 7900 XTX"]
+ state["rows"] = [{"index": 0, "name": row}, _VULKAN_ROWS[2]]
+ state["ordinal"] = 0
+ monkeypatch.setattr(pup, "_resident_cuda_ordinal", lambda: 0)
+ assert run()[0][-2:] == ["--device", "Vulkan0"]
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "started"
+ sidecar._cancel_idle_locked()
+
+
+def test_a_cancelled_warm_start_does_not_wait_out_the_vulkan_probe(warmable, monkeypatch):
+ # A blocking unload (handover, training start) must not sit behind a probe that can take its 15 s timeout.
+ from core.inference import llama_cpp
+
+ sidecar, launched, _memory, model, _mmproj = warmable
+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)
+ import torch
+
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(
+ llama_cpp.LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: True)
+ )
+ for var in (
+ "HIP_VISIBLE_DEVICES",
+ "GGML_BACKEND_PATH",
+ "LLAMA_ARG_DEVICE",
+ "LLAMA_ARG_MAIN_GPU",
+ ):
+ monkeypatch.delenv(var, raising = False)
+ entered, release = threading.Event(), threading.Event()
+
+ def _probe(b = None):
+ entered.set()
+ release.wait(10)
+ return list(_VULKAN_ROWS)
+
+ monkeypatch.setattr(llama_cpp.LlamaCppBackend, "_run_vulkan_probe", staticmethod(_probe))
+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})
+ out = []
+ warm = threading.Thread(
+ target = lambda: out.append(sidecar.warm(binary = "b", model = model, mmproj = None))
+ )
+ warm.start()
+ try:
+ assert entered.wait(5)
+ assert sidecar.cancel_warm_start()
+ unload = threading.Thread(target = sidecar.unload)
+ unload.start()
+ unload.join(3)
+ assert not unload.is_alive()
+ warm.join(3)
+ assert out == ["cancelled"] and launched == []
+ finally:
+ release.set()
+ warm.join(5)
+
+
+def test_maintenance_waits_for_a_probe_a_cancelled_warm_start_left_running(warmable, monkeypatch):
+ # The probe loads the llama.cpp tree's libraries; replacing that tree under it fails on Windows.
+ import torch
+
+ from core.inference import llama_cpp
+
+ sidecar, launched, _memory, model, _mmproj = warmable
+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(
+ llama_cpp.LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: True)
+ )
+ for var in (
+ "HIP_VISIBLE_DEVICES",
+ "GGML_BACKEND_PATH",
+ "LLAMA_ARG_DEVICE",
+ "LLAMA_ARG_MAIN_GPU",
+ ):
+ monkeypatch.delenv(var, raising = False)
+ entered, release = threading.Event(), threading.Event()
+
+ def _probe(b = None):
+ entered.set()
+ release.wait(10)
+ return list(_VULKAN_ROWS)
+
+ monkeypatch.setattr(llama_cpp.LlamaCppBackend, "_run_vulkan_probe", staticmethod(_probe))
+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})
+ out, phase = [], []
+ warm = threading.Thread(
+ target = lambda: out.append(sidecar.warm(binary = "b", model = model, mmproj = None))
+ )
+ warm.start()
+
+ def _maintain():
+ with sidecar.update_maintenance():
+ phase.append("replacing")
+
+ try:
+ assert entered.wait(5)
+ maint = threading.Thread(target = _maintain)
+ maint.start()
+ warm.join(3)
+ assert out == ["cancelled"]
+ maint.join(0.5)
+ assert phase == [] and maint.is_alive()
+ release.set()
+ maint.join(5)
+ assert phase == ["replacing"] and launched == []
+ finally:
+ release.set()
+ warm.join(5)
+
+
+def _prefetch_host(monkeypatch, cards, rows):
+ import torch
+
+ from core.inference import llama_cpp
+
+ calls, looked_up = [], []
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: cards)
+ monkeypatch.setattr(
+ llama_cpp.LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: True)
+ )
+ for var in (
+ "HIP_VISIBLE_DEVICES",
+ "GGML_BACKEND_PATH",
+ "LLAMA_ARG_DEVICE",
+ "LLAMA_ARG_MAIN_GPU",
+ ):
+ monkeypatch.delenv(var, raising = False)
+ monkeypatch.setattr(
+ llama_cpp.LlamaCppBackend,
+ "_run_vulkan_probe",
+ staticmethod(lambda b = None: calls.append(b) or list(rows)),
+ )
+ monkeypatch.setattr(pup, "_VULKAN_ROWS_TTL_S", 120.0)
+ monkeypatch.setattr(pup, "_resident_cuda_ordinal", lambda: 0)
+ monkeypatch.setattr(pup, "_vulkan_card_name", lambda o, t: looked_up.append(o) or ("x", None))
+ monkeypatch.setattr(pup, "_vulkan_probe_workers", [])
+ return calls, looked_up
+
+
+def test_a_warm_start_prefetch_retries_a_cached_empty_probe(monkeypatch):
+ # The launch a warm start stands in for re-probed a (maybe transient) empty answer; the prefetch does too.
+ calls, looked_up = _prefetch_host(monkeypatch, 1, _VULKAN_ROWS)
+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {("b", None): ([], time.monotonic())})
+ pup._prefetch_vulkan_rows("b")
+ assert calls == ["b"] and looked_up == [0]
+ # A cached non-empty answer is reused.
+ pup._prefetch_vulkan_rows("b")
+ assert calls == ["b"]
+
+
+def test_one_card_and_one_vulkan_row_skips_the_card_lookup(monkeypatch):
+ # That launch stays unpinned, so the prefetch reads no inventory for it.
+ calls, looked_up = _prefetch_host(monkeypatch, 1, _VULKAN_ROWS[:1])
+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})
+ pup._prefetch_vulkan_rows("b")
+ assert calls == ["b"] and looked_up == []
+
+
+def test_a_start_cancelled_before_its_probe_never_starts_one(monkeypatch):
+ calls, _looked_up = _prefetch_host(monkeypatch, 2, _VULKAN_ROWS)
+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})
+ cancel = threading.Event()
+ cancel.set()
+ with pytest.raises(pup.UpsamplerCancelledError):
+ pup._prefetch_vulkan_rows("b", cancel)
+ assert calls == [] and pup._vulkan_probe_workers == []
+
+
+def test_a_lone_vulkan_device_of_another_vendor_declines_the_warm_start(
+ spawn_argv, warmable, monkeypatch
+):
+ # A CUDA torch's image model on an RTX 4090 while Vulkan reaches only an AMD iGPU: not the same card.
+ import torch
+
+ state, run = spawn_argv
+ sidecar, launched, _memory, model, _mmproj = warmable
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ state["names"] = ["NVIDIA GeForce RTX 4090"]
+ state["ordinal"] = 0
+ state["rows"] = [{"index": 0, "name": "AMD Radeon Graphics (RADV RAPHAEL)"}]
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "placement"
+ # The explicit launch is unchanged: unpinned, as before Vulkan pinning.
+ argv, env = run()
+ assert "--device" not in argv
+ # Its own NVIDIA row warm-starts as before.
+ state["rows"] = [{"index": 0, "name": "NVIDIA GeForce RTX 4090"}]
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "started"
+ sidecar._cancel_idle_locked()
+
+
+def test_maintenance_never_joins_an_unstarted_probe(monkeypatch):
+ # A worker is registered only once started, so a maintenance pass cannot hit "cannot join thread".
+ import inspect
+ source = inspect.getsource(pup._prefetch_vulkan_rows)
+ assert source.index("worker.start()") < source.index("_vulkan_probe_workers[:]")
diff --git a/studio/backend/tests/test_sd_cpp_rocm_vulkan_fallback.py b/studio/backend/tests/test_sd_cpp_rocm_vulkan_fallback.py
index 6b91d1fc91c..4e7a60da677 100644
--- a/studio/backend/tests/test_sd_cpp_rocm_vulkan_fallback.py
+++ b/studio/backend/tests/test_sd_cpp_rocm_vulkan_fallback.py
@@ -3106,3 +3106,36 @@ def test_a_damaged_cli_beside_a_healthy_server_is_not_a_strike(fake_settings, mo
chosen = router.select_and_activate_engine(_detect_load_family(H3_REPO, None, "minimax-h3"))
assert chosen == "sd_cpp", chosen # the healthy server carries the load
assert _recorded_strikes(fake_settings) == 0 # and nothing was held against ROCm
+
+
+def test_one_mask_string_for_rocr_and_hip_keeps_rocrs_repeat_rule(monkeypatch):
+ # The same str object in both variables: ROCr still stops at the repeat, HIP still dedupes.
+ from core.inference import sd_cpp_backend
+
+ # ROCr is a Linux layer: Windows HIP ignores ROCR_VISIBLE_DEVICES.
+ monkeypatch.setattr(sd_cpp_backend.sys, "platform", "linux")
+
+ mask = "0,0,1"
+ env = {"ROCR_VISIBLE_DEVICES": mask, "HIP_VISIBLE_DEVICES": mask}
+ assert env["ROCR_VISIBLE_DEVICES"] is env["HIP_VISIBLE_DEVICES"]
+ assert sd_cpp_backend._physical_index_of(1, env = env) == (None, True)
+ assert sd_cpp_backend._physical_index_of(
+ 1, env = {"ROCR_VISIBLE_DEVICES": mask, "HIP_VISIBLE_DEVICES": "0,0,1 "}
+ ) == (None, True)
+
+
+def test_trademark_marks_do_not_split_a_card_name():
+ from core.inference import sd_cpp_backend
+
+ listing = "Vulkan0\tAMD Radeon Graphics (RADV RAPHAEL)\n"
+ assert (
+ sd_cpp_backend.sd_cpp_device_named(
+ "b", "AMD Radeon(TM) Graphics", position = 0, listing = listing
+ )
+ == "Vulkan0"
+ )
+ assert sd_cpp_backend._normalized_card_name(
+ "Intel(R) Arc(TM) B580"
+ ) == sd_cpp_backend._normalized_card_name("Intel Arc B580")
+ # Only the marks go: a model letter R or TM inside a word stays.
+ assert sd_cpp_backend._normalized_card_name("Radeon R9") == "radeonr9"