Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: FLUX.3 upsampler pin follows torch's vendor, single-GPU hosts launch as before, non-blocking Vulkan probe (#14491)
Daniel Han3f74ad129f89
diff --git a/studio/backend/core/inference/flux3_prompt_upsampler.py b/studio/backend/core/inference/flux3_prompt_upsampler.pyindex 443ce4c5d72..2fd235042a8 100644--- a/studio/backend/core/inference/flux3_prompt_upsampler.py+++ b/studio/backend/core/inference/flux3_prompt_upsampler.py@@ -938,13 +938,113 @@ def _user_device_pick(binary: Optional[str] = None) -> bool:return True-def _prefetch_vulkan_rows(binary: str) -> None:+def _vulkan_pin_possible(binary: str, torch) -> bool:+ """Whether a Vulkan pin is worth the probe: a torch card, a Vulkan build, nothing the user picked."""+ from core.inference.llama_cpp import LlamaCppBackend+ return (+ torch.cuda.device_count() >= 1+ and not _user_device_pick()+ and not str(os.environ.get("GGML_BACKEND_PATH", "")).strip()+ and bool(LlamaCppBackend._is_vulkan_backend(binary))+ and not (+ (os.environ.get("HIP_VISIBLE_DEVICES") or "").strip()+ and not LlamaCppBackend._torch_is_rocm(torch)+ )+ )+++def _prefetch_vulkan_rows(binary: str, cancel: Optional[threading.Event] = None) -> None:+ """Probe (and read the card lookup's inventory) before the render slot, waited for cancellably: the probe can take+ its 15 s timeout, which a blocking unload behind this warm start must not sit out."""try:- from core.inference.llama_cpp import LlamaCppBackend- if not _user_device_pick() and LlamaCppBackend._is_vulkan_backend(binary):- _vulkan_rows(binary)+ import torch+ if not _vulkan_pin_possible(binary, torch):+ returnexcept Exception: # noqa: BLE001- pass+ return++ def _work() -> None:+ try:+ # An empty answer (cached or fresh) is retried once, as the launch it stands in for did.+ rows = _vulkan_rows(binary, retry_empty = True) or _vulkan_rows(binary, retry_empty = True)+ # One row on one torch card launches unpinned, so it needs no card lookup.+ if rows and (len(rows) > 1 or torch.cuda.device_count() > 1):+ ordinal = _resident_cuda_ordinal()+ if ordinal is not None and ordinal >= 0:+ _vulkan_card_name(ordinal, torch)+ except Exception: # noqa: BLE001+ pass++ # A maintenance pass that cancelled this start before the probe began must not find one running.+ if cancel is not None and cancel.is_set():+ raise UpsamplerCancelledError("Cancelled.")+ worker = threading.Thread(target = _work, name = "flux3-pup-vulkan-probe", daemon = True)+ # Registered once started: joining a thread that has not started raises.+ worker.start()+ _vulkan_probe_workers[:] = [w for w in _vulkan_probe_workers if w.is_alive()] + [worker]+ while worker.is_alive():+ worker.join(0.1)+ if cancel is not None and cancel.is_set():+ raise UpsamplerCancelledError("Cancelled.")+++_vulkan_probe_workers: list = []+++def _wait_for_vulkan_probe(timeout: float = 20.0) -> None:+ """Join prefetches cancelled warm starts abandoned: their probes can still hold the llama.cpp tree's libraries."""+ deadline = time.monotonic() + timeout+ for worker in list(_vulkan_probe_workers):+ worker.join(max(0.0, deadline - time.monotonic()))+++def _vulkan_rows_came_back_empty(binary: str) -> bool:+ """A pinnable multi-GPU host whose cached Vulkan probe listed nothing: a warm start would stay unpinned for good.+ One torch card warm-starts unpinned as before."""+ try:+ import torch+ return (+ torch.cuda.device_count() > 1+ and _vulkan_pin_possible(binary, torch)+ and not _vulkan_rows(binary)+ )+ except Exception: # noqa: BLE001+ return False+++def _vulkan_card_name(ordinal: int, torch) -> tuple:+ """``physical_card_name`` on ROCm. A CUDA torch reads no HIP/ROCR mask and its cards are not amd-smi's, so that+ lookup would place it by an AMD card's position: name it in torch order, unplaced under a CUDA mask (Vulkan still+ lists the cards it hides)."""+ from core.inference.llama_cpp import LlamaCppBackend+ from core.inference.sd_cpp_backend import physical_card_name++ if LlamaCppBackend._torch_is_rocm(torch):+ return physical_card_name(ordinal)+ if not torch.cuda.is_available() or not 0 <= ordinal < torch.cuda.device_count():+ return None, None+ names = [+ (torch.cuda.get_device_name(i) or "").strip() for i in range(torch.cuda.device_count())+ ]+ name = names[ordinal]+ if not name:+ return None, None+ if (os.environ.get("CUDA_VISIBLE_DEVICES") or "").strip():+ return name, None+ # Between identical cards only PCI order matches Vulkan's; FASTEST_FIRST (CUDA's default) may not.+ if names.count(name) > 1 and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID":+ return name, None+ return name, names[:ordinal].count(name)+++def _vulkan_row_is_another_vendor(row_name: str, torch) -> bool:+ """A CUDA torch's card is NVIDIA's and a ROCm torch's AMD's: a Vulkan row naming neither is another card."""+ from core.inference.llama_cpp import LlamaCppBackend++ name = str(row_name or "").lower()+ if LlamaCppBackend._torch_is_rocm(torch):+ return not any(tag in name for tag in ("amd", "radeon", "instinct"))+ return "nvidia" not in namedef _vulkan_device_args(@@ -952,55 +1052,54 @@ def _vulkan_device_args(ordinal: Optional[int],retry_empty: bool = False,) -> Optional[list]:- """``--device Vulkan<i>`` for the image model's card (Vulkan ignores visibility masks); [] unpinned, None unknown."""+ """``--device Vulkan<i>`` for the image model's card (Vulkan ignores visibility masks); [] unpinned, None unknown.+ One torch card and one Vulkan device launch as before Vulkan pinning; non-Vulkan builds never probe."""if ordinal is None or ordinal < 0:return []try:- from core.inference.llama_cpp import LlamaCppBackend-import torch- # The user's own device pick wins (a Vulkan mask renumbers what the probe lists).- if _user_device_pick():- return []- # Backends loaded from GGML_BACKEND_PATH are not the ones beside the binary the probe would list.- if str(os.environ.get("GGML_BACKEND_PATH", "")).strip():- return None if torch.cuda.device_count() > 1 else []- if not LlamaCppBackend._is_vulkan_backend(binary):+ count = torch.cuda.device_count()+ if count < 1 or _user_device_pick():return []- # The card lookup reads HIP masks, which only a ROCm torch applies: elsewhere one could name the wrong card.- if (- os.environ.get("HIP_VISIBLE_DEVICES") or ""- ).strip() and not LlamaCppBackend._torch_is_rocm(torch):- return None if torch.cuda.device_count() > 1 else []- # Vulkan sees every physical GPU (an iGPU, cards a mask hides), so torch's count says nothing here.+ if not _vulkan_pin_possible(binary, torch):+ # Backends from GGML_BACKEND_PATH are not the ones the probe lists; a HIP mask on a CUDA torch steers nothing.+ from core.inference.llama_cpp import LlamaCppBackend+ if count == 1 or not (+ str(os.environ.get("GGML_BACKEND_PATH", "")).strip()+ or LlamaCppBackend._is_vulkan_backend(binary)+ ):+ return []+ return None+ # Vulkan also lists an iGPU and cards a mask hides (ggml's default takes discrete AND integrated GPUs).rows = _vulkan_rows(binary, retry_empty = retry_empty)if not rows:- return None if torch.cuda.device_count() > 1 else []+ return None if count > 1 else []+ if len(rows) == 1 and count == 1:+ # Nothing to choose, so no name match (a renamed card never declines a single-GPU host),+ # unless that one device is plainly another vendor's: then it is not the image model's card.+ return None if _vulkan_row_is_another_vendor(rows[0]["name"], torch) else []except Exception: # noqa: BLE001return []try:from core.inference.sd_cpp_backend import (_normalized_card_name,_without_driver_tag,- physical_card_name,sd_cpp_device_named,)# By name, as the image engine pins its Vulkan build: no torch ordinal maps onto Vulkan's index.listing = "".join(f"Vulkan{row['index']}\t{row['name']}\n" for row in rows)- name, position = physical_card_name(ordinal)+ name, position = _vulkan_card_name(ordinal, torch)device = sd_cpp_device_named(binary, name, position = position, listing = listing)if not device:return None# Exact model match: containment takes an `RX 7600 XT` for an `RX 7600`.- import torch-matched = next((row["name"] for row in rows if f"Vulkan{row['index']}" == device), "")exact = bool(name) and _normalized_card_name(_without_driver_tag(matched)) == _normalized_card_name(name)- if not exact and (len(rows) > 1 or torch.cuda.device_count() > 1):+ if not exact:return Nonereturn [] if len(rows) == 1 else ["--device", device]except Exception: # noqa: BLE001@@ -1194,10 +1293,14 @@ class PromptUpsamplerSidecar:self._update_in_progress = Truetry:self.cancel_pending_load()- with self._request_lock, self._lock:- was_active = self.loaded- self._release_locked()- yield was_active+ _wait_for_vulkan_probe()+ with self._request_lock:+ # Again once no warm start can still be about to start one.+ _wait_for_vulkan_probe()+ with self._lock:+ was_active = self.loaded+ self._release_locked()+ yield was_activefinally:self._update_in_progress = False@@ -1258,7 +1361,8 @@ class PromptUpsamplerSidecar:if layers == 0:cmd.append("--no-mmproj-offload")if layers:- cmd += _vulkan_device_args(binary, ordinal, retry_empty = True) or []+ # A warm start probed just before its slot: re-probing an empty answer here would run it twice.+ cmd += _vulkan_device_args(binary, ordinal, retry_empty = not self._warming) or []sock.close()if is_process_shutting_down():raise UpsamplerUnavailableError("Unsloth is shutting down.")@@ -1504,7 +1608,7 @@ class PromptUpsamplerSidecar:if _user_device_pick(binary) and _cuda_device_count() > 1:return "placement"# The Vulkan probe can take its full timeout, so it runs before the slot that renders queue behind.- _prefetch_vulkan_rows(binary)+ _prefetch_vulkan_rows(binary, self._load_cancel)# The engine's generation slot, held until the server is up: a render or model transition that arrives# meanwhile queues behind the start instead of overlapping it (a load cancels it first through# stop_for_handover). Not taken while one runs or a load is in flight.@@ -1512,6 +1616,9 @@ class PromptUpsamplerSidecar:if slot is None:return whyslot_held = slot+ # An empty (maybe transient) probe would leave the server unpinned for good; a real launch re-probes.+ if _vulkan_rows_came_back_empty(binary):+ return "placement"ordinal = _resident_cuda_ordinal()# Unpinnable on a multi-GPU host (the native engine records its card elsewhere): the server would split# over every visible card and the probe below would read the wrong one. Its first enhanced run starts it.diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex 05a8142c3cc..e4d04210ffa 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -748,8 +748,13 @@ def sd_cpp_vulkan_device_is_uma(_GGML_DEVICE_PREFIXES: tuple[str, ...] = ("CUDA", "ROCM", "VULKAN", "SYCL", "METAL", "OPENCL")+_TRADEMARK_RE = re.compile(r"\((?:tm|r)\)|[\u2122\u00ae]", re.IGNORECASE)++def _normalized_card_name(text: str) -> str:- """Letters and digits only, lowercased: `AMD Radeon RX 7900 XTX (RADV NAVI31)` then contains `AMD Radeon RX 7900 XTX`."""+ """Letters and digits only, lowercased, trademark marks dropped (HIP's `AMD Radeon(TM) Graphics` is RADV's+ `AMD Radeon Graphics`): `AMD Radeon RX 7900 XTX (RADV NAVI31)` then contains `AMD Radeon RX 7900 XTX`."""+ text = _TRADEMARK_RE.sub(" ", text)return "".join(character for character in text.lower() if character.isalnum())@@ -803,14 +808,14 @@ def _physical_index_of(ordinal: int, env: Optional[dict] = None) -> "tuple[Optioif opaque is not None:return None, Truevisible: "Optional[list[int]]" = None- for raw in (rocr, hip):+ for is_hip, raw in ((False, rocr), (True, hip)):if raw is None:continueentries = _mask_entries(raw)if entries is None:return None, True# clr keeps the first of a repeated HIP entry; ROCr stops at one (utils/hardware/amd.py).- if raw is hip:+ if is_hip:entries = list(dict.fromkeys(entries))elif len(set(entries)) < len(entries):entries = entries[: next(i for i, e in enumerate(entries) if e in entries[:i])]diff --git a/studio/backend/tests/test_flux3_prompt_upsampler.py b/studio/backend/tests/test_flux3_prompt_upsampler.pyindex 7ebb1d6eb11..cd3e45ad77e 100644--- a/studio/backend/tests/test_flux3_prompt_upsampler.py+++ b/studio/backend/tests/test_flux3_prompt_upsampler.py@@ -1088,11 +1088,11 @@ def test_vulkan_sole_device_must_be_the_image_models_card(spawn_argv, warmable,state, run = spawn_argvsidecar, launched, _memory, model, _mmproj = warmable- # torch holds the image model on the 7900 XTX, but the only Vulkan device is the iGPU.+ # torch holds the image model on the 7900 XTX, but Vulkan lists only the iGPU and an RX 7600.monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)state["names"] = ["AMD Radeon RX 7900 XTX"]state["ordinal"] = 0- state["rows"] = [_VULKAN_ROWS[0]]+ state["rows"] = _VULKAN_ROWS[:2]assert "--device" not in run()[0]assert sidecar.warm(binary = "b", model = model, mmproj = None) == "placement"@@ -2124,7 +2124,10 @@ def test_the_vulkan_probe_runs_before_the_render_slot(warmable, monkeypatch):# A slow probe under the slot would hold renders and handovers until its timeout.from core.inference import llama_cpp+ import torch+sidecar, _launched, _memory, model, _mmproj = warmable+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)order = []take = pup._take_render_slotmonkeypatch.setattr(pup, "_take_render_slot", lambda: order.append("slot") or take())@@ -2135,7 +2138,8 @@ def test_the_vulkan_probe_runs_before_the_render_slot(warmable, monkeypatch):pup, "_vulkan_rows", lambda b, retry_empty = False: order.append("probe") or [])sidecar.warm(binary = "b", model = model, mmproj = None)- assert order[:2] == ["probe", "slot"]+ # An empty answer is retried once, still before the slot.+ assert order[: order.index("slot")] == ["probe", "probe"]sidecar._cancel_idle_locked()@@ -2240,3 +2244,421 @@ def test_a_user_main_gpu_is_a_device_pick(monkeypatch):monkeypatch.setenv("LLAMA_ARG_MAIN_GPU", "1")assert pup._user_device_pick() is Trueassert pup._vulkan_device_args("b", 0) == []+++from core.inference import sd_cpp_backend as _sd_cpp_backend # noqa: E402++_REAL_PHYSICAL_POSITION_OF = _sd_cpp_backend._physical_position_of+++def _real_card_lookup(monkeypatch, amd_rows, hip_by_row):+ """Undo the fixture's position stub: the real lookup over a mocked amd-smi inventory and HIP id map."""+ from core.inference import sd_cpp_backend+ from utils.hardware import amd++ monkeypatch.setattr(sd_cpp_backend, "_physical_position_of", _REAL_PHYSICAL_POSITION_OF)+ inventory = {+ "devices": [{"index": i, "vendor": "amd", "name": name} for i, name in enumerate(amd_rows)]+ }+ monkeypatch.setattr(sd_cpp_backend, "_card_lookup_inventory", lambda: inventory)+ monkeypatch.setattr(amd, "get_hip_id_by_gpu_index", lambda: dict(hip_by_row))+++_MIXED_ROWS = [+ {"index": 0, "name": "AMD Radeon Graphics (RADV RAPHAEL)"},+ {"index": 1, "name": "AMD Radeon RX 7600 (RADV NAVI33)"},+ {"index": 2, "name": "NVIDIA GeForce RTX 4090"},+ {"index": 3, "name": "NVIDIA GeForce RTX 4090"},+]++[email protected]("ordinal, device", [(0, "Vulkan2"), (1, "Vulkan3")])+def test_a_cuda_torch_never_takes_an_amd_cards_position(spawn_argv, monkeypatch, ordinal, device):+ # Two RTX 4090s on a CUDA torch beside AMD cards amd-smi reads: HIP device 1 (the RX 7600) is no torch cuda:1.+ import torch++ state, run = spawn_argv+ _real_card_lookup(monkeypatch, ["AMD Radeon Graphics", "AMD Radeon RX 7600"], {0: 0, 1: 1})+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ state["names"] = ["NVIDIA GeForce RTX 4090", "NVIDIA GeForce RTX 4090"]+ state["rows"] = _MIXED_ROWS+ state["ordinal"] = ordinal+ monkeypatch.setenv("CUDA_DEVICE_ORDER", "PCI_BUS_ID")+ argv, env = run()+ assert argv[-2:] == ["--device", device] and env == {"CUDA_VISIBLE_DEVICES": str(ordinal)}++[email protected]("order", [None, "FASTEST_FIRST"])+def test_identical_cuda_cards_are_not_told_apart_outside_pci_order(+ spawn_argv, warmable, monkeypatch, order+):+ # CUDA's default FASTEST_FIRST may not match Vulkan's (PCI) order, so torch's position among twins is no pin.+ import torch++ state, run = spawn_argv+ sidecar, launched, _memory, model, _mmproj = warmable+ _real_card_lookup(monkeypatch, ["AMD Radeon Graphics", "AMD Radeon RX 7600"], {0: 0, 1: 1})+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)+ state["names"] = ["NVIDIA GeForce RTX 4090", "NVIDIA GeForce RTX 4090"]+ state["rows"] = _MIXED_ROWS+ state["ordinal"] = 1+ if order is None:+ monkeypatch.delenv("CUDA_DEVICE_ORDER", raising = False)+ else:+ monkeypatch.setenv("CUDA_DEVICE_ORDER", order)+ assert pup._vulkan_device_args("b", 1) is None+ assert "--device" not in run()[0]+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "placement"+ # A card with no twin needs no order to be found.+ state["names"] = ["NVIDIA GeForce RTX 4090", "AMD Radeon RX 7600"]+ assert pup._vulkan_device_args("b", 0) == ["--device", "Vulkan2"]+++ "var, value", [("CUDA_VISIBLE_DEVICES", "1,0"), ("ROCR_VISIBLE_DEVICES", "1,0")]+)+def test_a_cuda_torch_mask_never_steers_the_vulkan_pin_onto_a_guess(+ spawn_argv, warmable, monkeypatch, var, value+):+ # CVD reorders the identical pair (the order Vulkan cannot see); ROCR is not read by a CUDA torch at all.+ import torch++ state, run = spawn_argv+ sidecar, launched, _memory, model, _mmproj = warmable+ _real_card_lookup(monkeypatch, ["AMD Radeon Graphics", "AMD Radeon RX 7600"], {0: 0, 1: 1})+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)+ state["names"] = ["NVIDIA GeForce RTX 4090", "NVIDIA GeForce RTX 4090"]+ state["rows"] = _MIXED_ROWS+ state["ordinal"] = 1+ monkeypatch.setenv(var, value)+ expected = None if var == "CUDA_VISIBLE_DEVICES" else ["--device", "Vulkan3"]+ assert pup._vulkan_device_args("b", 1) == expected+ if expected is None:+ assert "--device" not in run()[0]+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "placement"++[email protected]("vulkan", [False, True])+def test_one_torch_card_and_one_vulkan_device_launch_as_before(+ spawn_argv, warmable, monkeypatch, vulkan+):+ # The pre-pinning launch: no --device, no mask, a start. A non-Vulkan build never probes; one Vulkan row (even a+ # renamed `Radeon(TM)` card) is nothing to choose between, so no name match can decline it.+ import torch++ from core.inference import llama_cpp++ state, run = spawn_argv+ sidecar, launched, _memory, model, _mmproj = warmable+ calls = []+ monkeypatch.setattr(+ llama_cpp.LlamaCppBackend,+ "_run_vulkan_probe",+ staticmethod(+ lambda b = None: calls.append(b)+ or [{"index": 0, "name": "AMD Radeon Graphics (RADV RAPHAEL)"}]+ ),+ )+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)+ state["vulkan"] = vulkan+ state["names"] = ["AMD Radeon(TM) Graphics"]+ state["ordinal"] = 0+ monkeypatch.setattr(pup, "_resident_cuda_ordinal", lambda: 0)+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "started"+ sidecar._cancel_idle_locked()+ argv, env = run()+ assert "--device" not in argv and env == {}+ assert bool(calls) is vulkan++[email protected]("cards", [1, 2])+def test_a_warm_start_retries_an_empty_vulkan_answer_once(spawn_argv, warmable, monkeypatch, cards):+ # The warm start retries an empty answer once, as the launch it stands in for did; its real spawn adds none.+ import torch++ from core.inference import llama_cpp++ state, _run = spawn_argv+ sidecar, launched, _memory, model, _mmproj = warmable+ calls = []+ monkeypatch.setattr(torch.cuda, "device_count", lambda: cards)+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: cards)+ monkeypatch.setattr(pup, "_resident_cuda_ordinal", lambda: 0)+ monkeypatch.setattr(pup, "_VULKAN_ROWS_TTL_S", 120.0)+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})+ monkeypatch.setattr(+ llama_cpp.LlamaCppBackend,+ "_run_vulkan_probe",+ staticmethod(lambda b = None: calls.append(b) or []),+ )+ monkeypatch.setattr(+ sidecar, "_spawn_server", pup.PromptUpsamplerSidecar._spawn_server.__get__(sidecar)+ )+ # Several cards: declined, not started unpinned; the first enhanced run re-probes (and pins, or launches as+ # before). One card goes on to the spawn unpinned, as before pinning.+ out = sidecar.warm(binary = "/x/llama-server", model = model, mmproj = None)+ assert (out == "placement") is (cards > 1)+ assert len(calls) == 2+++def test_maintenance_waits_for_every_abandoned_vulkan_probe(monkeypatch):+ # Two warm starts cancelled within one probe's timeout leave two probes holding the llama.cpp libraries.+ import torch++ from core.inference import llama_cpp++ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ monkeypatch.setattr(+ llama_cpp.LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: True)+ )+ for var in (+ "HIP_VISIBLE_DEVICES",+ "GGML_BACKEND_PATH",+ "LLAMA_ARG_DEVICE",+ "LLAMA_ARG_MAIN_GPU",+ ):+ monkeypatch.delenv(var, raising = False)+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})+ monkeypatch.setattr(pup, "_vulkan_probe_workers", [], raising = False)+ releases = [threading.Event(), threading.Event()]+ entered = []++ def _probe(b = None):+ release = releases[len(entered)]+ entered.append(b)+ release.wait(10)+ return []++ monkeypatch.setattr(llama_cpp.LlamaCppBackend, "_run_vulkan_probe", staticmethod(_probe))+ try:+ for _ in releases:+ cancel = threading.Event()+ threading.Timer(0.2, cancel.set).start()+ with pytest.raises(pup.UpsamplerCancelledError):+ pup._prefetch_vulkan_rows("b", cancel)+ releases[1].set()+ waiter = threading.Thread(target = pup._wait_for_vulkan_probe, args = (10,))+ waiter.start()+ waiter.join(0.5)+ assert waiter.is_alive()+ releases[0].set()+ waiter.join(5)+ assert not waiter.is_alive()+ finally:+ for release in releases:+ release.set()+++ "name, row",+ [+ ("AMD Radeon(TM) Graphics", "AMD Radeon Graphics (RADV RAPHAEL)"),+ ("AMD Radeon(TM) 8060S Graphics", "AMD Radeon 8060S Graphics (RADV GFX1151)"),+ ],+)+def test_trademark_marks_do_not_hide_the_card(spawn_argv, warmable, monkeypatch, name, row):+ import torch++ state, run = spawn_argv+ sidecar, launched, _memory, model, _mmproj = warmable+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)+ state["names"] = [name, "AMD Radeon RX 7900 XTX"]+ state["rows"] = [{"index": 0, "name": row}, _VULKAN_ROWS[2]]+ state["ordinal"] = 0+ monkeypatch.setattr(pup, "_resident_cuda_ordinal", lambda: 0)+ assert run()[0][-2:] == ["--device", "Vulkan0"]+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "started"+ sidecar._cancel_idle_locked()+++def test_a_cancelled_warm_start_does_not_wait_out_the_vulkan_probe(warmable, monkeypatch):+ # A blocking unload (handover, training start) must not sit behind a probe that can take its 15 s timeout.+ from core.inference import llama_cpp++ sidecar, launched, _memory, model, _mmproj = warmable+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)+ import torch++ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(+ llama_cpp.LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: True)+ )+ for var in (+ "HIP_VISIBLE_DEVICES",+ "GGML_BACKEND_PATH",+ "LLAMA_ARG_DEVICE",+ "LLAMA_ARG_MAIN_GPU",+ ):+ monkeypatch.delenv(var, raising = False)+ entered, release = threading.Event(), threading.Event()++ def _probe(b = None):+ entered.set()+ release.wait(10)+ return list(_VULKAN_ROWS)++ monkeypatch.setattr(llama_cpp.LlamaCppBackend, "_run_vulkan_probe", staticmethod(_probe))+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})+ out = []+ warm = threading.Thread(+ target = lambda: out.append(sidecar.warm(binary = "b", model = model, mmproj = None))+ )+ warm.start()+ try:+ assert entered.wait(5)+ assert sidecar.cancel_warm_start()+ unload = threading.Thread(target = sidecar.unload)+ unload.start()+ unload.join(3)+ assert not unload.is_alive()+ warm.join(3)+ assert out == ["cancelled"] and launched == []+ finally:+ release.set()+ warm.join(5)+++def test_maintenance_waits_for_a_probe_a_cancelled_warm_start_left_running(warmable, monkeypatch):+ # The probe loads the llama.cpp tree's libraries; replacing that tree under it fails on Windows.+ import torch++ from core.inference import llama_cpp++ sidecar, launched, _memory, model, _mmproj = warmable+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(+ llama_cpp.LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: True)+ )+ for var in (+ "HIP_VISIBLE_DEVICES",+ "GGML_BACKEND_PATH",+ "LLAMA_ARG_DEVICE",+ "LLAMA_ARG_MAIN_GPU",+ ):+ monkeypatch.delenv(var, raising = False)+ entered, release = threading.Event(), threading.Event()++ def _probe(b = None):+ entered.set()+ release.wait(10)+ return list(_VULKAN_ROWS)++ monkeypatch.setattr(llama_cpp.LlamaCppBackend, "_run_vulkan_probe", staticmethod(_probe))+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})+ out, phase = [], []+ warm = threading.Thread(+ target = lambda: out.append(sidecar.warm(binary = "b", model = model, mmproj = None))+ )+ warm.start()++ def _maintain():+ with sidecar.update_maintenance():+ phase.append("replacing")++ try:+ assert entered.wait(5)+ maint = threading.Thread(target = _maintain)+ maint.start()+ warm.join(3)+ assert out == ["cancelled"]+ maint.join(0.5)+ assert phase == [] and maint.is_alive()+ release.set()+ maint.join(5)+ assert phase == ["replacing"] and launched == []+ finally:+ release.set()+ warm.join(5)+++def _prefetch_host(monkeypatch, cards, rows):+ import torch++ from core.inference import llama_cpp++ calls, looked_up = [], []+ monkeypatch.setattr(torch.cuda, "device_count", lambda: cards)+ monkeypatch.setattr(+ llama_cpp.LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda b = None: True)+ )+ for var in (+ "HIP_VISIBLE_DEVICES",+ "GGML_BACKEND_PATH",+ "LLAMA_ARG_DEVICE",+ "LLAMA_ARG_MAIN_GPU",+ ):+ monkeypatch.delenv(var, raising = False)+ monkeypatch.setattr(+ llama_cpp.LlamaCppBackend,+ "_run_vulkan_probe",+ staticmethod(lambda b = None: calls.append(b) or list(rows)),+ )+ monkeypatch.setattr(pup, "_VULKAN_ROWS_TTL_S", 120.0)+ monkeypatch.setattr(pup, "_resident_cuda_ordinal", lambda: 0)+ monkeypatch.setattr(pup, "_vulkan_card_name", lambda o, t: looked_up.append(o) or ("x", None))+ monkeypatch.setattr(pup, "_vulkan_probe_workers", [])+ return calls, looked_up+++def test_a_warm_start_prefetch_retries_a_cached_empty_probe(monkeypatch):+ # The launch a warm start stands in for re-probed a (maybe transient) empty answer; the prefetch does too.+ calls, looked_up = _prefetch_host(monkeypatch, 1, _VULKAN_ROWS)+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {("b", None): ([], time.monotonic())})+ pup._prefetch_vulkan_rows("b")+ assert calls == ["b"] and looked_up == [0]+ # A cached non-empty answer is reused.+ pup._prefetch_vulkan_rows("b")+ assert calls == ["b"]+++def test_one_card_and_one_vulkan_row_skips_the_card_lookup(monkeypatch):+ # That launch stays unpinned, so the prefetch reads no inventory for it.+ calls, looked_up = _prefetch_host(monkeypatch, 1, _VULKAN_ROWS[:1])+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})+ pup._prefetch_vulkan_rows("b")+ assert calls == ["b"] and looked_up == []+++def test_a_start_cancelled_before_its_probe_never_starts_one(monkeypatch):+ calls, _looked_up = _prefetch_host(monkeypatch, 2, _VULKAN_ROWS)+ monkeypatch.setattr(pup, "_vulkan_rows_memo", {})+ cancel = threading.Event()+ cancel.set()+ with pytest.raises(pup.UpsamplerCancelledError):+ pup._prefetch_vulkan_rows("b", cancel)+ assert calls == [] and pup._vulkan_probe_workers == []+++def test_a_lone_vulkan_device_of_another_vendor_declines_the_warm_start(+ spawn_argv, warmable, monkeypatch+):+ # A CUDA torch's image model on an RTX 4090 while Vulkan reaches only an AMD iGPU: not the same card.+ import torch++ state, run = spawn_argv+ sidecar, launched, _memory, model, _mmproj = warmable+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ state["names"] = ["NVIDIA GeForce RTX 4090"]+ state["ordinal"] = 0+ state["rows"] = [{"index": 0, "name": "AMD Radeon Graphics (RADV RAPHAEL)"}]+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "placement"+ # The explicit launch is unchanged: unpinned, as before Vulkan pinning.+ argv, env = run()+ assert "--device" not in argv+ # Its own NVIDIA row warm-starts as before.+ state["rows"] = [{"index": 0, "name": "NVIDIA GeForce RTX 4090"}]+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "started"+ sidecar._cancel_idle_locked()+++def test_maintenance_never_joins_an_unstarted_probe(monkeypatch):+ # A worker is registered only once started, so a maintenance pass cannot hit "cannot join thread".+ import inspect+ source = inspect.getsource(pup._prefetch_vulkan_rows)+ assert source.index("worker.start()") < source.index("_vulkan_probe_workers[:]")diff --git a/studio/backend/tests/test_sd_cpp_rocm_vulkan_fallback.py b/studio/backend/tests/test_sd_cpp_rocm_vulkan_fallback.pyindex 6b91d1fc91c..4e7a60da677 100644--- a/studio/backend/tests/test_sd_cpp_rocm_vulkan_fallback.py+++ b/studio/backend/tests/test_sd_cpp_rocm_vulkan_fallback.py@@ -3106,3 +3106,36 @@ def test_a_damaged_cli_beside_a_healthy_server_is_not_a_strike(fake_settings, mochosen = router.select_and_activate_engine(_detect_load_family(H3_REPO, None, "minimax-h3"))assert chosen == "sd_cpp", chosen # the healthy server carries the loadassert _recorded_strikes(fake_settings) == 0 # and nothing was held against ROCm+++def test_one_mask_string_for_rocr_and_hip_keeps_rocrs_repeat_rule(monkeypatch):+ # The same str object in both variables: ROCr still stops at the repeat, HIP still dedupes.+ from core.inference import sd_cpp_backend++ # ROCr is a Linux layer: Windows HIP ignores ROCR_VISIBLE_DEVICES.+ monkeypatch.setattr(sd_cpp_backend.sys, "platform", "linux")++ mask = "0,0,1"+ env = {"ROCR_VISIBLE_DEVICES": mask, "HIP_VISIBLE_DEVICES": mask}+ assert env["ROCR_VISIBLE_DEVICES"] is env["HIP_VISIBLE_DEVICES"]+ assert sd_cpp_backend._physical_index_of(1, env = env) == (None, True)+ assert sd_cpp_backend._physical_index_of(+ 1, env = {"ROCR_VISIBLE_DEVICES": mask, "HIP_VISIBLE_DEVICES": "0,0,1 "}+ ) == (None, True)+++def test_trademark_marks_do_not_split_a_card_name():+ from core.inference import sd_cpp_backend++ listing = "Vulkan0\tAMD Radeon Graphics (RADV RAPHAEL)\n"+ assert (+ sd_cpp_backend.sd_cpp_device_named(+ "b", "AMD Radeon(TM) Graphics", position = 0, listing = listing+ )+ == "Vulkan0"+ )+ assert sd_cpp_backend._normalized_card_name(+ "Intel(R) Arc(TM) B580"+ ) == sd_cpp_backend._normalized_card_name("Intel Arc B580")+ # Only the marks go: a model letter R or TM inside a word stays.+ assert sd_cpp_backend._normalized_card_name("Radeon R9") == "radeonr9"