Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio sd.cpp: a lost Vulkan device is not a strike; Arc / ROCm keep Vulkan; a UHD iGPU beside a card no longer vetoes Vulkan (#14610)
Daniel Hanf4a21e236fa4
diff --git a/studio/backend/core/inference/diffusion_engine_router.py b/studio/backend/core/inference/diffusion_engine_router.pyindex 53092e865a7..f33a6b383ff 100644--- a/studio/backend/core/inference/diffusion_engine_router.py+++ b/studio/backend/core/inference/diffusion_engine_router.py@@ -240,6 +240,31 @@ def _accessible_render_vendors(vendors) -> set:return found+def _reachable_intel_arc_render_node() -> bool:+ """True when this process can open the render node of an Intel Arc itself (PCI class per node), not+ only some Intel node: beside a UHD iGPU, a container may mount the iGPU's node alone."""+ import glob++ from utils.hardware.hardware import _intel_pci_device_is_xpu_class++ for node in glob.glob("/dev/dri/renderD*"):+ if not os.access(node, os.R_OK | os.W_OK):+ continue+ device = f"/sys/class/drm/{os.path.basename(node)}/device"+ try:+ with open(f"{device}/vendor", encoding = "utf-8") as f:+ if _RENDER_NODE_VENDORS.get(f.read().strip().lower()) != "intel":+ continue+ except OSError:+ continue+ try:+ if _intel_pci_device_is_xpu_class(device) is True:+ return True+ except Exception: # noqa: BLE001 -- unknown class: not proof the Arc is reachable+ continue+ return False++def _vulkan_gpu_for_cpu_torch() -> bool:"""True when torch is CPU-only but the host has a GPU the Vulkan sd.cpp build can render on:an AMD card (Vega, RDNA1 and Polaris get no ROCm wheel) or an Intel Arc. Never beside NVIDIA,@@ -251,7 +276,7 @@ def _vulkan_gpu_for_cpu_torch() -> bool:# ggml reads this one itself; anything but a list of ordinals selects nothing or throws. Its# ordinals index the raw Vulkan list, so it never vouches for which card a mixed host gets.vk_mask = os.environ.get("GGML_VK_VISIBLE_DEVICES")- if vk_mask is not None and not re.fullmatch(r"\s*\d+\s*(,\s*\d+\s*)*", vk_mask):+ if vk_mask is not None and not re.fullmatch(r"\s*\d+(?:[\s,]+\d+)*[\s,]*", vk_mask):return Falseinventory = _physical_inventory()if inventory.get("unknown") or "nvidia" in (inventory.get("unanswered") or ()):@@ -278,12 +303,23 @@ def _vulkan_gpu_for_cpu_torch() -> bool:reachable = _accessible_render_vendors({d.get("vendor") for d in gpus})gpus = [d for d in gpus if d.get("vendor") in reachable]masked = _masked_vendors()- # Unpinned sd.cpp takes the first Vulkan device, which may be a UHD iGPU beside the card.if not gpus or any(d.get("vendor") in masked for d in gpus):return False- if not all(_renders(d) for d in gpus):- return False- return _hardware_vulkan_driver({d.get("vendor") for d in gpus}) and _vulkan_loader_loads()+ rendering = [d for d in gpus if _renders(d)]+ if len(rendering) < len(gpus):+ # Unpinned sd.cpp takes a discrete GPU before an iGPU (get_default_backend_name); a mask's ordinals may name the iGPU.+ if vk_mask is not None or not rendering:+ return False+ if any(d.get("vendor") != "intel" for d in gpus if not _renders(d)):+ return False+ # The vendor-level node check cannot tell an Arc's node from the iGPU's: an unopenable Arc vouches for nothing.+ if sys.platform != "win32" and not _reachable_intel_arc_render_node():+ rendering = [d for d in rendering if d.get("vendor") != "intel"]+ if not rendering:+ return False+ # The driver must belong to a card that renders: beside an AMD card, an Intel-only ICD drives just the iGPU.+ rendering_vendors = {d.get("vendor") for d in rendering}+ return _hardware_vulkan_driver(rendering_vendors) and _vulkan_loader_loads()# The ICD manifests that drive each vendor's cards (RADV as radeon_icd or radv_icd, AMDVLK /diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex bb9e4e661f2..ddc661581af 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -644,7 +644,7 @@ def sd_cpp_device_name_for_ordinal(binary: Optional[str], ordinal: Optional[int]choice, i.e. today's behaviour."""if not binary or ordinal is None:return None- text = _sd_cpp_probe_output(binary, "--list-devices")+ text = sd_cpp_device_listing(binary)if text is not None:for line in text.splitlines():name = line.split("\t", 1)[0].strip()@@ -673,8 +673,21 @@ _CUDA_DEVICE_CC_RE = re.compile()-# Last --list-devices answer per binary + (size, mtime_ns); only the capability read reuses it, never the verdict.+# Last --list-devices answer per binary. The accelerator verdict always re-probes (a load re-vets the listing on purpose)._LAST_DEVICE_LISTING: dict = {}+# What ggml reads to pick or hide devices.+_DEVICE_LISTING_ENV_VARS: tuple[str, ...] = (+ "GGML_VK_VISIBLE_DEVICES",+ "VK_ICD_FILENAMES",+ "VK_DRIVER_FILES",+ "VK_ADD_DRIVER_FILES",+ "VK_LOADER_DRIVERS_SELECT",+ "VK_LOADER_DRIVERS_DISABLE",+ "CUDA_VISIBLE_DEVICES",+ "HIP_VISIBLE_DEVICES",+ "ROCR_VISIBLE_DEVICES",+ "GPU_DEVICE_ORDINAL",+)def _binary_stat_identity(binary: str) -> Optional[tuple[int, int]]:@@ -685,13 +698,49 @@ def _binary_stat_identity(binary: str) -> Optional[tuple[int, int]]:return (st.st_size, st.st_mtime_ns)+def _hardware_inventory_generation() -> Optional[int]:+ try:+ from utils.hardware.hardware import physical_gpu_inventory_generation+ return physical_gpu_inventory_generation()+ except Exception: # noqa: BLE001 -- an identity read must never fail the probe+ return None+++def _device_listing_identity(binary: str) -> tuple:+ return (+ _binary_stat_identity(binary),+ tuple(os.environ.get(name) for name in _DEVICE_LISTING_ENV_VARS),+ _hardware_inventory_generation(),+ )++def _remember_device_listing(binary: Optional[str], text: Optional[str]) -> None:if not binary:returnif text is None:- # Keep the last good listing; readers check (size, mtime_ns), so a changed binary's listing is never served.+ # An unreadable probe or a timeout is never cached.return- _LAST_DEVICE_LISTING[binary] = (_binary_stat_identity(binary), text)+ _LAST_DEVICE_LISTING[binary] = (_device_listing_identity(binary), text)+++def _cached_device_listing(binary: str, *, require_file: bool = False) -> Optional[str]:+ seen = _LAST_DEVICE_LISTING.get(binary)+ if seen is None:+ return None+ identity = _device_listing_identity(binary)+ if require_file and identity[0] is None:+ return None+ return seen[1] if seen[0] == identity else None+++def sd_cpp_device_listing(binary: str) -> Optional[str]:+ """``binary --list-devices``, reusing the listing already taken of this file under this environment."""+ text = _cached_device_listing(binary, require_file = True)+ if text is None:+ text = _sd_cpp_probe_output(binary, "--list-devices")+ if _binary_stat_identity(binary) is not None:+ _remember_device_listing(binary, text)+ return textdef sd_cpp_cuda_compute_capability(@@ -705,12 +754,9 @@ def sd_cpp_cuda_compute_capability(only reads the listing the last accelerator verdict took of this file."""if not binary:return None- text = None- seen = _LAST_DEVICE_LISTING.get(binary)- if seen is not None and seen[0] == _binary_stat_identity(binary):- text = seen[1]- elif probe:- text = _sd_cpp_probe_output(binary, "--list-devices")+ text = _cached_device_listing(binary)+ if text is None and probe:+ text = sd_cpp_device_listing(binary)if text is None or not _CUDA_INIT_RE.search(text):return Nonecaps = {@@ -730,8 +776,7 @@ def sd_cpp_cuda_compute_capability(def sd_cpp_device_listing_known(binary: Optional[str]) -> bool:if not binary:return False- seen = _LAST_DEVICE_LISTING.get(binary)- return seen is not None and seen[0] == _binary_stat_identity(binary)+ return _cached_device_listing(binary) is not None# Runs when --list-devices timed out (possibly a stalled driver); CUDA calls have no deadline, so bound the wait.@@ -847,12 +892,9 @@ def sd_cpp_vulkan_device_is_uma(memory for such a device sums every heap, the GTT the pinned weights come from included. None when unsure."""if not binary:return None- text = None- seen = _LAST_DEVICE_LISTING.get(binary)- if seen is not None and seen[0] == _binary_stat_identity(binary):- text = seen[1]- elif probe:- text = _sd_cpp_probe_output(binary, "--list-devices")+ text = _cached_device_listing(binary)+ if text is None and probe:+ text = sd_cpp_device_listing(binary)if text is None:return Noneuma = {int(m.group(1)): m.group(2) == "1" for m in _VULKAN_DEVICE_UMA_RE.finditer(text)}@@ -1072,7 +1114,7 @@ def sd_cpp_device_named(wanted = _normalized_card_name(card_name)if not wanted:return None- text = listing if listing is not None else _sd_cpp_probe_output(binary, "--list-devices")+ text = listing if listing is not None else sd_cpp_device_listing(binary)if text is None:return None# Both readings: containment cannot tell `RX 7600` from `RX 7600 XT`, equality cannot see through a driver tag.@@ -1326,6 +1368,16 @@ def _note_failed_upgrade(accelerator: str) -> None:_ACCELERATOR_FALLBACK: dict[str, str] = {"rocm": "vulkan", "vulkan": "cpu"}+def _vulkan_cpu_rung_applies() -> bool:+ """Whether ambiguous failures can move the Vulkan build to CPU: only on the CPU-torch host routed to Vulkan. An XPU+ host (always Vulkan) and a ROCm host on its Vulkan rung leave the GPU only on decisive evidence. Unknown: as before."""+ try:+ from utils.hardware.hardware import DeviceType, get_device+ return get_device() == DeviceType.CPU+ except Exception: # noqa: BLE001+ return True++# The sonames the upstream ROCm archive imports and does not ship._ROCM_RUNTIME_SONAMES: tuple[str, ...] = ("libamdhip64.so.7",@@ -1436,12 +1488,9 @@ def sd_cpp_listed_device(file (no subprocess)."""if not binary:return None, None- text = None- seen = _LAST_DEVICE_LISTING.get(binary)- if seen is not None and seen[0] == _binary_stat_identity(binary):- text = seen[1]- elif probe:- text = _sd_cpp_probe_output(binary, "--list-devices")+ text = _cached_device_listing(binary)+ if text is None and probe:+ text = sd_cpp_device_listing(binary)if text is None:return None, Nonefor line in text.splitlines():@@ -1492,6 +1541,47 @@ _accelerator_runtime_failures: dict[str, dict] = {}# AMBIGUOUS failures needed to divert a host (a decisive one is enough alone)._AMBIGUOUS_FAILURE_STRIKES = 2+# Rule 2 stopped counting Vulkan device-lost errors: an older Vulkan record keeps only its decisive evidence.+_FAILURE_RULE = 2+++def _record_under_current_rule(klass: str, record: Optional[dict]) -> Optional[dict]:+ if not isinstance(record, dict) or klass != "vulkan":+ return record+ if int(record.get("rule", 1) or 1) >= _FAILURE_RULE:+ return record+ return _decisive_evidence_only(record)+++def _decisive_evidence_only(record: Optional[dict]) -> Optional[dict]:+ """*record* with its ambiguous strikes dropped, or None when nothing decisive is left."""+ if not isinstance(record, dict):+ return record+ proven_cards = {+ name: {"strikes": 0, "proven": True}+ for name, entry in (record.get("per_card") or {}).items()+ if isinstance(entry, dict) and entry.get("proven")+ }+ unscoped = _unscoped_evidence(record)+ # Proof without per-card tallies stays as written: record-wide, and scoped to its ``cards`` when it names any.+ whole = bool(record.get("proven")) and not isinstance(record.get("per_card"), dict)+ if not (proven_cards or unscoped["proven"] or whole):+ return None+ kept = {+ "strikes": 0,+ "proven": True,+ "fingerprint": record.get("fingerprint") or {},+ "rule": _FAILURE_RULE,+ }+ if record.get("cards"):+ # A card left in ``cards`` without its own entry would divert on another card's record-wide proof.+ kept["cards"] = [+ c for c in record["cards"] if whole or unscoped["proven"] or c in proven_cards+ ]+ if proven_cards:+ kept["per_card"] = proven_cards+ kept["unscoped"] = {"strikes": 0, "proven": unscoped["proven"]}+ return keptdef _accelerator_class_of(accelerator: Optional[str]) -> str:@@ -1662,10 +1752,15 @@ def _normalise_failure_record(key: str, value: object) -> Optional[dict]:except (TypeError, ValueError):strikes = 0fingerprint = value.get("fingerprint")+ try:+ rule = int(value.get("rule", 1) or 1)+ except (TypeError, ValueError):+ rule = 1record = {"strikes": max(strikes, 0),"proven": bool(value.get("proven", False)),"fingerprint": fingerprint if isinstance(fingerprint, dict) else {},+ "rule": rule,}cards = [str(card).strip() for card in (value.get("cards") or []) if str(card).strip()]if cards:@@ -1778,7 +1873,7 @@ def note_accelerator_runtime_failure(fingerprint = dict(fingerprint) if isinstance(fingerprint, dict) else _accelerator_fingerprint()records = _stored_accelerator_runtime_failures()records.update({k: v for k, v in _accelerator_runtime_failures.items() if k not in records})- previous = records.get(klass)+ previous = _record_under_current_rule(klass, records.get(klass))if previous is not None and not _fingerprint_still_applies(previous.get("fingerprint"), fingerprint):@@ -1816,6 +1911,7 @@ def note_accelerator_runtime_failure(# The union, for a caller that names no card and for the settings report."proven": bool(proven) or bool((previous or {}).get("proven", False)),"fingerprint": fingerprint,+ "rule": _FAILURE_RULE,}if cards:record["cards"] = cards@@ -1877,9 +1973,24 @@ def accelerator_runtime_failed(accelerator: Optional[str], card: Optional[str] =if not klass:return Falsefingerprint = _accelerator_fingerprint()- if _record_diverts(_accelerator_runtime_failures.get(klass), fingerprint, card):+ if _record_diverts(+ _applicable_record(klass, _accelerator_runtime_failures.get(klass)), fingerprint, card+ ):return True- return _record_diverts(_stored_accelerator_runtime_failures().get(klass), fingerprint, card)+ return _record_diverts(+ _applicable_record(klass, _stored_accelerator_runtime_failures().get(klass)),+ fingerprint,+ card,+ )+++def _applicable_record(klass: str, record: Optional[dict]) -> Optional[dict]:+ """The evidence in *record* that still counts on this host: old-rule ambiguous Vulkan strikes are+ dropped, and outside the CPU-torch route only decisive Vulkan evidence diverts."""+ record = _record_under_current_rule(klass, record)+ if klass == "vulkan" and not _vulkan_cpu_rung_applies():+ return _decisive_evidence_only(record)+ return recorddef off_torch_build_mismatch(off_torch: Any, binary: Optional[str]) -> Optional[str]:@@ -1952,7 +2063,8 @@ def accelerator_runtime_failure_state() -> dict:"fallback": _ACCELERATOR_FALLBACK.get(klass),"strikes": int(record.get("strikes", 0) or 0),"proven": bool(record.get("proven", False)),- "diverting": enabled and _record_diverts(record, fingerprint),+ "diverting": enabled+ and _record_diverts(_applicable_record(klass, record), fingerprint),"stale": not _fingerprint_still_applies(record.get("fingerprint"), fingerprint),})@@ -1991,8 +2103,7 @@ _ACCELERATOR_AMBIGUOUS_FAILURE_MARKERS: tuple[str, ...] = ("rocm error","rocblas error","memory access fault by gpu node",- "errordevicelost",- "vk_error_device_lost",+ # Not VK_ERROR_DEVICE_LOST: the Vulkan spec makes it recoverable by recreating the device (TDR, scheduler reset)."errorinitializationfailed","vk_error_initialization_failed",)@@ -2060,6 +2171,34 @@ def output_shows_capacity_failure(text: Optional[str]) -> bool:return any(marker in lowered for marker in _ACCELERATOR_CAPACITY_FAILURE_MARKERS)+# A lost Vulkan device: recoverable by creating a new device, so neither a build strike nor a fatal error.+_VULKAN_DEVICE_LOST_MARKERS: tuple[str, ...] = ("errordevicelost", "vk_error_device_lost")+++def output_shows_vulkan_device_lost(text: Optional[str]) -> bool:+ if not text:+ return False+ lowered = str(text).lower()+ return any(marker in lowered for marker in _VULKAN_DEVICE_LOST_MARKERS)+++# Appended when a lost Vulkan device repeats and no CPU fallback applies. Must match no failure marker.+_REPEATED_DEVICE_LOSS_HINT = (+ "The GPU lost its Vulkan device twice in a row (usually a driver reset during the render). Try Low VRAM "+ "memory mode, a smaller resolution or a smaller quant, or load the model again to retry on the GPU."+)+++def _device_loss_cpu_fallback_applies(state: Any) -> bool:+ """Whether a repeated Vulkan device loss may move this load to the CPU backend: only the CPU-torch Vulkan+ route (the one with a CPU rung, as on main), the Vulkan-to-CPU fallback enabled, and no user-picked GPU."""+ if getattr(state, "device", None) != "vulkan" or getattr(state, "off_torch_device", None):+ return False+ if getattr(state, "gpu_explicit", False):+ return False+ return fallback_accelerator_for("vulkan") == "cpu" and _vulkan_cpu_rung_applies()++def output_shows_accelerator_failure(text: Optional[str]) -> bool:"""True when sd-cli output names a failure of the GPU BUILD rather than of the request; an unrecognised failure persists nothing."""if not text:@@ -2734,6 +2873,8 @@ class _SdState:# UNSLOTH_DIFFUSION_SD_CPP_DEVICE placement: its label, and the env every spawn gets to open only that card.off_torch_device: Optional[str] = Nonechild_env: tuple[tuple[str, str], ...] = ()+ # The user picked the GPU (a gpu_ids pool or UNSLOTH_DIFFUSION_SD_CPP_DEVICE): never moved to CPU behind their back.+ gpu_explicit: bool = Falsedef spawn_env(self) -> Optional[dict[str, str]]:return dict(self.child_env) or None@@ -3071,6 +3212,9 @@ class SdCppDiffusionBackend:# sd-server started for an in-flight load, before it commits to _state; tracked so an unload can stop it# mid-startup.self._pending_server: Optional[SdCppServer] = None+ # Server a generation is stopping for a replacement: status() must not reap it as an idle crash, or the+ # handover reads an unload and kills the healthy replacement.+ self._replacing_server: Optional[SdCppServer] = Noneself._gen: Optional[_SdGen] = None# Set by _resolve_backend when it had to skip an accelerator install because the managed tree was still in# use; the load retries it once the tree is free.@@ -3361,6 +3505,7 @@ class SdCppDiffusionBackend:from core.inference.diffusion_engine_router import off_torch_sd_cpp_deviceoff_torch = off_torch_sd_cpp_device()+ gpu_explicit = bool(gpu_ids) or off_torch is not Noneif off_torch is not None:gpu_ids, gpu_ordinal = None, None# Same fallback the diffusers and video backends take: the route ranks the selection and passes the winner,@@ -3457,6 +3602,7 @@ class SdCppDiffusionBackend:memory_mode = memory_mode,speed_mode = speed_mode,gpu_ordinal = gpu_ordinal,+ gpu_explicit = gpu_explicit,off_torch = off_torch,_load_token = token,_cancel_event = cancel_event,@@ -3484,6 +3630,7 @@ class SdCppDiffusionBackend:memory_mode: Optional[str] = None,speed_mode: Optional[str] = None,gpu_ordinal: Optional[int] = None,+ gpu_explicit: bool = False,off_torch: Any = None,_load_token: int,_cancel_event: Optional[threading.Event] = None,@@ -3950,6 +4097,7 @@ class SdCppDiffusionBackend:selected_card = self._loading_card,off_torch_device = off_torch.label if off_torch is not None else None,child_env = tuple(sorted((spawn_env or {}).items())),+ gpu_explicit = gpu_explicit,)superseded = Falseorphan: Optional[SdCppServer] = None@@ -4890,6 +5038,7 @@ class SdCppDiffusionBackend:base_seed = int(seed) & ((1 << 63) - 1)images: list = []seeds: list[int] = []+# Stage LoRAs into a per-request subdir of the server lora-model-dir (so a prior request's adapters cannot# leak in); removed after.lora_payload: Optional[list[dict]] = None@@ -4913,6 +5062,9 @@ class SdCppDiffusionBackend:# images per job, so each chunk gets whatever is left rather than its own full budget and a long batch still# ends on time.deadline = time.monotonic() + NATIVE_GENERATION_TIMEOUT_S+ # One same-build respawn per request; a second loss moves this load to CPU where the rung applies. Persists nothing.+ device_respawned = False+ device_losses = 0hires_payload: Optional[dict] = Nonehires_staged: list[Path] = []@@ -4981,34 +5133,68 @@ class SdCppDiffusionBackend:cancel_event = cancel,total_timeout = max(deadline - time.monotonic(), 1.0),)- except RuntimeError as exc:+ except RuntimeError as first_exc:# A ggml unsupported-op abort killed the server: this graph cannot run on the GPU backend at all,- # so restart the model on the CPU backend once and retry this chunk. Any other death propagates.- server = self._restart_server_on_cpu_backend(state, str(exc), cancel)- if server is None:- raise- state = replace(state, server = server)- with self._lock:- if self._state is not None and self._state.server is not None:- self._state = state- if lora_payload is not None:- # The replacement server has its own scratch dir: stage the LoRAs there again.- lora_payload = _stage_loras(server)- if lora_payload is None:- raise RuntimeError(- "The CPU-backend native server has no LoRA directory for the selected LoRAs."+ # so restart the model on the CPU backend once and retry this chunk. A lost Vulkan device needs a+ # fresh device: respawn the same build once. Any other death propagates.+ exc = first_exc+ while True:+ moved_to_cpu = False+ server = self._restart_server_on_cpu_backend(state, str(exc), cancel)+ if server is None and output_shows_vulkan_device_lost(str(exc)):+ device_losses += 1+ if device_losses >= 2:+ server = self._restart_server_on_cpu_backend(+ state, str(exc), cancel, after_device_loss = True+ )+ if server is None:+ raise RuntimeError(+ f"{exc} {_REPEATED_DEVICE_LOSS_HINT}"+ ) from exc+ moved_to_cpu = True+ elif not device_respawned:+ server = self._respawn_server_after_device_loss(+ state, str(exc), cancel+ )+ device_respawned = server is not None+ if server is None:+ raise exc+ old_server = state.server+ state = replace(state, server = server)+ if moved_to_cpu:+ # Placement follows the server: the flags the CPU server got, no Vulkan pin.+ state = replace(+ state,+ device = "cpu",+ offload_flags = tuple(+ without_device_backend_flags(state.offload_flags)+ ),+ physical_gpu_id = None,)- payload["lora"] = list(lora_payload)- if hires_plan is not None:- # The replacement server has its own scratch dir: stage the upscaler there again.- hires_payload = _stage_hires(server)- payload["hires"] = dict(hires_payload)- blobs = server.img_gen(- payload,- on_step = self._on_log,- cancel_event = cancel,- total_timeout = max(deadline - time.monotonic(), 1.0),- )+ self._hand_over_replacement(state, server, old_server)+ # The old server's stop removed its scratch dir, and the staged adapters with it.+ if lora_payload is not None:+ lora_payload = _stage_loras(server)+ if lora_payload is None:+ raise RuntimeError(+ "The replacement native server has no LoRA directory for the selected LoRAs."+ )+ payload["lora"] = list(lora_payload)+ if hires_plan is not None:+ # The replacement has its own scratch dir: stage the upscaler there again.+ hires_payload = _stage_hires(server)+ payload["hires"] = dict(hires_payload)+ try:+ blobs = server.img_gen(+ payload,+ on_step = self._on_log,+ cancel_event = cancel,+ total_timeout = max(deadline - time.monotonic(), 1.0),+ )+ break+ except RuntimeError as retry_exc:+ exc = retry_exc+ device_losses = 0# All-or-nothing per chunk; a layered generation decodes layers + 1 images.expected = count * ((layers + 1) if layers else 1)if not cancel.is_set() and len(blobs) != expected:@@ -5034,8 +5220,117 @@ class SdCppDiffusionBackend:passreturn images, seeds- def _restart_server_on_cpu_backend(+ def _begin_replacement(self, state: _SdState) -> bool:+ """Mark ``state.server`` as being replaced, under ``_lock``, before the caller stops it, so status() reports+ the load as still loaded instead of reaping it. False when an unload already took the state: no restart."""+ with self._lock:+ if self._state is None or self._state.server is not state.server:+ return False+ self._replacing_server = state.server+ return True++ def _end_replacement(self, old_server: Any) -> None:+ with self._lock:+ if self._replacing_server is old_server:+ self._replacing_server = None++ def _hand_over_replacement(self, state: _SdState, server: SdCppServer, old_server: Any) -> None:+ """Publish a restarted sd-server into ``_state`` in the same locked step that drops it from+ ``_pending_server``, so an unload always sees it in one of the two. Only while ``_state`` still holds the+ server being replaced: when an unload cleared it or a superseding load replaced it, stop the replacement+ and cancel instead of generating on an untracked process or overwriting the newer load."""+ with self._lock:+ if self._pending_server is server:+ self._pending_server = None+ if self._replacing_server is old_server:+ self._replacing_server = None+ published = self._state is not None and self._state.server is old_server+ if published:+ self._state = state+ else:+ self._reserve_stop()+ if not published:+ self._stop_reserved(server)+ raise SdCppCancelled("sd-server generation was cancelled.")++ def _start_replacement(+ self, state: _SdState, server: SdCppServer, cancel: threading.Event, **start_kwargs: Any+ ) -> Optional[SdCppServer]:+ """Start a published-as-pending replacement sd-server for this generation; None when it fails.++ The start blocks for the model load (up to its startup timeout) while the caller holds+ ``_generate_lock``. ``cancel_generate()`` and a superseding load only set ``cancel``, so a watcher stops+ the pending server when that happens: the readiness wait bails at once and the generation raises+ ``SdCppCancelled`` instead of loading a replacement nobody wants."""+ done = threading.Event()++ def _watch() -> None:+ while not done.is_set():+ if cancel.wait(0.25):+ server.stop()+ return++ watcher = threading.Thread(target = _watch, name = "sd-server-replacement-cancel", daemon = True)+ watcher.start()+ failed = False+ try:+ server.start(state.files, **start_kwargs)+ except Exception: # noqa: BLE001 -- the original server error is the more useful one+ failed = True+ finally:+ done.set()+ watcher.join()+ if failed or cancel.is_set():+ with self._lock:+ if self._pending_server is server:+ self._pending_server = None+ server.stop()+ self._end_replacement(state.server)+ if cancel.is_set():+ raise SdCppCancelled("sd-server generation was cancelled.")+ return None+ # Still pending on success: _hand_over_replacement publishes it, so an unload never misses it.+ return server++ def _respawn_server_after_device_loss(self, state: _SdState, error_text: str, cancel: threading.Event+ ) -> Optional[SdCppServer]:+ """Restart this checkpoint's sd-server on the same build after a lost Vulkan device; None if that+ does not apply. The Vulkan spec makes VK_ERROR_DEVICE_LOST recoverable only by creating a new device,+ which the running server never does, so every later job on it would fail. No build strike: a lost+ device (a TDR or a scheduler reset) says nothing about whether this build runs here."""+ if not output_shows_vulkan_device_lost(error_text):+ return None+ if state.server is None or cancel.is_set():+ return None+ if not self._begin_replacement(state):+ return None+ logger.warning(+ "sd-server lost its Vulkan device; restarting it on the same build. Details: %s",+ error_text[:300],+ )+ state.server.stop()+ server = SdCppServer(state.server.binary)+ with self._lock:+ self._pending_server = server+ return self._start_replacement(+ state,+ server,+ cancel,+ vae_format = state.vae_format,+ offload = state.offload_flags,+ native_speed = state.native_speed,+ threads = state.threads,+ env = state.spawn_env(),+ )++ def _restart_server_on_cpu_backend(+ self,+ state: _SdState,+ error_text: str,+ cancel: threading.Event,+ *,+ after_device_loss: bool = False,) -> Optional[SdCppServer]:"""Relaunch this checkpoint's sd-server with ``--backend cpu``; None if that does not apply.@@ -5051,8 +5346,15 @@ class SdCppDiffusionBackend:(``--offload-to-cpu`` moves parameters, not compute), so the restart runs the samecheckpoint slower rather than not at all. Done once per load: the second abort, or any othercause of death, is surfaced to the caller.++ ``after_device_loss``: the respawned server lost its Vulkan device too (a host whose driver resets the+ card on every render). Only on the CPU-torch Vulkan route, the one route with a CPU rung, and never for a+ GPU the user picked. For this load only and nothing is persisted: the next load tries Vulkan again."""- if not is_ggml_unsupported_op_abort(error_text):+ if after_device_loss:+ if not _device_loss_cpu_fallback_applies(state):+ return None+ elif not is_ggml_unsupported_op_abort(error_text):return Noneif self._cpu_backend_forced or state.device == "cpu":return None # already on CPU: the abort is not a backend-placement problem@@ -5061,38 +5363,40 @@ class SdCppDiffusionBackend:server_binary = find_sd_server_binary()if not server_binary:return None- logger.warning(- "sd-server aborted on an op the '%s' backend cannot run; restarting on the CPU "- "backend (slower, but it completes). Details: %s",- state.device,- error_text[:300],- )+ if not self._begin_replacement(state):+ return None+ if after_device_loss:+ logger.warning(+ "sd-server lost its Vulkan device again after a restart; this model now renders on the CPU "+ "backend (slower) until it is loaded again, which tries Vulkan first. Details: %s",+ error_text[:300],+ )+ else:+ logger.warning(+ "sd-server aborted on an op the '%s' backend cannot run; restarting on the CPU "+ "backend (slower, but it completes). Details: %s",+ state.device,+ error_text[:300],+ )self._cpu_backend_forced = Truestate.server.stop()server = SdCppServer(server_binary)with self._lock:self._pending_server = server- try:- server.start(- state.files,- vae_format = state.vae_format,- # WITHOUT the device pin. sd.cpp joins repeated --backend values into one spec instead of replacing,- # and an explicit diffusion=CUDA0 outranks the bare `cpu` default, so leaving the pin on would restart- # the server onto the very backend that just aborted.- offload = without_device_backend_flags(state.offload_flags),- native_speed = state.native_speed,- threads = state.threads,- env = state.spawn_env(),- extra_args = list(CPU_BACKEND_FLAGS),- )- except Exception: # noqa: BLE001 -- the original abort is the more useful error- server.stop()- return None- finally:- with self._lock:- if self._pending_server is server:- self._pending_server = None- return server+ return self._start_replacement(+ state,+ server,+ cancel,+ vae_format = state.vae_format,+ # WITHOUT the device pin. sd.cpp joins repeated --backend values into one spec instead of replacing,+ # and an explicit diffusion=CUDA0 outranks the bare `cpu` default, so leaving the pin on would restart+ # the server onto the very backend that just aborted.+ offload = without_device_backend_flags(state.offload_flags),+ native_speed = state.native_speed,+ threads = state.threads,+ env = state.spawn_env(),+ extra_args = list(CPU_BACKEND_FLAGS),+ )def _generate_oneshot(self,@@ -5317,6 +5621,7 @@ class SdCppDiffusionBackend:self._active_generate_cancel.set()state = self._stateself._state = None+ self._replacing_server = Noneself._load_token += 1self._loading = None# Grab a mid-start() uncommitted server too so we can stop it (startup is abortable).@@ -5350,13 +5655,17 @@ class SdCppDiffusionBackend:state is not Noneand state.mode == "server"and state.server is not None+ and state.server is not self._replacing_serverand not state.server.is_alive()):- logger.warning("sd-server exited after load; clearing loaded state")with self._lock:- if self._state is state:+ # Re-checked under the lock: a generation stopping it for a replacement marks it first.+ reap = self._state is state and state.server is not self._replacing_server+ if reap:+ logger.warning("sd-server exited after load; clearing loaded state")self._state = None- state = None+ if reap:+ state = Noneif state is None:return {"loaded": False,diff --git a/studio/backend/tests/test_diffusion_hires.py b/studio/backend/tests/test_diffusion_hires.pyindex 56cc0dc9625..dafed138475 100644--- a/studio/backend/tests/test_diffusion_hires.py+++ b/studio/backend/tests/test_diffusion_hires.py@@ -930,8 +930,11 @@ def test_native_cpu_restart_restages_the_model_upscaler(tmp_path, monkeypatch):target = (96, 96),refine_steps = 4,)+ state = _State(family = _fam("sdxl"), server = _Server(old_dir, fail = True))+ # The replacement is published only while the backend still holds the server it replaces.+ b._state = stateimages, _ = b._generate_server(- _State(family = _fam("sdxl"), server = _Server(old_dir, fail = True)),+ state,prompt = "p",negative_prompt = None,width = 64,@@ -953,6 +956,7 @@ def test_native_cpu_restart_restages_the_model_upscaler(tmp_path, monkeypatch):new_dir,] # staged again in the new server's dirassert not any(p.exists() for p in staged_paths) # both removed after+ assert b._state.server is new_serverdef test_diffusers_status_names_why_hires_is_hidden():diff --git a/studio/backend/tests/test_sd_cpp_backend.py b/studio/backend/tests/test_sd_cpp_backend.pyindex d938c4ce07b..9a5d697fdcd 100644--- a/studio/backend/tests/test_sd_cpp_backend.py+++ b/studio/backend/tests/test_sd_cpp_backend.py@@ -5,9 +5,12 @@from __future__ import annotations+import os+import shutilimport threading-from pathlib import Path+import timeimport types+from pathlib import Pathimport pytestfrom PIL import Image@@ -2145,10 +2148,14 @@ def _server_lora_run(monkeypatch, tmp_path, *, fail_first):new_dir.mkdir()b = SdCppDiffusionBackend()monkeypatch.setattr(- b, "_restart_server_on_cpu_backend", lambda state, text, cancel: _Server(new_dir, False)+ b,+ "_restart_server_on_cpu_backend",+ lambda state, text, cancel, **kw: _Server(new_dir, False),)+ # Published like a real load: a restart hands over only while _state still holds the server it replaces.+ b._state = _State(family = detect_family("flux.1"), server = _Server(old_dir, fail_first))b._generate_server(- _State(family = detect_family("flux.1"), server = _Server(old_dir, fail_first)),+ b._state,prompt = "p",negative_prompt = None,width = 64,@@ -2973,6 +2980,501 @@ def test_the_native_progress_total_is_sized_where_the_assets_are_fetched(monkeypassert b._loading.expected_bytes == 2_000+def _png_bytes():+ import io++ from PIL import Image++ buf = io.BytesIO()+ Image.new("RGB", (8, 8)).save(buf, format = "PNG")+ return buf.getvalue()++[email protected]("losses, recovers", [(1, True), (2, False)])+def test_a_lost_vulkan_device_restarts_the_resident_server(monkeypatch, losses, recovers):+ # The resident server stays up on a dead device after VK_ERROR_DEVICE_LOST: start the same build again+ # (a fresh device) and retry the job once; a second loss in a row surfaces where no CPU rung applies (an Arc /+ # ROCm host keeps Vulkan). Neither records a strike.+ monkeypatch.setattr(bk, "_vulkan_cpu_rung_applies", lambda: False)+ events: list = []+ remaining = {"losses": losses}++ class _FakeServer:+ def __init__(self, binary):+ self.binary = binary+ events.append(("new", binary))++ def start(self, files, **kwargs):+ events.append(("start", kwargs.get("offload")))++ def stop(self):+ events.append(("stop", self.binary))++ def img_gen(self, payload, **_kw):+ if remaining["losses"]:+ remaining["losses"] -= 1+ raise RuntimeError("sd-server job failed: vk::Queue::submit: ErrorDeviceLost")+ return [_png_bytes()]++ b = _loaded_backend()+ b._state = _pinned_state(b, policy_flags = ("--offload-to-cpu",), device = "vulkan")+ object.__setattr__(b._state, "server", _FakeServer("/bin/sd-server-vulkan"))+ monkeypatch.setattr(bk, "SdCppServer", _FakeServer)+ monkeypatch.setattr(bk, "find_sd_server_binary", lambda: "/bin/sd-server-cpu")+ bk._accelerator_runtime_failures.clear()+ kwargs = dict(+ prompt = "a fox",+ negative_prompt = None,+ width = 64,+ height = 64,+ steps = 2,+ seed = 1,+ batch_size = 1,+ cfg_scale = None,+ flux_guidance = None,+ lora_resolved = [],+ cancel = threading.Event(),+ )+ if recovers:+ images, _seeds = b._generate_server(b._state, **kwargs)+ assert len(images) == 1+ else:+ with pytest.raises(RuntimeError, match = "ErrorDeviceLost"):+ b._generate_server(b._state, **kwargs)+ # The same build, with the load's own flags (device pin included), not the CPU backend.+ assert ("new", "/bin/sd-server-vulkan") in events[1:]+ assert ("start", b._state.offload_flags) in events or any(+ e[0] == "start" and "--offload-to-cpu" in (e[1] or ()) for e in events+ )+ assert not bk._accelerator_runtime_failures+++def _repeated_device_loss_backend(+ monkeypatch,+ *,+ losses,+ gpu_explicit = False,+):+ """A CPU-torch Vulkan load whose first ``losses`` jobs lose the device, on whichever server runs them."""+ events: list = []+ remaining = {"losses": losses}+ persisted: list = []++ class _FakeServer:+ lora_dir = None++ def __init__(self, binary):+ self.binary = binary++ def start(self, files, **kwargs):+ events.append(("start", self.binary, tuple(kwargs.get("extra_args") or ())))++ def stop(self):+ events.append(("stop", self.binary))++ def is_alive(self):+ return True++ def img_gen(self, payload, **_kw):+ if remaining["losses"]:+ remaining["losses"] -= 1+ raise RuntimeError("sd-server job failed: vk::Queue::submit: ErrorDeviceLost")+ return [_png_bytes()]++ b = _loaded_backend()+ b._state = _pinned_state(b, policy_flags = ("--offload-to-cpu",), device = "vulkan")+ object.__setattr__(b._state, "server", _FakeServer("/bin/sd-server-vulkan"))+ object.__setattr__(b._state, "mode", "server")+ object.__setattr__(b._state, "gpu_explicit", gpu_explicit)+ monkeypatch.setattr(bk, "SdCppServer", _FakeServer)+ monkeypatch.setattr(bk, "find_sd_server_binary", lambda: "/bin/sd-server-vulkan")+ monkeypatch.setattr(bk, "_vulkan_cpu_rung_applies", lambda: True)+ monkeypatch.setattr(+ bk, "fallback_accelerator_for", lambda acc: "cpu" if acc == "vulkan" else None+ )+ monkeypatch.setattr(+ bk, "_persist_accelerator_runtime_failures", lambda records: persisted.append(records)+ )+ monkeypatch.setattr(bk, "_stored_accelerator_runtime_failures", lambda: {})+ bk._accelerator_runtime_failures.clear()+ return b, events, persisted+++def test_a_vulkan_device_lost_again_after_the_respawn_renders_this_load_on_cpu(monkeypatch):+ # A host whose driver resets the card on every render (a Windows TDR on a slow iGPU) must not fail every request:+ # the respawned server losing the device too moves this load alone to the CPU backend. Nothing is persisted, so+ # the next load resolves the Vulkan build again.+ b, events, persisted = _repeated_device_loss_backend(monkeypatch, losses = 2)+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())+ assert len(images) == 1+ starts = [e for e in events if e[0] == "start"]+ # First the same-build respawn on Vulkan, then the CPU backend.+ assert [s[2] for s in starts] == [(), ("--backend", "cpu")]+ assert b._cpu_backend_forced is True+ # The published state follows the server to CPU: status() no longer reports a Vulkan renderer, and the flags are+ # the ones the CPU server got (the Vulkan device pin dropped).+ assert b._state.device == "cpu"+ assert b._state.physical_gpu_id is None+ assert "--backend" not in b._state.offload_flags+ assert b._state.offload_flags == ("--offload-to-cpu",)+ assert b.status()["device"] == "cpu"+ assert not persisted and not bk._accelerator_runtime_failures+ # The failure text is not a strike either, and the next load still prefers Vulkan.+ bk.note_accelerator_failure_from_output(+ "/bin/sd-server-vulkan", "sd-server job failed: vk::Queue::submit: ErrorDeviceLost"+ )+ assert not persisted and not bk._accelerator_runtime_failures+ assert bk.preferred_accelerator("vulkan") == "vulkan"+ # Later requests on this load keep the CPU server without another restart.+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())+ assert len(images) == 1+ assert len([e for e in events if e[0] == "start"]) == 2+++def test_a_single_vulkan_device_loss_stays_on_vulkan(monkeypatch):+ b, events, persisted = _repeated_device_loss_backend(monkeypatch, losses = 1)+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())+ assert len(images) == 1+ assert [e[2] for e in events if e[0] == "start"] == [()]+ assert b._cpu_backend_forced is False+ assert b._state.device == "vulkan"+ assert not persisted++[email protected]("why", ["explicit_gpu", "no_cpu_rung", "fallback_disabled"])+def test_a_repeated_vulkan_device_loss_without_a_cpu_rung_errors_with_a_hint(monkeypatch, why):+ # A user-picked GPU is never moved to CPU behind their back, and an Arc / ROCm host (no CPU rung) or a disabled+ # Vulkan fallback keeps Vulkan: the second loss surfaces, with what to try.+ b, events, persisted = _repeated_device_loss_backend(+ monkeypatch, losses = 2, gpu_explicit = why == "explicit_gpu"+ )+ if why == "no_cpu_rung":+ monkeypatch.setattr(bk, "_vulkan_cpu_rung_applies", lambda: False)+ if why == "fallback_disabled":+ monkeypatch.setattr(bk, "fallback_accelerator_for", lambda acc: None)+ with pytest.raises(RuntimeError, match = "ErrorDeviceLost") as info:+ b._generate_server(b._state, **_gen_kwargs())+ assert "twice in a row" in str(info.value)+ assert not bk.output_shows_accelerator_failure(str(info.value))+ assert all(e[2] != ("--backend", "cpu") for e in events if e[0] == "start")+ assert b._cpu_backend_forced is False+ assert not persisted++[email protected]("losses, device", [(1, "vulkan"), (2, "cpu")])+def test_a_device_loss_replacement_restages_the_hires_upscaler(+ monkeypatch, tmp_path, losses, device+):+ # Every replacement server (the same-build respawn after one lost device, the CPU backend after a second) has its+ # own scratch dir, and the old server's stop removed the upscaler staged in its own: the retry must stage the model+ # upscaler again under the server that runs it, or sd.cpp cannot find the hires upscaler file.+ from core.inference import diffusion_hires as hires++ b, events, persisted = _repeated_device_loss_backend(monkeypatch, losses = losses)+ server_cls = type(b._state.server)+ dirs: dict = {}+ seen: list = []++ def _lora_dir(self):+ if id(self) not in dirs:+ dirs[id(self)] = tmp_path / f"srv{len(dirs)}"+ dirs[id(self)].mkdir()+ return str(dirs[id(self)])++ real_img_gen = server_cls.img_gen++ def _img_gen(self, payload, **kw):+ seen.append((self.lora_dir, dict(payload["hires"])))+ return real_img_gen(self, payload, **kw)++ monkeypatch.setattr(server_cls, "lora_dir", property(_lora_dir), raising = False)+ monkeypatch.setattr(server_cls, "img_gen", _img_gen)+ staged: list = []++ def _stage(+ upscaler_id,+ scratch_dir,+ *,+ cancel = None,+ ):+ dest = Path(scratch_dir) / "hires_x.pth"+ dest.write_bytes(b"w")+ staged.append(dest)+ return "hires_x", dest++ monkeypatch.setattr(hires, "stage_sd_cpp_upscaler", _stage)+ plan = hires.SdCppHiresPlan(+ params = hires.HiresParams(upscaler = "realesrgan_x4plus"),+ base_width = 64,+ base_height = 64,+ target = (96, 96),+ refine_steps = 2,+ )+ images, _seeds = b._generate_server(b._state, hires_plan = plan, **_gen_kwargs())+ assert len(images) == 1+ assert b._state.device == device+ assert not persisted+ # One server per attempt, each sent the upscaler staged in its own dir.+ assert len(seen) == losses + 1+ assert len({d for d, _ in seen}) == losses + 1+ assert all(h["upscaler"] == "hires_x" for _, h in seen)+ assert [str(p.parent) for p in staged] == [d for d, _ in seen]+ assert b._state.server.lora_dir == seen[-1][0]+ assert not any(p.exists() for p in staged)+++def test_an_explicit_gpu_pool_marks_the_native_load_explicit(monkeypatch):+ seen: dict = {}+ monkeypatch.setattr(+ "core.inference.diffusion_engine_router.off_torch_sd_cpp_device", lambda *a, **k: None+ )+ b = SdCppDiffusionBackend(engine = _FakeEngine())+ monkeypatch.setattr(b, "_run_load", lambda **kw: seen.update(kw))+ monkeypatch.setattr(bk, "resolve_selected_cuda_ordinal", lambda ids: None, raising = False)+ b.begin_load(+ repo_id = "unsloth/Z-Image-Turbo-GGUF", gguf_filename = "z-image-turbo-Q4_K_M.gguf", gpu_ids = [0]+ )+ deadline = time.monotonic() + 5+ while "gpu_explicit" not in seen and time.monotonic() < deadline:+ time.sleep(0.01)+ assert seen.get("gpu_explicit") is True+++def _device_loss_backend(+ monkeypatch,+ tmp_path,+ *,+ on_start = None,+):+ """A loaded backend whose first job loses its Vulkan device; each server gets its own lora dir."""+ servers: list = []+ payloads: list = []++ class _FakeServer:+ def __init__(self, binary):+ self.binary = binary+ self.lora_dir = str(tmp_path / f"srv{len(servers)}")+ os.makedirs(self.lora_dir, exist_ok = True)+ self.stopped = False+ servers.append(self)++ def start(self, files, **kwargs):+ if on_start is not None:+ on_start(self)++ def stop(self):+ self.stopped = True+ shutil.rmtree(self.lora_dir, ignore_errors = True)++ def is_alive(self):+ return not self.stopped++ def img_gen(self, payload, **_kw):+ payloads.append((self, dict(payload)))+ if len(payloads) == 1:+ raise RuntimeError("sd-server job failed: vk::Queue::submit: ErrorDeviceLost")+ return [_png_bytes()]++ b = _loaded_backend()+ b._state = _pinned_state(b, policy_flags = (), device = "vulkan")+ object.__setattr__(b._state, "server", _FakeServer("/bin/sd-server-vulkan"))+ object.__setattr__(b._state, "mode", "server")+ monkeypatch.setattr(bk, "SdCppServer", _FakeServer)+ bk._accelerator_runtime_failures.clear()+ return b, servers, payloads+++def _gen_kwargs(lora_resolved = ()):+ return dict(+ prompt = "a fox",+ negative_prompt = None,+ width = 64,+ height = 64,+ steps = 2,+ seed = 1,+ batch_size = 1,+ cfg_scale = None,+ flux_guidance = None,+ lora_resolved = list(lora_resolved),+ cancel = threading.Event(),+ )+++def test_a_device_loss_retry_restages_the_loras_under_the_new_server(monkeypatch, tmp_path):+ # The old server's stop removes its scratch dir and the staged adapters: the retry must stage them again.+ from core.inference import diffusion_lora++ def _materialize(+ resolved,+ stage,+ real_files = False,+ ):+ os.makedirs(stage, exist_ok = True)+ (Path(stage) / "adapter.gguf").write_bytes(b"x")+ return [types.SimpleNamespace(path = str(Path(stage) / "adapter.gguf"), weight = 0.8)]++ monkeypatch.setattr(diffusion_lora, "materialize_native_dir", _materialize)+ b, servers, payloads = _device_loss_backend(monkeypatch, tmp_path)+ images, _seeds = b._generate_server(b._state, **_gen_kwargs(lora_resolved = ["adapter"]))+ assert len(images) == 1 and len(servers) == 2+ retry_server, retry_payload = payloads[-1]+ assert retry_server is servers[1]+ (entry,) = retry_payload["lora"]+ # Staged under the replacement's own lora dir (the file exists there when the retry runs).+ assert entry["path"] != payloads[0][1]["lora"][0]["path"]+ assert entry["multiplier"] == 0.8+++def test_an_unload_during_the_respawn_leaves_no_live_server(monkeypatch, tmp_path):+ # unload() racing the restart must find the replacement (pending) or the handover must stop it.+ holder: dict = {}++ def _unload_mid_start(server):+ holder["backend"].unload()++ b, servers, _payloads = _device_loss_backend(monkeypatch, tmp_path, on_start = _unload_mid_start)+ holder["backend"] = b+ monkeypatch.setattr(b, "status", lambda: {})+ with pytest.raises(SdCppCancelled):+ b._generate_server(b._state, **_gen_kwargs())+ assert len(servers) == 2+ assert all(s.stopped for s in servers)+ assert b._state is None and b._pending_server is None+++def test_a_status_poll_during_the_respawn_keeps_the_replacement(monkeypatch, tmp_path):+ # The UI polls /images/status while the respawn runs. The old server is already stopped then, and the idle-crash+ # reap used to clear _state, so the handover read an unload and killed the healthy replacement.+ holder: dict = {}+ polls: list = []++ def _poll_mid_start(server):+ polls.append(holder["backend"].status()["loaded"])++ b, servers, _payloads = _device_loss_backend(monkeypatch, tmp_path, on_start = _poll_mid_start)+ holder["backend"] = b+ images, _seeds = b._generate_server(b._state, **_gen_kwargs())+ assert len(images) == 1 and len(servers) == 2+ assert polls == [True]+ assert b._state is not None and b._state.server is servers[1]+ assert not servers[1].stopped and b._replacing_server is None+ # Once the generation is done, a server that really died is reaped again.+ servers[1].stopped = True+ assert b.status()["loaded"] is False+++def test_a_status_poll_during_the_cpu_backend_restart_keeps_the_load(monkeypatch):+ import dataclasses++ class _FakeServer:+ def __init__(self, binary):+ self.binary = binary+ self.stopped = False++ def start(self, files, **kwargs):+ polls.append(b.status()["loaded"])++ def stop(self):+ self.stopped = True++ def is_alive(self):+ return not self.stopped++ polls: list = []+ b = _loaded_backend()+ b._state = _pinned_state(b, policy_flags = ("--offload-to-cpu",))+ object.__setattr__(b._state, "server", _FakeServer("/bin/sd-server"))+ object.__setattr__(b._state, "mode", "server")+ monkeypatch.setattr(bk, "find_sd_server_binary", lambda: "/bin/sd-server")+ monkeypatch.setattr(bk, "SdCppServer", _FakeServer)+ state = b._state+ server = b._restart_server_on_cpu_backend(+ state,+ "ggml_metal_op_encode_impl: unsupported op 'MUL_MAT' -> ggml_abort",+ threading.Event(),+ )+ assert server is not None and polls == [True] and b._state is state+ b._hand_over_replacement(dataclasses.replace(state, server = server), server, state.server)+ assert b._state.server is server and not server.stopped+++def test_a_superseding_load_during_the_respawn_is_not_overwritten(monkeypatch, tmp_path):+ # A newer load that commits while the replacement starts owns _state: the handover must not publish the old+ # model's replacement over it, and the replacement must not be left running untracked.+ import dataclasses++ holder: dict = {}++ def _supersede_mid_start(server):+ b = holder["backend"]+ newer = _FakeNewer("/bin/sd-server-newer")+ holder["newer"] = newer+ with b._lock:+ b._state = dataclasses.replace(b._state, server = newer)++ class _FakeNewer:+ def __init__(self, binary):+ self.binary = binary+ self.stopped = False++ def stop(self):+ self.stopped = True++ def is_alive(self):+ return not self.stopped++ b, servers, payloads = _device_loss_backend(+ monkeypatch, tmp_path, on_start = _supersede_mid_start+ )+ holder["backend"] = b+ monkeypatch.setattr(b, "status", lambda: {})+ with pytest.raises(SdCppCancelled):+ b._generate_server(b._state, **_gen_kwargs())+ assert len(servers) == 2 and servers[1].stopped # the stale replacement is stopped+ assert b._state.server is holder["newer"] and not holder["newer"].stopped+ assert len(payloads) == 1 and b._pending_server is None+++def test_a_cancel_during_the_respawn_stops_the_replacement_start(monkeypatch, tmp_path):+ # cancel_generate() (or a superseding load) only sets the generation's event; the replacement start blocks for the+ # model load while _generate_lock is held, so it must notice and bail instead of loading for the full timeout.+ kwargs = _gen_kwargs()+ cancel = kwargs["cancel"]++ def _cancel_mid_start(server):+ cancel.set()+ deadline = time.monotonic() + 5.0+ while time.monotonic() < deadline:+ if server.stopped:+ raise RuntimeError("sd-server startup aborted before ready")+ time.sleep(0.01)++ b, servers, payloads = _device_loss_backend(monkeypatch, tmp_path, on_start = _cancel_mid_start)+ monkeypatch.setattr(b, "status", lambda: {})+ started = time.monotonic()+ with pytest.raises(SdCppCancelled):+ b._generate_server(b._state, **kwargs)+ assert time.monotonic() - started < 4.0+ assert len(servers) == 2 and all(s.stopped for s in servers)+ assert len(payloads) == 1 # no retry job was posted+ assert b._pending_server is None and b._replacing_server is None+++def test_an_unload_before_the_respawn_starts_skips_it(monkeypatch, tmp_path):+ # unload() wins outright when it lands first: no replacement is started at all.+ b, servers, _payloads = _device_loss_backend(monkeypatch, tmp_path)+ state = b._state+ monkeypatch.setattr(b, "status", lambda: {})+ b.unload()+ assert (+ b._respawn_server_after_device_loss(+ state, "vk::Queue::submit: ErrorDeviceLost", threading.Event()+ )+ is None+ )+ assert len(servers) == 1 and b._replacing_server is None++@pytest.mark.parametrize("device,ccs,expected",[diff --git a/studio/backend/tests/test_sd_cpp_vulkan_cpu_torch.py b/studio/backend/tests/test_sd_cpp_vulkan_cpu_torch.pyindex 6b411607f28..ac200bcb42e 100644--- a/studio/backend/tests/test_sd_cpp_vulkan_cpu_torch.py+++ b/studio/backend/tests/test_sd_cpp_vulkan_cpu_torch.py@@ -70,6 +70,7 @@ def _pinned_host(monkeypatch):monkeypatch.setattr(amd, "the_vulkan_loader_has_no_usable_driver", lambda: False)monkeypatch.setattr(r, "_hardware_vulkan_driver", lambda vendors: True)monkeypatch.setattr(r, "_nvidia_adapter_present", lambda: False)+ monkeypatch.setattr(r, "_reachable_intel_arc_render_node", lambda: True, raising = False)@pytest.mark.parametrize(@@ -77,8 +78,8 @@ def _pinned_host(monkeypatch):[((_VEGA,), "vulkan"),((_ARC_LINUX,), "vulkan"),- # Unpinned sd.cpp could take the UHD iGPU first.- ((_UHD_LINUX, _VEGA), "auto"),+ # Unpinned sd.cpp takes a discrete GPU before an iGPU: the card renders, not the UHD iGPU.+ ((_UHD_LINUX, _VEGA), "vulkan"),((_ARC_LINUX, _VEGA), "vulkan"),((_UHD_LINUX,), "auto"),((), "auto"),@@ -143,9 +144,43 @@ def test_an_unmounted_uhd_node_does_not_block_the_amd_card(monkeypatch):monkeypatch.setattr(r, "_physical_inventory", lambda: _host(_UHD_LINUX, _VEGA))monkeypatch.setattr(r, "_accessible_render_vendors", lambda vendors: {"amd"} & set(vendors))assert r.image_install_accelerator("cpu") == "vulkan"- # Both nodes mounted: the UHD iGPU could be taken first.+ # Both nodes mounted: the dedicated card is still the one sd.cpp takes.monkeypatch.setattr(r, "_accessible_render_vendors", lambda vendors: set(vendors))- assert r.image_install_accelerator("cpu") == "auto"+ assert r.image_install_accelerator("cpu") == "vulkan"++[email protected]("arc_node_reachable, expected", [(True, "vulkan"), (False, "auto")])+def test_a_uhd_beside_an_arc_needs_the_arcs_own_node(monkeypatch, arc_node_reachable, expected):+ # The vendor-level node check passes on the iGPU's node alone: a container mounting only it sees the UHD.+ monkeypatch.setattr(r, "_physical_inventory", lambda: _host(_UHD_LINUX, _ARC_LINUX))+ monkeypatch.setattr(+ r, "_reachable_intel_arc_render_node", lambda: arc_node_reachable, raising = False+ )+ assert r.image_install_accelerator("cpu") == expected+++ "arc_node_reachable, drivers, expected",+ [+ # The Arc node is not mounted and only the Intel driver loads: Intel Vulkan sees the UHD alone and the AMD+ # card has no driver, so the iGPU would render.+ (False, {"intel"}, "auto"),+ # The AMD card's own driver loads: it renders, whatever the Arc's node.+ (False, {"amd"}, "vulkan"),+ (False, {"amd", "intel"}, "vulkan"),+ # The Arc's node is reachable: the Intel driver drives the Arc, a discrete GPU sd.cpp takes first.+ (True, {"intel"}, "vulkan"),+ ],+)+def test_an_arc_beside_an_amd_card_and_a_uhd_still_needs_its_own_node(+ monkeypatch, arc_node_reachable, drivers, expected+):+ monkeypatch.setattr(r, "_physical_inventory", lambda: _host(_VEGA, _ARC_LINUX, _UHD_LINUX))+ monkeypatch.setattr(+ r, "_reachable_intel_arc_render_node", lambda: arc_node_reachable, raising = False+ )+ monkeypatch.setattr(r, "_hardware_vulkan_driver", lambda vendors: bool(set(vendors) & drivers))+ assert r.image_install_accelerator("cpu") == expected@pytest.mark.parametrize("value", ["", "default"])@@ -164,15 +199,20 @@ def test_a_malformed_ggml_vulkan_mask_keeps_cpu(monkeypatch, value):@pytest.mark.parametrize("output",[- "vk::Queue::submit: ErrorDeviceLost",- "VK_ERROR_DEVICE_LOST","vk::createInstance: ErrorIncompatibleDriver",+ "vk::createInstance: ErrorInitializationFailed",],)def test_vulkan_failures_are_recognised(output):assert sd_cpp_backend.output_shows_accelerator_failure(output)[email protected]("output", ["vk::Queue::submit: ErrorDeviceLost", "VK_ERROR_DEVICE_LOST"])+def test_a_lost_vulkan_device_is_not_a_build_failure(output):+ # Recoverable by recreating the device (a TDR, a scheduler timeout): no evidence about the build.+ assert not sd_cpp_backend.output_shows_accelerator_failure(output)++def test_an_incompatible_vulkan_driver_is_decisive():assert sd_cpp_backend.output_shows_decisive_accelerator_failure("ErrorIncompatibleDriver")assert not sd_cpp_backend.output_shows_decisive_accelerator_failure("ErrorDeviceLost")@@ -401,6 +441,16 @@ def test_the_render_node_and_the_driver_must_share_a_vendor(monkeypatch):assert r.image_install_accelerator("cpu") == "vulkan"+def test_a_uhd_beside_an_amd_card_needs_the_cards_own_driver(monkeypatch):+ # Both nodes mounted, but only an Intel ICD loads: Vulkan then enumerates the iGPU alone and the AMD card is not a+ # Vulkan device at all, so the UHD would render.+ monkeypatch.setattr(r, "_physical_inventory", lambda: _host(_UHD_LINUX, _VEGA))+ monkeypatch.setattr(r, "_hardware_vulkan_driver", lambda vendors: "intel" in vendors)+ assert r.image_install_accelerator("cpu") == "auto"+ monkeypatch.setattr(r, "_hardware_vulkan_driver", lambda vendors: "amd" in vendors)+ assert r.image_install_accelerator("cpu") == "vulkan"++@pytest.mark.parametrize("records,live,expected",[@@ -433,3 +483,176 @@ def test_windows_nvidia_without_nvidia_smi_is_found_in_the_adapter_registry(monkeypatch.setattr(hw, "_windows_live_adapter_names", lambda: live)assert r._nvidia_adapter_present() is expectedassert seen == [0x10DE]+++def _empty_failure_records(monkeypatch, stored = None):+ monkeypatch.setattr(sd_cpp_backend, "_accelerator_runtime_failures", {})+ monkeypatch.setattr(+ sd_cpp_backend, "_stored_accelerator_runtime_failures", lambda: dict(stored or {})+ )+ monkeypatch.setattr(sd_cpp_backend, "_persist_accelerator_runtime_failures", lambda _r: None)+ monkeypatch.setattr(sd_cpp_backend, "_accelerator_fingerprint", lambda *_a: {"bundle": "t"})++[email protected]("cpu_torch", [True, False])+def test_two_lost_vulkan_devices_keep_the_vulkan_build(monkeypatch, cpu_torch):+ """An Arc / ROCm-on-Vulkan host (and a CPU-torch one) that lost the device twice stays on the GPU build."""+ _empty_failure_records(monkeypatch)+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: cpu_torch)+ monkeypatch.setattr(sd_cpp_backend, "_installed_accelerator_of", lambda _b: "vulkan")+ for _ in range(2):+ sd_cpp_backend.note_accelerator_failure_from_output(+ "/x/sd-cli", "vk::Queue::submit: ErrorDeviceLost"+ )+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "vulkan"+++def test_ambiguous_vulkan_strikes_move_only_a_cpu_torch_host(monkeypatch):+ """Two ambiguous failures: the CPU-torch route falls to CPU sd.cpp, an XPU / ROCm host keeps Vulkan."""+ _empty_failure_records(monkeypatch)+ for _ in range(2):+ sd_cpp_backend.note_accelerator_runtime_failure("vulkan", proven = False)+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: True)+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "cpu"+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: False)+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "vulkan"+ # Decisive evidence still moves every host: a build that cannot run must not leave it with nothing.+ sd_cpp_backend.note_accelerator_runtime_failure("vulkan", proven = True)+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "cpu"+++def test_a_vulkan_record_from_before_device_lost_stopped_counting_is_ignored(monkeypatch):+ """Two ambiguous strikes persisted under the old rule may be two lost devices: they no longer divert."""+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: True)+ old = {"vulkan": {"strikes": 2, "proven": False, "fingerprint": {"bundle": "t"}}}+ _empty_failure_records(monkeypatch, stored = old)+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "vulkan"+ # A decisive one written then still holds.+ proven = {"vulkan": {"strikes": 1, "proven": True, "fingerprint": {"bundle": "t"}}}+ _empty_failure_records(monkeypatch, stored = proven)+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "cpu"+++def test_an_old_rule_record_keeps_only_the_proven_cards_known():+ """Card A proven, card B ambiguous only, no unscoped proof: B must not divert on A's failure."""+ old = {+ "strikes": 2,+ "proven": True,+ "fingerprint": {"bundle": "t"},+ "cards": ["Card A", "Card B"],+ "per_card": {+ "Card A": {"strikes": 1, "proven": True},+ "Card B": {"strikes": 1, "proven": False},+ },+ }+ kept = sd_cpp_backend._decisive_evidence_only(old)+ assert kept["cards"] == ["Card A"]+ fp = {"bundle": "t"}+ assert sd_cpp_backend._record_diverts(kept, fp, card = "Card A") is True+ assert sd_cpp_backend._record_diverts(kept, fp, card = "Card B") is False++[email protected]("card,diverts", [("Card A", True), ("Card B", False), (None, True)])+def test_an_old_rule_proof_without_tallies_keeps_its_cards(card, diverts):+ """Proof naming cards but no per-card tallies diverted only those cards; migrating it must not widen it."""+ old = {"strikes": 1, "proven": True, "fingerprint": {"bundle": "t"}, "cards": ["Card A"]}+ fp = {"bundle": "t"}+ assert sd_cpp_backend._record_diverts(old, fp, card = card) is diverts+ kept = sd_cpp_backend._record_under_current_rule("vulkan", old)+ assert sd_cpp_backend._record_diverts(kept, fp, card = card) is diverts+++def test_the_fallback_status_counts_the_same_evidence_as_the_runtime(monkeypatch):+ """Settings must not report a diversion the runtime decision no longer makes."""+ monkeypatch.setattr(sd_cpp_backend, "sd_cpp_vulkan_fallback_enabled", lambda: True)+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: True)+ old = {"vulkan": {"strikes": 2, "proven": False, "fingerprint": {"bundle": "t"}}}+ _empty_failure_records(monkeypatch, stored = old)+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "vulkan"+ assert sd_cpp_backend.accelerator_runtime_failure_state()["diverting"] is False+ # Current ambiguous strikes off the CPU-torch route: kept on Vulkan, and reported so.+ _empty_failure_records(monkeypatch)+ for _ in range(2):+ sd_cpp_backend.note_accelerator_runtime_failure("vulkan", proven = False)+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: False)+ assert sd_cpp_backend.preferred_accelerator("vulkan") == "vulkan"+ assert sd_cpp_backend.accelerator_runtime_failure_state()["diverting"] is False+ monkeypatch.setattr(sd_cpp_backend, "_vulkan_cpu_rung_applies", lambda: True)+ assert sd_cpp_backend.accelerator_runtime_failure_state()["diverting"] is True++[email protected]("value", ["0 1", "0, 1", "1 "])+def test_a_space_separated_ggml_vulkan_mask_is_a_mask(monkeypatch, value):+ # ggml reads commas as spaces.+ monkeypatch.setenv("GGML_VK_VISIBLE_DEVICES", value)+ assert r.image_install_accelerator("cpu") == "vulkan"+++_LISTING = (+ "CPU\tAMD Ryzen 9\nVulkan0\tAMD Radeon RX 7800 XT (RADV NAVI32)\nROCm0\tAMD Radeon RX 7800 XT\n"+)+++def test_device_listing_readers_share_one_probe(monkeypatch, tmp_path):+ binary = tmp_path / "sd-cli"+ binary.write_bytes(b"x")+ calls: list = []++ def _probe(_binary, *args):+ calls.append(args)+ return _LISTING++ monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", _probe)+ assert sd_cpp_backend.sd_cpp_device_name_for_ordinal(str(binary), 0) == "ROCm0"+ sd_cpp_backend.sd_cpp_device_named(str(binary), "AMD Radeon RX 7800 XT")+ assert sd_cpp_backend.sd_cpp_listed_device(str(binary))[0] == "vulkan"+ assert calls == [("--list-devices",)]+ # A different device selection is a different listing.+ monkeypatch.setenv("GGML_VK_VISIBLE_DEVICES", "0")+ sd_cpp_backend.sd_cpp_listed_device(str(binary))+ assert len(calls) == 2+++def test_a_hardware_inventory_change_reprobes_the_listing(monkeypatch, tmp_path):+ # Same file, same env, but the eGPU was unplugged (or the driver reset): the old listing names a device that is gone.+ from utils.hardware import gpu_query, hardware++ binary = tmp_path / "sd-cli"+ binary.write_bytes(b"x")+ calls: list = []++ def _probe(_binary, *args):+ calls.append(args)+ return _LISTING++ monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", _probe)+ sd_cpp_backend.sd_cpp_listed_device(str(binary))+ sd_cpp_backend.sd_cpp_listed_device(str(binary))+ assert len(calls) == 1+ # An explicit re-detection (hotplug, driver reset).+ gpu_query.invalidate_static("test hotplug")+ sd_cpp_backend.sd_cpp_listed_device(str(binary))+ assert len(calls) == 2+ # A periodic inventory refresh that found a different device set.+ before = [{"vendor": "amd", "name": "AMD Radeon RX 7800 XT"}]+ monkeypatch.setattr(hardware, "_physical_gpu_inventory_cache", (0.0, {"devices": before}))+ monkeypatch.setattr(hardware, "_probe_physical_gpu_inventory", lambda: {"devices": []})+ hardware._run_physical_gpu_inventory_probe()+ sd_cpp_backend.sd_cpp_listed_device(str(binary))+ assert len(calls) == 3+ # A refresh that found the same devices keeps the listing.+ hardware._run_physical_gpu_inventory_probe()+ sd_cpp_backend.sd_cpp_listed_device(str(binary))+ assert len(calls) == 3+++def test_an_unreadable_listing_is_not_cached(monkeypatch, tmp_path):+ binary = tmp_path / "sd-cli"+ binary.write_bytes(b"x")+ answers = [None, _LISTING]+ monkeypatch.setattr(sd_cpp_backend, "_LAST_DEVICE_LISTING", {})+ monkeypatch.setattr(sd_cpp_backend, "_sd_cpp_probe_output", lambda *_a: answers.pop(0))+ assert sd_cpp_backend.sd_cpp_device_name_for_ordinal(str(binary), 0) is None+ assert sd_cpp_backend.sd_cpp_device_name_for_ordinal(str(binary), 0) == "ROCm0"diff --git a/studio/backend/utils/hardware/gpu_query.py b/studio/backend/utils/hardware/gpu_query.pyindex 32a1575c971..87fe66a9c9f 100644--- a/studio/backend/utils/hardware/gpu_query.py+++ b/studio/backend/utils/hardware/gpu_query.py@@ -206,6 +206,11 @@ def invalidate_static(reason: str = "") -> None:logger.debug("GPU inventory readings invalidated: %s", reason)+def static_generation() -> int:+ """Bumped by invalidate_static() and reset(): an inventory read before the bump may describe a different host."""+ return _static_gen++def reset() -> None:global _static_gen, _slow_until, _stats, _reset_epoch_events.invalidate_gpu_memory("reset")diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.pyindex 7a1cd2f9d1b..93b2650d0e0 100644--- a/studio/backend/utils/hardware/hardware.py+++ b/studio/backend/utils/hardware/hardware.py@@ -288,6 +288,8 @@ _physical_gpu_inventory_refreshing = False# One re-detection request per recovery, or a poll every few seconds starves detection._REDETECTION_REQUESTED = False_physical_gpu_inventory_cache: Optional[tuple[float, Dict[str, Any]]] = None+# Bumped when a refresh finds a different device set, so device-listing caches can tell a stale answer apart.+_physical_gpu_inventory_generation = 0# Same treatment for the torch side: classify_torch_build() calls availability probes that block on a wedged driver, and /api/health and /api/liveness reach it through the verdict._TORCH_BUILD_SNAPSHOT_TTL_SECONDS = 60.0_torch_build_snapshot_lock = threading.Lock()@@ -753,7 +755,7 @@ def _carry_unanswered_vendors_forward(def _run_physical_gpu_inventory_probe() -> Dict[str, Any]:"""One probe pass, stored in the cache. Never raises."""- global _physical_gpu_inventory_cache+ global _physical_gpu_inventory_cache, _physical_gpu_inventory_generationprevious = _physical_gpu_inventory_cachetry:inventory = _probe_physical_gpu_inventory()@@ -765,10 +767,19 @@ def _run_physical_gpu_inventory_probe() -> Dict[str, Any]:inventory = _carry_unanswered_vendors_forward(inventory, previous[1] if previous is not None else None)+ if previous is not None and previous[1].get("devices") != inventory.get("devices"):+ _physical_gpu_inventory_generation += 1_physical_gpu_inventory_cache = (time.monotonic(), inventory)return inventory+def physical_gpu_inventory_generation() -> int:+ """Changes whenever the GPUs this host has may have changed: a refresh that found a different device set, or an+ explicit re-detection (``gpu_query.invalidate_static``). Never probes."""+ from . import gpu_query+ return _physical_gpu_inventory_generation + gpu_query.static_generation()++def get_physical_gpu_inventory(*, block: bool = True) -> Dict[str, Any]:"""GPUs the OS enumerates, whether or not PyTorch can use them: ``{available, devices, sources, unknown}``. Runs regardless of DEVICE and never raises. ``block=False`` for anything on a request path -- the NVIDIA half shells out with a 10s timeout behind a lock, so a hung driver would stall the event loop every TTL; a non-blocking caller takes the last answer, kicks a daemon-thread refresh, and gets the new value later."""now = time.monotonic()